reguire gnupg1 on f>=31; sense the system to use gpg1 when installed

[nodemanager.git] / bwmon.py

diff --git a/bwmon.py b/bwmon.py
index 362820b..3a1f533 100644 (file)
--- a/bwmon.py
+++ b/bwmon.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3

 #
 # Average bandwidth monitoring script. Run periodically via NM db.sync to
 # enforce a soft limit on daily bandwidth usage for each slice. If a
 #
 # Average bandwidth monitoring script. Run periodically via NM db.sync to
 # enforce a soft limit on daily bandwidth usage for each slice. If a


@@ -14,23 +14,23 @@
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2008 The Trustees of Princeton University
 #
 # Faiyaz Ahmed <faiyaza@cs.princeton.edu>
 # Copyright (C) 2004-2008 The Trustees of Princeton University
 #

-# $Id$
-#

 
 import os
 import sys
 import time
 import pickle
 import socket
 
 import os
 import sys
 import time
 import pickle
 import socket

-import logger

 import copy
 import threading
 import copy
 import threading

-import tools

 
 

-import bwlimit
+import plnode.bwlimit as bwlimit
+
+import logger
+import tools

 import database
 import database

+from config import Config

 
 

-from sets import Set
+priority = 20

 
 # Defaults
 # Set DEBUG to True if you don't want to send emails
 
 # Defaults
 # Set DEBUG to True if you don't want to send emails


@@ -38,28 +38,36 @@ DEBUG = False
 # Set ENABLE to False to setup buckets, but not limit.
 ENABLE = True
 
 # Set ENABLE to False to setup buckets, but not limit.
 ENABLE = True
 

-datafile = "/var/lib/misc/bwmon.dat"
-
-try:
-    sys.path.append("/etc/planetlab")
-    from plc_config import *
-except:
-    DEBUG = True
-    logger.log("bwmon:  Warning: Configuration file /etc/planetlab/plc_config.py not found", 2)
-    logger.log("bwmon:  Running in DEBUG mode.  Logging to file and not emailing.", 1)
+DB_FILE = "/var/lib/nodemanager/bwmon.pickle"

 
 # Constants
 seconds_per_day = 24 * 60 * 60
 bits_per_byte = 8
 
 
 # Constants
 seconds_per_day = 24 * 60 * 60
 bits_per_byte = 8
 

+dev_default = tools.get_default_if()

 # Burst to line rate (or node cap).  Set by NM. in KBit/s
 # Burst to line rate (or node cap).  Set by NM. in KBit/s

-default_MaxRate = int(bwlimit.get_bwcap() / 1000)
+default_MaxRate = int(bwlimit.get_bwcap(dev_default) / 1000)

 default_Maxi2Rate = int(bwlimit.bwmax / 1000)
 default_Maxi2Rate = int(bwlimit.bwmax / 1000)

-# 5.4 Gbyte per day. 5.4 * 1024 k * 1024M * 1024G 
+# 5.4 Gbyte per day. 5.4 * 1024 k * 1024M * 1024G

 # 5.4 Gbyte per day max allowed transfered per recording period
 # 5.4 Gbyte per day max allowed transfered per recording period

-default_MaxKByte = 5662310
+# 5.4 Gbytes per day is aprox 512k/s for 24hrs (approx because original math was wrong
+# but its better to keep a higher byte total and keep people happy than correct
+# the problem and piss people off.
+# default_MaxKByte = 5662310
+
+# -- 6/1/09
+# llp wants to double these, so we use the following
+# 1mbit * 24hrs * 60mins * 60secs = bits/day
+# 1000000 * 24 * 60 * 60 / (1024 * 8)
+default_MaxKByte = 10546875
+

 # 16.4 Gbyte per day max allowed transfered per recording period to I2
 # 16.4 Gbyte per day max allowed transfered per recording period to I2

-default_Maxi2KByte = 17196646
+# default_Maxi2KByte = 17196646
+
+# -- 6/1/09
+# 3Mb/s for 24hrs a day (30.17 gigs)
+default_Maxi2KByte = 31640625
+

 # Default share quanta
 default_Share = 1
 
 # Default share quanta
 default_Share = 1
 


@@ -127,16 +135,17 @@ def slicemail(slice, subject, body):
     '''
     Front end to sendmail.  Sends email to slice alias with given subject and body.
     '''
     '''
     Front end to sendmail.  Sends email to slice alias with given subject and body.
     '''

-
-    sendmail = os.popen("/usr/sbin/sendmail -N never -t -f%s" % PLC_MAIL_SUPPORT_ADDRESS, "w")
+    config = Config()
+    sendmail = os.popen("/usr/sbin/sendmail -N never -t -f%s" % config.PLC_MAIL_SUPPORT_ADDRESS, "w")

 
     # Parsed from MyPLC config
 
     # Parsed from MyPLC config

-    to = [PLC_MAIL_MOM_LIST_ADDRESS]
+    to = [config.PLC_MAIL_MOM_LIST_ADDRESS]

 
     if slice is not None and slice != "root":
 
     if slice is not None and slice != "root":

-        to.append(PLC_MAIL_SLICE_ADDRESS.replace("SLICE", slice))
+        to.append(config.PLC_MAIL_SLICE_ADDRESS.replace("SLICE", slice))

 
 

-    header = {'from': "%s Support <%s>" % (PLC_NAME, PLC_MAIL_SUPPORT_ADDRESS),
+    header = {'from': "%s Support <%s>" % (config.PLC_NAME,
+                                           config.PLC_MAIL_SUPPORT_ADDRESS),

               'to': ", ".join(to),
               'version': sys.version.split(" ")[0],
               'subject': subject}
               'to': ", ".join(to),
               'version': sys.version.split(" ")[0],
               'subject': subject}


@@ -170,9 +179,9 @@ class Slice:
     i2bytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F)
     MaxKByte - total volume of data allowed
     ThreshKbyte - After thresh, cap node to (maxkbyte - bytes)/(time left in period)
     i2bytes - high bandwidth bytes transmitted at the beginning of the recording period (for I2 -F)
     MaxKByte - total volume of data allowed
     ThreshKbyte - After thresh, cap node to (maxkbyte - bytes)/(time left in period)

-    Maxi2KByte - same as MaxKByte, but for i2 
-    Threshi2Kbyte - same as Threshi2KByte, but for i2 
-    MaxRate - max_rate slice attribute. 
+    Maxi2KByte - same as MaxKByte, but for i2
+    Threshi2Kbyte - same as Threshi2KByte, but for i2
+    MaxRate - max_rate slice attribute.

     Maxi2Rate - max_exempt_rate slice attribute.
     Share - Used by Sirius to loan min rates
     Sharei2 - Used by Sirius to loan min rates for i2
     Maxi2Rate - max_exempt_rate slice attribute.
     Share - Used by Sirius to loan min rates
     Sharei2 - Used by Sirius to loan min rates for i2


@@ -187,9 +196,9 @@ class Slice:
         self.bytes = 0
         self.i2bytes = 0
         self.MaxRate = default_MaxRate
         self.bytes = 0
         self.i2bytes = 0
         self.MaxRate = default_MaxRate

-        self.MinRate = bwlimit.bwmin / 1000
+        self.MinRate = bwlimit.bwmin // 1000

         self.Maxi2Rate = default_Maxi2Rate
         self.Maxi2Rate = default_Maxi2Rate

-        self.Mini2Rate = bwlimit.bwmin / 1000
+        self.Mini2Rate = bwlimit.bwmin // 1000

         self.MaxKByte = default_MaxKByte
         self.ThreshKByte = int(.8 * self.MaxKByte)
         self.Maxi2KByte = default_Maxi2KByte
         self.MaxKByte = default_MaxKByte
         self.ThreshKByte = int(.8 * self.MaxKByte)
         self.Maxi2KByte = default_Maxi2KByte


@@ -200,12 +209,13 @@ class Slice:
         self.capped = False
 
         self.updateSliceTags(rspec)
         self.capped = False
 
         self.updateSliceTags(rspec)

-        bwlimit.set(xid = self.xid, 
-                minrate = self.MinRate * 1000, 
-                maxrate = self.MaxRate * 1000, 
-                maxexemptrate = self.Maxi2Rate * 1000,
-                minexemptrate = self.Mini2Rate * 1000,
-                share = self.Share)
+        bwlimit.set(
+            xid=self.xid, dev=dev_default,
+            minrate=self.MinRate * 1000,
+            maxrate=self.MaxRate * 1000,
+            maxexemptrate=self.Maxi2Rate * 1000,
+            minexemptrate=self.Mini2Rate * 1000,
+            share=self.Share)

 
     def __repr__(self):
         return self.name
 
     def __repr__(self):
         return self.name


@@ -218,57 +228,57 @@ class Slice:
 
         # Sanity check plus policy decision for MinRate:
         # Minrate cant be greater than 25% of MaxRate or NodeCap.
 
         # Sanity check plus policy decision for MinRate:
         # Minrate cant be greater than 25% of MaxRate or NodeCap.

-        MinRate = int(rspec.get("net_min_rate", bwlimit.bwmin / 1000))
+        MinRate = int(rspec.get("net_min_rate", bwlimit.bwmin // 1000))

         if MinRate > int(.25 * default_MaxRate):
             MinRate = int(.25 * default_MaxRate)
         if MinRate != self.MinRate:
             self.MinRate = MinRate
         if MinRate > int(.25 * default_MaxRate):
             MinRate = int(.25 * default_MaxRate)
         if MinRate != self.MinRate:
             self.MinRate = MinRate

-            logger.log("bwmon:  Updating %s: Min Rate = %s" %(self.name, self.MinRate))
+            logger.log("bwmon: Updating %s: Min Rate = %s" %(self.name, self.MinRate))

 
         MaxRate = int(rspec.get('net_max_rate', default_MaxRate))
         if MaxRate != self.MaxRate:
             self.MaxRate = MaxRate
 
         MaxRate = int(rspec.get('net_max_rate', default_MaxRate))
         if MaxRate != self.MaxRate:
             self.MaxRate = MaxRate

-            logger.log("bwmon:  Updating %s: Max Rate = %s" %(self.name, self.MaxRate))
+            logger.log("bwmon: Updating %s: Max Rate = %s" %(self.name, self.MaxRate))

 
 

-        Mini2Rate = int(rspec.get('net_i2_min_rate', bwlimit.bwmin / 1000))
+        Mini2Rate = int(rspec.get('net_i2_min_rate', bwlimit.bwmin // 1000))

         if Mini2Rate != self.Mini2Rate:
         if Mini2Rate != self.Mini2Rate:

-            self.Mini2Rate = Mini2Rate 
-            logger.log("bwmon:  Updating %s: Min i2 Rate = %s" %(self.name, self.Mini2Rate))
+            self.Mini2Rate = Mini2Rate
+            logger.log("bwmon: Updating %s: Min i2 Rate = %s" %(self.name, self.Mini2Rate))

 
         Maxi2Rate = int(rspec.get('net_i2_max_rate', default_Maxi2Rate))
         if Maxi2Rate != self.Maxi2Rate:
             self.Maxi2Rate = Maxi2Rate
 
         Maxi2Rate = int(rspec.get('net_i2_max_rate', default_Maxi2Rate))
         if Maxi2Rate != self.Maxi2Rate:
             self.Maxi2Rate = Maxi2Rate

-            logger.log("bwmon:  Updating %s: Max i2 Rate = %s" %(self.name, self.Maxi2Rate))
-                          
+            logger.log("bwmon: Updating %s: Max i2 Rate = %s" %(self.name, self.Maxi2Rate))
+

         MaxKByte = int(rspec.get('net_max_kbyte', default_MaxKByte))
         if MaxKByte != self.MaxKByte:
             self.MaxKByte = MaxKByte
         MaxKByte = int(rspec.get('net_max_kbyte', default_MaxKByte))
         if MaxKByte != self.MaxKByte:
             self.MaxKByte = MaxKByte

-            logger.log("bwmon:  Updating %s: Max KByte lim = %s" %(self.name, self.MaxKByte))
-                          
+            logger.log("bwmon: Updating %s: Max KByte lim = %s" %(self.name, self.MaxKByte))
+

         Maxi2KByte = int(rspec.get('net_i2_max_kbyte', default_Maxi2KByte))
         if Maxi2KByte != self.Maxi2KByte:
             self.Maxi2KByte = Maxi2KByte
         Maxi2KByte = int(rspec.get('net_i2_max_kbyte', default_Maxi2KByte))
         if Maxi2KByte != self.Maxi2KByte:
             self.Maxi2KByte = Maxi2KByte

-            logger.log("bwmon:  Updating %s: Max i2 KByte = %s" %(self.name, self.Maxi2KByte))
-                          
+            logger.log("bwmon: Updating %s: Max i2 KByte = %s" %(self.name, self.Maxi2KByte))
+

         ThreshKByte = int(rspec.get('net_thresh_kbyte', (MaxKByte * .8)))
         if ThreshKByte != self.ThreshKByte:
             self.ThreshKByte = ThreshKByte
         ThreshKByte = int(rspec.get('net_thresh_kbyte', (MaxKByte * .8)))
         if ThreshKByte != self.ThreshKByte:
             self.ThreshKByte = ThreshKByte

-            logger.log("bwmon:  Updating %s: Thresh KByte = %s" %(self.name, self.ThreshKByte))
-                          
+            logger.log("bwmon: Updating %s: Thresh KByte = %s" %(self.name, self.ThreshKByte))
+

         Threshi2KByte = int(rspec.get('net_i2_thresh_kbyte', (Maxi2KByte * .8)))
         Threshi2KByte = int(rspec.get('net_i2_thresh_kbyte', (Maxi2KByte * .8)))

-        if Threshi2KByte != self.Threshi2KByte:    
+        if Threshi2KByte != self.Threshi2KByte:

             self.Threshi2KByte = Threshi2KByte
             self.Threshi2KByte = Threshi2KByte

-            logger.log("bwmon:  Updating %s: i2 Thresh KByte = %s" %(self.name, self.Threshi2KByte))
- 
+            logger.log("bwmon: Updating %s: i2 Thresh KByte = %s" %(self.name, self.Threshi2KByte))
+

         Share = int(rspec.get('net_share', default_Share))
         if Share != self.Share:
             self.Share = Share
         Share = int(rspec.get('net_share', default_Share))
         if Share != self.Share:
             self.Share = Share

-            logger.log("bwmon:  Updating %s: Net Share = %s" %(self.name, self.Share))
+            logger.log("bwmon: Updating %s: Net Share = %s" %(self.name, self.Share))

 
         Sharei2 = int(rspec.get('net_i2_share', default_Share))
         if Sharei2 != self.Sharei2:
 
         Sharei2 = int(rspec.get('net_i2_share', default_Share))
         if Sharei2 != self.Sharei2:

-            self.Sharei2 = Sharei2 
-            logger.log("bwmon:  Updating %s: Net i2 Share = %s" %(self.name, self.i2Share))
+            self.Sharei2 = Sharei2
+            logger.log("bwmon: Updating %s: Net i2 Share = %s" %(self.name, self.i2Share))

 
 
     def reset(self, runningrates, rspec):
 
 
     def reset(self, runningrates, rspec):


@@ -280,7 +290,7 @@ class Slice:
         self.Share = runningrates.get('share', 1)
 
         # Query Node Manager for max rate overrides
         self.Share = runningrates.get('share', 1)
 
         # Query Node Manager for max rate overrides

-        self.updateSliceTags(rspec)    
+        self.updateSliceTags(rspec)

 
         # Reset baseline time
         self.time = time.time()
 
         # Reset baseline time
         self.time = time.time()


@@ -289,13 +299,13 @@ class Slice:
         self.bytes = runningrates.get('usedbytes', 0)
         self.i2bytes = runningrates.get('usedi2bytes', 0)
 
         self.bytes = runningrates.get('usedbytes', 0)
         self.i2bytes = runningrates.get('usedi2bytes', 0)
 

-        # Reset email 
+        # Reset email

         self.emailed = False
         # Reset flag
         self.capped = False
         # Reset rates.
         self.emailed = False
         # Reset flag
         self.capped = False
         # Reset rates.

-        maxrate = self.MaxRate * 1000 
-        minrate = self.MinRate * 1000 
+        maxrate = self.MaxRate * 1000
+        minrate = self.MinRate * 1000

         maxi2rate = self.Maxi2Rate * 1000
         mini2rate = self.Mini2Rate * 1000
 
         maxi2rate = self.Maxi2Rate * 1000
         mini2rate = self.Mini2Rate * 1000
 


@@ -304,13 +314,13 @@ class Slice:
          (maxi2rate != runningrates.get('maxexemptrate', 0)) or \
          (mini2rate != runningrates.get('minexemptrate', 0)) or \
          (self.Share != runningrates.get('share', 0)):
          (maxi2rate != runningrates.get('maxexemptrate', 0)) or \
          (mini2rate != runningrates.get('minexemptrate', 0)) or \
          (self.Share != runningrates.get('share', 0)):

-            logger.log("bwmon:  %s reset to %s/%s" % \
-                  (self.name,
-                   bwlimit.format_tc_rate(maxrate),
-                   bwlimit.format_tc_rate(maxi2rate)), 1)
-            bwlimit.set(xid = self.xid, 
-                minrate = self.MinRate * 1000, 
-                maxrate = self.MaxRate * 1000, 
+            logger.log("bwmon: %s reset to %s/%s" % \
+                           (self.name,
+                            bwlimit.format_tc_rate(maxrate),
+                            bwlimit.format_tc_rate(maxi2rate)))
+            bwlimit.set(xid = self.xid, dev = dev_default,
+                minrate = self.MinRate * 1000,
+                maxrate = self.MaxRate * 1000,

                 maxexemptrate = self.Maxi2Rate * 1000,
                 minexemptrate = self.Mini2Rate * 1000,
                 share = self.Share)
                 maxexemptrate = self.Maxi2Rate * 1000,
                 minexemptrate = self.Mini2Rate * 1000,
                 share = self.Share)


@@ -336,7 +346,7 @@ class Slice:
 
             # Cap low bandwidth burst rate
             message += template % params
 
             # Cap low bandwidth burst rate
             message += template % params

-            logger.log("bwmon:   ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)
+            logger.log("bwmon:  ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)

 
         if new_maxexemptrate != (self.Maxi2Rate * 1000):
             # Format template parameters for high bandwidth message
 
         if new_maxexemptrate != (self.Maxi2Rate * 1000):
             # Format template parameters for high bandwidth message


@@ -344,33 +354,33 @@ class Slice:
             params['bytes'] = format_bytes(usedi2bytes - self.i2bytes)
             params['limit'] = format_bytes(self.Maxi2KByte * 1024)
             params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate)
             params['bytes'] = format_bytes(usedi2bytes - self.i2bytes)
             params['limit'] = format_bytes(self.Maxi2KByte * 1024)
             params['new_maxrate'] = bwlimit.format_tc_rate(new_maxexemptrate)

- 
+

             message += template % params
             message += template % params

-            logger.log("bwmon:   ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)
-       
+            logger.log("bwmon:  ** %(slice)s %(class)s capped at %(new_maxrate)s/s " % params)
+

         # Notify slice
         if self.emailed == False:
             subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params
             if DEBUG:
         # Notify slice
         if self.emailed == False:
             subject = "pl_mom capped bandwidth of slice %(slice)s on %(hostname)s" % params
             if DEBUG:

-                logger.log("bwmon:  "+ subject)
-                logger.log("bwmon:  "+ message + (footer % params))
+                logger.log("bwmon: "+ subject)
+                logger.log("bwmon: "+ message + (footer % params))

             else:
                 self.emailed = True
             else:
                 self.emailed = True

-                logger.log("bwmon:  Emailing %s" % self.name)
+                logger.log("bwmon: Emailing %s" % self.name)

                 slicemail(self.name, subject, message + (footer % params))
 
 
     def update(self, runningrates, rspec):
         """
         Update byte counts and check if byte thresholds have been
                 slicemail(self.name, subject, message + (footer % params))
 
 
     def update(self, runningrates, rspec):
         """
         Update byte counts and check if byte thresholds have been

-        exceeded. If exceeded, cap to remaining bytes in limit over remaining time in period.  
+        exceeded. If exceeded, cap to remaining bytes in limit over remaining time in period.

         Recalculate every time module runs.
         """
         # cache share for later comparison
         runningrates['share'] = self.Share
 
         # Query Node Manager for max rate overrides
         Recalculate every time module runs.
         """
         # cache share for later comparison
         runningrates['share'] = self.Share
 
         # Query Node Manager for max rate overrides

-        self.updateSliceTags(rspec)    
+        self.updateSliceTags(rspec)

 
         usedbytes = runningrates['usedbytes']
         usedi2bytes = runningrates['usedi2bytes']
 
         usedbytes = runningrates['usedbytes']
         usedi2bytes = runningrates['usedi2bytes']


@@ -382,7 +392,8 @@ class Slice:
             bytesused = usedbytes - self.bytes
             timeused = int(time.time() - self.time)
             # Calcuate new rate. in bit/s
             bytesused = usedbytes - self.bytes
             timeused = int(time.time() - self.time)
             # Calcuate new rate. in bit/s

-            new_maxrate = int(((maxbyte - bytesused) * 8)/(period - timeused))
+            new_maxrate = int(((maxbyte - bytesused) * 8)
+                              / (period - timeused))

             # Never go under MinRate
             if new_maxrate < (self.MinRate * 1000):
                 new_maxrate = self.MinRate * 1000
             # Never go under MinRate
             if new_maxrate < (self.MinRate * 1000):
                 new_maxrate = self.MinRate * 1000


@@ -392,13 +403,14 @@ class Slice:
             # Sanity Check
             new_maxrate = self.MaxRate * 1000
             self.capped += False
             # Sanity Check
             new_maxrate = self.MaxRate * 1000
             self.capped += False

- 
+

         if usedi2bytes >= (self.i2bytes + (self.Threshi2KByte * 1024)):
             maxi2byte = self.Maxi2KByte * 1024
             i2bytesused = usedi2bytes - self.i2bytes
             timeused = int(time.time() - self.time)
             # Calcuate New Rate.
         if usedi2bytes >= (self.i2bytes + (self.Threshi2KByte * 1024)):
             maxi2byte = self.Maxi2KByte * 1024
             i2bytesused = usedi2bytes - self.i2bytes
             timeused = int(time.time() - self.time)
             # Calcuate New Rate.

-            new_maxi2rate = int(((maxi2byte - i2bytesused) * 8)/(period - timeused))
+            new_maxi2rate = int(((maxi2byte - i2bytesused) * 8)
+                                /(period - timeused))

             # Never go under MinRate
             if new_maxi2rate < (self.Mini2Rate * 1000):
                 new_maxi2rate = self.Mini2Rate * 1000
             # Never go under MinRate
             if new_maxi2rate < (self.Mini2Rate * 1000):
                 new_maxi2rate = self.Mini2Rate * 1000


@@ -414,11 +426,11 @@ class Slice:
         if (runningrates['maxrate'] != new_maxrate) or \
         (runningrates['minrate'] != self.MinRate * 1000) or \
         (runningrates['maxexemptrate'] != new_maxi2rate) or \
         if (runningrates['maxrate'] != new_maxrate) or \
         (runningrates['minrate'] != self.MinRate * 1000) or \
         (runningrates['maxexemptrate'] != new_maxi2rate) or \

-        (runningrates['minexemptrate'] != self.Mini2Rate * 1000) or \
+        ('minexemptrate' in runningrates and runningrates['minexemptrate'] != self.Mini2Rate * 1000) or \

         (runningrates['share'] != self.Share):
             # Apply parameters
         (runningrates['share'] != self.Share):
             # Apply parameters

-            bwlimit.set(xid = self.xid, 
-                minrate = self.MinRate * 1000, 
+            bwlimit.set(xid = self.xid, dev = dev_default,
+                minrate = self.MinRate * 1000,

                 maxrate = new_maxrate,
                 minexemptrate = self.Mini2Rate * 1000,
                 maxexemptrate = new_maxi2rate,
                 maxrate = new_maxrate,
                 minexemptrate = self.Mini2Rate * 1000,
                 maxexemptrate = new_maxi2rate,


@@ -435,12 +447,12 @@ def gethtbs(root_xid, default_xid):
     Turn off HTBs without names.
     """
     livehtbs = {}
     Turn off HTBs without names.
     """
     livehtbs = {}

-    for params in bwlimit.get():
+    for params in bwlimit.get(dev = dev_default):

         (xid, share,
          minrate, maxrate,
          minexemptrate, maxexemptrate,
          usedbytes, usedi2bytes) = params
         (xid, share,
          minrate, maxrate,
          minexemptrate, maxexemptrate,
          usedbytes, usedi2bytes) = params

-        
+

         name = bwlimit.get_slice(xid)
 
         if (name is None) \
         name = bwlimit.get_slice(xid)
 
         if (name is None) \


@@ -448,8 +460,8 @@ def gethtbs(root_xid, default_xid):
         and (xid != default_xid):
             # Orphaned (not associated with a slice) class
             name = "%d?" % xid
         and (xid != default_xid):
             # Orphaned (not associated with a slice) class
             name = "%d?" % xid

-            logger.log("bwmon:  Found orphaned HTB %s. Removing." %name, 1)
-            bwlimit.off(xid)
+            logger.log("bwmon: Found orphaned HTB %s. Removing." %name)
+            bwlimit.off(xid, dev = dev_default)

 
         livehtbs[xid] = {'share': share,
             'minrate': minrate,
 
         livehtbs[xid] = {'share': share,
             'minrate': minrate,


@@ -457,42 +469,48 @@ def gethtbs(root_xid, default_xid):
             'maxexemptrate': maxexemptrate,
             'minexemptrate': minexemptrate,
             'usedbytes': usedbytes,
             'maxexemptrate': maxexemptrate,
             'minexemptrate': minexemptrate,
             'usedbytes': usedbytes,

-            'name': name, 
+            'name': name,

             'usedi2bytes': usedi2bytes}
 
     return livehtbs
 
 def sync(nmdbcopy):
     """
             'usedi2bytes': usedi2bytes}
 
     return livehtbs
 
 def sync(nmdbcopy):
     """

-    Syncs tc, db, and bwmon.dat.  Then, starts new slices, kills old ones, and updates byte accounts for each running slice.  Sends emails and caps those that went over their limit.
+    Syncs tc, db, and bwmon.pickle.
+    Then, starts new slices, kills old ones, and updates byte accounts for each running slice.
+    Sends emails and caps those that went over their limit.

     """
     # Defaults
     """
     # Defaults

-    global datafile, \
+    global DB_FILE, \

         period, \
         default_MaxRate, \
         default_Maxi2Rate, \
         period, \
         default_MaxRate, \
         default_Maxi2Rate, \

-        default_MaxKByte,\
-        default_Maxi2KByte,\
-        default_Share
+        default_MaxKByte, \
+        default_Maxi2KByte, \
+        default_Share, \
+        dev_default

 
     # All slices
     names = []
 
     # All slices
     names = []

-    # Incase the limits have changed. 
-    default_MaxRate = int(bwlimit.get_bwcap() / 1000)
+    # In case the limits have changed.
+    default_MaxRate = int(bwlimit.get_bwcap(dev_default) / 1000)

     default_Maxi2Rate = int(bwlimit.bwmax / 1000)
 
     # Incase default isn't set yet.
     if default_MaxRate == -1:
         default_MaxRate = 1000000
 
     default_Maxi2Rate = int(bwlimit.bwmax / 1000)
 
     # Incase default isn't set yet.
     if default_MaxRate == -1:
         default_MaxRate = 1000000
 

+    # xxx $Id$
+    # with svn we used to have a trick to detect upgrades of this file
+    # this has gone with the move to git, without any noticeable effect on operations though

     try:
     try:

-        f = open(datafile, "r+")
-        logger.log("bwmon:  Loading %s" % datafile, 2)
+        f = open(DB_FILE, "r+")
+        logger.verbose("bwmon: Loading %s" % DB_FILE)

         (version, slices, deaddb) = pickle.load(f)
         f.close()
         # Check version of data file
         if version != "$Id$":
         (version, slices, deaddb) = pickle.load(f)
         f.close()
         # Check version of data file
         if version != "$Id$":

-            logger.log("bwmon:  Not using old version '%s' data file %s" % (version, datafile))
+            logger.log("bwmon: Not using old version '%s' data file %s" % (version, DB_FILE))

             raise Exception
     except Exception:
         version = "$Id$"
             raise Exception
     except Exception:
         version = "$Id$"


@@ -503,67 +521,67 @@ def sync(nmdbcopy):
     root_xid = bwlimit.get_xid("root")
     default_xid = bwlimit.get_xid("default")
 
     root_xid = bwlimit.get_xid("root")
     default_xid = bwlimit.get_xid("default")
 

-    # Since root is required for sanity, its not in the API/plc database, so pass {} 
+    # Since root is required for sanity, its not in the API/plc database, so pass {}

     # to use defaults.
     # to use defaults.

-    if root_xid not in slices.keys():
+    if root_xid not in list(slices.keys()):

         slices[root_xid] = Slice(root_xid, "root", {})
         slices[root_xid].reset({}, {})
         slices[root_xid] = Slice(root_xid, "root", {})
         slices[root_xid].reset({}, {})

-    
+

     # Used by bwlimit.  pass {} since there is no rspec (like above).
     # Used by bwlimit.  pass {} since there is no rspec (like above).

-    if default_xid not in slices.keys():
+    if default_xid not in list(slices.keys()):

         slices[default_xid] = Slice(default_xid, "default", {})
         slices[default_xid].reset({}, {})
 
     live = {}
     # Get running slivers that should be on this node (from plc). {xid: name}
     # db keys on name, bwmon keys on xid.  db doesnt have xid either.
         slices[default_xid] = Slice(default_xid, "default", {})
         slices[default_xid].reset({}, {})
 
     live = {}
     # Get running slivers that should be on this node (from plc). {xid: name}
     # db keys on name, bwmon keys on xid.  db doesnt have xid either.

-    for plcSliver in nmdbcopy.keys():
+    for plcSliver in list(nmdbcopy.keys()):

         live[bwlimit.get_xid(plcSliver)] = nmdbcopy[plcSliver]
 
         live[bwlimit.get_xid(plcSliver)] = nmdbcopy[plcSliver]
 

-    logger.log("bwmon:  Found %s instantiated slices" % live.keys().__len__(), 2)
-    logger.log("bwmon:  Found %s slices in dat file" % slices.values().__len__(), 2)
+    logger.verbose("bwmon: Found %s instantiated slices" % list(live.keys()).__len__())
+    logger.verbose("bwmon: Found %s slices in dat file" % list(slices.values()).__len__())

 
     # Get actual running values from tc.
     # Update slice totals and bandwidth. {xid: {values}}
     kernelhtbs = gethtbs(root_xid, default_xid)
 
     # Get actual running values from tc.
     # Update slice totals and bandwidth. {xid: {values}}
     kernelhtbs = gethtbs(root_xid, default_xid)

-    logger.log("bwmon:  Found %s running HTBs" % kernelhtbs.keys().__len__(), 2)
+    logger.verbose("bwmon: Found %s running HTBs" % list(kernelhtbs.keys()).__len__())

 
     # The dat file has HTBs for slices, but the HTBs aren't running
 
     # The dat file has HTBs for slices, but the HTBs aren't running

-    nohtbslices =  Set(slices.keys()) - Set(kernelhtbs.keys())
-    logger.log( "bwmon:  Found %s slices in dat but not running." % nohtbslices.__len__(), 2)
+    nohtbslices =  set(slices.keys()) - set(kernelhtbs.keys())
+    logger.verbose( "bwmon: Found %s slices in dat but not running." % nohtbslices.__len__())

     # Reset tc counts.
     for nohtbslice in nohtbslices:
     # Reset tc counts.
     for nohtbslice in nohtbslices:

-        if live.has_key(nohtbslice): 
+        if nohtbslice in live:

             slices[nohtbslice].reset( {}, live[nohtbslice]['_rspec'] )
         else:
             slices[nohtbslice].reset( {}, live[nohtbslice]['_rspec'] )
         else:

-            logger.log("bwmon:  Removing abondoned slice %s from dat." % nohtbslice)
+            logger.log("bwmon: Removing abondoned slice %s from dat." % nohtbslice)

             del slices[nohtbslice]
 
     # The dat file doesnt have HTB for the slice but kern has HTB
             del slices[nohtbslice]
 
     # The dat file doesnt have HTB for the slice but kern has HTB

-    slicesnodat = Set(kernelhtbs.keys()) - Set(slices.keys())
-    logger.log( "bwmon:  Found %s slices with HTBs but not in dat" % slicesnodat.__len__(), 2)
+    slicesnodat = set(kernelhtbs.keys()) - set(slices.keys())
+    logger.verbose( "bwmon: Found %s slices with HTBs but not in dat" % slicesnodat.__len__())

     for slicenodat in slicesnodat:
     for slicenodat in slicesnodat:

-        # But slice is running 
-        if live.has_key(slicenodat): 
+        # But slice is running
+        if slicenodat in live:

             # init the slice.  which means start accounting over since kernel
             # htb was already there.
             # init the slice.  which means start accounting over since kernel
             # htb was already there.

-            slices[slicenodat] = Slice(slicenodat, 
-                live[slicenodat]['name'], 
+            slices[slicenodat] = Slice(slicenodat,
+                live[slicenodat]['name'],

                 live[slicenodat]['_rspec'])
 
     # Get new slices.
     # Slices in GetSlivers but not running HTBs
                 live[slicenodat]['_rspec'])
 
     # Get new slices.
     # Slices in GetSlivers but not running HTBs

-    newslicesxids = Set(live.keys()) - Set(kernelhtbs.keys())
-    logger.log("bwmon:  Found %s new slices" % newslicesxids.__len__(), 2)
-       
+    newslicesxids = set(live.keys()) - set(kernelhtbs.keys())
+    logger.verbose("bwmon: Found %s new slices" % newslicesxids.__len__())
+

     # Setup new slices
     for newslice in newslicesxids:
         # Delegated slices dont have xids (which are uids) since they haven't been
         # instantiated yet.
     # Setup new slices
     for newslice in newslicesxids:
         # Delegated slices dont have xids (which are uids) since they haven't been
         # instantiated yet.

-        if newslice != None and live[newslice].has_key('_rspec') == True:
+        if newslice != None and ('_rspec' in live[newslice]) == True:

             # Check to see if we recently deleted this slice.
             # Check to see if we recently deleted this slice.

-            if live[newslice]['name'] not in deaddb.keys():
-                logger.log( "bwmon: New Slice %s" % live[newslice]['name'] )
+            if live[newslice]['name'] not in list(deaddb.keys()):
+                logger.log( "bwmon: new slice %s" % live[newslice]['name'] )

                 # _rspec is the computed rspec:  NM retrieved data from PLC, computed loans
                 # and made a dict of computed values.
                 slices[newslice] = Slice(newslice, live[newslice]['name'], live[newslice]['_rspec'])
                 # _rspec is the computed rspec:  NM retrieved data from PLC, computed loans
                 # and made a dict of computed values.
                 slices[newslice] = Slice(newslice, live[newslice]['name'], live[newslice]['_rspec'])


@@ -580,7 +598,7 @@ def sync(nmdbcopy):
                             "maxexemptrate": deadslice['slice'].Maxi2Rate * 1000,
                             "usedbytes": deadslice['htb']['usedbytes'] * 1000,
                             "usedi2bytes": deadslice['htb']['usedi2bytes'],
                             "maxexemptrate": deadslice['slice'].Maxi2Rate * 1000,
                             "usedbytes": deadslice['htb']['usedbytes'] * 1000,
                             "usedi2bytes": deadslice['htb']['usedi2bytes'],

-                            "share":deadslice['htb']['share']} 
+                            "share":deadslice['htb']['share']}

                 slices[newslice].reset(newvals, live[newslice]['_rspec'])
                 # Bring up to date
                 slices[newslice].update(newvals, live[newslice]['_rspec'])
                 slices[newslice].reset(newvals, live[newslice]['_rspec'])
                 # Bring up to date
                 slices[newslice].update(newvals, live[newslice]['_rspec'])


@@ -588,46 +606,46 @@ def sync(nmdbcopy):
                 del deaddb[deadslice['slice'].name]
                 del newvals
         else:
                 del deaddb[deadslice['slice'].name]
                 del newvals
         else:

-            logger.log("bwmon:  Slice %s doesn't have xid.  Skipping." % live[newslice]['name'])
+            logger.log("bwmon: Slice %s doesn't have xid.  Skipping." % live[newslice]['name'])

 
     # Move dead slices that exist in the pickle file, but
     # aren't instantiated by PLC into the dead dict until
     # recording period is over.  This is to avoid the case where a slice is dynamically created
     # and destroyed then recreated to get around byte limits.
 
     # Move dead slices that exist in the pickle file, but
     # aren't instantiated by PLC into the dead dict until
     # recording period is over.  This is to avoid the case where a slice is dynamically created
     # and destroyed then recreated to get around byte limits.

-    deadxids = Set(slices.keys()) - Set(live.keys())
-    logger.log("bwmon:  Found %s dead slices" % (deadxids.__len__() - 2), 2)
+    deadxids = set(slices.keys()) - set(live.keys())
+    logger.verbose("bwmon: Found %s dead slices" % (deadxids.__len__() - 2))

     for deadxid in deadxids:
         if deadxid == root_xid or deadxid == default_xid:
             continue
     for deadxid in deadxids:
         if deadxid == root_xid or deadxid == default_xid:
             continue

-        logger.log("bwmon:  removing dead slice %s " % deadxid)
-        if slices.has_key(deadxid) and kernelhtbs.has_key(deadxid):
+        logger.log("bwmon: removing dead slice %s " % deadxid)
+        if deadxid in slices and deadxid in kernelhtbs:

             # add slice (by name) to deaddb
             # add slice (by name) to deaddb

-            logger.log("bwmon:  Saving bandwidth totals for %s." % slices[deadxid].name)
+            logger.log("bwmon: Saving bandwidth totals for %s." % slices[deadxid].name)

             deaddb[slices[deadxid].name] = {'slice': slices[deadxid], 'htb': kernelhtbs[deadxid]}
             del slices[deadxid]
             deaddb[slices[deadxid].name] = {'slice': slices[deadxid], 'htb': kernelhtbs[deadxid]}
             del slices[deadxid]

-        if kernelhtbs.has_key(deadxid): 
-            logger.log("bwmon:  Removing HTB for %s." % deadxid, 2)
-            bwlimit.off(deadxid)
-    
+        if deadxid in kernelhtbs:
+            logger.verbose("bwmon: Removing HTB for %s." % deadxid)
+            bwlimit.off(deadxid, dev = dev_default)
+

     # Clean up deaddb
     # Clean up deaddb

-    for deadslice in deaddb.keys():
+    for deadslice in list(deaddb.keys()):

         if (time.time() >= (deaddb[deadslice]['slice'].time + period)):
         if (time.time() >= (deaddb[deadslice]['slice'].time + period)):

-            logger.log("bwmon:  Removing dead slice %s from dat." \
+            logger.log("bwmon: Removing dead slice %s from dat." \

                         % deaddb[deadslice]['slice'].name)
             del deaddb[deadslice]
 
     # Get actual running values from tc since we've added and removed buckets.
     # Update slice totals and bandwidth. {xid: {values}}
     kernelhtbs = gethtbs(root_xid, default_xid)
                         % deaddb[deadslice]['slice'].name)
             del deaddb[deadslice]
 
     # Get actual running values from tc since we've added and removed buckets.
     # Update slice totals and bandwidth. {xid: {values}}
     kernelhtbs = gethtbs(root_xid, default_xid)

-    logger.log("bwmon:  now %s running HTBs" % kernelhtbs.keys().__len__(), 2)
+    logger.verbose("bwmon: now %s running HTBs" % list(kernelhtbs.keys()).__len__())

 
     # Update all byte limites on all slices
 
     # Update all byte limites on all slices

-    for (xid, slice) in slices.iteritems():
+    for (xid, slice) in slices.items():

         # Monitor only the specified slices
         if xid == root_xid or xid == default_xid: continue
         if names and name not in names:
             continue
         # Monitor only the specified slices
         if xid == root_xid or xid == default_xid: continue
         if names and name not in names:
             continue

- 
+

         if (time.time() >= (slice.time + period)) or \
             (kernelhtbs[xid]['usedbytes'] < slice.bytes) or \
             (kernelhtbs[xid]['usedi2bytes'] < slice.i2bytes):
         if (time.time() >= (slice.time + period)) or \
             (kernelhtbs[xid]['usedbytes'] < slice.bytes) or \
             (kernelhtbs[xid]['usedi2bytes'] < slice.i2bytes):


@@ -637,31 +655,68 @@ def sync(nmdbcopy):
             # were re-initialized).
             slice.reset(kernelhtbs[xid], live[xid]['_rspec'])
         elif ENABLE:
             # were re-initialized).
             slice.reset(kernelhtbs[xid], live[xid]['_rspec'])
         elif ENABLE:

-            logger.log("bwmon:  Updating slice %s" % slice.name, 2)
+            logger.verbose("bwmon: Updating slice %s" % slice.name)

             # Update byte counts
             slice.update(kernelhtbs[xid], live[xid]['_rspec'])
 
             # Update byte counts
             slice.update(kernelhtbs[xid], live[xid]['_rspec'])
 

-    logger.log("bwmon:  Saving %s slices in %s" % (slices.keys().__len__(),datafile), 2)
-    f = open(datafile, "w")
+    logger.verbose("bwmon: Saving %s slices in %s" % (list(slices.keys()).__len__(), DB_FILE))
+    f = open(DB_FILE, "w")

     pickle.dump((version, slices, deaddb), f)
     f.close()
 
     pickle.dump((version, slices, deaddb), f)
     f.close()
 

+# doesnt use generic default interface because this runs as its own thread.
+# changing the config variable will not have an effect since GetSlivers: pass
+def getDefaults(nmdbcopy):
+    '''
+    Get defaults from default slice's slice attributes.
+    '''
+    status = True
+    # default slice
+    dfltslice = nmdbcopy.get(Config().PLC_SLICE_PREFIX+"_default")
+    if dfltslice:
+        if dfltslice['rspec']['net_max_rate'] == -1:
+            allOff()
+            status = False
+    return status
+
+
+def allOff():
+    """
+    Turn off all slice HTBs
+    """
+    # Get/set special slice IDs
+    root_xid = bwlimit.get_xid("root")
+    default_xid = bwlimit.get_xid("default")
+    kernelhtbs = gethtbs(root_xid, default_xid)
+    if len(kernelhtbs):
+        logger.log("bwmon: Disabling all running HTBs.")
+        for htb in list(kernelhtbs.keys()): bwlimit.off(htb, dev = dev_default)
+
+

 lock = threading.Event()
 def run():
 lock = threading.Event()
 def run():

-    """When run as a thread, wait for event, lock db, deep copy it, release it, run bwmon.GetSlivers(), then go back to waiting."""
-    logger.log("bwmon:  Thread started", 2)
+    """
+    When run as a thread, wait for event, lock db, deep copy it, release it,
+    run bwmon.GetSlivers(), then go back to waiting.
+    """
+    logger.verbose("bwmon: Thread started")

     while True:
         lock.wait()
     while True:
         lock.wait()

-        logger.log("bwmon:  Event received.  Running.", 2)
+        logger.verbose("bwmon: Event received.  Running.")

         database.db_lock.acquire()
         nmdbcopy = copy.deepcopy(database.db)
         database.db_lock.release()
         database.db_lock.acquire()
         nmdbcopy = copy.deepcopy(database.db)
         database.db_lock.release()

-        try:  sync(nmdbcopy)
-        except: logger.log_exc()
+        try:
+            if getDefaults(nmdbcopy) and len(bwlimit.tc("class show dev %s" % dev_default)) > 0:
+                # class show to check if net:InitNodeLimit:bwlimit.init has run.
+                sync(nmdbcopy)
+            else: logger.log("bwmon: BW limits DISABLED.")
+        except: logger.log_exc("bwmon failed")

         lock.clear()
 
 def start(*args):
     tools.as_daemon_thread(run)
 
 def GetSlivers(*args):
         lock.clear()
 
 def start(*args):
     tools.as_daemon_thread(run)
 
 def GetSlivers(*args):

+    logger.verbose ("bwmon: triggering dummy GetSlivers")

     pass
     pass