reguire gnupg1 on f>=31; sense the system to use gpg1 when installed

[nodemanager.git] / database.py

diff --git a/database.py b/database.py
index 1985b58..a7099bc 100644 (file)
--- a/database.py
+++ b/database.py
@@ -1,6 +1,4 @@
-# $Id$
-# $URL$
-
+#

 """The database houses information on slivers.  This information
 reaches the sliver manager in two different ways: one, through the
 GetSlivers() call made periodically; two, by users delivering tickets.
 """The database houses information on slivers.  This information
 reaches the sliver manager in two different ways: one, through the
 GetSlivers() call made periodically; two, by users delivering tickets.


@@ -15,26 +13,33 @@ In order to maintain service when the node reboots during a network
 partition, the database is constantly being dumped to disk.
 """
 
 partition, the database is constantly being dumped to disk.
 """
 

-import cPickle
+import sys
+
+import pickle

 import threading
 import time
 
 import threading
 import time
 

-import accounts
+import account

 import logger
 import tools
 import bwmon
 
 import logger
 import tools
 import bwmon
 

+# hopefully temporary
+# is there a good reason to have this done here and not in a plugin ?
+try:    from coresched_lxc import CoreSched
+except: from coresched_vs  import CoreSched
+

 # We enforce minimum allocations to keep the clueless from hosing their slivers.
 # Disallow disk loans because there's currently no way to punish slivers over quota.
 # We enforce minimum allocations to keep the clueless from hosing their slivers.
 # Disallow disk loans because there's currently no way to punish slivers over quota.

-MINIMUM_ALLOCATION = {'cpu_pct': 0, 
-                      'cpu_share': 1, 
-                      'net_min_rate': 0, 
-                      'net_max_rate': 8, 
-                      'net_i2_min_rate': 0, 
-                      'net_i2_max_rate': 8, 
+MINIMUM_ALLOCATION = {'cpu_pct': 0,
+                      'cpu_share': 1,
+                      'net_min_rate': 0,
+                      'net_max_rate': 8,
+                      'net_i2_min_rate': 0,
+                      'net_i2_max_rate': 8,

                       'net_share': 1,
                       }
                       'net_share': 1,
                       }

-LOANABLE_RESOURCES = MINIMUM_ALLOCATION.keys()
+LOANABLE_RESOURCES = list(MINIMUM_ALLOCATION.keys())

 
 DB_FILE = '/var/lib/nodemanager/database.pickle'
 
 
 DB_FILE = '/var/lib/nodemanager/database.pickle'
 


@@ -64,19 +69,19 @@ class Database(dict):
         self._min_timestamp = 0
 
     def _compute_effective_rspecs(self):
         self._min_timestamp = 0
 
     def _compute_effective_rspecs(self):

-        """Calculate the effects of loans and store the result in field _rspec. 
+        """Calculate the effects of loans and store the result in field _rspec.

 At the moment, we allow slivers to loan only those resources that they have received directly from PLC.
 At the moment, we allow slivers to loan only those resources that they have received directly from PLC.

-In order to do the accounting, we store three different rspecs: 
- * field 'rspec', which is the resources given by PLC; 
- * field '_rspec', which is the actual amount of resources the sliver has after all loans; 
- * and variable resid_rspec, which is the amount of resources the sliver 
+In order to do the accounting, we store three different rspecs:
+ * field 'rspec', which is the resources given by PLC;
+ * field '_rspec', which is the actual amount of resources the sliver has after all loans;
+ * and variable resid_rspec, which is the amount of resources the sliver

    has after giving out loans but not receiving any."""
         slivers = {}
    has after giving out loans but not receiving any."""
         slivers = {}

-        for name, rec in self.iteritems():
+        for name, rec in self.items():

             if 'rspec' in rec:
                 rec['_rspec'] = rec['rspec'].copy()
                 slivers[name] = rec
             if 'rspec' in rec:
                 rec['_rspec'] = rec['rspec'].copy()
                 slivers[name] = rec

-        for rec in slivers.itervalues():
+        for rec in slivers.values():

             eff_rspec = rec['_rspec']
             resid_rspec = rec['rspec'].copy()
             for target, resource_name, amount in rec.get('_loans', []):
             eff_rspec = rec['_rspec']
             resid_rspec = rec['rspec'].copy()
             for target, resource_name, amount in rec.get('_loans', []):


@@ -93,18 +98,20 @@ keys."""
         if rec['timestamp'] < self._min_timestamp: return
         name = rec['name']
         old_rec = self.get(name)
         if rec['timestamp'] < self._min_timestamp: return
         name = rec['name']
         old_rec = self.get(name)

-        if old_rec == None: self[name] = rec
+        if old_rec == None:
+            self[name] = rec

         elif rec['timestamp'] > old_rec['timestamp']:
         elif rec['timestamp'] > old_rec['timestamp']:

-            for key in old_rec.keys():
-                if not key.startswith('_'): del old_rec[key]
+            for key in list(old_rec.keys()):
+                if not key.startswith('_'):
+                    del old_rec[key]

             old_rec.update(rec)
 
     def set_min_timestamp(self, ts):
             old_rec.update(rec)
 
     def set_min_timestamp(self, ts):

-        """The ._min_timestamp member is the timestamp on the last comprehensive update.  
-We use it to determine if a record is stale.  
+        """The ._min_timestamp member is the timestamp on the last comprehensive update.
+We use it to determine if a record is stale.

 This method should be called whenever new GetSlivers() data comes in."""
         self._min_timestamp = ts
 This method should be called whenever new GetSlivers() data comes in."""
         self._min_timestamp = ts

-        for name, rec in self.items():
+        for name, rec in list(self.items()):

             if rec['timestamp'] < ts: del self[name]
 
     def sync(self):
             if rec['timestamp'] < ts: del self[name]
 
     def sync(self):


@@ -115,40 +122,48 @@ It may be necessary in the future to do something smarter."""
 
         # delete expired records
         now = time.time()
 
         # delete expired records
         now = time.time()

-        for name, rec in self.items():
+        for name, rec in list(self.items()):

             if rec.get('expires', now) < now: del self[name]
 
         self._compute_effective_rspecs()
 
             if rec.get('expires', now) < now: del self[name]
 
         self._compute_effective_rspecs()
 

+        try:
+            coresched = CoreSched()
+            coresched.adjustCores(self)
+        except:
+            logger.log_exc("database: exception while doing core sched")
+

         # create and destroy accounts as needed
         logger.verbose("database: sync : fetching accounts")
         # create and destroy accounts as needed
         logger.verbose("database: sync : fetching accounts")

-        existing_acct_names = accounts.all()
+        existing_acct_names = account.all()

         for name in existing_acct_names:
         for name in existing_acct_names:

-            if name not in self: 
+            if name not in self:

                 logger.verbose("database: sync : ensure_destroy'ing %s"%name)
                 logger.verbose("database: sync : ensure_destroy'ing %s"%name)

-                accounts.get(name).ensure_destroyed()
-        for name, rec in self.iteritems():
-            # protect this; if anything fails for a given sliver 
+                account.get(name).ensure_destroyed()
+        for name, rec in self.items():
+            # protect this; if anything fails for a given sliver

             # we still need the other ones to be handled
             try:
             # we still need the other ones to be handled
             try:

-                sliver = accounts.get(name)
-                logger.verbose("database: sync : looping on %s (shell account class from pwd %s)" %(name,sliver._get_class()))
+                sliver = account.get(name)
+                logger.verbose("database: sync : looping on %s (shell account class from pwd %s)" %(name, sliver._get_class()))

                 # Make sure we refresh accounts that are running
                 # Make sure we refresh accounts that are running

-                if rec['instantiation'] == 'plc-instantiated': 
+                if rec['instantiation'] == 'plc-instantiated':

                     logger.verbose ("database: sync : ensure_create'ing 'instantiation' sliver %s"%name)
                     sliver.ensure_created(rec)
                     logger.verbose ("database: sync : ensure_create'ing 'instantiation' sliver %s"%name)
                     sliver.ensure_created(rec)

-                elif rec['instantiation'] == 'nm-controller': 
+                elif rec['instantiation'] == 'nm-controller':

                     logger.verbose ("database: sync : ensure_create'ing 'nm-controller' sliver %s"%name)
                     sliver.ensure_created(rec)
                 # Back door to ensure PLC overrides Ticket in delegation.
                 elif rec['instantiation'] == 'delegated' and sliver._get_class() != None:
                     # if the ticket has been delivered and the nm-controller started the slice
                     # update rspecs and keep them up to date.
                     logger.verbose ("database: sync : ensure_create'ing 'nm-controller' sliver %s"%name)
                     sliver.ensure_created(rec)
                 # Back door to ensure PLC overrides Ticket in delegation.
                 elif rec['instantiation'] == 'delegated' and sliver._get_class() != None:
                     # if the ticket has been delivered and the nm-controller started the slice
                     # update rspecs and keep them up to date.

-                    if sliver.is_running(): 
+                    if sliver.is_running():

                         logger.verbose ("database: sync : ensure_create'ing 'delegated' sliver %s"%name)
                         sliver.ensure_created(rec)
                         logger.verbose ("database: sync : ensure_create'ing 'delegated' sliver %s"%name)
                         sliver.ensure_created(rec)

+            except SystemExit as e:
+                sys.exit(e)

             except:
             except:

-                logger.log_exc("database: sync failed to handle sliver",name=name)
+                logger.log_exc("database: sync failed to handle sliver", name=name)

 
         # Wake up bwmom to update limits.
         bwmon.lock.set()
 
         # Wake up bwmom to update limits.
         bwmon.lock.set()


@@ -166,18 +181,19 @@ It proceeds to handle dump requests forever."""
         while True:
             db_lock.acquire()
             while not dump_requested: db_cond.wait()
         while True:
             db_lock.acquire()
             while not dump_requested: db_cond.wait()

-            db_pickle = cPickle.dumps(db, cPickle.HIGHEST_PROTOCOL)
+            db_pickle = pickle.dumps(db, pickle.HIGHEST_PROTOCOL)

             dump_requested = False
             db_lock.release()
             dump_requested = False
             db_lock.release()

-            try: 
-                tools.write_file(DB_FILE, lambda f: f.write(db_pickle))
+            try:
+                tools.write_file(
+                    DB_FILE, lambda f: f.write(db_pickle), binary=True)

                 logger.log_database(db)
                 logger.log_database(db)

-            except: 
+            except:

                 logger.log_exc("database.start: failed to pickle/dump")
     global db
     try:
         f = open(DB_FILE)
                 logger.log_exc("database.start: failed to pickle/dump")
     global db
     try:
         f = open(DB_FILE)

-        try: db = cPickle.load(f)
+        try: db = pickle.load(f)

         finally: f.close()
     except IOError:
         logger.log ("database: Could not load %s -- starting from a fresh database"%DB_FILE)
         finally: f.close()
     except IOError:
         logger.log ("database: Could not load %s -- starting from a fresh database"%DB_FILE)