Use cgroup CPU scheduler
[util-vserver-pl.git] / python / vserver.py
index 590ec1b..fa58beb 100644 (file)
@@ -17,7 +17,6 @@ import resource
 import vserverimpl
 import cpulimit, bwlimit
 
-from vserverimpl import VS_SCHED_CPU_GUARANTEED as SCHED_CPU_GUARANTEED
 from vserverimpl import DLIMIT_INF
 from vserverimpl import VC_LIM_KEEP
 from vserverimpl import VLIMIT_NSOCK
@@ -42,6 +41,8 @@ RLIMITS = { "NSOCK": VLIMIT_NSOCK,
             "ANON": VLIMIT_ANON,
             "SHMEM": VLIMIT_SHMEM}
 
+CPU_SHARE_MULT = 1024
+
 # add in the platform supported rlimits
 for entry in resource.__dict__.keys():
     if entry.find("RLIMIT_")==0:
@@ -135,10 +136,9 @@ class VServer:
     INITSCRIPTS = [('/etc/rc.vinit', 'start'),
                    ('/etc/rc.d/rc', '%(runlevel)d')]
 
-    def __init__(self, name, vm_id = None, vm_running = None):
+    def __init__(self, name, vm_id = None, vm_running = None, logfile=None):
 
         self.name = name
-        self.rlimits_changed = False
         self.dir = "%s/%s" % (vserverimpl.VSERVER_BASEDIR, name)
         if not (os.path.isdir(self.dir) and
                 os.access(self.dir, os.R_OK | os.W_OK | os.X_OK)):
@@ -151,69 +151,80 @@ class VServer:
         if vm_running == None:
             vm_running = self.is_running()
         self.vm_running = vm_running
+        self.logfile = logfile
 
-    def have_limits_changed(self):
-        return self.rlimits_changed
+    # inspired from nodemanager's logger
+    def log(self,msg):
+        if self.logfile:
+            try:
+                fd = os.open(self.logfile,os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0600)
+                if not msg.endswith('\n'): msg += '\n'
+                os.write(fd, '%s: %s' % (time.asctime(time.gmtime()), msg))
+                os.close(fd)
+            except:
+                print '%s: (%s failed to open) %s'%(time.asctime(time.gmtime()),self.logfile,msg)
 
-    def set_rlimit_limit(self,type,hard,soft,minimum):
+    def set_rlimit(self, type, hard, soft, min):
         """Generic set resource limit function for vserver"""
         global RLIMITS
-        changed = False
-        try:
-            old_hard, old_soft, old_minimum = self.get_rlimit_limit(type)
-            if old_hard != VC_LIM_KEEP and old_hard <> hard: changed = True
-            if old_soft != VC_LIM_KEEP and old_soft <> soft: changed = True
-            if old_minimum != VC_LIM_KEEP and old_minimum <> minimum: changed = True
-            self.rlimits_changed = self.rlimits_changed or changed 
-        except OSError, e:
-            if self.is_running(): print "Unexpected error with getrlimit for running context %d" % self.ctx
+        update = False
 
-        resource_type = RLIMITS[type]
-        try:
-            ret = vserverimpl.setrlimit(self.ctx,resource_type,hard,soft,minimum)
-        except OSError, e:
-            if self.is_running(): print "Unexpected error with setrlimit for running context %d" % self.ctx
-
-    def set_rlimit_config(self,type,hard,soft,minimum):
-        """Generic set resource limit function for vserver"""
         if hard <> VC_LIM_KEEP:
             self.config.update('rlimits/%s.hard' % type.lower(), hard)
+            update = True
         if soft <> VC_LIM_KEEP:
             self.config.update('rlimits/%s.soft' % type.lower(), soft)
-        if minimum <> VC_LIM_KEEP:
-            self.config.update('rlimits/%s.min' % type.lower(), minimum)
-        self.set_rlimit_limit(type,hard,soft,minimum)
+            update = True
+        if min <> VC_LIM_KEEP:
+            self.config.update('rlimits/%s.min' % type.lower(), min)
+            update = True
 
-    def get_rlimit_limit(self,type):
-        """Generic get resource configuration function for vserver"""
-        global RLIMITS
-        resource_type = RLIMITS[type]
-        try:
-            ret = vserverimpl.getrlimit(self.ctx,resource_type)
-        except OSError, e:
-            print "Unexpected error with getrlimit for context %d" % self.ctx
-            ret = self.get_rlimit_config(type)
-        return ret
+        if self.is_running() and update:
+            resource_type = RLIMITS[type]
+            try:
+                vserverimpl.setrlimit(self.ctx, resource_type, hard, soft, min)
+            except OSError, e:
+                self.log("Error: setrlimit(%d, %s, %d, %d, %d): %s"
+                         % (self.ctx, type.lower(), hard, soft, min))
+
+        return update
 
-    def get_rlimit_config(self,type):
-        """Generic get resource configuration function for vserver"""
-        hard = int(self.config.get("rlimits/%s.hard"%type.lower(),VC_LIM_KEEP))
-        soft = int(self.config.get("rlimits/%s.soft"%type.lower(),VC_LIM_KEEP))
-        minimum = int(self.config.get("rlimits/%s.min"%type.lower(),VC_LIM_KEEP))
-        return (hard,soft,minimum)
+    def get_prefix_from_capabilities(self, capabilities, prefix):
+        split_caps = capabilities.split(',')
+        return ",".join(["%s" % (c) for c in split_caps if c.startswith(prefix.upper()) or c.startswith(prefix.lower())])
 
-    def set_capabilities(self, capabilities):
-        return vserverimpl.setbcaps(self.ctx, vserverimpl.text2bcaps(capabilities))
+    def get_bcaps_from_capabilities(self, capabilities):
+        return self.get_prefix_from_capabilities(capabilities, "cap_")
+
+    def get_ccaps_from_capabilities(self, capabilities):
+        return self.get_prefix_from_capabilities(capabilities, "vxc_")
 
     def set_capabilities_config(self, capabilities):
-        self.config.update('bcapabilities', capabilities)
-        self.set_capabilities(capabilities)
+        bcaps = self.get_bcaps_from_capabilities(capabilities)
+        ccaps = self.get_ccaps_from_capabilities(capabilities)
+        if len(bcaps) > 0:
+            bcaps += ","
+        bcaps += "CAP_NET_RAW"
+        self.config.update('bcapabilities', bcaps)
+        self.config.update('ccapabilities', ccaps)
+        ret = vserverimpl.setbcaps(self.ctx, vserverimpl.text2bcaps(bcaps))
+        if ret > 0:
+            return ret
+        return vserverimpl.setccaps(self.ctx, vserverimpl.text2ccaps(ccaps))
 
     def get_capabilities(self):
-        return vserverimpl.bcaps2text(vserverimpl.getbcaps(self.ctx))
+        bcaps = vserverimpl.bcaps2text(vserverimpl.getbcaps(self.ctx))
+        ccaps = vserverimpl.ccaps2text(vserverimpl.getccaps(self.ctx))
+        if bcaps and ccaps:
+            ccaps = "," + ccaps
+        return (bcaps + ccaps)
  
     def get_capabilities_config(self):
-        return self.config.get('bcapabilities', '')
+        bcaps = self.config.get('bcapabilities', '')
+        ccaps = self.config.get('ccapabilities', '')
+        if bcaps and ccaps:
+            ccaps = "," + ccaps
+        return (bcaps + ccaps)
 
     def set_ipaddresses(self, addresses):
         vserverimpl.netremove(self.ctx, "all")
@@ -221,6 +232,7 @@ class VServer:
             vserverimpl.netadd(self.ctx, a)
 
     def set_ipaddresses_config(self, addresses):
+        return
         i = 0
         for a in addresses.split(","):
             self.config.update("interfaces/%d/ip" % i, a)
@@ -244,6 +256,13 @@ class VServer:
         # No clean way to do this right now.
         return None
 
+    def get_unshare_netns_config(self):
+        try:
+            unshare_netns = int(self.config.get('spaces/net'))
+        except:
+            unshare_netns = 0;
+        return unshare_netns;
+
     def __do_chroot(self):
         os.chroot(self.dir)
         os.chdir("/")
@@ -271,7 +290,7 @@ class VServer:
             try:
                 vserverimpl.unsetdlimit(self.dir, self.ctx)
             except OSError, e:
-                print "Unexpected error with unsetdlimit for context %d" % self.ctx
+                self.log("Unexpected error with unsetdlimit for context %d" % self.ctx)
             return
 
         if self.vm_running:
@@ -282,7 +301,6 @@ class VServer:
             block_usage = self.disk_blocks
             inode_usage = self.disk_inodes
 
-
         try:
             vserverimpl.setdlimit(self.dir,
                                   self.ctx,
@@ -292,7 +310,7 @@ class VServer:
                                   vserverimpl.DLIMIT_INF,  # inode limit
                                   2)   # %age reserved for root
         except OSError, e:
-            print "Unexpected error with setdlimit for context %d" % self.ctx
+            self.log("Unexpected error with setdlimit for context %d" % self.ctx)
 
 
         self.config.update('dlimits/0/space_total', block_limit)
@@ -313,29 +331,30 @@ class VServer:
 
         return block_limit
 
-    def set_sched_config(self, cpu_share, sched_flags):
+    def set_sched_config(self, cpu_min, cpu_share):
 
         """ Write current CPU scheduler parameters to the vserver
-        configuration file. This method does not modify the kernel CPU
-        scheduling parameters for this context. """
-
-        if sched_flags & SCHED_CPU_GUARANTEED:
-            cpu_guaranteed = cpu_share
-        else:
-            cpu_guaranteed = 0
-        self.config.update('sched/fill-rate2', cpu_share)
-        self.config.update('sched/fill-rate', cpu_guaranteed)
-
-        if self.vm_running:
-            self.set_sched(cpu_share, sched_flags)
-
-    def set_sched(self, cpu_share, sched_flags = 0):
-        """ Update kernel CPU scheduling parameters for this context. """
-        vserverimpl.setsched(self.ctx, cpu_share, sched_flags)
+        configuration file. Currently, 'cpu_min' is not supported. """
+        self.config.update('cgroup/cpu.shares', cpu_share * CPU_SHARE_MULT)
+        if self.is_running():
+            self.set_sched(cpu_min, cpu_share)
+
+    def set_sched(self, cpu_min, cpu_share):
+        """ Update kernel CPU scheduling parameters for this context.
+        Currently, 'cpu_min' is not supported. """
+        try:
+            cgroup = open('/dev/cgroup/%s/cpu.shares' % name, 'w')
+            cgroup.write('%s' % (cpu_share * CPU_SHARE_MULT))
+            cgroup.close()
+        except:
+            pass
 
     def get_sched(self):
-        # have no way of querying scheduler right now on a per vserver basis
-        return (-1, False)
+        try:
+            cpu_share = int(int(self.config.get('cgroup/cpu.shares')) / CPU_SHARE_MULT)
+        except:
+            cpu_share = False
+        return (-1, cpu_share)
 
     def set_bwlimit(self, minrate = bwlimit.bwmin, maxrate = None,
                     exempt_min = None, exempt_max = None,
@@ -365,11 +384,11 @@ class VServer:
             print >>state_file, "%u" % self.ctx
             state_file.close()
 
-        if vserverimpl.chcontext(self.ctx, vserverimpl.text2bcaps(self.get_capabilities_config())):
+        if vserverimpl.chcontext(self.ctx, vserverimpl.text2bcaps(self.get_capabilities_config()), self.get_unshare_netns_config()):
             self.set_resources()
             vserverimpl.setup_done(self.ctx)
 
-    def __prep(self, runlevel, log):
+    def __prep(self, runlevel):
 
         """ Perform all the crap that the vserver script does before
         actually executing the startup scripts. """
@@ -409,65 +428,23 @@ class VServer:
             raise ex
 
     def enter(self):
-        self.config.cache_it()
-        self.__do_chroot()
-        self.__do_chcontext(None)
-
-    def start(self, wait, runlevel = 3):
-        self.vm_running = True
-        self.rlimits_changed = False
+        subprocess.call("/usr/sbin/vserver %s enter" % self.name, shell=True)
 
-        child_pid = os.fork()
-        if child_pid == 0:
+    def start(self, runlevel = 3):
+        if (os.fork() != 0):
+            # Parent should just return.
+            self.vm_running = True
+            return
+        else:
             # child process
             try:
-                # get a new session
-                os.setsid()
-
-                # open state file to record vserver info
-                state_file = open("/var/run/vservers/%s" % self.name, "w")
-
-                # use /dev/null for stdin, /var/log/boot.log for stdout/err
-                fd = os.open("/dev/null", os.O_RDONLY)
-                if fd != 0:
-                    os.dup2(fd, 0)
-                    os.close(fd)
-                self.config.cache_it()
-                self.__do_chroot()
-                log = open("/var/log/boot.log", "w", 0)
-                if log.fileno() != 1:
-                    os.dup2(log.fileno(), 1)
-                os.dup2(1, 2)
-
-                print >>log, ("%s: starting the virtual server %s" %
-                              (time.asctime(time.gmtime()), self.name))
-
-                # perform pre-init cleanup
-                self.__prep(runlevel, log)
-
-                # execute each init script in turn
-                # XXX - we don't support all scripts that vserver script does
-                self.__do_chcontext(state_file)
-                for cmd in self.INITSCRIPTS:
-                     try:
-                         # enter vserver context
-                         arg_subst = { 'runlevel': runlevel }
-                         cmd_args = [cmd[0]] + map(lambda x: x % arg_subst,
-                                                   cmd[1:])
-                         print >>log, "executing '%s'" % " ".join(cmd_args)
-                         os.spawnvp(os.P_NOWAIT,cmd[0],cmd_args)
-                     except:
-                         traceback.print_exc()
-                         os._exit(1)
-
+                subprocess.call("/usr/sbin/vserver %s start" % self.name, 
+                                shell=True)
             # we get here due to an exception in the top-level child process
             except Exception, ex:
-                traceback.print_exc()
+                self.log(traceback.format_exc())
             os._exit(0)
 
-        # parent process
-        return child_pid
-
     def set_resources(self):
 
         """ Called when vserver context is entered for first time,
@@ -476,6 +453,13 @@ class VServer:
         pass
 
     def init_disk_info(self):
+        try:
+            dlimit = vserverimpl.getdlimit(self.dir, self.ctx)
+            self.disk_blocks = dlimit[0]
+            self.disk_inodes = dlimit[2]
+            return self.disk_blocks * 1024
+        except Exception, e:
+            pass
         cmd = "/usr/sbin/vdu --script --space --inodes --blocksize 1024 --xid %d %s" % (self.ctx, self.dir)
         p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
@@ -496,10 +480,16 @@ class VServer:
         return self.disk_blocks * 1024
 
     def stop(self, signal = signal.SIGKILL):
-        vserverimpl.killall(self.ctx, signal)
         self.vm_running = False
-        self.rlimits_changed = False
+        subprocess.call("/usr/sbin/vserver %s stop" % self.name, shell=True)
+
+    def setname(self, slice_id):
+        '''Set vcVHI_CONTEXT field in kernel to slice_id'''
+        vserverimpl.setname(self.ctx, slice_id)
 
+    def getname(self):
+        '''Get vcVHI_CONTEXT field in kernel'''
+        return vserverimpl.getname(self.ctx)
 
 
 def create(vm_name, static = False, ctor = VServer):
@@ -518,3 +508,12 @@ def create(vm_name, static = False, ctor = VServer):
     vm_id = pwd.getpwnam(vm_name)[2]
 
     return ctor(vm_name, vm_id)
+
+
+def close_nonstandard_fds():
+    """Close all open file descriptors other than 0, 1, and 2."""
+    _SC_OPEN_MAX = 4
+    for fd in range(3, os.sysconf(_SC_OPEN_MAX)):
+        try: os.close(fd)
+        except OSError: pass  # most likely an fd that isn't open
+