-#
-
"""LibVirt slivers"""
-import accounts
-import logger
+import sys
+import os, os.path
+import re
import subprocess
-import os
+import pprint
+import random
+
import libvirt
-import sys
-import shutil
-from string import Template
+from account import Account
+import logger
+import plnode.bwlimit as bwlimit
+import cgroups
-states = {
- libvirt.VIR_DOMAIN_NOSTATE: 'no state',
- libvirt.VIR_DOMAIN_RUNNING: 'running',
- libvirt.VIR_DOMAIN_BLOCKED: 'blocked on resource',
- libvirt.VIR_DOMAIN_PAUSED: 'paused by user',
+STATES = {
+ libvirt.VIR_DOMAIN_NOSTATE: 'no state',
+ libvirt.VIR_DOMAIN_RUNNING: 'running',
+ libvirt.VIR_DOMAIN_BLOCKED: 'blocked on resource',
+ libvirt.VIR_DOMAIN_PAUSED: 'paused by user',
libvirt.VIR_DOMAIN_SHUTDOWN: 'being shut down',
- libvirt.VIR_DOMAIN_SHUTOFF: 'shut off',
- libvirt.VIR_DOMAIN_CRASHED: 'crashed',
+ libvirt.VIR_DOMAIN_SHUTOFF: 'shut off',
+ libvirt.VIR_DOMAIN_CRASHED: 'crashed',
}
-class Sliver_LV(accounts.Account):
- """This class wraps LibVirt commands"""
-
- SHELL = '/bin/sh'
+# with fedora24 and (broken) libvirt-python-1.3.3-3,
+# the following symbols are not available
+# kashyap on IRC reported that libvirt-python-1.3.5-1.fc24.x86_64
+# did not have the issue though
+try:
+ REASONS = {
+ # 0
+ libvirt.VIR_CONNECT_CLOSE_REASON_ERROR: 'Misc I/O error',
+ # 1
+ libvirt.VIR_CONNECT_CLOSE_REASON_EOF: 'End-of-file from server',
+ # 2
+ libvirt.VIR_CONNECT_CLOSE_REASON_KEEPALIVE: 'Keepalive timer triggered',
+ # 3
+ libvirt.VIR_CONNECT_CLOSE_REASON_CLIENT: 'Client requested it',
+ }
+except:
+ REASONS = {
+ # libvirt.VIR_CONNECT_CLOSE_REASON_ERROR
+ 0 : 'Misc I/O error',
+ # libvirt.VIR_CONNECT_CLOSE_REASON_EOF
+ 1 : 'End-of-file from server',
+ # libvirt.VIR_CONNECT_CLOSE_REASON_KEEPALIVE
+ 2 : 'Keepalive timer triggered',
+ # libvirt.VIR_CONNECT_CLOSE_REASON_CLIENT
+ 3 : 'Client requested it',
+ }
+ logger.log("WARNING : using hard-wired constants instead of symbolic names for CONNECT_CLOSE*")
- # Need to add a tag at myplc to actually use this account
- # type = 'sliver.LIBVIRT'
- TYPE = 'sliver.LIBVIRT'
-
+connections = dict()
- @staticmethod
- def create(name, rec = None):
- ''' Create dirs, copy fs image, lxc_create '''
- logger.verbose ('sliver_libvirt: %s create'%(name))
- dir = '/vservers/%s'%(name)
-
- # Template for libvirt sliver configuration
- template = Template(open('/vservers/.lvref/config_template.xml').read())
- config = template.substitute(name=name)
-
- lxc_log = '%s/log'%(dir)
-
- # Get the type of image from vref myplc tags specified as:
- # pldistro = lxc
- # fcdistro = squeeze
- # arch x86_64
- vref = rec['vref']
- if vref is None:
- logger.log("sliver_libvirt: %s: WARNING - no vref attached defaults to lxc-debian"%(name))
- vref = "lxc-squeeze-x86_64"
-
- # check the template exists -- there's probably a better way..
- if not os.path.isdir ("/vservers/.lvref/%s"%vref):
- logger.log ("sliver_libvirt: %s: ERROR Could not create sliver - reference image %s not found"%(name,vref))
- return
-
- # Copy the reference image fs
- # shutil.copytree("/vservers/.lvref/%s"%vref, "/vservers/%s"%name, symlinks=True)
- command = ['cp', '-r', '/vservers/.lvref/%s'%vref, '/vservers/%s'%name]
- logger.log_call(command, timeout=15*60)
-
- # Set hostname
- file('/vservers/%s/etc/hostname' % name, 'w').write(name)
-
- # Add slices group if not already present
- command = ['/usr/sbin/groupadd slices']
- logger.log_call(command, timeout=15*60)
- # Add unix account
- command = ['/usr/sbin/useradd', '-g', 'slices', '-s', '/bin/sh', name, '-p', '*']
- logger.log_call(command, timeout=15*60)
-
- # Get a connection and lookup for the sliver before actually
- # defining it, just in case it was already defined.
- conn = Sliver_LV.getConnection()
- try:
- dom = conn.lookupByName(name)
- except:
- dom = conn.defineXML(config)
- logger.verbose('lxc_create: %s -> %s'%(name, Sliver_LV.info(dom)))
+# Common Libvirt code
- @staticmethod
- def destroy(name):
- logger.verbose ('sliver_libvirt: %s destroy'%(name))
-
- dir = '/vservers/%s'%(name)
- lxc_log = '%s/lxc.log'%(dir)
+class Sliver_Libvirt(Account):
- conn = Sliver_LV.getConnection()
+ # Helper methods
+
+ @staticmethod
+ def getConnection(sliver_type):
+ """
+ returns a connection to the underlying libvirt service
+ a single connection is created and shared among slivers
+ this call ensures the connection is alive
+ and will reconnect if it appears to be necessary
+ """
+ # sliver_type comes from rec['type'] and is of the form sliver.{LXC,QEMU}
+ # so we need to lower case to lxc/qemu
+ vtype = sliver_type.split('.')[1].lower()
+ uri = vtype + ':///'
+ if uri not in connections:
+ # create connection
+ conn = libvirt.open(uri)
+ connections[uri] = conn
+ return conn
+ else:
+ # connection already available : check for health
+ conn = connections[uri]
+ # see if a reconnection is needed
+ try:
+ numDomains = conn.numOfDomains()
+ except:
+ logger.log("libvirt connection to {} looks broken - reconnecting".format(uri))
+ conn = libvirt.open(uri)
+ # if this fails then an expection is thrown outside of this function
+ numDomains = conn.numOfDomains()
+ return conn
- try:
- command = ['/usr/sbin/userdel', '-r', name]
- logger.log_call(command, timeout=15*60)
-
- # Destroy libvirt domain
- dom = conn.lookupByName(name)
- dom.destroy()
- dom.undefine()
-
- # Remove rootfs of destroyed domain
- shutil.rmtree("/vservers/%s"%name)
- except:
- logger.verbose('sliver_libvirt: Unexpected error on %s: %s'%(name, sys.exc_info()[0]))
-
def __init__(self, rec):
self.name = rec['name']
- logger.verbose ('sliver_libvirt: %s init'%(self.name))
-
- self.dir = '/vservers/%s'%(self.name)
-
+ logger.verbose ('sliver_libvirt: {} init'.format(self.name))
+
# Assume the directory with the image and config files
# are in place
-
+
self.keys = ''
self.rspec = {}
self.slice_id = rec['slice_id']
- self.disk_usage_initialized = False
- self.initscript = ''
self.enabled = True
- conn = Sliver_LV.getConnection()
+ self.conn = Sliver_Libvirt.getConnection(rec['type'])
+ self.xid = bwlimit.get_xid(self.name)
+
+ dom = None
try:
- self.container = conn.lookupByName(self.name)
+ dom = self.conn.lookupByName(self.name)
except:
- logger.verbose('sliver_libvirt: Unexpected error on %s: %s'%(self.name, sys.exc_info()[0]))
+ logger.log('sliver_libvirt: Domain {} does not exist. ' \
+ 'Will try to create it again.'.format(self.name))
+ self.__class__.create(rec['name'], rec)
+ dom = self.conn.lookupByName(self.name)
+ self.dom = dom
+
+ @staticmethod
+ def dom_details (dom):
+ output = ""
+ output += " id={} - OSType={}".format(dom.ID(), dom.OSType())
+ # calling state() seems to be working fine
+ (state, reason) = dom.state()
+ output += " state={}, reason={}".format(STATES.get(state, state),
+ REASONS.get(reason, reason))
+ try:
+ # try to use info() - this however does not work for some reason on f20
+ # info cannot get info operation failed: Cannot read cputime for domain
+ [state, maxmem, mem, ncpu, cputime] = dom.info()
+ output += " [info: state={}, maxmem = {}, mem = {}, ncpu = {}, cputime = {}]"\
+ .format(STATES.get(state, state), maxmem, mem, ncpu, cputime)
+ except:
+ # too bad but libvirt.py prints out stuff on stdout when this fails, don't know how to get rid of that..
+ output += " [info: not available]"
+ return output
+
+ def __repr__(self):
+ ''' Helper method to get a "nice" output of the domain struct for debug purposes'''
+ output = "Domain {}".format(self.name)
+ dom = self.dom
+ if dom is None:
+ output += " [no attached dom ?!?]"
+ else:
+ output += Sliver_Libvirt.dom_details (dom)
+ return output
+
+ # Thierry : I am not quite sure if /etc/libvirt/lxc/<>.xml holds a reliably up-to-date
+ # copy of the sliver XML config; I feel like issuing a virsh dumpxml first might be safer
+ def repair_veth(self):
+ # See workaround email, 2-14-2014, "libvirt 1.2.1 rollout"
+ xmlfilename = "/etc/libvirt/lxc/{}.xml".format(self.name)
+ with open(xmlfilename) as xmlfile:
+ xml = xmlfile.read()
+ veths = re.findall("<target dev='veth[0-9]*'/>", xml)
+ veths = [x[13:-3] for x in veths]
+ for veth in veths:
+ command = ["ip", "link", "delete", veth]
+ logger.log_call(command)
+
+ logger.log("trying to redefine the VM")
+ command = [ "virsh", "define", xmlfilename ]
+ logger.log_call(command)
- def configure(self, rec):
- ''' Allocate resources and fancy configuration stuff '''
- logger.verbose('sliver_libvirt: %s configure'%(self.name))
- accounts.Account.configure(self, rec)
-
def start(self, delay=0):
- ''' Just start the sliver '''
- print "LIBVIRT %s start"%(self.name)
+ '''Just start the sliver'''
+ logger.verbose('sliver_libvirt: {} start'.format(self.name))
# Check if it's running to avoid throwing an exception if the
- # domain was already running, create actually means start
+ # domain was already running
if not self.is_running():
- self.container.create()
+ try:
+ # create actually means start
+ self.dom.create()
+ except Exception as e:
+ # XXX smbaker: attempt to resolve slivers that are stuck in
+ # "failed to allocate free veth".
+ if "ailed to allocate free veth" in str(e):
+ logger.log("failed to allocate free veth on {}".format(self.name))
+ self.repair_veth()
+ logger.log("trying dom.create again")
+ self.dom.create()
+ else:
+ raise
else:
- logger.verbose('sliver_libvirt: sliver %s already started'%(self.name))
-
+ logger.verbose('sliver_libvirt: sliver {} already started'.format(self.name))
+
+ # After the VM is started... we can play with the virtual interface
+ # Create the ebtables rule to mark the packets going out from the virtual
+ # interface to the actual device so the filter canmatch against the mark
+ bwlimit.ebtables("-A INPUT -i veth{} -j mark --set-mark {}"
+ .format(self.xid, self.xid))
+
+ ### this is confusing, because it seems it is not used in fact
def stop(self):
- logger.verbose('sliver_libvirt: %s stop'%(self.name))
-
+ logger.verbose('sliver_libvirt: {} stop'.format(self.name))
+
+ # Remove the ebtables rule before stopping
+ bwlimit.ebtables("-D INPUT -i veth{} -j mark --set-mark {}"
+ .format(self.xid, self.xid))
+
try:
- self.container.destroy()
+ self.dom.destroy()
except:
- print "Unexpected error:", sys.exc_info()[0]
-
+ logger.log_exc("in sliver_libvirt.stop", name=self.name)
+
def is_running(self):
''' Return True if the domain is running '''
- logger.verbose('sliver_libvirt: %s is_running'%(self.name))
- try:
- [state, _, _, _, _] = self.container.info()
- if state == libvirt.VIR_DOMAIN_RUNNING:
- logger.verbose('sliver_libvirt: %s is RUNNING'%(self.name))
- return True
+ (state, _) = self.dom.state()
+ result = (state == libvirt.VIR_DOMAIN_RUNNING)
+ logger.verbose('sliver_libvirt.is_running: {} => {}'
+ .format(self, result))
+ return result
+
+ def configure(self, rec):
+
+ #sliver.[LXC/QEMU] tolower case
+ #sliver_type = rec['type'].split('.')[1].lower()
+
+ #BASE_DIR = '/cgroup/libvirt/{}/{}/'.format(sliver_type, self.name)
+
+ # Disk allocation
+ # No way through cgroups... figure out how to do that with user/dir quotas.
+ # There is no way to do quota per directory. Chown-ing would create
+ # problems as username namespaces are not yet implemented (and thus, host
+ # and containers share the same name ids
+
+ # Btrfs support quota per volumes
+
+ if "rspec" in rec and "tags" in rec["rspec"]:
+ if cgroups.get_cgroup_path(self.name) == None:
+ # If configure is called before start, then the cgroups won't exist
+ # yet. NM will eventually re-run configure on the next iteration.
+ # TODO: Add a post-start configure, and move this stuff there
+ logger.log("Configure: postponing tag check on {} as cgroups are not yet populated"
+ .format(self.name))
else:
- info = Sliver_LV.info(self.container)
- logger.verbose('sliver_libvirt: %s is NOT RUNNING...\n%s'%(self.name, info))
- return False
- except:
- print "Unexpected error:", sys.exc_info()
+ tags = rec["rspec"]["tags"]
+ # It will depend on the FS selection
+ if 'disk_max' in tags:
+ disk_max = tags['disk_max']
+ if disk_max == 0:
+ # unlimited
+ pass
+ else:
+ # limit to certain number
+ pass
+
+ # Memory allocation
+ if 'memlock_hard' in tags:
+ mem = str(int(tags['memlock_hard']) * 1024) # hard limit in bytes
+ cgroups.write(self.name, 'memory.limit_in_bytes', mem, subsystem="memory")
+ if 'memlock_soft' in tags:
+ mem = str(int(tags['memlock_soft']) * 1024) # soft limit in bytes
+ cgroups.write(self.name, 'memory.soft_limit_in_bytes', mem, subsystem="memory")
+
+ # CPU allocation
+ # Only cpu_shares until figure out how to provide limits and guarantees
+ # (RT_SCHED?)
+ if 'cpu_share' in tags:
+ cpu_share = tags['cpu_share']
+ cgroups.write(self.name, 'cpu.shares', cpu_share)
+
+ # Call the upper configure method (ssh keys...)
+ Account.configure(self, rec)
- ''' PRIVATE/HELPER/STATIC METHODS '''
@staticmethod
- def getConnection():
- ''' Helper method to get a connection to the LXC driver of Libvirt '''
- conn = libvirt.open('lxc:///')
- if conn == None:
- print 'Failed to open connection to LXC hypervisor'
- sys.exit(1)
- else: return conn
+ def get_unique_vif():
+ return 'veth{}'.format(random.getrandbits(32))
+ # A placeholder until we get true VirtualInterface objects
@staticmethod
- def info(dom):
- ''' Helper method to get a "nice" output of the info struct for debug'''
- [state, maxmem, mem, ncpu, cputime] = dom.info()
- return '%s is %s, maxmem = %s, mem = %s, ncpu = %s, cputime = %s' % (dom.name(), states.get(state, state), maxmem, mem, ncpu, cputime)
+ def get_interfaces_xml(rec):
+ xml = """
+ <interface type='network'>
+ <source network='default'/>
+ <target dev='{}'/>
+ </interface>
+""".format(Sliver_Libvirt.get_unique_vif())
+ try:
+ tags = rec['rspec']['tags']
+ if 'interface' in tags:
+ interfaces = eval(tags['interface'])
+ if not isinstance(interfaces, (list, tuple)):
+ # if interface is not a list, then make it into a singleton list
+ interfaces = [interfaces]
+ tag_xml = ""
+ for interface in interfaces:
+ if 'vlan' in interface:
+ vlanxml = "<vlan><tag id='{}'/></vlan>".format(interface['vlan'])
+ else:
+ vlanxml = ""
+ if 'bridge' in interface:
+ tag_xml = tag_xml + """
+ <interface type='bridge'>
+ <source bridge='{}'/>
+ {}
+ <virtualport type='openvswitch'/>
+ <target dev='{}'/>
+ </interface>
+ """.format(interface['bridge'], vlanxml, Sliver_Libvirt.get_unique_vif())
+ else:
+ tag_xml = tag_xml + """
+ <interface type='network'>
+ <source network='default'/>
+ <target dev='{}'/>
+ </interface>
+ """.format(Sliver_Libvirt.get_unique_vif())
+ xml = tag_xml
+ logger.log('sliver_libvirty.py: interface XML is: {}'.format(xml))
+
+ except:
+ logger.log('sliver_libvirt.py: ERROR parsing "interface" tag for slice {}'.format(rec['name']))
+ logger.log('sliver_libvirt.py: tag value: {}'.format(tags['interface']))
+ return xml
git://git.onelab.eu / nodemanager.git / blobdiff