10 from pwd import getpwnam
11 from string import Template
13 # vsys probably should not be a plugin
14 # the thing is, the right way to handle stuff would be that
15 # if slivers get created by doing a,b,c
16 # then they should be deleted by doing c,b,a
17 # the current ordering model for vsys plugins completely fails to capture that
18 from plugins.vsys import removeSliverFromVsys, startService as vsysStartService
23 import plnode.bwlimit as bwlimit
24 from initscript import Initscript
25 from account import Account
26 from sliver_libvirt import Sliver_Libvirt
30 class Sliver_LXC(Sliver_Libvirt, Initscript):
31 """This class wraps LXC commands"""
33 SHELL = '/usr/sbin/vsh'
35 # Need to add a tag at myplc to actually use this account
38 REF_IMG_BASE_DIR = '/vservers/.lvref'
39 CON_BASE_DIR = '/vservers'
41 def __init__(self, rec):
43 Sliver_Libvirt.__init__(self, rec)
44 Initscript.__init__(self, name)
46 def configure(self, rec):
47 logger.log('========== sliver_lxc.configure {}'.format(self.name))
48 Sliver_Libvirt.configure(self, rec)
50 # in case we update nodemanager..
51 self.install_and_enable_vinit()
52 # do the configure part from Initscript
53 Initscript.configure(self, rec)
55 # remember configure() always gets called *before* start()
56 # in particular the slice initscript
57 # is expected to be in place already at this point
58 def start(self, delay=0):
59 logger.log('==================== sliver_lxc.start {}'.format(self.name))
60 if 'enabled' in self.rspec and self.rspec['enabled'] <= 0:
61 logger.log('sliver_lxc: not starting {}, is not enabled'.format(self.name))
63 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
64 self.install_and_enable_vinit()
65 # expose .ssh for omf_friendly slivers
66 if 'tags' in self.rspec and 'omf_control' in self.rspec['tags']:
67 Account.mount_ssh_dir(self.name)
68 # logger.log("NM is exiting for debug - just about to start {}".format(self.name))
70 Sliver_Libvirt.start(self, delay)
72 def rerun_slice_vinit(self):
73 """This is called at startup, and whenever the initscript code changes"""
74 logger.log("sliver_lxc.rerun_slice_vinit {}".format(self.name))
75 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl --system daemon-reload"\
77 command = plain.split()
78 logger.log_call(command, timeout=3)
79 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl restart vinit.service"\
81 command = plain.split()
82 logger.log_call(command, timeout=3)
86 def create(name, rec=None):
88 Create dirs, copy fs image, lxc_create
90 logger.verbose('sliver_lxc: {} create'.format(name))
91 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
95 vref = "lxc-f18-x86_64"
96 logger.log("sliver_libvirt: {}: WARNING - no vref attached, using hard-wired default {}"
99 # compute guest arch from vref
100 # essentially we want x86_64 (default) or i686 here for libvirt
102 (x, y, arch) = vref.split('-')
103 arch = "x86_64" if arch.find("64") >= 0 else "i686"
107 # Get the type of image from vref myplc tags specified as:
113 tags = rec['rspec']['tags']
120 refImgDir = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, vref)
121 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
123 # check the template exists -- there's probably a better way..
124 if not os.path.isdir(refImgDir):
125 logger.log('sliver_lxc: {}: ERROR Could not create sliver - reference image {} not found'
127 logger.log('sliver_lxc: {}: ERROR Expected reference image in {}'.format(name, refImgDir))
130 # this hopefully should be fixed now
131 # # in fedora20 we have some difficulty in properly cleaning up /vservers/<slicename>
132 # # also note that running e.g. btrfs subvolume create /vservers/.lvref/image /vservers/foo
133 # # behaves differently, whether /vservers/foo exists or not:
134 # # if /vservers/foo does not exist, it creates /vservers/foo
135 # # but if it does exist, then it creates /vservers/foo/image !!
136 # # so we need to check the expected container rootfs does not exist yet
137 # # this hopefully could be removed in a future release
138 # if os.path.exists (containerDir):
139 # logger.log("sliver_lxc: {}: WARNING cleaning up pre-existing {}".format(name, containerDir))
140 # command = ['btrfs', 'subvolume', 'delete', containerDir]
141 # logger.log_call(command, BTRFS_TIMEOUT)
143 # if os.path.exists (containerDir):
144 # logger.log('sliver_lxc: {}: ERROR Could not create sliver - could not clean up empty {}'
145 # .format(name, containerDir))
148 # Snapshot the reference image fs
149 # this assumes the reference image is in its own subvolume
150 command = ['btrfs', 'subvolume', 'snapshot', refImgDir, containerDir]
151 if not logger.log_call(command, timeout=BTRFS_TIMEOUT):
152 logger.log('sliver_lxc: ERROR Could not create BTRFS snapshot at', containerDir)
154 command = ['chmod', '755', containerDir]
155 logger.log_call(command)
157 # TODO: set quotas...
159 # Set hostname. A valid hostname cannot have '_'
160 #with open(os.path.join(containerDir, 'etc/hostname'), 'w') as f:
161 # print >>f, name.replace('_', '-')
163 # Add slices group if not already present
165 group = grp.getgrnam('slices')
167 command = ['/usr/sbin/groupadd', 'slices']
168 logger.log_call(command)
170 # Add unix account (TYPE is specified in the subclass)
171 command = ['/usr/sbin/useradd', '-g', 'slices', '-s', Sliver_LXC.SHELL, name, '-p', '*']
172 logger.log_call(command)
173 command = ['mkdir', '/home/{}/.ssh'.format(name)]
174 logger.log_call(command)
176 # Create PK pair keys to connect from the host to the guest without
177 # password... maybe remove the need for authentication inside the
179 command = ['su', '-s', '/bin/bash', '-c',
180 'ssh-keygen -t rsa -N "" -f /home/{}/.ssh/id_rsa'.format(name)]
181 logger.log_call(command)
183 command = ['chown', '-R', '{}:slices'.format(name), '/home/{}/.ssh'.format(name)]
184 logger.log_call(command)
186 command = ['mkdir', '{}/root/.ssh'.format(containerDir)]
187 logger.log_call(command)
189 command = ['cp', '/home/{}/.ssh/id_rsa.pub'.format(name),
190 '{}/root/.ssh/authorized_keys'.format(containerDir)]
191 logger.log_call(command)
193 logger.log("creating /etc/slicename file in {}".format(os.path.join(containerDir, 'etc/slicename')))
195 with open(os.path.join(containerDir, 'etc/slicename'), 'w') as f:
198 logger.log_exc("exception while creating /etc/slicename")
201 with open(os.path.join(containerDir, 'etc/slicefamily'), 'w') as f:
204 logger.log_exc("exception while creating /etc/slicefamily")
208 uid = getpwnam(name).pw_uid
210 # keyerror will happen if user id was not created successfully
211 logger.log_exc("exception while getting user id")
214 logger.log("uid is {}".format(uid))
215 command = ['mkdir', '{}/home/{}'.format(containerDir, name)]
216 logger.log_call(command)
217 command = ['chown', name, '{}/home/{}'.format(containerDir, name)]
218 logger.log_call(command)
219 etcpasswd = os.path.join(containerDir, 'etc/passwd')
220 etcgroup = os.path.join(containerDir, 'etc/group')
221 if os.path.exists(etcpasswd):
222 # create all accounts with gid=1001 - i.e. 'slices' like it is in the root context
224 logger.log("adding user {name} id {uid} gid {slices_gid} to {etcpasswd}"
225 .format(**(locals())))
227 with open(etcpasswd, 'a') as passwdfile:
228 passwdfile.write("{name}:x:{uid}:{slices_gid}::/home/{name}:/bin/bash\n"
231 logger.log_exc("exception while updating {}".format(etcpasswd))
232 logger.log("adding group slices with gid {slices_gid} to {etcgroup}"
235 with open(etcgroup, 'a') as groupfile:
236 groupfile.write("slices:x:{slices_gid}\n"
239 logger.log_exc("exception while updating {}".format(etcgroup))
240 sudoers = os.path.join(containerDir, 'etc/sudoers')
241 if os.path.exists(sudoers):
243 with open(sudoers, 'a') as f:
244 f.write("{} ALL=(ALL) NOPASSWD: ALL\n".format(name))
246 logger.log_exc("exception while updating /etc/sudoers")
248 # customizations for the user environment - root or slice uid
249 # we save the whole business in /etc/planetlab.profile
250 # and source this file for both root and the slice uid's .profile
251 # prompt for slice owner, + LD_PRELOAD for transparently wrap bind
252 pl_profile = os.path.join(containerDir, "etc/planetlab.profile")
253 ld_preload_text = """# by default, we define this setting so that calls to bind(2),
254 # when invoked on 0.0.0.0, get transparently redirected to the public interface of this node
255 # see https://svn.planet-lab.org/wiki/LxcPortForwarding"""
256 usrmove_path_text = """# VM's before Features/UsrMove need /bin and /sbin in their PATH"""
257 usrmove_path_code = """
259 if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
260 if [ "$2" = "after" ] ; then
268 pathmunge /sbin after
271 with open(pl_profile, 'w') as f:
272 f.write("export PS1='{}@\H \$ '\n".format(name))
273 f.write("{}\n".format(ld_preload_text))
274 f.write("export LD_PRELOAD=/etc/planetlab/lib/bind_public.so\n")
275 f.write("{}\n".format(usrmove_path_text))
276 f.write("{}\n".format(usrmove_path_code))
278 # make sure this file is sourced from both root's and slice's .profile
279 enforced_line = "[ -f /etc/planetlab.profile ] && source /etc/planetlab.profile\n"
280 for path in [ 'root/.profile', 'home/{}/.profile'.format(name) ]:
281 from_root = os.path.join(containerDir, path)
282 # if dir is not yet existing let's forget it for now
283 if not os.path.isdir(os.path.dirname(from_root)): continue
286 with open(from_root) as f:
287 contents = f.readlines()
288 for content in contents:
289 if content == enforced_line:
294 with open(from_root, "a") as user_profile:
295 user_profile.write(enforced_line)
296 # in case we create the slice's .profile when writing
297 if from_root.find("/home") >= 0:
298 command = ['chown', '{}:slices'.format(name), from_root]
299 logger.log_call(command)
301 # Lookup for xid and create template after the user is created so we
302 # can get the correct xid based on the name of the slice
303 xid = bwlimit.get_xid(name)
305 # Template for libvirt sliver configuration
306 template_filename_sliceimage = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, 'lxc_template.xml')
307 if os.path.isfile(template_filename_sliceimage):
308 logger.verbose("Using XML template {}".format(template_filename_sliceimage))
309 template_filename = template_filename_sliceimage
311 logger.log("Cannot find XML template {}".format(template_filename_sliceimage))
314 interfaces = Sliver_Libvirt.get_interfaces_xml(rec)
317 with open(template_filename) as f:
318 template = Template(f.read())
319 xml = template.substitute(name=name, xid=xid, interfaces=interfaces, arch=arch)
321 logger.log('Failed to parse or use XML template file {}'.format(template_filename))
324 # Lookup for the sliver before actually
325 # defining it, just in case it was already defined.
327 dom = conn.lookupByName(name)
329 dom = conn.defineXML(xml)
330 logger.verbose('lxc_create: {} -> {}'.format(name, Sliver_Libvirt.dom_details(dom)))
335 # umount .ssh directory - only if mounted
336 Account.umount_ssh_dir(name)
337 logger.verbose ('sliver_lxc: {} destroy'.format(name))
338 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
340 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
343 # Destroy libvirt domain
344 dom = conn.lookupByName(name)
346 logger.verbose('sliver_lxc.destroy: Domain {} does not exist!'.format(name))
349 # Slivers with vsys running will fail the subvolume delete
350 # removeSliverFromVsys return True if it stops vsys, telling us to start it again later
351 vsys_stopped = removeSliverFromVsys (name)
354 logger.log("sliver_lxc.destroy: destroying domain {}".format(name))
357 logger.verbose("sliver_lxc.destroy: Domain {} not running... continuing.".format(name))
360 logger.log("sliver_lxc.destroy: undefining domain {}".format(name))
363 logger.verbose('sliver_lxc.destroy: Domain {} is not defined... continuing.'.format(name))
365 # Remove user after destroy domain to force logout
366 command = ['/usr/sbin/userdel', '-f', '-r', name]
367 logger.log_call(command)
369 # Remove rootfs of destroyed domain
370 command = ['/usr/bin/rm', '-rf', containerDir]
371 logger.log_call(command, timeout=BTRFS_TIMEOUT)
374 logger.log("-TMP-ls-l {}".format(name))
375 command = ['ls', '-lR', containerDir]
376 logger.log_call(command)
377 logger.log("-TMP-vsys-status")
378 command = ['/usr/bin/systemctl', 'status', 'vsys']
379 logger.log_call(command)
382 # Remove rootfs of destroyed domain
383 command = ['btrfs', 'subvolume', 'delete', containerDir]
384 logger.log_call(command, timeout=BTRFS_TIMEOUT)
386 # For some reason I am seeing this :
387 #log_call: running command btrfs subvolume delete /vservers/inri_sl1
388 #log_call: ERROR: cannot delete '/vservers/inri_sl1' - Device or resource busy
389 #log_call: Delete subvolume '/vservers/inri_sl1'
390 #log_call:end command (btrfs subvolume delete /vservers/inri_sl1) returned with code 1
392 # something must have an open handle to a file in there, but I can't find out what it is
393 # the following code aims at gathering data on what is going on in the system at this point in time
394 # note that some time later (typically when the sliver gets re-created) the same
395 # attempt at deleting the subvolume does work
396 # also lsof never shows anything relevant; this is painful..
398 if not os.path.exists(containerDir):
399 logger.log('sliver_lxc.destroy: {} cleanly destroyed.'.format(name))
402 #logger.log("-TMP-cwd {} : {}".format(name, os.getcwd()))
403 # also lsof never shows anything relevant; this is painful..
404 #logger.log("-TMP-lsof {}".format(name))
406 #logger.log_call(command)
407 logger.log("-TMP-ls-l {}".format(name))
408 command = ['ls', '-lR', containerDir]
409 logger.log_call(command)
410 logger.log("-TMP-lsof")
412 logger.log_call(command)
413 if os.path.exists(containerDir):
414 logger.log('sliver_lxc.destroy: ERROR could not cleanly destroy {} - giving up'.format(name))