10 from pwd import getpwnam
11 from string import Template
13 # vsys probably should not be a plugin
14 # the thing is, the right way to handle stuff would be that
15 # if slivers get created by doing a,b,c
16 # then they should be deleted by doing c,b,a
17 # the current ordering model for vsys plugins completely fails to capture that
18 from plugins.vsys import removeSliverFromVsys, startService as vsysStartService
23 import plnode.bwlimit as bwlimit
24 from initscript import Initscript
25 from account import Account
26 from sliver_libvirt import Sliver_Libvirt
30 class Sliver_LXC(Sliver_Libvirt, Initscript):
31 """This class wraps LXC commands"""
33 SHELL = '/usr/sbin/vsh'
35 # Need to add a tag at myplc to actually use this account
38 REF_IMG_BASE_DIR = '/vservers/.lvref'
39 CON_BASE_DIR = '/vservers'
41 def __init__(self, rec):
43 Sliver_Libvirt.__init__(self, rec)
44 Initscript.__init__(self, name)
46 def configure(self, rec):
47 logger.log('========== sliver_lxc.configure {}'.format(self.name))
48 Sliver_Libvirt.configure(self, rec)
50 # in case we update nodemanager..
51 self.install_and_enable_vinit()
52 # do the configure part from Initscript
53 Initscript.configure(self, rec)
55 # remember configure() always gets called *before* start()
56 # in particular the slice initscript
57 # is expected to be in place already at this point
58 def start(self, delay=0):
59 logger.log('==================== sliver_lxc.start {}'.format(self.name))
60 if 'enabled' in self.rspec and self.rspec['enabled'] <= 0:
61 logger.log('sliver_lxc: not starting %s, is not enabled'%self.name)
63 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
64 self.install_and_enable_vinit()
65 # expose .ssh for omf_friendly slivers
66 if 'tags' in self.rspec and 'omf_control' in self.rspec['tags']:
67 Account.mount_ssh_dir(self.name)
68 # logger.log("NM is exiting for debug - just about to start {}".format(self.name))
70 Sliver_Libvirt.start(self, delay)
72 def rerun_slice_vinit(self):
73 """This is called at startup, and whenever the initscript code changes"""
74 logger.log("sliver_lxc.rerun_slice_vinit {}".format(self.name))
75 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl --system daemon-reload"\
77 command = plain.split()
78 logger.log_call(command, timeout=3)
79 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl restart vinit.service"\
81 command = plain.split()
82 logger.log_call(command, timeout=3)
86 def create(name, rec=None):
88 Create dirs, copy fs image, lxc_create
90 logger.verbose('sliver_lxc: %s create' % name)
91 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
95 vref = "lxc-f18-x86_64"
96 logger.log("sliver_libvirt: %s: WARNING - no vref attached, using hard-wired default %s" % (name, vref))
98 # compute guest arch from vref
99 # essentially we want x86_64 (default) or i686 here for libvirt
101 (x, y, arch) = vref.split('-')
102 arch = "x86_64" if arch.find("64") >= 0 else "i686"
106 # Get the type of image from vref myplc tags specified as:
112 tags = rec['rspec']['tags']
121 refImgDir = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, vref)
122 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
124 # check the template exists -- there's probably a better way..
125 if not os.path.isdir(refImgDir):
126 logger.log('sliver_lxc: %s: ERROR Could not create sliver - reference image %s not found' % (name, vref))
127 logger.log('sliver_lxc: %s: ERROR Expected reference image in %s'%(name, refImgDir))
130 # this hopefully should be fixed now
131 # # in fedora20 we have some difficulty in properly cleaning up /vservers/<slicename>
132 # # also note that running e.g. btrfs subvolume create /vservers/.lvref/image /vservers/foo
133 # # behaves differently, whether /vservers/foo exists or not:
134 # # if /vservers/foo does not exist, it creates /vservers/foo
135 # # but if it does exist, then it creates /vservers/foo/image !!
136 # # so we need to check the expected container rootfs does not exist yet
137 # # this hopefully could be removed in a future release
138 # if os.path.exists (containerDir):
139 # logger.log("sliver_lxc: %s: WARNING cleaning up pre-existing %s"%(name, containerDir))
140 # command = ['btrfs', 'subvolume', 'delete', containerDir]
141 # logger.log_call(command, BTRFS_TIMEOUT)
143 # if os.path.exists (containerDir):
144 # logger.log('sliver_lxc: %s: ERROR Could not create sliver - could not clean up empty %s'%(name, containerDir))
147 # Snapshot the reference image fs
148 # this assumes the reference image is in its own subvolume
149 command = ['btrfs', 'subvolume', 'snapshot', refImgDir, containerDir]
150 if not logger.log_call(command, timeout=BTRFS_TIMEOUT):
151 logger.log('sliver_lxc: ERROR Could not create BTRFS snapshot at', containerDir)
153 command = ['chmod', '755', containerDir]
154 logger.log_call(command)
156 # TODO: set quotas...
158 # Set hostname. A valid hostname cannot have '_'
159 #with open(os.path.join(containerDir, 'etc/hostname'), 'w') as f:
160 # print >>f, name.replace('_', '-')
162 # Add slices group if not already present
164 group = grp.getgrnam('slices')
166 command = ['/usr/sbin/groupadd', 'slices']
167 logger.log_call(command)
169 # Add unix account (TYPE is specified in the subclass)
170 command = ['/usr/sbin/useradd', '-g', 'slices', '-s', Sliver_LXC.SHELL, name, '-p', '*']
171 logger.log_call(command)
172 command = ['mkdir', '/home/%s/.ssh'%name]
173 logger.log_call(command)
175 # Create PK pair keys to connect from the host to the guest without
176 # password... maybe remove the need for authentication inside the
178 command = ['su', '-s', '/bin/bash', '-c', 'ssh-keygen -t rsa -N "" -f /home/%s/.ssh/id_rsa'%(name)]
179 logger.log_call(command)
181 command = ['chown', '-R', '%s.slices'%name, '/home/%s/.ssh'%name]
182 logger.log_call(command)
184 command = ['mkdir', '%s/root/.ssh'%containerDir]
185 logger.log_call(command)
187 command = ['cp', '/home/%s/.ssh/id_rsa.pub'%name, '%s/root/.ssh/authorized_keys'%containerDir]
188 logger.log_call(command)
190 logger.log("creating /etc/slicename file in %s" % os.path.join(containerDir, 'etc/slicename'))
192 file(os.path.join(containerDir, 'etc/slicename'), 'w').write(name)
194 logger.log_exc("exception while creating /etc/slicename")
197 file(os.path.join(containerDir, 'etc/slicefamily'), 'w').write(vref)
199 logger.log_exc("exception while creating /etc/slicefamily")
203 uid = getpwnam(name).pw_uid
205 # keyerror will happen if user id was not created successfully
206 logger.log_exc("exception while getting user id")
209 logger.log("uid is %d" % uid)
210 command = ['mkdir', '%s/home/%s' % (containerDir, name)]
211 logger.log_call(command)
212 command = ['chown', name, '%s/home/%s' % (containerDir, name)]
213 logger.log_call(command)
214 etcpasswd = os.path.join(containerDir, 'etc/passwd')
215 etcgroup = os.path.join(containerDir, 'etc/group')
216 if os.path.exists(etcpasswd):
217 # create all accounts with gid=1001 - i.e. 'slices' like it is in the root context
219 logger.log("adding user %(name)s id %(uid)d gid %(slices_gid)d to %(etcpasswd)s" % (locals()))
221 file(etcpasswd, 'a').write("%(name)s:x:%(uid)d:%(slices_gid)d::/home/%(name)s:/bin/bash\n" % locals())
223 logger.log_exc("exception while updating %s"%etcpasswd)
224 logger.log("adding group slices with gid %(slices_gid)d to %(etcgroup)s"%locals())
226 file(etcgroup, 'a').write("slices:x:%(slices_gid)d\n"%locals())
228 logger.log_exc("exception while updating %s"%etcgroup)
229 sudoers = os.path.join(containerDir, 'etc/sudoers')
230 if os.path.exists(sudoers):
232 file(sudoers, 'a').write("%s ALL=(ALL) NOPASSWD: ALL\n" % name)
234 logger.log_exc("exception while updating /etc/sudoers")
236 # customizations for the user environment - root or slice uid
237 # we save the whole business in /etc/planetlab.profile
238 # and source this file for both root and the slice uid's .profile
239 # prompt for slice owner, + LD_PRELOAD for transparently wrap bind
240 pl_profile = os.path.join(containerDir, "etc/planetlab.profile")
241 ld_preload_text = """# by default, we define this setting so that calls to bind(2),
242 # when invoked on 0.0.0.0, get transparently redirected to the public interface of this node
243 # see https://svn.planet-lab.org/wiki/LxcPortForwarding"""
244 usrmove_path_text = """# VM's before Features/UsrMove need /bin and /sbin in their PATH"""
245 usrmove_path_code = """
247 if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
248 if [ "$2" = "after" ] ; then
256 pathmunge /sbin after
259 with open(pl_profile, 'w') as f:
260 f.write("export PS1='%s@\H \$ '\n"%(name))
261 f.write("%s\n"%ld_preload_text)
262 f.write("export LD_PRELOAD=/etc/planetlab/lib/bind_public.so\n")
263 f.write("%s\n"%usrmove_path_text)
264 f.write("%s\n"%usrmove_path_code)
266 # make sure this file is sourced from both root's and slice's .profile
267 enforced_line = "[ -f /etc/planetlab.profile ] && source /etc/planetlab.profile\n"
268 for path in [ 'root/.profile', 'home/%s/.profile'%name ]:
269 from_root = os.path.join(containerDir, path)
270 # if dir is not yet existing let's forget it for now
271 if not os.path.isdir(os.path.dirname(from_root)): continue
274 contents = file(from_root).readlines()
275 for content in contents:
276 if content == enforced_line:
281 with open(from_root, "a") as user_profile:
282 user_profile.write(enforced_line)
283 # in case we create the slice's .profile when writing
284 if from_root.find("/home") >= 0:
285 command = ['chown', '%s:slices'%name, from_root]
286 logger.log_call(command)
288 # Lookup for xid and create template after the user is created so we
289 # can get the correct xid based on the name of the slice
290 xid = bwlimit.get_xid(name)
292 # Template for libvirt sliver configuration
293 template_filename_sliceimage = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, 'lxc_template.xml')
294 if os.path.isfile (template_filename_sliceimage):
295 logger.verbose("Using XML template %s"%template_filename_sliceimage)
296 template_filename = template_filename_sliceimage
298 logger.log("Cannot find XML template %s"%template_filename_sliceimage)
301 interfaces = Sliver_Libvirt.get_interfaces_xml(rec)
304 with open(template_filename) as f:
305 template = Template(f.read())
306 xml = template.substitute(name=name, xid=xid, interfaces=interfaces, arch=arch)
308 logger.log('Failed to parse or use XML template file %s'%template_filename)
311 # Lookup for the sliver before actually
312 # defining it, just in case it was already defined.
314 dom = conn.lookupByName(name)
316 dom = conn.defineXML(xml)
317 logger.verbose('lxc_create: %s -> %s'%(name, Sliver_Libvirt.dom_details(dom)))
322 # umount .ssh directory - only if mounted
323 Account.umount_ssh_dir(name)
324 logger.verbose ('sliver_lxc: %s destroy'%(name))
325 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
327 containerDir = Sliver_LXC.CON_BASE_DIR + '/%s'%(name)
330 # Destroy libvirt domain
331 dom = conn.lookupByName(name)
333 logger.verbose('sliver_lxc.destroy: Domain %s does not exist!' % name)
336 # Slivers with vsys running will fail the subvolume delete
337 # removeSliverFromVsys return True if it stops vsys, telling us to start it again later
338 vsys_stopped = removeSliverFromVsys (name)
341 logger.log("sliver_lxc.destroy: destroying domain %s"%name)
344 logger.verbose('sliver_lxc.destroy: Domain %s not running... continuing.' % name)
347 logger.log("sliver_lxc.destroy: undefining domain %s"%name)
350 logger.verbose('sliver_lxc.destroy: Domain %s is not defined... continuing.' % name)
352 # Remove user after destroy domain to force logout
353 command = ['/usr/sbin/userdel', '-f', '-r', name]
354 logger.log_call(command)
356 # Remove rootfs of destroyed domain
357 command = ['/usr/bin/rm', '-rf', containerDir]
358 logger.log_call(command, timeout=BTRFS_TIMEOUT)
361 logger.log("-TMP-ls-l %s"%name)
362 command = ['ls', '-lR', containerDir]
363 logger.log_call(command)
364 logger.log("-TMP-vsys-status")
365 command = ['/usr/bin/systemctl', 'status', 'vsys']
366 logger.log_call(command)
369 # Remove rootfs of destroyed domain
370 command = ['btrfs', 'subvolume', 'delete', containerDir]
371 logger.log_call(command, timeout=BTRFS_TIMEOUT)
373 # For some reason I am seeing this :
374 #log_call: running command btrfs subvolume delete /vservers/inri_sl1
375 #log_call: ERROR: cannot delete '/vservers/inri_sl1' - Device or resource busy
376 #log_call: Delete subvolume '/vservers/inri_sl1'
377 #log_call:end command (btrfs subvolume delete /vservers/inri_sl1) returned with code 1
379 # something must have an open handle to a file in there, but I can't find out what it is
380 # the following code aims at gathering data on what is going on in the system at this point in time
381 # note that some time later (typically when the sliver gets re-created) the same
382 # attempt at deleting the subvolume does work
383 # also lsof never shows anything relevant; this is painful..
385 if not os.path.exists(containerDir):
386 logger.log('sliver_lxc.destroy: %s cleanly destroyed.'%name)
389 #logger.log("-TMP-cwd %s : %s"%(name, os.getcwd()))
390 # also lsof never shows anything relevant; this is painful..
391 #logger.log("-TMP-lsof %s"%name)
393 #logger.log_call(command)
394 logger.log("-TMP-ls-l %s"%name)
395 command = ['ls', '-lR', containerDir]
396 logger.log_call(command)
397 logger.log("-TMP-lsof")
399 logger.log_call(command)
400 if os.path.exists(containerDir):
401 logger.log('sliver_lxc.destroy: ERROR could not cleanly destroy %s - giving up'%name)