10 from pwd import getpwnam
11 from string import Template
13 # vsys probably should not be a plugin
14 # the thing is, the right way to handle stuff would be that
15 # if slivers get created by doing a,b,c
16 # then they should be deleted by doing c,b,a
17 # the current ordering model for vsys plugins completely fails to capture that
18 from plugins.vsys import removeSliverFromVsys, startService as vsysStartService
23 import plnode.bwlimit as bwlimit
24 from initscript import Initscript
25 from account import Account
26 from sliver_libvirt import Sliver_Libvirt
30 class Sliver_LXC(Sliver_Libvirt, Initscript):
31 """This class wraps LXC commands"""
33 SHELL = '/usr/sbin/vsh'
35 # Need to add a tag at myplc to actually use this account
38 REF_IMG_BASE_DIR = '/vservers/.lvref'
39 CON_BASE_DIR = '/vservers'
41 def __init__(self, rec):
43 Sliver_Libvirt.__init__(self,rec)
44 Initscript.__init__(self,name)
46 def configure(self, rec):
47 Sliver_Libvirt.configure(self, rec)
49 # in case we update nodemanager..
50 self.install_and_enable_vinit()
51 # do the configure part from Initscript
52 Initscript.configure(self, rec)
54 def start(self, delay=0):
55 if 'enabled' in self.rspec and self.rspec['enabled'] <= 0:
56 logger.log('sliver_lxc: not starting %s, is not enabled'%self.name)
58 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
59 self.install_and_enable_vinit()
60 # expose .ssh for omf_friendly slivers
61 if 'tags' in self.rspec and 'omf_control' in self.rspec['tags']:
62 Account.mount_ssh_dir(self.name)
63 Sliver_Libvirt.start(self, delay)
64 # if a change has occured in the slice initscript, reflect this in /etc/init.d/vinit.slice
65 self.refresh_slice_vinit(force=True)
67 def rerun_slice_vinit(self):
68 """This is called at startup, and whenever the initscript code changes"""
69 logger.log("sliver_lxc.rerun_slice_vinit {}".format(self.name))
70 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl --system daemon-reload"\
72 command = plain.split()
73 logger.log_call(command, timeout=3)
74 plain = "virsh -c lxc:/// lxc-enter-namespace --noseclabel -- {} /usr/bin/systemctl restart vinit.service"\
76 command = plain.split()
77 logger.log_call(command, timeout=3)
81 def create(name, rec=None):
83 Create dirs, copy fs image, lxc_create
85 logger.verbose('sliver_lxc: %s create' % name)
86 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
90 vref = "lxc-f18-x86_64"
91 logger.log("sliver_libvirt: %s: WARNING - no vref attached, using hard-wired default %s" % (name,vref))
93 # compute guest arch from vref
94 # essentially we want x86_64 (default) or i686 here for libvirt
96 (x, y, arch) = vref.split('-')
97 arch = "x86_64" if arch.find("64")>=0 else "i686"
101 # Get the type of image from vref myplc tags specified as:
107 tags = rec['rspec']['tags']
116 refImgDir = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, vref)
117 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
119 # check the template exists -- there's probably a better way..
120 if not os.path.isdir(refImgDir):
121 logger.log('sliver_lxc: %s: ERROR Could not create sliver - reference image %s not found' % (name,vref))
122 logger.log('sliver_lxc: %s: ERROR Expected reference image in %s'%(name,refImgDir))
125 # this hopefully should be fixed now
126 # # in fedora20 we have some difficulty in properly cleaning up /vservers/<slicename>
127 # # also note that running e.g. btrfs subvolume create /vservers/.lvref/image /vservers/foo
128 # # behaves differently, whether /vservers/foo exists or not:
129 # # if /vservers/foo does not exist, it creates /vservers/foo
130 # # but if it does exist, then it creates /vservers/foo/image !!
131 # # so we need to check the expected container rootfs does not exist yet
132 # # this hopefully could be removed in a future release
133 # if os.path.exists (containerDir):
134 # logger.log("sliver_lxc: %s: WARNING cleaning up pre-existing %s"%(name,containerDir))
135 # command = ['btrfs', 'subvolume', 'delete', containerDir]
136 # logger.log_call(command, BTRFS_TIMEOUT)
138 # if os.path.exists (containerDir):
139 # logger.log('sliver_lxc: %s: ERROR Could not create sliver - could not clean up empty %s'%(name,containerDir))
142 # Snapshot the reference image fs
143 # this assumes the reference image is in its own subvolume
144 command = ['btrfs', 'subvolume', 'snapshot', refImgDir, containerDir]
145 if not logger.log_call(command, timeout=BTRFS_TIMEOUT):
146 logger.log('sliver_lxc: ERROR Could not create BTRFS snapshot at', containerDir)
148 command = ['chmod', '755', containerDir]
149 logger.log_call(command)
151 # TODO: set quotas...
153 # Set hostname. A valid hostname cannot have '_'
154 #with open(os.path.join(containerDir, 'etc/hostname'), 'w') as f:
155 # print >>f, name.replace('_', '-')
157 # Add slices group if not already present
159 group = grp.getgrnam('slices')
161 command = ['/usr/sbin/groupadd', 'slices']
162 logger.log_call(command)
164 # Add unix account (TYPE is specified in the subclass)
165 command = ['/usr/sbin/useradd', '-g', 'slices', '-s', Sliver_LXC.SHELL, name, '-p', '*']
166 logger.log_call(command)
167 command = ['mkdir', '/home/%s/.ssh'%name]
168 logger.log_call(command)
170 # Create PK pair keys to connect from the host to the guest without
171 # password... maybe remove the need for authentication inside the
173 command = ['su', '-s', '/bin/bash', '-c', 'ssh-keygen -t rsa -N "" -f /home/%s/.ssh/id_rsa'%(name)]
174 logger.log_call(command)
176 command = ['chown', '-R', '%s.slices'%name, '/home/%s/.ssh'%name]
177 logger.log_call(command)
179 command = ['mkdir', '%s/root/.ssh'%containerDir]
180 logger.log_call(command)
182 command = ['cp', '/home/%s/.ssh/id_rsa.pub'%name, '%s/root/.ssh/authorized_keys'%containerDir]
183 logger.log_call(command)
185 logger.log("creating /etc/slicename file in %s" % os.path.join(containerDir,'etc/slicename'))
187 file(os.path.join(containerDir,'etc/slicename'), 'w').write(name)
189 logger.log_exc("exception while creating /etc/slicename")
192 file(os.path.join(containerDir,'etc/slicefamily'), 'w').write(vref)
194 logger.log_exc("exception while creating /etc/slicefamily")
198 uid = getpwnam(name).pw_uid
200 # keyerror will happen if user id was not created successfully
201 logger.log_exc("exception while getting user id")
204 logger.log("uid is %d" % uid)
205 command = ['mkdir', '%s/home/%s' % (containerDir, name)]
206 logger.log_call(command)
207 command = ['chown', name, '%s/home/%s' % (containerDir, name)]
208 logger.log_call(command)
209 etcpasswd = os.path.join(containerDir, 'etc/passwd')
210 etcgroup = os.path.join(containerDir, 'etc/group')
211 if os.path.exists(etcpasswd):
212 # create all accounts with gid=1001 - i.e. 'slices' like it is in the root context
214 logger.log("adding user %(name)s id %(uid)d gid %(slices_gid)d to %(etcpasswd)s" % (locals()))
216 file(etcpasswd,'a').write("%(name)s:x:%(uid)d:%(slices_gid)d::/home/%(name)s:/bin/bash\n" % locals())
218 logger.log_exc("exception while updating %s"%etcpasswd)
219 logger.log("adding group slices with gid %(slices_gid)d to %(etcgroup)s"%locals())
221 file(etcgroup,'a').write("slices:x:%(slices_gid)d\n"%locals())
223 logger.log_exc("exception while updating %s"%etcgroup)
224 sudoers = os.path.join(containerDir, 'etc/sudoers')
225 if os.path.exists(sudoers):
227 file(sudoers,'a').write("%s ALL=(ALL) NOPASSWD: ALL\n" % name)
229 logger.log_exc("exception while updating /etc/sudoers")
231 # customizations for the user environment - root or slice uid
232 # we save the whole business in /etc/planetlab.profile
233 # and source this file for both root and the slice uid's .profile
234 # prompt for slice owner, + LD_PRELOAD for transparently wrap bind
235 pl_profile=os.path.join(containerDir,"etc/planetlab.profile")
236 ld_preload_text="""# by default, we define this setting so that calls to bind(2),
237 # when invoked on 0.0.0.0, get transparently redirected to the public interface of this node
238 # see https://svn.planet-lab.org/wiki/LxcPortForwarding"""
239 usrmove_path_text="""# VM's before Features/UsrMove need /bin and /sbin in their PATH"""
240 usrmove_path_code="""
242 if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
243 if [ "$2" = "after" ] ; then
251 pathmunge /sbin after
254 with open(pl_profile,'w') as f:
255 f.write("export PS1='%s@\H \$ '\n"%(name))
256 f.write("%s\n"%ld_preload_text)
257 f.write("export LD_PRELOAD=/etc/planetlab/lib/bind_public.so\n")
258 f.write("%s\n"%usrmove_path_text)
259 f.write("%s\n"%usrmove_path_code)
261 # make sure this file is sourced from both root's and slice's .profile
262 enforced_line = "[ -f /etc/planetlab.profile ] && source /etc/planetlab.profile\n"
263 for path in [ 'root/.profile', 'home/%s/.profile'%name ]:
264 from_root=os.path.join(containerDir,path)
265 # if dir is not yet existing let's forget it for now
266 if not os.path.isdir(os.path.dirname(from_root)): continue
269 contents=file(from_root).readlines()
270 for content in contents:
271 if content==enforced_line: found=True
274 with open(from_root,"a") as user_profile:
275 user_profile.write(enforced_line)
276 # in case we create the slice's .profile when writing
277 if from_root.find("/home")>=0:
278 command=['chown','%s:slices'%name,from_root]
279 logger.log_call(command)
281 # Lookup for xid and create template after the user is created so we
282 # can get the correct xid based on the name of the slice
283 xid = bwlimit.get_xid(name)
285 # Template for libvirt sliver configuration
286 template_filename_sliceimage = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR,'lxc_template.xml')
287 if os.path.isfile (template_filename_sliceimage):
288 logger.log("WARNING: using compat template %s"%template_filename_sliceimage)
289 template_filename=template_filename_sliceimage
291 logger.log("Cannot find XML template %s"%template_filename_sliceimage)
294 interfaces = Sliver_Libvirt.get_interfaces_xml(rec)
297 with open(template_filename) as f:
298 template = Template(f.read())
299 xml = template.substitute(name=name, xid=xid, interfaces=interfaces, arch=arch)
301 logger.log('Failed to parse or use XML template file %s'%template_filename)
304 # Lookup for the sliver before actually
305 # defining it, just in case it was already defined.
307 dom = conn.lookupByName(name)
309 dom = conn.defineXML(xml)
310 logger.verbose('lxc_create: %s -> %s'%(name, Sliver_Libvirt.dom_details(dom)))
315 # umount .ssh directory - only if mounted
316 Account.umount_ssh_dir(name)
317 logger.verbose ('sliver_lxc: %s destroy'%(name))
318 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
320 containerDir = Sliver_LXC.CON_BASE_DIR + '/%s'%(name)
323 # Destroy libvirt domain
324 dom = conn.lookupByName(name)
326 logger.verbose('sliver_lxc.destroy: Domain %s does not exist!' % name)
329 # Slivers with vsys running will fail the subvolume delete
330 # removeSliverFromVsys return True if it stops vsys, telling us to start it again later
331 vsys_stopped = removeSliverFromVsys (name)
334 logger.log("sliver_lxc.destroy: destroying domain %s"%name)
337 logger.verbose('sliver_lxc.destroy: Domain %s not running... continuing.' % name)
340 logger.log("sliver_lxc.destroy: undefining domain %s"%name)
343 logger.verbose('sliver_lxc.destroy: Domain %s is not defined... continuing.' % name)
345 # Remove user after destroy domain to force logout
346 command = ['/usr/sbin/userdel', '-f', '-r', name]
347 logger.log_call(command)
349 # Remove rootfs of destroyed domain
350 command = ['/usr/bin/rm', '-rf', containerDir]
351 logger.log_call(command, timeout=BTRFS_TIMEOUT)
354 logger.log("-TMP-ls-l %s"%name)
355 command = ['ls', '-lR', containerDir]
356 logger.log_call(command)
357 logger.log("-TMP-vsys-status")
358 command = ['/usr/bin/systemctl', 'status', 'vsys']
359 logger.log_call(command)
362 # Remove rootfs of destroyed domain
363 command = ['btrfs', 'subvolume', 'delete', containerDir]
364 logger.log_call(command, timeout=BTRFS_TIMEOUT)
366 # For some reason I am seeing this :
367 #log_call: running command btrfs subvolume delete /vservers/inri_sl1
368 #log_call: ERROR: cannot delete '/vservers/inri_sl1' - Device or resource busy
369 #log_call: Delete subvolume '/vservers/inri_sl1'
370 #log_call:end command (btrfs subvolume delete /vservers/inri_sl1) returned with code 1
372 # something must have an open handle to a file in there, but I can't find out what it is
373 # the following code aims at gathering data on what is going on in the system at this point in time
374 # note that some time later (typically when the sliver gets re-created) the same
375 # attempt at deleting the subvolume does work
376 # also lsof never shows anything relevant; this is painful..
378 if not os.path.exists(containerDir):
379 logger.log('sliver_lxc.destroy: %s cleanly destroyed.'%name)
382 #logger.log("-TMP-cwd %s : %s"%(name,os.getcwd()))
383 # also lsof never shows anything relevant; this is painful..
384 #logger.log("-TMP-lsof %s"%name)
386 #logger.log_call(command)
387 logger.log("-TMP-ls-l %s"%name)
388 command = ['ls', '-lR', containerDir]
389 logger.log_call(command)
390 logger.log("-TMP-lsof")
392 logger.log_call(command)
393 if os.path.exists(containerDir):
394 logger.log('sliver_lxc.destroy: ERROR could not cleanly destroy %s - giving up'%name)