10 from pwd import getpwnam
11 from string import Template
13 # vsys probably should not be a plugin
14 # the thing is, the right way to handle stuff would be that
15 # if slivers get created by doing a,b,c
16 # then they should be deleted by doing c,b,a
17 # the current ordering model for vsys plugins completely fails to capture that
18 from plugins.vsys import removeSliverFromVsys, startService as vsysStartService
23 import plnode.bwlimit as bwlimit
24 from initscript import Initscript
25 from account import Account
26 from sliver_libvirt import Sliver_Libvirt
30 class Sliver_LXC(Sliver_Libvirt, Initscript):
31 """This class wraps LXC commands"""
33 SHELL = '/usr/sbin/vsh'
35 # Need to add a tag at myplc to actually use this account
38 REF_IMG_BASE_DIR = '/vservers/.lvref'
39 CON_BASE_DIR = '/vservers'
41 def __init__ (self, rec):
43 Sliver_Libvirt.__init__ (self,rec)
44 Initscript.__init__ (self,name)
46 def configure (self, rec):
47 Sliver_Libvirt.configure (self,rec)
49 # in case we update nodemanager..
50 self.install_and_enable_vinit()
51 # do the configure part from Initscript
52 Initscript.configure(self,rec)
54 def start(self, delay=0):
55 if 'enabled' in self.rspec and self.rspec['enabled'] <= 0:
56 logger.log('sliver_lxc: not starting %s, is not enabled'%self.name)
58 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
59 self.install_and_enable_vinit()
60 # expose .ssh for omf_friendly slivers
61 if 'tags' in self.rspec and 'omf_control' in self.rspec['tags']:
62 Account.mount_ssh_dir(self.name)
63 Sliver_Libvirt.start (self, delay)
64 # if a change has occured in the slice initscript, reflect this in /etc/init.d/vinit.slice
65 self.refresh_slice_vinit()
67 def rerun_slice_vinit (self):
68 """This is called whenever the initscript code changes"""
69 # xxx - todo - not sure exactly how to:
70 # (.) invoke something in the guest
71 # (.) which options of systemctl should be used to trigger a restart
72 # should not prevent the first run from going fine hopefully
73 logger.log("WARNING: sliver_lxc.rerun_slice_vinit not implemented yet")
76 def create(name, rec=None):
77 ''' Create dirs, copy fs image, lxc_create '''
78 logger.verbose ('sliver_lxc: %s create'%(name))
79 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
83 vref = "lxc-f18-x86_64"
84 logger.log("sliver_libvirt: %s: WARNING - no vref attached, using hard-wired default %s" % (name,vref))
86 # compute guest arch from vref
87 # essentially we want x86_64 (default) or i686 here for libvirt
89 (x,y,arch)=vref.split('-')
90 arch = "x86_64" if arch.find("64")>=0 else "i686"
94 # Get the type of image from vref myplc tags specified as:
100 tags = rec['rspec']['tags']
109 refImgDir = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, vref)
110 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
112 # check the template exists -- there's probably a better way..
113 if not os.path.isdir(refImgDir):
114 logger.log('sliver_lxc: %s: ERROR Could not create sliver - reference image %s not found' % (name,vref))
115 logger.log('sliver_lxc: %s: ERROR Expected reference image in %s'%(name,refImgDir))
118 # this hopefully should be fixed now
119 # # in fedora20 we have some difficulty in properly cleaning up /vservers/<slicename>
120 # # also note that running e.g. btrfs subvolume create /vservers/.lvref/image /vservers/foo
121 # # behaves differently, whether /vservers/foo exists or not:
122 # # if /vservers/foo does not exist, it creates /vservers/foo
123 # # but if it does exist, then it creates /vservers/foo/image !!
124 # # so we need to check the expected container rootfs does not exist yet
125 # # this hopefully could be removed in a future release
126 # if os.path.exists (containerDir):
127 # logger.log("sliver_lxc: %s: WARNING cleaning up pre-existing %s"%(name,containerDir))
128 # command = ['btrfs', 'subvolume', 'delete', containerDir]
129 # logger.log_call(command, BTRFS_TIMEOUT)
131 # if os.path.exists (containerDir):
132 # logger.log('sliver_lxc: %s: ERROR Could not create sliver - could not clean up empty %s'%(name,containerDir))
135 # Snapshot the reference image fs (assume the reference image is in its own
137 command = ['btrfs', 'subvolume', 'snapshot', refImgDir, containerDir]
138 if not logger.log_call(command, timeout=BTRFS_TIMEOUT):
139 logger.log('sliver_lxc: ERROR Could not create BTRFS snapshot at', containerDir)
141 command = ['chmod', '755', containerDir]
142 logger.log_call(command)
144 # TODO: set quotas...
146 # Set hostname. A valid hostname cannot have '_'
147 #with open(os.path.join(containerDir, 'etc/hostname'), 'w') as f:
148 # print >>f, name.replace('_', '-')
150 # Add slices group if not already present
152 group = grp.getgrnam('slices')
154 command = ['/usr/sbin/groupadd', 'slices']
155 logger.log_call(command)
157 # Add unix account (TYPE is specified in the subclass)
158 command = ['/usr/sbin/useradd', '-g', 'slices', '-s', Sliver_LXC.SHELL, name, '-p', '*']
159 logger.log_call(command)
160 command = ['mkdir', '/home/%s/.ssh'%name]
161 logger.log_call(command)
163 # Create PK pair keys to connect from the host to the guest without
164 # password... maybe remove the need for authentication inside the
166 command = ['su', '-s', '/bin/bash', '-c', 'ssh-keygen -t rsa -N "" -f /home/%s/.ssh/id_rsa'%(name)]
167 logger.log_call(command)
169 command = ['chown', '-R', '%s.slices'%name, '/home/%s/.ssh'%name]
170 logger.log_call(command)
172 command = ['mkdir', '%s/root/.ssh'%containerDir]
173 logger.log_call(command)
175 command = ['cp', '/home/%s/.ssh/id_rsa.pub'%name, '%s/root/.ssh/authorized_keys'%containerDir]
176 logger.log_call(command)
178 logger.log("creating /etc/slicename file in %s" % os.path.join(containerDir,'etc/slicename'))
180 file(os.path.join(containerDir,'etc/slicename'), 'w').write(name)
182 logger.log_exc("exception while creating /etc/slicename")
185 file(os.path.join(containerDir,'etc/slicefamily'), 'w').write(vref)
187 logger.log_exc("exception while creating /etc/slicefamily")
191 uid = getpwnam(name).pw_uid
193 # keyerror will happen if user id was not created successfully
194 logger.log_exc("exception while getting user id")
197 logger.log("uid is %d" % uid)
198 command = ['mkdir', '%s/home/%s' % (containerDir, name)]
199 logger.log_call(command)
200 command = ['chown', name, '%s/home/%s' % (containerDir, name)]
201 logger.log_call(command)
202 etcpasswd = os.path.join(containerDir, 'etc/passwd')
203 etcgroup = os.path.join(containerDir, 'etc/group')
204 if os.path.exists(etcpasswd):
205 # create all accounts with gid=1001 - i.e. 'slices' like it is in the root context
207 logger.log("adding user %(name)s id %(uid)d gid %(slices_gid)d to %(etcpasswd)s" % (locals()))
209 file(etcpasswd,'a').write("%(name)s:x:%(uid)d:%(slices_gid)d::/home/%(name)s:/bin/bash\n" % locals())
211 logger.log_exc("exception while updating %s"%etcpasswd)
212 logger.log("adding group slices with gid %(slices_gid)d to %(etcgroup)s"%locals())
214 file(etcgroup,'a').write("slices:x:%(slices_gid)d\n"%locals())
216 logger.log_exc("exception while updating %s"%etcgroup)
217 sudoers = os.path.join(containerDir, 'etc/sudoers')
218 if os.path.exists(sudoers):
220 file(sudoers,'a').write("%s ALL=(ALL) NOPASSWD: ALL\n" % name)
222 logger.log_exc("exception while updating /etc/sudoers")
224 # customizations for the user environment - root or slice uid
225 # we save the whole business in /etc/planetlab.profile
226 # and source this file for both root and the slice uid's .profile
227 # prompt for slice owner, + LD_PRELOAD for transparently wrap bind
228 pl_profile=os.path.join(containerDir,"etc/planetlab.profile")
229 ld_preload_text="""# by default, we define this setting so that calls to bind(2),
230 # when invoked on 0.0.0.0, get transparently redirected to the public interface of this node
231 # see https://svn.planet-lab.org/wiki/LxcPortForwarding"""
232 usrmove_path_text="""# VM's before Features/UsrMove need /bin and /sbin in their PATH"""
233 usrmove_path_code="""
235 if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
236 if [ "$2" = "after" ] ; then
244 pathmunge /sbin after
247 with open(pl_profile,'w') as f:
248 f.write("export PS1='%s@\H \$ '\n"%(name))
249 f.write("%s\n"%ld_preload_text)
250 f.write("export LD_PRELOAD=/etc/planetlab/lib/bind_public.so\n")
251 f.write("%s\n"%usrmove_path_text)
252 f.write("%s\n"%usrmove_path_code)
254 # make sure this file is sourced from both root's and slice's .profile
255 enforced_line = "[ -f /etc/planetlab.profile ] && source /etc/planetlab.profile\n"
256 for path in [ 'root/.profile', 'home/%s/.profile'%name ]:
257 from_root=os.path.join(containerDir,path)
258 # if dir is not yet existing let's forget it for now
259 if not os.path.isdir(os.path.dirname(from_root)): continue
262 contents=file(from_root).readlines()
263 for content in contents:
264 if content==enforced_line: found=True
267 with open(from_root,"a") as user_profile:
268 user_profile.write(enforced_line)
269 # in case we create the slice's .profile when writing
270 if from_root.find("/home")>=0:
271 command=['chown','%s:slices'%name,from_root]
272 logger.log_call(command)
274 # Lookup for xid and create template after the user is created so we
275 # can get the correct xid based on the name of the slice
276 xid = bwlimit.get_xid(name)
278 # Template for libvirt sliver configuration
279 template_filename_sliceimage = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR,'lxc_template.xml')
280 if os.path.isfile (template_filename_sliceimage):
281 logger.log("WARNING: using compat template %s"%template_filename_sliceimage)
282 template_filename=template_filename_sliceimage
284 logger.log("Cannot find XML template %s"%template_filename_sliceimage)
287 interfaces = Sliver_Libvirt.get_interfaces_xml(rec)
290 with open(template_filename) as f:
291 template = Template(f.read())
292 xml = template.substitute(name=name, xid=xid, interfaces=interfaces, arch=arch)
294 logger.log('Failed to parse or use XML template file %s'%template_filename)
297 # Lookup for the sliver before actually
298 # defining it, just in case it was already defined.
300 dom = conn.lookupByName(name)
302 dom = conn.defineXML(xml)
303 logger.verbose('lxc_create: %s -> %s'%(name, Sliver_Libvirt.dom_details(dom)))
308 # umount .ssh directory - only if mounted
309 Account.umount_ssh_dir(name)
310 logger.verbose ('sliver_lxc: %s destroy'%(name))
311 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
313 containerDir = Sliver_LXC.CON_BASE_DIR + '/%s'%(name)
316 # Destroy libvirt domain
317 dom = conn.lookupByName(name)
319 logger.verbose('sliver_lxc.destroy: Domain %s does not exist!' % name)
322 # Slivers with vsys running will fail the subvolume delete
323 # removeSliverFromVsys return True if it stops vsys, telling us to start it again later
324 vsys_stopped = removeSliverFromVsys (name)
327 logger.log("sliver_lxc.destroy: destroying domain %s"%name)
330 logger.verbose('sliver_lxc.destroy: Domain %s not running... continuing.' % name)
333 logger.log("sliver_lxc.destroy: undefining domain %s"%name)
336 logger.verbose('sliver_lxc.destroy: Domain %s is not defined... continuing.' % name)
338 # Remove user after destroy domain to force logout
339 command = ['/usr/sbin/userdel', '-f', '-r', name]
340 logger.log_call(command)
342 # Remove rootfs of destroyed domain
343 command = ['/usr/bin/rm', '-rf', containerDir]
344 logger.log_call(command, timeout=BTRFS_TIMEOUT)
347 logger.log("-TMP-ls-l %s"%name)
348 command = ['ls', '-lR', containerDir]
349 logger.log_call(command)
350 logger.log("-TMP-vsys-status")
351 command = ['/usr/bin/systemctl', 'status', 'vsys']
352 logger.log_call(command)
355 # Remove rootfs of destroyed domain
356 command = ['btrfs', 'subvolume', 'delete', containerDir]
357 logger.log_call(command, timeout=BTRFS_TIMEOUT)
359 # For some reason I am seeing this :
360 #log_call: running command btrfs subvolume delete /vservers/inri_sl1
361 #log_call: ERROR: cannot delete '/vservers/inri_sl1' - Device or resource busy
362 #log_call: Delete subvolume '/vservers/inri_sl1'
363 #log_call:end command (btrfs subvolume delete /vservers/inri_sl1) returned with code 1
365 # something must have an open handle to a file in there, but I can't find out what it is
366 # the following code aims at gathering data on what is going on in the system at this point in time
367 # note that some time later (typically when the sliver gets re-created) the same
368 # attempt at deleting the subvolume does work
369 # also lsof never shows anything relevant; this is painful..
371 if not os.path.exists(containerDir):
372 logger.log('sliver_lxc.destroy: %s cleanly destroyed.'%name)
375 #logger.log("-TMP-cwd %s : %s"%(name,os.getcwd()))
376 # also lsof never shows anything relevant; this is painful..
377 #logger.log("-TMP-lsof %s"%name)
379 #logger.log_call(command)
380 logger.log("-TMP-ls-l %s"%name)
381 command = ['ls', '-lR', containerDir]
382 logger.log_call(command)
383 logger.log("-TMP-lsof")
385 logger.log_call(command)
386 if os.path.exists(containerDir):
387 logger.log('sliver_lxc.destroy: ERROR could not cleanly destroy %s - giving up'%name)
389 if vsys_stopped: vsysStartService()