10 from pwd import getpwnam
11 from string import Template
13 # vsys probably should not be a plugin
14 # the thing is, the right way to handle stuff would be that
15 # if slivers get created by doing a,b,c
16 # then they sohuld be delted by doing c,b,a
17 # the current ordering model for vsys plugins completely fails to capture that
18 from plugins.vsys import removeSliverFromVsys, startService as vsysStartService
23 import plnode.bwlimit as bwlimit
24 from initscript import Initscript
25 from account import Account
26 from sliver_libvirt import Sliver_Libvirt
28 class Sliver_LXC(Sliver_Libvirt, Initscript):
29 """This class wraps LXC commands"""
31 SHELL = '/usr/sbin/vsh'
33 # Need to add a tag at myplc to actually use this account
36 REF_IMG_BASE_DIR = '/vservers/.lvref'
37 CON_BASE_DIR = '/vservers'
39 def __init__ (self, rec):
41 Sliver_Libvirt.__init__ (self,rec)
42 Initscript.__init__ (self,name)
44 def configure (self, rec):
45 Sliver_Libvirt.configure (self,rec)
47 # in case we update nodemanager..
48 self.install_and_enable_vinit()
49 # do the configure part from Initscript
50 Initscript.configure(self,rec)
52 def start(self, delay=0):
53 if 'enabled' in self.rspec and self.rspec['enabled'] <= 0:
54 logger.log('sliver_lxc: not starting %s, is not enabled'%self.name)
56 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
57 self.install_and_enable_vinit()
58 # expose .ssh for omf_friendly slivers
59 if 'tags' in self.rspec and 'omf_control' in self.rspec['tags']:
60 Account.mount_ssh_dir(self.name)
61 Sliver_Libvirt.start (self, delay)
62 # if a change has occured in the slice initscript, reflect this in /etc/init.d/vinit.slice
63 self.refresh_slice_vinit()
65 def rerun_slice_vinit (self):
66 """This is called whenever the initscript code changes"""
67 # xxx - todo - not sure exactly how to:
68 # (.) invoke something in the guest
69 # (.) which options of systemctl should be used to trigger a restart
70 # should not prevent the first run from going fine hopefully
71 logger.log("WARNING: sliver_lxc.rerun_slice_vinit not implemented yet")
74 def create(name, rec=None):
75 ''' Create dirs, copy fs image, lxc_create '''
76 logger.verbose ('sliver_lxc: %s create'%(name))
77 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
79 # Get the type of image from vref myplc tags specified as:
85 tags = rec['rspec']['tags']
93 vref = "lxc-f18-x86_64"
94 logger.log("sliver_libvirt: %s: WARNING - no vref attached, using hard-wired default %s" % (name,vref))
96 refImgDir = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR, vref)
97 containerDir = os.path.join(Sliver_LXC.CON_BASE_DIR, name)
99 # check the template exists -- there's probably a better way..
100 if not os.path.isdir(refImgDir):
101 logger.log('sliver_lxc: %s: ERROR Could not create sliver - reference image %s not found' % (name,vref))
102 logger.log('sliver_lxc: %s: ERROR Expected reference image in %s'%(name,refImgDir))
105 # in fedora20 we have some difficulty in properly cleaning up /vservers/<slicename>
106 # also note that running e.g. btrfs subvolume create /vservers/.lvref/image /vservers/foo
107 # behaves differently, whether /vservers/foo exists or not:
108 # if /vservers/foo does not exist, it creates /vservers/foo
109 # but if it does exist, then it creates /vservers/foo/image !!
110 # so we need to check the expected container rootfs does not exist yet
111 if not os.path.exists (containerDir):
114 # if it's empty then let's clean it up
115 if not os.listdir(containerDir):
116 # clean up rootfs as userdel will only take care of /home/<slicename>
117 logger.log("sliver_lxc: %s: WARNING cleaning up empty %s"%(name,containerDir))
118 command = ['btrfs', 'subvolume', 'delete', containerDir]
119 logger.log_call(command, timeout=60)
121 if os.path.exists (containerDir):
122 logger.log('sliver_lxc: %s: ERROR Could not create sliver - could not clean up empty %s'%(name,containerDir))
125 logger.log('sliver_lxc: %s: ERROR Could not create sliver - could not clean up pre-existing %s'%(name,containerDir))
128 # Snapshot the reference image fs (assume the reference image is in its own
130 command = ['btrfs', 'subvolume', 'snapshot', refImgDir, containerDir]
131 if not logger.log_call(command, timeout=15*60):
132 logger.log('sliver_lxc: ERROR Could not create BTRFS snapshot at', containerDir)
134 command = ['chmod', '755', containerDir]
135 logger.log_call(command, timeout=15*60)
137 # TODO: set quotas...
139 # Set hostname. A valid hostname cannot have '_'
140 #with open(os.path.join(containerDir, 'etc/hostname'), 'w') as f:
141 # print >>f, name.replace('_', '-')
143 # Add slices group if not already present
145 group = grp.getgrnam('slices')
147 command = ['/usr/sbin/groupadd', 'slices']
148 logger.log_call(command, timeout=15*60)
150 # Add unix account (TYPE is specified in the subclass)
151 command = ['/usr/sbin/useradd', '-g', 'slices', '-s', Sliver_LXC.SHELL, name, '-p', '*']
152 logger.log_call(command, timeout=15*60)
153 command = ['mkdir', '/home/%s/.ssh'%name]
154 logger.log_call(command, timeout=15*60)
156 # Create PK pair keys to connect from the host to the guest without
157 # password... maybe remove the need for authentication inside the
159 command = ['su', '-s', '/bin/bash', '-c', 'ssh-keygen -t rsa -N "" -f /home/%s/.ssh/id_rsa'%(name)]
160 logger.log_call(command, timeout=60)
162 command = ['chown', '-R', '%s.slices'%name, '/home/%s/.ssh'%name]
163 logger.log_call(command, timeout=30)
165 command = ['mkdir', '%s/root/.ssh'%containerDir]
166 logger.log_call(command, timeout=10)
168 command = ['cp', '/home/%s/.ssh/id_rsa.pub'%name, '%s/root/.ssh/authorized_keys'%containerDir]
169 logger.log_call(command, timeout=30)
171 logger.log("creating /etc/slicename file in %s" % os.path.join(containerDir,'etc/slicename'))
173 file(os.path.join(containerDir,'etc/slicename'), 'w').write(name)
175 logger.log_exc("exception while creating /etc/slicename")
178 file(os.path.join(containerDir,'etc/slicefamily'), 'w').write(vref)
180 logger.log_exc("exception while creating /etc/slicefamily")
184 uid = getpwnam(name).pw_uid
186 # keyerror will happen if user id was not created successfully
187 logger.log_exc("exception while getting user id")
190 logger.log("uid is %d" % uid)
191 command = ['mkdir', '%s/home/%s' % (containerDir, name)]
192 logger.log_call(command, timeout=10)
193 command = ['chown', name, '%s/home/%s' % (containerDir, name)]
194 logger.log_call(command, timeout=10)
195 etcpasswd = os.path.join(containerDir, 'etc/passwd')
196 etcgroup = os.path.join(containerDir, 'etc/group')
197 if os.path.exists(etcpasswd):
198 # create all accounts with gid=1001 - i.e. 'slices' like it is in the root context
200 logger.log("adding user %(name)s id %(uid)d gid %(slices_gid)d to %(etcpasswd)s" % (locals()))
202 file(etcpasswd,'a').write("%(name)s:x:%(uid)d:%(slices_gid)d::/home/%(name)s:/bin/bash\n" % locals())
204 logger.log_exc("exception while updating %s"%etcpasswd)
205 logger.log("adding group slices with gid %(slices_gid)d to %(etcgroup)s"%locals())
207 file(etcgroup,'a').write("slices:x:%(slices_gid)d\n"%locals())
209 logger.log_exc("exception while updating %s"%etcgroup)
210 sudoers = os.path.join(containerDir, 'etc/sudoers')
211 if os.path.exists(sudoers):
213 file(sudoers,'a').write("%s ALL=(ALL) NOPASSWD: ALL\n" % name)
215 logger.log_exc("exception while updating /etc/sudoers")
217 # customizations for the user environment - root or slice uid
218 # we save the whole business in /etc/planetlab.profile
219 # and source this file for both root and the slice uid's .profile
220 # prompt for slice owner, + LD_PRELOAD for transparently wrap bind
221 pl_profile=os.path.join(containerDir,"etc/planetlab.profile")
222 ld_preload_text="""# by default, we define this setting so that calls to bind(2),
223 # when invoked on 0.0.0.0, get transparently redirected to the public interface of this node
224 # see https://svn.planet-lab.org/wiki/LxcPortForwarding"""
225 usrmove_path_text="""# VM's before Features/UsrMove need /bin and /sbin in their PATH"""
226 usrmove_path_code="""
228 if ! echo $PATH | /bin/egrep -q "(^|:)$1($|:)" ; then
229 if [ "$2" = "after" ] ; then
237 pathmunge /sbin after
240 with open(pl_profile,'w') as f:
241 f.write("export PS1='%s@\H \$ '\n"%(name))
242 f.write("%s\n"%ld_preload_text)
243 f.write("export LD_PRELOAD=/etc/planetlab/lib/bind_public.so\n")
244 f.write("%s\n"%usrmove_path_text)
245 f.write("%s\n"%usrmove_path_code)
247 # make sure this file is sourced from both root's and slice's .profile
248 enforced_line = "[ -f /etc/planetlab.profile ] && source /etc/planetlab.profile\n"
249 for path in [ 'root/.profile', 'home/%s/.profile'%name ]:
250 from_root=os.path.join(containerDir,path)
251 # if dir is not yet existing let's forget it for now
252 if not os.path.isdir(os.path.dirname(from_root)): continue
255 contents=file(from_root).readlines()
256 for content in contents:
257 if content==enforced_line: found=True
260 with open(from_root,"a") as user_profile:
261 user_profile.write(enforced_line)
262 # in case we create the slice's .profile when writing
263 if from_root.find("/home")>=0:
264 command=['chown','%s:slices'%name,from_root]
265 logger.log_call(command,timeout=5)
267 # Lookup for xid and create template after the user is created so we
268 # can get the correct xid based on the name of the slice
269 xid = bwlimit.get_xid(name)
271 # Template for libvirt sliver configuration
272 template_filename_sliceimage = os.path.join(Sliver_LXC.REF_IMG_BASE_DIR,'lxc_template.xml')
273 if os.path.isfile (template_filename_sliceimage):
274 logger.log("WARNING: using compat template %s"%template_filename_sliceimage)
275 template_filename=template_filename_sliceimage
277 logger.log("Cannot find XML template %s"%template_filename_sliceimage)
280 interfaces = Sliver_Libvirt.get_interfaces_xml(rec)
283 with open(template_filename) as f:
284 template = Template(f.read())
285 xml = template.substitute(name=name, xid=xid, interfaces=interfaces, arch=arch)
287 logger.log('Failed to parse or use XML template file %s'%template_filename)
290 # Lookup for the sliver before actually
291 # defining it, just in case it was already defined.
293 dom = conn.lookupByName(name)
295 dom = conn.defineXML(xml)
296 logger.verbose('lxc_create: %s -> %s'%(name, Sliver_Libvirt.dom_details(dom)))
301 # umount .ssh directory - only if mounted
302 Account.umount_ssh_dir(name)
303 logger.verbose ('sliver_lxc: %s destroy'%(name))
304 conn = Sliver_Libvirt.getConnection(Sliver_LXC.TYPE)
306 containerDir = Sliver_LXC.CON_BASE_DIR + '/%s'%(name)
309 # Destroy libvirt domain
310 dom = conn.lookupByName(name)
312 logger.verbose('sliver_lxc.destroy: Domain %s does not exist!' % name)
315 # Slivers with vsys running will fail the subvolume delete
316 # removeSliverFromVsys return True if it stops vsys, telling us to start it again later
317 vsys_stopped = removeSliverFromVsys (name)
320 logger.log("sliver_lxc.destroy: destroying domain %s"%name)
323 logger.verbose('sliver_lxc.destroy: Domain %s not running... continuing.' % name)
326 logger.log("sliver_lxc.destroy: undefining domain %s"%name)
329 logger.verbose('sliver_lxc.destroy: Domain %s is not defined... continuing.' % name)
331 # Remove user after destroy domain to force logout
332 command = ['/usr/sbin/userdel', '-f', '-r', name]
333 logger.log_call(command, timeout=15*60)
335 # Remove rootfs of destroyed domain
336 command = ['btrfs', 'subvolume', 'delete', containerDir]
337 logger.log_call(command, timeout=10)
339 # For some reason I am seeing this :
340 #log_call: running command btrfs subvolume delete /vservers/inri_sl1
341 #log_call: ERROR: cannot delete '/vservers/inri_sl1' - Device or resource busy
342 #log_call: Delete subvolume '/vservers/inri_sl1'
343 #log_call:end command (btrfs subvolume delete /vservers/inri_sl1) returned with code 1
345 # something must have an open handle to a file in there, but I can't find out what it is
346 # the following code aims at gathering data on what is going on in the system at this point in time
347 # note that some time later (typically when the sliver gets re-created) the same
348 # attempt at deleting the subvolume does work
349 # also lsof never shows anything relevant; this is painful..
351 if not os.path.exists(containerDir):
352 logger.log('sliver_lxc.destroy: %s cleanly destroyed.'%name)
354 logger.log("-TMP-cwd %s : %s"%(name,os.getcwd()))
355 logger.log("-TMP-lsof %s"%name)
357 logger.log_call(command)
358 logger.log("-TMP-ls-l %s"%name)
359 command = ['ls', '-l', containerDir]
360 logger.log_call(command)
361 if os.path.exists(containerDir):
362 logger.log('sliver_lxc.destroy: ERROR could not cleanly destroy %s - giving up'%name)
364 if vsys_stopped: vsysStartService()