5 There are a couple of tricky things going on here. First, the kernel
6 needs disk usage information in order to enforce the quota. However,
7 determining disk usage redundantly strains the disks. Thus, the
8 Sliver_VS.disk_usage_initialized flag is used to determine whether
9 this initialization has been made.
11 Second, it's not currently possible to set the scheduler parameters
12 for a sliver unless that sliver has a running process. /bin/vsh helps
13 us out by reading the configuration file so that it can set the
14 appropriate limits after entering the sliver context. Making the
15 syscall that actually sets the parameters gives a harmless error if no
16 process is running. Thus we keep vm_running on when setting scheduler
17 parameters so that set_sched_params() always makes the syscall, and we
18 don't have to guess if there is a running process or not.
26 from threading import BoundedSemaphore
29 # the util-vserver-pl module
36 # special constant that tells vserver to keep its existing settings
37 KEEP_LIMIT = vserver.VC_LIM_KEEP
39 # populate the sliver/vserver specific default allocations table,
40 # which is used to look for slice attributes
41 DEFAULT_ALLOCATION = {}
42 for rlimit in vserver.RLIMITS.keys():
44 DEFAULT_ALLOCATION["%s_min"%rlim]=KEEP_LIMIT
45 DEFAULT_ALLOCATION["%s_soft"%rlim]=KEEP_LIMIT
46 DEFAULT_ALLOCATION["%s_hard"%rlim]=KEEP_LIMIT
48 class Sliver_VS(accounts.Account, vserver.VServer):
49 """This class wraps vserver.VServer to make its interface closer to what we need."""
52 TYPE = 'sliver.VServer'
53 _init_disk_info_sem = BoundedSemaphore()
55 def __init__(self, rec):
57 logger.verbose ('sliver_vs: %s init'%name)
59 logger.log("sliver_vs: %s: first chance..."%name)
60 vserver.VServer.__init__(self, name,logfile='/var/log/nodemanager')
61 except Exception, err:
62 if not isinstance(err, vserver.NoSuchVServer):
63 # Probably a bad vserver or vserver configuration file
64 logger.log_exc("sliver_vs:__init__ (first chance) %s",name=name)
65 logger.log('sliver_vs: %s: recreating bad vserver' % name)
67 self.create(name, rec)
68 logger.log("sliver_vs: %s: second chance..."%name)
69 vserver.VServer.__init__(self, name,logfile='/var/log/nodemanager')
72 self.slice_id = rec['slice_id']
73 self.disk_usage_initialized = False
79 def create(name, rec = None):
80 logger.verbose('sliver_vs: %s: create'%name)
83 logger.log("sliver_vs: %s: ERROR - no vref attached, this is unexpected"%(name))
85 # band-aid for short period as old API doesn't have GetSliceFamily function
87 vref = "planetlab-f8-i386"
89 # used to look in /etc/planetlab/family,
90 # now relies on the 'GetSliceFamily' extra attribute in GetSlivers()
91 # which for legacy is still exposed here as the 'vref' key
93 # check the template exists -- there's probably a better way..
94 if not os.path.isdir ("/vservers/.vref/%s"%vref):
95 logger.log ("sliver_vs: %s: ERROR Could not create sliver - vreference image %s not found"%(name,vref))
100 (x,y,arch)=vref.split('-')
101 # mh, this of course applies when 'vref' is e.g. 'netflow'
102 # and that's not quite right
106 def personality (arch):
107 personality="linux32"
108 if arch.find("64")>=0:
109 personality="linux64"
114 command += ['/bin/bash','-x',]
115 command += ['/usr/sbin/vuseradd', ]
116 if 'attributes' in rec and 'isolate_loopback' in rec['attributes'] and rec['attributes']['isolate_loopback'] == '1':
118 # the vsliver imge to use
119 command += [ '-t', vref, ]
122 # logger.log_call(['/usr/sbin/vuseradd', '-t', vref, name, ], timeout=15*60)
123 logger.log_call(command, timeout=15*60)
124 # export slicename to the slice in /etc/slicename
125 file('/vservers/%s/etc/slicename' % name, 'w').write(name)
126 file('/vservers/%s/etc/slicefamily' % name, 'w').write(vref)
127 # set personality: only if needed (if arch's differ)
128 if tools.root_context_arch() != arch:
129 file('/etc/vservers/%s/personality' % name, 'w').write(personality(arch)+"\n")
130 logger.log('sliver_vs: %s: set personality to %s'%(name,personality(arch)))
134 # logger.log_call(['/usr/sbin/vuserdel', name, ])
135 logger.log_call(['/bin/bash','-x','/usr/sbin/vuserdel', name, ])
137 def configure(self, rec):
138 # in case we update nodemanager..
139 self.install_and_enable_vinit()
141 new_rspec = rec['_rspec']
142 if new_rspec != self.rspec:
143 self.rspec = new_rspec
146 new_initscript = rec['initscript']
147 if new_initscript != self.initscript:
148 self.initscript = new_initscript
149 # not used anymore, we always check against the installed script
150 #self.initscriptchanged = True
151 self.refresh_slice_vinit()
153 accounts.Account.configure(self, rec) # install ssh keys
155 # unconditionnally install and enable the generic vinit script
156 # mimicking chkconfig for enabling the generic vinit script
157 # this is hardwired for runlevel 3
158 def install_and_enable_vinit (self):
159 vinit_source="/usr/share/NodeManager/sliver-initscripts/vinit"
160 vinit_script="/vservers/%s/etc/rc.d/init.d/vinit"%self.name
161 rc3_link="/vservers/%s/etc/rc.d/rc3.d/S99vinit"%self.name
162 rc3_target="../init.d/vinit"
164 code=file(vinit_source).read()
165 if tools.replace_file_with_string(vinit_script,code,chmod=0755):
166 logger.log("vsliver_vs: %s: installed generic vinit rc script"%self.name)
167 # create symlink for runlevel 3
168 if not os.path.islink(rc3_link):
170 logger.log("vsliver_vs: %s: creating runlevel3 symlink %s"%(self.name,rc3_link))
171 os.symlink(rc3_target,rc3_link)
173 logger.log_exc("vsliver_vs: %s: failed to create runlevel3 symlink %s"%rc3_link)
175 def rerun_slice_vinit(self):
176 command = "/usr/sbin/vserver %s exec /etc/rc.d/init.d/vinit restart" % (self.name)
177 logger.log("vsliver_vs: %s: Rerunning slice initscript: %s" % (self.name, command))
178 subprocess.call(command + "&", stdin=open('/dev/null', 'r'), stdout=open('/dev/null', 'w'), stderr=subprocess.STDOUT, shell=True)
180 # this one checks for the existence of the slice initscript
181 # install or remove the slice inistscript, as instructed by the initscript tag
182 def refresh_slice_vinit(self):
184 sliver_initscript="/vservers/%s/etc/rc.d/init.d/vinit.slice"%self.name
185 if tools.replace_file_with_string(sliver_initscript,code,remove_if_empty=True,chmod=0755):
187 logger.log("vsliver_vs: %s: Installed new initscript in %s"%(self.name,sliver_initscript))
188 if self.is_running():
189 # Only need to rerun the initscript if the vserver is
190 # already running. If the vserver isn't running, then the
191 # initscript will automatically be started by
192 # /etc/rc.d/vinit when the vserver is started.
193 self.rerun_slice_vinit()
195 logger.log("vsliver_vs: %s: Removed obsolete initscript %s"%(self.name,sliver_initscript))
197 # bind mount root side dir to sliver side
198 # needs to be done before sliver starts
199 def expose_ssh_dir (self):
201 root_ssh="/home/%s/.ssh"%self.name
202 sliver_ssh="/vservers/%s/home/%s/.ssh"%(self.name,self.name)
203 # any of both might not exist yet
204 for path in [root_ssh,sliver_ssh]:
205 if not os.path.exists (path):
207 if not os.path.isdir (path):
209 mounts=file('/proc/mounts').read()
210 if mounts.find(sliver_ssh)<0:
212 subprocess.call("mount --bind -o ro %s %s"%(root_ssh,sliver_ssh),shell=True)
213 logger.log("expose_ssh_dir: %s mounted into slice %s"%(root_ssh,self.name))
215 logger.log_exc("expose_ssh_dir with slice %s failed"%self.name)
217 def start(self, delay=0):
218 if self.rspec['enabled'] <= 0:
219 logger.log('sliver_vs: not starting %s, is not enabled'%self.name)
221 logger.log('sliver_vs: %s: starting in %d seconds' % (self.name, delay))
223 # the generic /etc/init.d/vinit script is permanently refreshed, and enabled
224 self.install_and_enable_vinit()
225 # expose .ssh for omf_friendly slivers
226 if 'omf_control' in self.rspec['tags']:
227 self.expose_ssh_dir()
228 # if a change has occured in the slice initscript, reflect this in /etc/init.d/vinit.slice
229 self.refresh_slice_vinit()
230 child_pid = os.fork()
232 # VServer.start calls fork() internally,
233 # so just close the nonstandard fds and fork once to avoid creating zombies
234 tools.close_nonstandard_fds()
235 vserver.VServer.start(self)
238 os.waitpid(child_pid, 0)
241 logger.log('sliver_vs: %s: stopping' % self.name)
242 vserver.VServer.stop(self)
244 def is_running(self):
245 return vserver.VServer.is_running(self)
247 def set_resources(self):
248 disk_max = self.rspec['disk_max']
249 logger.log('sliver_vs: %s: setting max disk usage to %d KiB' % (self.name, disk_max))
250 try: # if the sliver is over quota, .set_disk_limit will throw an exception
251 if not self.disk_usage_initialized:
252 self.vm_running = False
253 Sliver_VS._init_disk_info_sem.acquire()
254 logger.log('sliver_vs: %s: computing disk usage: beginning' % self.name)
255 # init_disk_info is inherited from VServer
256 try: self.init_disk_info()
257 finally: Sliver_VS._init_disk_info_sem.release()
258 logger.log('sliver_vs: %s: computing disk usage: ended' % self.name)
259 self.disk_usage_initialized = True
260 vserver.VServer.set_disklimit(self, max(disk_max, self.disk_blocks))
262 logger.log_exc('sliver_vs: failed to set max disk usage',name=self.name)
264 # get/set the min/soft/hard values for all of the vserver
265 # related RLIMITS. Note that vserver currently only
266 # implements support for hard limits.
267 for limit in vserver.RLIMITS.keys():
269 minimum = self.rspec['%s_min'%type]
270 soft = self.rspec['%s_soft'%type]
271 hard = self.rspec['%s_hard'%type]
272 update = self.set_rlimit(limit, hard, soft, minimum)
274 logger.log('sliver_vs: %s: setting rlimit %s to (%d, %d, %d)'
275 % (self.name, type, hard, soft, minimum))
277 self.set_capabilities_config(self.rspec['capabilities'])
278 if self.rspec['capabilities']:
279 logger.log('sliver_vs: %s: setting capabilities to %s' % (self.name, self.rspec['capabilities']))
281 cpu_pct = self.rspec['cpu_pct']
282 cpu_share = self.rspec['cpu_share']
285 for key in self.rspec.keys():
286 if key.find('sysctl.') == 0:
287 sysctl=key.split('.')
289 # /etc/vservers/<guest>/sysctl/<id>/
290 dirname = "/etc/vservers/%s/sysctl/%s" % (self.name, count)
292 os.makedirs(dirname, 0755)
295 setting = open("%s/setting" % dirname, "w")
296 setting.write("%s\n" % key.lstrip("sysctl."))
298 value = open("%s/value" % dirname, "w")
299 value.write("%s\n" % self.rspec[key])
303 logger.log("sliver_vs: %s: writing %s=%s"%(self.name,key,self.rspec[key]))
305 logger.log("sliver_vs: %s: could not set %s=%s"%(self.name,key,self.rspec[key]))
306 logger.log("sliver_vs: %s: error = %s"%(self.name,e))
309 if self.rspec['enabled'] > 0:
311 logger.log('sliver_vs: %s: setting cpu reservation to %d%%' % (self.name, cpu_pct))
316 logger.log('sliver_vs: %s: setting cpu share to %d' % (self.name, cpu_share))
320 self.set_sched_config(cpu_pct, cpu_share)
321 # if IP address isn't set (even to 0.0.0.0), sliver won't be able to use network
322 if self.rspec['ip_addresses'] != '0.0.0.0':
323 logger.log('sliver_vs: %s: setting IP address(es) to %s' % \
324 (self.name, self.rspec['ip_addresses']))
326 if 'isolate_loopback' in self.rspec['tags']:
327 add_loopback = self.rspec['tags']['isolate_loopback'] != "1"
328 self.set_ipaddresses_config(self.rspec['ip_addresses'], add_loopback)
330 #logger.log("sliver_vs: %s: Setting name to %s" % (self.name, self.slice_id))
331 #self.setname(self.slice_id)
332 #logger.log("sliver_vs: %s: Storing slice id of %s for PlanetFlow" % (self.name, self.slice_id))
334 vserver_config_path = '/etc/vservers/%s'%self.name
335 if not os.path.exists (vserver_config_path):
336 os.makedirs (vserver_config_path)
337 file('%s/slice_id'%vserver_config_path, 'w').write("%d\n"%self.slice_id)
338 logger.log("sliver_vs: Recorded slice id %d for slice %s"%(self.slice_id,self.name))
340 logger.log("sliver_vs: Could not record slice_id for slice %s. Error: %s"%(self.name,str(e)))
342 logger.log_exc("sliver_vs: Error recording slice id: %s"%str(e),name=self.name)
345 if self.enabled == False:
349 if False: # Does not work properly yet.
350 if self.have_limits_changed():
351 logger.log('sliver_vs: %s: limits have changed --- restarting' % self.name)
353 while self.is_running() and stopcount > 0:
357 stopcount = stopcount - 1
360 else: # tell vsh to disable remote login by setting CPULIMIT to 0
361 logger.log('sliver_vs: %s: disabling remote login' % self.name)
362 self.set_sched_config(0, 0)