3 # Runs once a day to "fix" nodes in various ways
5 # Mark Huang <mlhuang@cs.princeton.edu>
6 # Copyright (C) 2005 The Trustees of Princeton University
11 PATH=/sbin:/usr/sbin:$PATH
13 # Parse PLC configuration
14 if [ -r /etc/planetlab/plc_config ] ; then
15 . /etc/planetlab/plc_config
20 PIDFILE=/var/run/pl_mop.pid
23 if [ -f $PIDFILE ] ; then
24 if kill -0 `cat $PIDFILE` >/dev/null 2>&1 ; then
25 logger -p info -t pl_mom "$0 (`cat $PIDFILE`) already running"
31 # Clean up stale lock files
32 trap "rm -f $PIDFILE" EXIT
34 # Run a command and log its output to syslog
36 eval $* 2>&1 | logger -p info -t "pl_mom: $1"
39 # OpenSSH server 3.8 and above refuse login for "locked"
40 # accounts. Replace "!!" with "*" in /etc/shadow for all VServer
43 echo "* Fixing /etc/shadow"
46 for file in /etc/vservers/*.conf pl_admin.conf site_admin.conf ; do
47 slice=$(basename ${file%*.conf})
48 if grep -q "$slice:\!\!" /etc/shadow ; then
49 sed -i -e "s/$slice:\!\!:\(.*\)/$slice:*:\1/" /etc/shadow
54 # keep essential services running
56 for service in sshd pl_sshd swapmon nm fprobe-ulog codemux; do
57 echo "* Checking $service"
58 status=$(service $service status)
59 if [ $? -ne 0 ] || echo $status 2>&1 | grep -q stopped ; then
60 echo "* Restarting $service"
61 service $service start
66 # kill all the processes running in slice contexts
68 vps -A | awk '(int($2) > 1) { system("vkill -c " $2 " -s 9 " $1); }'
69 # unmounts all the /proc and /dev/pts mounts in each vserver
71 while grep -q /vservers/ /proc/mounts && [ $tries -gt 0 ] ; do
73 awk '(/vservers\//) { sub(/\\040.*$/, ""); print "Unmounting " $2; system("umount " $2); }' /proc/mounts
77 # /vservers gets re-mounted read-only by the kernel if an ext3 journal
80 echo "* Fixing /vservers"
82 # test to see if /vservers is mounted read-only
83 mkdir -p /vservers/.vtmp
84 tmp=$(mktemp /vservers/.vtmp/fixit.XXXXXX)
85 if [ $? -eq 0 ] ; then
90 # kill all processes running in slice contexts
94 pidfile=/var/run/vcached.pid
95 if [ -r "$pidfile" ] ; then
101 if umount /vservers ; then
102 # install expect if necessary
103 if ! rpm -q expect ; then
104 yum -y install expect
107 # tell expect to hit the 'y' key every time fsck asks
108 expect -c 'set timeout 3600; spawn fsck /dev/mapper/planetlab-vservers; expect "<y>?" { send "y\r"; exp_continue }'
110 # blow away the vserver cache
111 rm -rf /vservers/.vcache/*
113 # XXX re-mount /vservers
116 # shutdown instead to avoid clearing disk quotas
117 shutdown -r now "/vservers filesystem repaired, rebooting"
119 echo "Unable to unmount /vservers!" >&2
122 # allow vcached to run again
126 kill_duplicate_ssh() {
127 echo "* Killing stale duplicate SSH instances"
129 # count the number of SSH instances started by each slice
130 ps -C sshd -o command= |
133 while read instances sshd slice priv ; do
134 # kill all old instances
135 if [ $instances -gt 10 ] ; then
136 ps -C sshd -o pid=,start_time=,command= |
137 grep "$slice \[priv\]" |
138 while read pid start_time command ; do
139 start_time=$(date -d "$start_time" +%s)
140 min=$(date -d "6 hours ago" +%s)
141 if [ $start_time -lt $min ] ; then
142 echo "* Killing $slice sshd pid $pid"
151 pids=$(vps aux | awk '$1 != "root" && $14 == "/usr/share/NodeManager/nm.py" {print $2}')
152 for pid in $pids ; do
153 line=$(vps aux | grep $pid)
154 echo NM found in slice. Killing PID $pid
161 # For whatever reason, Some NM's, after fork and chcontext...don't chcontext. Kill them.
162 pids=$(ps aux | awk '$1 != "root" && $12 == "/usr/share/NodeManager/nm.py" {print $2}')
163 for pid in $pids ; do
164 line=$(ps aux | grep $pid)
165 echo NM found not belonging to root. Killing PID $pid
172 # if there is more than one nm running around, kill them, then nm restart
173 pids=$(ps aux | awk '$1 == "root" && $12 == "/usr/share/NodeManager/nm.py" {print $2}')
175 for pid in $pids ; do
178 if [ $i -gt 1 ] ; then
180 echo "More than 1 NM found belonging to root. Restarting NM."
182 pids=$(ps aux | awk '$1 == "root" && $12 == "/usr/share/NodeManager/nm.py" {print $2}')
183 for pid in $pids ; do
189 # XXX kill zombie slices
204 run kill_duplicate_ssh