X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=pl_mom.pl;h=576975e45d9016f1dae175525f3dfc2d5c4c9533;hb=708e302a5e0db162ab47116070105bfcedd2a60b;hp=1e16c3c2a231171fe1841bac4b39c7e1a4f952a8;hpb=1beb79232d2015f6c4eb40c4fab2b1ccda9fc85a;p=mom.git diff --git a/pl_mom.pl b/pl_mom.pl index 1e16c3c..576975e 100755 --- a/pl_mom.pl +++ b/pl_mom.pl @@ -3,14 +3,13 @@ use POSIX qw(setsid); use Sys::Syslog; use Sys::Hostname; +use RPC::XML; #use LWP::Simple; $debug = 0; $proc = "pl_mom"; $alias_addr = "pl-mom\@planet-lab.org"; $from_addr = "support\@planet-lab.org"; -#$bwcap = "1.5Mbit"; -$bwcap = "off"; if (! $debug) { $kill_thresh = 90; @@ -18,30 +17,37 @@ if (! $debug) { $log_thresh = 85; $change_thresh = 5; $min_thresh = 10; - $byte_cutoff = 16000000000; # 16GB + #$bwcap_default = "off"; + $bwcap_default = "1.5Mbit"; + $cutoff_default = "16200000000"; # 16GB, for 1.5Mbit cap $bwmon_sleep = 900; $sendmail = "/usr/sbin/sendmail -t -f$from_addr"; + $vservers = "/etc/vservers"; $pidfile = "/var/run/$proc.pid"; $rebootfile = "/var/lib/misc/pl_mom.reboot"; $daily_log = "/var/lib/misc/pl_mom.daily"; $daily_stamp = "/var/lib/misc/pl_mom.stamp"; $configfile = "/etc/planetlab/pl_mom.conf"; + $capfile = "/var/lib/misc/pl_mom.oldcaps"; } else { $kill_thresh = 2; $reboot_thresh = 20; $log_thresh = 2; $change_thresh = 5; - $min_thresh = 10; - $byte_cutoff = 16000; + $min_thresh = 2; + $bwcap_default = "1Kbit"; + $cutoff_default = "10800"; $bwmon_sleep = 10; $sendmail = "cat"; + $vservers = "./debug"; $pidfile = "./$proc.pid"; - $rebootfile = "./pl_mom.reboot"; - $daily_log = "./pl_mom.daily"; - $daily_stamp = "./pl_mom.stamp"; - $configfile = "./pl_mom.conf"; + $rebootfile = "./debug/pl_mom.reboot"; + $daily_log = "./debug/pl_mom.daily"; + $daily_stamp = "./debug/pl_mom.stamp"; + $configfile = "./debug/pl_mom.conf"; + $capfile = "./debug/pl_mom.oldcaps"; } $sleep = 30; @@ -71,9 +77,10 @@ if (! $pid) { $pid = fork(); if (! $pid) { syslog ("info", "pl_mom: Launching bandwidth monitor"); - if ($bwcap =~ /off/) { - syslog("info", "pl_mom: Bandwidth capping is off"); + if ($bwcap_default =~ /off/) { + syslog("info", "pl_mom: Max rate unlimited by default"); } + reset_bandwidth_caps(); bandwidth_monitor(); die (0); } @@ -105,7 +112,7 @@ while (1) { } else { my $id = `id -u $hog`; chomp($id); - my $top = `chcontext --ctx $id top -b -n 1`; + my $top = `/usr/sbin/chcontext --ctx $id /usr/bin/top -b -n 1`; syslog ("warning", "pl_mom: Resetting slice $hog"); if (! $debug) { slice_reset($hog); @@ -130,7 +137,8 @@ sub reboot_kicker { system("touch $rebootfile"); if (! $debug) { - system("shutdown -r now"); + #system("shutdown -r now"); + system("/bin/sync; /sbin/reboot -f"); } die (0); } @@ -158,74 +166,42 @@ sub bandwidth_monitor { # Also may want a list of slices that are exempt from capping. if (defined(%Start)) { undef %Start; } if (defined(%Now)) { undef %Now; } + if (defined(%Cap)) { undef %Cap; } - if (defined(%Cap)) { - # Reset bandwidth limits here - chomp($cap = `cat /etc/planetlab/bwcap`); - foreach $sliceid ( sort ( keys %Cap ) ) { - $slice = get_slice_name($sliceid); - syslog("info", "pl_mom: Uncapping bandwidth of $slice"); - cap_bandwidth ($slice, $cap); - } - undef %Cap; - } + reset_bandwidth_caps(); syslog("info", "pl_mom: Beginning bandwidth monitoring for $now"); } - # Get baseline counts - `touch $daily_log`; - open (BASE, "+<$daily_log") || - print "Cannot open $daily_log; $!\n"; - while () { - my ($sliceid, $bytecount) = split(/ /); - $Start{$sliceid} = $bytecount; - } - - $status = `tc -s -d qdisc show`; - @lines = split(/\n/, $status); - for ($i = 0; $i < @lines; $i++) { - if ($lines[$i] =~ /qdisc pfifo/) { - $lines[$i] =~ s/^ +//; - @fields = split(/ /, $lines[$i]); - $slice = $fields[2]; - $slice =~ s/://; - - if ($slice != 9999) { - $lines[$i+1] =~ s/^ +//; - @fields = split(/ /, $lines[$i+1]); - $bytes = $fields[1]; - #if ($bytes) {print "Slice $slice sent $bytes bytes\n";} - - if (! defined($Start{$slice})) { - print BASE "$slice $bytes\n"; - $Start{$slice} = $bytes; - } - $Now{$slice} = $bytes; + get_slice_names(); + get_baseline_counts(); + get_slice_limits(); + + foreach $slice ( sort (keys %Start) ) { + if (defined $Now{$slice}) { + $today = $Now{$slice} - $Start{$slice}; + if (! (defined ($Cutoff{$slice})||$bwcap_default =~ /off/)) { + $Cutoff{$slice} = $cutoff_default; + $Maxrate{$slice} = $bwcap_default; } - } - } - close (BASE); - - if (!($bwcap =~ /off/)) { - foreach $slice ( sort (keys %Start) ) { - if (defined $Now{$slice}) { - $today = $Now{$slice} - $Start{$slice}; - if ($today >= $byte_cutoff && ! defined($Cap{$slice})) { - $Cap{$slice} = "sent"; - $slicename = get_slice_name($slice); - if ($slicename) { - bw_cap_mail($slicename); - # Cap bandwidth here - cap_bandwidth($slicename, $bwcap); - } else { - syslog("warning", "pl_mom: Could not find slice ". - "name for slice ID $slice"); - } + if ($debug) { + if ($today) { + $cutoff = defined($Cutoff{$slice}) + ? $Cutoff{$slice} : ""; + print "Slice $slice sent $today bytes; ". + "cutoff $cutoff\n"; } - } else { - # Token bucket for this slice is gone! } + if (defined ($Cutoff{$slice}) && + $today >= $Cutoff{$slice} && + ! defined($Cap{$slice})) { + $Cap{$slice} = "sent"; + bw_cap_mail($slice); + log_bandwidth_cap($slice, $Maxrate{$slice}); + cap_bandwidth($slice, $Maxrate{$slice}); + } + } else { + # Token bucket for this slice is gone! } } @@ -249,6 +225,113 @@ sub read_config_file { } } +sub get_slice_names { + # Read slice names from /etc/passwd + if (defined (%Name)) { undef %Name; } + open (PASSWD, ") { + my ($slicename, $passwd, $sliceid) = split(/:/); + $Name{$sliceid} = $slicename; + } + close PASSWD; +} + +sub get_baseline_counts { + `touch $daily_log`; + open (BASE, "+<$daily_log") || + print "Cannot open $daily_log; $!\n"; + while () { + my ($slice, $bytecount) = split(/ /); + $Start{$slice} = $bytecount; + } + + my $status = `tc -s -d qdisc show`; + my $sliceid = 0xffff; + @Lines = split(/\n/, $status); + foreach $line ( @Lines ) { + if ($line =~ /qdisc pfifo (.*): dev/) { + $sliceid = hex($1); + # "Capped" buckets all begin with 0x1000. Ignore the root + # (0x1000) and default (0x1fff) buckets, as well as + # "exempt" buckets that begin with 0x2000 (or anything + # other than 0x1000). + if (($sliceid & 0xf000) == 0x1000 && + $sliceid != 0x1000 && $sliceid != 0x1fff) { + $sliceid = $sliceid & 0x0fff; + } else { + $sliceid = 0xffff; + } + } else { + if ($line =~ /Sent (.*) bytes/) { + my $bytes = $1; + if ($sliceid != 0xffff) { + my $slice = $Name{$sliceid}; + if ($debug && $bytes) { + print "Slice: $slice ($sliceid), bytes $bytes\n"; + } + if (! defined($Start{$slice})) { + print BASE "$slice $bytes\n"; + $Start{$slice} = $bytes; + } + $Now{$slice} = $bytes; + } + } + } + } + close (BASE); +} + +sub get_slice_limits { + if (defined %Maxrate) { undef %Maxrate; } + if (defined %Cutoff) { undef %Cutoff; } + if (-e $vservers) { + my $result = `grep -H "^BWAVGRATE" $vservers/*.conf`; + chomp ($result); + my @Lines = split(/\n/,$result); + foreach $line ( @Lines ) { + if ($line =~ /\/([^\/]*).conf:BWAVGRATE=(.*)[Mm]bit/) { + $slice = $1; + $limit = $2."Mbit"; + $cutoff = ($2 * 1000000 * 86400)/8; + } else { + if ($line =~ /\/([^\/]*).conf:BWAVGRATE=(.*)[Kk]bit/) { + $slice = $1; + $limit = $2."Kbit"; + $cutoff = ($2 * 1000 * 86400)/8; + } else { + die "Could not parse line $line"; + } + } + $Maxrate{$slice} = $limit; + $Cutoff{$slice} = $cutoff; + } + } +} + +sub reset_bandwidth_caps { + if (-e $capfile) { + open(CAP, "<$capfile") or die "Cannot open $capfile: $!"; + while () { + chomp(); + ($slicename, $oldcap) = split(/ /); + syslog("info", "pl_mom: Restoring bandwidth cap of $oldcap ". + "to $slicename"); + cap_bandwidth ($slicename, $oldcap); + } + close CAP; + unlink($capfile); + } +} + +sub log_bandwidth_cap { + ($slicename, $cap) = @_; + syslog("warning", "pl_mom: Capping bandwidth of slice ". + "$slicename at $cap until midnight GMT."); + # Save current cap to $capfile + system("echo $slicename `bwlimit getcap $slicename` >> $capfile"); +} + sub send_mail { # Arg 0: recipient addresses, comma-separated string # Arg 1: subject line @@ -258,20 +341,24 @@ sub send_mail { my $subject = "Subject: $_[1]\n"; my $msg = $_[2]; - open(SENDMAIL, "|$sendmail") or die "Cannot open $sendmail: $!"; - print SENDMAIL $to; - print SENDMAIL $from; - print SENDMAIL $subject; - print SENDMAIL "Content-type: text/plain\n\n"; - print SENDMAIL $msg; - close(SENDMAIL); + if ($debug) { + print $to; + print $subject; + } else { + open(SENDMAIL, "|$sendmail") or die "Cannot open $sendmail: $!"; + print SENDMAIL $to; + print SENDMAIL $from; + print SENDMAIL $subject; + print SENDMAIL "Content-type: text/plain\n\n"; + print SENDMAIL $msg; + close(SENDMAIL); + } } sub cap_bandwidth { - # Arg 0: slice name - # Arg 1: bandwidth cap for 'tc' - `bwlimit setcap $_[0] $_[1]`; - `bwlimit on $_[0]`; + ($slicename, $cap) = @_; + system("bwlimit setcap $slicename $cap"); + system("bwlimit on $slicename"); } sub get_date { @@ -282,36 +369,16 @@ sub get_date { return $date; } -sub get_slice_name { - # Arg 0: slice ID - - # Need to map slice id to slice name; is there a sensor? - # For now, get it from /etc/passwd - my $name = ""; - open (PASSWD, ") { - my ($slicename, $passwd, $sliceid) = split(/:/); - if ($sliceid == $_[0]) { - $name = $slicename; - } - } - close PASSWD; - return $name; -} - sub bw_cap_mail { + my ($slicename) = @_; my $hostname = hostname(); my $date = get_date(); - my $sent = int($byte_cutoff/1000000000); - - # Put this here because this is where we have the - syslog("warning", "pl_mom: Capping bandwidth of slice ". - "$slicename at $bwcap until midnight GMT."); + my $sent = int($Cutoff{$slicename}/(1024*1024)); + my $bwcap = $Maxrate{$slicename}; - send_mail($alias_addr, + send_mail("$alias_addr, $slicename\@slices.planet-lab.org", "$proc capped bandwidth of slice $slicename on $hostname", - "Slice $slicename has transmitted more than ${sent}GB today". + "Slice $slicename has transmitted more than ${sent}MB today". " on $hostname. ". "Its bandwidth will be capped at $bwcap until midnight GMT.". "\n\n$date $hostname bwcap $slicename\n"); @@ -412,10 +479,10 @@ sub swap_used { sub get_slice_info { if (! $debug) { - #$content = get "http://127.0.0.1:3100/slicestat"; $content = `curl -s http://127.0.0.1:3100/slicestat`; } else { - $content = `cat ../pl_mom-deploy/slicestat` + #$content = `cat ../pl_mom-deploy/slicestat` + $content = `curl -s http://127.0.0.1:3100/slicestat`; } my @lines = split(/\n/, $content); %Slice = ();