From 391310e122de0536c08f62bd46acd3b3b7b13964 Mon Sep 17 00:00:00 2001 From: Thierry Parmentelat Date: Fri, 16 Nov 2007 15:14:21 +0000 Subject: [PATCH] * tentative merge of onelab myplc * configuration works a slightly different way, using plc-config-tty is recommended * a few new convenience commands * improved packaging (plcapi doc, yum.conf, sudoers) * more robust for invoking service plc through ssh --- Makefile | 37 +++++ build.functions | 128 ++++++++++++--- build.sh | 91 +++++++++-- build_devel.sh | 20 ++- check-ssl-peering.py | 125 +++++++++++++++ clean-empty-dirs.py | 55 +++++++ db-config | 4 +- plc_config.xml => default_config.xml | 14 +- dns-config | 2 +- doc/Makefile | 10 +- guest.init | 28 +++- host.init | 76 ++++++++- mtail.py | 229 +++++++++++++++++++++++++++ myplc.spec | 37 +++-- planetlab-f7-plc.lst | 49 ++++++ planetlab-fc4-plc.lst | 49 ++++++ planetlab-fc6-plc.lst | 49 ++++++ plc-config | 42 +++-- plc-config-tty | 42 +++-- plc-map.py | 125 +++++++++++++++ plc.d/api | 9 +- plc.d/bootcd | 2 +- plc.d/bootmanager | 2 +- plc.d/crond | 30 ++-- plc.d/db | 24 ++- plc.d/dns | 2 +- plc.d/functions | 13 +- plc.d/gpg | 2 +- plc.d/httpd | 17 +- plc.d/mail | 4 +- plc.d/network | 2 +- plc.d/packages | 8 +- plc.d/postgresql | 8 +- plc.d/ssh | 2 +- plc.d/ssl | 2 +- plc.d/syslog | 2 +- plc_config.dtd | 2 +- plc_config.py | 2 +- plc_devel_config.xml | 18 +-- refresh-peer.py | 36 +++++ 40 files changed, 1235 insertions(+), 164 deletions(-) create mode 100644 Makefile create mode 100755 check-ssl-peering.py create mode 100755 clean-empty-dirs.py rename plc_config.xml => default_config.xml (97%) create mode 100755 mtail.py create mode 100644 planetlab-f7-plc.lst create mode 100644 planetlab-fc4-plc.lst create mode 100644 planetlab-fc6-plc.lst create mode 100755 plc-map.py create mode 100755 refresh-peer.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9f819f4 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +# +# $Id: Makefile 1078 2007-11-15 13:38:27Z thierry $ +# + +BINARIES = plc-config plc-config-tty db-config dns-config refresh-peer.py plc-map.py clean-empty-dirs.py mtail.py +INIT_SCRIPTS = api bootcd bootmanager crond db dns functions gpg httpd mail network packages postgresql ssh ssl syslog + +INITS=$(addprefix plc.d/,$(INIT_SCRIPTS)) + +########## make sync PLCHOST=hostname +ifdef PLCHOST +PLCSSH:=root@$(PLCHOST) +endif + +LOCAL_RSYNC_EXCLUDES := --exclude '*.pyc' +RSYNC_EXCLUDES := --exclude .svn --exclude CVS --exclude '*~' --exclude TAGS $(LOCAL_RSYNC_EXCLUDES) +RSYNC_COND_DRY_RUN := $(if $(findstring n,$(MAKEFLAGS)),--dry-run,) +RSYNC := rsync -a -v $(RSYNC_COND_DRY_RUN) $(RSYNC_EXCLUDES) + +sync: +ifeq (,$(PLCSSH)) + echo "sync: You must define target host as PLCHOST on the command line" + echo " e.g. make sync PLCHOST=private.one-lab.org" ; exit 1 +else + +$(RSYNC) host.init $(PLCSSH):/etc/init.d/plc + +$(RSYNC) guest.init $(PLCSSH):/plc/root/etc/init.d/plc + +$(RSYNC) $(BINARIES) $(PLCSSH):/plc/root/usr/bin + +$(RSYNC) $(INITS) $(PLCSSH):/plc/root/etc/plc.d + +$(RSYNC) plc_config.py $(PLCSSH):/plc/root/usr/lib/python2.4/site-packages/plc_config.py + +$(RSYNC) default_config.xml $(PLCSSH):/plc/data/etc/planetlab/default_config.xml + @echo XXXXXXXX You might consider running the following command + @echo ssh $(PLCSSH) chroot /plc/root service plc start +endif + + +tags: + find . -type f | egrep -v '.svn/|~$$' | xargs etags diff --git a/build.functions b/build.functions index cef3f49..8eb3ce9 100644 --- a/build.functions +++ b/build.functions @@ -6,7 +6,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: build.functions,v 1.10.2.1 2007/08/30 16:39:07 mef Exp $ +# $Id: build.functions 1086 2007-11-15 14:17:45Z thierry $ # PATH=/sbin:/bin:/usr/sbin:/usr/bin @@ -48,37 +48,117 @@ set -x # Make a basic chroot at the specified location given the specified # configuration. -make_chroot() { +make_chroot_from_lst() { root=$1 - config=$2 + lst=$2 - # Get group list - groups= - while read group ; do - groups="$groups -g \"$group\"" - done < <(./plc-config --groups $config) - - # Get package list - packages= - while read package ; do - packages="$packages -p \"$package\"" - done < <(./plc-config --packages $config) + packages=$(pl_getPackagesOptions $lst) + groups=$(pl_getGroupsOptions $lst) pl_setup_chroot $root $packages $groups } # Move specified directories out of the chroot and into a "data" # directory that will be bind mounted on /data inside the chroot. -move_datadirs() { - root=$1 - data=$2 - shift 2 - pl_move_dirs $root $data /data "$@" -} +#move_datadirs() { +# root=$1 +# data=$2 +# shift 2 +# pl_move_dirs $root $data /data "$@" +#} # Make loopback filesystem from specified location -make_image() { - root=$1 - image=$2 - pl_make_image $root $image 100000000 +#make_image() { +# root=$1 +# image=$2 +# pl_make_image $root $image 100000000 +#} + +function yum_conf_to_build_host () { + BUILD_HOST=$(hostname) + cat < $php + +

Build-time error - could not locate documentation $html

+__header_no_doc__ + else + # insert header, makes sure we have a trailing eol + (cat << __header_doc__ ; cat $html ) > $php + +__header_doc__ + # ignore ed return status + set +e + # cuts off around the + # preserves the 4 first lines that we just added as a header + ed -s $php << __ed_script__ +/BODY/ +/>/ +s,><,<, +5,-d +$ +?/BODY? +s,><.*,>, ++ +;d +w +q +__ed_script__ + set -e + fi +} diff --git a/build.sh b/build.sh index 8f2ee3a..3352f1f 100755 --- a/build.sh +++ b/build.sh @@ -3,7 +3,7 @@ # Builds MyPLC, either inside the MyPLC development environment in # devel/root (if PLC_DEVEL_BOOTSTRAP is true), or in the current host # environment (may be itself a MyPLC development environment or a -# Fedora environment with the appropriate development packages +# Fedora Core 4 environment with the appropriate development packages # installed). # # root.img (loopback image) @@ -13,14 +13,17 @@ # data/root (root's homedir) # # Mark Huang -# Marc E. Fiuczynski -# Copyright (C) 2006-2007 The Trustees of Princeton University +# Copyright (C) 2006 The Trustees of Princeton University # -# $Id: build.sh,v 1.41.2.1 2007/08/30 16:39:08 mef Exp $ +# $Id: build.sh 1095 2007-11-16 09:52:30Z thierry $ # . build.functions +# pldistro expected as $1 - defaults to planetlab +pldistro=planetlab +[ -n "$@" ] && pldistro=$1 + # These directories are allowed to grow to unspecified size, so they # are stored as symlinks to the /data partition. mkfedora and yum # expect some of them to be real directories, however. @@ -46,7 +49,9 @@ pl_fixdirs root "${datadirs[@]}" echo "* myplc: Installing base filesystem" mkdir -p root data -make_chroot root plc_config.xml + +lst=${pldistro}-${pl_DISTRO_NAME}-plc.lst +make_chroot_from_lst root $lst # Install configuration scripts echo "* myplc: Installing configuration scripts" @@ -56,6 +61,10 @@ install -D -m 755 plc-config root/usr/bin/plc-config install -D -m 755 plc-config-tty root/usr/bin/plc-config-tty install -D -m 755 db-config root/usr/bin/db-config install -D -m 755 dns-config root/usr/bin/dns-config +install -D -m 755 plc-map.py root/usr/bin/plc-map.py +install -D -m 755 clean-empty-dirs.py root/usr/bin/clean-empty-dirs.py +install -D -m 755 mtail.py root/usr/bin/mtail.py +install -D -m 755 check-ssl-peering.py root/usr/bin/check-ssl-peering.py # Install initscripts echo "* myplc: Installing initscripts" @@ -72,17 +81,71 @@ chroot root sh -c 'chkconfig --add plc; chkconfig plc on' #$srcdir/plc/scripts/gen-static-content.py \ #root/usr/bin/ -# Install web pages -echo "* myplc: Installing web pages" -mkdir -p root/var/www/html -rsync -a $srcdir/WWW/ root/var/www/html/ - -# Install Drupal rewrite rules -install -D -m 644 $srcdir/WWW/drupal.conf root/etc/httpd/conf.d/drupal.conf +### Thierry Parmentelat - april 16 2007 +### from now on we package plcwww separately in the plcwww rpm +#### Install web pages +###echo "* myplc: Installing web pages" +###mkdir -p root/var/www/html +###rsync -a $srcdir/new_plc_www/ root/var/www/html/ + +#### Install Drupal rewrite rules +###install -D -m 644 $srcdir/new_plc_www/drupal.conf root/etc/httpd/conf.d/drupal.conf + +### Thierry Parmentelat - april 16 2007 +# fetch the release stamp from the build if any +# I could not come up with any more sensitive scheme +if [ -f ../../../SOURCES/myplc-release ] ; then + cp ../../../SOURCES/myplc-release myplc-release +else + echo "No build information found in ../.." > myplc-release +fi +# install it in /etc/myplc-release +install -m 444 myplc-release root/etc/myplc-release + +### Thierry Parmentelat - april 16 2007 +# fix the yum.conf as produced by mkfedora +# so we can use the build's fc4 mirror for various installs/upgrades +# within the chroot jail +# yum_conf_to_build_host is defined in build.functions +yum_conf_to_build_host > root/etc/yum.conf + +### Thierry Parmentelat - may 16 2007 +# the node-dependent image generation script requires root privilege +# to perform various mount operations +sudoers_bootcustom_apache > root/etc/sudoers +chown root:root root/etc/sudoers +chmod 400 root/etc/sudoers + +### Thierry Parmentelat - july 20 2007 +# we now build the myplc doc +# beware that making the pdf file somehow overwrites the html +make -C doc myplc.pdf +rm -f doc/myplc.html +make -C doc myplc.html + +# install at the same place as plcapi - better ideas ? +for doc in myplc.html myplc.pdf ; do + install -m 644 doc/$doc root/usr/share/plc_api/doc/$doc +done + +# we now build the plcapi doc +# this generates a drupal php file from a docbook-generated html +# quick & dirty +docbook_html_to_drupal "OneLab PLCAPI Documentation" \ + root/usr/share/plc_api/doc/PLCAPI.html \ + root/var/www/html/planetlab/doc/plcapi.php +# pdf just get copied +install -m 644 root/usr/share/plc_api/doc/PLCAPI.pdf root/var/www/html/planetlab/doc/plcapi.pdf + +docbook_html_to_drupal "Myplc User Guide" \ + root/usr/share/plc_api/doc/myplc.html \ + root/var/www/html/planetlab/doc/myplc.php +# pdf just get copied +install -m 644 root/usr/share/plc_api/doc/myplc.pdf root/var/www/html/planetlab/doc/myplc.pdf # Install configuration file echo "* myplc: Installing configuration file" -install -D -m 444 $config data/etc/planetlab/default_config.xml +install -D -m 444 default_config.xml data/etc/planetlab/default_config.xml install -D -m 444 plc_config.dtd data/etc/planetlab/plc_config.dtd # handle root's homedir and tweak root prompt @@ -107,7 +170,7 @@ rm -f data/var/www/html/boot/bootmanager.sh # Initialize node RPMs directory. The PlanetLab-Bootstrap.tar.bz2 # tarball already contains all of the node RPMs pre-installed. Only # updates or optional packages should be placed in this directory. -install -D -m 644 $pl_YUMGROUPSXML \ +install -D -m 644 $pl_DISTRO_YUMGROUPS \ data/var/www/html/install-rpms/planetlab/yumgroups.xml # Make image out of directory diff --git a/build_devel.sh b/build_devel.sh index 1005bde..1f755ec 100755 --- a/build_devel.sh +++ b/build_devel.sh @@ -13,12 +13,15 @@ # devel/data/root (root's home dir) # # Mark Huang -# Marc E. Fiuczynski -# Copyright (C) 2006-2007 The Trustees of Princeton University +# Copyright (C) 2006 The Trustees of Princeton University # -# $Id: build_devel.sh,v 1.11 2007/08/31 02:33:04 mef Exp $ +# $Id: build_devel.sh 1078 2007-11-15 13:38:27Z thierry $ # +echo "$0" not supported anymore +echo "need to figure a way to handle space in group names in .lst files" +exit 1 + . build.functions # These directories are allowed to grow to unspecified size, so they @@ -38,7 +41,8 @@ pl_fixdirs devel/root "${datadirs[@]}" echo "* myplc-devel: Installing base filesystem" mkdir -p devel/root -make_chroot devel/root plc_devel_config.xml +# xxx need be pldistro & fcdistro dependant +make_chroot_from_lst devel/root planetlab-devel.lst # Install configuration file echo "* myplc-devel: Installing configuration file" @@ -65,9 +69,11 @@ echo "* myplc-devel: Adding build user" uid=${SUDO_UID:-2000} gid=${SUDO_GID:-2000} if ! grep -q "Automated Build" devel/root/etc/passwd ; then - chroot devel/root sh -c "groupadd -o -g $gid build; \ -useradd -o -c 'Automated Build' -u $uid -g $gid -n -d /data/build -M -s /bin/bash build; \ -exit 0" + chroot devel/root < + +GetPeerName + + + + +AuthMethod +gpg + + +name +%s + + +signature +%s + + + + + +"""%(local_peername,signature) + return post + + def check_url (self,url,local_peername,remote_peername,cert,timeout=10,verbose=1): + curl=pycurl.Curl() + curl.setopt(pycurl.NOSIGNAL, 1) + + # Follow redirections + curl.setopt(pycurl.FOLLOWLOCATION, 1) + curl.setopt(pycurl.URL, str(url)) + cert_path = str(cert) + curl.setopt(pycurl.CAINFO, cert_path) + curl.setopt(pycurl.SSL_VERIFYPEER, 2) + + # Set connection timeout + if timeout: + curl.setopt(pycurl.CONNECTTIMEOUT, timeout) + curl.setopt(pycurl.TIMEOUT, timeout) + + curl.setopt(pycurl.VERBOSE, verbose) + + # Post request + curl.setopt(pycurl.POST, 1) + curl.setopt(pycurl.POSTFIELDS, self.getpeername_post_request(local_peername)) + + import StringIO + b = StringIO.StringIO() + curl.setopt(pycurl.WRITEFUNCTION, b.write) + + try: + curl.perform() + errcode = curl.getinfo(pycurl.HTTP_CODE) + response = b.getvalue() + print 'xmlrpc answer',response + if response.find('Failed') >= 0: + print 'FAILURE : failed to authenticate ?' + return False + elif response.find(remote_peername) <0: + print 'FAILURE : xmlrpc round trip OK but peername does not match' + return False + else: + print 'SUCCESS' + return True + + except pycurl.error, err: + (errcode, errmsg) = err + if errcode == 60: + print 'FAILURE', "SSL certificate validation failed, %r"%(errmsg) + elif errcode != 200: + print 'FAILURE', "HTTP error %d, errmsg %r" % (errcode,errmsg) + return False + + def main (self): + from optparse import OptionParser + usage="%prog [options] local-peername remote-peername cacert hostname [ .. hostname ]" + parser=OptionParser(usage=usage) + parser.add_option('-s','--secret',default='/etc/planetlab/secring.gpg', + dest='PLC_ROOT_GPG_KEY', + help='local GPG secret ring') + parser.add_option('-p','--public',default='/etc/planetlab/pubring.gpg', + dest='PLC_ROOT_GPG_KEY_PUB', + help='local GPG public ring') + (self.options, args) = parser.parse_args() + + if len(args) < 4: + parser.print_help() + sys.exit(2) + arg=0 + local_peername=args[arg] ; arg+=1 + remote_peername=args[arg] ; arg+=1 + cacert=args[arg]; arg+=1 + ok=False + for hostname in args[arg:]: +# this does not seem to make any difference +# for url_format in [ 'https://%s:443/PLCAPI/' , 'https://%s/PLCAPI/' ]: + for url_format in [ 'https://%s/PLCAPI/' ]: + url=url_format%hostname + print '============================== Checking url=',url + if self.check_url(url,local_peername,remote_peername,cacert): + ok=True + if ok: + return 0 + else: + return 1 + +if __name__ == '__main__': + sys.exit(check_ssl().main()) diff --git a/clean-empty-dirs.py b/clean-empty-dirs.py new file mode 100755 index 0000000..59817f2 --- /dev/null +++ b/clean-empty-dirs.py @@ -0,0 +1,55 @@ +#!/usr/bin/python +### +### $Id: clean-empty-dirs.py 495 2007-06-11 07:20:50Z thierry $ +### +### utility script for cleaning empty directories +### useful to clean up /var/tmp/bootmedium +### + +""" +Usage: $0 dir [ .. dir] +scans all provided directories and prunes any empty directory under +the arg directories are always preserved +""" + +import os,sys + +### cleans up a everything under a given root +def clean_root (path, cleanRoot = False): + + if not os.path.isdir(path): + return + + # scan dir contents + files=os.listdir(path) + + for x in files: + fullpath=os.path.join(path, x) + if os.path.isfile(fullpath): + # we do not remove files + return + elif os.path.isdir(fullpath): + clean_root(fullpath,True) + + if (cleanRoot): + # rescan, and clean if empty + files=os.listdir(path) + if not files: + os.rmdir(path) + +ERROR_STR= """Error removing %(path)s, %(error)s """ + +def main (dirs): + + for dir in dirs: + try: + if dir.index("/") != 0: + print "%s: args must be absolute paths"%(sys.argv[0]) + print "%s: %s ignored"%(sys.argv[0],dir) + else: + clean_root(dir) + except OSError, (errno, strerror): + print ERROR_STR % {'path' : path, 'error': strerror } + +if __name__ == '__main__': + main (sys.argv[1:]) diff --git a/db-config b/db-config index 6651f61..cbdc3fe 100755 --- a/db-config +++ b/db-config @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: db-config,v 1.23 2007/07/02 18:44:10 tmack Exp $ +# $Id: db-config 1078 2007-11-15 13:38:27Z thierry $ # from plc_config import PLCConfiguration @@ -54,7 +54,7 @@ def main(): 'name': plc['name'] + " Central", 'abbreviated_name': plc['name'], 'login_base': plc['slice_prefix'], - 'is_public': True, + 'is_public': False, 'url': url, 'max_slices': 100 } diff --git a/plc_config.xml b/default_config.xml similarity index 97% rename from plc_config.xml rename to default_config.xml index 3c8b7d3..637b352 100644 --- a/plc_config.xml +++ b/default_config.xml @@ -6,7 +6,7 @@ Default PLC configuration file Mark Huang Copyright (C) 2006 The Trustees of Princeton University -$Id: plc_config.xml,v 1.21 2007/08/24 07:19:27 mef Exp $ +$Id: default_config.xml 1078 2007-11-15 13:38:27Z thierry $ --> @@ -517,6 +517,7 @@ $Id: plc_config.xml,v 1.21 2007/08/24 07:19:27 mef Exp $ + plc PlanetLab Central @@ -610,6 +611,17 @@ $Id: plc_config.xml,v 1.21 2007/08/24 07:19:27 mef Exp $ bootcd bootmanager + + + plcwww + + + sudo + + + vim-minimal + python-imaging + diff --git a/dns-config b/dns-config index b018558..8551985 100755 --- a/dns-config +++ b/dns-config @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: dns-config,v 1.1 2006/05/26 19:57:30 mlhuang Exp $ +# $Id: dns-config 129 2007-03-20 12:04:03Z thierry $ # from plc_config import PLCConfiguration diff --git a/doc/Makefile b/doc/Makefile index 2e347bb..6a9823e 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -4,13 +4,19 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: Makefile,v 1.5 2006/07/18 22:41:44 mlhuang Exp $ +# $Id: Makefile 704 2007-07-20 14:15:43Z thierry $ # vpath GenDoc.xsl ../../plc_www/doc vpath %_config.xml .. -all: myplc.pdf myplc.php +# dont redo php by default, this requires plc_www (see above) +# that we build separately (has no doc/ subdir anyway) +# note that the build host (myplc-devel) needs ghostscript +# that we added only on 20 july 2007 +all: myplc.pdf myplc.html + +static: pyplc.php .PHONY: all diff --git a/guest.init b/guest.init index c2e5aae..0b7e473 100755 --- a/guest.init +++ b/guest.init @@ -6,7 +6,7 @@ # # description: Manages all PLC services on this machine # -# $Id: guest.init,v 1.20 2006/08/08 23:19:52 mlhuang Exp $ +# $Id: guest.init 635 2007-07-05 11:08:14Z thierry $ # # Source function library and configuration @@ -20,11 +20,27 @@ verbose=0 # being run. The idea is that when the configuration changes, "service # plc restart" is called, all dependencies are fixed up, and # everything just works. + +### NOTE. +# we want the resulting myplc to be able to easily skip +# some steps. e.g. the packages step takes ages if you install +# all rpms under the repository. +# We skip steps whose name contains a dot (.) or a tilde (~) +# this way the operations would just rename a step name e.g. +# cd /etc/plc.d +# mv packages packages.hide +# +# The drawback is, this stuff does not survive an rpm update +# but that's maybe a good thing, that all is done at first start +### + steps=($( for step in /etc/plc.d/* ; do - if [ -f $step -a -x $step ] ; then + stepname=$(basename $step) + plainstepname=$(echo $stepname | sed -e 's,\.,,' -e 's,~,,') + if [ -f $step -a -x $step -a "$stepname" = "$plainstepname" ] ; then priority=$(sed -ne 's/# priority: \(.*\)/\1/p' $step) - echo $priority $(basename $step) + echo $priority $stepname fi done | sort -n | cut -d' ' -f2 )) @@ -37,10 +53,10 @@ reload () # Regenerate the main configuration file from default values # overlaid with site-specific and current values. + # Thierry -- 2007-07-05 : values in plc_config.xml are *not* taken into account here files=( /etc/planetlab/default_config.xml - /etc/planetlab/configs/*.xml - /etc/planetlab/plc_config.xml + /etc/planetlab/configs/site.xml ) for file in "${files[@]}" ; do if [ -n "$force" -o $file -nt /etc/planetlab/plc_config.xml ] ; then @@ -48,7 +64,7 @@ reload () plc-config --xml "${files[@]}" >$tmp if [ $? -eq 0 ] ; then mv $tmp /etc/planetlab/plc_config.xml - chmod 644 /etc/planetlab/plc_config.xml + chmod 444 /etc/planetlab/plc_config.xml else echo "PLC: Warning: Invalid configuration file(s) detected" rm -f $tmp diff --git a/host.init b/host.init index f207e00..a299e82 100755 --- a/host.init +++ b/host.init @@ -6,7 +6,7 @@ # # description: Manages all PLC services on this machine # -# $Id: host.init,v 1.8 2006/07/06 17:43:52 mlhuang Exp $ +# $Id: host.init 1078 2007-11-15 13:38:27Z thierry $ # PATH=/sbin:/bin:/usr/bin:/usr/sbin @@ -128,6 +128,58 @@ mountstatus_plc () done } +# safestop : tries to stop normally; if that fails, kills processes that are still using /plc +# needs the lsof rpm in the root context (should be a dependency of the myplc spec) +function check_command () +{ + command=$1; shift + found=$(type -p $command) + if [ -z "$found" ] ; then + echo "$COMMAND : requires command $command, was not found - exiting" + exit 1 + fi +} + + +### when process stil use /plc/root, we cannot umount it +function kill_all () +{ + [ -n "$DEBUG" ] && set -x + check_command lsof + + echo -n "Killing processes using $PLC_ROOT and $PLC_DATA: " + # initialize process list + former_process_list="unlikely" + + # we ignore unknown uids for now, since we run in the chroot jail anyway + # not too sure about that though, + while true; do + # get the list of processes - collapse and remove empty lines + process_list=$(lsof -t +D $PLC_ROOT +D $PLC_DATA) + if [ -z "$process_list" ] ; then + # we are done, let's bail out + success "$PLC_ROOT clear" ; echo ; return + fi + if [ "$process_list" = "$former_process_list" ] ; then + # we are stuck, no progress since last time : exit on error + failure "$PLC_ROOT locked" ; echo ; return + fi + # record for next loop + former_process_list="$process_list" + # kill them + kill $process_list + sleep 2 + # check there are dead + for pid in $process_list ; do + ps -o pid $pid &> /dev/null + if [ "$?" = 0 ] ; then + [ -n "$DEBUG" ] && echo "$pid survived kill - forcing kill -9" + kill -9 $pid + fi + done + done +} + # Get command shift $(($OPTIND - 1)) command=$1 @@ -142,6 +194,7 @@ case "$command" in restart) stop $* + ERRORS=0 start $* ;; @@ -150,11 +203,28 @@ case "$command" in ;; mount|umount|mountstatus) - ${command}_plc $* + ${command}_plc + ;; + + kill) + kill_all + ;; + + safestop) + stop + ### Checking : we might need to run kill + mounted=$(mountstatus_plc) + if [ -n "$mounted" ] ; then + echo "Umount failed : killing remaining processes and trying again" + ERRORS=0 + kill_all + ERRORS=0 + stop + fi ;; *) - echo "Usage: $0 {start|stop|restart|reload|mount|umount|mountstatus}" + echo "Usage: $0 {start|stop|restart|reload|mount|umount|mountstatus|kill|safestop}" RETVAL=1 ;; esac diff --git a/mtail.py b/mtail.py new file mode 100755 index 0000000..1fb5b68 --- /dev/null +++ b/mtail.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python + +''' +Does tail -f on log files in a given directory. +The display is in chronological order of the logged lines, +given that the first column of log files is timestamp. +It can be altered to fit other formats too +''' + +import os, sys, time +from optparse import OptionParser + +class mtail: + + subversion_id = "$Id: mtail.py 571 2007-06-22 21:38:07Z thierry $" + + default_time_format = "%H:%M:%S" + + def __init__ (self, args ): + + # internal structure for tracking changes + self.files = {} + # parse command-line args : will set options and args + self.parse_args(args) + # initialize + self.scan_files() + + def parse_args (self, args): + usage = """usage: %prog [options] file-or-dir ... +example: +# %prog -e '*access*' /var/log""" + parser=OptionParser(usage=usage,version=self.subversion_id) + # tail_period + parser.add_option("-p","--period", type="int", dest="tail_period", default=1, + help="Files check period in seconds") + # rescan_period + parser.add_option("-d","--dir-period", type="int", dest="rescan_period", default=20, + help="Directories rescan period in seconds") + # time format + parser.add_option("-f","--format", dest="time_format", default=mtail.default_time_format, + help="Time format, defaults to " + mtail.default_time_format) + # show time + parser.add_option("-r","--raw", action="store_true", dest="show_time", default=True, + help="Suppresses time display") + + # note for exclusion patterns + parser.add_option("-e","--exclude", action="append", dest="excludes", default=[], + help="Exclusion pattern -- can be specified multiple times applies on files not explicitly mentioned on the command-line") + + parser.add_option("-u","--usual",action="store_true",dest="plc_mode",default=False, + help="Shortcut for watching /var/log with default settings") + + # verbosity + parser.add_option("-v","--verbose", action="store_true", dest="verbose", default=False, + help="Run in verbose mode") + + (self.options, self.args) = parser.parse_args(args) + self.optparse = parser + + ### plc shortcuts + if self.options.plc_mode: + self.options.excludes.append('*access_log') + self.options.excludes.append('*request_log') + self.options.excludes.append('*.swp') + self.args.append('/var/log') + + if self.options.verbose: + print 'Version:',self.subversion_id + print 'Options:',self.options + print 'Arguments:',self.args + + def file_size (self,filename): + return os.stat(filename)[6] + + def number_files (self): + return len(self.files) + + # scans given arguments, and updates files accordingly + # can be run several times + def scan_files (self) : + + if self.options.verbose: + print 'entering scan_files, files=',self.files + + # mark entries in files as pre-existing + for key in self.files: + self.files[key]['old-file']=True + + # refreshes the proper set of filenames + filenames = [] + for arg in self.args: + if self.options.verbose: + print 'scan_files -- Considering arg',arg + if os.path.isfile (arg): + filenames += [ arg ] + elif os.path.isdir (arg) : + filenames += self.walk (arg) + else: + print "mtail : no such file or directory %s -- ignored"%arg + + # updates files + for filename in filenames : + # known file + if self.files.has_key(filename): + size = self.file_size(filename) + offset = self.files[filename]['size'] + if size > offset: + self.show_file_end(filename,offset,size) + self.files[filename]['size']=size + elif size < offset: + self.show_file_when_size_decreased(filename,offset,size) + del self.files[filename]['old-file'] + else: + # enter file with current size + # if we didn't set format yet, it's because we are initializing + try: + self.format + self.show_now() + print self.format%filename,"new file" + self.show_file_end(filename,0,self.file_size(filename)) + except: + pass + self.files[filename]={'size':self.file_size(filename)} + + # cleanup + # avoid side-effects on the current loop basis + read_filenames = self.files.keys() + for filename in read_filenames: + if self.files[filename].has_key('old-file'): + self.show_now() + print self.format%filename,"file has gone" + del self.files[filename] + + # compute margin and format + if not filenames: + print sys.argv[0],": WARNING : no file in scope" + self.format="%s" + else: + self.margin=max(*[len(f) for f in filenames]) + self.format="%%%ds"%self.margin + if self.options.verbose: + print 'Current set of files:',filenames + + def tail_files (self): + + if self.options.verbose: + print 'tail_files' + for filename in self.files: + size = self.file_size(filename) + offset = self.files[filename]['size'] + if size != offset: + self.show_file_end(filename,offset,size) + self.files[filename]['size']=size + + def show_now (self): + if self.options.show_time: + label=time.strftime(self.options.time_format,time.localtime()) + print label, + + def show_file_end (self, filename, offset, size): + file = open(filename,"r") + file.seek(offset) + line=file.read(size-offset) + self.show_now() + print self.format%filename,'----------------------------------------' + print line + file.close() + + def show_file_when_size_decreased (self, filename, offset, size): + print self.format%filename,'---------- file size decreased ---------', + if self.options.verbose: + print 'size during last check',offset,'current size',size + else: + print '' + + # get all files under a directory + def walk ( self, root ): + import fnmatch, os, string + + # initialize + result = [] + + # must have at least root folder + try: + names = os.listdir(root) + except os.error: + return result + + # check each file + for name in names: + fullname = os.path.normpath(os.path.join(root, name)) + + # a file : check for excluded, otherwise append + if os.path.isfile(fullname): + try: + for exclude in self.options.excludes: + if fnmatch.fnmatch(name, exclude): + raise Exception('excluded') + result.append(fullname) + except: + pass + # a dir : let's recurse - avoid symlinks for anti-loop + elif os.path.isdir(fullname) and not os.path.islink(fullname): + result = result + self.walk( fullname ) + + return result + + def run (self): + + if self.number_files() == 0: + self.optparse.print_help() + sys.exit(1) + counter = 0 + + while 1: + ## hit the period ? + # dont do this twice at startup + if (counter !=0 and counter % self.options.rescan_period == 0): + self.scan_files() + + if (counter % self.options.tail_period == 0): + self.tail_files() + + time.sleep(1) + counter += 1 + +### +if __name__ == '__main__': + mtail (sys.argv[1:]).run() diff --git a/myplc.spec b/myplc.spec index 0c58219..0d23663 100644 --- a/myplc.spec +++ b/myplc.spec @@ -1,34 +1,45 @@ -Vendor: PlanetLab -Packager: PlanetLab Central -Distribution: PlanetLab 4.0 -URL: http://cvs.planet-lab.org/cvs/myplc +# +# $Id: myplc.spec 1087 2007-11-15 14:25:23Z thierry $ +# +%define url $URL: svn+ssh://thierry@svn.planet-lab.org/svn/PLCAPI/trunk/PLCAPI.spec $ + +%define name myplc +%define version 4.0 +%define subversion 15 + +%define release %{subversion}%{?pldistro:.%{pldistro}}%{?date:.%{date}} Summary: PlanetLab Central (PLC) Portable Installation -Name: myplc -Version: 0.5 -Release: 5%{?pldistro:.%{pldistro}}%{?date:.%{date}} +Name: %{name} +Version: %{version} +Release: %{release} License: PlanetLab Group: Applications/Systems Source0: %{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root +Vendor: PlanetLab +Packager: PlanetLab Central +Distribution: PlanetLab 4.0 +URL: %(echo %{url} | cut -d ' ' -f 2) + %define debug_package %{nil} %description MyPLC is a complete PlanetLab Central (PLC) portable installation contained within a chroot jail. The default installation consists of a web server, an XML-RPC API server, a boot server, and a database -server: the core components of PLC. The installation may be customized -through a graphical interface. All PLC services are started up and +server: the core components of PLC. All PLC services are started up and shut down through a single System V init script installed in the host -system. +system. The related Web Interface is now separately packaged +in the PLCWWW component. %prep %setup -q %build pushd MyPLC -./build.sh +./build.sh %{pldistro} popd %install @@ -73,7 +84,7 @@ fi %pre if [ -x %{_sysconfdir}/init.d/plc ] ; then - %{_sysconfdir}/init.d/plc stop + %{_sysconfdir}/init.d/plc safestop fi # Old versions of myplc used to ship with a bootstrapped database and @@ -129,7 +140,7 @@ fi %preun # 0 = erase, 1 = upgrade if [ $1 -eq 0 ] ; then - %{_sysconfdir}/init.d/plc stop + %{_sysconfdir}/init.d/plc safestop if [ -x /sbin/chkconfig ] ; then /sbin/chkconfig plc off /sbin/chkconfig --del plc diff --git a/planetlab-f7-plc.lst b/planetlab-f7-plc.lst new file mode 100644 index 0000000..949b4ab --- /dev/null +++ b/planetlab-f7-plc.lst @@ -0,0 +1,49 @@ +# first draft, extracted from plc_config.xml with --packages +# the groups list was 'PlanetLab Central' but turned out to be an unknown group +package:bzip2 +package:sendmail-cf +package:tar +package:less +package:perl-GD +package:sudo +package:openssl +package:xmlsec1 +package:kernel-vserver +package:plcwww +package:gd +package:expect +package:php-pgsql +package:curl +package:rpm +package:httpd +package:rsync +package:mod_python +package:mod_ssl +package:bootmanager +package:python-devel +package:SOAPpy +package:vixie-cron +package:yum +package:php-gd +package:PLCAPI +package:vim-minimal +package:PyXML +package:sendmail +package:python +package:createrepo +package:postgresql-python +package:cpio +package:postgresql-server +package:wget +package:php +package:xmlsec1-openssl +package:postgresql +package:openssh +package:cvs +package:dev +package:python-imaging +package:bootcd +package:dnsmasq +package:diffutils +package:gzip +package:findutils diff --git a/planetlab-fc4-plc.lst b/planetlab-fc4-plc.lst new file mode 100644 index 0000000..949b4ab --- /dev/null +++ b/planetlab-fc4-plc.lst @@ -0,0 +1,49 @@ +# first draft, extracted from plc_config.xml with --packages +# the groups list was 'PlanetLab Central' but turned out to be an unknown group +package:bzip2 +package:sendmail-cf +package:tar +package:less +package:perl-GD +package:sudo +package:openssl +package:xmlsec1 +package:kernel-vserver +package:plcwww +package:gd +package:expect +package:php-pgsql +package:curl +package:rpm +package:httpd +package:rsync +package:mod_python +package:mod_ssl +package:bootmanager +package:python-devel +package:SOAPpy +package:vixie-cron +package:yum +package:php-gd +package:PLCAPI +package:vim-minimal +package:PyXML +package:sendmail +package:python +package:createrepo +package:postgresql-python +package:cpio +package:postgresql-server +package:wget +package:php +package:xmlsec1-openssl +package:postgresql +package:openssh +package:cvs +package:dev +package:python-imaging +package:bootcd +package:dnsmasq +package:diffutils +package:gzip +package:findutils diff --git a/planetlab-fc6-plc.lst b/planetlab-fc6-plc.lst new file mode 100644 index 0000000..949b4ab --- /dev/null +++ b/planetlab-fc6-plc.lst @@ -0,0 +1,49 @@ +# first draft, extracted from plc_config.xml with --packages +# the groups list was 'PlanetLab Central' but turned out to be an unknown group +package:bzip2 +package:sendmail-cf +package:tar +package:less +package:perl-GD +package:sudo +package:openssl +package:xmlsec1 +package:kernel-vserver +package:plcwww +package:gd +package:expect +package:php-pgsql +package:curl +package:rpm +package:httpd +package:rsync +package:mod_python +package:mod_ssl +package:bootmanager +package:python-devel +package:SOAPpy +package:vixie-cron +package:yum +package:php-gd +package:PLCAPI +package:vim-minimal +package:PyXML +package:sendmail +package:python +package:createrepo +package:postgresql-python +package:cpio +package:postgresql-server +package:wget +package:php +package:xmlsec1-openssl +package:postgresql +package:openssh +package:cvs +package:dev +package:python-imaging +package:bootcd +package:dnsmasq +package:diffutils +package:gzip +package:findutils diff --git a/plc-config b/plc-config index 645b1d0..e7b74fd 100755 --- a/plc-config +++ b/plc-config @@ -6,7 +6,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: plc-config,v 1.1.1.1 2006/03/27 17:36:46 mlhuang Exp $ +# $Id: plc-config 1078 2007-11-15 13:38:27Z thierry $ # import sys @@ -54,6 +54,11 @@ Usage: %s [OPTION]... [FILES] sys.exit(1) +def deprecated (message): + print "%s: deprecated usage"%sys.argv[0] + print message + sys.exit(1) + def main(): plc = PLCConfiguration() fileobjs = [] @@ -65,7 +70,7 @@ def main(): save = False # Standard options - shortopts = "hs" + shortopts = "hs:" longopts = ["shell", "bash", "python", "php", "xml", @@ -76,7 +81,7 @@ def main(): "category=", "variable=", "value=", "group=", "package=", "type=", "help", - "save"] + "save="] try: (opts, argv) = getopt.gnu_getopt(sys.argv[1:], shortopts, longopts) @@ -96,11 +101,14 @@ def main(): elif opt == "--variables": output = plc.output_variables elif opt == "--packages": - output = plc.output_packages +# output = plc.output_packages + deprecated("option --packages deprecated -- use .lst files instead") elif opt == "--groups": - output = plc.output_groups +# output = plc.output_groups + deprecated("option --groups deprecated -- use .lst files instead") elif opt == "--comps": - output = plc.output_comps +# output = plc.output_comps + deprecated("option --comps deprecated -- use .lst files instead") elif opt == "--category": category['id'] = optval elif opt == "--variable": @@ -108,13 +116,18 @@ def main(): elif opt == "--value": variable['value'] = optval elif opt == "--group": - group['id'] = optval +# group['id'] = optval + deprecated("option --group deprecated -- use .lst files instead") elif opt == "--package": - package['name'] = optval +# package['name'] = optval + deprecated("option --package deprecated -- use .lst files instead") elif opt == "--type": package['type'] = optval elif opt == '-s' or opt == "--save": - save = True + if not optval: + usage() + print 'parsed save option',optval + save = optval elif opt == '-h' or opt == "--help": usage() @@ -147,7 +160,16 @@ def main(): # --save if save: - plc.save() + # create directory if needed + # so that plc.d/{api,postgres} can create configs/site.xml + dirname = os.path.dirname (save) + if (not os.path.exists (dirname)): + os.makedirs(dirname,0755) + if (not os.path.exists (dirname)): + print "Cannot create dir %s - exiting" % dirname + sys.exit(1) + + plc.save(save) if __name__ == '__main__': diff --git a/plc-config-tty b/plc-config-tty index 79f7233..d9559dc 100755 --- a/plc-config-tty +++ b/plc-config-tty @@ -20,8 +20,8 @@ import getopt from plc_config import PLCConfiguration #################### -release_id = "$Id: plc-config-tty,v 1.10 2006/12/12 10:14:44 thierry Exp $" -release_rev = "$Revision: 1.10 $" +release_id = "$Id: plc-config-tty 635 2007-07-05 11:08:14Z thierry $" +release_rev = "$Revision: 635 $" def init_flavour (flavour): global service @@ -57,18 +57,18 @@ def init_flavour (flavour): global mainloop_usage mainloop_usage= """Available commands: Uppercase versions give variables comments, when available --u/U\t\t\tEdit usual variables --w\t\t\tWrite & consolidate --r\t\t\tRestart %s service --q\t\t\tQuit (without saving) --h/?\t\t\tThis help + u/U\t\t\tEdit usual variables + w\t\t\tWrite & consolidate + r\t\t\tRestart %s service + q\t\t\tQuit (without saving) + h/?\t\t\tThis help --- -l/L [|]\tShow Locally modified variables/values --s/S [|]\tShow variables/values (all, in category, single) --e/E [|]\tEdit variables (all, in category, single) + l/L [|]\tShow Locally modified variables/values + s/S [|]\tShow variables/values (all, in category, single) + e/E [|]\tEdit variables (all, in category, single) --- --c\t\t\tList categories --v/V [|]List Variables (all, in category, single) + c\t\t\tList categories + v/V [|]List Variables (all, in category, single) --- Typical usage involves: u, [l,] w, r, q """ % service @@ -182,6 +182,7 @@ def consolidate (default_config, site_config, consolidated_config): return print ("Merged\n\t%s\nand\t%s\ninto\t%s"%(default_config,site_config, consolidated_config)) + os.system("set -x ; service plc reload") #################### def restart_plc (): @@ -405,23 +406,16 @@ def mainloop (cdef, cread, cwrite, default_config, site_config, consolidated_con print ("Unknown command >%s< -- use h for help" % answer) #################### +# creates directory for file if not yet existing def check_dir (config_file): dirname = os.path.dirname (config_file) if (not os.path.exists (dirname)): - print "Config file %s located under a non-existing directory" % config_file - answer=raw_input("Want to create %s [y]/n ? " % dirname) - answer = answer.lower() - if (answer == 'n'): - print "Cannot proceed - good bye" + os.makedirs(dirname,0755) + if (not os.path.exists (dirname)): + print "Cannot create dir %s - exiting" % dirname sys.exit(1) else: - os.makedirs(dirname,0755) - if (not os.path.exists (dirname)): - print "Cannot create dir %s - exiting" % dirname - sys.exit(1) - else: - print "Created directory %s" % dirname - + print "Created directory %s" % dirname #################### def main (): diff --git a/plc-map.py b/plc-map.py new file mode 100755 index 0000000..932afc5 --- /dev/null +++ b/plc-map.py @@ -0,0 +1,125 @@ +#!/usr/bin/env plcsh + +import Image, ImageDraw + +####### first - rustic - linear positioning on a map +def circle (image, percentX, percentY, radiusX, radiusY, colorIn, colorOut): + + imageX, imageY = image.size + centerX = int(imageX*percentX) + centerY = int(imageY*percentY) + x = max (0, min (centerX,imageX)) + y = max (0, min (centerY,imageY)) + + x1 = x - radiusX + x2 = x + radiusX + y1 = y - radiusY + y2 = y + radiusY + + draw = ImageDraw.Draw (image) + draw.chord((x1,y1,x2,y2), 0, 360, fill=colorIn, outline=colorOut ) + del draw + +latitude={'top':65., + 'bottom': 35.5} +longitude={'left':-11., + 'right':58.} + +def render_site (site, image, sx, sy, cIn, cOut): + if site['longitude'] is not None and site['latitude'] is not None: + px=float(longitude['left']-site['longitude'])/float(longitude['left']-longitude['right']) + py=float(latitude['top']-site['latitude'])/float(latitude['top']-latitude['bottom']) + if (px<0 or py<0 or px>1 or py>1): + return + + circle(image,px,py,sx,sy,cIn,cOut) + +def make_image(): + path = '/var/www/html/sites/' + original = path + 'map.png' + live = path + 'livemap.png' + + # map characteristics, in degrees. + # latitude : positive is north + # longitude : positive is east + + # circle radius in pixels + sxLocal,syLocal=7,7 + # circle in and out colors + cInLocal , cOutLocal = '#566b8a','#bbbbbb' + + # same for federating / foreign sites + sxForeign,syForeign=6,6 + cInForeign , cOutForeign = '#acb3a4', '#444444' + + image = Image.open(original) + + for site in GetSites({'~peer_id':None,'enabled':True}): + render_site (site, image, sxForeign, syForeign, cInForeign , cOutForeign) + # local sites go last to be more visible + for site in GetSites({'peer_id':None,'enabled':True}): + render_site (site, image, sxLocal, syLocal, cInLocal , cOutLocal) + + image.save (live) + +########## second - way simpler - export sites as a list to javascript for rendering with googlemap +js_prelude=""" +function Site (lat,lon,site_id,name,peer_id,peername,nb_nodes) { + this.lat=lat; + this.lon=lon; + this.site_id=site_id; + this.name=name; + this.peer_id=peer_id; + this.peername=peername; + this.nb_nodes=nb_nodes; +} +""" + +def locate_peer (peers,peer_id): + for peer in peers: + if peer['peer_id']==peer_id: + return peer + return {'peername':'Cannot locate peer'} + +def js_site (site,peers): + # some sites come with lat or lon being None + lat = site['latitude'] + if not lat: + lat=0 + lon = site['longitude'] + if not lon: + lon=0 + # build javascript text + jstext="new Site(" + jstext += str(lat) + "," + str(lon) + "," + jstext += str(site['site_id']) + "," + # needs html encoding for wierd chars + jstext += '"' + site['name'].encode("utf-8") + '"' + ',' + if not site['peer_id']: + jstext += '0,""' +',' + else: + peer=locate_peer(peers,site['peer_id']) + jstext += str(site['peer_id']) + ',"' + peer['peername'].encode("utf-8") + '"' + ',' + jstext += str(len(site['node_ids'])) + jstext += ')\n' + return jstext + +def make_javascript(): + outputname="/var/www/html/sites/plc-sites.js" + f=open(outputname,"w") + f.write(js_prelude) + columns=['latitude','longitude','site_id','name','peer_id','node_ids'] + f.write("allSites=new Array(\n") + # writes foreign sites first + foreign_sites=GetSites({'~peer_id':None},columns) + peers=GetPeers({}) + local_sites=GetSites({'peer_id':None},columns) + f.write(",".join([js_site(site,peers) for site in foreign_sites+local_sites])) + f.write(");") + +def main (): + make_image () + make_javascript () + +if __name__ == '__main__': + main () diff --git a/plc.d/api b/plc.d/api index 1f9d138..4740f67 100755 --- a/plc.d/api +++ b/plc.d/api @@ -8,12 +8,13 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: api,v 1.8 2007/01/19 20:05:05 mlhuang Exp $ +# $Id: api 635 2007-07-05 11:08:14Z thierry $ # # Source function library and configuration . /etc/plc.d/functions . /etc/planetlab/plc_config +local_config=/etc/planetlab/configs/site.xml # Be verbose set -x @@ -31,7 +32,8 @@ case "$1" in # password. if [ -z "$PLC_API_MAINTENANCE_PASSWORD" ] ; then PLC_API_MAINTENANCE_PASSWORD=$(uuidgen) - plc-config --category=plc_api --variable=maintenance_password --value="$PLC_API_MAINTENANCE_PASSWORD" --save + plc-config --category=plc_api --variable=maintenance_password --value="$PLC_API_MAINTENANCE_PASSWORD" --save=$local_config $local_config + service plc reload fi # Make sure that all PLC servers are allowed to access the API @@ -46,7 +48,8 @@ case "$1" in done ) | sort -u)) PLC_API_MAINTENANCE_SOURCES=${PLC_API_MAINTENANCE_SOURCES[*]} - plc-config --category=plc_api --variable=maintenance_sources --value="$PLC_API_MAINTENANCE_SOURCES" --save + plc-config --category=plc_api --variable=maintenance_sources --value="$PLC_API_MAINTENANCE_SOURCES" --save=$local_config $local_config + service plc reload result "$MESSAGE" ;; diff --git a/plc.d/bootcd b/plc.d/bootcd index 87aa0e2..bc0986c 100755 --- a/plc.d/bootcd +++ b/plc.d/bootcd @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: bootcd,v 1.3 2006/06/23 20:29:22 mlhuang Exp $ +# $Id: bootcd 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/bootmanager b/plc.d/bootmanager index 8a48e3b..f36b9f0 100755 --- a/plc.d/bootmanager +++ b/plc.d/bootmanager @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: bootmanager,v 1.3 2006/06/23 20:29:22 mlhuang Exp $ +# $Id: bootmanager 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/crond b/plc.d/crond index 0c58bd0..c828ea7 100755 --- a/plc.d/crond +++ b/plc.d/crond @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: crond,v 1.9 2007/02/07 23:21:27 mlhuang Exp $ +# $Id: crond 1078 2007-11-15 13:38:27Z thierry $ # # Source function library and configuration @@ -36,26 +36,26 @@ SHELL=/bin/bash PATH=/sbin:/bin:/usr/sbin:/usr/bin MAILTO=$MAILTO HOME=/ - +# # minute hour day-of-month month day-of-week user command EOF # Run all jobs once at startup - if [ "$PLC_BOOT_ENABLED" = "1" ] ; then - echo "*/5 * * * * root gen-slices-xml-05.py" >>/etc/cron.d/plc.cron - gen-slices-xml-05.py - check +# if [ "$PLC_BOOT_ENABLED" = "1" ] ; then +# echo "*/5 * * * * root gen-slices-xml-05.py" >>/etc/cron.d/plc.cron +# gen-slices-xml-05.py +# check fi if [ "$PLC_WWW_ENABLED" = "1" ] ; then - echo "*/15 * * * * root gen-static-content.py" >>/etc/cron.d/plc.cron - echo "*/15 * * * * root gen-sites-xml.py" >>/etc/cron.d/plc.cron +# echo "*/15 * * * * root gen-static-content.py" >>/etc/cron.d/plc.cron +# echo "*/15 * * * * root gen-sites-xml.py" >>/etc/cron.d/plc.cron echo "00 * * * * wget -O - -q http://localhost/cron.php" >>/etc/cron.d/plc.cron - gen-static-content.py - check - gen-sites-xml.py - check +# gen-static-content.py +# check +# gen-sites-xml.py +# check wget -O - -q http://localhost/cron.php check fi @@ -72,6 +72,12 @@ EOF check fi + if [ "$PLC_WWW_ENABLED" = "1" ] ; then + echo "*/15 * * * * root clean-empty-dirs.py /var/tmp/bootmedium" >> /etc/cron.d/plc.cron + clean-empty-dirs.py /var/tmp/bootmedium + check + fi + plc_daemon crond check diff --git a/plc.d/db b/plc.d/db index 5bf4784..8f5fc14 100755 --- a/plc.d/db +++ b/plc.d/db @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: db,v 1.7 2007/01/31 19:53:20 mlhuang Exp $ +# $Id: db 316 2007-04-24 17:25:09Z thierry $ # # Source function library and configuration @@ -36,22 +36,33 @@ function migrate_db() extension=${script##*.} if [ $index -gt $subversion ] ; then if [ "$extension" = "sql" ] ; then + dialog " - $script (dbdumped)" + dump_planetlab_db "before-$script" psql -U $PLC_DB_USER -f $file $PLC_DB_NAME elif [ -x $file ] ; then + dialog " - $script (dbdumped)" + dump_planetlab_db "before-$script" $file + else + dialog "\nWarning: migration $file not executable" fi check fi done } -# Dumps the database -function dump_db() +# Dumps the database - optional argument to specify filename suffix +function dump_planetlab_db() { - dump=/var/lib/pgsql/backups/$(date +"$PLC_DB_NAME.%Y-%m-%d-%H-%M.sql") + if [ -n "$1" ] ; then suffix="-$1" ; else suffix="" ; fi + dump=/var/lib/pgsql/backups/$(date +"$PLC_DB_NAME.%Y-%m-%d-%H-%M-%S${suffix}.sql") pg_dump -U $PLC_DB_USER $PLC_DB_NAME > $dump check - dump=/var/lib/pgsql/backups/$(date +"drupal.%Y-%m-%d-%H-%M.sql") +} + +function dump_drupal_db() +{ + dump=/var/lib/pgsql/backups/$(date +"drupal.%Y-%m-%d-%H-%M-%S.sql") pg_dump -U $PLC_DB_USER drupal > $dump check } @@ -107,7 +118,8 @@ EOF MESSAGE=$"Dumping the databases in /var/lib/pgsql/backups" dialog "$MESSAGE" - dump_db + dump_planetlab_db + dump_drupal_db result "$MESSAGE" ;; diff --git a/plc.d/dns b/plc.d/dns index 8c8f87c..1727e8d 100755 --- a/plc.d/dns +++ b/plc.d/dns @@ -8,7 +8,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: dns,v 1.2 2006/05/26 19:57:30 mlhuang Exp $ +# $Id: dns 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/functions b/plc.d/functions index 15318f0..d4cb0a5 100644 --- a/plc.d/functions +++ b/plc.d/functions @@ -5,7 +5,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: functions,v 1.7 2007/01/19 17:12:45 mlhuang Exp $ +# $Id: functions 545 2007-06-18 10:56:31Z thierry $ # export PATH=/sbin:/bin:/usr/bin:/usr/sbin @@ -56,7 +56,16 @@ plc_daemon () [ -n "${pid:-}" -a -z "${force:-}" ] && return # And start it up. - (exec -a plc_${base} $*) + # Thierry -- June 18 2007 + # when invoking, e.g. service plc start httpd from an ssh connection + # ssh stupidly hangs when everything is done + # it turns out the forked ssh daemon exhibits the following stack at that point + # (gdb) where + # #0 0x001d6402 in __kernel_vsyscall () + # #1 0x003c2e7d in ___newselect_nocancel () from /lib/libc.so.6 + # #2 0x004387b4 in main () from /usr/sbin/sshd + # So I figured the various file descriptors used were not properly closed + (exec 3>&- 4>&- ; exec -a plc_${base} $*) ret=$? if [ -f /var/run/${base}.pid ] ; then diff --git a/plc.d/gpg b/plc.d/gpg index 66983c7..7f146a9 100755 --- a/plc.d/gpg +++ b/plc.d/gpg @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: gpg,v 1.8 2006/12/15 20:16:16 mlhuang Exp $ +# $Id: gpg 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/httpd b/plc.d/httpd index f614f51..993b543 100755 --- a/plc.d/httpd +++ b/plc.d/httpd @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: httpd,v 1.11 2007/02/06 16:24:13 mlhuang Exp $ +# $Id: httpd 350 2007-05-11 10:58:44Z thierry $ # # Source function library and configuration @@ -140,6 +140,10 @@ EOF Redirect /index.html http://$PLC_WWW_HOST:$PLC_WWW_PORT/ EOF fi + cat <>$plc_conf # Make alpina-logs directory writable for bootmanager log upload @@ -191,6 +195,17 @@ define('PLANETLAB_SUPPORT_EMAIL_ONLY', PLC_MAIL_SUPPORT_ADDRESS); ?> EOF + ## patch php.ini + # memory limit + sed -i -e 's,^memory_limit = 8M *;,memory_limit = 24M ; patch myplc -- ,' $php_ini + # log_errors : is On by default + # error_log + if ! grep '^error_log *=' $php_ini > /dev/null ; then + echo 'error_log = /var/log/php.log' >> $php_ini + touch /var/log/php.log + chmod 666 /var/log/php.log + fi + plc_daemon httpd check diff --git a/plc.d/mail b/plc.d/mail index fe31b25..2b3eade 100755 --- a/plc.d/mail +++ b/plc.d/mail @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: mail,v 1.3 2006/06/23 20:29:22 mlhuang Exp $ +# $Id: mail 554 2007-06-19 15:11:22Z thierry $ # # Source function library and configuration @@ -30,7 +30,7 @@ case "$1" in # without a warning, so that the API can send out mail. echo "apache" >/etc/mail/trusted-users - service sendmail start + (exec 3>&- 4>&- ; service sendmail start) check result "$MESSAGE" diff --git a/plc.d/network b/plc.d/network index d93f4e8..9df70e4 100755 --- a/plc.d/network +++ b/plc.d/network @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: network,v 1.5 2006/06/23 20:29:22 mlhuang Exp $ +# $Id: network 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/packages b/plc.d/packages index 453f9c6..6b02bd5 100755 --- a/plc.d/packages +++ b/plc.d/packages @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: packages,v 1.7 2007/02/04 01:05:28 mlhuang Exp $ +# $Id: packages 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration @@ -81,16 +81,16 @@ case "$1" in # Old command is yum-arch if [ $yum_arch -eq 1 ] ; then - yum-arch $repository + yum-arch $repository | tr '\r' '\n' | grep -v '^ *$' check fi # New command is createrepo if [ $createrepo -eq 1 ] ; then if [ -f $repository/yumgroups.xml ] ; then - createrepo -g yumgroups.xml $repository + createrepo -g yumgroups.xml $repository | tr '\r' '\n' | grep -v '^ *$' else - createrepo $repository + createrepo $repository | tr '\r' '\n' | grep -v '^ *$' fi check fi diff --git a/plc.d/postgresql b/plc.d/postgresql index 18ca081..4512536 100755 --- a/plc.d/postgresql +++ b/plc.d/postgresql @@ -7,12 +7,13 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: postgresql,v 1.11 2007/02/05 19:11:06 mlhuang Exp $ +# $Id: postgresql 635 2007-07-05 11:08:14Z thierry $ # # Source function library and configuration . /etc/plc.d/functions . /etc/planetlab/plc_config +local_config=/etc/planetlab/configs/site.xml # Be verbose set -x @@ -29,7 +30,7 @@ export PGPORT=$PLC_DB_PORT postgresql_start () { # start() always returns 0 - service postgresql start + (exec 3>&- 4>&- ; service postgresql start) # status() will still return 0 even while still initializing if status postmaster && [ -f /var/lock/subsys/postgresql ] ; then @@ -119,7 +120,8 @@ case "$1" in # Create/update the unprivileged database user and password if [ -z "$PLC_DB_PASSWORD" ] ; then PLC_DB_PASSWORD=$(uuidgen) - plc-config --category=plc_db --variable=password --value="$PLC_DB_PASSWORD" --save + plc-config --category=plc_db --variable=password --value="$PLC_DB_PASSWORD" --save=$local_config $local_config + service plc reload fi if ! psql -U $PLC_DB_USER -c "" template1 >/dev/null 2>&1 ; then psql -U postgres -c "CREATE USER $PLC_DB_USER PASSWORD '$PLC_DB_PASSWORD'" template1 diff --git a/plc.d/ssh b/plc.d/ssh index d629074..a76ba83 100755 --- a/plc.d/ssh +++ b/plc.d/ssh @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: ssh,v 1.2 2006/04/25 21:18:19 mlhuang Exp $ +# $Id: ssh 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/ssl b/plc.d/ssl index c181156..517b1f6 100755 --- a/plc.d/ssl +++ b/plc.d/ssl @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: ssl,v 1.11 2007/01/18 18:44:18 mlhuang Exp $ +# $Id: ssl 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc.d/syslog b/plc.d/syslog index 4a9a206..c5a2427 100755 --- a/plc.d/syslog +++ b/plc.d/syslog @@ -10,7 +10,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: syslog,v 1.2 2006/04/25 21:18:19 mlhuang Exp $ +# $Id: syslog 129 2007-03-20 12:04:03Z thierry $ # # Source function library and configuration diff --git a/plc_config.dtd b/plc_config.dtd index 9eea2fa..1d2a753 100644 --- a/plc_config.dtd +++ b/plc_config.dtd @@ -4,7 +4,7 @@ Specification for PLC configuration files Mark Huang Copyright (C) 2006 The Trustees of Princeton University -$Id$ +$Id: plc_config.dtd 129 2007-03-20 12:04:03Z thierry $ --> diff --git a/plc_config.py b/plc_config.py index 35ff200..b034e89 100644 --- a/plc_config.py +++ b/plc_config.py @@ -7,7 +7,7 @@ # Mark Huang # Copyright (C) 2006 The Trustees of Princeton University # -# $Id: plc_config.py,v 1.4 2006/07/17 21:29:15 mlhuang Exp $ +# $Id: plc_config.py 1078 2007-11-15 13:38:27Z thierry $ # import xml.dom.minidom diff --git a/plc_devel_config.xml b/plc_devel_config.xml index 35feea3..7bb06ef 100644 --- a/plc_devel_config.xml +++ b/plc_devel_config.xml @@ -6,7 +6,7 @@ Default PLC build environment configuration file Mark Huang Copyright (C) 2006 The Trustees of Princeton University -$Id: plc_devel_config.xml,v 1.10 2007/08/24 07:19:27 mef Exp $ +$Id: plc_devel_config.xml 1078 2007-11-15 13:38:27Z thierry $ --> @@ -59,6 +59,7 @@ $Id: plc_devel_config.xml,v 1.10 2007/08/24 07:19:27 mef Exp $ + development-libs Development Libraries @@ -103,9 +104,6 @@ $Id: plc_devel_config.xml,v 1.10 2007/08/24 07:19:27 mef Exp $ Additional tools required to build PlanetLab software. - - findutils - @@ -127,17 +125,9 @@ $Id: plc_devel_config.xml,v 1.10 2007/08/24 07:19:27 mef Exp $ tetex-latex gcc-c++ - - ocaml - ocaml-ocamldoc - inotify-tools-devel - libpcap libpcap-devel - mysql - mysql-devel - mysql-server linuxdoc-tools @@ -161,14 +151,14 @@ $Id: plc_devel_config.xml,v 1.10 2007/08/24 07:19:27 mef Exp $ rsync + ghostscript - + docbook-utils-pdf postgresql-devel php-devel SOAPpy PyXML - expat-devel diff --git a/refresh-peer.py b/refresh-peer.py new file mode 100755 index 0000000..b1a7a3f --- /dev/null +++ b/refresh-peer.py @@ -0,0 +1,36 @@ +#!/usr/bin/env plcsh +# $Id: refresh-peer.py 154 2007-03-28 14:15:55Z thierry $ + +import sys,os,time + +def Run (peername): + timestring=time.strftime("%Y-%m-%d-%H-%M-%S") + print 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',peername + print 'RefreshPeer on %s - starting on %s'%(peername,timestring) + print 'xxxxxxxxxx' + sys.stdout.flush() + start=time.time() + result=RefreshPeer(peername) + finish=time.time() + + print 'Total duration',finish-start + print 'xxxxxxxxxx timers:' + keys=result.keys() + keys.sort() + for key in keys: + print key,result[key] + sys.stdout.flush() + sys.stderr.flush() + +def RunInLog (peername): + logname="/var/log/refresh-peer-%s.log"%(peername) + sys.stdout=open(logname,'a') + sys.stderr=sys.stdout + Run(peername) + sys.stderr.close() + sys.stdout.close() + +if __name__ == "__main__": + + for peername in sys.argv[1:]: + RunInLog (peername) -- 2.43.0