From 464898a6577bafdd22d212557f6341fe57349afb Mon Sep 17 00:00:00 2001 From: root Date: Fri, 13 May 2011 19:00:19 +0000 Subject: [PATCH] Several fixes to configuration, dependencies, and support tools. Added missing dependencies to .spec Added configurable dbuser, dbname, and dbhost >= myplc-5.0-18 Added external commands to fetch.py Added googlevis java script template --- Monitor.spec | 7 +- monitor-server.init | 13 +- monitor/wrapper/plccache.py | 3 +- tools/automate/fetch.py | 164 +++++++++++------- tools/getsshkeys.py | 2 +- tools/setup-agent.sh | 2 +- .../monitorweb/static/javascript/googlevis.js | 43 +++++ ...{monitorweb-httpd.conf => monitorweb.conf} | 0 8 files changed, 160 insertions(+), 74 deletions(-) create mode 100644 web/MonitorWeb/monitorweb/static/javascript/googlevis.js rename web/{monitorweb-httpd.conf => monitorweb.conf} (100%) diff --git a/Monitor.spec b/Monitor.spec index 68ed180..3bb70c4 100644 --- a/Monitor.spec +++ b/Monitor.spec @@ -96,10 +96,13 @@ Requires: openssh-clients Requires: perl-libwww-perl Requires: perl-IO-Socket-SSL Requires: MySQL-python +Requires: perl-DBD-mysql Requires: nmap +Requires: mailx Requires: nc Requires: rt3 Requires: traceroute +Requires: sendmail Requires: plewww-plekit Requires: pcucontrol @@ -173,7 +176,7 @@ install -D -m 644 nagios/monitor-nagios.cron $RPM_BUILD_ROOT/%{_sysconfdir}/cron install -D -m 644 monitor-server.cron $RPM_BUILD_ROOT/%{_sysconfdir}/cron.d/monitor-server.cron # apache configuration -install -D -m 644 web/monitorweb-httpd.conf $RPM_BUILD_ROOT/etc/httpd/conf.d/ +install -D -m 644 web/monitorweb.conf $RPM_BUILD_ROOT/etc/httpd/conf.d/ # we'll install monitor in site-packages install rest to # /usr/share/monitor @@ -283,6 +286,8 @@ export TMPDIR=/var/tmp/ #easy_install -UZ http://files.turbogears.org/eggs/TurboGears-1.0.7-py2.5.egg #easy_install -UZ http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.5.3.tar.gz #easy_install -UZ Elixir +chkconfig sendmail on +chkconfig crond on # crazy openssl libs for racadm binary ln -s /lib/libssl.so.0.9.8b /usr/lib/libssl.so.2 diff --git a/monitor-server.init b/monitor-server.init index 32d1209..544b47e 100644 --- a/monitor-server.init +++ b/monitor-server.init @@ -33,8 +33,8 @@ pghba_conf=$PGDATA/pg_hba.conf export PGPORT=$PLC_DB_PORT -MONITOR_DB_USER="monitoruser" -MONITOR_DB_NAME="monitor" +MONITOR_DB_USER=$PLC_MONITOR_DBUSER +MONITOR_DB_NAME=$PLC_MONITOR_DBNAME WROTE_PG_CONFIG= @@ -123,14 +123,14 @@ MONITOR_IP=${PLC_MONITOR_IP} PLC_WWW_HOSTNAME=${PLC_WWW_HOST} PLC_NAME=${PLC_NAME} -# used for debug mode -email=${PLC_MONITOR_CC_EMAIL} +email=${PLC_MONITOR_EMAIL} # all messages will appear to be from this address from_email=${PLC_MONITOR_EMAIL} # a separate address for support messages support_email=${PLC_MAIL_SUPPORT_ADDRESS} # mailing list copied on all out-going messages cc_email=${PLC_MONITOR_CC_EMAIL} +exception_email=${PLC_MONITOR_CC_EMAIL} # these are reserved values RT_DB_HOST=${PLC_RT_HOST} @@ -139,8 +139,7 @@ RT_DB_PASSWORD= RT_DB_NAME= [monitordatabase] -monitor_dburi=postgres://${MONITOR_DB_USER}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${MONITOR_DB_NAME} -zabbix_dburi=postgres://${ZABBIX_DB_USER}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${ZABBIX_DB_NAME} +monitor_dburi=postgres://${MONITOR_DB_USER}:${PLC_MONITOR_DBPASSWORD}@${PLC_MONITOR_DBHOST}:5432/${MONITOR_DB_NAME} cachetime=60 @@ -209,7 +208,7 @@ function start_tg_server () { stop_tg_server pushd ${MONITORPATH}/web/MonitorWeb/ - mkdir -p /var/log/monitor/monitorweb/ + mkdir -p /var/lib/monitor/monitorweb/ cp /var/log/monitorweb.log /var/lib/monitor/monitorweb/`date +%Y-%m-%d-%H-%M`-monitorweb.log ./start-monitorweb.py ${MONITORPATH}/web/MonitorWeb/prod.cfg &> /var/log/monitorweb.log & popd diff --git a/monitor/wrapper/plccache.py b/monitor/wrapper/plccache.py index 4778a7d..d818e97 100755 --- a/monitor/wrapper/plccache.py +++ b/monitor/wrapper/plccache.py @@ -163,7 +163,8 @@ def sync(): l_nodes = plc.api.GetNodes({'peer_id':None}, ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level', 'version', 'last_updated', 'date_created', 'key', - 'last_contact', 'pcu_ids', 'interface_ids']) + 'last_contact', 'pcu_ids', 'interface_ids', + 'last_boot', 'last_download', 'last_pcu_reboot', 'last_pcu_confirmation']) l_pcus = plc.api.GetPCUs() print >>sys.stderr, "sync sites" diff --git a/tools/automate/fetch.py b/tools/automate/fetch.py index 35cb9da..01fb399 100755 --- a/tools/automate/fetch.py +++ b/tools/automate/fetch.py @@ -10,72 +10,110 @@ from monitor import parser as parsermodule from monitor import common from automate import * +def build_vx_args_external(shell_cmd): + args = shell_cmd.split() + return args + +def vx_start_external(filelist,outdir,cmd, timeout=0, threadcount=20): + args = build_vx_args_external(cmd) + vxargs.start(None, threadcount, filelist, outdir, False, args, timeout) + def build_vx_args(shell_cmd): - ssh_options="-q -o UserKnownHostsFile=junkssh -o StrictHostKeyChecking=no" - cmd="""ssh %s root@{} """ % ssh_options - args = cmd.split() - args.append(shell_cmd) - return args + ssh_options="-q -o UserKnownHostsFile=junkssh -o StrictHostKeyChecking=no" + cmd="""ssh %s root@{} """ % ssh_options + args = cmd.split() + args.append(shell_cmd) + return args def vx_start(filelist,outdir,cmd, timeout=0): - args = build_vx_args(cmd) - vxargs.start(None, 20, filelist, outdir, False, args, timeout) + args = build_vx_args(cmd) + vxargs.start(None, 20, filelist, outdir, False, args, timeout) + +def get_hostlist_from_myops(filter): + ret = [] + curl_url = "curl -s 'http://myops.planet-lab.org:5984/query" + curl_params = "fields=hostname&skip_header&filter=%s'" % filter + curl_cmd = "%s?%s" % (curl_url, curl_params) + print curl_cmd + f = os.popen(curl_cmd, 'r') + for h in f.read().split(): + ret.append((h, '')) + return ret if __name__ == "__main__": - parser = parsermodule.getParser(['nodesets']) - parser.set_defaults(outdir=None, - timeout=0, - simple=False, - run=False, - cmdfile=None,) - - parser.add_option("", "--timeout", dest="timeout", metavar="seconds", - help="Number of seconds to wait before timing out on host.") - parser.add_option("", "--outdir", dest="outdir", metavar="dirname", - help="Name of directory to place output") - parser.add_option("", "--cmd", dest="cmdfile", metavar="filename", - help="Name of file that contains a unix-to-csv command " + \ - "to run on the hosts.") - - config = parsermodule.parse_args(parser) - - if config.outdir == None: - outdir="checkhosts" - else: - outdir=config.outdir - - if not os.path.exists(outdir): - os.system('mkdir -p %s' % outdir) - - #if config.site is not None or \ - # config.nodeselect is not None or \ - # config.nodegroup is not None: - # print "TODO: implement support for nodeselect and site queries." - # print "%s %s %s" % (config.site, config.nodeselect, config.nodegroup) - # sys.exit(1) - - nodelist = common.get_nodeset(config) - - if len(nodelist) > 0: - filelist = [ (x, '') for x in nodelist ] - elif os.path.exists(str(config.nodelist)) and os.path.isfile(config.nodelist): - filelist = vxargs.getListFromFile(open(config.nodelist,'r')) - elif os.path.exists(str(config.nodelist)) and os.path.isdir(config.nodelist): - filelist = get_hostlist_from_dir(config.nodelist) - elif config.node is not None: - filelist = [(config.node, '')] - else: - # probably no such file. - raise Exception("No such file %s" % config.nodelist) - - if config.cmdfile == None: - f = open("command.txt",'r') - cmd = f.read() - else: - f = open(config.cmdfile,'r') - cmd = f.read() - - print filelist - - vx_start(filelist, outdir, cmd, int(config.timeout)) + parser = parsermodule.getParser(['nodesets']) + parser.set_defaults(outdir=None, + timeout=0, + simple=False, + threadcount=20, + external=False, + myopsfilter=None, + run=False, + template=None, + cmdfile=None,) + + parser.add_option("", "--timeout", dest="timeout", metavar="seconds", + help="Number of seconds to wait before timing out on host.") + parser.add_option("", "--myopsfilter", dest="myopsfilter", metavar="", + help="filter string to pass directly to myops query") + parser.add_option("", "--outdir", dest="outdir", metavar="dirname", + help="Name of directory to place output") + parser.add_option("", "--cmd", dest="cmdfile", metavar="filename", + help="Name of file that contains a unix-to-csv command " + \ + "to run on the hosts.") + parser.add_option("", "--external", dest="external", action="store_true", + help="Run commands external to the server. The default is internal.") + parser.add_option("", "--template", dest="template", + help="Command template for external commands; substitutes {} with hostname.") + parser.add_option("", "--threadcount", dest="threadcount", + help="Number of simultaneous threads: default 20.") + + config = parsermodule.parse_args(parser) + + if config.outdir == None: + outdir="checkhosts" + else: + outdir=config.outdir + + if not os.path.exists(outdir): + os.system('mkdir -p %s' % outdir) + + #if config.site is not None or \ + # config.nodeselect is not None or \ + # config.nodegroup is not None: + # print "TODO: implement support for nodeselect and site queries." + # print "%s %s %s" % (config.site, config.nodeselect, config.nodegroup) + # sys.exit(1) + if config.myopsfilter is not None: + filelist = get_hostlist_from_myops(config.myopsfilter) + else: + nodelist = common.get_nodeset(config) + if len(nodelist) > 0: + filelist = [ (x, '') for x in nodelist ] + elif os.path.exists(str(config.nodelist)) and os.path.isfile(config.nodelist): + filelist = vxargs.getListFromFile(open(config.nodelist,'r')) + elif os.path.exists(str(config.nodelist)) and os.path.isdir(config.nodelist): + filelist = get_hostlist_from_dir(config.nodelist) + elif config.node is not None: + filelist = [(config.node, '')] + else: + # probably no such file. + raise Exception("No such file %s" % config.nodelist) + + if config.cmdfile == None and config.template == None: + f = open("command.txt",'r') + cmd = f.read() + elif config.template is not None and config.external: + cmd = config.template + else: + f = open(config.cmdfile,'r') + cmd = f.read() + + print filelist + + if config.external or config.template is not None: + vx_start_external(filelist, outdir, cmd, int(config.timeout), int(config.threadcount)) + else: + vx_start(filelist, outdir, cmd, int(config.timeout)) + diff --git a/tools/getsshkeys.py b/tools/getsshkeys.py index 0104deb..6d04ce2 100755 --- a/tools/getsshkeys.py +++ b/tools/getsshkeys.py @@ -4,7 +4,7 @@ import sys sys.path.append('.') sys.path.append('..') -from monitor.util.file import * +from monitor.util.sshknownhosts import * def main(hosts): k = SSHKnownHosts() diff --git a/tools/setup-agent.sh b/tools/setup-agent.sh index b78dcf0..9cae55b 100755 --- a/tools/setup-agent.sh +++ b/tools/setup-agent.sh @@ -2,7 +2,7 @@ # NOTE: Must be an absolute path to guarantee it is read. INSTALLPATH=/usr/share/monitor/ -$INSTALLPATH/shconfig.py > $INSTALLPATH/monitorconfig.sh +$INSTALLPATH/commands/shconfig.py > $INSTALLPATH/monitorconfig.sh source $INSTALLPATH/monitorconfig.sh cd ${MONITOR_SCRIPT_ROOT} set -e diff --git a/web/MonitorWeb/monitorweb/static/javascript/googlevis.js b/web/MonitorWeb/monitorweb/static/javascript/googlevis.js new file mode 100644 index 0000000..f9c98db --- /dev/null +++ b/web/MonitorWeb/monitorweb/static/javascript/googlevis.js @@ -0,0 +1,43 @@ +/* + * NOTE: requires the Google JSAPI using a key for your domain. + + */ +google.load('visualization', '1', {'packages':['annotatedtimeline']}); +google.load("jquery", "1"); + +function getData2() { + // To see the data that this visualization uses, browse to + var link = 'http://spreadsheets.google.com/tq?key=0AubvfWUvLDrpdGs2eGdwNndDUEZSV2hfNFprTkQxUkE&hl=en&authkey=CO3y2JoO&sheet=NodeHistory'; + var query = new google.visualization.Query(link); + + // Apply query language. + //console.log("run select"); + query.setQuery('SELECT A,B+E,I,J,G+H,K,L'); + // Send the query with a callback function. + query.send(handleQueryResponse); +} +function handleQueryResponse(response) { + if (response.isError()) { + alert('Error in query: ' + response.getMessage() + ' ' + response.getDetailedMessage()); + return; + } + var data = response.getDataTable(); + //console.log("loaded data"); + //console.log(data); + if ( data ) { + drawChart2(data); + } +} +function drawChart2(data) { + //data.insertColumn(2, "string", "title1"); + //data.insertColumn(3, "string", "text1"); + //data.insertColumn(5, "string", "title1"); + //data.insertColumn(6, "string", "text1"); + data.setColumnLabel(1, "Production Nodes"); + data.setColumnLabel(4, "Failing Nodes"); + console.log(data); + var chart = new google.visualization.AnnotatedTimeLine(document.getElementById('chart_div')); + var st = new Date(); + st.setDate(st.getDate()-60); + chart.draw(data, {displayAnnotations: true, zoomStartTime: st, zoomEndTime: new Date()}); +} diff --git a/web/monitorweb-httpd.conf b/web/monitorweb.conf similarity index 100% rename from web/monitorweb-httpd.conf rename to web/monitorweb.conf -- 2.43.0