From da62714c855a89f1b50d21844fa8f1e709850ebb Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Thu, 20 Nov 2008 21:58:02 +0000 Subject: [PATCH] setup both monitor and zabbix databases in /etc/plc.d/monitor tweak the infovacuum model to work with the new db. still much to do there. --- findbad.py | 18 ++++- findbadpcu.py | 4 +- monitor-server.init | 74 +++++++++++++------ monitor/database/infovacuum/__init__.py | 6 +- monitor/database/infovacuum/actionrecord.py | 56 +++++++------- .../{findbadrecord.py => findbad.py} | 0 nodequery.py | 3 +- 7 files changed, 100 insertions(+), 61 deletions(-) rename monitor/database/infovacuum/{findbadrecord.py => findbad.py} (100%) diff --git a/findbad.py b/findbad.py index 1e412bc..41cbc45 100755 --- a/findbad.py +++ b/findbad.py @@ -11,7 +11,10 @@ import threading from monitor import util from monitor.util import command from monitor import config -from monitor.database import FindbadNodeRecordSync, FindbadNodeRecord + +from monitor.database.infovacuum import FindbadNodeRecordSync, FindbadNodeRecord +from monitor.database.dborm import mon_session as session + from monitor.sources import comon from monitor.wrapper import plc, plccache @@ -277,6 +280,9 @@ def recordPingAndSSH(request, result): observed_category = values['category'], ) fbnodesync.round = global_round + fbnodesync.flush() + fbsync.flush() + fbrec.flush() count += 1 print "%d %s %s" % (count, nodename, values) @@ -300,8 +306,9 @@ def checkAndRecordState(l_nodes, cohash): # CREATE all the work requests for nodename in l_nodes: fbnodesync = FindbadNodeRecordSync.findby_or_create(hostname=nodename, if_new_set={'round':0}) - node_round = fbnodesync.round + fbnodesync.flush() + if node_round < global_round: # recreate node stats when refreshed #print "%s" % nodename @@ -333,6 +340,7 @@ def checkAndRecordState(l_nodes, cohash): print FindbadNodeRecordSync.query.count() print FindbadNodeRecord.query.count() + session.flush() def main(): global global_round @@ -346,14 +354,16 @@ def main(): global_round += 1 fbsync.round = global_round + fbsync.flush() + cotop = comon.Comon() # lastcotop measures whether cotop is actually running. this is a better # metric than sshstatus, or other values from CoMon cotop_url = COMON_COTOPURL # history information for all nodes - #cohash = {} - cohash = cotop.coget(cotop_url) + cohash = {} + #cohash = cotop.coget(cotop_url) l_nodes = plccache.l_nodes if config.nodelist: f_nodes = util.file.getListFromFile(config.nodelist) diff --git a/findbadpcu.py b/findbadpcu.py index 1af600c..8345637 100755 --- a/findbadpcu.py +++ b/findbadpcu.py @@ -15,7 +15,8 @@ import threading import monitor from monitor.pcu import reboot from monitor import config -from monitor.database import FindbadPCURecordSync, FindbadPCURecord +from monitor.database.infovacuum import FindbadPCURecordSync, FindbadPCURecord +from monitor.database.dborm import mon_session as session from monitor import util from monitor.wrapper import plc, plccache from nodequery import pcu_select @@ -340,6 +341,7 @@ def checkAndRecordState(l_pcus, cohash): print FindbadPCURecordSync.query.count() print FindbadPCURecord.query.count() + session.flush() def main(): diff --git a/monitor-server.init b/monitor-server.init index f8e5a02..30cd6e2 100644 --- a/monitor-server.init +++ b/monitor-server.init @@ -33,34 +33,46 @@ export PGPORT=$PLC_DB_PORT ZABBIX_DB_USER="zabbixuser" ZABBIX_DB_NAME="zabbix" +MONITOR_DB_USER="monitoruser" +MONITOR_DB_NAME="monitor" + +WROTE_PG_CONFIG= + +if [ -z "$PLC_MONITOR_IP" ] ; then + PLC_MONITOR_IP=$( gethostbyname $PLC_MONITOR_HOST ) +fi + function check_pg_hba () { + NAME=$1 + USER=$2 #### SETUP ACCESS to this user and database mkdir -p $PGDATA/pg_hba.conf.d - ZABCONF=$PGDATA/pg_hba.conf.d/zabbix.conf - if [ ! -f $ZABCONF ] ; then - echo "host $ZABBIX_DB_NAME $ZABBIX_DB_USER 127.0.0.1/32 password" > $ZABCONF - echo "host $ZABBIX_DB_NAME $ZABBIX_DB_USER $PLC_MONITOR_IP/32 password" >> $ZABCONF + CONF=$PGDATA/pg_hba.conf.d/${NAME}.conf + if [ ! -f $CONF ] ; then + echo "host $NAME $USER 127.0.0.1/32 password" > $CONF + echo "host $NAME $USER $PLC_MONITOR_IP/32 password" >> $CONF - # NOTE: restart db to enable access by users granted above. - service plc restart posgresql + WROTE_PG_CONFIG="true" fi } function check_user_and_db() { CREATED= + NAME=$1 + USER=$2 # confirm user is present or create it - user_present=$( psql -U postgres -c "select * from pg_user;" -d template1 | grep $ZABBIX_DB_USER ) + user_present=$( psql -U postgres -c "select * from pg_user;" -d template1 | grep $USER ) if [ -z $user_present ] ; then - createuser --no-superuser --no-createdb --no-createrole --login --unencrypted --echo $ZABBIX_DB_USER -U postgres + createuser --no-superuser --no-createdb --no-createrole --login --unencrypted --echo $USER -U postgres CREATED="true" fi # confirm database is present or create it - db_present=$( psql -U postgres -c "select * from pg_database;" -d template1 | grep $ZABBIX_DB_NAME ) + db_present=$( psql -U postgres -c "select * from pg_database;" -d template1 | grep $NAME ) if [ -z $db_present ] ; then - createdb --owner=$ZABBIX_DB_USER $ZABBIX_DB_NAME -U postgres + createdb --owner=$USER $NAME -U postgres CREATED="true" fi @@ -73,7 +85,7 @@ function check_user_and_db() CREATED="true" fi if [ -n "$CREATED" ] ; then - psql -d template1 -U postgres -c "ALTER USER $ZABBIX_DB_USER WITH PASSWORD '$PLC_MONITOR_DBPASSWORD';" + psql -d template1 -U postgres -c "ALTER USER $USER WITH PASSWORD '$PLC_MONITOR_DBPASSWORD';" fi } @@ -84,8 +96,13 @@ function if_present_load () psql -d $ZABBIX_DB_NAME -U $ZABBIX_DB_USER < $file fi } +function check_monitor_schema_and_data() +{ + # NOTE: call create_all() to setup the database from the infovacuum model. + python -c "from monitor.database.infovacuum import *; from elixir import create_all; create_all()" +} -function check_schema_and_data() +function check_zabbix_schema_and_data() { schema_present=$( psql -U $ZABBIX_DB_USER $ZABBIX_DB_NAME -c "\d;" < /dev/null | grep hosts ) if [ -z $schema_present ] ; then @@ -97,7 +114,7 @@ function check_schema_and_data() fi } -function check_templates_and_import () +function check_zabbix_templates_and_import () { # LOG IN COOKIE_FILE=/tmp/cookiejar.txt @@ -141,9 +158,6 @@ function check_monitor_conf () { MONITOR_CONFIG=/etc/monitor.conf - if [ -z "$PLC_MONITOR_IP" ] ; then - PLC_MONITOR_IP=$( gethostbyname $PLC_MONITOR_HOST ) - fi # Using plcsh add default, monitor user plcsh </dev/null @@ -195,12 +209,15 @@ support_email=${PLC_MAIL_SUPPORT_ADDRESS} cc_email= [monitordatabase] -monitor_dburi=postgres://monitoruser:${PLC_MONITOR_DBPASSWORD}@localhost:5432/monitor -zabbix_dburi=postgres://zabbixuser:${PLC_MONITOR_DBPASSWORD}@localhost:5432/zabbix +monitor_dburi=postgres://${MONITOR_DB_NAME}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${MONITOR_DB_NAME} +zabbix_dburi=postgres://${ZABBIX_DB_USER}:${PLC_MONITOR_DBPASSWORD}@localhost:5432/${ZABBIX_DB_NAME} cachetime=60 +# Evaluated as true or false [commandline] +cachecalls=True + echo=False debug=False mail=True @@ -280,10 +297,20 @@ case "$1" in dialog "$MESSAGE" # DATABASE acces, creation, and data loading - check_pg_hba - check_user_and_db - check_schema_and_data - check_templates_and_import + check_pg_hba $MONITOR_DB_NAME $MONITOR_DB_USER + check_user_and_db $MONITOR_DB_NAME $MONITOR_DB_USER + check_monitor_schema_and_data + + check_pg_hba $ZABBIX_DB_NAME $ZABBIX_DB_USER + check_user_and_db $ZABBIX_DB_NAME $ZABBIX_DB_USER + + if [ -n "$WROTE_PG_CONFIG" ] ; then + # NOTE: restart db to enable access by users granted above. + service plc restart postgresql + fi + + check_zabbix_schema_and_data + check_zabbix_templates_and_import # WRITE default /etc/monitor.conf check_monitor_conf @@ -293,7 +320,6 @@ case "$1" in check_zab_agentd check_zab_webconfig - result "$MESSAGE" ;; @@ -306,7 +332,7 @@ case "$1" in $MONITORPATH/zabbix/zabbixsync.py --setupglobal &> /var/log/monitor-server # import any templates - check_templates_and_import + check_zabbix_templates_and_import service plc start monitor diff --git a/monitor/database/infovacuum/__init__.py b/monitor/database/infovacuum/__init__.py index 9642d30..c726776 100644 --- a/monitor/database/infovacuum/__init__.py +++ b/monitor/database/infovacuum/__init__.py @@ -40,7 +40,7 @@ def findby_or_create(cls, if_new_set={}, **kwargs): return result Entity.findby_or_create = classmethod(findby_or_create) -from monitor.database.infovacuum.model.actionrecord import * -from monitor.database.infovacuum.model.findbadrecord import * -from monitor.database.infovacuum.model.historyrecord import * +from monitor.database.infovacuum.actionrecord import * +from monitor.database.infovacuum.findbad import * +from monitor.database.infovacuum.historyrecord import * setup_all() diff --git a/monitor/database/infovacuum/actionrecord.py b/monitor/database/infovacuum/actionrecord.py index 968403f..2569e35 100644 --- a/monitor/database/infovacuum/actionrecord.py +++ b/monitor/database/infovacuum/actionrecord.py @@ -9,34 +9,34 @@ from monitor.database.dborm import mon_metadata, mon_session __metadata__ = mon_metadata __session__ = mon_session -class IssueType(Entity): - shortname = Field(String, default=None) - description = Field(String, default=None) - issue_record = ManyToMany('IssueRecord') +#class IssueType(Entity): +# shortname = Field(String, default=None) +# description = Field(String, default=None) +# issue_record = ManyToMany('IssueRecord') -class IssueRecord(Entity): - date_created = Field(DateTime,default=datetime.now) - date_last_updated = Field(DateTime,default=datetime.now) - date_action_taken = Field(DateTime,default=datetime.now) - - hostname = Field(String,default=None) - loginbase = Field(String) - - ticket_id = Field(Integer, default=0) - rt = Field(PickleType, default=None) - - # open, paused, closed - status = Field(String, default="open") - - take_action = Field(Boolean, default=False) - send_email = Field(Boolean, default=True) - - message_series = Field(String, default="nodedown") - message_index = Field(Integer, default=0) - penalty_level = Field(Integer, default=0) - - issue_type = ManyToMany('IssueType') - actions = OneToMany('ActionRecord', order_by='-date_created') +#class IssueRecord(Entity): +# date_created = Field(DateTime,default=datetime.now) +# date_last_updated = Field(DateTime,default=datetime.now) +# date_action_taken = Field(DateTime,default=datetime.now) +# +# hostname = Field(String,default=None) +# loginbase = Field(String) +# +# ticket_id = Field(Integer, default=0) +# rt = Field(PickleType, default=None) +# +# # open, paused, closed +# status = Field(String, default="open") +# +# take_action = Field(Boolean, default=False) +# send_email = Field(Boolean, default=True) +# +# message_series = Field(String, default="nodedown") +# message_index = Field(Integer, default=0) +# penalty_level = Field(Integer, default=0) +# +# issue_type = ManyToMany('IssueType') +# actions = OneToMany('ActionRecord', order_by='-date_created') class ActionRecord(Entity): @@ -50,7 +50,7 @@ class ActionRecord(Entity): hostname = Field(String,default=None) loginbase = Field(String) - issue = ManyToOne('IssueRecord') + #issue = ManyToOne('IssueRecord') # NOTE: this is the parent relation to fb records. first create the # action record, then append to this value all of the findbad records we # want to have in our set. diff --git a/monitor/database/infovacuum/findbadrecord.py b/monitor/database/infovacuum/findbad.py similarity index 100% rename from monitor/database/infovacuum/findbadrecord.py rename to monitor/database/infovacuum/findbad.py diff --git a/nodequery.py b/nodequery.py index 71c62bc..5287e48 100755 --- a/nodequery.py +++ b/nodequery.py @@ -17,7 +17,8 @@ from monitor.pcu import reboot from monitor.wrapper import plc, plccache api = plc.getAuthAPI() -from monitor.database import FindbadNodeRecord, FindbadPCURecord +from monitor.database.infovacuum import FindbadNodeRecordSync, FindbadNodeRecord +from monitor.database.dborm import mon_session as session from monitor import util from monitor import config -- 2.43.0