From 8e90984231a680415cc3f6a60f008b3029ad4548 Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Sat, 22 Nov 2008 00:43:36 +0000 Subject: [PATCH] completed updates to the info model. various files updated with new paths and import strings. --- findbad.py | 3 +- findbadpcu.py | 13 +++++-- monitor-server.cron | 2 +- monitor-server.init | 5 +-- monitor/database/dborm.py | 3 -- monitor/database/info/__init__.py | 3 +- monitor/database/info/findbad.py | 5 +-- monitor/database/info/model.py | 1 + monitor/wrapper/plc.py | 10 ++++-- nodequery.py | 3 +- prep_power_users.py | 60 ++++++++++++++++++++----------- setup.py | 4 +-- zabbix/zabbixsite.py | 17 +++++++++ 13 files changed, 89 insertions(+), 40 deletions(-) diff --git a/findbad.py b/findbad.py index 41cbc45..77dd120 100755 --- a/findbad.py +++ b/findbad.py @@ -12,8 +12,7 @@ from monitor import util from monitor.util import command from monitor import config -from monitor.database.infovacuum import FindbadNodeRecordSync, FindbadNodeRecord -from monitor.database.dborm import mon_session as session +from monitor.database.info.model import FindbadNodeRecordSync, FindbadNodeRecord, session from monitor.sources import comon from monitor.wrapper import plc, plccache diff --git a/findbadpcu.py b/findbadpcu.py index 8345637..f7a3576 100755 --- a/findbadpcu.py +++ b/findbadpcu.py @@ -15,8 +15,7 @@ import threading import monitor from monitor.pcu import reboot from monitor import config -from monitor.database.infovacuum import FindbadPCURecordSync, FindbadPCURecord -from monitor.database.dborm import mon_session as session +from monitor.database.info.model import FindbadPCURecordSync, FindbadPCURecord, session from monitor import util from monitor.wrapper import plc, plccache from nodequery import pcu_select @@ -275,7 +274,7 @@ def recordPingAndSSH(request, result): fbrec = FindbadPCURecord( date_checked=datetime.fromtimestamp(values['date_checked']), - record=fbsync.round, + round=fbsync.round, plc_pcuid=pcu_id, plc_pcu_stats=values['plc_pcu_stats'], dns_status=values['dnsmatch'], @@ -284,6 +283,11 @@ def recordPingAndSSH(request, result): reboot_trial_status="%s" % values['reboot'], ) fbnodesync.round = global_round + + fbnodesync.flush() + fbsync.flush() + fbrec.flush() + count += 1 print "%d %s %s" % (count, nodename, values) @@ -309,6 +313,7 @@ def checkAndRecordState(l_pcus, cohash): for pcuname in l_pcus: pcu_id = int(pcuname) fbnodesync = FindbadPCURecordSync.findby_or_create(plc_pcuid=pcu_id, if_new_set={'round' : 0}) + fbnodesync.flush() node_round = fbnodesync.round if node_round < global_round: @@ -360,6 +365,8 @@ def main(): global_round += 1 fbsync.round = global_round + fbsync.flush() + if config.site is not None: api = plc.getAuthAPI() site = api.GetSites(config.site) diff --git a/monitor-server.cron b/monitor-server.cron index 86e829c..f6062d6 100644 --- a/monitor-server.cron +++ b/monitor-server.cron @@ -2,5 +2,5 @@ # automated actions for debug nodes. 01 * * * * root /usr/share/monitor/automate.sh 2>&1 > /usr/share/monitor/monitor.log - +01 * * * * root /usr/share/monitor/zabbix/zabbixsync.py 2>&1 > /usr/share/monitor/zabbixsync.log diff --git a/monitor-server.init b/monitor-server.init index 30cd6e2..dc984ad 100644 --- a/monitor-server.init +++ b/monitor-server.init @@ -98,8 +98,8 @@ function if_present_load () } function check_monitor_schema_and_data() { - # NOTE: call create_all() to setup the database from the infovacuum model. - python -c "from monitor.database.infovacuum import *; from elixir import create_all; create_all()" + # NOTE: call create_all() to setup the database from the info model. + python -c "from monitor.database.info.model import *; from elixir import create_all; create_all()" } function check_zabbix_schema_and_data() @@ -241,6 +241,7 @@ function check_zab_server () -e "s#DBName=.*#DBName=$ZABBIX_DB_NAME#g" \ -e "s#DBUser=.*#DBUser=$ZABBIX_DB_USER#g" \ -e "s#DBPassword=.*#DBPassword=$PLC_MONITOR_DBPASSWORD#g" \ + -e "s/.*ExternalScripts=.*/ExternalScripts=${MONITORPATH}\/zabbix/g" \ ${ZABBIXCFG}/zabbix_server.conf > $TMP_FILE cat $TMP_FILE > ${ZABBIXCFG}/zabbix_server.conf fi diff --git a/monitor/database/dborm.py b/monitor/database/dborm.py index 76cbe63..5e54f11 100644 --- a/monitor/database/dborm.py +++ b/monitor/database/dborm.py @@ -9,6 +9,3 @@ mon_session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflus zab_metadata = sqlalchemy.MetaData() zab_metadata.bind = sqlalchemy.create_engine(config.zabbix_dburi, echo=config.echo) zab_session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=False,autocommit=True)) - -#from monitor.database.infovacuum.model import * -#from monitor.database.zabbixapi.model import * diff --git a/monitor/database/info/__init__.py b/monitor/database/info/__init__.py index 8fabb45..9c3df82 100644 --- a/monitor/database/info/__init__.py +++ b/monitor/database/info/__init__.py @@ -1,4 +1,3 @@ -#__connection__ = hub = PackageHub('infovacuum') try: import pkg_resources pkg_resources.require("SQLAlchemy>=0.4.9") @@ -27,6 +26,8 @@ from elixir import Entity, Field, OneToMany, ManyToOne, ManyToMany from elixir import options_defaults, using_options, setup_all from elixir import String, Unicode, Integer, DateTime options_defaults['autosetup'] = False +# NOTE: due to the module path length, this is required. +options_defaults['shortnames'] = True from monitor.database.dborm import mon_metadata, mon_session __metadata__ = mon_metadata diff --git a/monitor/database/info/findbad.py b/monitor/database/info/findbad.py index f147a92..da7775a 100644 --- a/monitor/database/info/findbad.py +++ b/monitor/database/info/findbad.py @@ -79,12 +79,13 @@ class FindbadNodeRecord(Entity): class FindbadPCURecord(Entity): @classmethod def get_all_latest(cls): - fbsync = cls.get_by(hostname="global") + fbsync = FindbadPCURecordSync.get_by(plc_pcuid=0) + print "round: ", fbsync.round return cls.query.filter_by(round=fbsync.round) @classmethod def get_latest_by(cls, **kwargs): - fbsync = cls.get_by(hostname="global") + fbsync = FindbadPCURecordSync.get_by(plc_pcuid=0) kwargs['round'] = fbsync.round return cls.query.filter_by(**kwargs) # ACCOUNTING diff --git a/monitor/database/info/model.py b/monitor/database/info/model.py index 5dc92a6..151f428 100644 --- a/monitor/database/info/model.py +++ b/monitor/database/info/model.py @@ -1,3 +1,4 @@ from monitor.database.info.action import * from monitor.database.info.findbad import * from monitor.database.info.history import * +from monitor.database.dborm import mon_session as session diff --git a/monitor/wrapper/plc.py b/monitor/wrapper/plc.py index 76db7d8..2ddb061 100644 --- a/monitor/wrapper/plc.py +++ b/monitor/wrapper/plc.py @@ -22,8 +22,14 @@ except: logger = logging.getLogger("monitor") class Auth: - def __init__(self): - self.auth = {'AuthMethod': "anonymous"} + def __init__(self, username=None, password=None): + if username==None and password==None: + self.auth = {'AuthMethod': "anonymous"} + else: + self.auth = {'Username' : username, + 'AuthMethod' : 'password', + 'AuthString' : password} + # NOTE: this host is used by default when there are no auth files. XMLRPC_SERVER="https://boot.planet-lab.org/PLCAPI/" diff --git a/nodequery.py b/nodequery.py index 5287e48..ff78b58 100755 --- a/nodequery.py +++ b/nodequery.py @@ -17,8 +17,7 @@ from monitor.pcu import reboot from monitor.wrapper import plc, plccache api = plc.getAuthAPI() -from monitor.database.infovacuum import FindbadNodeRecordSync, FindbadNodeRecord -from monitor.database.dborm import mon_session as session +from monitor.database.info.model import FindbadNodeRecordSync, FindbadNodeRecord, session from monitor import util from monitor import config diff --git a/prep_power_users.py b/prep_power_users.py index a2c5047..8e0219e 100755 --- a/prep_power_users.py +++ b/prep_power_users.py @@ -1,10 +1,10 @@ #!/usr/bin/python -import plc +from monitor.wrapper import plc import sys import time -email_list = [ 'monitor@planet-lab.org', +email_list = [ 'soltesz@cs.princeton.edu', # 'soltesz@cs.princeton.edu', # 'justin@cs.arizona.edu', # 'bakers@cs.arizona.edu', @@ -15,21 +15,20 @@ email_list = [ 'monitor@planet-lab.org', # 'vivek@cs.princeton.edu', ] -import config -api = plc.PLC(config.API_AUTH, config.API_SERVER) -api06 = plc.PLC(config.API_AUTH06, config.API_SERVER06) - +from monitor import config +api = plc.PLC(plc.Auth(config.API_AUTH_USER, config.API_AUTH_PASSWORD).auth, config.API_SERVER) +api06 = plc.PLC(plc.Auth(config.API06_AUTH_USER, config.API06_AUTH_PASSWORD).auth, config.API06_SERVER) # add planetlab-15.cs.princeton.edu, and use the key on the CD. -id = api06.AddNode(1, {'boot_state': 'rins', 'model': 'Dell Optiplex', - 'hostname' : 'planetlab-15.cs.princeton.edu', - 'version' : '3.3'}) -api06.AddNodeNetwork(id, {'ip': '128.112.139.39', - 'type' : 'ipv4', - 'is_primary' : True, - 'method' : 'dhcp', }) -api06.UpdateNode(id, {'key': "wptNagk8SgRxzN1lXfKMAjUYhQbOBymKnKg9Uv0LwGM"}) - +#id = api06.AddNode(1, {'boot_state': 'rins', 'model': 'Dell Optiplex', +# 'hostname' : 'planetlab-15.cs.princeton.edu', +# 'version' : '3.3'}) +#api06.AddNodeNetwork(id, {'ip': '128.112.139.39', +# 'type' : 'ipv4', +## 'is_primary' : True, +# 'method' : 'dhcp', }) +#api06.UpdateNode(id, {'key': "wptNagk8SgRxzN1lXfKMAjUYhQbOBymKnKg9Uv0LwGM"}) +# #print "adding vsys attributes" #api06.AddSliceAttribute('princeton_slicestat', 'vsys', 'pl-ps') @@ -77,6 +76,13 @@ def slice_exists(slice): except: return False +def node_exists(node): + x = api06.GetNodes({'hostname':node['hostname']}) + if len(x) == 0: + return False + else: + return True + # Renew slices slices = api06.GetSlices() for slice in slices: @@ -114,11 +120,25 @@ for email in email_list: sites = api.GetSites(user['site_ids']) print "Adding sites:", for site in sites: - if not site_exists(site): - print "%s" % site['login_base'], - api06.AddSite(site) - api06.AddPersonToSite(user['email'], site['login_base']) - sys.stdout.flush() + nodes = api.GetNodes(site['node_ids']) + + if not site_exists(site): + print "%s" % site['login_base'] + api06.AddSite(site) + api06.AddPersonToSite(user['email'], site['login_base']) + sys.stdout.flush() + + for node in nodes: + if not node_exists(node): + id = api06.AddNode(site['login_base'], {'boot_state': node['boot_state'], + 'model': node['model'], + 'hostname' : node['hostname'], + 'version' : node['version']}) + nnets = api.GetNodeNetworks(node['nodenetwork_ids']) + for nnet in nnets: + del nnet['nodenetwork_id'] + del nnet['nodenetwork_setting_ids'] + api06.AddNodeNetwork(id, nnet) print "" nodes = api06.GetNodes() diff --git a/setup.py b/setup.py index 0e4f001..a682b2d 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ from distutils.core import setup #from setuptools import setup, find_packages packages=['monitor', 'monitor.database', 'monitor.database.zabbixapi', - 'monitor.database.infovacuum', 'monitor.pcu', - 'monitor.sources', 'monitor.util', 'monitor.wrapper' ] + 'monitor.database.info', 'monitor.pcu', 'monitor.sources', + 'monitor.util', 'monitor.wrapper' ] #packages = find_packages(exclude=('Rpyc', 'www', 'ssh', 'pyssh', #'Rpyc.Demo', 'Rpyc.Servers', 'www.HyperText')) diff --git a/zabbix/zabbixsite.py b/zabbix/zabbixsite.py index 241e739..e3ddc21 100755 --- a/zabbix/zabbixsite.py +++ b/zabbix/zabbixsite.py @@ -122,6 +122,23 @@ def setup_global(): }) return +def merge_iplist(iplist): + # TODO: rewrite addresses as x.x.x.y-z rather than x.x.x.y,x.x.x.z if y-z==1 + ips = iplist.split(',') + ips.sort() + prev=None + newlist="" + for ip in ips: + fields = ip.split('.') + first = ".".join(fields[:2]) + last = int(fields[3]) + if prev: + if last - prev == 1: + pass + prev=last + newlist += "%s," + return newlist[:-1] + def setup_site(loginbase, techemail, piemail, iplist): # TODO: send a message when host is discovered. -- 2.43.0