added cacheset to parser to allow for plccache refreshing.
removed ' from emailZabbix.py due to runtime errors with zabbix_server
# an API server or so on.
# TODO: create real monitorconfig.py from monitorconfig-default.py
# TODO: create monitorconfig.php using phpconfig.py
-# TODO: create symlink in /var/lib/monitor-server for chroot environments
+# TODO: create symlink in /var/lib/monitor for chroot environments
# TODO: update the content of automate_pl03.sh
# TODO: Use the installed version of bootcd to create custom boot images. ( or, use the api now).
#!/bin/bash
# NOTE: Must be an absolute path to guarantee it is read.
-INSTALLPATH=/usr/share/monitor-server/
+INSTALLPATH=/usr/share/monitor/
# Generate an 'sh' style file full of variables in monitor.conf
$INSTALLPATH/shconfig.py > $INSTALLPATH/monitorconfig.sh
source $INSTALLPATH/monitorconfig.sh
# SERVER PATHS
MONITOR_HOSTNAME=monitor.planet-lab.org
-MONITOR_SCRIPT_ROOT=/usr/share/monitor-server
-MONITOR_DATA_ROOT=/var/lib/monitor-server
-MONITOR_ARCHIVE_ROOT=/var/lib/monitor-server/archive-pdb
+MONITOR_SCRIPT_ROOT=/usr/share/monitor
+MONITOR_DATA_ROOT=/var/lib/monitor
+MONITOR_ARCHIVE_ROOT=/var/lib/monitor/archive-pdb
MONITOR_IP=
MONITOR_HOSTNAME=
# Runs every three hours to poll all nodes and PCUs, as well as take some
# automated actions for debug nodes.
-01 * * * * root /usr/share/monitor-server/automate.sh 2>&1 > /usr/share/monitor-server/monitor.log
+01 * * * * root /usr/share/monitor/automate.sh 2>&1 > /usr/share/monitor/monitor.log
. /etc/planetlab/plc_config
local_config=/etc/planetlab/configs/site.xml
-MONITORPATH=/usr/share/monitor-server
+MONITORPATH=/usr/share/monitor
# Be verbose
set -x
{
# LOG IN
COOKIE_FILE=/tmp/cookiejar.txt
- TEMPLATES_DIR=/usr/share/monitor/zabbix/templates
+ rm -f ${COOKIE_FILE}
+ TEMPLATES_DIR=${MONITORPATH}/zabbix/templates
curl -s --cookie $COOKIE_FILE --cookie-jar $COOKIE_FILE \
--form "enter=Enter" \
--form "name=Admin" \
API_AUTH_PASSWORD=${PLC_MONITOR_DBPASSWORD}
# SERVER PATHS
-MONITOR_SCRIPT_ROOT=/usr/share/monitor-server
-MONITOR_DATA_ROOT=/var/lib/monitor-server
-MONITOR_ARCHIVE_ROOT=/var/lib/monitor-server/archive-pdb
+MONITOR_SCRIPT_ROOT=${MONITORPATH}
+MONITOR_DATA_ROOT=/var/lib/monitor
+MONITOR_ARCHIVE_ROOT=/var/lib/monitor/archive-pdb
MONITOR_HOSTNAME=${PLC_MONITOR_HOST}
MONITOR_IP=${PLC_MONITOR_IP}
return true or false based on whether the modified time of the cached
file is within 'length' minutes.
"""
+ if hasattr(config, 'cachecalls') and not config.cachecalls:
+ # don't use cached calls if cachecalls is false
+ return False
+
try:
t = lastModified(name, type)
except:
#print getattr(cls,f)
return
+ # NOTE: using the literal --> ' <-- character in messages will cause the db or zabbix_server to fail.
+
nodedown_one_subject="Server {HOSTNAME} is unreachable: First Notice"
nodedown_one = """
Hello,
-We hope that you're having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
+We hope that you are having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
{HOSTNAME} : Since {EVENT.AGE}
-We're writing because we need your help returning them to their regular operation.
+We are writing because we need your help returning them to their regular operation.
To help, please confirm that a verison 3.0 or greater BootCD is installed in the machine. Then, after checking that the node is properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we are seeing. Once the machine has come back up, please visit the monitor status page to verify that your node is accessible.
sudo /usr/sbin/vps ax
-If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contact's Guide:
+If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contacts Guide:
https://www.planet-lab.org/doc/guides/bootcdsetup
-There is no need to respond to this message unless there are any console messages relating to the node's failure. In this case, please report them to PlanetLab support (%(support_email)s) so we can help resolve the issue. Including this message in your reply will help us coordinate our records with the actions you've taken.
+There is no need to respond to this message unless there are any console messages relating to the node failure. In this case, please report them to PlanetLab support (%(support_email)s) so we can help resolve the issue. Including this message in your reply will help us coordinate our records with the actions you have taken.
Finally, you can track the current status of your machines using this Google Gadget:
nodedown_two = """
Hello,
-We hope that you're having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
+We hope that you are having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
{HOSTNAME} : Since {EVENT.AGE}
-We're writing again because our previous correspondence, sent only to the registered Technical Contact, has gone unacknowledged for at least a week, and we need your help returning these machines to their regular operation. We understand that machine maintenance can take time. So, while we wait for the machines to return to their regular operation slice creation has been suspended at your site. No new slices may be created, but the existing slices and services running within them will be unaffected.
+We are writing again because our previous correspondence, sent only to the registered Technical Contact, has gone unacknowledged for at least a week, and we need your help returning these machines to their regular operation. We understand that machine maintenance can take time. So, while we wait for the machines to return to their regular operation slice creation has been suspended at your site. No new slices may be created, but the existing slices and services running within them will be unaffected.
To help, please confirm that a verison 3.0 or greater BootCD is installed in the machine. Then, after checking that the node is properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we are seeing. Once the machine has come back up, please visit the monitor status page to verify that your node is accessible.
If the machine has booted successfully, you may check directly by logging in with your site_admin account, and running:
sudo /usr/sbin/vps ax
-If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contact's Guide:
+If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contacts Guide:
https://www.planet-lab.org/doc/guides/bootcdsetup
nodedown_three ="""
Hello,
-We hope that you're having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
+We hope that you are having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
{HOSTNAME} : Since {EVENT.AGE}
-We understand that machine maintenance can take time. We're writing again because our previous correspondences, sent first to the registered Technical Contact then the the Site PI, have gone unacknowledged for at least two weeks, and we need your help returning these machines to their regular operation. This is the third time attempting to contact someone in regard to these machines at your site. So, while we wait for the machines to return to their regular operation all current slice activity will be suspended. Current experiments will be stopped and will not be be able to start again until there is evidence that you have begun to help with the maintenance of these machines.
+We understand that machine maintenance can take time. We are writing again because our previous correspondences, sent first to the registered Technical Contact then the the Site PI, have gone unacknowledged for at least two weeks, and we need your help returning these machines to their regular operation. This is the third time attempting to contact someone in regard to these machines at your site. So, while we wait for the machines to return to their regular operation all current slice activity will be suspended. Current experiments will be stopped and will not be be able to start again until there is evidence that you have begun to help with the maintenance of these machines.
To help, please confirm that a verison 3.0 or greater BootCD is installed in the machine. Then, after checking that the node is properly networked, power cycle the machine. Note that rebooting the machine may not fully resolve the problems we are seeing. Once the machine has come back up, please visit the monitor status page to verify that your node is accessible.
sudo /usr/sbin/vps ax
-If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contact's Guide:
+If you have a BootCD older than 3.0, you will need to create a new BootImage on CD or USB. You can find instructions for this at the Technical Contacts Guide:
https://www.planet-lab.org/doc/guides/bootcdsetup
nodedown_four="""
Hello,
-We hope that you're having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
+We hope that you are having a good day. As part of PlanetLab node monitoring, we noticed the following node is down at your site:
{HOSTNAME} : Since {EVENT.AGE}
"debug is enabled or for bcc when it is not")
return parser
+def parseSetCacheSet(parser=None):
+ if parser == None:
+ parser = OptionParser()
+
+ parser.set_defaults(cachecalls=True, cachetime=60)
+ parser.add_option("", "--nocache", dest="cachecalls", action="store_false",
+ help="When using PLCCache objects, temporarily disable the use of caching. i.e. refresh")
+ parser.add_option("", "--cachetime", dest="cachetime",
+ help="How long to preserve a cached value. Minutes")
+ return parser
+
def parseSetNodeSets(parser=None):
if parser == None:
parser = OptionParser()
else:
p = parser
+ if 'cacheset' in parsesets:
+ p = parseSetCacheSet(p)
+
if 'nodesets' in parsesets:
p = parseSetNodeSets(p)
if 'defaults' in parsesets:
<application>Processes</application>
</applications>
</item>
- <item type="0" key="proc.num[httpd]" value_type="3">
+ <item type="0" key="proc.num[plc_httpd]" value_type="3">
<description>Number of running processes apache</description>
<ipmi_sensor></ipmi_sensor>
<delay>60</delay>
<trigger>
<description>Apache is not running on {HOSTNAME}</description>
<type>0</type>
- <expression>{{HOSTNAME}:proc.num[httpd].last(0)}<1</expression>
+ <expression>{{HOSTNAME}:proc.num[plc_httpd].last(0)}<1</expression>
<url></url>
<status>0</status>
<priority>3</priority>
print "checking zabbix server host info"
zabbixserver = Host.get_by(host="ZABBIX Server")
if zabbixserver:
+ # TODO: verify that this works. it has failed once on fresh
+ # install... not sure why.
print "Removing default Zabbix server entry"
zabbixserver.delete()
# copying that the php code does during a host add.
# NOTE: Instead, reformat any *xml.in templates and import those
# during /etc/plc.d/monitor sync
- for file in glob.glob("/usr/share/monitor/templates/*.xml.in"):
+ for file in glob.glob("%s/zabbix/templates/*.xml.in" config.MONITOR_SCRIPT_ROOT):
if 'zabbix_server' in file:
buf = loadFile(file)
args = {'hostname' : config.MONITOR_HOSTNAME, 'ip' : config.MONITOR_IP}
print "checking scripts"
script1 = Script.find_or_create(name="RebootNode",
set_if_new = {
- 'command':"/usr/share/monitor-server/reboot.py {HOST.CONN}",
+ 'command':"%s/reboot.py {HOST.CONN}" % config.MONITOR_SCRIPT_ROOT,
'host_access':3 # r/w)
})
script2 = Script.find_or_create(name="NMap",
# TODO: remove old users that are no longer in the plcdb.
# TODO: consider creating two user groups for Tech & PI emails
+ # NOTE: setup default valus for EMAIL
+ mailtxt.reformat({'hostname' : config.MONITOR_HOSTNAME,
+ 'support_email' : config.support_email})
+
+ # NOTE: verify arguments
+ if len(iplist) > 255:
+ raise Exception("iplist length is too long!")
+
BI_WEEKLY_ESC_PERIOD = int(60*60*24)
- BI_WEEKLY_ESC_PERIOD = int(60) # testing...
+ #BI_WEEKLY_ESC_PERIOD = int(60) # testing...
# User Group
site_user_group = UsrGrp.find_or_create(name=USERGROUP_NAME % loginbase)
key_="system.uname", ports=10050) )
)
if dr.iprange != iplist:
- if len(iplist) < 255:
- dr.iprange = iplist
- else:
- raise Exception("iplist length is too long!")
+ dr.iprange = iplist
# DISCOVERY ACTION for these servers
]
else:
# TODO: verify iplist is up-to-date
- pass
+ # NOTE: len(a.actioncondition_list) > 0
+ ip_condition = a.actioncondition_list[0]
+ assert ip_condition.conditiontype == defines.CONDITION_TYPE_DHOST_IP
+ if ip_condition.value != iplist:
+ ip_condition.value = iplist
# ESCALATION ACTION for these servers
ea = Action.find_or_create(name=escalation_action_name,
esc_step_from=10, esc_step_to=10,
esc_period=0,
shortdata="",
- longdata="%s:/usr/share/monitor-server/checkslices.py {HOSTNAME} disablesite" % config.MONITOR_HOSTNAME,
+ longdata="%s:%s/checkslices.py {HOSTNAME} disablesite" % ( config.MONITOR_HOSTNAME, config.MONITOR_SCRIPT_ROOT ),
operationcondition_list=[ OperationConditionNotAck() ]),
ActionOperation(operationtype=defines.OPERATION_TYPE_MESSAGE,
shortdata=mailtxt.nodedown_two_subject,
esc_step_from=17, esc_step_to=17,
esc_period=0,
shortdata="",
- longdata="%s:/usr/share/monitor-server/checkslices.py {HOSTNAME} disableslices" % config.MONITOR_HOSTNAME,
+ longdata="%s:%s/checkslices.py {HOSTNAME} disableslices" % ( config.MONITOR_HOSTNAME, config.MONITOR_SCRIPT_ROOT ),
# TODO: send notice to users of slices
operationcondition_list=[ OperationConditionNotAck() ]),
ActionOperation(operationtype=defines.OPERATION_TYPE_MESSAGE,
esc_step_from=21, esc_step_to=0,
esc_period=int(BI_WEEKLY_ESC_PERIOD*3.5),
shortdata="",
- longdata="%s:/usr/share/monitor-server/checkslices.py {HOSTNAME} forever" % config.MONITOR_HOSTNAME,
+ longdata="%s:%s/checkslices.py {HOSTNAME} forever" % ( config.MONITOR_HOSTNAME, config.MONITOR_SCRIPT_ROOT),
operationcondition_list=[ OperationConditionNotAck() ]),
ActionOperation(operationtype=defines.OPERATION_TYPE_MESSAGE,
shortdata=mailtxt.nodedown_four_subject,
if __name__=="__main__":
from monitor import parser as parsermodule
- parser = parsermodule.getParser()
+ parser = parsermodule.getParser(['cacheset'])
parser.set_defaults( setupglobal=False, syncsite=True, site=None)
parser.add_option("", "--setupglobal", action="store_true", dest="setupglobal",
help="Setup global settings.")
session.flush()
if opts.syncsite:
- api = plc.getAuthAPI()
+ api = plc.getCachedAuthAPI()
query = {'peer_id' : None}
if opts.site:
query.update({'login_base' : opts.site})
git://git.onelab.eu / monitor.git / commitdiff