From 45fea95bb898f254ea7e987d7417e9091885fbee Mon Sep 17 00:00:00 2001 From: Stephen Soltesz Date: Wed, 2 Sep 2009 18:34:48 +0000 Subject: [PATCH] added actionlist_template to display action list consistently on different pages added upload() method to enable myops to collect bootmanager and other logs added pcu error messages to policy setup clarified messages in emailTxt.py added log_path field to ActionRecord model to save relative bm log paths added SQL upgrade script for new actionrecord field distinguish between bm.log collection form bootman.py and directly uploaded logs removed unnecessary bootmanager action logs when nothing is done. fixed a bug in determining whether comon's dir was running on a node --- monitor/bootman.py | 93 +++++++++++++++---- monitor/common.py | 1 - monitor/database/info/action.py | 2 + monitor/database/info/interface.py | 5 +- monitor/scanapi.py | 2 +- monitor/util/__init__.py | 3 + monitor/util/file.py | 2 +- monitor/wrapper/emailTxt.py | 26 +++--- policy.py | 2 + upgrade/monitor-server-3.0-20.sql | 3 + web/MonitorWeb/monitorweb/controllers.py | 65 +++++++------ .../templates/actionlist_template.kid | 63 +++++++++++++ .../monitorweb/templates/detailview.kid | 42 +-------- .../monitorweb/templates/simpleview.kid | 41 +------- 14 files changed, 207 insertions(+), 143 deletions(-) create mode 100644 upgrade/monitor-server-3.0-20.sql create mode 100644 web/MonitorWeb/monitorweb/templates/actionlist_template.kid diff --git a/monitor/bootman.py b/monitor/bootman.py index 52a8da2..25bd9f4 100755 --- a/monitor/bootman.py +++ b/monitor/bootman.py @@ -2,8 +2,6 @@ # Attempt to reboot a node in debug state. - - import os import sys import time @@ -14,7 +12,6 @@ import subprocess from sets import Set from monitor.getsshkeys import SSHKnownHosts - from monitor.Rpyc import SocketConnection, Async from monitor.Rpyc.Utils import * @@ -36,11 +33,33 @@ from pcucontrol.transports.ssh import pxssh as pxssh from pcucontrol.transports.ssh import fdpexpect as fdpexpect from pcucontrol.transports.ssh import pexpect as pexpect - - api = plc.getAuthAPI() fb = None +def bootmanager_log_name(hostname): + t_stamp = time.strftime("%Y-%m-%d-%H:%M") + base_filename = "%s-bm.%s.log" % (t_stamp, hostname) + short_target_filename = os.path.join('history', base_filename) + return short_target_filename + +def bootmanager_log_action(hostname, short_log_path, logtype="bm.log"): + try: + node = FindbadNodeRecord.get_latest_by(hostname=hostname) + loginbase = PlcSite.query.get(node.plc_node_stats['site_id']).plc_site_stats['login_base'] + err = "" + except: + loginbase = "unknown" + err = traceback.format_exc() + + act = ActionRecord(loginbase=loginbase, + hostname=hostname, + action='log', + action_type=logtype, + log_path=short_log_path, + error_string=err) + session.flush(); session.clear() + return + class ExceptionDoubleSSHError(Exception): pass @@ -76,9 +95,10 @@ class NodeConnection: return log def get_bootmanager_log(self): - t_stamp = time.strftime("%Y-%m-%d-%H:%M") - download(self.c, "/tmp/bm.log", "%s/history/%s-bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node)) - os.system("cp %s/history/%s-bm.%s.log %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, t_stamp, self.node, config.MONITOR_BOOTMANAGER_LOG, self.node)) + bm_name = bootmanager_log_name(self.node) + download(self.c, "/tmp/bm.log", "%s/%s" % (config.MONITOR_BOOTMANAGER_LOG, bm_name)) + bootmanager_log_action(self.node, bm_name, "collected_bm.log") + os.system("cp %s/%s %s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, bm_name, config.MONITOR_BOOTMANAGER_LOG, self.node)) log = open("%s/bm.%s.log" % (config.MONITOR_BOOTMANAGER_LOG, self.node), 'r') return log @@ -255,13 +275,12 @@ class PlanetLabSession: self.setup_host() def get_connection(self, config): - conn = NodeConnection(SocketConnection("localhost", self.port), self.node, config) - #i = 0 - #while i < 3: - # print i, conn.c.modules.sys.path - # print conn.c.modules.os.path.exists('/tmp/source') - # i+=1 - # time.sleep(1) + try: + conn = NodeConnection(SocketConnection("localhost", self.port), self.node, config) + except: + # NOTE: try twice since this can sometimes fail the first time. If + # it fails again, let it go. + conn = NodeConnection(SocketConnection("localhost", self.port), self.node, config) return conn def setup_host(self): @@ -438,12 +457,14 @@ class DebugInterface: "bminit-cfg-auth-getplc-installinit-validate-rebuildinitrd-netcfg-disk-update4-update3-update3-exception-protoerror-protoerror-debug-validate-done", "bminit-cfg-auth-protoerror-exception-update-debug-validate-exception-done", "bminit-cfg-auth-getplc-update-debug-done", + "bminit-cfg-auth-protoerror2-debug-done", "bminit-cfg-auth-getplc-exception-protoerror-update-protoerror-debug-done", "bminit-cfg-auth-protoerror-exception-update-protoerror-debug-done", "bminit-cfg-auth-protoerror-exception-update-bootupdatefail-authfail-debug-done", "bminit-cfg-auth-protoerror-exception-update-debug-done", "bminit-cfg-auth-getplc-exception-protoerror-update-debug-done", "bminit-cfg-auth-getplc-implementerror-update-debug-done", + "bminit-cfg-auth-authfail2-protoerror2-debug-done", ]: sequences.update({n : "restart_bootmanager_boot"}) @@ -474,6 +495,7 @@ class DebugInterface: "bminit-cfg-auth-getplc-update-installinit-validate-exception-noinstall-update-debug-validate-done", "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-update-debug-validate-bmexceptvgscan-done", "bminit-cfg-auth-getplc-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done", + "bminit-cfg-auth-getplc-update-installinit-validate-bmexceptvgscan-exception-noinstall-debug-validate-bmexceptvgscan-done", ]: sequences.update({n : "restart_bootmanager_rins"}) @@ -482,6 +504,8 @@ class DebugInterface: "bminit-cfg-auth-bootcheckfail-authfail-exception-update-bootupdatefail-authfail-debug-done", "bminit-cfg-auth-bootcheckfail-authfail-exception-update-debug-validate-exception-done", "bminit-cfg-auth-bootcheckfail-authfail-exception-authfail-debug-validate-exception-done", + "bminit-cfg-auth-authfail-debug-done", + "bminit-cfg-auth-authfail2-authfail-debug-done", ]: sequences.update({n: "repair_node_keys"}) @@ -554,6 +578,8 @@ class DebugInterface: "bminit-cfg-auth-getplc-hardware-exception-noblockdev-hardwarerequirefail-update-debug-done", "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done", "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-update-debug-done", + "bminit-cfg-auth-getplc-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done", + "bminit-cfg-auth-getplc-update-hardware-noblockdev-exception-hardwarerequirefail-debug-validate-bmexceptvgscan-done", ]: sequences.update({n : "noblockdevice_notice"}) @@ -586,7 +612,7 @@ class DebugInterface: steps = [ ('scsierror' , 'SCSI error : <\d+ \d+ \d+ \d+> return code = 0x\d+'), ('ioerror' , 'end_request: I/O error, dev sd\w+, sector \d+'), - ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION byte \w+ = \w+'), + ('ccisserror' , 'cciss: cmd \w+ has CHECK CONDITION'), ('buffererror', 'Buffer I/O error on device dm-\d, logical block \d+'), @@ -657,6 +683,7 @@ class DebugInterface: ('bmexceptrmfail', 'Unable to remove directory tree: /tmp/mnt'), ('exception' , 'Exception'), ('nocfg' , 'Found configuration file planet.cnf on floppy, but was unable to parse it.'), + ('protoerror2' , '500 Internal Server Error'), ('protoerror' , 'XML RPC protocol error'), ('nodehostname' , 'Configured node hostname does not resolve'), ('implementerror', 'Implementation Error'), @@ -682,8 +709,9 @@ class DebugInterface: ('nospace' , "No space left on device"), ('nonode' , 'Failed to authenticate call: No such node'), ('authfail' , 'Failed to authenticate call: Call could not be authenticated'), - ('bootcheckfail' , 'BootCheckAuthentication'), - ('bootupdatefail' , 'BootUpdateNode'), + ('authfail2' , 'Authentication Failed'), + ('bootcheckfail' , 'BootCheckAuthentication'), + ('bootupdatefail' , 'BootUpdateNode'), ] return steps @@ -732,7 +760,7 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): debugnode = DebugInterface(hostname) conn = debugnode.getConnection() - if type(conn) == type(False): return "error" + if type(conn) == type(False): return "connect_failed" boot_state = conn.get_boot_state() if boot_state != "debug": @@ -838,10 +866,17 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): # the keys either are in sync or were forced in sync. # so try to start BM again. conn.restart_bootmanager(conn.get_nodestate()) - pass else: # there was some failure to synchronize the keys. print "...Unable to repair node keys on %s" %hostname + if not found_within(recent_actions, 'nodeconfig_notice', 3.5): + args = {} + args['hostname'] = hostname + sitehist.sendMessage('nodeconfig_notice', **args) + conn.dump_plconf_file() + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "unknownsequence_notice": args = {} @@ -862,6 +897,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args['hostname'] = hostname sitehist.sendMessage('nodeconfig_notice', **args) conn.dump_plconf_file() + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "nodenetwork_email": @@ -871,6 +909,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args['bmlog'] = conn.get_bootmanager_log().read() sitehist.sendMessage('nodeconfig_notice', **args) conn.dump_plconf_file() + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "noblockdevice_notice": @@ -880,6 +921,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args['hostname'] = hostname sitehist.sendMessage('noblockdevice_notice', **args) + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "baddisk_notice": # MAKE An ACTION record that this host has failed hardware. May @@ -894,6 +938,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): sitehist.sendMessage('baddisk_notice', **args) #conn.set_nodestate('disabled') + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "minimalhardware_notice": if not found_within(recent_actions, 'minimalhardware_notice', 7): @@ -902,6 +949,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args['hostname'] = hostname args['bmlog'] = conn.get_bootmanager_log().read() sitehist.sendMessage('minimalhardware_notice', **args) + else: + # NOTE: do not add a new action record + return "" elif sequences[s] == "baddns_notice": if not found_within(recent_actions, 'baddns_notice', 1): @@ -923,6 +973,9 @@ def restore_basic(sitehist, hostname, config=None, forced_action=None): args['interface_id'] = net['interface_id'] sitehist.sendMessage('baddns_notice', **args) + else: + # NOTE: do not add a new action record + return "" return bootman_action diff --git a/monitor/common.py b/monitor/common.py index d3dc895..75f76e4 100644 --- a/monitor/common.py +++ b/monitor/common.py @@ -3,7 +3,6 @@ import time import struct from monitor import reboot from monitor import util -from monitor import database from monitor.wrapper import plc from datetime import datetime, timedelta diff --git a/monitor/database/info/action.py b/monitor/database/info/action.py index f324daa..6554de8 100644 --- a/monitor/database/info/action.py +++ b/monitor/database/info/action.py @@ -112,6 +112,8 @@ class ActionRecord(Entity): # NOTE: in case an exception is thrown while trying to perform an action. error_string = Field(String, default=None) + log_path = Field(String, default=None) + #issue = ManyToOne('IssueRecord') # NOTE: this is the parent relation to fb records. first create the # action record, then append to this value all of the findbad records we diff --git a/monitor/database/info/interface.py b/monitor/database/info/interface.py index ef7d510..06c83a0 100644 --- a/monitor/database/info/interface.py +++ b/monitor/database/info/interface.py @@ -114,7 +114,7 @@ class SiteInterface(HistorySiteRecord): # those errors. args = {'loginbase' : self.db.loginbase, - 'penalty_level' : self.db.penalty_level, + 'penalty_level' : -self.db.penalty_level, 'monitor_hostname' : config.MONITOR_HOSTNAME, 'support_email' : config.support_email, 'plc_name' : config.PLC_NAME, @@ -201,7 +201,8 @@ class SiteInterface(HistorySiteRecord): action_type='bootmanager_restore', error_string="") - act = ActionRecord(loginbase=self.db.loginbase, + if ret: + act = ActionRecord(loginbase=self.db.loginbase, hostname=hostname, action='reboot', action_type='bootmanager_' + ret, diff --git a/monitor/scanapi.py b/monitor/scanapi.py index d0ed72b..454bcd5 100644 --- a/monitor/scanapi.py +++ b/monitor/scanapi.py @@ -277,7 +277,7 @@ EOF """) continue_slice_check = True oval = values['princeton_comon_dir'] - if "princeton_comon_dir" in oval: + if "princeton_comon" in oval: values['princeton_comon_dir'] = True else: values['princeton_comon_dir'] = False diff --git a/monitor/util/__init__.py b/monitor/util/__init__.py index b3268d9..98802a3 100644 --- a/monitor/util/__init__.py +++ b/monitor/util/__init__.py @@ -3,6 +3,9 @@ __all__=["writepid","removepid","daemonize"] import time import math +# import local file.py +import file + def diff_time(timestamp, abstime=True): now = time.time() if timestamp == None: diff --git a/monitor/util/file.py b/monitor/util/file.py index 43ff968..f26c71d 100644 --- a/monitor/util/file.py +++ b/monitor/util/file.py @@ -24,7 +24,7 @@ def loadFile(file): return buf def dumpFile(file, buf): - f = open(file, 'w') + f = open(file, 'wb') f.write(buf) f.close() return diff --git a/monitor/wrapper/emailTxt.py b/monitor/wrapper/emailTxt.py index 3381d44..63254d0 100644 --- a/monitor/wrapper/emailTxt.py +++ b/monitor/wrapper/emailTxt.py @@ -66,6 +66,10 @@ You can learn more details about the problem by visiting the link below. https://%(monitor_hostname)s/monitor/pcuview?loginbase=%(loginbase)s +If you need to change the PCU configuration in the PLC database: + + https://%(plc_hostname)s/db/sites/pcu.php?id=%(plc_pcuid)s + We would like to save you time by taking care of as many administrative situations for your site's machines as possible without disturbing you. Errors like these prevent us from being able to remotely administer your machines, and so we must solicit your help using messages like these. So, any help and time that you can offer now to help us remotely administer your machines will pay off for you in the future. @@ -193,9 +197,9 @@ Thank you very much for your help, Legend: - 0 - no penalties applied - 1 - site is disabled. no new slices can be created. - 2+ - all existing slices will be disabled. + 0 - no penalties applied + -1 - site is disabled. no new slices can be created. + -2 - all existing slices will be disabled. """) increase_penalty=("""Privilege reduced for site %(loginbase)s""", @@ -214,10 +218,10 @@ Thank you very much for your help, Legend: - 0 - no penalty applied - 1 - site is disabled. no new slices can be created. - 2+ - all existing slices will be disabled. - """) + 0 - no penalties applied + -1 - site is disabled. no new slices can be created. + -2 - all existing slices will be disabled. +""") newbootcd_notice=("""Host %(hostname)s needs a new BootImage""", """ We noticed the following node has an out-dated BootImage: @@ -248,7 +252,7 @@ This may be the case for a number of reasons: * the hard disk was physically removed, * the hard disk cable is loose or disconnected, -Please help us investigate and let us know if there's anything that we can do to assist in getting your machine up and running again. +Please help us investigate and let us know if there is anything that we can do to assist in getting your machine up and running again. Thank you for your help, -- %(plc_name)s (%(support_email)s) @@ -265,11 +269,11 @@ The only step that you need to take is to choose which media you prefer, either %(url_list)s -Instructions to burn or copy these All-in-One images to the appropriate media are available in the Technical Contact's Guide. +Instructions to burn or copy these All-in-One images to the appropriate media are available in the Technical Contacts Guide. https://%(plc_hostname)s/doc/guides/bootcdsetup -If your node returns to normal operation after following these directions, then there's no need to respond to this message. However, if there are any console messages relating to the node's failure, please report them to PlanetLab support (%(support_email)s) so we can help resolve the issue. Including this message in your reply will help us coordinate our records with the actions you've taken. +If your node returns to normal operation after following these directions, then there is no need to respond to this message. However, if there are any console messages relating to the nodes failure, please report them to PlanetLab support (%(support_email)s) so we can help resolve the issue. Including this message in your reply will help us coordinate our records with the actions you have taken. Thank you for your help, -- %(plc_name)s (%(support_email)s) @@ -341,7 +345,7 @@ The output of `dmesg` follows: nodeconfig_notice=(""" Please Update Configuration file for PlanetLab node %(hostname)s""", -"""As part of PlanetLab node monitoring, we noticed %(hostname)s has an out-dated plnode.txt file. +"""As part of PlanetLab node monitoring, we noticed %(hostname)s has an out-dated configuration. Either our boot scripts cannot find it because the boot media is corrupted, or it has no NODE_ID or a mis-matched HOSTNAME. This can happen either due to a configuration mistake at your site, with bad information entered into our database, or after a necessary software upgrade. To resolve the issue we require your assistance. All that is needed is to visit: diff --git a/policy.py b/policy.py index cb5d93f..d3af8e5 100755 --- a/policy.py +++ b/policy.py @@ -110,9 +110,11 @@ def main(hostnames, sitenames): if fbpcu: args['pcu_name'] = fbpcu.pcu_name() args['pcu_errors'] = fbpcu.pcu_errors() + args['plc_pcuid'] = fbpcu.plc_pcuid else: args['pcu_name'] = "error looking up pcu name" args['pcu_errors'] = "" + args['plc_pcuid'] = 0 args['hostname'] = host sitehist.sendMessage('pcuerror_notice', **args) diff --git a/upgrade/monitor-server-3.0-20.sql b/upgrade/monitor-server-3.0-20.sql new file mode 100644 index 0000000..03b665e --- /dev/null +++ b/upgrade/monitor-server-3.0-20.sql @@ -0,0 +1,3 @@ +-- If there's an existing database, these commands will upgrade it to the +-- current version +ALTER TABLE actionrecord ADD COLUMN log_path varchar DEFAULT NULL; diff --git a/web/MonitorWeb/monitorweb/controllers.py b/web/MonitorWeb/monitorweb/controllers.py index 7915ca1..be51959 100644 --- a/web/MonitorWeb/monitorweb/controllers.py +++ b/web/MonitorWeb/monitorweb/controllers.py @@ -12,8 +12,11 @@ from monitor.database.info.model import * #from monitor.database.zabbixapi.model import * from monitor_xmlrpc import MonitorXmlrpcServer +from monitor import util from monitor import reboot +from monitor import bootman from monitor import scanapi +from monitor import config import time from monitor.wrapper.plccache import plcdb_hn2lb as site_hn2lb @@ -174,6 +177,9 @@ def prep_pcu_for_display(pcu): agg.dns_short_status = 'Mismatch' return agg +class ActionListWidget(widgets.Widget): + pass + class NodeWidget(widgets.Widget): pass @@ -578,7 +584,8 @@ class Root(controllers.RootController, MonitorXmlrpcServer): for pcuid_key in pcus: pcuquery += [pcus[pcuid_key]] - return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, since=since, exceptions=exceptions) + actionlist_widget = ActionListWidget(template='monitorweb.templates.actionlist_template') + return dict(sitequery=sitequery, pcuquery=pcuquery, nodequery=nodequery, actions=actions_list, actionlist_widget=actionlist_widget, since=since, exceptions=exceptions) # TODO: add form validation @@ -829,40 +836,46 @@ class Root(controllers.RootController, MonitorXmlrpcServer): return dict(results=results) @expose(template="monitorweb.templates.actionlist") - def actionlist(self, action_type='down_notice', since=7, loginbase=None): + def actionlist(self, since=7, action_type=None, loginbase=None): try: since = int(since) except: since = 7 + acts_query = ActionRecord.query.filter( + ActionRecord.date_created >= datetime.now() - timedelta(since) + ) if loginbase: - acts = ActionRecord.query.filter_by(loginbase=loginbase - ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since) - ).order_by(ActionRecord.date_created.desc()) - else: - acts = ActionRecord.query.filter(ActionRecord.action_type==action_type - ).filter(ActionRecord.date_created >= datetime.now() - timedelta(since) - ).order_by(ActionRecord.date_created.desc()) + acts_query = acts_query.filter_by(loginbase=loginbase) + + if action_type: + acts_query = acts_query.filter(ActionRecord.action_type==action_type) + + acts = acts_query.order_by(ActionRecord.date_created.desc()) + query = [ a for a in acts ] return dict(actions=query, action_type=action_type, since=since) @cherrypy.expose() def upload(self, log, **keywords): - print "got data" - data = log.file.read() - target_file_name = os.path.join(os.getcwd(), log.filename) - # open file in binary mode for writing - - f = open(target_file_name, 'wb') - print "write data" - f.write(data) - f.close() - - #flash("File uploaded successfully: %s saved as: %s" \ - # % (upload_file.filename, target_file_name)) - #u = UploadedFile(filename=upload_file.filename, - # abspath=target_file_name, size=0) - print "redirecting " - - #redirect("monitor") + hostname = None + logtype = None + logtype_list = ['bm.log', ] + + if 'hostname' in keywords: + hostname = keywords['hostname'] + if 'type' in keywords and keywords['type'] in logtype_list: + logtype = keywords['type'] + + if not hostname: return "" + if not logtype: return "unknown logtype: %s" % logtype + + short_target_filename = bootman.bootmanager_log_name(hostname) + abs_target_filename = os.path.join(config.MONITOR_BOOTMANAGER_LOG, short_target_filename) + print "write data: %s" % abs_target_filename + util.file.dumpFile(abs_target_filename, log.file.read()) + bootman.bootmanager_log_action(hostname, short_target_filename, logtype) + + print "redirecting 3" + return dict() diff --git a/web/MonitorWeb/monitorweb/templates/actionlist_template.kid b/web/MonitorWeb/monitorweb/templates/actionlist_template.kid new file mode 100644 index 0000000..afdbd4e --- /dev/null +++ b/web/MonitorWeb/monitorweb/templates/actionlist_template.kid @@ -0,0 +1,63 @@ + + + +

Actions Over the Last ${since} Days

+

+ There are no recent actions taken for this site. +

+ + + + + + + + + + + + + + + + + + + + + + + + + +
AtMyOps acted onUsingMessage/LogErrors
+ + ${act.hostname} + + + ${act.loginbase} + + + + ${act.message_id} + + + + orig bm log + + + +
+ diff --git a/web/MonitorWeb/monitorweb/templates/detailview.kid b/web/MonitorWeb/monitorweb/templates/detailview.kid index ec7b131..14639f8 100644 --- a/web/MonitorWeb/monitorweb/templates/detailview.kid +++ b/web/MonitorWeb/monitorweb/templates/detailview.kid @@ -197,47 +197,7 @@ from links import *
-

Actions Over the Last ${since} Days

-

- There are no recent actions taken for this site. -

- - - - - - - - - - - - - - - - - - - - - - - -
DateAction taken onAction TypeMessage IDErrors
- - ${act.hostname} - - - ${act.loginbase} - - - ${act.message_id} - - - latest bm log - - 

+	${actionlist_widget.display(since=since, actions=actions)}
 
 	
-					
-					
-							${act.message_id}
-					
-						latest bm log
-					
-					
-						

-				
-			
-		
-
+	${actionlist_widget.display(since=since, actions=actions)}
 	
-- 
2.43.0