import sys
from monitor.wrapper import plc
+from monitor.generic import *
+from monitor.database.info.model import *
from monitor import database
from monitor import config
-
-def dsites_from_lsites(l_sites):
- d_sites = {}
- id2lb = {}
- for site in l_sites:
- if not site['login_base'] in d_sites:
- d_sites[site['login_base']] = site
- id2lb[site['site_id']] = site['login_base']
- else:
- #print "Two sites have the same login_base value %s!" % site['login_base']
- sys.exit(1)
- return (d_sites, id2lb)
-
-def dsn_from_dsln(d_sites, id2lb, l_nodes):
- lb2hn = {}
- dsn = {}
- hn2lb = {}
- for id in id2lb:
- if id2lb[id] not in lb2hn:
- lb2hn[id2lb[id]] = []
-
- for node in l_nodes:
- # this won't reach sites without nodes, which I guess isn't a problem.
- if node['site_id'] in id2lb.keys():
- login_base = id2lb[node['site_id']]
- else:
- print "%s has a foreign site_id %s" % (node['hostname'],
- node['site_id'])
- continue
- for i in id2lb:
- print i, " ", id2lb[i]
- raise Exception, "Node has missing site id!! %s %d" %(node['hostname'], node['site_id'])
- if not login_base in dsn:
- lb2hn[login_base] = []
- dsn[login_base] = {}
- dsn[login_base]['plc'] = d_sites[login_base]
- dsn[login_base]['monitor'] = {} # event log, or something
-
- hostname = node['hostname']
- lb2hn[login_base].append(node)
- dsn[login_base][hostname] = {}
- dsn[login_base][hostname]['plc'] = node
- dsn[login_base][hostname]['comon'] = {}
- dsn[login_base][hostname]['monitor'] = {}
-
- hn2lb[hostname] = login_base
- return (dsn, hn2lb, lb2hn)
-
-def create_netid2ip(l_nodes, l_nodenetworks):
- netid2ip = {}
- for node in l_nodes:
- for netid in node['nodenetwork_ids']:
- found = False
- for nn in l_nodenetworks:
- if nn['nodenetwork_id'] == netid:
- found = True
- netid2ip[netid] = nn['ip']
- if not found:
- print "ERROR! %s" % node
-
- return netid2ip
+import profile
l_sites = None
l_nodes = None
l_pcus = None
-l_nodenetworks = None
plcdb_hn2lb = None
plcdb_lb2hn = None
-plcdb_netid2ip = None
+plcdb_id2lb = None
+
+class CachedPLC(plc.PLC):
+
+ def _param_to_str(self, name, *params):
+ fields = len(params)
+ retstr = ""
+ retstr += "%s-" % name
+ for x in params:
+ retstr += "%s-" % x
+ return retstr[:-1]
+
+ def __getattr__(self, name):
+ method = getattr(self.api, name)
+ if method is None:
+ raise AssertionError("method does not exist")
+
+ def run_or_returncached(*params):
+ cachename = self._param_to_str(name, *params)
+ #print "cachename is %s" % cachename
+ if hasattr(config, 'refresh'):
+ refresh = config.refresh
+ else:
+ refresh = False
+
+ if 'Get' in name:
+ if not database.cachedRecently(cachename):
+ load_old_cache = False
+ try:
+ values = method(self.auth, *params)
+ except:
+ print "Call %s FAILED: Using old cached data" % cachename
+ load_old_cache = True
+
+ if load_old_cache:
+ values = database.dbLoad(cachename)
+ else:
+ database.dbDump(cachename, values)
+
+ return values
+ else:
+ values = database.dbLoad(cachename)
+ return values
+ else:
+ return method(self.auth, *params)
+
+ return run_or_returncached
+
+cacheapi = CachedPLC(plc.auth.auth, plc.auth.server)
def init():
+ import traceback
+ #print "IMPORTING PLCCACHE: ",
+ #traceback.print_stack()
global l_sites
global l_nodes
global l_pcus
- global l_nodenetworks
global plcdb_hn2lb
global plcdb_lb2hn
- global plcdb_netid2ip
-
- api = plc.getCachedAuthAPI()
- l_sites = api.GetSites({'peer_id':None},
- ['login_base', 'site_id', 'abbreviated_name', 'latitude',
- 'longitude', 'max_slices', 'slice_ids', 'node_ids', 'enabled' ])
- l_nodes = api.GetNodes({'peer_id':None},
- ['hostname', 'node_id', 'ports', 'site_id', 'version', 'last_updated',
- 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids'])
- l_pcus = api.GetPCUs()
- l_nodenetworks = api.GetNodeNetworks()
-
- (d_sites,id2lb) = dsites_from_lsites(l_sites)
- (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes)
- netid2ip = create_netid2ip(l_nodes, l_nodenetworks)
+ global plcdb_id2lb
+ print >>sys.stderr, "initing plccache"
+
+ print >>sys.stderr, "collecting plcsites"
+ dbsites = PlcSite.query.all()
+ l_sites = [ s.plc_site_stats for s in dbsites ]
+
+ print >>sys.stderr, "collecting plcnodes"
+ dbnodes = PlcNode.query.all()
+ l_nodes = [ s.plc_node_stats for s in dbnodes ]
+
+ print >>sys.stderr, "collecting plcpcus"
+ dbpcus = PlcPCU2.query.all()
+ l_pcus = []
+ for s in dbpcus:
+ pcu = {}
+ for k in ['username', 'protocol', 'node_ids', 'ip',
+ 'pcu_id', 'hostname', 'site_id', 'notes',
+ 'model', 'password', 'ports']:
+ pcu[k] = getattr(s, k)
+ l_pcus.append(pcu)
+
+ print >>sys.stderr, "building id2lb"
+ (d_sites,id2lb) = dsites_from_lsites_id(l_sites)
+ print >>sys.stderr, "building lb2hn"
+ (plcdb, hn2lb, lb2hn, exclude) = dsn_from_dsln(d_sites, id2lb, l_nodes)
plcdb_hn2lb = hn2lb
plcdb_lb2hn = lb2hn
- plcdb_netid2ip = netid2ip
-
- return l_nodes
-
-
-def create_plcdb():
-
- # get sites, and stats
- l_sites = plc.getSites({'peer_id':None}, ['login_base', 'site_id', 'abbreviated_name', 'latitude', 'longitude',
- 'max_slices', 'slice_ids', 'node_ids' ])
- if len(l_sites) == 0:
- sys.exit(1)
- (d_sites,id2lb) = dsites_from_lsites(l_sites)
-
- # get nodes at each site, and
- l_nodes = plc.getNodes({'peer_id':None}, ['hostname', 'node_id', 'ports', 'site_id', 'version',
- 'last_updated', 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids'])
-
- l_nodenetworks = plc.getNodeNetworks()
- (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes)
- netid2ip = create_netid2ip(l_nodes, l_nodenetworks)
-
- # save information for future.
- id2lb = id2lb
- hn2lb = hn2lb
- db = plcdb
-
- if ('cachenodes' in dir(config) and config.cachenodes) or \
- 'cachenodes' not in dir(config):
- database.dbDump("plcdb_hn2lb", hn2lb)
- database.dbDump("plcdb_lb2hn", lb2hn)
- database.dbDump("plcdb_netid2ip", netid2ip)
- database.dbDump("l_plcnodenetworks", l_nodenetworks)
- database.dbDump("l_plcnodes", l_nodes)
- database.dbDump("l_plcsites", l_sites)
+ plcdb_id2lb = id2lb
+
+ l_nodes = filter(lambda x: x['hostname'] not in exclude, l_nodes)
- return l_nodes
+ return
+
+def GetNodesByIds(ids):
+ ret = []
+ for node_id in ids:
+ node = PlcNode.get_by(node_id=node_id)
+ ret.append(node.plc_node_stats)
+ return ret
+
+def GetNodesBySite(loginbase):
+ site = PlcSite.get_by(loginbase=loginbase)
+ return GetNodesByIds(site.plc_site_stats['node_ids'])
+
+def GetNodeByName(hostname):
+ print "GetNodeByName %s" % hostname
+ node = PlcNode.get_by(hostname=hostname)
+ return node.plc_node_stats
+
+def GetSitesByName(sitelist):
+ ret = []
+ for site in sitelist:
+ site = PlcSite.get_by(loginbase=site)
+ ret.append(site.plc_site_stats)
+ return ret
+
+def GetSitesById(idlist):
+ ret = []
+ for site_id in idlist:
+ site = PlcSite.get_by(site_id=site_id)
+ ret.append(site.plc_site_stats)
+ return ret
+
+def deleteExtra(l_plc, objectClass=PlcSite, dbKey='loginbase', plcKey='login_base'):
+ dbobjs = objectClass.query.all()
+ dbobj_key = [ getattr(s, dbKey) for s in dbobjs ]
+ plcobj_key = [ s[plcKey] for s in l_plc ]
+ extra_key = set(dbobj_key) - set(plcobj_key)
+ for obj in extra_key:
+ print >>sys.stderr, "deleting %s" % obj
+ dbobj = objectClass.get_by(**{dbKey : obj})
+ dbobj.delete()
+
+def conv(s):
+ # strip non-ascii characters to prvent errors
+ r = s
+ if type(s) in (str,unicode):
+ r = "".join([x for x in s if ord(x) < 128])
+ return r
+
+def sync():
+ l_sites = plc.api.GetSites({'peer_id':None},
+ ['login_base', 'site_id', 'abbreviated_name', 'latitude',
+ 'longitude', 'max_slices', 'slice_ids', 'node_ids',
+ 'enabled', 'date_created' ])
+ l_nodes = plc.api.GetNodes({'peer_id':None},
+ ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level',
+ 'version', 'last_updated', 'date_created', 'key',
+ 'last_contact', 'pcu_ids', 'interface_ids'])
+ l_pcus = plc.api.GetPCUs()
+
+ print >>sys.stderr, "sync sites"
+ for site in l_sites:
+ dbsite = PlcSite.findby_or_create(site_id=site['site_id'])
+ dbsite.loginbase = site['login_base']
+ dbsite.date_checked = datetime.now()
+ dbsite.plc_site_stats = site
+ deleteExtra(l_sites, PlcSite, 'loginbase', 'login_base')
+ deleteExtra(l_sites, HistorySiteRecord, 'loginbase', 'login_base')
+ session.flush()
+
+ print >>sys.stderr, "sync pcus"
+ for pcu in l_pcus:
+ dbpcu = PlcPCU2.findby_or_create(pcu_id=pcu['pcu_id'])
+ dbpcu.date_checked = datetime.now()
+ for key in pcu.keys():
+ print >>sys.stderr, "setting %s = %s" % (key, conv(pcu[key]))
+ setattr(dbpcu, key, conv(pcu[key]))
+
+ deleteExtra(l_pcus, PlcPCU2, 'pcu_id', 'pcu_id')
+ deleteExtra(l_pcus, HistoryPCURecord, 'plc_pcuid', 'pcu_id')
+ deleteExtra(l_pcus, FindbadPCURecord, 'plc_pcuid', 'pcu_id')
+ session.flush()
+
+ print >>sys.stderr, "sync nodes"
+ for node in l_nodes:
+ dbnode = PlcNode.findby_or_create(node_id=node['node_id'])
+ dbnode.hostname = node['hostname']
+ dbnode.date_checked = datetime.now()
+ dbnode.plc_node_stats = node
+ deleteExtra(l_nodes, PlcNode, 'node_id', 'node_id')
+ deleteExtra(l_nodes, HistoryNodeRecord, 'plc_nodeid', 'node_id')
+ deleteExtra(l_nodes, PlcNode, 'hostname', 'hostname')
+ deleteExtra(l_nodes, HistoryNodeRecord, 'hostname', 'hostname')
+ deleteExtra(l_nodes, FindbadNodeRecord, 'hostname', 'hostname')
+ session.flush()
+
+ init()
+
+ return
if __name__ == '__main__':
- create_plcdb()
+ sync()
else:
- print "calling plccache init()"
init()
git://git.onelab.eu / monitor.git / blobdiff