X-Git-Url: http://git.onelab.eu/?p=monitor.git;a=blobdiff_plain;f=monitor%2Fwrapper%2Fplccache.py;h=4778a7dba819a72dc3d33694af1f74ff36db1516;hp=73a6e57779ef82c924f04a2d5edeb28030403c37;hb=f4f26439ae2db33f8f9a55e1a3350f6ed4f78278;hpb=8c989c864f4113c3f0969f5ec5fe86f047f84256 diff --git a/monitor/wrapper/plccache.py b/monitor/wrapper/plccache.py index 73a6e57..4778a7d 100755 --- a/monitor/wrapper/plccache.py +++ b/monitor/wrapper/plccache.py @@ -2,137 +2,211 @@ import sys from monitor.wrapper import plc +from monitor.generic import * +from monitor.database.info.model import * from monitor import database from monitor import config - -def dsites_from_lsites(l_sites): - d_sites = {} - id2lb = {} - for site in l_sites: - if not site['login_base'] in d_sites: - d_sites[site['login_base']] = site - id2lb[site['site_id']] = site['login_base'] - else: - #print "Two sites have the same login_base value %s!" % site['login_base'] - sys.exit(1) - return (d_sites, id2lb) - -def dsn_from_dsln(d_sites, id2lb, l_nodes): - lb2hn = {} - dsn = {} - hn2lb = {} - for node in l_nodes: - # this won't reach sites without nodes, which I guess isn't a problem. - if node['site_id'] in id2lb.keys(): - login_base = id2lb[node['site_id']] - else: - print "%s has a foreign site_id %s" % (node['hostname'], - node['site_id']) - continue - for i in id2lb: - print i, " ", id2lb[i] - raise Exception, "Node has missing site id!! %s %d" %(node['hostname'], node['site_id']) - if not login_base in dsn: - lb2hn[login_base] = [] - dsn[login_base] = {} - dsn[login_base]['plc'] = d_sites[login_base] - dsn[login_base]['monitor'] = {} # event log, or something - - hostname = node['hostname'] - lb2hn[login_base].append(node) - dsn[login_base][hostname] = {} - dsn[login_base][hostname]['plc'] = node - dsn[login_base][hostname]['comon'] = {} - dsn[login_base][hostname]['monitor'] = {} - - hn2lb[hostname] = login_base - return (dsn, hn2lb, lb2hn) - -def create_netid2ip(l_nodes, l_nodenetworks): - netid2ip = {} - for node in l_nodes: - for netid in node['nodenetwork_ids']: - found = False - for nn in l_nodenetworks: - if nn['nodenetwork_id'] == netid: - found = True - netid2ip[netid] = nn['ip'] - if not found: - print "ERROR! %s" % node - - return netid2ip +import profile l_sites = None l_nodes = None l_pcus = None -l_nodenetworks = None plcdb_hn2lb = None plcdb_lb2hn = None -plcdb_netid2ip = None +plcdb_id2lb = None + +class CachedPLC(plc.PLC): + + def _param_to_str(self, name, *params): + fields = len(params) + retstr = "" + retstr += "%s-" % name + for x in params: + retstr += "%s-" % x + return retstr[:-1] + + def __getattr__(self, name): + method = getattr(self.api, name) + if method is None: + raise AssertionError("method does not exist") + + def run_or_returncached(*params): + cachename = self._param_to_str(name, *params) + #print "cachename is %s" % cachename + if hasattr(config, 'refresh'): + refresh = config.refresh + else: + refresh = False + + if 'Get' in name: + if not database.cachedRecently(cachename): + load_old_cache = False + try: + values = method(self.auth, *params) + except: + print "Call %s FAILED: Using old cached data" % cachename + load_old_cache = True + + if load_old_cache: + values = database.dbLoad(cachename) + else: + database.dbDump(cachename, values) + + return values + else: + values = database.dbLoad(cachename) + return values + else: + return method(self.auth, *params) + + return run_or_returncached + +cacheapi = CachedPLC(plc.auth.auth, plc.auth.server) def init(): + import traceback + #print "IMPORTING PLCCACHE: ", + #traceback.print_stack() global l_sites global l_nodes global l_pcus - global l_nodenetworks global plcdb_hn2lb global plcdb_lb2hn - global plcdb_netid2ip - - api = plc.getCachedAuthAPI() - l_sites = api.GetSites({'peer_id':None}, - ['login_base', 'site_id', 'abbreviated_name', 'latitude', - 'longitude', 'max_slices', 'slice_ids', 'node_ids' ]) - l_nodes = api.GetNodes({'peer_id':None}, - ['hostname', 'node_id', 'ports', 'site_id', 'version', 'last_updated', - 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids']) - l_pcus = api.GetPCUs() - l_nodenetworks = api.GetNodeNetworks() - - (d_sites,id2lb) = dsites_from_lsites(l_sites) - (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes) - netid2ip = create_netid2ip(l_nodes, l_nodenetworks) + global plcdb_id2lb + print >>sys.stderr, "initing plccache" + + print >>sys.stderr, "collecting plcsites" + dbsites = PlcSite.query.all() + l_sites = [ s.plc_site_stats for s in dbsites ] + + print >>sys.stderr, "collecting plcnodes" + dbnodes = PlcNode.query.all() + l_nodes = [ s.plc_node_stats for s in dbnodes ] + + print >>sys.stderr, "collecting plcpcus" + dbpcus = PlcPCU2.query.all() + l_pcus = [] + for s in dbpcus: + pcu = {} + for k in ['username', 'protocol', 'node_ids', 'ip', + 'pcu_id', 'hostname', 'site_id', 'notes', + 'model', 'password', 'ports']: + pcu[k] = getattr(s, k) + l_pcus.append(pcu) + + print >>sys.stderr, "building id2lb" + (d_sites,id2lb) = dsites_from_lsites_id(l_sites) + print >>sys.stderr, "building lb2hn" + (plcdb, hn2lb, lb2hn, exclude) = dsn_from_dsln(d_sites, id2lb, l_nodes) plcdb_hn2lb = hn2lb plcdb_lb2hn = lb2hn - plcdb_netid2ip = netid2ip - - return l_nodes - -def create_plcdb(): - - # get sites, and stats - l_sites = plc.getSites({'peer_id':None}, ['login_base', 'site_id', 'abbreviated_name', 'latitude', 'longitude', - 'max_slices', 'slice_ids', 'node_ids' ]) - if len(l_sites) == 0: - sys.exit(1) - (d_sites,id2lb) = dsites_from_lsites(l_sites) - - # get nodes at each site, and - l_nodes = plc.getNodes({'peer_id':None}, ['hostname', 'node_id', 'ports', 'site_id', 'version', - 'last_updated', 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids']) - - l_nodenetworks = plc.getNodeNetworks() - (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes) - netid2ip = create_netid2ip(l_nodes, l_nodenetworks) - - # save information for future. - id2lb = id2lb - hn2lb = hn2lb - db = plcdb - - if ('cachenodes' in dir(config) and config.cachenodes) or \ - 'cachenodes' not in dir(config): - database.dbDump("plcdb_hn2lb", hn2lb) - database.dbDump("plcdb_lb2hn", lb2hn) - database.dbDump("plcdb_netid2ip", netid2ip) - database.dbDump("l_plcnodenetworks", l_nodenetworks) - database.dbDump("l_plcnodes", l_nodes) - database.dbDump("l_plcsites", l_sites) - - return l_nodes + plcdb_id2lb = id2lb + + l_nodes = filter(lambda x: x['hostname'] not in exclude, l_nodes) + return + +def GetNodesByIds(ids): + ret = [] + for node_id in ids: + node = PlcNode.get_by(node_id=node_id) + ret.append(node.plc_node_stats) + return ret + +def GetNodesBySite(loginbase): + site = PlcSite.get_by(loginbase=loginbase) + return GetNodesByIds(site.plc_site_stats['node_ids']) + +def GetNodeByName(hostname): + print "GetNodeByName %s" % hostname + node = PlcNode.get_by(hostname=hostname) + return node.plc_node_stats + +def GetSitesByName(sitelist): + ret = [] + for site in sitelist: + site = PlcSite.get_by(loginbase=site) + ret.append(site.plc_site_stats) + return ret + +def GetSitesById(idlist): + ret = [] + for site_id in idlist: + site = PlcSite.get_by(site_id=site_id) + ret.append(site.plc_site_stats) + return ret + +def deleteExtra(l_plc, objectClass=PlcSite, dbKey='loginbase', plcKey='login_base'): + dbobjs = objectClass.query.all() + dbobj_key = [ getattr(s, dbKey) for s in dbobjs ] + plcobj_key = [ s[plcKey] for s in l_plc ] + extra_key = set(dbobj_key) - set(plcobj_key) + for obj in extra_key: + print >>sys.stderr, "deleting %s" % obj + dbobj = objectClass.get_by(**{dbKey : obj}) + dbobj.delete() + +def conv(s): + # strip non-ascii characters to prvent errors + r = s + if type(s) in (str,unicode): + r = "".join([x for x in s if ord(x) < 128]) + return r + +def sync(): + l_sites = plc.api.GetSites({'peer_id':None}, + ['login_base', 'site_id', 'abbreviated_name', 'latitude', + 'longitude', 'max_slices', 'slice_ids', 'node_ids', + 'enabled', 'date_created' ]) + l_nodes = plc.api.GetNodes({'peer_id':None}, + ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level', + 'version', 'last_updated', 'date_created', 'key', + 'last_contact', 'pcu_ids', 'interface_ids']) + l_pcus = plc.api.GetPCUs() + + print >>sys.stderr, "sync sites" + for site in l_sites: + dbsite = PlcSite.findby_or_create(site_id=site['site_id']) + dbsite.loginbase = site['login_base'] + dbsite.date_checked = datetime.now() + dbsite.plc_site_stats = site + deleteExtra(l_sites, PlcSite, 'loginbase', 'login_base') + deleteExtra(l_sites, HistorySiteRecord, 'loginbase', 'login_base') + session.flush() + + print >>sys.stderr, "sync pcus" + for pcu in l_pcus: + dbpcu = PlcPCU2.findby_or_create(pcu_id=pcu['pcu_id']) + dbpcu.date_checked = datetime.now() + for key in pcu.keys(): + print >>sys.stderr, "setting %s = %s" % (key, conv(pcu[key])) + setattr(dbpcu, key, conv(pcu[key])) + + deleteExtra(l_pcus, PlcPCU2, 'pcu_id', 'pcu_id') + deleteExtra(l_pcus, HistoryPCURecord, 'plc_pcuid', 'pcu_id') + deleteExtra(l_pcus, FindbadPCURecord, 'plc_pcuid', 'pcu_id') + session.flush() + + print >>sys.stderr, "sync nodes" + for node in l_nodes: + dbnode = PlcNode.findby_or_create(node_id=node['node_id']) + dbnode.hostname = node['hostname'] + dbnode.date_checked = datetime.now() + dbnode.plc_node_stats = node + deleteExtra(l_nodes, PlcNode, 'node_id', 'node_id') + deleteExtra(l_nodes, HistoryNodeRecord, 'plc_nodeid', 'node_id') + deleteExtra(l_nodes, PlcNode, 'hostname', 'hostname') + deleteExtra(l_nodes, HistoryNodeRecord, 'hostname', 'hostname') + deleteExtra(l_nodes, FindbadNodeRecord, 'hostname', 'hostname') + session.flush() + + init() + + return if __name__ == '__main__': - create_plcdb() + sync() +else: + init()