X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=monitor%2Fwrapper%2Fplccache.py;h=7b1d258e8477450db96b75265fd314ec32f71468;hb=0a896f10303078562b737c83fd08dc25f0973c58;hp=73a6e57779ef82c924f04a2d5edeb28030403c37;hpb=8c989c864f4113c3f0969f5ec5fe86f047f84256;p=monitor.git diff --git a/monitor/wrapper/plccache.py b/monitor/wrapper/plccache.py index 73a6e57..7b1d258 100755 --- a/monitor/wrapper/plccache.py +++ b/monitor/wrapper/plccache.py @@ -2,8 +2,8 @@ import sys from monitor.wrapper import plc -from monitor import database -from monitor import config +from monitor.database.info.model import * +import profile def dsites_from_lsites(l_sites): d_sites = {} @@ -14,19 +14,24 @@ def dsites_from_lsites(l_sites): id2lb[site['site_id']] = site['login_base'] else: #print "Two sites have the same login_base value %s!" % site['login_base'] - sys.exit(1) + #sys.exit(1) + continue return (d_sites, id2lb) def dsn_from_dsln(d_sites, id2lb, l_nodes): lb2hn = {} dsn = {} hn2lb = {} + for id in id2lb: + if id2lb[id] not in lb2hn: + lb2hn[id2lb[id]] = [] + for node in l_nodes: # this won't reach sites without nodes, which I guess isn't a problem. if node['site_id'] in id2lb.keys(): login_base = id2lb[node['site_id']] else: - print "%s has a foreign site_id %s" % (node['hostname'], + print >>sys.stderr, "%s has a foreign site_id %s" % (node['hostname'], node['site_id']) continue for i in id2lb: @@ -48,91 +53,145 @@ def dsn_from_dsln(d_sites, id2lb, l_nodes): hn2lb[hostname] = login_base return (dsn, hn2lb, lb2hn) -def create_netid2ip(l_nodes, l_nodenetworks): - netid2ip = {} - for node in l_nodes: - for netid in node['nodenetwork_ids']: - found = False - for nn in l_nodenetworks: - if nn['nodenetwork_id'] == netid: - found = True - netid2ip[netid] = nn['ip'] - if not found: - print "ERROR! %s" % node - - return netid2ip - l_sites = None l_nodes = None l_pcus = None -l_nodenetworks = None plcdb_hn2lb = None plcdb_lb2hn = None -plcdb_netid2ip = None +plcdb_id2lb = None def init(): + import traceback + #print "IMPORTING PLCCACHE: ", + #traceback.print_stack() global l_sites global l_nodes global l_pcus - global l_nodenetworks global plcdb_hn2lb global plcdb_lb2hn - global plcdb_netid2ip - - api = plc.getCachedAuthAPI() - l_sites = api.GetSites({'peer_id':None}, - ['login_base', 'site_id', 'abbreviated_name', 'latitude', - 'longitude', 'max_slices', 'slice_ids', 'node_ids' ]) - l_nodes = api.GetNodes({'peer_id':None}, - ['hostname', 'node_id', 'ports', 'site_id', 'version', 'last_updated', - 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids']) - l_pcus = api.GetPCUs() - l_nodenetworks = api.GetNodeNetworks() - + global plcdb_id2lb + print >>sys.stderr, "initing plccache" + + print >>sys.stderr, "collecting plcsites" + dbsites = PlcSite.query.all() + l_sites = [ s.plc_site_stats for s in dbsites ] + + print >>sys.stderr, "collecting plcnodes" + dbnodes = PlcNode.query.all() + l_nodes = [ s.plc_node_stats for s in dbnodes ] + + print >>sys.stderr, "collecting plcpcus" + dbpcus = PlcPCU2.query.all() + l_pcus = [] + for s in dbpcus: + pcu = {} + for k in ['username', 'protocol', 'node_ids', 'ip', + 'pcu_id', 'hostname', 'site_id', 'notes', + 'model', 'password', 'ports']: + pcu[k] = getattr(s, k) + l_pcus.append(pcu) + + print >>sys.stderr, "building id2lb" (d_sites,id2lb) = dsites_from_lsites(l_sites) + print >>sys.stderr, "building lb2hn" (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes) - netid2ip = create_netid2ip(l_nodes, l_nodenetworks) plcdb_hn2lb = hn2lb plcdb_lb2hn = lb2hn - plcdb_netid2ip = netid2ip + plcdb_id2lb = id2lb - return l_nodes - -def create_plcdb(): - - # get sites, and stats - l_sites = plc.getSites({'peer_id':None}, ['login_base', 'site_id', 'abbreviated_name', 'latitude', 'longitude', - 'max_slices', 'slice_ids', 'node_ids' ]) - if len(l_sites) == 0: - sys.exit(1) - (d_sites,id2lb) = dsites_from_lsites(l_sites) + return + +def GetNodesByIds(ids): + ret = [] + for node_id in ids: + node = PlcNode.get_by(node_id=node_id) + ret.append(node.plc_node_stats) + return ret + +def GetNodesBySite(loginbase): + site = PlcSite.get_by(loginbase=loginbase) + return GetNodesByIds(site.plc_site_stats['node_ids']) + +def GetNodeByName(hostname): + node = PlcNode.get_by(hostname=hostname) + return node.plc_node_stats + +def GetSitesByName(sitelist): + ret = [] + for site in sitelist: + site = PlcSite.get_by(loginbase=site) + ret.append(site.plc_site_stats) + return ret + +def GetSitesById(idlist): + ret = [] + for site_id in idlist: + site = PlcSite.get_by(site_id=site_id) + ret.append(site.plc_site_stats) + return ret + +def deleteExtra(l_plc, objectClass=PlcSite, dbKey='loginbase', plcKey='login_base'): + dbobjs = objectClass.query.all() + dbobj_key = [ getattr(s, dbKey) for s in dbobjs ] + plcobj_key = [ s[plcKey] for s in l_plc ] + extra_key = set(dbobj_key) - set(plcobj_key) + for obj in extra_key: + print >>sys.stderr, "deleting %s" % obj + dbobj = objectClass.get_by(**{dbKey : obj}) + dbobj.delete() + +def sync(): + l_sites = plc.api.GetSites({'peer_id':None}, + ['login_base', 'site_id', 'abbreviated_name', 'latitude', + 'longitude', 'max_slices', 'slice_ids', 'node_ids', + 'enabled', 'date_created' ]) + l_nodes = plc.api.GetNodes({'peer_id':None}, + ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level', + 'version', 'last_updated', 'date_created', 'key', + 'last_contact', 'pcu_ids', 'interface_ids']) + l_pcus = plc.api.GetPCUs() + + print >>sys.stderr, "sync sites" + for site in l_sites: + dbsite = PlcSite.findby_or_create(site_id=site['site_id']) + dbsite.loginbase = site['login_base'] + dbsite.date_checked = datetime.now() + dbsite.plc_site_stats = site + deleteExtra(l_sites, PlcSite, 'loginbase', 'login_base') + deleteExtra(l_sites, HistorySiteRecord, 'loginbase', 'login_base') + session.flush() + + print >>sys.stderr, "sync pcus" + for pcu in l_pcus: + dbpcu = PlcPCU2.findby_or_create(pcu_id=pcu['pcu_id']) + dbpcu.date_checked = datetime.now() + for key in pcu.keys(): + print >>sys.stderr, "setting %s = %s" % (key, pcu[key]) + setattr(dbpcu, key, pcu[key]) + + deleteExtra(l_pcus, PlcPCU2, 'pcu_id', 'pcu_id') + deleteExtra(l_pcus, HistoryPCURecord, 'plc_pcuid', 'pcu_id') + deleteExtra(l_pcus, FindbadPCURecord, 'plc_pcuid', 'pcu_id') + session.flush() + + print >>sys.stderr, "sync nodes" + for node in l_nodes: + dbnode = PlcNode.findby_or_create(node_id=node['node_id']) + dbnode.hostname = node['hostname'] + dbnode.date_checked = datetime.now() + dbnode.plc_node_stats = node + deleteExtra(l_nodes, PlcNode, 'hostname', 'hostname') + deleteExtra(l_nodes, HistoryNodeRecord, 'hostname', 'hostname') + deleteExtra(l_nodes, FindbadNodeRecord, 'hostname', 'hostname') + session.flush() - # get nodes at each site, and - l_nodes = plc.getNodes({'peer_id':None}, ['hostname', 'node_id', 'ports', 'site_id', 'version', - 'last_updated', 'date_created', 'last_contact', 'pcu_ids', 'nodenetwork_ids']) + init() - l_nodenetworks = plc.getNodeNetworks() - (plcdb, hn2lb, lb2hn) = dsn_from_dsln(d_sites, id2lb, l_nodes) - netid2ip = create_netid2ip(l_nodes, l_nodenetworks) - - # save information for future. - id2lb = id2lb - hn2lb = hn2lb - db = plcdb - - if ('cachenodes' in dir(config) and config.cachenodes) or \ - 'cachenodes' not in dir(config): - database.dbDump("plcdb_hn2lb", hn2lb) - database.dbDump("plcdb_lb2hn", lb2hn) - database.dbDump("plcdb_netid2ip", netid2ip) - database.dbDump("l_plcnodenetworks", l_nodenetworks) - database.dbDump("l_plcnodes", l_nodes) - database.dbDump("l_plcsites", l_sites) - - return l_nodes - + return if __name__ == '__main__': - create_plcdb() + sync() +else: + init()