monitor/wrapper/plccache.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 from monitor.wrapper import plc
   5 from monitor.generic import *
   6 from monitor.database.info.model import *
   7 from monitor import database
   8 from monitor import config
   9 import profile
  10
  11 l_sites = None
  12 l_nodes = None
  13 l_pcus = None
  14
  15 plcdb_hn2lb = None
  16 plcdb_lb2hn = None
  17 plcdb_id2lb = None
  18
  19 class CachedPLC(plc.PLC):
  20
  21         def _param_to_str(self, name, *params):
  22                 fields = len(params)
  23                 retstr = ""
  24                 retstr += "%s-" % name
  25                 for x in params:
  26                         retstr += "%s-" % x
  27                 return retstr[:-1]
  28
  29         def __getattr__(self, name):
  30                 method = getattr(self.api, name)
  31                 if method is None:
  32                         raise AssertionError("method does not exist")
  33
  34                 def run_or_returncached(*params):
  35                         cachename = self._param_to_str(name, *params)
  36                         #print "cachename is %s" % cachename
  37                         if hasattr(config, 'refresh'):
  38                                 refresh = config.refresh
  39                         else:
  40                                 refresh = False
  41
  42                         if 'Get' in name:
  43                                 if not database.cachedRecently(cachename):
  44                                         load_old_cache = False
  45                                         try:
  46                                                 values = method(self.auth, *params)
  47                                         except:
  48                                                 print "Call %s FAILED: Using old cached data" % cachename
  49                                                 load_old_cache = True
  50
  51                                         if load_old_cache:
  52                                                 values = database.dbLoad(cachename)
  53                                         else:
  54                                                 database.dbDump(cachename, values)
  55
  56                                         return values
  57                                 else:
  58                                         values = database.dbLoad(cachename)
  59                                         return values
  60                         else:
  61                                 return method(self.auth, *params)
  62
  63                 return run_or_returncached
  64
  65 cacheapi = CachedPLC(plc.auth.auth, plc.auth.server)
  66
  67 def init():
  68         import traceback
  69         #print "IMPORTING PLCCACHE: ",
  70         #traceback.print_stack()
  71         global l_sites
  72         global l_nodes
  73         global l_pcus
  74         global plcdb_hn2lb
  75         global plcdb_lb2hn
  76         global plcdb_id2lb
  77         print >>sys.stderr, "initing plccache"
  78
  79         print >>sys.stderr, "collecting plcsites"
  80         dbsites = PlcSite.query.all()
  81         l_sites = [ s.plc_site_stats for s in dbsites ]
  82
  83         print >>sys.stderr, "collecting plcnodes"
  84         dbnodes = PlcNode.query.all()
  85         l_nodes = [ s.plc_node_stats for s in dbnodes ]
  86
  87         print >>sys.stderr, "collecting plcpcus"
  88         dbpcus = PlcPCU2.query.all()
  89         l_pcus = []
  90         for s in dbpcus:
  91                 pcu = {}
  92                 for k in ['username', 'protocol', 'node_ids', 'ip',
  93                                   'pcu_id', 'hostname', 'site_id', 'notes',
  94                                   'model', 'password', 'ports']:
  95                         pcu[k] = getattr(s, k)
  96                 l_pcus.append(pcu)
  97
  98         print >>sys.stderr, "building id2lb"
  99         (d_sites,id2lb) = dsites_from_lsites_id(l_sites)
 100         print >>sys.stderr, "building lb2hn"
 101         (plcdb, hn2lb, lb2hn, exclude) = dsn_from_dsln(d_sites, id2lb, l_nodes)
 102
 103         plcdb_hn2lb = hn2lb
 104         plcdb_lb2hn = lb2hn
 105         plcdb_id2lb = id2lb
 106
 107         l_nodes = filter(lambda x: x['hostname'] not in exclude, l_nodes)
 108
 109         return
 110
 111 def GetNodesByIds(ids):
 112         ret = []
 113         for node_id in ids:
 114                 node = PlcNode.get_by(node_id=node_id)
 115                 ret.append(node.plc_node_stats)
 116         return ret
 117
 118 def GetNodesBySite(loginbase):
 119         site = PlcSite.get_by(loginbase=loginbase)
 120         return GetNodesByIds(site.plc_site_stats['node_ids'])
 121
 122 def GetNodeByName(hostname):
 123         print "GetNodeByName %s" % hostname
 124         node = PlcNode.get_by(hostname=hostname)
 125         return node.plc_node_stats
 126
 127 def GetSitesByName(sitelist):
 128         ret = []
 129         for site in sitelist:
 130                 site = PlcSite.get_by(loginbase=site)
 131                 ret.append(site.plc_site_stats)
 132         return ret
 133
 134 def GetSitesById(idlist):
 135         ret = []
 136         for site_id in idlist:
 137                 site = PlcSite.get_by(site_id=site_id)
 138                 ret.append(site.plc_site_stats)
 139         return ret
 140
 141 def deleteExtra(l_plc, objectClass=PlcSite, dbKey='loginbase', plcKey='login_base'):
 142         dbobjs = objectClass.query.all()
 143         dbobj_key = [ getattr(s, dbKey) for s in dbobjs ]
 144         plcobj_key = [ s[plcKey] for s in l_plc ]
 145         extra_key = set(dbobj_key) - set(plcobj_key)
 146         for obj in extra_key:
 147                 print >>sys.stderr, "deleting %s" % obj
 148                 dbobj = objectClass.get_by(**{dbKey : obj})
 149                 dbobj.delete()
 150
 151 def conv(s):
 152     # strip non-ascii characters to prvent errors
 153     r = s
 154     if type(s) in (str,unicode):
 155         r = "".join([x for x in s if ord(x) < 128])
 156     return r
 157
 158 def sync():
 159         l_sites = plc.api.GetSites({'peer_id':None},
 160                                                 ['login_base', 'site_id', 'abbreviated_name', 'latitude',
 161                                                 'longitude', 'max_slices', 'slice_ids', 'node_ids',
 162                                                 'enabled', 'date_created' ])
 163         l_nodes = plc.api.GetNodes({'peer_id':None},
 164                                                 ['hostname', 'node_id', 'ports', 'site_id', 'boot_state', 'run_level',
 165                                                  'version', 'last_updated', 'date_created', 'key',
 166                                                  'last_contact', 'pcu_ids', 'interface_ids'])
 167         l_pcus = plc.api.GetPCUs()
 168
 169         print >>sys.stderr, "sync sites"
 170         for site in l_sites:
 171                 dbsite = PlcSite.findby_or_create(site_id=site['site_id'])
 172                 dbsite.loginbase = site['login_base']
 173                 dbsite.date_checked = datetime.now()
 174                 dbsite.plc_site_stats = site
 175         deleteExtra(l_sites, PlcSite, 'loginbase', 'login_base')
 176         deleteExtra(l_sites, HistorySiteRecord, 'loginbase', 'login_base')
 177         session.flush()
 178
 179         print >>sys.stderr, "sync pcus"
 180         for pcu in l_pcus:
 181                 dbpcu = PlcPCU2.findby_or_create(pcu_id=pcu['pcu_id'])
 182                 dbpcu.date_checked = datetime.now()
 183                 for key in pcu.keys():
 184                         print >>sys.stderr, "setting %s  = %s" % (key, conv(pcu[key]))
 185                         setattr(dbpcu, key, conv(pcu[key]))
 186
 187         deleteExtra(l_pcus, PlcPCU2, 'pcu_id', 'pcu_id')
 188         deleteExtra(l_pcus, HistoryPCURecord, 'plc_pcuid', 'pcu_id')
 189         deleteExtra(l_pcus, FindbadPCURecord, 'plc_pcuid', 'pcu_id')
 190         session.flush()
 191
 192         print >>sys.stderr, "sync nodes"
 193         for node in l_nodes:
 194                 dbnode = PlcNode.findby_or_create(node_id=node['node_id'])
 195                 dbnode.hostname = node['hostname']
 196                 dbnode.date_checked = datetime.now()
 197                 dbnode.plc_node_stats = node
 198         deleteExtra(l_nodes, PlcNode, 'node_id', 'node_id')
 199         deleteExtra(l_nodes, HistoryNodeRecord, 'plc_nodeid', 'node_id')
 200         deleteExtra(l_nodes, PlcNode, 'hostname', 'hostname')
 201         deleteExtra(l_nodes, HistoryNodeRecord, 'hostname', 'hostname')
 202         deleteExtra(l_nodes, FindbadNodeRecord, 'hostname', 'hostname')
 203         session.flush()
 204
 205         init()
 206
 207         return
 208
 209 if __name__ == '__main__':
 210         sync()
 211 else:
 212         init()