From 972db1ffcae67b46a19c7eb5d53fecff9f67606f Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Tue, 30 Sep 2014 11:31:56 -0700 Subject: [PATCH] rename sanity_check to consistency_check; run it from an external program rather than as part of observer --- planetstack/hpc_observer/fsck.py | 79 +++++++++++++++++++ .../hpc_observer/steps/sync_cdnprefix.py | 13 ++- .../steps/sync_contentprovider.py | 11 ++- .../hpc_observer/steps/sync_originserver.py | 11 ++- .../steps/sync_serviceprovider.py | 11 ++- 5 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 planetstack/hpc_observer/fsck.py diff --git a/planetstack/hpc_observer/fsck.py b/planetstack/hpc_observer/fsck.py new file mode 100644 index 0000000..82258a7 --- /dev/null +++ b/planetstack/hpc_observer/fsck.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +import argparse +import imp +import inspect +import os +import sys +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings") +sys.path.append("/opt/planetstack") +from planetstack.config import Config +from util.logger import Logger, logging +from observer.syncstep import SyncStep + +try: + from django import setup as django_setup # django 1.7 +except: + django_setup = False + +logger = Logger(level=logging.INFO) + +class PlanetStackConsistencyCheck: + def __init__(self): + self.sync_steps = [] + self.load_sync_step_modules() + + def load_sync_step_modules(self, step_dir=None): + if step_dir is None: + if hasattr(Config(), "observer_steps_dir"): + step_dir = Config().observer_steps_dir + else: + step_dir = "/opt/planetstack/observer/steps" + + for fn in os.listdir(step_dir): + pathname = os.path.join(step_dir,fn) + if os.path.isfile(pathname) and fn.endswith(".py") and (fn!="__init__.py"): + module = imp.load_source(fn[:-3],pathname) + for classname in dir(module): + c = getattr(module, classname, None) + + # make sure 'c' is a descendent of SyncStep and has a + # provides field (this eliminates the abstract base classes + # since they don't have a provides) + + if inspect.isclass(c) and issubclass(c, SyncStep) and hasattr(c,"provides") and (c not in self.sync_steps): + self.sync_steps.append(c) + logger.info('loaded sync steps: %s' % ",".join([x.__name__ for x in self.sync_steps])) + + def run(self): + updated = True + while updated: + updated = False + + for step in self.sync_steps: + if hasattr(step, "consistency_check"): + updated = updated or step(driver=None).consistency_check() + + if updated: + logger.info('re-running consistency checks because something changed') + +def main(): + if not "-C" in sys.argv: + print >> sys.stderr, "You probably wanted to use -C /opt/planetstack/hpc_observer/hpc_observer_config" + + # Generate command line parser + parser = argparse.ArgumentParser(usage='%(prog)s [options]') + # smbaker: util/config.py parses sys.argv[] directly to get config file name; include the option here to avoid + # throwing unrecognized argument exceptions + parser.add_argument('-C', '--config', dest='config_file', action='store', default="/opt/planetstack/plstackapi_config", + help='Name of config file.') + args = parser.parse_args() + + if django_setup: # 1.7 + django_setup() + + cc = PlanetStackConsistencyCheck() + cc.run() + +if __name__ == '__main__': + main() + diff --git a/planetstack/hpc_observer/steps/sync_cdnprefix.py b/planetstack/hpc_observer/steps/sync_cdnprefix.py index 55b2326..dfa6ef8 100644 --- a/planetstack/hpc_observer/steps/sync_cdnprefix.py +++ b/planetstack/hpc_observer/steps/sync_cdnprefix.py @@ -25,11 +25,14 @@ class SyncCDNPrefix(SyncStep, HpcLibrary): HpcLibrary.__init__(self) def fetch_pending(self, deleted): - self.sanity_check() + #self.consistency_check() return SyncStep.fetch_pending(self, deleted) - def sanity_check(self): + def consistency_check(self): + # set to true if something changed + result=False + # sanity check to make sure our PS objects have CMI objects behind them all_p_ids = [x["cdn_prefix_id"] for x in self.client.onev.ListAll("CDNPrefix")] @@ -48,10 +51,14 @@ class SyncCDNPrefix(SyncStep, HpcLibrary): logger.info("CDN Prefix %s was not found on CMI" % p.cdn_prefix_id) p.cdn_prefix_id=None p.save() + result = True if (p.defaultOriginServer!=None) and (all_origins.get(p.cdn_prefix_id,None) != p.defaultOriginServer.url): logger.info("CDN Prefix %s does not have default origin server on CMI" % str(p)) - p.save() # this will set updated>enacted and force run + p.save() # this will set updated>enacted and force observer to re-sync + result = True + + return result def sync_record(self, cp): logger.info("sync'ing cdn prefix %s" % str(cp)) diff --git a/planetstack/hpc_observer/steps/sync_contentprovider.py b/planetstack/hpc_observer/steps/sync_contentprovider.py index cebe186..b3c2f66 100644 --- a/planetstack/hpc_observer/steps/sync_contentprovider.py +++ b/planetstack/hpc_observer/steps/sync_contentprovider.py @@ -25,11 +25,14 @@ class SyncContentProvider(SyncStep, HpcLibrary): HpcLibrary.__init__(self) def fetch_pending(self, deleted): - self.sanity_check() + #self.consistency_check() return SyncStep.fetch_pending(self, deleted) - def sanity_check(self): + def consistency_check(self): + # set to true if something changed + result=False + # sanity check to make sure our PS objects have CMI objects behind them all_cp_ids = [x["content_provider_id"] for x in self.client.onev.ListAll("ContentProvider")] for cp in ContentProvider.objects.all(): @@ -37,11 +40,13 @@ class SyncContentProvider(SyncStep, HpcLibrary): logger.info("Content provider %s was not found on CMI" % cp.content_provider_id) cp.content_provider_id=None cp.save() + result = True + + return result def sync_record(self, cp): logger.info("sync'ing content provider %s" % str(cp)) account_name = self.make_account_name(cp.name) - print "XXX", cp.name, account_name if (not cp.serviceProvider) or (not cp.serviceProvider.service_provider_id): return diff --git a/planetstack/hpc_observer/steps/sync_originserver.py b/planetstack/hpc_observer/steps/sync_originserver.py index 92ac0f4..d18c672 100644 --- a/planetstack/hpc_observer/steps/sync_originserver.py +++ b/planetstack/hpc_observer/steps/sync_originserver.py @@ -1,6 +1,7 @@ import os import sys import base64 + from django.db.models import F, Q from planetstack.config import Config from observer.syncstep import SyncStep @@ -25,11 +26,14 @@ class SyncOriginServer(SyncStep, HpcLibrary): HpcLibrary.__init__(self) def fetch_pending(self, deleted): - self.sanity_check() + #self.consistency_check() return SyncStep.fetch_pending(self, deleted) - def sanity_check(self): + def consistency_check(self): + # set to true if something changed + result=False + # sanity check to make sure our PS objects have CMI objects behind them all_ors_ids = [x["origin_server_id"] for x in self.client.onev.ListAll("OriginServer")] for ors in OriginServer.objects.all(): @@ -40,6 +44,9 @@ class SyncOriginServer(SyncStep, HpcLibrary): logger.info("origin server %s was not found on CMI" % ors.origin_server_id) ors.origin_server_id=None ors.save() + result = True + + return result def sync_record(self, ors): logger.info("sync'ing origin server %s" % str(ors)) diff --git a/planetstack/hpc_observer/steps/sync_serviceprovider.py b/planetstack/hpc_observer/steps/sync_serviceprovider.py index 78fd196..ca0ba6a 100644 --- a/planetstack/hpc_observer/steps/sync_serviceprovider.py +++ b/planetstack/hpc_observer/steps/sync_serviceprovider.py @@ -25,11 +25,14 @@ class SyncServiceProvider(SyncStep, HpcLibrary): HpcLibrary.__init__(self) def fetch_pending(self, deleted): - self.sanity_check() + #self.consistency_check() return SyncStep.fetch_pending(self, deleted) - def sanity_check(self): + def consistency_check(self): + # set to true if something changed + result=False + # sanity check to make sure our PS objects have CMI objects behind them all_sp_ids = [x["service_provider_id"] for x in self.client.onev.ListAll("ServiceProvider")] for sp in ServiceProvider.objects.all(): @@ -37,11 +40,13 @@ class SyncServiceProvider(SyncStep, HpcLibrary): logger.info("Service provider %s was not found on CMI" % sp.service_provider_id) sp.service_provider_id=None sp.save() + result = True + + return result def sync_record(self, sp): logger.info("sync'ing service provider %s" % str(sp)) account_name = self.make_account_name(sp.name) - print "XXX", sp.name, account_name sp_dict = {"account": account_name, "name": sp.name, "enabled": sp.enabled} if not sp.service_provider_id: id = self.client.onev.Create("ServiceProvider", sp_dict) -- 2.43.0