From 04c94ad7d56a164d1f1a5d05c3b5ced5d792bae2 Mon Sep 17 00:00:00 2001 From: Sapan Bhatia Date: Mon, 2 Sep 2013 18:00:28 -0400 Subject: [PATCH] Wide range of debugging changes --- planetstack/observer/event_loop.py | 52 ++- planetstack/observer/steps/__init__.py | 20 +- .../observer/steps/sync_external_routes.py | 16 +- .../observer/steps/sync_network_slivers.py | 108 ++--- planetstack/observer/steps/sync_networks.py | 67 +-- .../observer/steps/sync_site_privileges.py | 4 +- planetstack/observer/steps/sync_sites.py | 2 + .../observer/steps/sync_slice_memberships.py | 5 +- planetstack/observer/steps/sync_slices.py | 96 ++-- planetstack/observer/steps/sync_sliver_ips.py | 6 +- planetstack/observer/steps/sync_slivers.py | 18 +- planetstack/observer/steps/sync_users.py | 46 +- planetstack/observer/syncstep.py | 5 +- planetstack/planetstack/config.py | 420 +++++++++--------- 14 files changed, 447 insertions(+), 418 deletions(-) diff --git a/planetstack/observer/event_loop.py b/planetstack/observer/event_loop.py index 6803398..671bdc3 100644 --- a/planetstack/observer/event_loop.py +++ b/planetstack/observer/event_loop.py @@ -12,6 +12,7 @@ from openstack.manager import OpenStackManager from util.logger import Logger, logging, logger #from timeout import timeout from planetstack.config import Config +from observer.steps import * debug_mode = False @@ -42,6 +43,7 @@ def toposort(g, steps): order = [] marked = [] + while sources: n = sources.pop() try: @@ -51,19 +53,16 @@ def toposort(g, steps): marked.append(m) except KeyError: pass - if (n in steps): - order.append(n) - + order.append(n) return order class PlanetStackObserver: - sync_steps = ['SyncNetworks','SyncNetworkSlivers','SyncSites','SyncSitePrivileges','SyncSlices','SyncSliceMemberships','SyncSlivers','SyncSliverIps'] + sync_steps = [SyncNetworks,SyncNetworkSlivers,SyncSites,SyncSitePrivileges,SyncSlices,SyncSliceMemberships,SyncSlivers,SyncSliverIps] def __init__(self): # The Condition object that gets signalled by Feefie events self.load_sync_steps() self.event_cond = threading.Condition() - self.load_enacted() def wait_for_event(self, timeout): self.event_cond.acquire() @@ -77,42 +76,52 @@ class PlanetStackObserver: self.event_cond.release() def load_sync_steps(self): - dep_path = Config().pl_dependency_path + dep_path = Config().observer_dependency_path try: # This contains dependencies between records, not sync steps self.model_dependency_graph = json.loads(open(dep_path).read()) except Exception,e: raise e - backend_path = Config().backend_dependency_path + backend_path = Config().observer_backend_dependency_path try: # This contains dependencies between backend records self.backend_dependency_graph = json.loads(open(backend_path).read()) except Exception,e: - raise e + # We can work without a backend graph + self.backend_dependency_graph = {} provides_dict = {} - for s in sync_steps: + for s in self.sync_steps: for m in s.provides: - provides_dict[m]=s.__name__ + try: + provides_dict[m.__name__].append(s.__name__) + except KeyError: + provides_dict[m.__name__]=[s.__name__] + step_graph = {} - for k,v in model_dependency_graph.iteritems(): + for k,v in self.model_dependency_graph.iteritems(): try: - source = provides_dict[k] - for m in v: - try: - dest = provides_dict[m] - except KeyError: - pass - # no deps, pass - step_graph[source]=dest + for source in provides_dict[k]: + for m in v: + try: + for dest in provides_dict[m]: + # no deps, pass + try: + step_graph[source].append(dest) + except: + step_graph[source]=[dest] + except KeyError: + pass except KeyError: pass # no dependencies, pass - if (backend_dependency_graph): + import pdb + pdb.set_trace() + if (self.backend_dependency_graph): backend_dict = {} for s in sync_steps: for m in s.serves: @@ -135,7 +144,8 @@ class PlanetStackObserver: dependency_graph = step_graph - self.ordered_steps = toposort(dependency_graph, steps) + self.ordered_steps = toposort(dependency_graph, self.sync_steps) + print "Order of steps=",self.ordered_steps self.load_run_times() diff --git a/planetstack/observer/steps/__init__.py b/planetstack/observer/steps/__init__.py index a239587..6d7adb8 100644 --- a/planetstack/observer/steps/__init__.py +++ b/planetstack/observer/steps/__init__.py @@ -1,10 +1,10 @@ -from .syncexternalroutes import SyncExternalRoutes -from .syncnetworkslivers import SyncNetworkSlivers -from .syncnetworks import SyncNetworks -from .syncsiteprivileges import SyncSitePrivileges -from .syncsites import SyncSites -from .syncslicememberships import SyncSliceMemberships -from .syncslices import SyncSlices -from .syncsliverips import SyncSliverIps -from .syncslivers import SyncSlivers -from .syncusers import SyncUsers +from .sync_external_routes import SyncExternalRoutes +from .sync_network_slivers import SyncNetworkSlivers +from .sync_networks import SyncNetworks +from .sync_site_privileges import SyncSitePrivileges +from .sync_sites import SyncSites +from .sync_slice_memberships import SyncSliceMemberships +from .sync_slices import SyncSlices +from .sync_sliver_ips import SyncSliverIps +from .sync_slivers import SyncSlivers +from .sync_users import SyncUsers diff --git a/planetstack/observer/steps/sync_external_routes.py b/planetstack/observer/steps/sync_external_routes.py index fda66bd..6c22c8b 100644 --- a/planetstack/observer/steps/sync_external_routes.py +++ b/planetstack/observer/steps/sync_external_routes.py @@ -1,16 +1,20 @@ import os import base64 from planetstack.config import Config +from observer.syncstep import SyncStep class SyncExternalRoutes(SyncStep): # XXX what does this provide? requested_interval = 86400 # This step is slow like a pig. Let's run it infrequently + def __init__(self): + pass + def call(self): routes = self.manager.driver.get_external_routes() - subnets = self.manager.driver.shell.quantum.list_subnets()['subnets'] - for subnet in subnets: - try: - self.manager.driver.add_external_route(subnet, routes) - except: - logger.log_exc("failed to add external route for subnet %s" % subnet) + subnets = self.manager.driver.shell.quantum.list_subnets()['subnets'] + for subnet in subnets: + try: + self.manager.driver.add_external_route(subnet, routes) + except: + logger.log_exc("failed to add external route for subnet %s" % subnet) diff --git a/planetstack/observer/steps/sync_network_slivers.py b/planetstack/observer/steps/sync_network_slivers.py index 3870924..9e24fae 100644 --- a/planetstack/observer/steps/sync_network_slivers.py +++ b/planetstack/observer/steps/sync_network_slivers.py @@ -1,73 +1,75 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.network import * class SyncNetworkSlivers(OpenStackSyncStep): requested_interval = 3600 provides=[NetworkSliver] def call(self): - networkSlivers = NetworkSliver.objects.all() - networkSlivers_by_id = {} - networkSlivers_by_port = {} - for networkSliver in networkSlivers: - networkSlivers_by_id[networkSliver.id] = networkSliver - networkSlivers_by_port[networkSliver.port_id] = networkSliver + networkSlivers = NetworkSliver.objects.all() + networkSlivers_by_id = {} + networkSlivers_by_port = {} + for networkSliver in networkSlivers: + networkSlivers_by_id[networkSliver.id] = networkSliver + networkSlivers_by_port[networkSliver.port_id] = networkSliver - networks = Network.objects.all() - networks_by_id = {} - for network in networks: - networks_by_id[network.network_id] = network + networks = Network.objects.all() + networks_by_id = {} + for network in networks: + networks_by_id[network.network_id] = network - slivers = Sliver.objects.all() - slivers_by_instance_id = {} - for sliver in slivers: - slivers_by_instance_id[sliver.instance_id] = sliver + slivers = Sliver.objects.all() + slivers_by_instance_id = {} + for sliver in slivers: + slivers_by_instance_id[sliver.instance_id] = sliver - ports = self.manager.driver.shell.quantum.list_ports()["ports"] - for port in ports: - if port["id"] in networkSlivers_by_port: - # we already have it - print "already accounted for port", port["id"] - continue + ports = self.manager.driver.shell.quantum.list_ports()["ports"] + for port in ports: + if port["id"] in networkSlivers_by_port: + # we already have it + print "already accounted for port", port["id"] + continue - if port["device_owner"] != "compute:nova": - # we only want the ports that connect to instances - continue + if port["device_owner"] != "compute:nova": + # we only want the ports that connect to instances + continue - network = networks_by_id.get(port['network_id'], None) - if not network: - #print "no network for port", port["id"], "network", port["network_id"] - continue + network = networks_by_id.get(port['network_id'], None) + if not network: + #print "no network for port", port["id"], "network", port["network_id"] + continue - sliver = slivers_by_instance_id.get(port['device_id'], None) - if not sliver: - print "no sliver for port", port["id"], "device_id", port['device_id'] - continue + sliver = slivers_by_instance_id.get(port['device_id'], None) + if not sliver: + print "no sliver for port", port["id"], "device_id", port['device_id'] + continue - if network.template.sharedNetworkId is not None: - # If it's a shared network template, then more than one network - # object maps to the quantum network. We have to do a whole bunch - # of extra work to find the right one. - networks = network.template.network_set.all() - network = None - for candidate_network in networks: - if (candidate_network.owner == sliver.slice): - print "found network", candidate_network - network = candidate_network + if network.template.sharedNetworkId is not None: + # If it's a shared network template, then more than one network + # object maps to the quantum network. We have to do a whole bunch + # of extra work to find the right one. + networks = network.template.network_set.all() + network = None + for candidate_network in networks: + if (candidate_network.owner == sliver.slice): + print "found network", candidate_network + network = candidate_network - if not network: - print "failed to find the correct network for a shared template for port", port["id"], "network", port["network_id"] - continue + if not network: + print "failed to find the correct network for a shared template for port", port["id"], "network", port["network_id"] + continue - if not port["fixed_ips"]: - print "port", port["id"], "has no fixed_ips" - continue + if not port["fixed_ips"]: + print "port", port["id"], "has no fixed_ips" + continue -# print "XXX", port +# print "XXX", port - ns = NetworkSliver(network=network, - sliver=sliver, - ip=port["fixed_ips"][0]["ip_address"], - port_id=port["id"]) - ns.save() + ns = NetworkSliver(network=network, + sliver=sliver, + ip=port["fixed_ips"][0]["ip_address"], + port_id=port["id"]) + ns.save() diff --git a/planetstack/observer/steps/sync_networks.py b/planetstack/observer/steps/sync_networks.py index eb8fe86..e64f0a4 100644 --- a/planetstack/observer/steps/sync_networks.py +++ b/planetstack/observer/steps/sync_networks.py @@ -1,6 +1,8 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.network import * class SyncNetworks(OpenStackSyncStep): provides=[Network] @@ -8,44 +10,43 @@ class SyncNetworks(OpenStackSyncStep): def save_network(self, network): if not network.network_id: - if network.template.sharedNetworkName: - network.network_id = network.template.sharedNetworkId - (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id) - else: - network_name = network.name + if network.template.sharedNetworkName: + network.network_id = network.template.sharedNetworkId + (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id) + else: + network_name = network.name - # create network - os_network = self.driver.create_network(network_name, shared=True) - network.network_id = os_network['id'] + # create network + os_network = self.driver.create_network(network_name, shared=True) + network.network_id = os_network['id'] - # create router - router = self.driver.create_router(network_name) - network.router_id = router['id'] + # create router + router = self.driver.create_router(network_name) + network.router_id = router['id'] - # create subnet - next_subnet = self.get_next_subnet() - cidr = str(next_subnet.cidr) - ip_version = next_subnet.version - start = str(next_subnet[2]) - end = str(next_subnet[-2]) - subnet = self.driver.create_subnet(name=network_name, - network_id = network.network_id, - cidr_ip = cidr, - ip_version = ip_version, - start = start, - end = end) - network.subnet = cidr - network.subnet_id = subnet['id'] + # create subnet + next_subnet = self.get_next_subnet() + cidr = str(next_subnet.cidr) + ip_version = next_subnet.version + start = str(next_subnet[2]) + end = str(next_subnet[-2]) + subnet = self.driver.create_subnet(name=network_name, + network_id = network.network_id, + cidr_ip = cidr, + ip_version = ip_version, + start = start, + end = end) + network.subnet = cidr + network.subnet_id = subnet['id'] - @require_enabled def sync_record(self, site): if network.owner and network.owner.creator: - try: - # update manager context - self.driver.init_caller(network.owner.creator, network.owner.name) - self.save_network(network) - logger.info("saved network: %s" % (network)) - except Exception,e: - logger.log_exc("save network failed: %s" % network) + try: + # update manager context + self.driver.init_caller(network.owner.creator, network.owner.name) + self.save_network(network) + logger.info("saved network: %s" % (network)) + except Exception,e: + logger.log_exc("save network failed: %s" % network) raise e diff --git a/planetstack/observer/steps/sync_site_privileges.py b/planetstack/observer/steps/sync_site_privileges.py index 273fd33..ac0dbac 100644 --- a/planetstack/observer/steps/sync_site_privileges.py +++ b/planetstack/observer/steps/sync_site_privileges.py @@ -1,13 +1,15 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.site import * class SyncSitePrivileges(OpenStackSyncStep): requested_interval=0 provides=[SitePrivilege] def sync_record(self, user): - if site_priv.user.kuser_id and site_priv.site.tenant_id: + if site_priv.user.kuser_id and site_priv.site.tenant_id: self.driver.add_user_role(site_priv.user.kuser_id, site_priv.site.tenant_id, site_priv.role.role_type) diff --git a/planetstack/observer/steps/sync_sites.py b/planetstack/observer/steps/sync_sites.py index 59f32e6..1f7a0f8 100644 --- a/planetstack/observer/steps/sync_sites.py +++ b/planetstack/observer/steps/sync_sites.py @@ -1,6 +1,8 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.site import Site class SyncSites(OpenStackSyncStep): provides=[Site] diff --git a/planetstack/observer/steps/sync_slice_memberships.py b/planetstack/observer/steps/sync_slice_memberships.py index c97bd49..66953f1 100644 --- a/planetstack/observer/steps/sync_slice_memberships.py +++ b/planetstack/observer/steps/sync_slice_memberships.py @@ -1,6 +1,8 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.slice import * class SyncSliceMemberships(OpenStackSyncStep): requested_interval=0 @@ -9,5 +11,4 @@ class SyncSliceMemberships(OpenStackSyncStep): if slice_memb.user.kuser_id and slice_memb.slice.tenant_id: self.driver.add_user_role(slice_memb.user.kuser_id, slice_memb.slice.tenant_id, - slice_memb.role.role_type) - + slice_memb.role.role_type) diff --git a/planetstack/observer/steps/sync_slices.py b/planetstack/observer/steps/sync_slices.py index fb787db..81ed925 100644 --- a/planetstack/observer/steps/sync_slices.py +++ b/planetstack/observer/steps/sync_slices.py @@ -1,56 +1,58 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.slice import Slice class SyncSlices(OpenStackSyncStep): provides=[Slice] requested_interval=0 def sync_record(self, slice): if not slice.tenant_id: - nova_fields = {'tenant_name': slice.name, - 'description': slice.description, - 'enabled': slice.enabled} - tenant = self.driver.create_tenant(**nova_fields) - slice.tenant_id = tenant.id - - # XXX give caller an admin role at the tenant they've created - self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin') - - # refresh credentials using this tenant - self.driver.shell.connect(username=self.driver.shell.keystone.username, - password=self.driver.shell.keystone.password, - tenant=tenant.name) - - # create network - network = self.driver.create_network(slice.name) - slice.network_id = network['id'] - - # create router - router = self.driver.create_router(slice.name) - slice.router_id = router['id'] - - # create subnet - next_subnet = self.get_next_subnet() - cidr = str(next_subnet.cidr) - ip_version = next_subnet.version - start = str(next_subnet[2]) - end = str(next_subnet[-2]) - subnet = self.driver.create_subnet(name=slice.name, - network_id = network['id'], - cidr_ip = cidr, - ip_version = ip_version, - start = start, - end = end) - slice.subnet_id = subnet['id'] - # add subnet as interface to slice's router - self.driver.add_router_interface(router['id'], subnet['id']) - # add external route - self.driver.add_external_route(subnet) - - - if slice.id and slice.tenant_id: - self.driver.update_tenant(slice.tenant_id, - description=slice.description, - enabled=slice.enabled) - - slice.save() + nova_fields = {'tenant_name': slice.name, + 'description': slice.description, + 'enabled': slice.enabled} + tenant = self.driver.create_tenant(**nova_fields) + slice.tenant_id = tenant.id + + # XXX give caller an admin role at the tenant they've created + self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin') + + # refresh credentials using this tenant + self.driver.shell.connect(username=self.driver.shell.keystone.username, + password=self.driver.shell.keystone.password, + tenant=tenant.name) + + # create network + network = self.driver.create_network(slice.name) + slice.network_id = network['id'] + + # create router + router = self.driver.create_router(slice.name) + slice.router_id = router['id'] + + # create subnet + next_subnet = self.get_next_subnet() + cidr = str(next_subnet.cidr) + ip_version = next_subnet.version + start = str(next_subnet[2]) + end = str(next_subnet[-2]) + subnet = self.driver.create_subnet(name=slice.name, + network_id = network['id'], + cidr_ip = cidr, + ip_version = ip_version, + start = start, + end = end) + slice.subnet_id = subnet['id'] + # add subnet as interface to slice's router + self.driver.add_router_interface(router['id'], subnet['id']) + # add external route + self.driver.add_external_route(subnet) + + + if slice.id and slice.tenant_id: + self.driver.update_tenant(slice.tenant_id, + description=slice.description, + enabled=slice.enabled) + + slice.save() diff --git a/planetstack/observer/steps/sync_sliver_ips.py b/planetstack/observer/steps/sync_sliver_ips.py index 93f8cb3..d69fd5d 100644 --- a/planetstack/observer/steps/sync_sliver_ips.py +++ b/planetstack/observer/steps/sync_sliver_ips.py @@ -1,6 +1,8 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.sliver import Sliver class SyncSliverIps(OpenStackSyncStep): provides=[Sliver] @@ -13,11 +15,11 @@ class SyncSliverIps(OpenStackSyncStep): self.manager.init_admin(tenant=sliver.slice.name) servers = self.manager.driver.shell.nova.servers.findall(id=sliver.instance_id) if not servers: - continue + return server = servers[0] ips = server.addresses.get(sliver.slice.name, []) if not ips: - continue + return sliver.ip = ips[0]['addr'] sliver.save() logger.info("saved sliver ip: %s %s" % (sliver, ips[0])) diff --git a/planetstack/observer/steps/sync_slivers.py b/planetstack/observer/steps/sync_slivers.py index 46f78fd..adab39d 100644 --- a/planetstack/observer/steps/sync_slivers.py +++ b/planetstack/observer/steps/sync_slivers.py @@ -1,6 +1,8 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.sliver import Sliver class SyncSlivers(OpenStackSyncStep): provides=[Sliver] @@ -13,15 +15,15 @@ class SyncSlivers(OpenStackSyncStep): pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key] pubkeys.append(sliver.creator.public_key) instance = self.driver.spawn_instance(name=sliver.name, - key_name = sliver.creator.keyname, - image_id = sliver.image.image_id, - hostname = sliver.node.name, - pubkeys = pubkeys, - nics = nics ) + key_name = sliver.creator.keyname, + image_id = sliver.image.image_id, + hostname = sliver.node.name, + pubkeys = pubkeys, + nics = nics ) sliver.instance_id = instance.id sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name') - if sliver.instance_id and ("numberCores" in sliver.changed_fields): - self.driver.update_instance_metadata(sliver.instance_id, {"cpu_cores": str(sliver.numberCores)}) + if sliver.instance_id and ("numberCores" in sliver.changed_fields): + self.driver.update_instance_metadata(sliver.instance_id, {"cpu_cores": str(sliver.numberCores)}) - sliver.save() + sliver.save() diff --git a/planetstack/observer/steps/sync_users.py b/planetstack/observer/steps/sync_users.py index 1e93034..3f509ef 100644 --- a/planetstack/observer/steps/sync_users.py +++ b/planetstack/observer/steps/sync_users.py @@ -1,33 +1,35 @@ import os import base64 from planetstack.config import Config +from observer.openstacksyncstep import OpenStackSyncStep +from core.models.user import User class SyncUsers(OpenStackSyncStep): provides=[User] requested_interval=0 def sync_record(self, user): - name = user.email[:user.email.find('@')] - user_fields = {'name': name, - 'email': user.email, - 'password': hashlib.md5(user.password).hexdigest()[:6], - 'enabled': True} - if not user.kuser_id: - keystone_user = self.driver.create_user(**user_fields) - user.kuser_id = keystone_user.id - else: - self.driver.update_user(user.kuser_id, user_fields) + name = user.email[:user.email.find('@')] + user_fields = {'name': name, + 'email': user.email, + 'password': hashlib.md5(user.password).hexdigest()[:6], + 'enabled': True} + if not user.kuser_id: + keystone_user = self.driver.create_user(**user_fields) + user.kuser_id = keystone_user.id + else: + self.driver.update_user(user.kuser_id, user_fields) - if user.site: - self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user') - if user.is_admin: - self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin') - else: - # may have admin role so attempt to remove it - self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin') + if user.site: + self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user') + if user.is_admin: + self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin') + else: + # may have admin role so attempt to remove it + self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin') - if user.public_key: - self.init_caller(user, user.site.login_base) - self.save_key(user.public_key, user.keyname) - self.init_admin() + if user.public_key: + self.init_caller(user, user.site.login_base) + self.save_key(user.public_key, user.keyname) + self.init_admin() - user.save() + user.save() diff --git a/planetstack/observer/syncstep.py b/planetstack/observer/syncstep.py index f3eb4ba..68e9f99 100644 --- a/planetstack/observer/syncstep.py +++ b/planetstack/observer/syncstep.py @@ -39,14 +39,13 @@ class SyncStep: return Sliver.objects.filter(ip=None) def check_dependencies(self, obj): - for dep in dependencies: + for dep in self.dependencies: peer_object = getattr(obj, dep.name.lowercase()) if (peer_object.pk==dep.pk): raise DependencyFailed - def call(self, failed=failed_objects): + def call(self, failed=[]): pending = self.fetch_pending() - failed = [] for o in pending: if (not self.depends_on(o, failed)): try: diff --git a/planetstack/planetstack/config.py b/planetstack/planetstack/config.py index 7927803..b9abd3a 100644 --- a/planetstack/planetstack/config.py +++ b/planetstack/planetstack/config.py @@ -13,228 +13,228 @@ default_config = \ """ def isbool(v): - return v.lower() in ("true", "false") + return v.lower() in ("true", "false") def str2bool(v): - return v.lower() in ("true", "1") + return v.lower() in ("true", "1") class Config: - def __init__(self, config_file='/opt/planetstack/plstackapi_config'): - self._files = [] - self.config_path = os.path.dirname(config_file) - self.config = ConfigParser.ConfigParser() - self.filename = config_file - if not os.path.isfile(self.filename): - self.create(self.filename) - self.load(self.filename) + def __init__(self, config_file='/opt/planetstack/plstackapi_config'): + self._files = [] + self.config_path = os.path.dirname(config_file) + self.config = ConfigParser.ConfigParser() + self.filename = config_file + if not os.path.isfile(self.filename): + self.create(self.filename) + self.load(self.filename) - def _header(self): - header = """ + def _header(self): + header = """ DO NOT EDIT. This file was automatically generated at %s from: %s """ % (time.asctime(), os.linesep.join(self._files)) - # Get rid of the surrounding newlines - return header.strip().split(os.linesep) - - def create(self, filename): - if not os.path.exists(os.path.dirname(filename)): - os.makedirs(os.path.dirname(filename)) - configfile = open(filename, 'w') - configfile.write(default_config) - configfile.close() - - - def load(self, filename): - if filename: - try: - self.config.read(filename) - except ConfigParser.MissingSectionHeaderError: - if filename.endswith('.xml'): - self.load_xml(filename) - else: - self.load_shell(filename) - self._files.append(filename) - self.set_attributes() - - def load_xml(self, filename): - xml = XML(filename) - categories = xml.xpath('//configuration/variables/category') - for category in categories: - section_name = category.get('id') - if not self.config.has_section(section_name): - self.config.add_section(section_name) - options = category.xpath('./variablelist/variable') - for option in options: - option_name = option.get('id') - value = option.xpath('./value')[0].text - if not value: - value = "" - self.config.set(section_name, option_name, value) - - def load_shell(self, filename): - f = open(filename, 'r') - for line in f: - try: - if line.startswith('#'): - continue - parts = line.strip().split("=") - if len(parts) < 2: - continue - option = parts[0] - value = parts[1].replace('"', '').replace("'","") - section, var = self.locate_varname(option, strict=False) - if section and var: - self.set(section, var, value) - except: - pass - f.close() - - def locate_varname(self, varname, strict=True): - varname = varname.lower() - sections = self.config.sections() - section_name = "" - var_name = "" - for section in sections: - if varname.startswith(section.lower()) and len(section) > len(section_name): - section_name = section.lower() - var_name = varname.replace(section_name, "")[1:] - if strict and not self.config.has_option(section_name, var_name): - raise ConfigParser.NoOptionError(var_name, section_name) - return (section_name, var_name) - - def set_attributes(self): - sections = self.config.sections() - for section in sections: - for item in self.config.items(section): - name = "%s_%s" % (section, item[0]) - value = item[1] - if isbool(value): - value = str2bool(value) - elif value.isdigit(): - value = int(value) - setattr(self, name, value) - setattr(self, name.upper(), value) - - - def verify(self, config1, config2, validate_method): - return True - - def validate_type(self, var_type, value): - return True - - @staticmethod - def is_xml(config_file): - try: - x = Xml(config_file) - return True - except: - return False - - @staticmethod - def is_ini(config_file): - try: - c = ConfigParser.ConfigParser() - c.read(config_file) - return True - except ConfigParser.MissingSectionHeaderError: - return False - - - def dump(self, sections = []): - sys.stdout.write(output_python()) - - def output_python(self, encoding = "utf-8"): - buf = codecs.lookup(encoding)[3](StringIO()) - buf.writelines(["# " + line + os.linesep for line in self._header()]) - - for section in self.sections(): - buf.write("[%s]%s" % (section, os.linesep)) - for (name,value) in self.items(section): - buf.write("%s=%s%s" % (name,value,os.linesep)) - buf.write(os.linesep) - return buf.getvalue() - - def output_shell(self, show_comments = True, encoding = "utf-8"): - """ - Return variables as a shell script. - """ - - buf = codecs.lookup(encoding)[3](StringIO()) - buf.writelines(["# " + line + os.linesep for line in self._header()]) - - for section in self.sections(): - for (name,value) in self.items(section): - # bash does not have the concept of NULL - if value: - option = "%s_%s" % (section.upper(), name.upper()) - if isbool(value): - value = str(str2bool(value)) - elif not value.isdigit(): - value = '"%s"' % value - buf.write(option + "=" + value + os.linesep) - return buf.getvalue() - - def output_php(self, encoding = "utf-8"): - """ - Return variables as a PHP script. - """ - - buf = codecs.lookup(encoding)[3](StringIO()) - buf.write("" + os.linesep) - - return buf.getvalue() - - def output_xml(self, encoding = "utf-8"): - pass - - def output_variables(self, encoding="utf-8"): - """ - Return list of all variable names. - """ - - buf = codecs.lookup(encoding)[3](StringIO()) - for section in self.sections(): - for (name,value) in self.items(section): - option = "%s_%s" % (section,name) - buf.write(option + os.linesep) - - return buf.getvalue() - pass - - def write(self, filename=None): - if not filename: - filename = self.filename - configfile = open(filename, 'w') - self.config.write(configfile) - - def save(self, filename=None): - self.write(filename) - - def __getattr__(self, attr): - return getattr(self.config, attr) + # Get rid of the surrounding newlines + return header.strip().split(os.linesep) + + def create(self, filename): + if not os.path.exists(os.path.dirname(filename)): + os.makedirs(os.path.dirname(filename)) + configfile = open(filename, 'w') + configfile.write(default_config) + configfile.close() + + + def load(self, filename): + if filename: + try: + self.config.read(filename) + except ConfigParser.MissingSectionHeaderError: + if filename.endswith('.xml'): + self.load_xml(filename) + else: + self.load_shell(filename) + self._files.append(filename) + self.set_attributes() + + def load_xml(self, filename): + xml = XML(filename) + categories = xml.xpath('//configuration/variables/category') + for category in categories: + section_name = category.get('id') + if not self.config.has_section(section_name): + self.config.add_section(section_name) + options = category.xpath('./variablelist/variable') + for option in options: + option_name = option.get('id') + value = option.xpath('./value')[0].text + if not value: + value = "" + self.config.set(section_name, option_name, value) + + def load_shell(self, filename): + f = open(filename, 'r') + for line in f: + try: + if line.startswith('#'): + continue + parts = line.strip().split("=") + if len(parts) < 2: + continue + option = parts[0] + value = parts[1].replace('"', '').replace("'","") + section, var = self.locate_varname(option, strict=False) + if section and var: + self.set(section, var, value) + except: + pass + f.close() + + def locate_varname(self, varname, strict=True): + varname = varname.lower() + sections = self.config.sections() + section_name = "" + var_name = "" + for section in sections: + if varname.startswith(section.lower()) and len(section) > len(section_name): + section_name = section.lower() + var_name = varname.replace(section_name, "")[1:] + if strict and not self.config.has_option(section_name, var_name): + raise ConfigParser.NoOptionError(var_name, section_name) + return (section_name, var_name) + + def set_attributes(self): + sections = self.config.sections() + for section in sections: + for item in self.config.items(section): + name = "%s_%s" % (section, item[0]) + value = item[1] + if isbool(value): + value = str2bool(value) + elif value.isdigit(): + value = int(value) + setattr(self, name, value) + setattr(self, name.upper(), value) + + + def verify(self, config1, config2, validate_method): + return True + + def validate_type(self, var_type, value): + return True + + @staticmethod + def is_xml(config_file): + try: + x = Xml(config_file) + return True + except: + return False + + @staticmethod + def is_ini(config_file): + try: + c = ConfigParser.ConfigParser() + c.read(config_file) + return True + except ConfigParser.MissingSectionHeaderError: + return False + + + def dump(self, sections = []): + sys.stdout.write(output_python()) + + def output_python(self, encoding = "utf-8"): + buf = codecs.lookup(encoding)[3](StringIO()) + buf.writelines(["# " + line + os.linesep for line in self._header()]) + + for section in self.sections(): + buf.write("[%s]%s" % (section, os.linesep)) + for (name,value) in self.items(section): + buf.write("%s=%s%s" % (name,value,os.linesep)) + buf.write(os.linesep) + return buf.getvalue() + + def output_shell(self, show_comments = True, encoding = "utf-8"): + """ + Return variables as a shell script. + """ + + buf = codecs.lookup(encoding)[3](StringIO()) + buf.writelines(["# " + line + os.linesep for line in self._header()]) + + for section in self.sections(): + for (name,value) in self.items(section): + # bash does not have the concept of NULL + if value: + option = "%s_%s" % (section.upper(), name.upper()) + if isbool(value): + value = str(str2bool(value)) + elif not value.isdigit(): + value = '"%s"' % value + buf.write(option + "=" + value + os.linesep) + return buf.getvalue() + + def output_php(self, encoding = "utf-8"): + """ + Return variables as a PHP script. + """ + + buf = codecs.lookup(encoding)[3](StringIO()) + buf.write("" + os.linesep) + + return buf.getvalue() + + def output_xml(self, encoding = "utf-8"): + pass + + def output_variables(self, encoding="utf-8"): + """ + Return list of all variable names. + """ + + buf = codecs.lookup(encoding)[3](StringIO()) + for section in self.sections(): + for (name,value) in self.items(section): + option = "%s_%s" % (section,name) + buf.write(option + os.linesep) + + return buf.getvalue() + pass + + def write(self, filename=None): + if not filename: + filename = self.filename + configfile = open(filename, 'w') + self.config.write(configfile) + + def save(self, filename=None): + self.write(filename) + + def __getattr__(self, attr): + return getattr(self.config, attr) if __name__ == '__main__': - filename = None - if len(sys.argv) > 1: - filename = sys.argv[1] - config = Config(filename) - else: - config = Config() - config.dump() + filename = None + if len(sys.argv) > 1: + filename = sys.argv[1] + config = Config(filename) + else: + config = Config() + config.dump() -- 2.43.0