From b30aa088e8b27aae6e2b188f22f8bce5983169f3 Mon Sep 17 00:00:00 2001 From: Scott Baker Date: Fri, 3 Jan 2014 08:36:00 -0800 Subject: [PATCH] hpc observer, wip --- planetstack/hpc_observer/hpc_observer_config | 33 ++++++ planetstack/hpc_observer/hpclib.py | 103 ++++++++++++++++++ planetstack/hpc_observer/run.sh | 1 + planetstack/hpc_observer/start.sh | 1 + .../hpc_observer/steps/sync_hpcservices.py | 89 +++++++++++++++ planetstack/hpc_observer/stop.sh | 1 + planetstack/hpc_observer/upload.sh | 3 + 7 files changed, 231 insertions(+) create mode 100644 planetstack/hpc_observer/hpc_observer_config create mode 100644 planetstack/hpc_observer/hpclib.py create mode 100755 planetstack/hpc_observer/run.sh create mode 100755 planetstack/hpc_observer/start.sh create mode 100644 planetstack/hpc_observer/steps/sync_hpcservices.py create mode 100755 planetstack/hpc_observer/stop.sh create mode 100755 planetstack/hpc_observer/upload.sh diff --git a/planetstack/hpc_observer/hpc_observer_config b/planetstack/hpc_observer/hpc_observer_config new file mode 100644 index 0000000..13bfaaf --- /dev/null +++ b/planetstack/hpc_observer/hpc_observer_config @@ -0,0 +1,33 @@ + +[plc] +name=plc +deployment=VICCI + +[db] +name=planetstack +user=plstackuser +password=2uMDYtJK +host=localhost +port=5432 + +[api] +host=128.112.171.237 +port=8000 +ssl_key=None +ssl_cert=None +ca_ssl_cert=None +ratelimit_enabled=0 +omf_enabled=0 +mail_support_address=support@localhost +nova_enabled=True + +[observer] +dependency_graph=/opt/planetstack/hpc_observer/model-deps +steps_dir=/opt/planetstack/hpc_observer/steps +deleters_dir=/opt/planetstack/hpc_ovserver/deleters +log_file=/var/log/hpc.log +driver=None + +[feefie] +client_id='vicci_dev_central' +user_id='pl' diff --git a/planetstack/hpc_observer/hpclib.py b/planetstack/hpc_observer/hpclib.py new file mode 100644 index 0000000..e91a468 --- /dev/null +++ b/planetstack/hpc_observer/hpclib.py @@ -0,0 +1,103 @@ +import os +import base64 +import sys + +if __name__ == '__main__': + sys.path.append("/opt/planetstack") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings") + +from planetstack.config import Config +from core.models import Service +from hpc.models import HpcService +from requestrouter.models import RequestRouterService +from util.logger import Logger, logging + +logger = Logger(level=logging.INFO) + +class HpcLibrary: + def extract_slice_info(self, service): + """ Produce a dict that describes the slices for the CMI + + slice_coblitz = + service_coblitz = + hostname_coblitz = + hostnames_coblitz = ,,... + + slice_cmi = + ... + """ + + slicenames = {} + slicehosts = {} + for slice in service.service.all(): + name = slice.name + if not ("_" in name): + continue + + if "coblitz" in name: + slicenames["coblitz"] = name + slicehosts["coblitz"] = [sliver.node.name for sliver in slice.slivers.all()] + elif "cmi" in name: + slicenames["cmi"] = name + slicehosts["cmi"] = [sliver.node.name for sliver in slice.slivers.all()] + elif "dnsredir" in name: + slicenames["dnsredir"] = name + slicehosts["dnsredir"] = [sliver.node.name for sliver in slice.slivers.all()] + elif "dnsdemux" in name: + slicenames["dnsdemux"] = name + slicehosts["dnsdemux"] = [sliver.node.name for sliver in slice.slivers.all()] + + base_hrn = None + if "coblitz" in slicenames: + base_hrn = slicenames["coblitz"].split("_")[0] + + mapping = {} + mapping["base_hrn"] = base_hrn + for (k,v) in slicenames.items(): + mapping["slice_" + k] = v + mapping["service_" + k] = v.split("_",1)[1] + for (k,v) in slicehosts.items(): + mapping["hostname_" + k] = v[0] + mapping["hostnames_" + k] = ",".join(v) + + return mapping + + def write_slices_file(self, hpc_service, rr_service): + mapping = self.extract_slice_info(hpc_service) + rr_mapping = self.extract_slice_info(rr_service) + + mapping.update(rr_mapping) + + print mapping + + fn = "/tmp/slices" + + f = open(fn, "w") + f.write(""" +ENABLE_PLC=True +ENABLE_PS=False +BASE_HRN="%(base_hrn)s" +RELEVANT_SERVICE_NAMES=['%(service_coblitz)s', '%(service_dnsredir)s', '%(service_dnsdemux)s'] +COBLITZ_SLICE_NAME="%(slice_coblitz)s" +COBLITZ_SLICE_ID=1 +COBLITZ_PS_SLICE_NAME="%(slice_coblitz)s" +DNSREDIR_SLICE_NAME="%(slice_dnsredir)s" +DNSREDIR_SLICE_ID=2 +DNSREDIR_PS_SLICE_NAME="%(slice_dnsredir)s" +DNSDEMUX_SLICE_NAME="%(slice_dnsdemux)s" +DNSDEMUX_SLICE_ID=3 +DNSDEMUX_PS_SLICE_NAME="%(slice_dnsdemux)s" +CMI_URL="http://%(hostname_cmi)s" +CMI_HTTP_PORT="8004" +CMI_HTTPS_PORT="8003" +PUPPET_MASTER_HOSTNAME="%(hostname_cmi)s" +PUPPET_MASTER_PORT="8140" +""" % mapping) + +if __name__ == '__main__': + hpc_service = HpcService.objects.get() + rr_service = RequestRouterService.objects.get() + lib = HpcLibrary() + lib.write_slices_file(hpc_service, rr_service) + + diff --git a/planetstack/hpc_observer/run.sh b/planetstack/hpc_observer/run.sh new file mode 100755 index 0000000..3369a4a --- /dev/null +++ b/planetstack/hpc_observer/run.sh @@ -0,0 +1 @@ +python hpc-backend.py -C /opt/planetstack/hpc_observer/hpc_observer_config diff --git a/planetstack/hpc_observer/start.sh b/planetstack/hpc_observer/start.sh new file mode 100755 index 0000000..b4df9f5 --- /dev/null +++ b/planetstack/hpc_observer/start.sh @@ -0,0 +1 @@ +nohup python hpc-backend.py -C /opt/planetstack/hpc_observer/hpc_observer_config > /dev/null 2>&1 & diff --git a/planetstack/hpc_observer/steps/sync_hpcservices.py b/planetstack/hpc_observer/steps/sync_hpcservices.py new file mode 100644 index 0000000..049b316 --- /dev/null +++ b/planetstack/hpc_observer/steps/sync_hpcservices.py @@ -0,0 +1,89 @@ +import os +import base64 +from django.db.models import F, Q +from planetstack.config import Config +from observer.syncstep import SyncStep +from core.models import Service +from hpc.models import HpcService +from util.logger import Logger, logging + +logger = Logger(level=logging.INFO) + +class SyncHpcService(SyncStep): + provides=[HpcService] + requested_interval=0 + + def fetch_pending(self): + return HpcService.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) + + def extract_slice_info(hpc_service): + """ Produce a dict that describes the slices for the CMI + + slice_coblitz = + service_coblitz = + hostname_coblitz = + hostnames_coblitz = ,,... + + slice_cmi = + ... + """ + + slicenames = {} + slicehosts = {} + for slice in hpc_service.service.all(): + name = slice.name + if not ("_" in name): + continue + + if "coblitz" in name: + slicenames["coblitz"] = name + slicehosts["coblitz"] = [sliver.node.name for sliver in slice.slivers.all()] + elif "cmi" in name: + slicenames["cmi"] = name + slicehosts["cmi"] = [sliver.node.name for sliver in slice.slivers.all()] + + base_hrn = None + if "coblitz" in slicenames: + base_hrn = slicenames["coblitz"].split("_")[0] + + mapping = {} + mapping["base_hrn"] = base_hrn + for (k,v) in slicenames.items(): + mapping["slice_" + k] = v + mapping["service_" + k] = v.split("_",1)[1] + for (k,v) in slicehosts.items() + mapping["hostname_" + k] = v[0] + mapping["hostnames_" + k] = ",".join(v) + + return mapping + + def write_slices_file(self, hpc_service): + mapping = self.extract_slicenames(hpc_service) + + fn = "/tmp/slices" + + f = open(fn, "w") + f.write(""" +ENABLE_PLC=True +ENABLE_PS=False +BASE_HRN="%(base_hrn)" +RELEVANT_SERVICE_NAMES=['%(service_coblitz)', '%(service_dnsredir)', '%(service_dnsdemux)'] +COBLITZ_SLICE_NAME="%(slice_coblitz)" +COBLITZ_SLICE_ID=1 +COBLITZ_PS_SLICE_NAME="%(slice_coblitz)" +DNSREDIR_SLICE_NAME="%(slice_dnsredir)" +DNSREDIR_SLICE_ID=2 +DNSREDIR_PS_SLICE_NAME="%(slice_dnsredir)" +DNSDEMUX_SLICE_NAME="%(slice_dnsdemux)" +DNSDEMUX_SLICE_ID=3 +DNSDEMUX_PS_SLICE_NAME="%(slice_dnsdemux)" +CMI_URL="http://%(hostname_cmi)" +CMI_HTTP_PORT="8004" +CMI_HTTPS_PORT="8003" +PUPPET_MASTER_HOSTNAME="%(hostname_cmi)" +PUPPET_MASTER_PORT="8140" +""") + + def sync_record(self, hpc_service): + logger.info("sync'ing hpc_service %s" % str(hpc_service)) + hpc_service.save() diff --git a/planetstack/hpc_observer/stop.sh b/planetstack/hpc_observer/stop.sh new file mode 100755 index 0000000..632f7a6 --- /dev/null +++ b/planetstack/hpc_observer/stop.sh @@ -0,0 +1 @@ +pkill -9 -f hpc-backend.py diff --git a/planetstack/hpc_observer/upload.sh b/planetstack/hpc_observer/upload.sh new file mode 100755 index 0000000..1a1b90d --- /dev/null +++ b/planetstack/hpc_observer/upload.sh @@ -0,0 +1,3 @@ +scp hpc_observer_config run.sh start.sh stop.sh root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/ +scp steps/*.py root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/steps/ +scp deleters/*.py root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/deleters/ -- 2.43.0