hpc observer, wip
authorScott Baker <smbaker@gmail.com>
Fri, 3 Jan 2014 16:36:00 +0000 (08:36 -0800)
committerScott Baker <smbaker@gmail.com>
Fri, 3 Jan 2014 16:36:00 +0000 (08:36 -0800)
planetstack/hpc_observer/hpc_observer_config [new file with mode: 0644]
planetstack/hpc_observer/hpclib.py [new file with mode: 0644]
planetstack/hpc_observer/run.sh [new file with mode: 0755]
planetstack/hpc_observer/start.sh [new file with mode: 0755]
planetstack/hpc_observer/steps/sync_hpcservices.py [new file with mode: 0644]
planetstack/hpc_observer/stop.sh [new file with mode: 0755]
planetstack/hpc_observer/upload.sh [new file with mode: 0755]

diff --git a/planetstack/hpc_observer/hpc_observer_config b/planetstack/hpc_observer/hpc_observer_config
new file mode 100644 (file)
index 0000000..13bfaaf
--- /dev/null
@@ -0,0 +1,33 @@
+
+[plc]
+name=plc
+deployment=VICCI
+
+[db]
+name=planetstack
+user=plstackuser
+password=2uMDYtJK
+host=localhost
+port=5432
+
+[api]
+host=128.112.171.237
+port=8000
+ssl_key=None
+ssl_cert=None
+ca_ssl_cert=None
+ratelimit_enabled=0
+omf_enabled=0
+mail_support_address=support@localhost
+nova_enabled=True
+
+[observer]
+dependency_graph=/opt/planetstack/hpc_observer/model-deps
+steps_dir=/opt/planetstack/hpc_observer/steps
+deleters_dir=/opt/planetstack/hpc_ovserver/deleters
+log_file=/var/log/hpc.log
+driver=None
+
+[feefie]
+client_id='vicci_dev_central'
+user_id='pl'
diff --git a/planetstack/hpc_observer/hpclib.py b/planetstack/hpc_observer/hpclib.py
new file mode 100644 (file)
index 0000000..e91a468
--- /dev/null
@@ -0,0 +1,103 @@
+import os
+import base64
+import sys
+
+if __name__ == '__main__':
+    sys.path.append("/opt/planetstack")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings")
+
+from planetstack.config import Config
+from core.models import Service
+from hpc.models import HpcService
+from requestrouter.models import RequestRouterService
+from util.logger import Logger, logging
+
+logger = Logger(level=logging.INFO)
+
+class HpcLibrary:
+    def extract_slice_info(self, service):
+        """ Produce a dict that describes the slices for the CMI
+
+            slice_coblitz = <name of coblitz slice>
+            service_coblitz = <name of coblitz service>
+            hostname_coblitz = <name of first coblitz slice>
+            hostnames_coblitz = <name_of_first_cob_slice>,<name_of_second_cob_slice>,...
+
+            slice_cmi = <name of cmi slice>
+            ...
+        """
+
+        slicenames = {}
+        slicehosts = {}
+        for slice in service.service.all():
+            name = slice.name
+            if not ("_" in name):
+                continue
+
+            if "coblitz" in name:
+                slicenames["coblitz"] = name
+                slicehosts["coblitz"] = [sliver.node.name for sliver in slice.slivers.all()]
+            elif "cmi" in name:
+                slicenames["cmi"] = name
+                slicehosts["cmi"] = [sliver.node.name for sliver in slice.slivers.all()]
+            elif "dnsredir" in name:
+                slicenames["dnsredir"] = name
+                slicehosts["dnsredir"] = [sliver.node.name for sliver in slice.slivers.all()]
+            elif "dnsdemux" in name:
+                slicenames["dnsdemux"] = name
+                slicehosts["dnsdemux"] = [sliver.node.name for sliver in slice.slivers.all()]
+
+        base_hrn = None
+        if "coblitz" in slicenames:
+            base_hrn = slicenames["coblitz"].split("_")[0]
+
+        mapping = {}
+        mapping["base_hrn"] = base_hrn
+        for (k,v) in slicenames.items():
+            mapping["slice_" + k] = v
+            mapping["service_" + k] = v.split("_",1)[1]
+        for (k,v) in slicehosts.items():
+            mapping["hostname_" + k] = v[0]
+            mapping["hostnames_" + k] = ",".join(v)
+
+        return mapping
+
+    def write_slices_file(self, hpc_service, rr_service):
+        mapping = self.extract_slice_info(hpc_service)
+        rr_mapping = self.extract_slice_info(rr_service)
+
+        mapping.update(rr_mapping)
+
+        print mapping
+
+        fn = "/tmp/slices"
+
+        f = open(fn, "w")
+        f.write("""
+ENABLE_PLC=True
+ENABLE_PS=False
+BASE_HRN="%(base_hrn)s"
+RELEVANT_SERVICE_NAMES=['%(service_coblitz)s', '%(service_dnsredir)s', '%(service_dnsdemux)s']
+COBLITZ_SLICE_NAME="%(slice_coblitz)s"
+COBLITZ_SLICE_ID=1
+COBLITZ_PS_SLICE_NAME="%(slice_coblitz)s"
+DNSREDIR_SLICE_NAME="%(slice_dnsredir)s"
+DNSREDIR_SLICE_ID=2
+DNSREDIR_PS_SLICE_NAME="%(slice_dnsredir)s"
+DNSDEMUX_SLICE_NAME="%(slice_dnsdemux)s"
+DNSDEMUX_SLICE_ID=3
+DNSDEMUX_PS_SLICE_NAME="%(slice_dnsdemux)s"
+CMI_URL="http://%(hostname_cmi)s"
+CMI_HTTP_PORT="8004"
+CMI_HTTPS_PORT="8003"
+PUPPET_MASTER_HOSTNAME="%(hostname_cmi)s"
+PUPPET_MASTER_PORT="8140"
+""" % mapping)
+
+if __name__ == '__main__':
+    hpc_service = HpcService.objects.get()
+    rr_service = RequestRouterService.objects.get()
+    lib = HpcLibrary()
+    lib.write_slices_file(hpc_service, rr_service)
+
+
diff --git a/planetstack/hpc_observer/run.sh b/planetstack/hpc_observer/run.sh
new file mode 100755 (executable)
index 0000000..3369a4a
--- /dev/null
@@ -0,0 +1 @@
+python hpc-backend.py  -C /opt/planetstack/hpc_observer/hpc_observer_config
diff --git a/planetstack/hpc_observer/start.sh b/planetstack/hpc_observer/start.sh
new file mode 100755 (executable)
index 0000000..b4df9f5
--- /dev/null
@@ -0,0 +1 @@
+nohup python hpc-backend.py  -C /opt/planetstack/hpc_observer/hpc_observer_config > /dev/null 2>&1 &
diff --git a/planetstack/hpc_observer/steps/sync_hpcservices.py b/planetstack/hpc_observer/steps/sync_hpcservices.py
new file mode 100644 (file)
index 0000000..049b316
--- /dev/null
@@ -0,0 +1,89 @@
+import os
+import base64
+from django.db.models import F, Q
+from planetstack.config import Config
+from observer.syncstep import SyncStep
+from core.models import Service
+from hpc.models import HpcService
+from util.logger import Logger, logging
+
+logger = Logger(level=logging.INFO)
+
+class SyncHpcService(SyncStep):
+    provides=[HpcService]
+    requested_interval=0
+
+    def fetch_pending(self):
+        return HpcService.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))
+
+    def extract_slice_info(hpc_service):
+        """ Produce a dict that describes the slices for the CMI
+
+            slice_coblitz = <name of coblitz slice>
+            service_coblitz = <name of coblitz service>
+            hostname_coblitz = <name of first coblitz slice>
+            hostnames_coblitz = <name_of_first_cob_slice>,<name_of_second_cob_slice>,...
+
+            slice_cmi = <name of cmi slice>
+            ...
+        """
+
+        slicenames = {}
+        slicehosts = {}
+        for slice in hpc_service.service.all():
+            name = slice.name
+            if not ("_" in name):
+                continue
+
+            if "coblitz" in name:
+                slicenames["coblitz"] = name
+                slicehosts["coblitz"] = [sliver.node.name for sliver in slice.slivers.all()]
+            elif "cmi" in name:
+                slicenames["cmi"] = name
+                slicehosts["cmi"] = [sliver.node.name for sliver in slice.slivers.all()]
+
+        base_hrn = None
+        if "coblitz" in slicenames:
+            base_hrn = slicenames["coblitz"].split("_")[0]
+
+        mapping = {}
+        mapping["base_hrn"] = base_hrn
+        for (k,v) in slicenames.items():
+            mapping["slice_" + k] = v
+            mapping["service_" + k] = v.split("_",1)[1]
+        for (k,v) in slicehosts.items()
+            mapping["hostname_" + k] = v[0]
+            mapping["hostnames_" + k] = ",".join(v)
+
+        return mapping
+
+    def write_slices_file(self, hpc_service):
+        mapping = self.extract_slicenames(hpc_service)
+
+        fn = "/tmp/slices"
+
+        f = open(fn, "w")
+        f.write("""
+ENABLE_PLC=True
+ENABLE_PS=False
+BASE_HRN="%(base_hrn)"
+RELEVANT_SERVICE_NAMES=['%(service_coblitz)', '%(service_dnsredir)', '%(service_dnsdemux)']
+COBLITZ_SLICE_NAME="%(slice_coblitz)"
+COBLITZ_SLICE_ID=1
+COBLITZ_PS_SLICE_NAME="%(slice_coblitz)"
+DNSREDIR_SLICE_NAME="%(slice_dnsredir)"
+DNSREDIR_SLICE_ID=2
+DNSREDIR_PS_SLICE_NAME="%(slice_dnsredir)"
+DNSDEMUX_SLICE_NAME="%(slice_dnsdemux)"
+DNSDEMUX_SLICE_ID=3
+DNSDEMUX_PS_SLICE_NAME="%(slice_dnsdemux)"
+CMI_URL="http://%(hostname_cmi)"
+CMI_HTTP_PORT="8004"
+CMI_HTTPS_PORT="8003"
+PUPPET_MASTER_HOSTNAME="%(hostname_cmi)"
+PUPPET_MASTER_PORT="8140"
+""")
+
+    def sync_record(self, hpc_service):
+        logger.info("sync'ing hpc_service %s" % str(hpc_service))
+        hpc_service.save()
diff --git a/planetstack/hpc_observer/stop.sh b/planetstack/hpc_observer/stop.sh
new file mode 100755 (executable)
index 0000000..632f7a6
--- /dev/null
@@ -0,0 +1 @@
+pkill -9 -f hpc-backend.py
diff --git a/planetstack/hpc_observer/upload.sh b/planetstack/hpc_observer/upload.sh
new file mode 100755 (executable)
index 0000000..1a1b90d
--- /dev/null
@@ -0,0 +1,3 @@
+scp hpc_observer_config run.sh start.sh  stop.sh  root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/
+scp steps/*.py root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/steps/
+scp deleters/*.py root@viccidev1.cs.princeton.edu:/opt/planetstack/hpc_observer/deleters/