Merge branch 'master' of git://git.planet-lab.org/plstackapi
authorSapan <sapanb@cs.princeton.edu>
Wed, 3 Dec 2014 19:50:39 +0000 (14:50 -0500)
committerSapan <sapanb@cs.princeton.edu>
Wed, 3 Dec 2014 19:50:39 +0000 (14:50 -0500)
planetstack/openstack_observer/ansible.py
planetstack/openstack_observer/steps/delete_slivers.yaml [new file with mode: 0644]
planetstack/openstack_observer/steps/sync_site_deployments.py
planetstack/openstack_observer/steps/sync_slice_deployments.py
planetstack/openstack_observer/steps/sync_slivers.py
planetstack/openstack_observer/steps/sync_slivers.yaml
planetstack/openstack_observer/steps/sync_user_deployments.py

index 5eec7d9..8570c17 100755 (executable)
@@ -3,11 +3,16 @@ import jinja2
 import tempfile
 import os
 import json
+import pdb
+import string
+import random
 
 try:
     step_dir = Config().observer_steps_dir
+    sys_dir = Config().observer_sys_dir
 except:
     step_dir = '/opt/planetstack/observer/steps'
+    sys_dir = '/opt/opencloud'
 
 os_template_loader = jinja2.FileSystemLoader( searchpath=step_dir)
 os_template_env = jinja2.Environment(loader=os_template_loader)
@@ -18,48 +23,63 @@ def parse_output(msg):
     print msg
 
     for l in lines:
-       magic_str = 'ok: [127.0.0.1] => '
-       magic_str2 = 'changed: [127.0.0.1] => '
+        magic_str = 'ok: [127.0.0.1] => '
+        magic_str2 = 'changed: [127.0.0.1] => '
         if (l.startswith(magic_str)):
-           w = len(magic_str)
-           str = l[w:]
-           d = json.loads(str)
-           results.append(d)
-       elif (l.startswith(magic_str2)):
-           w = len(magic_str2)
-           str = l[w:]
-           d = json.loads(str)
-           results.append(d)
+            w = len(magic_str)
+            str = l[w:]
+            d = json.loads(str)
+            results.append(d)
+        elif (l.startswith(magic_str2)):
+            w = len(magic_str2)
+            str = l[w:]
+            d = json.loads(str)
+            results.append(d)
 
 
     return results
-           
-def run_template(name, opts):
+
+def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
+    return ''.join(random.choice(chars) for _ in range(size))
+
+def run_template(name, opts,path=''):
     template = os_template_env.get_template(name)
     buffer = template.render(opts)
-    
-    f = tempfile.NamedTemporaryFile(mode='w')
+
+    import pdb
+
+    #f = open('/tmp/obsans','w')
+    try:
+        objname = opts['ansible_tag']
+    except:
+        objname= id_generator()
+    os.system('mkdir -p %s'%'/'.join([sys_dir,path]))
+    fqp = '/'.join([sys_dir,path,objname])
+
+
+    f = open(fqp,'w')
     f.write(buffer)
     f.flush()
-    
-    run = os.popen('/opt/planetstack/observer/run_ansible '+f.name)
+
+    #os.system('cp %s %s-backup'%(fqp,fqp))
+    run = os.popen('/opt/planetstack/observer/run_ansible '+fqp)
     msg = run.read()
     status = run.close()
 
     try:
-       ok_results = parse_output(msg)
+        ok_results = parse_output(msg)
     except ValueError,e:
-       print str(e)
-       raise e
+        print str(e)
+        raise e
     return ok_results
 
 def main():
-       run_template('ansible/sync_user_deployments.yaml',{ "endpoint" : "http://172.31.38.128:5000/v2.0/",
-                 "name" : "Sapan Bhatia",
-                 "email": "gwsapan@gmail.com",
-                 "password": "foobar",
-                 "admin_user":"admin",
-                 "admin_password":"6a789bf69dd647e2",
-                 "admin_tenant":"admin",
-                "tenant":"demo",
-                 "roles":['user','admin'] })
+    run_template('ansible/sync_user_deployments.yaml',{ "endpoint" : "http://172.31.38.128:5000/v2.0/",
+             "name" : "Sapan Bhatia",
+             "email": "gwsapan@gmail.com",
+             "password": "foobar",
+             "admin_user":"admin",
+             "admin_password":"6a789bf69dd647e2",
+             "admin_tenant":"admin",
+             "tenant":"demo",
+             "roles":['user','admin'] })
diff --git a/planetstack/openstack_observer/steps/delete_slivers.yaml b/planetstack/openstack_observer/steps/delete_slivers.yaml
new file mode 100644 (file)
index 0000000..fa6b879
--- /dev/null
@@ -0,0 +1,8 @@
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks:
+
+  - nova_compute:
+      state: absent
+      name: {{ name }}
index b5e9f9a..8b2d3b8 100644 (file)
@@ -18,10 +18,11 @@ class SyncSiteDeployments(OpenStackSyncStep):
                         'admin_password': site_deployment.deployment.admin_password,
                         'admin_tenant': 'admin',
                         'tenant': site_deployment.site.login_base,
+                                'ansible_tag': '%s@%s'%(site_deployment.site.login_base,site_deployment.deployment.name), # name of ansible playbook
                         'tenant_description': site_deployment.site.name}
 
        rendered = template.render(tenant_fields)
-       res = run_template('sync_site_deployments.yaml', tenant_fields)
+       res = run_template('sync_site_deployments.yaml', tenant_fields, path='site_deployments')
 
        if (len(res)==1):
                site_deployment.tenant_id = res[0]['id']
index 4c358d7..e8d760d 100644 (file)
@@ -52,9 +52,10 @@ class SyncSliceDeployments(OpenStackSyncStep):
                                                              deployment=slice_deployment.deployment)            
        if not deployment_users:
            logger.info("slice createor %s has not accout at deployment %s" % (slice_deployment.slice.creator, slice_deployment.deployment.name))
-           roles = []
+           roles = ['admin']
+           deployment_user = 'Unknown user'
        else:
-           deployment_user = deployment_users[0]
+           deployment_user = deployment_users[0].user.email
            roles = ['admin']
            
        max_instances=int(slice_deployment.slice.max_slivers)
@@ -65,10 +66,11 @@ class SyncSliceDeployments(OpenStackSyncStep):
                         'tenant': slice_deployment.slice.name,
                         'tenant_description': slice_deployment.slice.description,
                         'roles':roles,
-                        'name':deployment_user.user.email,
+                        'name':deployment_user,
+                        'ansible_tag':'%s@%s'%(slice_deployment.slice.name,slice_deployment.deployment.name),
                         'max_instances':max_instances}
 
-       res = run_template('sync_slice_deployments.yaml', tenant_fields)
+       res = run_template('sync_slice_deployments.yaml', tenant_fields, path='slice_deployments')
        expected_num = len(roles)+1
        if (len(res)!=expected_num):
            raise Exception('Could not sync tenants for slice %s'%slice_deployment.slice.name)
index 8237896..cf41446 100644 (file)
@@ -14,7 +14,7 @@ logger = Logger(level=logging.INFO)
 def escape(s):
     s = s.replace('\n',r'\n').replace('"',r'\"')
     return s
-    
+
 class SyncSlivers(OpenStackSyncStep):
     provides=[Sliver]
     requested_interval=0
@@ -27,7 +27,7 @@ class SyncSlivers(OpenStackSyncStep):
         logger.info("sync'ing sliver:%s slice:%s deployment:%s " % (sliver, sliver.slice.name, sliver.node.deployment))
 
         metadata_update = {}
-       if (sliver.numberCores):
+        if (sliver.numberCores):
             metadata_update["cpu_cores"] = str(sliver.numberCores)
 
         for tag in sliver.slice.tags.all():
@@ -37,88 +37,102 @@ class SyncSlivers(OpenStackSyncStep):
         # public keys
         slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice)
         pubkeys = set([sm.user.public_key for sm in slice_memberships if sm.user.public_key])
-       if sliver.creator.public_key:
-           pubkeys.add(sliver.creator.public_key)
+        if sliver.creator.public_key:
+            pubkeys.add(sliver.creator.public_key)
 
         if sliver.slice.creator.public_key:
-            pubkeys.add(sliver.slice.creator.public_key) 
+            pubkeys.add(sliver.slice.creator.public_key)
 
-       nics = []
-       networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)]   
-       network_deployments = NetworkDeployments.objects.filter(network__in=networks, 
-                                                               deployment=sliver.node.deployment)
+        nics = []
+        networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)]
+        network_deployments = NetworkDeployments.objects.filter(network__in=networks,
+                                                                deployment=sliver.node.deployment)
 
-       for network_deployment in network_deployments:
-           if network_deployment.network.template.visibility == 'private' and \
-              network_deployment.network.template.translation == 'none' and network_deployment.net_id: 
-               nics.append(network_deployment.net_id)
+        for network_deployment in network_deployments:
+            if network_deployment.network.template.visibility == 'private' and \
+               network_deployment.network.template.translation == 'none' and network_deployment.net_id:
+                nics.append(network_deployment.net_id)
 
-       # now include network template
-       network_templates = [network.template.sharedNetworkName for network in networks \
-                            if network.template.sharedNetworkName]
+        # now include network template
+        network_templates = [network.template.sharedNetworkName for network in networks \
+                             if network.template.sharedNetworkName]
 
         #driver = self.driver.client_driver(caller=sliver.creator, tenant=sliver.slice.name, deployment=sliver.deploymentNetwork)
         driver = self.driver.admin_driver(tenant='admin', deployment=sliver.deploymentNetwork)
-       nets = driver.shell.quantum.list_networks()['networks']
-       for net in nets:
-           if net['name'] in network_templates: 
-               nics.append(net['id']) 
-
-       if (not nics):
-           for net in nets:
-               if net['name']=='public':
-                   nics.append(net['id'])
-
-       # look up image id
-       deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name)
-       image_id = None
-       images = deployment_driver.shell.glanceclient.images.list()
-       for image in images:
-           if image.name == sliver.image.name or not image_id:
-               image_id = image.id
-               
-       # look up key name at the deployment
-       # create/fetch keypair
-       keyname = None
-       keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\
-                 sliver.slice.name
-       key_fields =  {'name': keyname,
-                      'public_key': sliver.creator.public_key}
-           
-
-       userData = self.get_userdata(sliver)
-       if sliver.userData:
-           userData = sliver.userData
-           
-       sliver_name = '@'.join([sliver.slice.name,sliver.node.name])
-       tenant_fields = {'endpoint':sliver.node.deployment.auth_url,
-                    'admin_user': sliver.node.deployment.admin_user,
-                    'admin_password': sliver.node.deployment.admin_password,
-                    'admin_tenant': 'admin',
-                    'tenant': sliver.slice.name,
-                    'tenant_description': sliver.slice.description,
-                    'name':sliver_name,
-                    'image_id':image_id,
-                    'key_name':keyname,
-                    'flavor_id':1,
-                    'nics':nics,
-                    'meta':metadata_update,
-                    'key':key_fields,
-                    'user_data':r'%s'%escape(userData)}
-
-       res = run_template('sync_slivers.yaml', tenant_fields)
-       if (len(res)!=2):
-           raise Exception('Could not sync sliver %s'%sliver.slice.name)
-       else:
-           sliver_id = res[1]['id'] # 0 is for the key
+        nets = driver.shell.quantum.list_networks()['networks']
+        for net in nets:
+            if net['name'] in network_templates:
+                nics.append(net['id'])
+
+        if (not nics):
+            for net in nets:
+                if net['name']=='public':
+                    nics.append(net['id'])
+
+        # look up image id
+        deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name)
+        image_id = None
+        images = deployment_driver.shell.glanceclient.images.list()
+        for image in images:
+            if image.name == sliver.image.name or not image_id:
+                image_id = image.id
+
+        # look up key name at the deployment
+        # create/fetch keypair
+        keyname = None
+        keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\
+                  sliver.slice.name
+        key_fields =  {'name': keyname,
+                       'public_key': sliver.creator.public_key}
+
+
+        userData = self.get_userdata(sliver)
+        if sliver.userData:
+            userData = sliver.userData
+
+        try:
+            legacy = Config().observer_legacy
+        except:
+            legacy = False
+
+        if (legacy):
+            host_filter = sliver.node.name.split('.',1)[0]
+        else:
+            host_filter = sliver.node.name
+
+        availability_zone_filter = 'nova:%s'%host_filter
+        sliver_name = '@'.join([sliver.slice.name,sliver.node.name])
+        tenant_fields = {'endpoint':sliver.node.deployment.auth_url,
+                     'admin_user': sliver.node.deployment.admin_user,
+                     'admin_password': sliver.node.deployment.admin_password,
+                     'admin_tenant': 'admin',
+                     'tenant': sliver.slice.name,
+                     'availability_zone': availability_zone_filter,
+                     'tenant_description': sliver.slice.description,
+                     'name':sliver_name,
+                     'ansible_tag':sliver_name,
+                     'image_id':image_id,
+                     'key_name':keyname,
+                     'flavor_id':3,
+                     'nics':nics,
+                     'meta':metadata_update,
+                     'key':key_fields,
+                     'user_data':r'%s'%escape(userData)}
+
+        res = run_template('sync_slivers.yaml', tenant_fields, path='slivers')
+        if (len(res)!=2):
+            raise Exception('Could not sync sliver %s'%sliver.slice.name)
+        else:
+            sliver_id = res[1]['id'] # 0 is for the key
 
             sliver.instance_id = sliver_id
             sliver.instance_name = sliver_name
-            sliver.save()    
+            sliver.save()
 
     def delete_record(self, sliver):
-        if sliver.instance_id:
-            driver = self.driver.client_driver(caller=sliver.creator, 
-                                               tenant=sliver.slice.name,
-                                               deployment=sliver.deploymentNetwork.name)
-            driver.destroy_instance(sliver.instance_id)
+        sliver_name = '@'.join([sliver.slice.name,sliver.node.name])
+        tenant_fields = {'name':sliver_name,
+                         'ansible_tag':sliver_name
+                        }
+        res = run_template('delete_slivers.yaml', tenant_fields, path='slivers')
+
index 5f45d5c..6085434 100644 (file)
@@ -3,34 +3,35 @@
   connection: local
   tasks:
   - nova_keypair:
-       state: present
-       auth_url: {{ endpoint }}
-       login_username: {{ admin_user }}
-       login_password: {{ admin_password }}
-       login_tenant_name: {{ admin_tenant }}
-       name: {{ key_name }}
-       public_key: "{{ key.public_key}}"
-       
+      state: present
+      auth_url: {{ endpoint }}
+      login_username: {{ admin_user }}
+      login_password: {{ admin_password }}
+      login_tenant_name: {{ admin_tenant }}
+      name: {{ key_name }}
+      public_key: "{{ key.public_key}}"
+
   - nova_compute:
-       state: present
-       auth_url: {{ endpoint }}
-       login_username: {{ admin_user }}
-       login_password: {{ admin_password }}
-       login_tenant_name: {{ admin_tenant }}
-       name: {{ name }}
-       image_id: {{ image_id }}
-       key_name: {{ key_name }}
-       wait_for: 200
-       flavor_id: {{ flavor_id }}
-       user_data: "{{ user_data }}"
-       nics:
-       {% for net in nics %}  
-         - net-id: {{ net }}
-       {% endfor %}
+      state: present
+      auth_url: {{ endpoint }}
+      login_username: {{ admin_user }}
+      login_password: {{ admin_password }}
+      login_tenant_name: {{ admin_tenant }}
+      availability_zone: {{ availability_zone }}
+      name: {{ name }}
+      image_id: {{ image_id }}
+      key_name: {{ key_name }}
+      wait_for: 200
+      flavor_id: {{ flavor_id }}
+      user_data: "{{ user_data }}"
+      nics:
+      {% for net in nics %}
+          - net-id: {{ net }}
+      {% endfor %}
 
-       {% if meta %}
-       meta:
-       {% for k,v in meta.items() %}  
-         {{ k }} : "{{ v }}"
-       {% endfor %}
-       {% endif %}
+      {% if meta %}
+      meta:
+      {% for k,v in meta.items() %}
+          {{ k }} : "{{ v }}"
+      {% endfor %}
+      {% endif %}
index 2e8256f..821e67e 100644 (file)
@@ -57,6 +57,7 @@ class SyncUserDeployments(OpenStackSyncStep):
 
         user_fields = {'endpoint':user_deployment.deployment.auth_url,
                       'name': user_deployment.user.email,
+                      'ansible_tag': '%s@%s'%(user_deployment.user.email,user_deployment.deployment.name),
                        'email': user_deployment.user.email,
                        'password': hashlib.md5(user_deployment.user.password).hexdigest()[:6],
                        'admin_user': user_deployment.deployment.admin_user,
@@ -66,7 +67,7 @@ class SyncUserDeployments(OpenStackSyncStep):
                       'tenant':tenant_name}    
        
        rendered = template.render(user_fields)
-       res = run_template('sync_user_deployments.yaml', user_fields)
+       res = run_template('sync_user_deployments.yaml', user_fields, 'user_deployments')
 
        # results is an array in which each element corresponds to an 
        # "ok" string received per operation. If we get as many oks as