From: Sapan Date: Wed, 3 Dec 2014 19:50:39 +0000 (-0500) Subject: Merge branch 'master' of git://git.planet-lab.org/plstackapi X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=43e37ab9119107c59488919f67f448c07a0f9a88;hp=2d37706dfa9b336cd14496cd7a90311d7f70d243;p=plstackapi.git Merge branch 'master' of git://git.planet-lab.org/plstackapi --- diff --git a/planetstack/openstack_observer/ansible.py b/planetstack/openstack_observer/ansible.py index 5eec7d9..8570c17 100755 --- a/planetstack/openstack_observer/ansible.py +++ b/planetstack/openstack_observer/ansible.py @@ -3,11 +3,16 @@ import jinja2 import tempfile import os import json +import pdb +import string +import random try: step_dir = Config().observer_steps_dir + sys_dir = Config().observer_sys_dir except: step_dir = '/opt/planetstack/observer/steps' + sys_dir = '/opt/opencloud' os_template_loader = jinja2.FileSystemLoader( searchpath=step_dir) os_template_env = jinja2.Environment(loader=os_template_loader) @@ -18,48 +23,63 @@ def parse_output(msg): print msg for l in lines: - magic_str = 'ok: [127.0.0.1] => ' - magic_str2 = 'changed: [127.0.0.1] => ' + magic_str = 'ok: [127.0.0.1] => ' + magic_str2 = 'changed: [127.0.0.1] => ' if (l.startswith(magic_str)): - w = len(magic_str) - str = l[w:] - d = json.loads(str) - results.append(d) - elif (l.startswith(magic_str2)): - w = len(magic_str2) - str = l[w:] - d = json.loads(str) - results.append(d) + w = len(magic_str) + str = l[w:] + d = json.loads(str) + results.append(d) + elif (l.startswith(magic_str2)): + w = len(magic_str2) + str = l[w:] + d = json.loads(str) + results.append(d) return results - -def run_template(name, opts): + +def id_generator(size=6, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + +def run_template(name, opts,path=''): template = os_template_env.get_template(name) buffer = template.render(opts) - - f = tempfile.NamedTemporaryFile(mode='w') + + import pdb + + #f = open('/tmp/obsans','w') + try: + objname = opts['ansible_tag'] + except: + objname= id_generator() + os.system('mkdir -p %s'%'/'.join([sys_dir,path])) + fqp = '/'.join([sys_dir,path,objname]) + + + f = open(fqp,'w') f.write(buffer) f.flush() - - run = os.popen('/opt/planetstack/observer/run_ansible '+f.name) + + #os.system('cp %s %s-backup'%(fqp,fqp)) + run = os.popen('/opt/planetstack/observer/run_ansible '+fqp) msg = run.read() status = run.close() try: - ok_results = parse_output(msg) + ok_results = parse_output(msg) except ValueError,e: - print str(e) - raise e + print str(e) + raise e return ok_results def main(): - run_template('ansible/sync_user_deployments.yaml',{ "endpoint" : "http://172.31.38.128:5000/v2.0/", - "name" : "Sapan Bhatia", - "email": "gwsapan@gmail.com", - "password": "foobar", - "admin_user":"admin", - "admin_password":"6a789bf69dd647e2", - "admin_tenant":"admin", - "tenant":"demo", - "roles":['user','admin'] }) + run_template('ansible/sync_user_deployments.yaml',{ "endpoint" : "http://172.31.38.128:5000/v2.0/", + "name" : "Sapan Bhatia", + "email": "gwsapan@gmail.com", + "password": "foobar", + "admin_user":"admin", + "admin_password":"6a789bf69dd647e2", + "admin_tenant":"admin", + "tenant":"demo", + "roles":['user','admin'] }) diff --git a/planetstack/openstack_observer/steps/delete_slivers.yaml b/planetstack/openstack_observer/steps/delete_slivers.yaml new file mode 100644 index 0000000..fa6b879 --- /dev/null +++ b/planetstack/openstack_observer/steps/delete_slivers.yaml @@ -0,0 +1,8 @@ +--- +- hosts: 127.0.0.1 + connection: local + tasks: + + - nova_compute: + state: absent + name: {{ name }} diff --git a/planetstack/openstack_observer/steps/sync_site_deployments.py b/planetstack/openstack_observer/steps/sync_site_deployments.py index b5e9f9a..8b2d3b8 100644 --- a/planetstack/openstack_observer/steps/sync_site_deployments.py +++ b/planetstack/openstack_observer/steps/sync_site_deployments.py @@ -18,10 +18,11 @@ class SyncSiteDeployments(OpenStackSyncStep): 'admin_password': site_deployment.deployment.admin_password, 'admin_tenant': 'admin', 'tenant': site_deployment.site.login_base, + 'ansible_tag': '%s@%s'%(site_deployment.site.login_base,site_deployment.deployment.name), # name of ansible playbook 'tenant_description': site_deployment.site.name} rendered = template.render(tenant_fields) - res = run_template('sync_site_deployments.yaml', tenant_fields) + res = run_template('sync_site_deployments.yaml', tenant_fields, path='site_deployments') if (len(res)==1): site_deployment.tenant_id = res[0]['id'] diff --git a/planetstack/openstack_observer/steps/sync_slice_deployments.py b/planetstack/openstack_observer/steps/sync_slice_deployments.py index 4c358d7..e8d760d 100644 --- a/planetstack/openstack_observer/steps/sync_slice_deployments.py +++ b/planetstack/openstack_observer/steps/sync_slice_deployments.py @@ -52,9 +52,10 @@ class SyncSliceDeployments(OpenStackSyncStep): deployment=slice_deployment.deployment) if not deployment_users: logger.info("slice createor %s has not accout at deployment %s" % (slice_deployment.slice.creator, slice_deployment.deployment.name)) - roles = [] + roles = ['admin'] + deployment_user = 'Unknown user' else: - deployment_user = deployment_users[0] + deployment_user = deployment_users[0].user.email roles = ['admin'] max_instances=int(slice_deployment.slice.max_slivers) @@ -65,10 +66,11 @@ class SyncSliceDeployments(OpenStackSyncStep): 'tenant': slice_deployment.slice.name, 'tenant_description': slice_deployment.slice.description, 'roles':roles, - 'name':deployment_user.user.email, + 'name':deployment_user, + 'ansible_tag':'%s@%s'%(slice_deployment.slice.name,slice_deployment.deployment.name), 'max_instances':max_instances} - res = run_template('sync_slice_deployments.yaml', tenant_fields) + res = run_template('sync_slice_deployments.yaml', tenant_fields, path='slice_deployments') expected_num = len(roles)+1 if (len(res)!=expected_num): raise Exception('Could not sync tenants for slice %s'%slice_deployment.slice.name) diff --git a/planetstack/openstack_observer/steps/sync_slivers.py b/planetstack/openstack_observer/steps/sync_slivers.py index 8237896..cf41446 100644 --- a/planetstack/openstack_observer/steps/sync_slivers.py +++ b/planetstack/openstack_observer/steps/sync_slivers.py @@ -14,7 +14,7 @@ logger = Logger(level=logging.INFO) def escape(s): s = s.replace('\n',r'\n').replace('"',r'\"') return s - + class SyncSlivers(OpenStackSyncStep): provides=[Sliver] requested_interval=0 @@ -27,7 +27,7 @@ class SyncSlivers(OpenStackSyncStep): logger.info("sync'ing sliver:%s slice:%s deployment:%s " % (sliver, sliver.slice.name, sliver.node.deployment)) metadata_update = {} - if (sliver.numberCores): + if (sliver.numberCores): metadata_update["cpu_cores"] = str(sliver.numberCores) for tag in sliver.slice.tags.all(): @@ -37,88 +37,102 @@ class SyncSlivers(OpenStackSyncStep): # public keys slice_memberships = SlicePrivilege.objects.filter(slice=sliver.slice) pubkeys = set([sm.user.public_key for sm in slice_memberships if sm.user.public_key]) - if sliver.creator.public_key: - pubkeys.add(sliver.creator.public_key) + if sliver.creator.public_key: + pubkeys.add(sliver.creator.public_key) if sliver.slice.creator.public_key: - pubkeys.add(sliver.slice.creator.public_key) + pubkeys.add(sliver.slice.creator.public_key) - nics = [] - networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)] - network_deployments = NetworkDeployments.objects.filter(network__in=networks, - deployment=sliver.node.deployment) + nics = [] + networks = [ns.network for ns in NetworkSlice.objects.filter(slice=sliver.slice)] + network_deployments = NetworkDeployments.objects.filter(network__in=networks, + deployment=sliver.node.deployment) - for network_deployment in network_deployments: - if network_deployment.network.template.visibility == 'private' and \ - network_deployment.network.template.translation == 'none' and network_deployment.net_id: - nics.append(network_deployment.net_id) + for network_deployment in network_deployments: + if network_deployment.network.template.visibility == 'private' and \ + network_deployment.network.template.translation == 'none' and network_deployment.net_id: + nics.append(network_deployment.net_id) - # now include network template - network_templates = [network.template.sharedNetworkName for network in networks \ - if network.template.sharedNetworkName] + # now include network template + network_templates = [network.template.sharedNetworkName for network in networks \ + if network.template.sharedNetworkName] #driver = self.driver.client_driver(caller=sliver.creator, tenant=sliver.slice.name, deployment=sliver.deploymentNetwork) driver = self.driver.admin_driver(tenant='admin', deployment=sliver.deploymentNetwork) - nets = driver.shell.quantum.list_networks()['networks'] - for net in nets: - if net['name'] in network_templates: - nics.append(net['id']) - - if (not nics): - for net in nets: - if net['name']=='public': - nics.append(net['id']) - - # look up image id - deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name) - image_id = None - images = deployment_driver.shell.glanceclient.images.list() - for image in images: - if image.name == sliver.image.name or not image_id: - image_id = image.id - - # look up key name at the deployment - # create/fetch keypair - keyname = None - keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\ - sliver.slice.name - key_fields = {'name': keyname, - 'public_key': sliver.creator.public_key} - - - userData = self.get_userdata(sliver) - if sliver.userData: - userData = sliver.userData - - sliver_name = '@'.join([sliver.slice.name,sliver.node.name]) - tenant_fields = {'endpoint':sliver.node.deployment.auth_url, - 'admin_user': sliver.node.deployment.admin_user, - 'admin_password': sliver.node.deployment.admin_password, - 'admin_tenant': 'admin', - 'tenant': sliver.slice.name, - 'tenant_description': sliver.slice.description, - 'name':sliver_name, - 'image_id':image_id, - 'key_name':keyname, - 'flavor_id':1, - 'nics':nics, - 'meta':metadata_update, - 'key':key_fields, - 'user_data':r'%s'%escape(userData)} - - res = run_template('sync_slivers.yaml', tenant_fields) - if (len(res)!=2): - raise Exception('Could not sync sliver %s'%sliver.slice.name) - else: - sliver_id = res[1]['id'] # 0 is for the key + nets = driver.shell.quantum.list_networks()['networks'] + for net in nets: + if net['name'] in network_templates: + nics.append(net['id']) + + if (not nics): + for net in nets: + if net['name']=='public': + nics.append(net['id']) + + # look up image id + deployment_driver = self.driver.admin_driver(deployment=sliver.deploymentNetwork.name) + image_id = None + images = deployment_driver.shell.glanceclient.images.list() + for image in images: + if image.name == sliver.image.name or not image_id: + image_id = image.id + + # look up key name at the deployment + # create/fetch keypair + keyname = None + keyname = sliver.creator.email.lower().replace('@', 'AT').replace('.', '') +\ + sliver.slice.name + key_fields = {'name': keyname, + 'public_key': sliver.creator.public_key} + + + userData = self.get_userdata(sliver) + if sliver.userData: + userData = sliver.userData + + try: + legacy = Config().observer_legacy + except: + legacy = False + + if (legacy): + host_filter = sliver.node.name.split('.',1)[0] + else: + host_filter = sliver.node.name + + availability_zone_filter = 'nova:%s'%host_filter + sliver_name = '@'.join([sliver.slice.name,sliver.node.name]) + tenant_fields = {'endpoint':sliver.node.deployment.auth_url, + 'admin_user': sliver.node.deployment.admin_user, + 'admin_password': sliver.node.deployment.admin_password, + 'admin_tenant': 'admin', + 'tenant': sliver.slice.name, + 'availability_zone': availability_zone_filter, + 'tenant_description': sliver.slice.description, + 'name':sliver_name, + 'ansible_tag':sliver_name, + 'image_id':image_id, + 'key_name':keyname, + 'flavor_id':3, + 'nics':nics, + 'meta':metadata_update, + 'key':key_fields, + 'user_data':r'%s'%escape(userData)} + + res = run_template('sync_slivers.yaml', tenant_fields, path='slivers') + if (len(res)!=2): + raise Exception('Could not sync sliver %s'%sliver.slice.name) + else: + sliver_id = res[1]['id'] # 0 is for the key sliver.instance_id = sliver_id sliver.instance_name = sliver_name - sliver.save() + sliver.save() def delete_record(self, sliver): - if sliver.instance_id: - driver = self.driver.client_driver(caller=sliver.creator, - tenant=sliver.slice.name, - deployment=sliver.deploymentNetwork.name) - driver.destroy_instance(sliver.instance_id) + sliver_name = '@'.join([sliver.slice.name,sliver.node.name]) + tenant_fields = {'name':sliver_name, + 'ansible_tag':sliver_name + } + res = run_template('delete_slivers.yaml', tenant_fields, path='slivers') + diff --git a/planetstack/openstack_observer/steps/sync_slivers.yaml b/planetstack/openstack_observer/steps/sync_slivers.yaml index 5f45d5c..6085434 100644 --- a/planetstack/openstack_observer/steps/sync_slivers.yaml +++ b/planetstack/openstack_observer/steps/sync_slivers.yaml @@ -3,34 +3,35 @@ connection: local tasks: - nova_keypair: - state: present - auth_url: {{ endpoint }} - login_username: {{ admin_user }} - login_password: {{ admin_password }} - login_tenant_name: {{ admin_tenant }} - name: {{ key_name }} - public_key: "{{ key.public_key}}" - + state: present + auth_url: {{ endpoint }} + login_username: {{ admin_user }} + login_password: {{ admin_password }} + login_tenant_name: {{ admin_tenant }} + name: {{ key_name }} + public_key: "{{ key.public_key}}" + - nova_compute: - state: present - auth_url: {{ endpoint }} - login_username: {{ admin_user }} - login_password: {{ admin_password }} - login_tenant_name: {{ admin_tenant }} - name: {{ name }} - image_id: {{ image_id }} - key_name: {{ key_name }} - wait_for: 200 - flavor_id: {{ flavor_id }} - user_data: "{{ user_data }}" - nics: - {% for net in nics %} - - net-id: {{ net }} - {% endfor %} + state: present + auth_url: {{ endpoint }} + login_username: {{ admin_user }} + login_password: {{ admin_password }} + login_tenant_name: {{ admin_tenant }} + availability_zone: {{ availability_zone }} + name: {{ name }} + image_id: {{ image_id }} + key_name: {{ key_name }} + wait_for: 200 + flavor_id: {{ flavor_id }} + user_data: "{{ user_data }}" + nics: + {% for net in nics %} + - net-id: {{ net }} + {% endfor %} - {% if meta %} - meta: - {% for k,v in meta.items() %} - {{ k }} : "{{ v }}" - {% endfor %} - {% endif %} + {% if meta %} + meta: + {% for k,v in meta.items() %} + {{ k }} : "{{ v }}" + {% endfor %} + {% endif %} diff --git a/planetstack/openstack_observer/steps/sync_user_deployments.py b/planetstack/openstack_observer/steps/sync_user_deployments.py index 2e8256f..821e67e 100644 --- a/planetstack/openstack_observer/steps/sync_user_deployments.py +++ b/planetstack/openstack_observer/steps/sync_user_deployments.py @@ -57,6 +57,7 @@ class SyncUserDeployments(OpenStackSyncStep): user_fields = {'endpoint':user_deployment.deployment.auth_url, 'name': user_deployment.user.email, + 'ansible_tag': '%s@%s'%(user_deployment.user.email,user_deployment.deployment.name), 'email': user_deployment.user.email, 'password': hashlib.md5(user_deployment.user.password).hexdigest()[:6], 'admin_user': user_deployment.deployment.admin_user, @@ -66,7 +67,7 @@ class SyncUserDeployments(OpenStackSyncStep): 'tenant':tenant_name} rendered = template.render(user_fields) - res = run_template('sync_user_deployments.yaml', user_fields) + res = run_template('sync_user_deployments.yaml', user_fields, 'user_deployments') # results is an array in which each element corresponds to an # "ok" string received per operation. If we get as many oks as