planetstack/openstack/manager.py

   1 import os
   2 #os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings")
   3 import string
   4 import random
   5 import hashlib
   6 from datetime import datetime
   7
   8 from netaddr import IPAddress, IPNetwork
   9 from planetstack import settings
  10 from django.core import management
  11 from core.models import *
  12 from planetstack.config import Config
  13 try:
  14     from openstack.client import OpenStackClient
  15     from openstack.driver import OpenStackDriver
  16     has_openstack = True
  17 except:
  18     has_openstack = False
  19
  20 manager_enabled = Config().api_nova_enabled
  21
  22
  23 def random_string(size=6):
  24     return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(size))
  25
  26 def require_enabled(callable):
  27     def wrapper(*args, **kwds):
  28         if manager_enabled and has_openstack:
  29             return callable(*args, **kwds)
  30         else:
  31             return None
  32     return wrapper
  33
  34
  35 class OpenStackManager:
  36
  37     def __init__(self, auth={}, caller=None):
  38         self.client = None
  39         self.driver = None
  40         self.caller = None
  41         self.has_openstack = has_openstack
  42         self.enabled = manager_enabled
  43
  44         if has_openstack and manager_enabled:
  45             if auth:
  46                 try:
  47                     self.init_user(auth, caller)
  48                 except:
  49                     # if this fails then it meanse the caller doesn't have a
  50                     # role at the slice's tenant. if the caller is an admin
  51                     # just use the admin client/manager.
  52                     if caller and caller.is_admin:
  53                         self.init_admin()
  54                     else: raise
  55             else:
  56                 self.init_admin()
  57
  58     @require_enabled
  59     def init_caller(self, caller, tenant):
  60         auth = {'username': caller.email,
  61                 'password': hashlib.md5(caller.password).hexdigest()[:6],
  62                 'tenant': tenant}
  63         self.client = OpenStackClient(**auth)
  64         self.driver = OpenStackDriver(client=self.client)
  65         self.caller = caller
  66
  67     @require_enabled
  68     def init_admin(self, tenant=None):
  69         # use the admin credentials
  70         self.client = OpenStackClient(tenant=tenant)
  71         self.driver = OpenStackDriver(client=self.client)
  72         self.caller = self.driver.admin_user
  73         self.caller.kuser_id = self.caller.id
  74
  75     @require_enabled
  76     def save_role(self, role):
  77         if not role.role:
  78             keystone_role = self.driver.create_role(role.role_type)
  79             role.role = keystone_role.id
  80
  81     @require_enabled
  82     def delete_role(self, role):
  83         if role.role:
  84             self.driver.delete_role({'id': role.role})
  85
  86     @require_enabled
  87     def save_key(self, key, name):
  88         key_fields = {'name': name,
  89                       'public_key': key}
  90         nova_key = self.driver.create_keypair(**key_fields)
  91
  92     @require_enabled
  93     def delete_key(self, key):
  94         if key.nkey_id:
  95             self.driver.delete_keypair(key.nkey_id)
  96
  97     @require_enabled
  98     def save_user(self, user):
  99         name = user.email[:user.email.find('@')]
 100         user_fields = {'name': name,
 101                        'email': user.email,
 102                        'password': hashlib.md5(user.password).hexdigest()[:6],
 103                        'enabled': True}
 104         if not user.kuser_id:
 105             keystone_user = self.driver.create_user(**user_fields)
 106             user.kuser_id = keystone_user.id
 107         else:
 108             self.driver.update_user(user.kuser_id, user_fields)
 109
 110         if user.site:
 111             self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user')
 112             if user.is_admin:
 113                 self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 114             else:
 115                 # may have admin role so attempt to remove it
 116                 self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 117
 118         if user.public_key:
 119             self.init_caller(user, user.site.login_base)
 120             self.save_key(user.public_key, user.keyname)
 121             self.init_admin()
 122
 123         user.save()
 124         user.enacted = datetime.now()
 125         user.save(update_fields=['enacted'])
 126
 127     @require_enabled
 128     def delete_user(self, user):
 129         if user.kuser_id:
 130             self.driver.delete_user(user.kuser_id)
 131
 132     @require_enabled
 133     def save_site(self, site, add_role=True):
 134         if not site.tenant_id:
 135             tenant = self.driver.create_tenant(tenant_name=site.login_base,
 136                                                description=site.name,
 137                                                enabled=site.enabled)
 138             site.tenant_id = tenant.id
 139             # give caller an admin role at the tenant they've created
 140             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 141
 142         # update the record
 143         if site.id and site.tenant_id:
 144             self.driver.update_tenant(site.tenant_id,
 145                                       description=site.name,
 146                                       enabled=site.enabled)
 147
 148         # commit the updated record
 149         site.save()
 150         site.enacted = datetime.now()
 151         site.save(update_fields=['enacted']) # enusre enacted > updated
 152
 153
 154     @require_enabled
 155     def delete_site(self, site):
 156         if site.tenant_id:
 157             self.driver.delete_tenant(site.tenant_id)
 158
 159     @require_enabled
 160     def save_site_privilege(self, site_priv):
 161         if site_priv.user.kuser_id and site_priv.site.tenant_id:
 162             self.driver.add_user_role(site_priv.user.kuser_id,
 163                                       site_priv.site.tenant_id,
 164                                       site_priv.role.role_type)
 165         site_priv.enacted = datetime.now()
 166         site_priv.save(update_fields=['enacted'])
 167
 168
 169     @require_enabled
 170     def delete_site_privilege(self, site_priv):
 171         self.driver.delete_user_role(site_priv.user.kuser_id,
 172                                      site_priv.site.tenant_id,
 173                                      site_priv.role.role_type)
 174
 175     @require_enabled
 176     def save_slice(self, slice):
 177         if not slice.tenant_id:
 178             nova_fields = {'tenant_name': slice.name,
 179                    'description': slice.description,
 180                    'enabled': slice.enabled}
 181             tenant = self.driver.create_tenant(**nova_fields)
 182             slice.tenant_id = tenant.id
 183
 184             # give caller an admin role at the tenant they've created
 185             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 186
 187             # refresh credentials using this tenant
 188             self.driver.shell.connect(username=self.driver.shell.keystone.username,
 189                                       password=self.driver.shell.keystone.password,
 190                                       tenant=tenant.name)
 191
 192             # create network
 193             network = self.driver.create_network(slice.name)
 194             slice.network_id = network['id']
 195
 196             # create router
 197             router = self.driver.create_router(slice.name)
 198             slice.router_id = router['id']
 199
 200             # create subnet
 201             next_subnet = self.get_next_subnet()
 202             cidr = str(next_subnet.cidr)
 203             ip_version = next_subnet.version
 204             start = str(next_subnet[2])
 205             end = str(next_subnet[-2])
 206             subnet = self.driver.create_subnet(name=slice.name,
 207                                                network_id = network['id'],
 208                                                cidr_ip = cidr,
 209                                                ip_version = ip_version,
 210                                                start = start,
 211                                                end = end)
 212             slice.subnet_id = subnet['id']
 213             # add subnet as interface to slice's router
 214             self.driver.add_router_interface(router['id'], subnet['id'])
 215             # add external route
 216             self.driver.add_external_route(subnet)
 217
 218
 219         if slice.id and slice.tenant_id:
 220             self.driver.update_tenant(slice.tenant_id,
 221                                       description=slice.description,
 222                                       enabled=slice.enabled)
 223
 224         slice.save()
 225         slice.enacted = datetime.now()
 226         slice.save(update_fields=['enacted'])
 227
 228     @require_enabled
 229     def delete_slice(self, slice):
 230         if slice.tenant_id:
 231             self._delete_slice(slice.tenant_id, slice.network_id,
 232                                slice.router_id, slice.subnet_id)
 233     @require_enabled
 234     def _delete_slice(self, tenant_id, network_id, router_id, subnet_id):
 235         self.driver.delete_router_interface(slice.router_id, slice.subnet_id)
 236         self.driver.delete_subnet(slice.subnet_id)
 237         self.driver.delete_router(slice.router_id)
 238         self.driver.delete_network(slice.network_id)
 239         self.driver.delete_tenant(slice.tenant_id)
 240         # delete external route
 241         subnet = None
 242         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 243         for snet in subnets:
 244             if snet['id'] == slice.subnet_id:
 245                 subnet = snet
 246         if subnet:
 247             self.driver.delete_external_route(subnet)
 248
 249
 250     @require_enabled
 251     def save_slice_membership(self, slice_memb):
 252         if slice_memb.user.kuser_id and slice_memb.slice.tenant_id:
 253             self.driver.add_user_role(slice_memb.user.kuser_id,
 254                                       slice_memb.slice.tenant_id,
 255                                       slice_memb.role.role_type)
 256         slice_memb.enacted = datetime.now()
 257         slice_memb.save(update_fields=['enacted'])
 258
 259
 260     @require_enabled
 261     def delete_slice_membership(self, slice_memb):
 262         self.driver.delete_user_role(slice_memb.user.kuser_id,
 263                                      slice_memb.slice.tenant_id,
 264                                      slice_memb.role.role_type)
 265
 266
 267     @require_enabled
 268     def get_next_subnet(self):
 269         # limit ourself to 10.0.x.x for now
 270         valid_subnet = lambda net: net.startswith('10.0')
 271         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 272         ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
 273                 if valid_subnet(subnet['cidr'])]
 274         ints.sort()
 275         last_ip = IPAddress(ints[-1])
 276         last_network = IPNetwork(str(last_ip) + "/24")
 277         next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
 278         return next_network
 279
 280     @require_enabled
 281     def save_subnet(self, subnet):
 282         if not subnet.subnet_id:
 283             quantum_subnet = self.driver.create_subnet(name= subnet.slice.name,
 284                                           network_id=subnet.slice.network_id,
 285                                           cidr_ip = subnet.cidr,
 286                                           ip_version=subnet.ip_version,
 287                                           start = subnet.start,
 288                                           end = subnet.end)
 289             subnet.subnet_id = quantum_subnet['id']
 290             # add subnet as interface to slice's router
 291             self.driver.add_router_interface(subnet.slice.router_id, subnet.subnet_id)
 292             #add_route = 'route add -net %s dev br-ex gw 10.100.0.5' % self.cidr
 293             #commands.getstatusoutput(add_route)
 294
 295
 296     @require_enabled
 297     def delete_subnet(self, subnet):
 298         if subnet.subnet_id:
 299             self.driver.delete_router_interface(subnet.slice.router_id, subnet.subnet_id)
 300             self.driver.delete_subnet(subnet.subnet_id)
 301             #del_route = 'route del -net %s' % self.cidr
 302             #commands.getstatusoutput(del_route)
 303
 304     def get_requested_networks(self, slice):
 305         network_ids = [x.network_id for x in slice.networks.all()]
 306
 307         if slice.network_id is not None:
 308             network_ids.append(slice.network_id)
 309
 310         networks = []
 311         for network_id in network_ids:
 312             networks.append({"net-id": network_id})
 313
 314         return networks
 315
 316     @require_enabled
 317     def save_sliver(self, sliver):
 318         if not sliver.instance_id:
 319             if (sliver.slice.name == "smbaker-slice-8") or (sliver.slice.name.startswith("smbaker-slice-net")):
 320                 # only inflict this pain on myself, for now...
 321                 nics = self.get_requested_networks(sliver.slice)
 322             else:
 323                 nics = None
 324             file("/tmp/scott-manager","a").write("slice: %s\nreq: %s\n" % (str(sliver.slice.name), str(nics)))
 325             slice_memberships = SliceMembership.objects.filter(slice=sliver.slice)
 326             pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
 327             pubkeys.append(sliver.creator.public_key)
 328             instance = self.driver.spawn_instance(name=sliver.name,
 329                                    key_name = sliver.creator.keyname,
 330                                    image_id = sliver.image.image_id,
 331                                    hostname = sliver.node.name,
 332                                    pubkeys = pubkeys,
 333                                    nics = nics )
 334             sliver.instance_id = instance.id
 335             sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name')
 336
 337         if sliver.instance_id and ("numberCores" in sliver.changed_fields):
 338             self.driver.update_instance_metadata(sliver.instance_id, {"cpu_cores": str(sliver.numberCores)})
 339
 340         sliver.save()
 341         sliver.enacted = datetime.now()
 342         sliver.save(update_fields=['enacted'])
 343
 344     @require_enabled
 345     def delete_sliver(self, sliver):
 346         if sliver.instance_id:
 347             self.driver.destroy_instance(sliver.instance_id)
 348
 349
 350     def refresh_nodes(self):
 351         # collect local nodes
 352         nodes = Node.objects.all()
 353         nodes_dict = {}
 354         for node in nodes:
 355             if 'viccidev10' not in node.name:
 356                 nodes_dict[node.name] = node
 357
 358         deployment = Deployment.objects.filter(name='VICCI')[0]
 359         login_bases = ['princeton', 'stanford', 'gt', 'uw', 'mpisws']
 360         sites = Site.objects.filter(login_base__in=login_bases)
 361         # collect nova nodes:
 362         compute_nodes = self.client.nova.hypervisors.list()
 363
 364         compute_nodes_dict = {}
 365         for compute_node in compute_nodes:
 366             compute_nodes_dict[compute_node.hypervisor_hostname] = compute_node
 367
 368         # add new nodes:
 369         new_node_names = set(compute_nodes_dict.keys()).difference(nodes_dict.keys())
 370         i = 0
 371         max = len(sites)
 372         for name in new_node_names:
 373             if i == max:
 374                 i = 0
 375             site = sites[i]
 376             node = Node(name=compute_nodes_dict[name].hypervisor_hostname,
 377                         site=site,
 378                         deployment=deployment)
 379             node.save()
 380             i+=1
 381
 382         # remove old nodes
 383         old_node_names = set(nodes_dict.keys()).difference(compute_nodes_dict.keys())
 384         Node.objects.filter(name__in=old_node_names).delete()
 385
 386     def refresh_images(self):
 387         from core.models.image import Image
 388         # collect local images
 389         images = Image.objects.all()
 390         images_dict = {}
 391         for image in images:
 392             images_dict[image.name] = image
 393
 394         # collect glance images
 395         glance_images = self.client.glance.get_images()
 396         glance_images_dict = {}
 397         for glance_image in glance_images:
 398             glance_images_dict[glance_image['name']] = glance_image
 399
 400         # add new images
 401         new_image_names = set(glance_images_dict.keys()).difference(images_dict.keys())
 402         for name in new_image_names:
 403             image = Image(image_id=glance_images_dict[name]['id'],
 404                           name=glance_images_dict[name]['name'],
 405                           disk_format=glance_images_dict[name]['disk_format'],
 406                           container_format=glance_images_dict[name]['container_format'])
 407             image.save()
 408
 409         # remove old images
 410         old_image_names = set(images_dict.keys()).difference(glance_images_dict.keys())
 411         Image.objects.filter(name__in=old_image_names).delete()
 412
 413     @require_enabled
 414     def save_network(self, network):
 415         if not network.network_id:
 416             if network.template.sharedNetworkName:
 417                 network.network_id = network.template.sharedNetworkId
 418                 (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id)
 419             else:
 420                 network_name = network.name
 421
 422                 # create network
 423                 os_network = self.driver.create_network(network_name)
 424                 network.network_id = os_network['id']
 425
 426                 # create router
 427                 router = self.driver.create_router(network_name)
 428                 network.router_id = router['id']
 429
 430                 # create subnet
 431                 next_subnet = self.get_next_subnet()
 432                 cidr = str(next_subnet.cidr)
 433                 ip_version = next_subnet.version
 434                 start = str(next_subnet[2])
 435                 end = str(next_subnet[-2])
 436                 subnet = self.driver.create_subnet(name=network_name,
 437                                                    network_id = network.network_id,
 438                                                    cidr_ip = cidr,
 439                                                    ip_version = ip_version,
 440                                                    start = start,
 441                                                    end = end)
 442                 network.subnet = cidr
 443                 network.subnet_id = subnet['id']
 444                 # add subnet as interface to slice's router
 445                 self.driver.add_router_interface(router['id'], subnet['id'])
 446                 # add external route
 447                 self.driver.add_external_route(subnet)
 448
 449         network.save()
 450         network.enacted = datetime.now()
 451         network.save(update_fields=['enacted'])
 452
 453     def delete_network(self, network):
 454         if (network.router_id) and (network.subnet_id):
 455             self.driver.delete_router_interface(network.router_id, network.subnet_id)
 456         if network.subnet_id:
 457             self.driver.delete_subnet(network.subnet_id)
 458         if network.router_id:
 459             self.driver.delete_router(network.router_id)
 460         if network.network_id:
 461             self.driver.delete_network(network.network_id)
 462
 463     def save_network_template(self, template):
 464         if (template.sharedNetworkName) and (not template.sharedNetworkId):
 465             os_networks = self.driver.shell.quantum.list_networks(name=template.sharedNetworkName)['networks']
 466             if os_networks:
 467                 template.sharedNetworkId = os_networks[0]["id"]
 468
 469         template.save()
 470         template.enacted = datetime.now()
 471         template.save(update_fields=['enacted'])
 472
 473     def find_or_make_template_for_network(self, name):
 474         """ Given a network name, try to guess the right template for it """
 475
 476         # templates for networks we may encounter
 477         if name=='nat-net':
 478             template_dict = None # {"name": "private-nat", "visibility": "private", "translation": "nat"}
 479         elif name=='sharednet1':
 480             template_dict = {"name": "dedicated-public", "visibility": "public", "translation": "none"}
 481         else:
 482             template_dict = {"name": "private", "visibility": "private", "translation": "none"}
 483
 484         # if we have an existing template return it
 485         templates = NetworkTemplate.objects.filter(name=template_dict["name"])
 486         if templates:
 487             return templates[0]
 488
 489         if template_dict == None:
 490             return None
 491
 492         template = NetworkTemplate(**template_dict)
 493         template.save()
 494         return template
 495
 496     def refresh_network_templates(self):
 497         for template in NetworkTemplate.objects.all():
 498             if (template.sharedNetworkName) and (not template.sharedNetworkId):
 499                  # this will cause us to try to fill in the sharedNetworkId
 500                  self.save_network_template(template)
 501
 502     def refresh_networks(self):
 503         # get a list of all networks in the model
 504
 505         networks = Network.objects.all()
 506         networks_by_name = {}
 507         networks_by_id = {}
 508         for network in networks:
 509             networks_by_name[network.name] = network
 510             networks_by_id[network.network_id] = network
 511
 512         # Get a list of all shared networks in OS
 513
 514         os_networks = self.driver.shell.quantum.list_networks()['networks']
 515         os_networks_by_name = {}
 516         os_networks_by_id = {}
 517         for os_network in os_networks:
 518             os_networks_by_name[os_network['name']] = os_network
 519             os_networks_by_id[os_network['id']] = os_network
 520
 521         for (uuid, os_network) in os_networks_by_id.items():
 522             #print "checking OS network", os_network['name']
 523             if (os_network['shared']) and (uuid not in networks_by_id):
 524                 # Only automatically create shared networks. This is for Andy's
 525                 # nat-net and sharednet1.
 526
 527                 owner_slice = Slice.objects.get(tenant_id = os_network['tenant_id'])
 528                 template = self.find_or_make_template_for_network(os_network['name'])
 529
 530                 if (template is None):
 531                     # This is our way of saying we don't want to auto-instantiate
 532                     # this network type.
 533                     continue
 534
 535                 (subnet_id, subnet) = self.driver.get_network_subnet(os_network['id'])
 536
 537                 if owner_slice:
 538                     #print "creating model object for OS network", os_network['name']
 539                     new_network = Network(name = os_network['name'],
 540                                           template = template,
 541                                           owner = owner_slice,
 542                                           network_id = uuid,
 543                                           subnet_id = subnet_id)
 544                     new_network.save()
 545
 546         for (network_id, network) in networks_by_id.items():
 547             # If the network disappeared from OS, then reset its network_id to None
 548             if (network.network_id is not None) and (network.network_id not in os_networks_by_id):
 549                 network.network_id = None
 550
 551             # If no OS object exists, then saving the network will create one
 552             if (network.network_id is None):
 553                 #print "creating OS network for", network.name
 554                 self.save_network(network)
 555             else:
 556                 pass #print "network", network.name, "has its OS object"
 557
 558