planetstack/openstack/manager.py

   1 import os
   2 #os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings")
   3 import string
   4 import random
   5 import hashlib
   6 from datetime import datetime
   7
   8 from netaddr import IPAddress, IPNetwork
   9 from planetstack import settings
  10 from django.core import management
  11 from core.models import *
  12 from planetstack.config import Config
  13 try:
  14     from openstack.client import OpenStackClient
  15     from openstack.driver import OpenStackDriver
  16     has_openstack = True
  17 except:
  18     has_openstack = False
  19
  20 manager_enabled = Config().api_nova_enabled
  21
  22
  23 def random_string(size=6):
  24     return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(size))
  25
  26 def require_enabled(callable):
  27     def wrapper(*args, **kwds):
  28         if manager_enabled and has_openstack:
  29             return callable(*args, **kwds)
  30         else:
  31             return None
  32     return wrapper
  33
  34
  35 class OpenStackManager:
  36
  37     def __init__(self, auth={}, caller=None):
  38         self.client = None
  39         self.driver = None
  40         self.caller = None
  41         self.has_openstack = has_openstack
  42         self.enabled = manager_enabled
  43
  44         if has_openstack and manager_enabled:
  45             if auth:
  46                 try:
  47                     self.init_user(auth, caller)
  48                 except:
  49                     # if this fails then it meanse the caller doesn't have a
  50                     # role at the slice's tenant. if the caller is an admin
  51                     # just use the admin client/manager.
  52                     if caller and caller.is_admin:
  53                         self.init_admin()
  54                     else: raise
  55             else:
  56                 self.init_admin()
  57
  58     @require_enabled
  59     def init_caller(self, caller, tenant):
  60         auth = {'username': caller.email,
  61                 'password': hashlib.md5(caller.password).hexdigest()[:6],
  62                 'tenant': tenant}
  63         self.client = OpenStackClient(**auth)
  64         self.driver = OpenStackDriver(client=self.client)
  65         self.caller = caller
  66
  67     @require_enabled
  68     def init_admin(self, tenant=None):
  69         # use the admin credentials
  70         self.client = OpenStackClient(tenant=tenant)
  71         self.driver = OpenStackDriver(client=self.client)
  72         self.caller = self.driver.admin_user
  73         self.caller.kuser_id = self.caller.id
  74
  75     @require_enabled
  76     def save_role(self, role):
  77         if not role.role:
  78             keystone_role = self.driver.create_role(role.role_type)
  79             role.role = keystone_role.id
  80
  81     @require_enabled
  82     def delete_role(self, role):
  83         if role.role:
  84             self.driver.delete_role({'id': role.role})
  85
  86     @require_enabled
  87     def save_key(self, key, name):
  88         key_fields = {'name': name,
  89                       'public_key': key}
  90         nova_key = self.driver.create_keypair(**key_fields)
  91
  92     @require_enabled
  93     def delete_key(self, key):
  94         if key.nkey_id:
  95             self.driver.delete_keypair(key.nkey_id)
  96
  97     @require_enabled
  98     def save_user(self, user):
  99         name = user.email[:user.email.find('@')]
 100         user_fields = {'name': name,
 101                        'email': user.email,
 102                        'password': hashlib.md5(user.password).hexdigest()[:6],
 103                        'enabled': True}
 104         if not user.kuser_id:
 105             keystone_user = self.driver.create_user(**user_fields)
 106             user.kuser_id = keystone_user.id
 107         else:
 108             self.driver.update_user(user.kuser_id, user_fields)
 109
 110         if user.site:
 111             self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user')
 112             if user.is_admin:
 113                 self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 114             else:
 115                 # may have admin role so attempt to remove it
 116                 self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 117
 118         if user.public_key:
 119             self.init_caller(user, user.site.login_base)
 120             self.save_key(user.public_key, user.keyname)
 121             self.init_admin()
 122
 123         user.save()
 124         user.enacted = datetime.now()
 125         user.save(update_fields=['enacted'])
 126
 127     @require_enabled
 128     def delete_user(self, user):
 129         if user.kuser_id:
 130             self.driver.delete_user(user.kuser_id)
 131
 132     @require_enabled
 133     def save_site(self, site, add_role=True):
 134         if not site.tenant_id:
 135             tenant = self.driver.create_tenant(tenant_name=site.login_base,
 136                                                description=site.name,
 137                                                enabled=site.enabled)
 138             site.tenant_id = tenant.id
 139             # give caller an admin role at the tenant they've created
 140             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 141
 142         # update the record
 143         if site.id and site.tenant_id:
 144             self.driver.update_tenant(site.tenant_id,
 145                                       description=site.name,
 146                                       enabled=site.enabled)
 147
 148         # commit the updated record
 149         site.save()
 150         site.enacted = datetime.now()
 151         site.save(update_fields=['enacted']) # enusre enacted > updated
 152
 153
 154     @require_enabled
 155     def delete_site(self, site):
 156         if site.tenant_id:
 157             self.driver.delete_tenant(site.tenant_id)
 158
 159     @require_enabled
 160     def save_site_privilege(self, site_priv):
 161         if site_priv.user.kuser_id and site_priv.site.tenant_id:
 162             self.driver.add_user_role(site_priv.user.kuser_id,
 163                                       site_priv.site.tenant_id,
 164                                       site_priv.role.role_type)
 165         site_priv.enacted = datetime.now()
 166         site_priv.save(update_fields=['enacted'])
 167
 168
 169     @require_enabled
 170     def delete_site_privilege(self, site_priv):
 171         self.driver.delete_user_role(site_priv.user.kuser_id,
 172                                      site_priv.site.tenant_id,
 173                                      site_priv.role.role_type)
 174
 175     @require_enabled
 176     def save_slice(self, slice):
 177         if not slice.tenant_id:
 178             nova_fields = {'tenant_name': slice.name,
 179                    'description': slice.description,
 180                    'enabled': slice.enabled}
 181             tenant = self.driver.create_tenant(**nova_fields)
 182             slice.tenant_id = tenant.id
 183
 184             # give caller an admin role at the tenant they've created
 185             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 186
 187             # refresh credentials using this tenant
 188             self.driver.shell.connect(username=self.driver.shell.keystone.username,
 189                                       password=self.driver.shell.keystone.password,
 190                                       tenant=tenant.name)
 191
 192             # create network
 193             network = self.driver.create_network(slice.name)
 194             slice.network_id = network['id']
 195
 196             # create router
 197             router = self.driver.create_router(slice.name)
 198             slice.router_id = router['id']
 199
 200             # create subnet
 201             next_subnet = self.get_next_subnet()
 202             cidr = str(next_subnet.cidr)
 203             ip_version = next_subnet.version
 204             start = str(next_subnet[2])
 205             end = str(next_subnet[-2])
 206             subnet = self.driver.create_subnet(name=slice.name,
 207                                                network_id = network['id'],
 208                                                cidr_ip = cidr,
 209                                                ip_version = ip_version,
 210                                                start = start,
 211                                                end = end)
 212             slice.subnet_id = subnet['id']
 213             # add subnet as interface to slice's router
 214             self.driver.add_router_interface(router['id'], subnet['id'])
 215             # add external route
 216             self.driver.add_external_route(subnet)
 217
 218
 219         if slice.id and slice.tenant_id:
 220             self.driver.update_tenant(slice.tenant_id,
 221                                       description=slice.description,
 222                                       enabled=slice.enabled)
 223
 224         slice.save()
 225         slice.enacted = datetime.now()
 226         slice.save(update_fields=['enacted'])
 227
 228     @require_enabled
 229     def delete_slice(self, slice):
 230         if slice.tenant_id:
 231             self._delete_slice(slice.tenant_id, slice.network_id,
 232                                slice.router_id, slice.subnet_id)
 233     @require_enabled
 234     def _delete_slice(self, tenant_id, network_id, router_id, subnet_id):
 235         self.driver.delete_router_interface(slice.router_id, slice.subnet_id)
 236         self.driver.delete_subnet(slice.subnet_id)
 237         self.driver.delete_router(slice.router_id)
 238         self.driver.delete_network(slice.network_id)
 239         self.driver.delete_tenant(slice.tenant_id)
 240         # delete external route
 241         subnet = None
 242         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 243         for snet in subnets:
 244             if snet['id'] == slice.subnet_id:
 245                 subnet = snet
 246         if subnet:
 247             self.driver.delete_external_route(subnet)
 248
 249
 250     @require_enabled
 251     def save_slice_membership(self, slice_memb):
 252         if slice_memb.user.kuser_id and slice_memb.slice.tenant_id:
 253             self.driver.add_user_role(slice_memb.user.kuser_id,
 254                                       slice_memb.slice.tenant_id,
 255                                       slice_memb.role.role_type)
 256         slice_memb.enacted = datetime.now()
 257         slice_memb.save(update_fields=['enacted'])
 258
 259
 260     @require_enabled
 261     def delete_slice_membership(self, slice_memb):
 262         self.driver.delete_user_role(slice_memb.user.kuser_id,
 263                                      slice_memb.slice.tenant_id,
 264                                      slice_memb.role.role_type)
 265
 266
 267     @require_enabled
 268     def get_next_subnet(self):
 269         # limit ourself to 10.0.x.x for now
 270         valid_subnet = lambda net: net.startswith('10.0')
 271         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 272         ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
 273                 if valid_subnet(subnet['cidr'])]
 274         ints.sort()
 275         last_ip = IPAddress(ints[-1])
 276         last_network = IPNetwork(str(last_ip) + "/24")
 277         next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
 278         return next_network
 279
 280     @require_enabled
 281     def save_subnet(self, subnet):
 282         if not subnet.subnet_id:
 283             quantum_subnet = self.driver.create_subnet(name= subnet.slice.name,
 284                                           network_id=subnet.slice.network_id,
 285                                           cidr_ip = subnet.cidr,
 286                                           ip_version=subnet.ip_version,
 287                                           start = subnet.start,
 288                                           end = subnet.end)
 289             subnet.subnet_id = quantum_subnet['id']
 290             # add subnet as interface to slice's router
 291             self.driver.add_router_interface(subnet.slice.router_id, subnet.subnet_id)
 292             #add_route = 'route add -net %s dev br-ex gw 10.100.0.5' % self.cidr
 293             #commands.getstatusoutput(add_route)
 294
 295
 296     @require_enabled
 297     def delete_subnet(self, subnet):
 298         if subnet.subnet_id:
 299             self.driver.delete_router_interface(subnet.slice.router_id, subnet.subnet_id)
 300             self.driver.delete_subnet(subnet.subnet_id)
 301             #del_route = 'route del -net %s' % self.cidr
 302             #commands.getstatusoutput(del_route)
 303
 304     def get_requested_networks(self, slice):
 305         network_ids = [x.network_id for x in slice.networks.all()]
 306
 307         if slice.network_id is not None:
 308             network_ids.append(slice.network_id)
 309
 310         networks = []
 311         for network_id in network_ids:
 312             networks.append({"uuid": network_id})
 313
 314         return networks
 315
 316     @require_enabled
 317     def save_sliver(self, sliver):
 318         if not sliver.instance_id:
 319             if (sliver.slice.name == "smbaker-slice-8") or (sliver.slice.name.startswith("smbaker-slice-net")):
 320                 # only inflict this pain on myself, for now...
 321                 requested_networks = self.get_requested_networks(sliver.slice)
 322             else:
 323                 requested_networks = None
 324             slice_memberships = SliceMembership.objects.filter(slice=sliver.slice)
 325             pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
 326             pubkeys.append(sliver.creator.public_key)
 327             instance = self.driver.spawn_instance(name=sliver.name,
 328                                    key_name = sliver.creator.keyname,
 329                                    image_id = sliver.image.image_id,
 330                                    hostname = sliver.node.name,
 331                                    pubkeys = pubkeys,
 332                                    networks = requested_networks )
 333             sliver.instance_id = instance.id
 334             sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name')
 335
 336         if sliver.instance_id and ("numberCores" in sliver.changed_fields):
 337             self.driver.update_instance_metadata(sliver.instance_id, {"cpu_cores": str(sliver.numberCores)})
 338
 339         sliver.save()
 340         sliver.enacted = datetime.now()
 341         sliver.save(update_fields=['enacted'])
 342
 343     @require_enabled
 344     def delete_sliver(self, sliver):
 345         if sliver.instance_id:
 346             self.driver.destroy_instance(sliver.instance_id)
 347
 348
 349     def refresh_nodes(self):
 350         # collect local nodes
 351         nodes = Node.objects.all()
 352         nodes_dict = {}
 353         for node in nodes:
 354             if 'viccidev10' not in node.name:
 355                 nodes_dict[node.name] = node
 356
 357         deployment = Deployment.objects.filter(name='VICCI')[0]
 358         login_bases = ['princeton', 'stanford', 'gt', 'uw', 'mpisws']
 359         sites = Site.objects.filter(login_base__in=login_bases)
 360         # collect nova nodes:
 361         compute_nodes = self.client.nova.hypervisors.list()
 362
 363         compute_nodes_dict = {}
 364         for compute_node in compute_nodes:
 365             compute_nodes_dict[compute_node.hypervisor_hostname] = compute_node
 366
 367         # add new nodes:
 368         new_node_names = set(compute_nodes_dict.keys()).difference(nodes_dict.keys())
 369         i = 0
 370         max = len(sites)
 371         for name in new_node_names:
 372             if i == max:
 373                 i = 0
 374             site = sites[i]
 375             node = Node(name=compute_nodes_dict[name].hypervisor_hostname,
 376                         site=site,
 377                         deployment=deployment)
 378             node.save()
 379             i+=1
 380
 381         # remove old nodes
 382         old_node_names = set(nodes_dict.keys()).difference(compute_nodes_dict.keys())
 383         Node.objects.filter(name__in=old_node_names).delete()
 384
 385     def refresh_images(self):
 386         from core.models.image import Image
 387         # collect local images
 388         images = Image.objects.all()
 389         images_dict = {}
 390         for image in images:
 391             images_dict[image.name] = image
 392
 393         # collect glance images
 394         glance_images = self.client.glance.get_images()
 395         glance_images_dict = {}
 396         for glance_image in glance_images:
 397             glance_images_dict[glance_image['name']] = glance_image
 398
 399         # add new images
 400         new_image_names = set(glance_images_dict.keys()).difference(images_dict.keys())
 401         for name in new_image_names:
 402             image = Image(image_id=glance_images_dict[name]['id'],
 403                           name=glance_images_dict[name]['name'],
 404                           disk_format=glance_images_dict[name]['disk_format'],
 405                           container_format=glance_images_dict[name]['container_format'])
 406             image.save()
 407
 408         # remove old images
 409         old_image_names = set(images_dict.keys()).difference(glance_images_dict.keys())
 410         Image.objects.filter(name__in=old_image_names).delete()
 411
 412     @require_enabled
 413     def save_network(self, network):
 414         if not network.network_id:
 415             if network.template.sharedNetworkName:
 416                 network.network_id = network.template.sharedNetworkId
 417                 (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id)
 418             else:
 419                 network_name = network.name
 420
 421                 # create network
 422                 os_network = self.driver.create_network(network_name)
 423                 network.network_id = os_network['id']
 424
 425                 # create router
 426                 router = self.driver.create_router(network_name)
 427                 network.router_id = router['id']
 428
 429                 # create subnet
 430                 next_subnet = self.get_next_subnet()
 431                 cidr = str(next_subnet.cidr)
 432                 ip_version = next_subnet.version
 433                 start = str(next_subnet[2])
 434                 end = str(next_subnet[-2])
 435                 subnet = self.driver.create_subnet(name=network_name,
 436                                                    network_id = network.network_id,
 437                                                    cidr_ip = cidr,
 438                                                    ip_version = ip_version,
 439                                                    start = start,
 440                                                    end = end)
 441                 network.subnet = cidr
 442                 network.subnet_id = subnet['id']
 443                 # add subnet as interface to slice's router
 444                 self.driver.add_router_interface(router['id'], subnet['id'])
 445                 # add external route
 446                 self.driver.add_external_route(subnet)
 447
 448         network.save()
 449         network.enacted = datetime.now()
 450         network.save(update_fields=['enacted'])
 451
 452     def delete_network(self, network):
 453         if (network.router_id) and (network.subnet_id):
 454             self.driver.delete_router_interface(network.router_id, network.subnet_id)
 455         if network.subnet_id:
 456             self.driver.delete_subnet(network.subnet_id)
 457         if network.router_id:
 458             self.driver.delete_router(network.router_id)
 459         if network.network_id:
 460             self.driver.delete_network(network.network_id)
 461
 462     def save_network_template(self, template):
 463         if (template.sharedNetworkName) and (not template.sharedNetworkId):
 464             os_networks = self.driver.shell.quantum.list_networks(name=template.sharedNetworkName)['networks']
 465             if os_networks:
 466                 template.sharedNetworkId = os_networks[0]["id"]
 467
 468         template.save()
 469         template.enacted = datetime.now()
 470         template.save(update_fields=['enacted'])
 471
 472     def find_or_make_template_for_network(self, name):
 473         """ Given a network name, try to guess the right template for it """
 474
 475         # templates for networks we may encounter
 476         if name=='nat-net':
 477             template_dict = None # {"name": "private-nat", "visibility": "private", "translation": "nat"}
 478         elif name=='sharednet1':
 479             template_dict = {"name": "dedicated-public", "visibility": "public", "translation": "none"}
 480         else:
 481             template_dict = {"name": "private", "visibility": "private", "translation": "none"}
 482
 483         # if we have an existing template return it
 484         templates = NetworkTemplate.objects.filter(name=template_dict["name"])
 485         if templates:
 486             return templates[0]
 487
 488         if template_dict == None:
 489             return None
 490
 491         template = NetworkTemplate(**template_dict)
 492         template.save()
 493         return template
 494
 495     def refresh_network_templates(self):
 496         for template in NetworkTemplate.objects.all():
 497             if (template.sharedNetworkName) and (not template.sharedNetworkId):
 498                  # this will cause us to try to fill in the sharedNetworkId
 499                  self.save_network_template(template)
 500
 501     def refresh_networks(self):
 502         # get a list of all networks in the model
 503
 504         networks = Network.objects.all()
 505         networks_by_name = {}
 506         networks_by_id = {}
 507         for network in networks:
 508             networks_by_name[network.name] = network
 509             networks_by_id[network.network_id] = network
 510
 511         # Get a list of all shared networks in OS
 512
 513         os_networks = self.driver.shell.quantum.list_networks()['networks']
 514         os_networks_by_name = {}
 515         os_networks_by_id = {}
 516         for os_network in os_networks:
 517             os_networks_by_name[os_network['name']] = os_network
 518             os_networks_by_id[os_network['id']] = os_network
 519
 520         for (uuid, os_network) in os_networks_by_id.items():
 521             #print "checking OS network", os_network['name']
 522             if (os_network['shared']) and (uuid not in networks_by_id):
 523                 # Only automatically create shared networks. This is for Andy's
 524                 # nat-net and sharednet1.
 525
 526                 owner_slice = Slice.objects.get(tenant_id = os_network['tenant_id'])
 527                 template = self.find_or_make_template_for_network(os_network['name'])
 528
 529                 if (template is None):
 530                     # This is our way of saying we don't want to auto-instantiate
 531                     # this network type.
 532                     continue
 533
 534                 (subnet_id, subnet) = self.driver.get_network_subnet(os_network['id'])
 535
 536                 if owner_slice:
 537                     #print "creating model object for OS network", os_network['name']
 538                     new_network = Network(name = os_network['name'],
 539                                           template = template,
 540                                           owner = owner_slice,
 541                                           network_id = uuid,
 542                                           subnet_id = subnet_id)
 543                     new_network.save()
 544
 545         for (network_id, network) in networks_by_id.items():
 546             # If the network disappeared from OS, then reset its network_id to None
 547             if (network.network_id is not None) and (network.network_id not in os_networks_by_id):
 548                 network.network_id = None
 549
 550             # If no OS object exists, then saving the network will create one
 551             if (network.network_id is None):
 552                 #print "creating OS network for", network.name
 553                 self.save_network(network)
 554             else:
 555                 pass #print "network", network.name, "has its OS object"
 556
 557