planetstack/openstack/manager.py

   1 import os
   2 #os.environ.setdefault("DJANGO_SETTINGS_MODULE", "planetstack.settings")
   3 import string
   4 import random
   5 import hashlib
   6 from datetime import datetime
   7
   8 from netaddr import IPAddress, IPNetwork
   9 from planetstack import settings
  10 from django.core import management
  11 from core.models import *
  12 from planetstack.config import Config
  13 try:
  14     from openstack.client import OpenStackClient
  15     from openstack.driver import OpenStackDriver
  16     has_openstack = True
  17 except:
  18     has_openstack = False
  19
  20 manager_enabled = Config().api_nova_enabled
  21
  22
  23 def random_string(size=6):
  24     return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(size))
  25
  26 def require_enabled(callable):
  27     def wrapper(*args, **kwds):
  28         if manager_enabled and has_openstack:
  29             return callable(*args, **kwds)
  30         else:
  31             return None
  32     return wrapper
  33
  34
  35 class OpenStackManager:
  36
  37     def __init__(self, auth={}, caller=None):
  38         self.client = None
  39         self.driver = None
  40         self.caller = None
  41         self.has_openstack = has_openstack
  42         self.enabled = manager_enabled
  43
  44         if has_openstack and manager_enabled:
  45             if auth:
  46                 try:
  47                     self.init_user(auth, caller)
  48                 except:
  49                     # if this fails then it meanse the caller doesn't have a
  50                     # role at the slice's tenant. if the caller is an admin
  51                     # just use the admin client/manager.
  52                     if caller and caller.is_admin:
  53                         self.init_admin()
  54                     else: raise
  55             else:
  56                 self.init_admin()
  57
  58     @require_enabled
  59     def init_caller(self, caller, tenant):
  60         auth = {'username': caller.email,
  61                 'password': hashlib.md5(caller.password).hexdigest()[:6],
  62                 'tenant': tenant}
  63         self.client = OpenStackClient(**auth)
  64         self.driver = OpenStackDriver(client=self.client)
  65         self.caller = caller
  66
  67     @require_enabled
  68     def init_admin(self, tenant=None):
  69         # use the admin credentials
  70         self.client = OpenStackClient(tenant=tenant)
  71         self.driver = OpenStackDriver(client=self.client)
  72         self.caller = self.driver.admin_user
  73         self.caller.kuser_id = self.caller.id
  74
  75     @require_enabled
  76     def save_role(self, role):
  77         if not role.role:
  78             keystone_role = self.driver.create_role(role.role_type)
  79             role.role = keystone_role.id
  80
  81     @require_enabled
  82     def delete_role(self, role):
  83         if role.role:
  84             self.driver.delete_role({'id': role.role})
  85
  86     @require_enabled
  87     def save_key(self, key, name):
  88         key_fields = {'name': name,
  89                       'public_key': key}
  90         nova_key = self.driver.create_keypair(**key_fields)
  91
  92     @require_enabled
  93     def delete_key(self, key):
  94         if key.nkey_id:
  95             self.driver.delete_keypair(key.nkey_id)
  96
  97     @require_enabled
  98     def save_user(self, user):
  99         name = user.email[:user.email.find('@')]
 100         user_fields = {'name': name,
 101                        'email': user.email,
 102                        'password': hashlib.md5(user.password).hexdigest()[:6],
 103                        'enabled': True}
 104         if not user.kuser_id:
 105             keystone_user = self.driver.create_user(**user_fields)
 106             user.kuser_id = keystone_user.id
 107         else:
 108             self.driver.update_user(user.kuser_id, user_fields)
 109
 110         if user.site:
 111             self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'user')
 112             if user.is_admin:
 113                 self.driver.add_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 114             else:
 115                 # may have admin role so attempt to remove it
 116                 self.driver.delete_user_role(user.kuser_id, user.site.tenant_id, 'admin')
 117
 118         if user.public_key:
 119             self.init_caller(user, user.site.login_base)
 120             self.save_key(user.public_key, user.keyname)
 121             self.init_admin()
 122
 123         user.save()
 124         user.enacted = datetime.now()
 125         user.save(update_fields=['enacted'])
 126
 127     @require_enabled
 128     def delete_user(self, user):
 129         if user.kuser_id:
 130             self.driver.delete_user(user.kuser_id)
 131
 132     @require_enabled
 133     def save_site(self, site, add_role=True):
 134         if not site.tenant_id:
 135             tenant = self.driver.create_tenant(tenant_name=site.login_base,
 136                                                description=site.name,
 137                                                enabled=site.enabled)
 138             site.tenant_id = tenant.id
 139             # give caller an admin role at the tenant they've created
 140             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 141
 142         # update the record
 143         if site.id and site.tenant_id:
 144             self.driver.update_tenant(site.tenant_id,
 145                                       description=site.name,
 146                                       enabled=site.enabled)
 147
 148         # commit the updated record
 149         site.save()
 150         site.enacted = datetime.now()
 151         site.save(update_fields=['enacted']) # enusre enacted > updated
 152
 153
 154     @require_enabled
 155     def delete_site(self, site):
 156         if site.tenant_id:
 157             self.driver.delete_tenant(site.tenant_id)
 158
 159     @require_enabled
 160     def save_site_privilege(self, site_priv):
 161         if site_priv.user.kuser_id and site_priv.site.tenant_id:
 162             self.driver.add_user_role(site_priv.user.kuser_id,
 163                                       site_priv.site.tenant_id,
 164                                       site_priv.role.role_type)
 165         site_priv.enacted = datetime.now()
 166         site_priv.save(update_fields=['enacted'])
 167
 168
 169     @require_enabled
 170     def delete_site_privilege(self, site_priv):
 171         self.driver.delete_user_role(site_priv.user.kuser_id,
 172                                      site_priv.site.tenant_id,
 173                                      site_priv.role.role_type)
 174
 175     @require_enabled
 176     def save_slice(self, slice):
 177         if not slice.tenant_id:
 178             nova_fields = {'tenant_name': slice.name,
 179                    'description': slice.description,
 180                    'enabled': slice.enabled}
 181             tenant = self.driver.create_tenant(**nova_fields)
 182             slice.tenant_id = tenant.id
 183
 184             # give caller an admin role at the tenant they've created
 185             self.driver.add_user_role(self.caller.kuser_id, tenant.id, 'admin')
 186
 187             # refresh credentials using this tenant
 188             self.driver.shell.connect(username=self.driver.shell.keystone.username,
 189                                       password=self.driver.shell.keystone.password,
 190                                       tenant=tenant.name)
 191
 192             # create network
 193             network = self.driver.create_network(slice.name)
 194             slice.network_id = network['id']
 195
 196             # create router
 197             router = self.driver.create_router(slice.name)
 198             slice.router_id = router['id']
 199
 200             # create subnet
 201             next_subnet = self.get_next_subnet()
 202             cidr = str(next_subnet.cidr)
 203             ip_version = next_subnet.version
 204             start = str(next_subnet[2])
 205             end = str(next_subnet[-2])
 206             subnet = self.driver.create_subnet(name=slice.name,
 207                                                network_id = network['id'],
 208                                                cidr_ip = cidr,
 209                                                ip_version = ip_version,
 210                                                start = start,
 211                                                end = end)
 212             slice.subnet_id = subnet['id']
 213             # add subnet as interface to slice's router
 214             self.driver.add_router_interface(router['id'], subnet['id'])
 215             # add external route
 216             self.driver.add_external_route(subnet)
 217
 218
 219         if slice.id and slice.tenant_id:
 220             self.driver.update_tenant(slice.tenant_id,
 221                                       description=slice.description,
 222                                       enabled=slice.enabled)
 223
 224         slice.save()
 225         slice.enacted = datetime.now()
 226         slice.save(update_fields=['enacted'])
 227
 228     @require_enabled
 229     def delete_slice(self, slice):
 230         if slice.tenant_id:
 231             self._delete_slice(slice.tenant_id, slice.network_id,
 232                                slice.router_id, slice.subnet_id)
 233     @require_enabled
 234     def _delete_slice(self, tenant_id, network_id, router_id, subnet_id):
 235         self.driver.delete_router_interface(slice.router_id, slice.subnet_id)
 236         self.driver.delete_subnet(slice.subnet_id)
 237         self.driver.delete_router(slice.router_id)
 238         self.driver.delete_network(slice.network_id)
 239         self.driver.delete_tenant(slice.tenant_id)
 240         # delete external route
 241         subnet = None
 242         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 243         for snet in subnets:
 244             if snet['id'] == slice.subnet_id:
 245                 subnet = snet
 246         if subnet:
 247             self.driver.delete_external_route(subnet)
 248
 249
 250     @require_enabled
 251     def save_slice_membership(self, slice_memb):
 252         if slice_memb.user.kuser_id and slice_memb.slice.tenant_id:
 253             self.driver.add_user_role(slice_memb.user.kuser_id,
 254                                       slice_memb.slice.tenant_id,
 255                                       slice_memb.role.role_type)
 256         slice_memb.enacted = datetime.now()
 257         slice_memb.save(update_fields=['enacted'])
 258
 259
 260     @require_enabled
 261     def delete_slice_membership(self, slice_memb):
 262         self.driver.delete_user_role(slice_memb.user.kuser_id,
 263                                      slice_memb.slice.tenant_id,
 264                                      slice_memb.role.role_type)
 265
 266
 267     @require_enabled
 268     def get_next_subnet(self):
 269         # limit ourself to 10.0.x.x for now
 270         valid_subnet = lambda net: net.startswith('10.0')
 271         subnets = self.driver.shell.quantum.list_subnets()['subnets']
 272         ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
 273                 if valid_subnet(subnet['cidr'])]
 274         ints.sort()
 275         last_ip = IPAddress(ints[-1])
 276         last_network = IPNetwork(str(last_ip) + "/24")
 277         next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
 278         return next_network
 279
 280     @require_enabled
 281     def save_subnet(self, subnet):
 282         if not subnet.subnet_id:
 283             quantum_subnet = self.driver.create_subnet(name= subnet.slice.name,
 284                                           network_id=subnet.slice.network_id,
 285                                           cidr_ip = subnet.cidr,
 286                                           ip_version=subnet.ip_version,
 287                                           start = subnet.start,
 288                                           end = subnet.end)
 289             subnet.subnet_id = quantum_subnet['id']
 290             # add subnet as interface to slice's router
 291             self.driver.add_router_interface(subnet.slice.router_id, subnet.subnet_id)
 292             #add_route = 'route add -net %s dev br-ex gw 10.100.0.5' % self.cidr
 293             #commands.getstatusoutput(add_route)
 294
 295
 296     @require_enabled
 297     def delete_subnet(self, subnet):
 298         if subnet.subnet_id:
 299             self.driver.delete_router_interface(subnet.slice.router_id, subnet.subnet_id)
 300             self.driver.delete_subnet(subnet.subnet_id)
 301             #del_route = 'route del -net %s' % self.cidr
 302             #commands.getstatusoutput(del_route)
 303
 304     def get_requested_networks(self, slice):
 305         network_ids = [x.network_id for x in slice.networks.all()]
 306
 307         if slice.network_id is not None:
 308             network_ids.append(slice.network_id)
 309
 310         networks = []
 311         for network_id in network_ids:
 312             networks.append({"net-id": network_id})
 313
 314         return networks
 315
 316     @require_enabled
 317     def save_sliver(self, sliver):
 318         metadata_update = {}
 319         if ("numberCores" in sliver.changed_fields):
 320             metadata_update["cpu_cores"] = str(sliver.numberCores)
 321
 322         for tag in sliver.slice.tags.all():
 323             if tag.name.startswith("sysctl-"):
 324                 metadata_update[tag.name] = tag.value
 325
 326         if not sliver.instance_id:
 327             nics = self.get_requested_networks(sliver.slice)
 328             for nic in nics:
 329                 # If a network hasn't been instantiated yet, then we'll fail
 330                 # during slice creation. Defer saving the sliver for now.
 331                 if not nic.get("net-id", None):
 332                     sliver.save()   # in case it hasn't been saved yet
 333                     return
 334             slice_memberships = SliceMembership.objects.filter(slice=sliver.slice)
 335             pubkeys = [sm.user.public_key for sm in slice_memberships if sm.user.public_key]
 336             pubkeys.append(sliver.creator.public_key)
 337             instance = self.driver.spawn_instance(name=sliver.name,
 338                                    key_name = sliver.creator.keyname,
 339                                    image_id = sliver.image.image_id,
 340                                    hostname = sliver.node.name,
 341                                    pubkeys = pubkeys,
 342                                    nics = nics,
 343                                    metadata = metadata_update )
 344             sliver.instance_id = instance.id
 345             sliver.instance_name = getattr(instance, 'OS-EXT-SRV-ATTR:instance_name')
 346         else:
 347             if metadata_update:
 348                 self.driver.update_instance_metadata(sliver.instance_id, metadata_update)
 349
 350         sliver.save()
 351         sliver.enacted = datetime.now()
 352         sliver.save(update_fields=['enacted'])
 353
 354     @require_enabled
 355     def delete_sliver(self, sliver):
 356         if sliver.instance_id:
 357             self.driver.destroy_instance(sliver.instance_id)
 358
 359
 360     def refresh_nodes(self):
 361         # collect local nodes
 362         nodes = Node.objects.all()
 363         nodes_dict = {}
 364         for node in nodes:
 365             if 'viccidev10' not in node.name:
 366                 nodes_dict[node.name] = node
 367
 368         deployment = Deployment.objects.filter(name='VICCI')[0]
 369         login_bases = ['princeton', 'stanford', 'gt', 'uw', 'mpisws']
 370         sites = Site.objects.filter(login_base__in=login_bases)
 371         # collect nova nodes:
 372         compute_nodes = self.client.nova.hypervisors.list()
 373
 374         compute_nodes_dict = {}
 375         for compute_node in compute_nodes:
 376             compute_nodes_dict[compute_node.hypervisor_hostname] = compute_node
 377
 378         # add new nodes:
 379         new_node_names = set(compute_nodes_dict.keys()).difference(nodes_dict.keys())
 380         i = 0
 381         max = len(sites)
 382         for name in new_node_names:
 383             if i == max:
 384                 i = 0
 385             site = sites[i]
 386             node = Node(name=compute_nodes_dict[name].hypervisor_hostname,
 387                         site=site,
 388                         deployment=deployment)
 389             node.save()
 390             i+=1
 391
 392         # remove old nodes
 393         old_node_names = set(nodes_dict.keys()).difference(compute_nodes_dict.keys())
 394         Node.objects.filter(name__in=old_node_names).delete()
 395
 396     def refresh_images(self):
 397         from core.models.image import Image
 398         # collect local images
 399         images = Image.objects.all()
 400         images_dict = {}
 401         for image in images:
 402             images_dict[image.name] = image
 403
 404         # collect glance images
 405         glance_images = self.client.glance.get_images()
 406         glance_images_dict = {}
 407         for glance_image in glance_images:
 408             glance_images_dict[glance_image['name']] = glance_image
 409
 410         # add new images
 411         new_image_names = set(glance_images_dict.keys()).difference(images_dict.keys())
 412         for name in new_image_names:
 413             image = Image(image_id=glance_images_dict[name]['id'],
 414                           name=glance_images_dict[name]['name'],
 415                           disk_format=glance_images_dict[name]['disk_format'],
 416                           container_format=glance_images_dict[name]['container_format'])
 417             image.save()
 418
 419         # remove old images
 420         old_image_names = set(images_dict.keys()).difference(glance_images_dict.keys())
 421         Image.objects.filter(name__in=old_image_names).delete()
 422
 423     @require_enabled
 424     def save_network(self, network):
 425         if not network.network_id:
 426             if network.template.shared_network_name:
 427                 network.network_id = network.template.shared_network_id
 428                 (network.subnet_id, network.subnet) = self.driver.get_network_subnet(network.network_id)
 429             else:
 430                 network_name = network.name
 431
 432                 # create network
 433                 os_network = self.driver.create_network(network_name, shared=True)
 434                 network.network_id = os_network['id']
 435
 436                 # create router
 437                 router = self.driver.create_router(network_name)
 438                 network.router_id = router['id']
 439
 440                 # create subnet
 441                 next_subnet = self.get_next_subnet()
 442                 cidr = str(next_subnet.cidr)
 443                 ip_version = next_subnet.version
 444                 start = str(next_subnet[2])
 445                 end = str(next_subnet[-2])
 446                 subnet = self.driver.create_subnet(name=network_name,
 447                                                    network_id = network.network_id,
 448                                                    cidr_ip = cidr,
 449                                                    ip_version = ip_version,
 450                                                    start = start,
 451                                                    end = end)
 452                 network.subnet = cidr
 453                 network.subnet_id = subnet['id']
 454                 # add subnet as interface to slice's router
 455                 self.driver.add_router_interface(router['id'], subnet['id'])
 456                 # add external route
 457                 self.driver.add_external_route(subnet)
 458
 459         network.save()
 460         network.enacted = datetime.now()
 461         network.save(update_fields=['enacted'])
 462
 463     def delete_network(self, network):
 464         if (network.router_id) and (network.subnet_id):
 465             self.driver.delete_router_interface(network.router_id, network.subnet_id)
 466         if network.subnet_id:
 467             self.driver.delete_subnet(network.subnet_id)
 468         if network.router_id:
 469             self.driver.delete_router(network.router_id)
 470         if network.network_id:
 471             self.driver.delete_network(network.network_id)
 472
 473     def save_network_template(self, template):
 474         if (template.shared_network_name) and (not template.shared_network_id):
 475             os_networks = self.driver.shell.quantum.list_networks(name=template.shared_network_name)['networks']
 476             if os_networks:
 477                 template.shared_network_id = os_networks[0]["id"]
 478
 479         template.save()
 480         template.enacted = datetime.now()
 481         template.save(update_fields=['enacted'])
 482
 483     def find_or_make_template_for_network(self, name):
 484         """ Given a network name, try to guess the right template for it """
 485
 486         # templates for networks we may encounter
 487         if name=='nat-net':
 488             template_dict = None # {"name": "private-nat", "visibility": "private", "translation": "nat"}
 489         elif name=='sharednet1':
 490             template_dict = {"name": "dedicated-public", "visibility": "public", "translation": "none"}
 491         else:
 492             template_dict = {"name": "private", "visibility": "private", "translation": "none"}
 493
 494         # if we have an existing template return it
 495         templates = NetworkTemplate.objects.filter(name=template_dict["name"])
 496         if templates:
 497             return templates[0]
 498
 499         if template_dict == None:
 500             return None
 501
 502         template = NetworkTemplate(**template_dict)
 503         template.save()
 504         return template
 505
 506     def refresh_network_templates(self):
 507         for template in NetworkTemplate.objects.all():
 508             if (template.shared_network_name) and (not template.shared_network_id):
 509                  # this will cause us to try to fill in the shared_network_id
 510                  self.save_network_template(template)
 511
 512     def refresh_networks(self):
 513         # get a list of all networks in the model
 514
 515         networks = Network.objects.all()
 516         networks_by_name = {}
 517         networks_by_id = {}
 518         for network in networks:
 519             networks_by_name[network.name] = network
 520             networks_by_id[network.network_id] = network
 521
 522         # Get a list of all shared networks in OS
 523
 524         os_networks = self.driver.shell.quantum.list_networks()['networks']
 525         os_networks_by_name = {}
 526         os_networks_by_id = {}
 527         for os_network in os_networks:
 528             os_networks_by_name[os_network['name']] = os_network
 529             os_networks_by_id[os_network['id']] = os_network
 530
 531         for (uuid, os_network) in os_networks_by_id.items():
 532             #print "checking OS network", os_network['name']
 533             if (os_network['shared']) and (uuid not in networks_by_id):
 534                 # Only automatically create shared networks. This is for Andy's
 535                 # nat-net and sharednet1.
 536
 537                 owner_slice = Slice.objects.get(tenant_id = os_network['tenant_id'])
 538                 template = self.find_or_make_template_for_network(os_network['name'])
 539
 540                 if (template is None):
 541                     # This is our way of saying we don't want to auto-instantiate
 542                     # this network type.
 543                     continue
 544
 545                 (subnet_id, subnet) = self.driver.get_network_subnet(os_network['id'])
 546
 547                 if owner_slice:
 548                     #print "creating model object for OS network", os_network['name']
 549                     new_network = Network(name = os_network['name'],
 550                                           template = template,
 551                                           owner = owner_slice,
 552                                           network_id = uuid,
 553                                           subnet_id = subnet_id)
 554                     new_network.save()
 555
 556         for (network_id, network) in networks_by_id.items():
 557             # If the network disappeared from OS, then reset its network_id to None
 558             if (network.network_id is not None) and (network.network_id not in os_networks_by_id):
 559                 network.network_id = None
 560
 561             # If no OS object exists, then saving the network will create one
 562             if (network.network_id is None):
 563                 #print "creating OS network for", network.name
 564                 self.save_network(network)
 565             else:
 566                 pass #print "network", network.name, "has its OS object"
 567
 568