planetstack/core/plus/views.py

   1 #views.py
   2 import functools
   3 import math
   4 import os
   5 import sys
   6 from django.views.generic import TemplateView, View
   7 import datetime
   8 from pprint import pprint
   9 import json
  10 from syndicate.models import *
  11 from core.models import *
  12 from hpc.models import ContentProvider
  13 from operator import attrgetter
  14 from django import template
  15 from django.views.decorators.csrf import csrf_exempt
  16 from django.http import HttpResponse, HttpResponseServerError, HttpResponseForbidden
  17 from django.core import urlresolvers
  18 from django.contrib.gis.geoip import GeoIP
  19 from django.db.models import Q
  20 from ipware.ip import get_ip
  21 from operator import itemgetter, attrgetter
  22 import traceback
  23 import socket
  24
  25 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
  26
  27 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
  28     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
  29 else:
  30     sys.path.append("/opt/planetstack/hpc_wizard")
  31 import hpc_wizard
  32 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
  33
  34 class DashboardWelcomeView(TemplateView):
  35     template_name = 'admin/dashboard/welcome.html'
  36
  37     def get(self, request, *args, **kwargs):
  38         context = self.get_context_data(**kwargs)
  39         context = getDashboardContext(request.user, context)
  40         return self.render_to_response(context=context)
  41
  42 class DashboardDynamicView(TemplateView):
  43     head_template = r"""{% extends "admin/dashboard/dashboard_base.html" %}
  44        {% load admin_static %}
  45        {% block content %}
  46     """
  47
  48     tail_template = r"{% endblock %}"
  49
  50     def get(self, request, name="root", *args, **kwargs):
  51         context = self.get_context_data(**kwargs)
  52         context = getDashboardContext(request.user, context)
  53
  54         if name=="root":
  55             return self.multiDashboardView(request, context)
  56         else:
  57             return self.singleDashboardView(request, name, context)
  58
  59     def readDashboard(self, fn):
  60         try:
  61             template= open("/opt/planetstack/templates/admin/dashboard/%s.html" % fn, "r").read()
  62             if (fn=="tenant"):
  63                 # fix for tenant view - it writes html to a div called tabs-5
  64                 template = '<div id="tabs-5"></div>' + template
  65             return template
  66         except:
  67             return "failed to open %s" % fn
  68
  69     def multiDashboardView(self, request, context):
  70         head_template = self.head_template
  71         tail_template = self.tail_template
  72
  73         body = """
  74          <div id="hometabs" >
  75          <ul id="suit_form_tabs" class="nav nav-tabs nav-tabs-suit" data-tab-prefix="suit-tab">
  76         """
  77
  78         dashboards = request.user.get_dashboards()
  79
  80         # customize is a special dashboard they always get
  81         customize = DashboardView.objects.filter(name="Customize")
  82         if customize:
  83             dashboards.append(customize[0])
  84
  85         for i,view in enumerate(dashboards):
  86             body = body + '<li><a href="#dashtab-%d">%s</a></li>\n' % (i, view.name)
  87
  88         body = body + "</ul>\n"
  89
  90         for i,view in enumerate(dashboards):
  91             url = view.url
  92             body = body + '<div id="dashtab-%d">\n' % i
  93             if url.startswith("template:"):
  94                 fn = url[9:]
  95                 body = body + self.readDashboard(fn)
  96             body = body + '</div>\n'
  97
  98         body=body+"</div>\n"
  99
 100         t = template.Template(head_template + body + self.tail_template)
 101
 102         response_kwargs = {}
 103         response_kwargs.setdefault('content_type', self.content_type)
 104         return self.response_class(\r
 105             request = request,\r
 106             template = t,\r
 107             context = context,\r
 108             **response_kwargs)
 109
 110     def singleDashboardView(self, request, name, context):
 111         head_template = self.head_template
 112         tail_template = self.tail_template
 113
 114         t = template.Template(head_template + self.readDashboard(name) + self.tail_template)
 115
 116         response_kwargs = {}
 117         response_kwargs.setdefault('content_type', self.content_type)
 118         return self.response_class(\r
 119             request = request,\r
 120             template = t,\r
 121             context = context,\r
 122             **response_kwargs)
 123
 124 def getDashboardContext(user, context={}, tableFormat = False):
 125         context = {}
 126
 127         userSliceData = getSliceInfo(user)
 128         if (tableFormat):
 129             context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
 130         else:
 131             context['userSliceInfo'] = userSliceData
 132         context['cdnData'] = getCDNOperatorData(wait=False)
 133         context['cdnContentProviders'] = getCDNContentProviderData()
 134
 135         (dashboards, unusedDashboards)= getDashboards(user)
 136         unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
 137         context['dashboards'] = dashboards
 138         context['unusedDashboards'] = unusedDashboards
 139
 140         return context
 141
 142 def getDashboards(user):
 143     #dashboards = sorted(list(user.dashboardViews.all()), key=attrgetter('order'))
 144     dashboards = user.get_dashboards()
 145
 146     dashboard_names = [d.name for d in dashboards]
 147
 148     unused_dashboard_names = []
 149     for dashboardView in DashboardView.objects.all():
 150         if not dashboardView.name in dashboard_names:
 151             unused_dashboard_names.append(dashboardView.name)
 152
 153     return (dashboard_names, unused_dashboard_names)
 154
 155 class TenantCreateSlice(View):
 156     def post(self, request, *args, **kwargs):
 157         if request.user.isReadOnlyUser():
 158             return HttpResponseForbidden("User is in read-only mode")
 159
 160         sliceName = request.POST.get("sliceName", "0")
 161         serviceClass = request.POST.get("serviceClass", "0")
 162         imageName = request.POST.get("imageName", "0")
 163         actionToDo = request.POST.get("actionToDo", "0")
 164         networkPorts = request.POST.get("network","0")
 165         mountDataSets = request.POST.get("mountDataSets","0")
 166         privateVolume = request.POST.get("privateVolume","0")
 167         if (actionToDo == "add"):
 168            serviceClass = ServiceClass.objects.get(name=serviceClass)
 169            site = request.user.site
 170            image = Image.objects.get(name=imageName)
 171            newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets)
 172            newSlice.save()
 173            privateTemplate="Private"
 174            publicTemplate="Public shared IPv4"\r
 175            privateNetworkName = sliceName+"-"+privateTemplate\r
 176            publicNetworkName = sliceName+"-"+publicTemplate\r
 177            slice=Slice.objects.get(name=sliceName)\r
 178            addNetwork(privateNetworkName,privateTemplate,slice)\r
 179            addNetwork(publicNetworkName,publicTemplate,slice)\r
 180            addOrModifyPorts(networkPorts,sliceName)\r
 181            if privateVolume=="true":\r
 182                 privateVolForSlice(request.user,sliceName)
 183         return HttpResponse("Slice created")
 184
 185 def privateVolForSlice(user,sliceName):
 186         if not hasPrivateVolume(sliceName):\r
 187            volumeName=createPrivateVolume(user,sliceName)\r
 188            readWrite="true"\r
 189            mountVolume(sliceName,volumeName,readWrite)
 190
 191 class TenantUpdateSlice(View):
 192     def post(self, request, *args, **kwargs):\r
 193         if request.user.isReadOnlyUser():\r
 194             return HttpResponseForbidden("User is in read-only mode")\r
 195 \r
 196         sliceName = request.POST.get("sliceName", "0")\r
 197         serviceClass = request.POST.get("serviceClass", "0")\r
 198         imageName = request.POST.get("imageName", "0")\r
 199         actionToDo = request.POST.get("actionToDo", "0")\r
 200         networkPorts = request.POST.get("networkPorts","0")\r
 201         dataSet = request.POST.get("dataSet","0")\r
 202         privateVolume = request.POST.get("privateVolume","0")\r
 203         slice = Slice.objects.all()\r
 204         for entry in slice:\r
 205                 serviceClass = ServiceClass.objects.get(name=serviceClass)\r
 206                 if(entry.name==sliceName):\r
 207                          if (actionToDo == "update"):\r
 208                                 setattr(entry,'serviceClass',serviceClass)\r
 209                                 setattr(entry,'imagePreference',imageName)\r
 210                                 setattr(entry,'mountDataSets',dataSet)\r
 211                                 entry.save()\r
 212                                 break\r
 213         addOrModifyPorts(networkPorts,sliceName)\r
 214         if privateVolume=="true":\r
 215                 privateVolForSlice(request.user,sliceName)\r
 216         return HttpResponse("Slice updated")\r
 217 \r
 218 def addNetwork(name,template,sliceName):\r
 219         networkTemplate=NetworkTemplate.objects.get(name=template)\r
 220         newNetwork = Network(name = name,\r
 221                               template = networkTemplate,\r
 222                               owner = sliceName)\r
 223         newNetwork.save()\r
 224         addNetworkSlice(newNetwork,sliceName)\r
 225 \r
 226 def addNetworkSlice(networkSlice,sliceName):\r
 227         newNetworkSlice=NetworkSlice(network =networkSlice,\r
 228                                      slice=sliceName)\r
 229         newNetworkSlice.save()\r
 230 \r
 231 def addOrModifyPorts(networkPorts,sliceName):\r
 232         networkList = Network.objects.all()\r
 233         networkInfo = []\r
 234         if networkPorts:\r
 235            for networkEntry in networkList:\r
 236                networkSlices = networkEntry.slices.all()\r
 237                for slice in networkSlices:\r
 238                    if slice.name==sliceName:\r
 239                           if networkEntry.template.name=="Public shared IPv4":\r
 240                              setattr(networkEntry,'ports',networkPorts)\r
 241                              networkEntry.save()\r
 242 \r
 243 def getTenantSliceInfo(user, tableFormat = False):
 244     tenantSliceDetails = {}
 245     tenantSliceData = getTenantInfo(user)
 246     tenantServiceClassData = getServiceClassInfo(user)
 247     if (tableFormat):
 248        tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
 249        tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
 250     else:
 251        tenantSliceDetails['userSliceInfo'] = tenantSliceData
 252     tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
 253     tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
 254     tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
 255     tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
 256     tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
 257     tenantSliceDetails['publicKey'] = getPublicKey(user)
 258     return tenantSliceDetails
 259
 260 def getTenantInfo(user):
 261     slices =Slice.objects.all()
 262     userSliceInfo = []
 263     for entry in slices:
 264        sliceName = Slice.objects.get(id=entry.id).name
 265        slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
 266        sliceServiceClass = entry.serviceClass.name
 267        preferredImage =  entry.imagePreference
 268        #sliceDataSet = entry.mountDataSets
 269        sliceNetwork = {}
 270        numSliver = 0
 271        sliceImage=""
 272        sliceSite = {}
 273        sliceNode = {}
 274        sliceInstance= {}
 275        #createPrivateVolume(user,sliceName)
 276        for sliver in slice.slivers.all():
 277             if sliver.node.site.name in BLESSED_SITES:
 278                 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
 279                 sliceImage = sliver.image.name
 280                 sliceNode[str(sliver)] = sliver.node.name
 281        numSliver = sum(sliceSite.values())
 282        numSites = len(sliceSite)
 283        userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'instanceNodePair':sliceNode})
 284     return userSliceInfo
 285
 286 def getTenantSitesInfo():
 287         tenantSiteInfo=[]
 288         for entry in Site.objects.all():
 289             if entry.name in BLESSED_SITES:
 290                  tenantSiteInfo.append({'siteName':entry.name})
 291         return tenantSiteInfo
 292
 293 def userSliceTableFormatter(data):
 294 #    pprint(data)
 295     formattedData = {
 296                      'rows' : data
 297                     }
 298     return formattedData
 299
 300 def getPublicKey(user):
 301         users=User.objects.all()\r
 302         for key in users:\r
 303                 if (str(key.email)==str(user)):\r
 304                         sshKey = key.public_key\r
 305         return sshKey
 306
 307 def getServiceClassInfo(user):
 308     serviceClassList = ServiceClass.objects.all()
 309     sliceInfo = []
 310     for entry in serviceClassList:
 311           sliceInfo.append({'serviceClass':entry.name})
 312     return sliceInfo
 313
 314 def getImageInfo(user):
 315     imageList = Image.objects.all()
 316     #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
 317     imageInfo = []
 318     for imageEntry in imageList:
 319           imageInfo.append({'Image':imageEntry.name})
 320           #imageInfo.append({'Image':imageEntry})
 321     return imageInfo
 322
 323 def createPrivateVolume(user, sliceName):
 324     caps = Volume.CAP_READ_DATA | Volume.CAP_WRITE_DATA | Volume.CAP_HOST_DATA
 325     getattr(Volume.default_gateway_caps,"read data") | \
 326            getattr(Volume.default_gateway_caps,"write data") | \
 327            getattr(Volume.default_gateway_caps,"host files")
 328     v = Volume(name="private_" + sliceName, owner_id=user, description="private volume for %s" % sliceName, blocksize=61440, private=True, archive=False, default_gateway_caps = caps)
 329     v.save()
 330     return v
 331
 332 SYNDICATE_REPLICATE_PORTNUM = 1025
 333
 334 def get_free_port():
 335     inuse={}
 336     inuse[SYNDICATE_REPLICATE_PORTNUM] = True
 337     for vs in VolumeSlice.objects.all():
 338         inuse[vs.peer_portnum]=True
 339         inuse[vs.replicate_portnum]=True
 340     for network in Network.objects.all():
 341         if not network.ports:
 342             continue
 343         network_ports = [x.strip() for x in network.ports.split(",")]
 344         for network_port in network_ports:
 345             try:
 346                 inuse[int(network_port)] = True
 347             except:
 348                 # in case someone has put a malformed port number in the list
 349                 pass
 350     for i in range(1025, 65535):
 351         if not inuse.get(i,False):
 352             return i
 353     return False
 354
 355 def mountVolume(sliceName, volumeName, readWrite):
 356     slice = Slice.objects.get(name=sliceName)
 357     volume = Volume.objects.get(name=volumeName)
 358     # choose some unused port numbers
 359     flags = Volume.CAP_READ_DATA
 360     if readWrite:
 361         flags = flags | Volume.CAP_WRITE_DATA
 362     vs = VolumeSlice(volume_id = volume, slice_id = slice, gateway_caps=flags, peer_portnum = get_free_port(), replicate_portnum = SYNDICATE_REPLICATE_PORTNUM)
 363     vs.save()
 364
 365 def hasPrivateVolume(sliceName):
 366      slice = Slice.objects.get(name=sliceName)
 367      for vs in VolumeSlice.objects.filter(slice_id=slice):
 368          if vs.volume_id.private:
 369              return True
 370      return False
 371
 372 def getMountDataSets():
 373         dataSetInfo=[]\r
 374         for volume in Volume.objects.all():\r
 375             if not volume.private:\r
 376                 dataSetInfo.append({'DataSet': volume.name})\r
 377 \r
 378         return dataSetInfo
 379
 380 def getDeploymentSites():
 381     deploymentList = Deployment.objects.all()
 382     deploymentInfo = []
 383     for entry in deploymentList:
 384         deploymentInfo.append({'DeploymentSite':entry.name})
 385     return deploymentInfo
 386
 387 def getSliceInfo(user):
 388     sliceList = Slice.objects.all()
 389     slicePrivs = SlicePrivilege.objects.filter(user=user)
 390     userSliceInfo = []
 391     for entry in slicePrivs:
 392
 393         slicename = Slice.objects.get(id=entry.slice.id).name
 394         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
 395         sliverList=Sliver.objects.all()
 396         sites_used = {}
 397         for sliver in slice.slivers.all():
 398              #sites_used['deploymentSites'] = sliver.node.deployment.name
 399              # sites_used[sliver.image.name] = sliver.image.name
 400              sites_used[sliver.node.site.name] = sliver.numberCores
 401         sliceid = Slice.objects.get(id=entry.slice.id).id
 402         try:
 403             sliverList = Sliver.objects.filter(slice=entry.slice.id)
 404             siteList = {}
 405             for x in sliverList:
 406                if x.node.site not in siteList:
 407                   siteList[x.node.site] = 1
 408             slivercount = len(sliverList)
 409             sitecount = len(siteList)
 410         except:
 411             traceback.print_exc()
 412             slivercount = 0
 413             sitecount = 0
 414
 415         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
 416                               'sitesUsed':sites_used,
 417                               'role': SliceRole.objects.get(id=entry.role.id).role,
 418                               'slivercount': slivercount,
 419                               'sitecount':sitecount})
 420
 421     return userSliceInfo
 422
 423 def getCDNContentProviderData():
 424     cps = []
 425     for dm_cp in ContentProvider.objects.all():
 426         cp = {"name": dm_cp.name,
 427               "account": dm_cp.account}
 428         cps.append(cp)
 429
 430     return cps
 431
 432 def getCDNOperatorData(randomizeData = False, wait=True):
 433     HPC_SLICE_NAME = "HyperCache"
 434
 435     bq = PlanetStackAnalytics()
 436
 437     rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
 438
 439     # wait=False on the first time the Dashboard is opened. This means we might
 440     # not have any rows yet. The dashboard code polls every 30 seconds, so it
 441     # will eventually pick them up.
 442
 443     if rows:
 444         rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
 445
 446         # dictionaryize the statistics rows by site name
 447         stats_rows = {}
 448         for row in rows:
 449             stats_rows[row["site"]] = row
 450     else:
 451         stats_rows = {}
 452
 453     slice = Slice.objects.filter(name=HPC_SLICE_NAME)
 454     if slice:
 455         slice_slivers = list(slice[0].slivers.all())
 456     else:
 457         slice_slivers = []
 458
 459     new_rows = {}
 460     for site in Site.objects.all():
 461         # compute number of slivers allocated in the data model
 462         allocated_slivers = 0
 463         for sliver in slice_slivers:
 464             if sliver.node.site == site:
 465                 allocated_slivers = allocated_slivers + 1
 466
 467         stats_row = stats_rows.get(site.name,{})
 468
 469         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
 470         cpu=float(max_cpu)/100.0
 471         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
 472
 473         # format it to what that CDN Operations View is expecting
 474         new_row = {"lat": float(site.location.longitude),
 475                "long": float(site.location.longitude),
 476                "lat": float(site.location.latitude),
 477                "health": 0,
 478                "numNodes": int(site.nodes.count()),
 479                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),     # measured number of slivers, from bigquery statistics
 480                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
 481                "siteUrl": str(site.site_url),
 482                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
 483                "load": max_cpu,
 484                "hot": float(hotness)}
 485         new_rows[str(site.name)] = new_row
 486
 487     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
 488     for (k,v) in new_rows.items():
 489         bad=False
 490         if v["numHPCSlivers"]==0:
 491             for v2 in new_rows.values():
 492                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
 493                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
 494                     if d<100:
 495                          bad=True
 496             if bad:
 497                 del new_rows[k]
 498
 499     return new_rows
 500
 501 def getPageSummary(request):
 502     slice = request.GET.get('slice', None)
 503     site = request.GET.get('site', None)
 504     node = request.GET.get('node', None)
 505
 506 class SimulatorView(View):
 507     def get(self, request, **kwargs):
 508         sim = json.loads(file("/tmp/simulator.json","r").read())
 509         text = "<html><head></head><body>"
 510         text += "Iteration: %d<br>" % sim["iteration"]
 511         text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
 512         text += "<table border=1>"
 513         text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
 514         for site in sim["site_load"].values():
 515             text += "<tr>"
 516             text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
 517                         (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
 518             text += "</tr>"
 519         text += "</table>"
 520         text += "</body></html>"
 521         return HttpResponse(text)
 522
 523 class DashboardUserSiteView(View):
 524     def get(self, request, **kwargs):
 525         return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
 526
 527 class TenantViewData(View):
 528     def get(self, request, **kwargs):
 529         return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
 530
 531 def haversine(site_lat, site_lon, lat, lon):
 532     d=0
 533     if lat and lon and site_lat and site_lon:
 534         site_lat = float(site_lat)
 535         site_lon = float(site_lon)
 536         lat = float(lat)
 537         lon = float(lon)
 538         R = 6378.1
 539         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
 540         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
 541         d = R * c
 542
 543     return d
 544
 545 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
 546     # try to pick a site we're already using
 547     has_slivers_here=False
 548     if slice:
 549         for sliver in slice.slivers.all():
 550             if sliver.node.site.name == site.name:
 551                 has_slivers_here=True
 552
 553     # Haversine method
 554     d = haversine(site.location.latitude, site.location.longitude, lat, lon)
 555
 556     return (-has_slivers_here, d)
 557
 558 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
 559     """ Returns list of sites, sorted from most favorable to least favorable """
 560     lat=None
 561     lon=None
 562     try:
 563         client_geo = GeoIP().city(user_ip)
 564         if client_geo:
 565             lat=float(client_geo["latitude"])
 566             lon=float(client_geo["longitude"])
 567     except:
 568         print "exception in geo code"
 569         traceback.print_exc()
 570
 571     sites = Site.objects.all()
 572     sites = [x for x in sites if x.name in BLESSED_SITES]
 573     sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
 574
 575     return sites
 576
 577 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
 578     sitesChanged = {}
 579
 580     # let's compute how many slivers are in use in each node of each site
 581     for site in siteList:
 582         site.nodeList = list(site.nodes.all())
 583         for node in site.nodeList:
 584             node.sliverCount = 0
 585             for sliver in node.slivers.all():
 586                  if sliver.slice.id == slice.id:
 587                      node.sliverCount = node.sliverCount + 1
 588
 589     # Allocate slivers to nodes
 590     # for now, assume we want to allocate all slivers from the same site
 591     nodes = siteList[0].nodeList
 592     while (count>0):
 593         # Sort the node list by number of slivers per node, then pick the
 594         # node with the least number of slivers.
 595         nodes = sorted(nodes, key=attrgetter("sliverCount"))
 596         node = nodes[0]
 597
 598         print "adding sliver at node", node.name, "of site", node.site.name
 599
 600         if not noAct:
 601             sliver = Sliver(name=node.name,
 602                         slice=slice,
 603                         node=node,
 604                         image = Image.objects.all()[0],
 605                         creator = User.objects.get(email=user),
 606                         deploymentNetwork=node.deployment,
 607                         numberCores =1 )
 608             sliver.save()
 609
 610         node.sliverCount = node.sliverCount + 1
 611
 612         count = count - 1
 613
 614         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
 615
 616     return sitesChanged
 617
 618 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
 619     sitesChanged = {}
 620     sliverList ={}
 621     if siteList:
 622         siteNames = [site.name for site in siteList]
 623     else:
 624         siteNames = None
 625
 626     for sliver in slice.slivers.all():
 627         if(not siteNames) or (sliver.node.site.name in siteNames):\r
 628                 node = sliver.node\r
 629                 sliverList[sliver.name]=node.name
 630
 631     for key in sliverList:
 632         if count>0:
 633             sliver = Sliver.objects.filter(name=key)[0]\r
 634             sliver.delete()\r
 635             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
 636             count=count-1\r
 637             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
 638 \r
 639     return sitesChanged
 640
 641 class TenantDeleteSliceView(View):
 642         def post(self,request):\r
 643                 if request.user.isReadOnlyUser():\r
 644                     return HttpResponseForbidden("User is in read-only mode")\r
 645                 sliceName = request.POST.get("sliceName",None)\r
 646                 slice = Slice.objects.get(name=sliceName)\r
 647                 #print slice, slice.id\r
 648                 sliceToDel=Slice(name=sliceName, id=slice.id)\r
 649                 sliceToDel.delete()
 650                 return HttpResponse("Slice deleted")
 651
 652 class TenantAddOrRemoveSliverView(View):
 653     """ Add or remove slivers from a Slice
 654
 655         Arguments:
 656             siteName - name of site. If not specified, PlanetStack will pick the
 657                        best site.,
 658             actionToDo - [add | rem]
 659             count - number of slivers to add or remove
 660             sliceName - name of slice
 661             noAct - if set, no changes will be made to db, but result will still
 662                     show which sites would have been modified.
 663
 664         Returns:
 665             Dictionary of sites that were modified, and the count of nodes
 666             that were added or removed at each site.
 667     """
 668     def post(self, request, *args, **kwargs):
 669         siteName = request.POST.get("siteName", None)
 670         actionToDo = request.POST.get("actionToDo", None)
 671         count = int(request.POST.get("count","0"))
 672         sliceName = request.POST.get("slice", None)
 673         noAct = request.POST.get("noAct", False)
 674
 675         if not sliceName:
 676             return HttpResponseServerError("No slice name given")
 677
 678         slice = Slice.objects.get(name=sliceName)
 679
 680         if siteName:
 681             siteList = [Site.objects.get(name=siteName)]
 682         else:
 683             siteList = None
 684
 685         if (actionToDo == "add"):
 686             user_ip = request.GET.get("ip", get_ip(request))
 687             if (siteList is None):
 688                 siteList = tenant_pick_sites(user, user_ip, slice, count)
 689
 690             sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
 691         elif (actionToDo == "rem"):
 692             sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
 693         else:
 694             return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
 695
 696         return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
 697
 698     def get(self, request, *args, **kwargs):
 699         request.POST = request.GET
 700         return self.post(request, *args, **kwargs)  # for testing REST in browser
 701         #return HttpResponseServerError("GET is not supported")
 702
 703 class TenantPickSitesView(View):
 704     """ primarily just for testing purposes """
 705     def get(self, request, *args, **kwargs):
 706         count = request.GET.get("count","0")
 707         slice = request.GET.get("slice",None)
 708         if slice:
 709             slice = Slice.objects.get(name=slice)
 710         ip = request.GET.get("ip", get_ip(request))
 711         sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
 712         sites = [x.name for x in sites]
 713         return HttpResponse(json.dumps(sites), mimetype='application/javascript')
 714
 715 class DashboardSummaryAjaxView(View):
 716     def get(self, request, **kwargs):
 717         def avg(x):
 718             if len(x)==0:
 719                 return 0
 720             return float(sum(x))/len(x)
 721
 722         sites = getCDNOperatorData().values()
 723
 724         sites = [site for site in sites if site["numHPCSlivers"]>0]
 725
 726         total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
 727         total_bandwidth = sum( [site["bandwidth"] for site in sites] )
 728         average_cpu = int(avg( [site["load"] for site in sites] ))
 729
 730         result= {"total_slivers": total_slivers,
 731                 "total_bandwidth": total_bandwidth,
 732                 "average_cpu": average_cpu}
 733
 734         return HttpResponse(json.dumps(result), mimetype='application/javascript')
 735
 736 class DashboardAddOrRemoveSliverView(View):
 737     # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
 738
 739     def post(self, request, *args, **kwargs):
 740         siteName = request.POST.get("site", None)
 741         actionToDo = request.POST.get("actionToDo", "0")
 742
 743         siteList = [Site.objects.get(name=siteName)]
 744         slice = Slice.objects.get(name="HyperCache")
 745
 746         if request.user.isReadOnlyUser():
 747             return HttpResponseForbidden("User is in read-only mode")
 748
 749         if (actionToDo == "add"):
 750             user_ip = request.GET.get("ip", get_ip(request))
 751             slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
 752         elif (actionToDo == "rem"):
 753             slice_decrease_slivers(request.user, siteList, slice, 1)
 754
 755         print '*' * 50
 756         print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
 757         return HttpResponse(json.dumps("Success"), mimetype='application/javascript')
 758
 759 class DashboardAjaxView(View):
 760     def get(self, request, **kwargs):
 761         return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
 762
 763 class DashboardAnalyticsAjaxView(View):
 764     def get(self, request, name="hello_world", **kwargs):
 765         if (name == "hpcSummary"):
 766             return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
 767         elif (name == "hpcUserSite"):
 768             return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
 769         elif (name == "hpcMap"):
 770             return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
 771         elif (name == "bigquery"):
 772             (mimetype, data) = DoPlanetStackAnalytics(request)
 773             return HttpResponse(data, mimetype=mimetype)
 774         else:
 775             return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
 776
 777 class DashboardCustomize(View):
 778     def post(self, request, *args, **kwargs):\r
 779         if request.user.isReadOnlyUser():\r
 780             return HttpResponseForbidden("User is in read-only mode")\r
 781 \r
 782         dashboards = request.POST.get("dashboards", None)\r
 783         if not dashboards:\r
 784             dashboards=[]\r
 785         else:\r
 786             dashboards = [x.strip() for x in dashboards.split(",")]\r
 787             dashboards = [DashboardView.objects.get(name=x) for x in dashboards]\r
 788 \r
 789         request.user.dashboardViews.all().delete()\r
 790 \r
 791         for i,dashboard in enumerate(dashboards):\r
 792             udbv = UserDashboardView(user=request.user, dashboardView=dashboard, order=i)\r
 793             udbv.save()\r
 794 \r
 795         return HttpResponse(json.dumps("Success"), mimetype='application/javascript')\r
 796
 797 class DashboardSliceInteractions(View):
 798     def get(self, request, name="users", **kwargs):
 799         colors = ["#005586", "#6ebe49", "orange", "#707170", "#00c4b3", "#077767", "dodgerblue", "#a79b94", "#c4e76a", "red"]
 800
 801         groups = []
 802         matrix = []
 803         slices = list(Slice.objects.all())
 804
 805         ids_by_slice = self.build_id_list(slices, name)
 806
 807         slices = [x for x in slices if (len(ids_by_slice[x])>0)]
 808
 809         for i,slice in enumerate(slices):
 810             groups.append({"name": slice.name, "color": colors[i%len(colors)]})
 811             row=self.buildMatrix(slice, slices, name, ids_by_slice)
 812             matrix.append(row)
 813
 814         result = {"groups": groups, "matrix": matrix}
 815
 816         if name=="users":
 817             result["title"] = "Slice interactions by user privilege"
 818             result["objectName"] = "users"
 819         elif name=="networks":
 820             result["title"] = "Slice interactions by network membership"
 821             result["objectName"] = "networks"
 822         elif name=="sites":
 823             result["title"] = "Slice interactions by site ownership"
 824             result["objectName"] = "sites"
 825         elif name=="sliver_sites":
 826             result["title"] = "Slice interactions by sliver sites"
 827             result["objectName"] = "sites"
 828         elif name=="sliver_nodes":
 829             result["title"] = "Slice interactions by sliver nodes"
 830             result["objectName"] = "nodes"
 831
 832         return HttpResponse(json.dumps(result), mimetype='application/javascript')
 833
 834     def build_id_list(self, slices, name):
 835         ids_by_slice = {}
 836         for slice in slices:
 837             # build up a list of object ids that are used by each slice
 838             ids_by_slice[slice] = self.getIds(slice, name)
 839
 840         return ids_by_slice
 841
 842     def buildMatrix(self, slice, slices, name, ids_by_slice):
 843         not_only_my_ids = []
 844
 845         # build up a list of object ids that are used by other slices
 846         for otherSlice in ids_by_slice.keys():
 847             if (slice != otherSlice):
 848                 for id in ids_by_slice[otherSlice]:
 849                     if not id in not_only_my_ids:
 850                         not_only_my_ids.append(id)
 851
 852         # build up a list of ids that are used only by the slice, and not
 853         # shared with any other slice
 854         only_my_ids = []
 855         for id in ids_by_slice[slice]:
 856              if id not in not_only_my_ids:
 857                   only_my_ids.append(id)
 858
 859         row = []
 860         for otherSlice in ids_by_slice.keys():
 861             if (otherSlice == slice):
 862                 row.append(len(only_my_ids))
 863             else:
 864                 row.append(self.inCommonIds(ids_by_slice[slice], ids_by_slice[otherSlice]))
 865
 866         return row
 867
 868     def getIds(self, slice, name):
 869         ids=[]
 870         if name=="users":
 871             for sp in slice.slice_privileges.all():
 872                     if sp.user.id not in ids:
 873                         ids.append(sp.user.id)
 874         elif name=="networks":
 875             for sp in slice.networkslice_set.all():
 876                     if sp.network.id not in ids:
 877                         ids.append(sp.network.id)
 878         elif name=="sites":
 879             ids = [slice.site.id]
 880         elif name=="sliver_sites":
 881             for sp in slice.slivers.all():
 882                  if sp.node.site.id not in ids:
 883                      ids.append(sp.node.site.id)
 884         elif name=="sliver_nodes":
 885             for sp in slice.slivers.all():
 886                  if sp.node.id not in ids:
 887                      ids.append(sp.node.id)
 888         return ids
 889
 890     def inCommonIds(self, ids1, ids2):
 891         count = 0
 892         for id in ids1:
 893             if id in ids2:
 894                 count+=1
 895         return count
 896
 897     def isEmpty(self, slice, name):
 898         if (name in ["users", "networks", "sites", "sliver_nodes", "sliver_sites"]):
 899             return (len(self.getIds(slice,name)) == 0)
 900