planetstack/core/plus/views.py

   1 #views.py
   2 import functools
   3 import math
   4 import os
   5 import sys
   6 from django.views.generic import TemplateView, View
   7 import datetime
   8 from pprint import pprint
   9 import json
  10 from core.models import *
  11 from operator import attrgetter
  12 from django.views.decorators.csrf import csrf_exempt
  13 from django.http import HttpResponse, HttpResponseServerError
  14 from django.core import urlresolvers
  15 from django.contrib.gis.geoip import GeoIP
  16 from ipware.ip import get_ip
  17 import traceback
  18 import socket
  19
  20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
  21
  22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
  23     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
  24 else:
  25     sys.path.append("/opt/planetstack/hpc_wizard")
  26 import hpc_wizard
  27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
  28
  29 class DashboardWelcomeView(TemplateView):
  30     template_name = 'admin/dashboard/welcome.html'
  31
  32     def get(self, request, *args, **kwargs):
  33         context = self.get_context_data(**kwargs)
  34         userDetails = getUserSliceInfo(request.user)
  35         #context['site'] = userDetails['site']
  36
  37         context['userSliceInfo'] = userDetails['userSliceInfo']
  38         context['cdnData'] = userDetails['cdnData']
  39         return self.render_to_response(context=context)
  40
  41 def getUserSliceInfo(user, tableFormat = False):
  42         userDetails = {}
  43 #        try:
  44 # //           site = Site.objects.filter(id=user.site.id)
  45 #  //      except:
  46 #   //         site = Site.objects.filter(name="Princeton")
  47 #    //    userDetails['sitename'] = site[0].name
  48 #     //   userDetails['siteid'] = site[0].id
  49
  50         userSliceData = getSliceInfo(user)
  51         if (tableFormat):
  52 #            pprint("*******      GET USER SLICE INFO")
  53             userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
  54         else:
  55             userDetails['userSliceInfo'] = userSliceData
  56         userDetails['cdnData'] = getCDNOperatorData(wait=False);
  57 #        pprint( userDetails)
  58         return userDetails
  59
  60 class TenantCreateSlice(View):
  61     def post(self, request, *args, **kwargs):
  62         sliceName = request.POST.get("sliceName", "0")
  63         serviceClass = request.POST.get("serviceClass", "0")
  64         imageName = request.POST.get("imageName", "0")
  65         actionToDo = request.POST.get("actionToDo", "0")
  66         print sliceName
  67         if (actionToDo == "add"):
  68            serviceClass = ServiceClass.objects.get(name=serviceClass)
  69            site = request.user.site
  70            #image = Image.objects.get(name=imageName)
  71            newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
  72            newSlice.save()
  73         return newSlice
  74
  75
  76 def getTenantSliceInfo(user, tableFormat = False):
  77     tenantSliceDetails = {}
  78     tenantSliceData = getTenantInfo(user)
  79     tenantServiceClassData = getServiceClassInfo(user)
  80     if (tableFormat):
  81        tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
  82        tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
  83     else:
  84        tenantSliceDetails['userSliceInfo'] = tenantSliceData
  85     tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
  86     tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
  87     tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
  88     tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
  89     tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
  90     tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
  91     return tenantSliceDetails
  92
  93
  94 def getTenantInfo(user):
  95     slices =Slice.objects.all()
  96     userSliceInfo = []
  97     for entry in slices:
  98        sliceName = Slice.objects.get(id=entry.id).name
  99        slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
 100        sliceServiceClass = entry.serviceClass.name
 101        preferredImage =  entry.imagePreference
 102        numSliver = 0
 103        sliceImage=""
 104        sliceSite = {}
 105        for sliver in slice.slivers.all():
 106             numSliver +=sliver.numberCores
 107            # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
 108             if sliver.node.site.name in BLESSED_SITES:
 109                 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
 110                 sliceImage = sliver.image.name
 111        numSites = len(sliceSite)
 112        userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
 113     return userSliceInfo
 114
 115 def getTenantSitesInfo():
 116         tenantSiteInfo=[]
 117         for entry in Site.objects.all():
 118             if entry.name in BLESSED_SITES:
 119                  tenantSiteInfo.append({'siteName':entry.name})
 120         return tenantSiteInfo
 121
 122 def userSliceTableFormatter(data):
 123 #    pprint(data)
 124     formattedData = {
 125                      'rows' : data
 126                     }
 127     return formattedData
 128
 129 def getServiceClassInfo(user):
 130     serviceClassList = ServiceClass.objects.all()
 131     sliceInfo = []
 132     for entry in serviceClassList:
 133           sliceInfo.append({'serviceClass':entry.name})
 134     return sliceInfo
 135
 136 def getImageInfo(user):
 137     #imageList = Image.objects.all()
 138     imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
 139     imageInfo = []
 140     for imageEntry in imageList:
 141           #imageInfo.append({'Image':imageEntry.name})
 142           imageInfo.append({'Image':imageEntry})
 143     return imageInfo
 144
 145 def getMountDataSets():
 146         dataSetList = ['GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
 147         dataSetInfo = []\r
 148         for entry in dataSetList:\r
 149                 dataSetInfo.append({'DataSet':entry})\r
 150         return dataSetInfo
 151
 152 def getNetworkInfo(user):
 153    #networkList = Network.objects.all()
 154     networkList = ['Private Only','Private and Publicly Routable']
 155     networkInfo = []
 156     for networkEntry in networkList:
 157           #networkInfo.append({'Network':networkEntry.name})
 158           networkInfo.append({'Network':networkEntry})
 159     return networkInfo
 160
 161 def getDeploymentSites():
 162     deploymentList = Deployment.objects.all()
 163     deploymentInfo = []
 164     for entry in deploymentList:
 165         deploymentInfo.append({'DeploymentSite':entry.name})
 166     return deploymentInfo
 167
 168 def getSliceInfo(user):
 169     sliceList = Slice.objects.all()
 170     slicePrivs = SlicePrivilege.objects.filter(user=user)
 171     userSliceInfo = []
 172     for entry in slicePrivs:
 173
 174         slicename = Slice.objects.get(id=entry.slice.id).name
 175         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
 176         sliverList=Sliver.objects.all()
 177         sites_used = {}
 178         for sliver in slice.slivers.all():
 179              #sites_used['deploymentSites'] = sliver.node.deployment.name
 180              # sites_used[sliver.image.name] = sliver.image.name
 181              sites_used[sliver.node.site.name] = sliver.numberCores
 182         sliceid = Slice.objects.get(id=entry.slice.id).id
 183         try:
 184             sliverList = Sliver.objects.filter(slice=entry.slice.id)
 185             siteList = {}
 186             for x in sliverList:
 187                if x.node.site not in siteList:
 188                   siteList[x.node.site] = 1
 189             slivercount = len(sliverList)
 190             sitecount = len(siteList)
 191         except:
 192             traceback.print_exc()
 193             slivercount = 0
 194             sitecount = 0
 195
 196         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
 197                               'sitesUsed':sites_used,
 198                               'role': SliceRole.objects.get(id=entry.role.id).role,
 199                               'slivercount': slivercount,
 200                               'sitecount':sitecount})
 201
 202     return userSliceInfo
 203
 204 def getCDNOperatorData(randomizeData = False, wait=True):
 205     HPC_SLICE_NAME = "HyperCache"
 206
 207     bq = PlanetStackAnalytics()
 208
 209     rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
 210
 211     # if wait==False, then we could have no rows yet
 212
 213     stats_rows = {}
 214     if rows:
 215         rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
 216
 217         # dictionaryize the statistics rows by site name
 218         stats_rows = {}
 219         for row in rows:
 220             stats_rows[row["site"]] = row
 221
 222     slice = Slice.objects.get(name=HPC_SLICE_NAME)
 223     slice_slivers = list(slice.slivers.all())
 224
 225     new_rows = {}
 226     for site in Site.objects.all():
 227         # compute number of slivers allocated in the data model
 228         allocated_slivers = 0
 229         for sliver in slice_slivers:
 230             if sliver.node.site == site:
 231                 allocated_slivers = allocated_slivers + 1
 232
 233         stats_row = stats_rows.get(site.name,{})
 234
 235         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
 236         cpu=float(max_cpu)/100.0
 237         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
 238
 239         # format it to what that CDN Operations View is expecting
 240         new_row = {"lat": float(site.location.longitude),
 241                "long": float(site.location.longitude),
 242                "lat": float(site.location.latitude),
 243                "health": 0,
 244                "numNodes": int(site.nodes.count()),
 245                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),         # measured number of slivers, from bigquery statistics
 246                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
 247                "siteUrl": str(site.site_url),
 248                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
 249                "load": max_cpu,
 250                "hot": float(hotness)}
 251         new_rows[str(site.name)] = new_row
 252
 253     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
 254     for (k,v) in new_rows.items():
 255         bad=False
 256         if v["numHPCSlivers"]==0:
 257             for v2 in new_rows.values():
 258                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
 259                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
 260                     if d<100:
 261                          bad=True
 262             if bad:
 263                 del new_rows[k]
 264
 265     return new_rows
 266
 267 class SimulatorView(View):
 268     def get(self, request, **kwargs):
 269         sim = json.loads(file("/tmp/simulator.json","r").read())
 270         text = "<html><head></head><body>"
 271         text += "Iteration: %d<br>" % sim["iteration"]
 272         text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
 273         text += "<table border=1>"
 274         text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
 275         for site in sim["site_load"].values():
 276             text += "<tr>"
 277             text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
 278                         (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
 279             text += "</tr>"
 280         text += "</table>"
 281         text += "</body></html>"
 282         return HttpResponse(text)
 283
 284 class DashboardUserSiteView(View):
 285     def get(self, request, **kwargs):
 286         return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
 287
 288 class TenantViewData(View):
 289     def get(self, request, **kwargs):
 290         return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
 291
 292 def haversine(site_lat, site_lon, lat, lon):
 293     site_lat = float(site_lat)
 294     site_lon = float(site_lon)
 295     lat = float(lat)
 296     lon = float(lon)
 297     d=0
 298     if lat and lon and site_lat and site_lon:
 299         R = 6378.1
 300         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
 301         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
 302         d = R * c
 303
 304     return d
 305
 306 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
 307     # try to pick a site we're already using
 308     has_slivers_here=False
 309     if slice:
 310         for sliver in slice.slivers.all():
 311             if sliver.node.site.name == site.name:
 312                 has_slivers_here=True
 313
 314     # Haversine method
 315     d = haversine(site.location.latitude, site.location.longitude, lat, lon)
 316
 317     return (-has_slivers_here, d)
 318
 319 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
 320     """ Returns list of sites, sorted from most favorable to least favorable """
 321     lat=None
 322     lon=None
 323     try:
 324         client_geo = GeoIP().city(user_ip)
 325         if client_geo:
 326             lat=float(client_geo["latitude"])
 327             lon=float(client_geo["longitude"])
 328     except:
 329         print "exception in geo code"
 330         traceback.print_exc()
 331
 332     sites = Site.objects.all()
 333     sites = [x for x in sites if x.name in BLESSED_SITES]
 334     sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
 335
 336     return sites
 337
 338 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
 339     sitesChanged = {}
 340
 341     # let's compute how many slivers are in use in each node of each site
 342     for site in siteList:
 343         site.nodeList = list(site.nodes.all())
 344         for node in site.nodeList:
 345             node.sliverCount = 0
 346             for sliver in node.slivers.all():
 347                  if sliver.slice.id == slice.id:
 348                      node.sliverCount = node.sliverCount + 1
 349
 350     # Allocate slivers to nodes
 351     # for now, assume we want to allocate all slivers from the same site
 352     nodes = siteList[0].nodeList
 353     while (count>0):
 354         # Sort the node list by number of slivers per node, then pick the
 355         # node with the least number of slivers.
 356         nodes = sorted(nodes, key=attrgetter("sliverCount"))
 357         node = nodes[0]
 358
 359         print "adding sliver at node", node.name, "of site", node.site.name
 360
 361         if not noAct:
 362             sliver = Sliver(name=node.name,
 363                         slice=slice,
 364                         node=node,
 365                         image = Image.objects.all()[0],
 366                         creator = User.objects.get(email=user),
 367                         deploymentNetwork=node.deployment,
 368                         numberCores =1 )
 369             sliver.save()
 370
 371         node.sliverCount = node.sliverCount + 1
 372
 373         count = count - 1
 374
 375         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
 376
 377     return sitesChanged
 378
 379 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
 380     sitesChanged = {}
 381     sliverList ={}
 382     if siteList:
 383         siteNames = [site.name for site in siteList]
 384     else:
 385         siteNames = None
 386
 387     for sliver in slice.slivers.all():
 388         if(not siteNames) or (sliver.node.site.name in siteNames):\r
 389                 node = sliver.node\r
 390                 sliverList[sliver.name]=node.name
 391
 392     for key in sliverList:
 393         if count>0:
 394             sliver = Sliver.objects.filter(name=key)[0]\r
 395             sliver.delete()\r
 396             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
 397             count=count-1\r
 398             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
 399 \r
 400     return sitesChanged
 401
 402 class TenantDeleteSliceView(View):
 403         def post(self,request):\r
 404                 sliceName = request.POST.get("sliceName",None)\r
 405                 slice = Slice.objects.get(name=sliceName)\r
 406                 print slice, slice.id\r
 407                 sliceToDel=Slice(name=sliceName, id=slice.id)\r
 408                 sliceToDel.delete()
 409
 410 class TenantAddOrRemoveSliverView(View):
 411     """ Add or remove slivers from a Slice
 412
 413         Arguments:
 414             siteName - name of site. If not specified, PlanetStack will pick the
 415                        best site.,
 416             actionToDo - [add | rem]
 417             count - number of slivers to add or remove
 418             sliceName - name of slice
 419             noAct - if set, no changes will be made to db, but result will still
 420                     show which sites would have been modified.
 421
 422         Returns:
 423             Dictionary of sites that were modified, and the count of nodes
 424             that were added or removed at each site.
 425     """
 426     def post(self, request, *args, **kwargs):
 427         siteName = request.POST.get("siteName", None)
 428         actionToDo = request.POST.get("actionToDo", None)
 429         count = int(request.POST.get("count","0"))
 430         sliceName = request.POST.get("slice", None)
 431         noAct = request.POST.get("noAct", False)
 432
 433         if not sliceName:
 434             return HttpResponseServerError("No slice name given")
 435
 436         slice = Slice.objects.get(name=sliceName)
 437
 438         if siteName:
 439             siteList = [Site.objects.get(name=siteName)]
 440         else:
 441             siteList = None
 442
 443         if (actionToDo == "add"):
 444             user_ip = request.GET.get("ip", get_ip(request))
 445             if (siteList is None):
 446                 siteList = tenant_pick_sites(user, user_ip, slice, count)
 447
 448             sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
 449         elif (actionToDo == "rem"):
 450             sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
 451         else:
 452             return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
 453
 454         return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
 455
 456     def get(self, request, *args, **kwargs):
 457         request.POST = request.GET
 458         return self.post(request, *args, **kwargs)  # for testing REST in browser
 459         #return HttpResponseServerError("GET is not supported")
 460
 461 class TenantPickSitesView(View):
 462     """ primarily just for testing purposes """
 463     def get(self, request, *args, **kwargs):
 464         count = request.GET.get("count","0")
 465         slice = request.GET.get("slice",None)
 466         if slice:
 467             slice = Slice.objects.get(name=slice)
 468         ip = request.GET.get("ip", get_ip(request))
 469         sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
 470         sites = [x.name for x in sites]
 471         return HttpResponse(json.dumps(sites), mimetype='application/javascript')
 472
 473 class DashboardSummaryAjaxView(View):
 474     def get(self, request, **kwargs):
 475         def avg(x):
 476             return float(sum(x))/len(x)
 477
 478         sites = getCDNOperatorData().values()
 479
 480         sites = [site for site in sites if site["numHPCSlivers"]>0]
 481
 482         total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
 483         total_bandwidth = sum( [site["bandwidth"] for site in sites] )
 484         average_cpu = int(avg( [site["load"] for site in sites] ))
 485
 486         result= {"total_slivers": total_slivers,
 487                 "total_bandwidth": total_bandwidth,
 488                 "average_cpu": average_cpu}
 489
 490         return HttpResponse(json.dumps(result), mimetype='application/javascript')
 491
 492 class DashboardAddOrRemoveSliverView(View):
 493     # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
 494
 495     def post(self, request, *args, **kwargs):
 496         siteName = request.POST.get("site", None)
 497         actionToDo = request.POST.get("actionToDo", "0")
 498
 499         siteList = [Site.objects.get(name=siteName)]
 500         slice = Slice.objects.get(name="HyperCache")
 501
 502         if (actionToDo == "add"):
 503             user_ip = request.GET.get("ip", get_ip(request))
 504             slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
 505         elif (actionToDo == "rem"):
 506             slice_decrease_slivers(request.user, siteList, slice, 1)
 507
 508         print '*' * 50
 509         print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
 510         return HttpResponse('This is POST request ')
 511
 512 class DashboardAjaxView(View):
 513     def get(self, request, **kwargs):
 514         return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
 515
 516 class DashboardAnalyticsAjaxView(View):
 517     def get(self, request, name="hello_world", **kwargs):
 518         if (name == "hpcSummary"):
 519             return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
 520         elif (name == "hpcUserSite"):
 521             return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
 522         elif (name == "hpcMap"):
 523             return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
 524         elif (name == "bigquery"):
 525             (mimetype, data) = DoPlanetStackAnalytics(request)
 526             return HttpResponse(data, mimetype=mimetype)
 527         else:
 528             return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
 529