planetstack/core/dashboard/views/view_common.py

   1 import os
   2 import sys
   3 from django.views.generic import TemplateView, View
   4 import datetime
   5 from pprint import pprint
   6 import json
   7 from syndicate.models import *
   8 from core.models import *
   9 from hpc.models import ContentProvider
  10 from operator import attrgetter
  11 from django import template
  12 from django.views.decorators.csrf import csrf_exempt
  13 from django.http import HttpResponse, HttpResponseServerError, HttpResponseForbidden
  14 from django.core import urlresolvers
  15 from django.contrib.gis.geoip import GeoIP
  16 from django.db.models import Q
  17 from ipware.ip import get_ip
  18 from operator import itemgetter, attrgetter
  19 import traceback
  20 import math
  21
  22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
  23     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
  24 else:
  25     sys.path.append("/opt/planetstack/hpc_wizard")
  26 import hpc_wizard
  27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
  28
  29 def getDashboardContext(user, context={}, tableFormat = False):
  30         context = {}
  31
  32         userSliceData = getSliceInfo(user)
  33         if (tableFormat):
  34             context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
  35         else:
  36             context['userSliceInfo'] = userSliceData
  37         context['cdnData'] = getCDNOperatorData(wait=False)
  38         context['cdnContentProviders'] = getCDNContentProviderData()
  39
  40         (dashboards, unusedDashboards)= getDashboards(user)
  41         unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
  42         context['dashboards'] = dashboards
  43         context['unusedDashboards'] = unusedDashboards
  44
  45         return context
  46
  47 def getDashboards(user):
  48     dashboards = user.get_dashboards()
  49
  50     dashboard_names = [d.name for d in dashboards]
  51
  52     unused_dashboard_names = []
  53     for dashboardView in DashboardView.objects.all():
  54         if not dashboardView.name in dashboard_names:
  55             unused_dashboard_names.append(dashboardView.name)
  56
  57     return (dashboard_names, unused_dashboard_names)
  58
  59 def getSliceInfo(user):
  60     sliceList = Slice.objects.all()
  61     slicePrivs = SlicePrivilege.objects.filter(user=user)
  62     userSliceInfo = []
  63     for entry in slicePrivs:
  64
  65         slicename = Slice.objects.get(id=entry.slice.id).name
  66         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
  67         sliverList=Sliver.objects.all()
  68         sites_used = {}
  69         for sliver in slice.slivers.all():
  70              #sites_used['deploymentSites'] = sliver.node.deployment.name
  71              # sites_used[sliver.image.name] = sliver.image.name
  72              sites_used[sliver.node.site.name] = sliver.numberCores
  73         sliceid = Slice.objects.get(id=entry.slice.id).id
  74         try:
  75             sliverList = Sliver.objects.filter(slice=entry.slice.id)
  76             siteList = {}
  77             for x in sliverList:
  78                if x.node.site not in siteList:
  79                   siteList[x.node.site] = 1
  80             slivercount = len(sliverList)
  81             sitecount = len(siteList)
  82         except:
  83             traceback.print_exc()
  84             slivercount = 0
  85             sitecount = 0
  86
  87         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
  88                               'sitesUsed':sites_used,
  89                               'role': SliceRole.objects.get(id=entry.role.id).role,
  90                               'slivercount': slivercount,
  91                               'sitecount':sitecount})
  92
  93     return userSliceInfo
  94
  95 def getCDNContentProviderData():
  96     cps = []
  97     for dm_cp in ContentProvider.objects.all():
  98         cp = {"name": dm_cp.name,
  99               "account": dm_cp.account}
 100         cps.append(cp)
 101
 102     return cps
 103
 104 def getCDNOperatorData(randomizeData = False, wait=True):
 105     HPC_SLICE_NAME = "HyperCache"
 106
 107     bq = PlanetStackAnalytics()
 108
 109     rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
 110
 111     # wait=False on the first time the Dashboard is opened. This means we might
 112     # not have any rows yet. The dashboard code polls every 30 seconds, so it
 113     # will eventually pick them up.
 114
 115     if rows:
 116         rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
 117
 118         # dictionaryize the statistics rows by site name
 119         stats_rows = {}
 120         for row in rows:
 121             stats_rows[row["site"]] = row
 122     else:
 123         stats_rows = {}
 124
 125     slice = Slice.objects.filter(name=HPC_SLICE_NAME)
 126     if slice:
 127         slice_slivers = list(slice[0].slivers.all())
 128     else:
 129         slice_slivers = []
 130
 131     new_rows = {}
 132     for site in Site.objects.all():
 133         # compute number of slivers allocated in the data model
 134         allocated_slivers = 0
 135         for sliver in slice_slivers:
 136             if sliver.node.site == site:
 137                 allocated_slivers = allocated_slivers + 1
 138
 139         stats_row = stats_rows.get(site.name,{})
 140
 141         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
 142         cpu=float(max_cpu)/100.0
 143         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
 144
 145         # format it to what that CDN Operations View is expecting
 146         new_row = {"lat": float(site.location.longitude),
 147                "long": float(site.location.longitude),
 148                "lat": float(site.location.latitude),
 149                "health": 0,
 150                "numNodes": int(site.nodes.count()),
 151                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),     # measured number of slivers, from bigquery statistics
 152                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
 153                "siteUrl": str(site.site_url),
 154                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
 155                "load": max_cpu,
 156                "hot": float(hotness)}
 157         new_rows[str(site.name)] = new_row
 158
 159     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
 160     for (k,v) in new_rows.items():
 161         bad=False
 162         if v["numHPCSlivers"]==0:
 163             for v2 in new_rows.values():
 164                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
 165                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
 166                     if d<100:
 167                          bad=True
 168             if bad:
 169                 del new_rows[k]
 170
 171     return new_rows
 172
 173 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
 174     sitesChanged = {}
 175
 176     # let's compute how many slivers are in use in each node of each site
 177     for site in siteList:
 178         site.nodeList = list(site.nodes.all())
 179         for node in site.nodeList:
 180             node.sliverCount = 0
 181             for sliver in node.slivers.all():
 182                  if sliver.slice.id == slice.id:
 183                      node.sliverCount = node.sliverCount + 1
 184
 185     # Allocate slivers to nodes
 186     # for now, assume we want to allocate all slivers from the same site
 187     nodes = siteList[0].nodeList
 188     while (count>0):
 189         # Sort the node list by number of slivers per node, then pick the
 190         # node with the least number of slivers.
 191         nodes = sorted(nodes, key=attrgetter("sliverCount"))
 192         node = nodes[0]
 193
 194         print "adding sliver at node", node.name, "of site", node.site.name
 195
 196         if not noAct:
 197             sliver = Sliver(name=node.name,
 198                         slice=slice,
 199                         node=node,
 200                         image = Image.objects.all()[0],
 201                         creator = User.objects.get(email=user),
 202                         deploymentNetwork=node.deployment,
 203                         numberCores =1 )
 204             sliver.save()
 205
 206         node.sliverCount = node.sliverCount + 1
 207
 208         count = count - 1
 209
 210         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
 211
 212     return sitesChanged
 213
 214 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
 215     sitesChanged = {}
 216     sliverList ={}
 217     if siteList:
 218         siteNames = [site.name for site in siteList]
 219     else:
 220         siteNames = None
 221
 222     for sliver in slice.slivers.all():
 223         if(not siteNames) or (sliver.node.site.name in siteNames):\r
 224                 node = sliver.node\r
 225                 sliverList[sliver.name]=node.name
 226
 227     for key in sliverList:
 228         if count>0:
 229             sliver = Sliver.objects.filter(name=key)[0]\r
 230             sliver.delete()\r
 231             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
 232             count=count-1\r
 233             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
 234 \r
 235     return sitesChanged
 236
 237 def haversine(site_lat, site_lon, lat, lon):
 238     d=0
 239     if lat and lon and site_lat and site_lon:
 240         site_lat = float(site_lat)
 241         site_lon = float(site_lon)
 242         lat = float(lat)
 243         lon = float(lon)
 244         R = 6378.1
 245         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
 246         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
 247         d = R * c
 248
 249     return d
 250
 251 def userSliceTableFormatter(data):
 252     formattedData = {
 253                      'rows' : data
 254                     }
 255     return formattedData