planetstack/core/dashboard/views/view_common.py

   1 import os
   2 import sys
   3 from django.views.generic import TemplateView, View
   4 import datetime
   5 from pprint import pprint
   6 import json
   7 from syndicate_storage.models import *
   8 from core.models import *
   9 from hpc.models import ContentProvider
  10 from operator import attrgetter
  11 from django import template
  12 from django.views.decorators.csrf import csrf_exempt
  13 from django.http import HttpResponse, HttpResponseServerError, HttpResponseForbidden
  14 from django.core import urlresolvers
  15 from django.contrib.gis.geoip import GeoIP
  16 from django.db.models import Q
  17 from ipware.ip import get_ip
  18 from operator import itemgetter, attrgetter
  19 import traceback
  20 import math
  21
  22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
  23     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
  24 else:
  25     sys.path.append("/opt/planetstack/hpc_wizard")
  26 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
  27
  28 def getDashboardContext(user, context={}, tableFormat = False):
  29         context = {}
  30
  31         userSliceData = getSliceInfo(user)
  32         if (tableFormat):
  33             context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
  34         else:
  35             context['userSliceInfo'] = userSliceData
  36         context['cdnData'] = getCDNOperatorData(wait=False)
  37         context['cdnContentProviders'] = getCDNContentProviderData()
  38
  39         (dashboards, unusedDashboards)= getDashboards(user)
  40         unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
  41         context['dashboards'] = dashboards
  42         context['unusedDashboards'] = unusedDashboards
  43
  44         return context
  45
  46 def getDashboards(user):
  47     dashboards = user.get_dashboards()
  48
  49     dashboard_names = [d.name for d in dashboards]
  50
  51     unused_dashboard_names = []
  52     for dashboardView in DashboardView.objects.all():
  53         # do not show disabled dashboard views
  54         if not dashboardView.enabled:
  55             continue
  56         if not dashboardView.name in dashboard_names:
  57             unused_dashboard_names.append(dashboardView.name)
  58
  59     return (dashboard_names, unused_dashboard_names)
  60
  61 def getSliceInfo(user):
  62     sliceList = Slice.objects.all()
  63     slicePrivs = SlicePrivilege.objects.filter(user=user)
  64     userSliceInfo = []
  65     for entry in slicePrivs:
  66
  67         slice = Slice.objects.filter(id=entry.slice.id)
  68         if not slice:
  69             # the privilege is to a slice that doesn't exist
  70             print "data model consistency problem, slice %s doesn't exist" % entry.slice.id
  71             continue
  72         slice = slice[0]
  73         slicename = slice.name
  74         sliverList=Sliver.objects.all()
  75         sites_used = {}
  76         for sliver in slice.slivers.all():
  77              #sites_used['deploymentSites'] = sliver.node.deployment.name
  78              # sites_used[sliver.image.name] = sliver.image.name
  79              sites_used[sliver.node.site_deployment.site] = 1 #sliver.numberCores
  80         sliceid = Slice.objects.get(id=entry.slice.id).id
  81         try:
  82             sliverList = Sliver.objects.filter(slice=entry.slice.id)
  83             siteList = {}
  84             for x in sliverList:
  85                if x.node.site_deployment.site not in siteList:
  86                   siteList[x.node.site_deployment.site] = 1
  87             slivercount = len(sliverList)
  88             sitecount = len(siteList)
  89         except:
  90             traceback.print_exc()
  91             slivercount = 0
  92             sitecount = 0
  93
  94         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
  95                               'sitesUsed':sites_used,
  96                               'role': SliceRole.objects.get(id=entry.role.id).role,
  97                               'slivercount': slivercount,
  98                               'sitecount':sitecount})
  99
 100     return userSliceInfo
 101
 102 def getCDNContentProviderData():
 103     cps = []
 104     for dm_cp in ContentProvider.objects.all():
 105         cp = {"name": dm_cp.name,
 106               "account": dm_cp.account}
 107         cps.append(cp)
 108
 109     return cps
 110
 111 def getCDNOperatorData(randomizeData = False, wait=True):
 112     HPC_SLICE_NAME = "HyperCache"
 113
 114     bq = PlanetStackAnalytics()
 115
 116     rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
 117
 118     # wait=False on the first time the Dashboard is opened. This means we might
 119     # not have any rows yet. The dashboard code polls every 30 seconds, so it
 120     # will eventually pick them up.
 121
 122     if rows:
 123         rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
 124
 125         # dictionaryize the statistics rows by site name
 126         stats_rows = {}
 127         for row in rows:
 128             stats_rows[row["site"]] = row
 129     else:
 130         stats_rows = {}
 131
 132     slice = Slice.objects.filter(name=HPC_SLICE_NAME)
 133     if slice:
 134         slice_slivers = list(slice[0].slivers.all())
 135     else:
 136         slice_slivers = []
 137
 138     new_rows = {}
 139     for site in Site.objects.all():
 140         # compute number of slivers allocated in the data model
 141         allocated_slivers = 0
 142         for sliver in slice_slivers:
 143             if sliver.node.site == site:
 144                 allocated_slivers = allocated_slivers + 1
 145
 146         stats_row = stats_rows.get(site.name,{})
 147
 148         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
 149         cpu=float(max_cpu)/100.0
 150         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
 151
 152         try:
 153            lat=float(site.location.latitude)
 154            long=float(site.location.longitude)
 155         except:
 156            lat=0
 157            long=0
 158
 159         # format it to what that CDN Operations View is expecting
 160         new_row = {"lat": lat,
 161                "long": long,
 162                "health": 0,
 163                #"numNodes": int(site.nodes.count()),
 164                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),     # measured number of slivers, from bigquery statistics
 165                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
 166                "siteUrl": str(site.site_url),
 167                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
 168                "load": max_cpu,
 169                "hot": float(hotness)}
 170         new_rows[str(site.name)] = new_row
 171
 172     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
 173     for (k,v) in new_rows.items():
 174         bad=False
 175         if v["numHPCSlivers"]==0:
 176             for v2 in new_rows.values():
 177                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
 178                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
 179                     if d<100:
 180                          bad=True
 181             if bad:
 182                 del new_rows[k]
 183
 184     return new_rows
 185
 186 def slice_increase_slivers(user, user_ip, siteList, slice, image, count, noAct=False):
 187     sitesChanged = {}
 188
 189     # let's compute how many slivers are in use in each node of each site
 190     for site in siteList:
 191         site.nodeList = list(site.nodes.all())
 192         for node in site.nodeList:
 193             node.sliverCount = 0
 194             for sliver in node.slivers.all():
 195                  if sliver.slice.id == slice.id:
 196                      node.sliverCount = node.sliverCount + 1
 197
 198     # Allocate slivers to nodes
 199     # for now, assume we want to allocate all slivers from the same site
 200     nodes = siteList[0].nodeList
 201     while (count>0):
 202         # Sort the node list by number of slivers per node, then pick the
 203         # node with the least number of slivers.
 204         nodes = sorted(nodes, key=attrgetter("sliverCount"))
 205         node = nodes[0]
 206
 207         print "adding sliver at node", node.name, "of site", node.site.name
 208
 209         if not noAct:
 210             sliver = Sliver(name=node.name,
 211                         slice=slice,
 212                         node=node,
 213                         image = image,
 214                         creator = User.objects.get(email=user),
 215                         deploymentNetwork=node.deployment)
 216             sliver.save()
 217
 218         node.sliverCount = node.sliverCount + 1
 219
 220         count = count - 1
 221
 222         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
 223
 224     return sitesChanged
 225
 226 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
 227     sitesChanged = {}
 228     if siteList:
 229         siteNames = [site.name for site in siteList]
 230     else:
 231         siteNames = None
 232
 233     for sliver in list(slice.slivers.all()):
 234         if count>0:
 235             if(not siteNames) or (sliver.node.site.name in siteNames):
 236                 sliver.delete()\r
 237                 print "deleting sliver",sliver.name,"at node",sliver.node.name\r
 238                 count=count-1\r
 239                 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
 240 \r
 241     return sitesChanged
 242
 243 def haversine(site_lat, site_lon, lat, lon):
 244     d=0
 245     if lat and lon and site_lat and site_lon:
 246         site_lat = float(site_lat)
 247         site_lon = float(site_lon)
 248         lat = float(lat)
 249         lon = float(lon)
 250         R = 6378.1
 251         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
 252         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
 253         d = R * c
 254
 255     return d
 256
 257 def userSliceTableFormatter(data):
 258     formattedData = {
 259                      'rows' : data
 260                     }
 261     return formattedData