import datetime
from pprint import pprint
import json
+from syndicate.models import *
from core.models import *
+from hpc.models import ContentProvider
from operator import attrgetter
+from django import template
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse, HttpResponseServerError
from django.core import urlresolvers
from django.contrib.gis.geoip import GeoIP
from ipware.ip import get_ip
+from operator import itemgetter, attrgetter
import traceback
import socket
else:
sys.path.append("/opt/planetstack/hpc_wizard")
import hpc_wizard
-from planetstack_analytics import DoPlanetStackAnalytics
+from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
class DashboardWelcomeView(TemplateView):
template_name = 'admin/dashboard/welcome.html'
def get(self, request, *args, **kwargs):
context = self.get_context_data(**kwargs)
- userDetails = getUserSliceInfo(request.user)
- #context['site'] = userDetails['site']
-
- context['userSliceInfo'] = userDetails['userSliceInfo']
- context['cdnData'] = userDetails['cdnData']
+ context = getDashboardContext(request.user, context)
return self.render_to_response(context=context)
-def getUserSliceInfo(user, tableFormat = False):
- userDetails = {}
-# try:
-# // site = Site.objects.filter(id=user.site.id)
-# // except:
-# // site = Site.objects.filter(name="Princeton")
-# // userDetails['sitename'] = site[0].name
-# // userDetails['siteid'] = site[0].id
+class DashboardDynamicView(TemplateView):
+ head_template = r"""{% extends "admin/dashboard/dashboard_base.html" %}
+ {% load admin_static %}
+ {% block content %}
+ """
+
+ tail_template = r"{% endblock %}"
+
+ def get(self, request, name="root", *args, **kwargs):
+ context = self.get_context_data(**kwargs)
+ context = getDashboardContext(request.user, context)
+
+ if name=="root":
+ return self.multiDashboardView(request, context)
+ else:
+ return self.singleDashboardView(request, name, context)
+
+ def readDashboard(self, fn):
+ try:
+ template= open("/opt/planetstack/templates/admin/dashboard/%s.html" % fn, "r").read()
+ if (fn=="tenant"):
+ # fix for tenant view - it writes html to a div called tabs-5
+ template = '<div id="tabs-5"></div>' + template
+ if (fn=="slice_interactions"):
+ # fix for slice_interactions - it gives its container div a 40px
+ # margin, and then positions it's header at -40px
+ template = '<style>#tabs-4 { margin: 40px; font-size: 24px; font-weight: bold;} </style><div id="tabs-4">' + template + '</div>'
+ return template
+ except:
+ return "failed to open %s" % fn
+
+ def multiDashboardView(self, request, context):
+ head_template = self.head_template
+ tail_template = self.tail_template
+
+ body = """
+ <div id="hometabs" >
+ <ul id="suit_form_tabs" class="nav nav-tabs nav-tabs-suit" data-tab-prefix="suit-tab">
+ """
+
+ dashboards = request.user.get_dashboards()
+
+ # customize is a special dashboard they always get
+ customize = DashboardView.objects.filter(name="Customize")
+ if customize:
+ dashboards.append(customize[0])
+
+ for i,view in enumerate(dashboards):
+ body = body + '<li><a href="#dashtab-%d">%s</a></li>\n' % (i, view.name)
+
+ body = body + "</ul>\n"
+
+ for i,view in enumerate(dashboards):
+ url = view.url
+ body = body + '<div id="dashtab-%d">\n' % i
+ if url.startswith("template:"):
+ fn = url[9:]
+ body = body + self.readDashboard(fn)
+ body = body + '</div>\n'
+
+ body=body+"</div>\n"
+
+ t = template.Template(head_template + body + self.tail_template)
+
+ response_kwargs = {}
+ response_kwargs.setdefault('content_type', self.content_type)
+ return self.response_class(\r
+ request = request,\r
+ template = t,\r
+ context = context,\r
+ **response_kwargs)
+
+ def singleDashboardView(self, request, name, context):
+ head_template = self.head_template
+ tail_template = self.tail_template
+
+ t = template.Template(head_template + self.readDashboard(fn) + self.tail_template)
+
+ response_kwargs = {}
+ response_kwargs.setdefault('content_type', self.content_type)
+ return self.response_class(\r
+ request = request,\r
+ template = t,\r
+ context = context,\r
+ **response_kwargs)
+
+def getDashboardContext(user, context={}, tableFormat = False):
+ context = {}
userSliceData = getSliceInfo(user)
if (tableFormat):
-# pprint("******* GET USER SLICE INFO")
- userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
+ context['userSliceInfo'] = userSliceTableFormatter(userSliceData)
else:
- userDetails['userSliceInfo'] = userSliceData
- userDetails['cdnData'] = getCDNOperatorData();
-# pprint( userDetails)
- return userDetails
+ context['userSliceInfo'] = userSliceData
+ context['cdnData'] = getCDNOperatorData(wait=False)
+ context['cdnContentProviders'] = getCDNContentProviderData()
+
+ (dashboards, unusedDashboards)= getDashboards(user)
+ unusedDashboards=[x for x in unusedDashboards if x!="Customize"]
+ context['dashboards'] = dashboards
+ context['unusedDashboards'] = unusedDashboards
+
+ return context
+
+def getDashboards(user):
+ #dashboards = sorted(list(user.dashboardViews.all()), key=attrgetter('order'))
+ dashboards = user.get_dashboards()
+
+ dashboard_names = [d.name for d in dashboards]
+
+ unused_dashboard_names = []
+ for dashboardView in DashboardView.objects.all():
+ if not dashboardView.name in dashboard_names:
+ unused_dashboard_names.append(dashboardView.name)
+
+ return (dashboard_names, unused_dashboard_names)
class TenantCreateSlice(View):
def post(self, request, *args, **kwargs):
serviceClass = request.POST.get("serviceClass", "0")
imageName = request.POST.get("imageName", "0")
actionToDo = request.POST.get("actionToDo", "0")
- print sliceName
+ #network = request.POST.get("network","0")
+ mountDataSets = request.POST.get("mountDataSets","0")
if (actionToDo == "add"):
serviceClass = ServiceClass.objects.get(name=serviceClass)
site = request.user.site
- #image = Image.objects.get(name=imageName)
- newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
+ image = Image.objects.get(name=imageName)
+ newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets)
newSlice.save()
- return newSlice
-
-
+ return HttpResponse("Slice created")
+
+class TenantUpdateSlice(View):
+ def post(self, request, *args, **kwargs):\r
+ sliceName = request.POST.get("sliceName", "0")\r
+ serviceClass = request.POST.get("serviceClass", "0")\r
+ imageName = request.POST.get("imageName", "0")\r
+ actionToDo = request.POST.get("actionToDo", "0")\r
+ #network = request.POST.get("network","0")\r
+ dataSet = request.POST.get("dataSet","0")\r
+ slice = Slice.objects.all()\r
+ for entry in slice:\r
+ serviceClass = ServiceClass.objects.get(name=serviceClass)\r
+ if(entry.name==sliceName):\r
+ if (actionToDo == "update"):\r
+ setattr(entry,'serviceClass',serviceClass)\r
+ setattr(entry,'imagePreference',imageName)\r
+ #setattr(entry,'network',network)\r
+ setattr(entry,'mountDataSets',dataSet)\r
+ entry.save()\r
+ break\r
+ return HttpResponse("Slice updated")\r
+\r
def getTenantSliceInfo(user, tableFormat = False):
tenantSliceDetails = {}
tenantSliceData = getTenantInfo(user)
tenantSliceDetails['userSliceInfo'] = tenantSliceData
tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
- tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
+ #tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
- tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo());
+ tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
+ tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
+ tenantSliceDetails['publicKey'] = getPublicKey(user)
return tenantSliceDetails
-
def getTenantInfo(user):
slices =Slice.objects.all()
userSliceInfo = []
slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
sliceServiceClass = entry.serviceClass.name
preferredImage = entry.imagePreference
+ sliceDataSet = entry.mountDataSets
+ #sliceNetwork = entry.network
numSliver = 0
sliceImage=""
sliceSite = {}
+ sliceNode = {}
+ sliceInstance= {}
for sliver in slice.slivers.all():
- numSliver +=sliver.numberCores
- # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
if sliver.node.site.name in BLESSED_SITES:
- print "equal",sliver.node.site.name
sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
sliceImage = sliver.image.name
- userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
+ sliceNode[str(sliver)] = sliver.node.name
+ numSliver = sum(sliceSite.values())
+ numSites = len(sliceSite)
+ userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'instanceNodePair':sliceNode})
return userSliceInfo
def getTenantSitesInfo():
}
return formattedData
+def getPublicKey(user):
+ users=User.objects.all()\r
+ for key in users:\r
+ if (str(key.email)==str(user)):\r
+ sshKey = key.public_key\r
+ return sshKey
+
def getServiceClassInfo(user):
serviceClassList = ServiceClass.objects.all()
sliceInfo = []
def getImageInfo(user):
imageList = Image.objects.all()
+ #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
imageInfo = []
for imageEntry in imageList:
imageInfo.append({'Image':imageEntry.name})
+ #imageInfo.append({'Image':imageEntry})
return imageInfo
+def createPrivateVolume(user, sliceName):
+ caps = Volume.CAP_READ_DATA | Volume.CAP_WRITE_DATA | Volume.CAP_HOST_DATA
+ getattr(Volume.default_gateway_caps,"read data") | \
+ getattr(Volume.default_gateway_caps,"write data") | \
+ getattr(Volume.default_gateway_caps,"host files")
+ v = Volume(name="private_" + sliceName, owner_id=user, description="private volume for %s" % sliceName, blocksize=61440, private=True, archive=False, default_gateway_caps = caps)
+ v.save()
+
+SYNDICATE_REPLICATE_PORTNUM = 1025
+
+def get_free_port():
+ inuse={}
+ inuse[SYNDICATE_REPLICATE_PORTNUM] = True
+ for vs in VolumeSlice.objects.all():
+ inuse[vs.peer_portnum]=True
+ inuse[vs.replicate_portnum]=True
+ for network in Network.objects.all():
+ network_ports = [x.strip() for x in network.ports.split(",")]
+ for network_port in network_ports:
+ try:
+ inuse[int(network_port)] = True
+ except:
+ # in case someone has put a malformed port number in the list
+ pass
+ for i in range(1025, 65535):
+ if not inuse.get(i,False):
+ return i
+ return False
+
+def mountVolume(sliceName, volumeName, readWrite):
+ slice = Slice.objects.get(name=sliceName)
+ volume = Volume.objects.get(name=volumeName)
+ # choose some unused port numbers
+ flags = Volume.CAP_READ_DATA
+ if readWrite:
+ flags = flags | Volume.CAP_WRITE_DATA
+ vs = VolumeSlice(volume_id = volume, slice_id = slice, gateway_caps=flags, peer_portnum = get_free_port(), replicate_portnum = SYNDICATE_REPLICATE_PORTNUM)
+ vs.save()
+
+def hasPrivateVolume(sliceName):
+ slice = Slice.objects.get(name=sliceName)
+ for vs in VolumeSlice.objects.filter(slice_id=slice):
+ if vs.volume_id.private:
+ return True
+ return False
+
+def getMountDataSets():
+ dataSetInfo=[]\r
+ for volume in Volume.objects.all():\r
+ if not volume.private:\r
+ dataSetInfo.append({'DataSet': volume.name})\r
+\r
+ return dataSetInfo
+
def getNetworkInfo(user):
#networkList = Network.objects.all()
networkList = ['Private Only','Private and Publicly Routable']
return userSliceInfo
-def getCDNOperatorData(randomizeData = False):
- return hpc_wizard.get_hpc_wizard().get_sites_for_view()
+def getCDNContentProviderData():
+ cps = []
+ for dm_cp in ContentProvider.objects.all():
+ cp = {"name": dm_cp.name,
+ "account": dm_cp.account}
+ cps.append(cp)
+
+ return cps
+
+def getCDNOperatorData(randomizeData = False, wait=True):
+ HPC_SLICE_NAME = "HyperCache"
+
+ bq = PlanetStackAnalytics()
+
+ rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
+
+ # wait=False on the first time the Dashboard is opened. This means we might
+ # not have any rows yet. The dashboard code polls every 30 seconds, so it
+ # will eventually pick them up.
+
+ if rows:
+ rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
+
+ # dictionaryize the statistics rows by site name
+ stats_rows = {}
+ for row in rows:
+ stats_rows[row["site"]] = row
+ else:
+ stats_rows = {}
+
+ slice = Slice.objects.filter(name=HPC_SLICE_NAME)
+ if slice:
+ slice_slivers = list(slice[0].slivers.all())
+ else:
+ slice_slivers = []
+
+ new_rows = {}
+ for site in Site.objects.all():
+ # compute number of slivers allocated in the data model
+ allocated_slivers = 0
+ for sliver in slice_slivers:
+ if sliver.node.site == site:
+ allocated_slivers = allocated_slivers + 1
+
+ stats_row = stats_rows.get(site.name,{})
+
+ max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
+ cpu=float(max_cpu)/100.0
+ hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
+
+ # format it to what that CDN Operations View is expecting
+ new_row = {"lat": float(site.location.longitude),
+ "long": float(site.location.longitude),
+ "lat": float(site.location.latitude),
+ "health": 0,
+ "numNodes": int(site.nodes.count()),
+ "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
+ "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
+ "siteUrl": str(site.site_url),
+ "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
+ "load": max_cpu,
+ "hot": float(hotness)}
+ new_rows[str(site.name)] = new_row
+
+ # get rid of sites with 0 slivers that overlap other sites with >0 slivers
+ for (k,v) in new_rows.items():
+ bad=False
+ if v["numHPCSlivers"]==0:
+ for v2 in new_rows.values():
+ if (v!=v2) and (v2["numHPCSlivers"]>=0):
+ d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
+ if d<100:
+ bad=True
+ if bad:
+ del new_rows[k]
+
+ return new_rows
def getPageSummary(request):
slice = request.GET.get('slice', None)
site = request.GET.get('site', None)
node = request.GET.get('node', None)
-
class SimulatorView(View):
def get(self, request, **kwargs):
sim = json.loads(file("/tmp/simulator.json","r").read())
class DashboardUserSiteView(View):
def get(self, request, **kwargs):
- return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
+ return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
class TenantViewData(View):
def get(self, request, **kwargs):
return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
+def haversine(site_lat, site_lon, lat, lon):
+ d=0
+ if lat and lon and site_lat and site_lon:
+ site_lat = float(site_lat)
+ site_lon = float(site_lon)
+ lat = float(lat)
+ lon = float(lon)
+ R = 6378.1
+ a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
+ c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
+ d = R * c
+
+ return d
+
def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
# try to pick a site we're already using
has_slivers_here=False
has_slivers_here=True
# Haversine method
- d = 0
- site_lat = site.location.latitude
- site_lon = site.location.longitude
- if lat and lon and site_lat and site_lon:
- site_lat = float(site_lat)
- site_lon = float(site_lon)
- R = 6378.1
- a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
- c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
- d = R * c
+ d = haversine(site.location.latitude, site.location.longitude, lat, lon)
return (-has_slivers_here, d)
node.sliverCount = 0
for sliver in node.slivers.all():
if sliver.slice.id == slice.id:
- node.sliverCount = node.sliverCount +1
+ node.sliverCount = node.sliverCount + 1
# Allocate slivers to nodes
# for now, assume we want to allocate all slivers from the same site
def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
sitesChanged = {}
-
+ sliverList ={}
if siteList:
siteNames = [site.name for site in siteList]
else:
siteNames = None
for sliver in slice.slivers.all():
- if (count <= 0):
- break
-
- node = sliver.node
- if (not siteNames) or (node.site.name in siteNames):
- print "deleting sliver", sliver, "at node", node.name, "of site", node.site.name
- if not noAct:
- sliver.delete()
- count = count -1
-
- sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) - 1
-
+ if(not siteNames) or (sliver.node.site.name in siteNames):\r
+ node = sliver.node\r
+ sliverList[sliver.name]=node.name
+
+ for key in sliverList:
+ if count>0:
+ sliver = Sliver.objects.filter(name=key)[0]\r
+ sliver.delete()\r
+ print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
+ count=count-1\r
+ sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
+\r
return sitesChanged
+class TenantDeleteSliceView(View):
+ def post(self,request):\r
+ sliceName = request.POST.get("sliceName",None)\r
+ slice = Slice.objects.get(name=sliceName)\r
+ #print slice, slice.id\r
+ sliceToDel=Slice(name=sliceName, id=slice.id)\r
+ sliceToDel.delete()
+ return HttpResponse("Slice deleted")
+
class TenantAddOrRemoveSliverView(View):
""" Add or remove slivers from a Slice
class DashboardSummaryAjaxView(View):
def get(self, request, **kwargs):
- return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
+ def avg(x):
+ if len(x)==0:
+ return 0
+ return float(sum(x))/len(x)
+
+ sites = getCDNOperatorData().values()
+
+ sites = [site for site in sites if site["numHPCSlivers"]>0]
+
+ total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
+ total_bandwidth = sum( [site["bandwidth"] for site in sites] )
+ average_cpu = int(avg( [site["load"] for site in sites] ))
+
+ result= {"total_slivers": total_slivers,
+ "total_bandwidth": total_bandwidth,
+ "average_cpu": average_cpu}
+
+ return HttpResponse(json.dumps(result), mimetype='application/javascript')
class DashboardAddOrRemoveSliverView(View):
# TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
if (name == "hpcSummary"):
return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
elif (name == "hpcUserSite"):
- return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
+ return HttpResponse(json.dumps(getDashboardContext(request.user, tableFormat=True)), mimetype='application/javascript')
elif (name == "hpcMap"):
return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
elif (name == "bigquery"):
else:
return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
+class DashboardCustomize(View):
+ def post(self, request, *args, **kwargs):\r
+ dashboards = request.POST.get("dashboards", None)\r
+ if not dashboards:\r
+ dashboards=[]\r
+ else:\r
+ dashboards = [x.strip() for x in dashboards.split(",")]\r
+ dashboards = [DashboardView.objects.get(name=x) for x in dashboards]\r
+\r
+ request.user.dashboardViews.all().delete()\r
+\r
+ for i,dashboard in enumerate(dashboards):\r
+ udbv = UserDashboardView(user=request.user, dashboardView=dashboard, order=i)\r
+ udbv.save()\r
+\r
+ return HttpResponse("updated")\r
+
git://git.onelab.eu / plstackapi.git / blobdiff