from pprint import pprint
import json
from core.models import *
+from hpc.models import ContentProvider
from operator import attrgetter
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse, HttpResponseServerError
else:
sys.path.append("/opt/planetstack/hpc_wizard")
import hpc_wizard
-from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics
+from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
class DashboardWelcomeView(TemplateView):
template_name = 'admin/dashboard/welcome.html'
context['userSliceInfo'] = userDetails['userSliceInfo']
context['cdnData'] = userDetails['cdnData']
+ context['cdnContentProviders'] = userDetails['cdnContentProviders']
return self.render_to_response(context=context)
def getUserSliceInfo(user, tableFormat = False):
userDetails = {}
-# try:
-# // site = Site.objects.filter(id=user.site.id)
-# // except:
-# // site = Site.objects.filter(name="Princeton")
-# // userDetails['sitename'] = site[0].name
-# // userDetails['siteid'] = site[0].id
userSliceData = getSliceInfo(user)
if (tableFormat):
userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
else:
userDetails['userSliceInfo'] = userSliceData
- userDetails['cdnData'] = getCDNOperatorData();
-# pprint( userDetails)
+ userDetails['cdnData'] = getCDNOperatorData(wait=False)
+ userDetails['cdnContentProviders'] = getCDNContentProviderData()
return userDetails
class TenantCreateSlice(View):
serviceClass = request.POST.get("serviceClass", "0")
imageName = request.POST.get("imageName", "0")
actionToDo = request.POST.get("actionToDo", "0")
- print sliceName
+ network = request.POST.get("network","0")
+ mountDataSets = request.POST.get("mountDataSets","0")
if (actionToDo == "add"):
serviceClass = ServiceClass.objects.get(name=serviceClass)
site = request.user.site
- #image = Image.objects.get(name=imageName)
- newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
+ image = Image.objects.get(name=imageName)
+ newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
newSlice.save()
- return newSlice
-
+ return HttpResponse("Slice created")
+
+class TenantUpdateSlice(View):
+ def post(self, request, *args, **kwargs):\r
+ sliceName = request.POST.get("sliceName", "0")\r
+ serviceClass = request.POST.get("serviceClass", "0")\r
+ imageName = request.POST.get("imageName", "0")\r
+ actionToDo = request.POST.get("actionToDo", "0")\r
+ network = request.POST.get("network","0")\r
+ dataSet = request.POST.get("dataSet","0")\r
+ slice = Slice.objects.all()\r
+ for entry in slice:\r
+ serviceClass = ServiceClass.objects.get(name=serviceClass)\r
+ if(entry.name==sliceName):\r
+ if (actionToDo == "update"):\r
+ setattr(entry,'serviceClass',serviceClass)\r
+ setattr(entry,'imagePreference',imageName)\r
+ setattr(entry,'network',network)\r
+ setattr(entry,'mountDataSets',dataSet)\r
+ entry.save()\r
+ break\r
+ return HttpResponse("Slice updated")\r
+\r
+def update_slice(sliceName,**fields):
+ slice = Slice.objects.filter(name = sliceName)\r
+ for (k,v) in fields.items():\r
+ setattr(slice, k, v)\r
+ slice.save()\r
+ return slice
def getTenantSliceInfo(user, tableFormat = False):
tenantSliceDetails = {}
slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
sliceServiceClass = entry.serviceClass.name
preferredImage = entry.imagePreference
+ sliceDataSet = entry.mountDataSets
+ sliceNetwork = entry.network
numSliver = 0
sliceImage=""
sliceSite = {}
+ sliceNode = {}
+ sliceInstance= {}
for sliver in slice.slivers.all():
- numSliver +=sliver.numberCores
- # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
if sliver.node.site.name in BLESSED_SITES:
sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
sliceImage = sliver.image.name
+ sliceNode[str(sliver)] = sliver.node.name
+ numSliver = sum(sliceSite.values())
numSites = len(sliceSite)
- userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
+ userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
return userSliceInfo
def getTenantSitesInfo():
return sliceInfo
def getImageInfo(user):
- #imageList = Image.objects.all()
- imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
+ imageList = Image.objects.all()
+ #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
imageInfo = []
for imageEntry in imageList:
- #imageInfo.append({'Image':imageEntry.name})
- imageInfo.append({'Image':imageEntry})
+ imageInfo.append({'Image':imageEntry.name})
+ #imageInfo.append({'Image':imageEntry})
return imageInfo
def getMountDataSets():
- dataSetList = ['GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
+ dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
dataSetInfo = []\r
for entry in dataSetList:\r
dataSetInfo.append({'DataSet':entry})\r
return userSliceInfo
-def getCDNOperatorData(randomizeData = False):
+def getCDNContentProviderData():
+ cps = []
+ for dm_cp in ContentProvider.objects.all():
+ cp = {"name": dm_cp.name,
+ "account": dm_cp.account}
+ cps.append(cp)
+
+ return cps
+
+def getCDNOperatorData(randomizeData = False, wait=True):
+ HPC_SLICE_NAME = "HyperCache"
+
bq = PlanetStackAnalytics()
- #hpc_sliceNames = bq.service_to_sliceNames("HPC Service")
+ rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
- rows = bq.get_cached_query_results(bq.compose_latest_query())
+ # wait=False on the first time the Dashboard is opened. This means we might
+ # not have any rows yet. The dashboard code polls every 30 seconds, so it
+ # will eventually pick them up.
- #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames]
+ if rows:
+ rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
- rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
+ # dictionaryize the statistics rows by site name
+ stats_rows = {}
+ for row in rows:
+ stats_rows[row["site"]] = row
+ else:
+ stats_rows = {}
- bq.merge_datamodel_sites(rows, slice="HyperCache")
+ slice = Slice.objects.filter(name=HPC_SLICE_NAME)
+ if slice:
+ slice_slivers = list(slice[0].slivers.all())
+ else:
+ slice_slivers = []
new_rows = {}
- for row in rows:
- new_row = {"lat": float(row.get("lat", 0)),
- "long": float(row.get("long", 0)),
+ for site in Site.objects.all():
+ # compute number of slivers allocated in the data model
+ allocated_slivers = 0
+ for sliver in slice_slivers:
+ if sliver.node.site == site:
+ allocated_slivers = allocated_slivers + 1
+
+ stats_row = stats_rows.get(site.name,{})
+
+ max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
+ cpu=float(max_cpu)/100.0
+ hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
+
+ # format it to what that CDN Operations View is expecting
+ new_row = {"lat": float(site.location.longitude),
+ "long": float(site.location.longitude),
+ "lat": float(site.location.latitude),
"health": 0,
- "numNodes": int(row.get("numNodes",0)),
- "activeHPCSlivers": int(row.get("count_hostname", 0)),
- "numHPCSlivers": int(row.get("allocated_slivers", 0)),
- "siteUrl": str(row.get("url", "")),
- "hot": float(row.get("hotness", 0.0)),
- "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0),
- "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))}
- new_rows[str(row["site"])] = new_row
+ "numNodes": int(site.nodes.count()),
+ "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
+ "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
+ "siteUrl": str(site.site_url),
+ "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
+ "load": max_cpu,
+ "hot": float(hotness)}
+ new_rows[str(site.name)] = new_row
+
+ # get rid of sites with 0 slivers that overlap other sites with >0 slivers
+ for (k,v) in new_rows.items():
+ bad=False
+ if v["numHPCSlivers"]==0:
+ for v2 in new_rows.values():
+ if (v!=v2) and (v2["numHPCSlivers"]>=0):
+ d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
+ if d<100:
+ bad=True
+ if bad:
+ del new_rows[k]
return new_rows
def get(self, request, **kwargs):
return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
+def haversine(site_lat, site_lon, lat, lon):
+ d=0
+ if lat and lon and site_lat and site_lon:
+ site_lat = float(site_lat)
+ site_lon = float(site_lon)
+ lat = float(lat)
+ lon = float(lon)
+ R = 6378.1
+ a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
+ c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
+ d = R * c
+
+ return d
+
def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
# try to pick a site we're already using
has_slivers_here=False
has_slivers_here=True
# Haversine method
- d = 0
- site_lat = site.location.latitude
- site_lon = site.location.longitude
- if lat and lon and site_lat and site_lon:
- site_lat = float(site_lat)
- site_lon = float(site_lon)
- R = 6378.1
- a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
- c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
- d = R * c
+ d = haversine(site.location.latitude, site.location.longitude, lat, lon)
return (-has_slivers_here, d)
print slice, slice.id\r
sliceToDel=Slice(name=sliceName, id=slice.id)\r
sliceToDel.delete()
+ return HttpResponse("Slice deleted")
class TenantAddOrRemoveSliverView(View):
""" Add or remove slivers from a Slice
git://git.onelab.eu / plstackapi.git / blobdiff