6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData();
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
67 if (actionToDo == "add"):
68 serviceClass = ServiceClass.objects.get(name=serviceClass)
69 site = request.user.site
70 #image = Image.objects.get(name=imageName)
71 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
76 def getTenantSliceInfo(user, tableFormat = False):
77 tenantSliceDetails = {}
78 tenantSliceData = getTenantInfo(user)
79 tenantServiceClassData = getServiceClassInfo(user)
81 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
82 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
84 tenantSliceDetails['userSliceInfo'] = tenantSliceData
85 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
86 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
87 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
88 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
89 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
90 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
91 return tenantSliceDetails
94 def getTenantInfo(user):
95 slices =Slice.objects.all()
98 sliceName = Slice.objects.get(id=entry.id).name
99 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
100 sliceServiceClass = entry.serviceClass.name
101 preferredImage = entry.imagePreference
105 for sliver in slice.slivers.all():
106 numSliver +=sliver.numberCores
107 # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
108 if sliver.node.site.name in BLESSED_SITES:
109 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
110 sliceImage = sliver.image.name
111 numSites = len(sliceSite)
112 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
115 def getTenantSitesInfo():
117 for entry in Site.objects.all():
118 if entry.name in BLESSED_SITES:
119 tenantSiteInfo.append({'siteName':entry.name})
120 return tenantSiteInfo
122 def userSliceTableFormatter(data):
129 def getServiceClassInfo(user):
130 serviceClassList = ServiceClass.objects.all()
132 for entry in serviceClassList:
133 sliceInfo.append({'serviceClass':entry.name})
136 def getImageInfo(user):
137 #imageList = Image.objects.all()
138 imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
140 for imageEntry in imageList:
141 #imageInfo.append({'Image':imageEntry.name})
142 imageInfo.append({'Image':imageEntry})
145 def getMountDataSets():
146 dataSetList = ['GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
148 for entry in dataSetList:
\r
149 dataSetInfo.append({'DataSet':entry})
\r
152 def getNetworkInfo(user):
153 #networkList = Network.objects.all()
154 networkList = ['Private Only','Private and Publicly Routable']
156 for networkEntry in networkList:
157 #networkInfo.append({'Network':networkEntry.name})
158 networkInfo.append({'Network':networkEntry})
161 def getDeploymentSites():
162 deploymentList = Deployment.objects.all()
164 for entry in deploymentList:
165 deploymentInfo.append({'DeploymentSite':entry.name})
166 return deploymentInfo
168 def getSliceInfo(user):
169 sliceList = Slice.objects.all()
170 slicePrivs = SlicePrivilege.objects.filter(user=user)
172 for entry in slicePrivs:
174 slicename = Slice.objects.get(id=entry.slice.id).name
175 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
176 sliverList=Sliver.objects.all()
178 for sliver in slice.slivers.all():
179 #sites_used['deploymentSites'] = sliver.node.deployment.name
180 # sites_used[sliver.image.name] = sliver.image.name
181 sites_used[sliver.node.site.name] = sliver.numberCores
182 sliceid = Slice.objects.get(id=entry.slice.id).id
184 sliverList = Sliver.objects.filter(slice=entry.slice.id)
187 if x.node.site not in siteList:
188 siteList[x.node.site] = 1
189 slivercount = len(sliverList)
190 sitecount = len(siteList)
192 traceback.print_exc()
196 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
197 'sitesUsed':sites_used,
198 'role': SliceRole.objects.get(id=entry.role.id).role,
199 'slivercount': slivercount,
200 'sitecount':sitecount})
204 def getCDNOperatorData(randomizeData = False):
205 bq = PlanetStackAnalytics()
207 #hpc_sliceNames = bq.service_to_sliceNames("HPC Service")
209 rows = bq.get_cached_query_results(bq.compose_latest_query())
211 #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames]
213 rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
215 bq.merge_datamodel_sites(rows, slice="HyperCache")
219 new_row = {"lat": float(row.get("lat", 0)),
220 "long": float(row.get("long", 0)),
222 "numNodes": int(row.get("numNodes",0)),
223 "activeHPCSlivers": int(row.get("count_hostname", 0)),
224 "numHPCSlivers": int(row.get("allocated_slivers", 0)),
225 "siteUrl": str(row.get("url", "")),
226 "hot": float(row.get("hotness", 0.0)),
227 "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0),
228 "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))}
229 new_rows[str(row["site"])] = new_row
233 class SimulatorView(View):
234 def get(self, request, **kwargs):
235 sim = json.loads(file("/tmp/simulator.json","r").read())
236 text = "<html><head></head><body>"
237 text += "Iteration: %d<br>" % sim["iteration"]
238 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
239 text += "<table border=1>"
240 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
241 for site in sim["site_load"].values():
243 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
244 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
247 text += "</body></html>"
248 return HttpResponse(text)
250 class DashboardUserSiteView(View):
251 def get(self, request, **kwargs):
252 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
254 class TenantViewData(View):
255 def get(self, request, **kwargs):
256 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
258 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
259 # try to pick a site we're already using
260 has_slivers_here=False
262 for sliver in slice.slivers.all():
263 if sliver.node.site.name == site.name:
264 has_slivers_here=True
268 site_lat = site.location.latitude
269 site_lon = site.location.longitude
270 if lat and lon and site_lat and site_lon:
271 site_lat = float(site_lat)
272 site_lon = float(site_lon)
274 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
275 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
278 return (-has_slivers_here, d)
280 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
281 """ Returns list of sites, sorted from most favorable to least favorable """
285 client_geo = GeoIP().city(user_ip)
287 lat=float(client_geo["latitude"])
288 lon=float(client_geo["longitude"])
290 print "exception in geo code"
291 traceback.print_exc()
293 sites = Site.objects.all()
294 sites = [x for x in sites if x.name in BLESSED_SITES]
295 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
299 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
302 # let's compute how many slivers are in use in each node of each site
303 for site in siteList:
304 site.nodeList = list(site.nodes.all())
305 for node in site.nodeList:
307 for sliver in node.slivers.all():
308 if sliver.slice.id == slice.id:
309 node.sliverCount = node.sliverCount + 1
311 # Allocate slivers to nodes
312 # for now, assume we want to allocate all slivers from the same site
313 nodes = siteList[0].nodeList
315 # Sort the node list by number of slivers per node, then pick the
316 # node with the least number of slivers.
317 nodes = sorted(nodes, key=attrgetter("sliverCount"))
320 print "adding sliver at node", node.name, "of site", node.site.name
323 sliver = Sliver(name=node.name,
326 image = Image.objects.all()[0],
327 creator = User.objects.get(email=user),
328 deploymentNetwork=node.deployment,
332 node.sliverCount = node.sliverCount + 1
336 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
340 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
344 siteNames = [site.name for site in siteList]
348 for sliver in slice.slivers.all():
349 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
351 sliverList[sliver.name]=node.name
353 for key in sliverList:
355 sliver = Sliver.objects.filter(name=key)[0]
\r
357 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
359 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
363 class TenantDeleteSliceView(View):
364 def post(self,request):
\r
365 sliceName = request.POST.get("sliceName",None)
\r
366 slice = Slice.objects.get(name=sliceName)
\r
367 print slice, slice.id
\r
368 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
371 class TenantAddOrRemoveSliverView(View):
372 """ Add or remove slivers from a Slice
375 siteName - name of site. If not specified, PlanetStack will pick the
377 actionToDo - [add | rem]
378 count - number of slivers to add or remove
379 sliceName - name of slice
380 noAct - if set, no changes will be made to db, but result will still
381 show which sites would have been modified.
384 Dictionary of sites that were modified, and the count of nodes
385 that were added or removed at each site.
387 def post(self, request, *args, **kwargs):
388 siteName = request.POST.get("siteName", None)
389 actionToDo = request.POST.get("actionToDo", None)
390 count = int(request.POST.get("count","0"))
391 sliceName = request.POST.get("slice", None)
392 noAct = request.POST.get("noAct", False)
395 return HttpResponseServerError("No slice name given")
397 slice = Slice.objects.get(name=sliceName)
400 siteList = [Site.objects.get(name=siteName)]
404 if (actionToDo == "add"):
405 user_ip = request.GET.get("ip", get_ip(request))
406 if (siteList is None):
407 siteList = tenant_pick_sites(user, user_ip, slice, count)
409 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
410 elif (actionToDo == "rem"):
411 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
413 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
415 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
417 def get(self, request, *args, **kwargs):
418 request.POST = request.GET
419 return self.post(request, *args, **kwargs) # for testing REST in browser
420 #return HttpResponseServerError("GET is not supported")
422 class TenantPickSitesView(View):
423 """ primarily just for testing purposes """
424 def get(self, request, *args, **kwargs):
425 count = request.GET.get("count","0")
426 slice = request.GET.get("slice",None)
428 slice = Slice.objects.get(name=slice)
429 ip = request.GET.get("ip", get_ip(request))
430 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
431 sites = [x.name for x in sites]
432 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
434 class DashboardSummaryAjaxView(View):
435 def get(self, request, **kwargs):
437 return float(sum(x))/len(x)
439 sites = getCDNOperatorData().values()
441 sites = [site for site in sites if site["numHPCSlivers"]>0]
443 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
444 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
445 average_cpu = int(avg( [site["load"] for site in sites] ))
447 result= {"total_slivers": total_slivers,
448 "total_bandwidth": total_bandwidth,
449 "average_cpu": average_cpu}
451 return HttpResponse(json.dumps(result), mimetype='application/javascript')
453 class DashboardAddOrRemoveSliverView(View):
454 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
456 def post(self, request, *args, **kwargs):
457 siteName = request.POST.get("site", None)
458 actionToDo = request.POST.get("actionToDo", "0")
460 siteList = [Site.objects.get(name=siteName)]
461 slice = Slice.objects.get(name="HyperCache")
463 if (actionToDo == "add"):
464 user_ip = request.GET.get("ip", get_ip(request))
465 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
466 elif (actionToDo == "rem"):
467 slice_decrease_slivers(request.user, siteList, slice, 1)
470 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
471 return HttpResponse('This is POST request ')
473 class DashboardAjaxView(View):
474 def get(self, request, **kwargs):
475 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
477 class DashboardAnalyticsAjaxView(View):
478 def get(self, request, name="hello_world", **kwargs):
479 if (name == "hpcSummary"):
480 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
481 elif (name == "hpcUserSite"):
482 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
483 elif (name == "hpcMap"):
484 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
485 elif (name == "bigquery"):
486 (mimetype, data) = DoPlanetStackAnalytics(request)
487 return HttpResponse(data, mimetype=mimetype)
489 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')