6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData();
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
67 if (actionToDo == "add"):
68 serviceClass = ServiceClass.objects.get(name=serviceClass)
69 site = request.user.site
70 #image = Image.objects.get(name=imageName)
71 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
76 def getTenantSliceInfo(user, tableFormat = False):
77 tenantSliceDetails = {}
78 tenantSliceData = getTenantInfo(user)
79 tenantServiceClassData = getServiceClassInfo(user)
81 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
82 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
84 tenantSliceDetails['userSliceInfo'] = tenantSliceData
85 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
86 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
87 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
88 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
89 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo());
90 return tenantSliceDetails
93 def getTenantInfo(user):
94 slices =Slice.objects.all()
97 sliceName = Slice.objects.get(id=entry.id).name
98 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
99 sliceServiceClass = entry.serviceClass.name
100 preferredImage = entry.imagePreference
104 for sliver in slice.slivers.all():
105 numSliver +=sliver.numberCores
106 # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
107 if sliver.node.site.name in BLESSED_SITES:
108 print "equal",sliver.node.site.name
109 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
110 sliceImage = sliver.image.name
111 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
114 def getTenantSitesInfo():
116 for entry in Site.objects.all():
117 if entry.name in BLESSED_SITES:
118 tenantSiteInfo.append({'siteName':entry.name})
119 return tenantSiteInfo
121 def userSliceTableFormatter(data):
128 def getServiceClassInfo(user):
129 serviceClassList = ServiceClass.objects.all()
131 for entry in serviceClassList:
132 sliceInfo.append({'serviceClass':entry.name})
135 def getImageInfo(user):
136 imageList = Image.objects.all()
138 for imageEntry in imageList:
139 imageInfo.append({'Image':imageEntry.name})
142 def getNetworkInfo(user):
143 #networkList = Network.objects.all()
144 networkList = ['Private Only','Private and Publicly Routable']
146 for networkEntry in networkList:
147 #networkInfo.append({'Network':networkEntry.name})
148 networkInfo.append({'Network':networkEntry})
151 def getDeploymentSites():
152 deploymentList = Deployment.objects.all()
154 for entry in deploymentList:
155 deploymentInfo.append({'DeploymentSite':entry.name})
156 return deploymentInfo
158 def getSliceInfo(user):
159 sliceList = Slice.objects.all()
160 slicePrivs = SlicePrivilege.objects.filter(user=user)
162 for entry in slicePrivs:
164 slicename = Slice.objects.get(id=entry.slice.id).name
165 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
166 sliverList=Sliver.objects.all()
168 for sliver in slice.slivers.all():
169 #sites_used['deploymentSites'] = sliver.node.deployment.name
170 # sites_used[sliver.image.name] = sliver.image.name
171 sites_used[sliver.node.site.name] = sliver.numberCores
172 sliceid = Slice.objects.get(id=entry.slice.id).id
174 sliverList = Sliver.objects.filter(slice=entry.slice.id)
177 if x.node.site not in siteList:
178 siteList[x.node.site] = 1
179 slivercount = len(sliverList)
180 sitecount = len(siteList)
182 traceback.print_exc()
186 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
187 'sitesUsed':sites_used,
188 'role': SliceRole.objects.get(id=entry.role.id).role,
189 'slivercount': slivercount,
190 'sitecount':sitecount})
194 def getCDNOperatorData(randomizeData = False):
195 bq = PlanetStackAnalytics()
197 #hpc_sliceNames = bq.service_to_sliceNames("HPC Service")
199 rows = bq.get_cached_query_results(bq.compose_latest_query())
201 #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames]
203 rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"])
205 bq.merge_datamodel_sites(rows)
209 new_row = {"lat": float(row.get("lat", 0)),
210 "long": float(row.get("long", 0)),
212 "numNodes": int(row.get("numNodes",0)),
213 "numHPCSlivers": int(row.get("count_hostname", 0)),
214 "siteUrl": str(row.get("url", "")),
215 "hot": float(row.get("hotness", 0.0)),
216 "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0),
217 "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))}
218 new_rows[str(row["site"])] = new_row
222 class SimulatorView(View):
223 def get(self, request, **kwargs):
224 sim = json.loads(file("/tmp/simulator.json","r").read())
225 text = "<html><head></head><body>"
226 text += "Iteration: %d<br>" % sim["iteration"]
227 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
228 text += "<table border=1>"
229 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
230 for site in sim["site_load"].values():
232 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
233 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
236 text += "</body></html>"
237 return HttpResponse(text)
239 class DashboardUserSiteView(View):
240 def get(self, request, **kwargs):
241 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
243 class TenantViewData(View):
244 def get(self, request, **kwargs):
245 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
247 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
248 # try to pick a site we're already using
249 has_slivers_here=False
251 for sliver in slice.slivers.all():
252 if sliver.node.site.name == site.name:
253 has_slivers_here=True
257 site_lat = site.location.latitude
258 site_lon = site.location.longitude
259 if lat and lon and site_lat and site_lon:
260 site_lat = float(site_lat)
261 site_lon = float(site_lon)
263 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
264 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
267 return (-has_slivers_here, d)
269 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
270 """ Returns list of sites, sorted from most favorable to least favorable """
274 client_geo = GeoIP().city(user_ip)
276 lat=float(client_geo["latitude"])
277 lon=float(client_geo["longitude"])
279 print "exception in geo code"
280 traceback.print_exc()
282 sites = Site.objects.all()
283 sites = [x for x in sites if x.name in BLESSED_SITES]
284 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
288 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
291 # let's compute how many slivers are in use in each node of each site
292 for site in siteList:
293 site.nodeList = list(site.nodes.all())
294 for node in site.nodeList:
296 for sliver in node.slivers.all():
297 if sliver.slice.id == slice.id:
298 node.sliverCount = node.sliverCount + 1
300 # Allocate slivers to nodes
301 # for now, assume we want to allocate all slivers from the same site
302 nodes = siteList[0].nodeList
304 # Sort the node list by number of slivers per node, then pick the
305 # node with the least number of slivers.
306 nodes = sorted(nodes, key=attrgetter("sliverCount"))
309 print "adding sliver at node", node.name, "of site", node.site.name
312 sliver = Sliver(name=node.name,
315 image = Image.objects.all()[0],
316 creator = User.objects.get(email=user),
317 deploymentNetwork=node.deployment,
321 node.sliverCount = node.sliverCount + 1
325 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
329 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
333 siteNames = [site.name for site in siteList]
337 for sliver in slice.slivers.all():
342 if (not siteNames) or (node.site.name in siteNames):
343 print "deleting sliver", sliver, "at node", node.name, "of site", node.site.name
348 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) - 1
352 class TenantAddOrRemoveSliverView(View):
353 """ Add or remove slivers from a Slice
356 siteName - name of site. If not specified, PlanetStack will pick the
358 actionToDo - [add | rem]
359 count - number of slivers to add or remove
360 sliceName - name of slice
361 noAct - if set, no changes will be made to db, but result will still
362 show which sites would have been modified.
365 Dictionary of sites that were modified, and the count of nodes
366 that were added or removed at each site.
368 def post(self, request, *args, **kwargs):
369 siteName = request.POST.get("siteName", None)
370 actionToDo = request.POST.get("actionToDo", None)
371 count = int(request.POST.get("count","0"))
372 sliceName = request.POST.get("slice", None)
373 noAct = request.POST.get("noAct", False)
376 return HttpResponseServerError("No slice name given")
378 slice = Slice.objects.get(name=sliceName)
381 siteList = [Site.objects.get(name=siteName)]
385 if (actionToDo == "add"):
386 user_ip = request.GET.get("ip", get_ip(request))
387 if (siteList is None):
388 siteList = tenant_pick_sites(user, user_ip, slice, count)
390 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
391 elif (actionToDo == "rem"):
392 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
394 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
396 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
398 def get(self, request, *args, **kwargs):
399 request.POST = request.GET
400 return self.post(request, *args, **kwargs) # for testing REST in browser
401 #return HttpResponseServerError("GET is not supported")
403 class TenantPickSitesView(View):
404 """ primarily just for testing purposes """
405 def get(self, request, *args, **kwargs):
406 count = request.GET.get("count","0")
407 slice = request.GET.get("slice",None)
409 slice = Slice.objects.get(name=slice)
410 ip = request.GET.get("ip", get_ip(request))
411 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
412 sites = [x.name for x in sites]
413 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
415 class DashboardSummaryAjaxView(View):
416 def get(self, request, **kwargs):
418 return float(sum(x))/len(x)
420 sites = getCDNOperatorData().values()
422 sites = [site for site in sites if site["numHPCSlivers"]>0]
424 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
425 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
426 average_cpu = int(avg( [site["load"] for site in sites] ))
428 result= {"total_slivers": total_slivers,
429 "total_bandwidth": total_bandwidth,
430 "average_cpu": average_cpu}
432 return HttpResponse(json.dumps(result), mimetype='application/javascript')
434 class DashboardAddOrRemoveSliverView(View):
435 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
437 def post(self, request, *args, **kwargs):
438 siteName = request.POST.get("site", None)
439 actionToDo = request.POST.get("actionToDo", "0")
441 siteList = [Site.objects.get(name=siteName)]
442 slice = Slice.objects.get(name="HyperCache")
444 if (actionToDo == "add"):
445 user_ip = request.GET.get("ip", get_ip(request))
446 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
447 elif (actionToDo == "rem"):
448 slice_decrease_slivers(request.user, siteList, slice, 1)
451 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
452 return HttpResponse('This is POST request ')
454 class DashboardAjaxView(View):
455 def get(self, request, **kwargs):
456 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
458 class DashboardAnalyticsAjaxView(View):
459 def get(self, request, name="hello_world", **kwargs):
460 if (name == "hpcSummary"):
461 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
462 elif (name == "hpcUserSite"):
463 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
464 elif (name == "hpcMap"):
465 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
466 elif (name == "bigquery"):
467 (mimetype, data) = DoPlanetStackAnalytics(request)
468 return HttpResponse(data, mimetype=mimetype)
470 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')