6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData();
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
67 if (actionToDo == "add"):
68 serviceClass = ServiceClass.objects.get(name=serviceClass)
69 site = request.user.site
70 #image = Image.objects.get(name=imageName)
71 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
76 def getTenantSliceInfo(user, tableFormat = False):
77 tenantSliceDetails = {}
78 tenantSliceData = getTenantInfo(user)
79 tenantServiceClassData = getServiceClassInfo(user)
81 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
82 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
84 tenantSliceDetails['userSliceInfo'] = tenantSliceData
85 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
86 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
87 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
88 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
89 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
90 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
91 return tenantSliceDetails
94 def getTenantInfo(user):
95 slices =Slice.objects.all()
98 sliceName = Slice.objects.get(id=entry.id).name
99 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
100 sliceServiceClass = entry.serviceClass.name
101 preferredImage = entry.imagePreference
105 for sliver in slice.slivers.all():
106 numSliver +=sliver.numberCores
107 # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
108 if sliver.node.site.name in BLESSED_SITES:
109 print "equal",sliver.node.site.name
110 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
111 sliceImage = sliver.image.name
112 numSites = len(sliceSite)
113 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
116 def getTenantSitesInfo():
118 for entry in Site.objects.all():
119 if entry.name in BLESSED_SITES:
120 tenantSiteInfo.append({'siteName':entry.name})
121 return tenantSiteInfo
123 def userSliceTableFormatter(data):
130 def getServiceClassInfo(user):
131 serviceClassList = ServiceClass.objects.all()
133 for entry in serviceClassList:
134 sliceInfo.append({'serviceClass':entry.name})
137 def getImageInfo(user):
138 #imageList = Image.objects.all()
139 imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
141 for imageEntry in imageList:
142 #imageInfo.append({'Image':imageEntry.name})
143 imageInfo.append({'Image':imageEntry})
146 def getMountDataSets():
147 dataSetList = ['GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
149 for entry in dataSetList:
\r
150 dataSetInfo.append({'DataSet':entry})
\r
153 def getNetworkInfo(user):
154 #networkList = Network.objects.all()
155 networkList = ['Private Only','Private and Publicly Routable']
157 for networkEntry in networkList:
158 #networkInfo.append({'Network':networkEntry.name})
159 networkInfo.append({'Network':networkEntry})
162 def getDeploymentSites():
163 deploymentList = Deployment.objects.all()
165 for entry in deploymentList:
166 deploymentInfo.append({'DeploymentSite':entry.name})
167 return deploymentInfo
169 def getSliceInfo(user):
170 sliceList = Slice.objects.all()
171 slicePrivs = SlicePrivilege.objects.filter(user=user)
173 for entry in slicePrivs:
175 slicename = Slice.objects.get(id=entry.slice.id).name
176 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
177 sliverList=Sliver.objects.all()
179 for sliver in slice.slivers.all():
180 #sites_used['deploymentSites'] = sliver.node.deployment.name
181 # sites_used[sliver.image.name] = sliver.image.name
182 sites_used[sliver.node.site.name] = sliver.numberCores
183 sliceid = Slice.objects.get(id=entry.slice.id).id
185 sliverList = Sliver.objects.filter(slice=entry.slice.id)
188 if x.node.site not in siteList:
189 siteList[x.node.site] = 1
190 slivercount = len(sliverList)
191 sitecount = len(siteList)
193 traceback.print_exc()
197 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
198 'sitesUsed':sites_used,
199 'role': SliceRole.objects.get(id=entry.role.id).role,
200 'slivercount': slivercount,
201 'sitecount':sitecount})
205 def getCDNOperatorData(randomizeData = False):
206 bq = PlanetStackAnalytics()
208 #hpc_sliceNames = bq.service_to_sliceNames("HPC Service")
210 rows = bq.get_cached_query_results(bq.compose_latest_query())
212 #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames]
214 rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"])
216 bq.merge_datamodel_sites(rows)
220 new_row = {"lat": float(row.get("lat", 0)),
221 "long": float(row.get("long", 0)),
223 "numNodes": int(row.get("numNodes",0)),
224 "numHPCSlivers": int(row.get("count_hostname", 0)),
225 "siteUrl": str(row.get("url", "")),
226 "hot": float(row.get("hotness", 0.0)),
227 "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0),
228 "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))}
229 new_rows[str(row["site"])] = new_row
233 class SimulatorView(View):
234 def get(self, request, **kwargs):
235 sim = json.loads(file("/tmp/simulator.json","r").read())
236 text = "<html><head></head><body>"
237 text += "Iteration: %d<br>" % sim["iteration"]
238 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
239 text += "<table border=1>"
240 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
241 for site in sim["site_load"].values():
243 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
244 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
247 text += "</body></html>"
248 return HttpResponse(text)
250 class DashboardUserSiteView(View):
251 def get(self, request, **kwargs):
252 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
254 class TenantViewData(View):
255 def get(self, request, **kwargs):
256 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
258 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
259 # try to pick a site we're already using
260 has_slivers_here=False
262 for sliver in slice.slivers.all():
263 if sliver.node.site.name == site.name:
264 has_slivers_here=True
268 site_lat = site.location.latitude
269 site_lon = site.location.longitude
270 if lat and lon and site_lat and site_lon:
271 site_lat = float(site_lat)
272 site_lon = float(site_lon)
274 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
275 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
278 return (-has_slivers_here, d)
280 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
281 """ Returns list of sites, sorted from most favorable to least favorable """
285 client_geo = GeoIP().city(user_ip)
287 lat=float(client_geo["latitude"])
288 lon=float(client_geo["longitude"])
290 print "exception in geo code"
291 traceback.print_exc()
293 sites = Site.objects.all()
294 sites = [x for x in sites if x.name in BLESSED_SITES]
295 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
299 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
302 # let's compute how many slivers are in use in each node of each site
303 for site in siteList:
304 site.nodeList = list(site.nodes.all())
305 for node in site.nodeList:
307 for sliver in node.slivers.all():
308 if sliver.slice.id == slice.id:
309 node.sliverCount = node.sliverCount + 1
311 # Allocate slivers to nodes
312 # for now, assume we want to allocate all slivers from the same site
313 nodes = siteList[0].nodeList
315 # Sort the node list by number of slivers per node, then pick the
316 # node with the least number of slivers.
317 nodes = sorted(nodes, key=attrgetter("sliverCount"))
320 print "adding sliver at node", node.name, "of site", node.site.name
323 sliver = Sliver(name=node.name,
326 image = Image.objects.all()[0],
327 creator = User.objects.get(email=user),
328 deploymentNetwork=node.deployment,
332 node.sliverCount = node.sliverCount + 1
336 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
340 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
344 siteNames = [site.name for site in siteList]
348 for sliver in slice.slivers.all():
349 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
351 sliverList[sliver.name]=node.name
353 for key in sliverList:
355 sliver = Sliver.objects.filter(name=key)
\r
357 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
359 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) - 1
\r
363 class TenantDeleteSliceView(View):
364 def post(self,request):
\r
365 sliceName = request.POST.get("sliceName",None)
\r
366 slice = Slice.objects.get(name=sliceName)
\r
367 print slice, slice.id
\r
368 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
371 class TenantAddOrRemoveSliverView(View):
372 """ Add or remove slivers from a Slice
375 siteName - name of site. If not specified, PlanetStack will pick the
377 actionToDo - [add | rem]
378 count - number of slivers to add or remove
379 sliceName - name of slice
380 noAct - if set, no changes will be made to db, but result will still
381 show which sites would have been modified.
384 Dictionary of sites that were modified, and the count of nodes
385 that were added or removed at each site.
387 def post(self, request, *args, **kwargs):
388 siteName = request.POST.get("siteName", None)
389 actionToDo = request.POST.get("actionToDo", None)
390 count = int(request.POST.get("count","0"))
391 sliceName = request.POST.get("slice", None)
392 noAct = request.POST.get("noAct", False)
395 return HttpResponseServerError("No slice name given")
397 slice = Slice.objects.get(name=sliceName)
400 siteList = [Site.objects.get(name=siteName)]
404 if (actionToDo == "add"):
405 user_ip = request.GET.get("ip", get_ip(request))
406 if (siteList is None):
407 siteList = tenant_pick_sites(user, user_ip, slice, count)
409 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
410 elif (actionToDo == "rem"):
411 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
413 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
415 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
417 def get(self, request, *args, **kwargs):
418 request.POST = request.GET
419 return self.post(request, *args, **kwargs) # for testing REST in browser
420 #return HttpResponseServerError("GET is not supported")
422 class TenantPickSitesView(View):
423 """ primarily just for testing purposes """
424 def get(self, request, *args, **kwargs):
425 count = request.GET.get("count","0")
426 slice = request.GET.get("slice",None)
428 slice = Slice.objects.get(name=slice)
429 ip = request.GET.get("ip", get_ip(request))
430 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
431 sites = [x.name for x in sites]
432 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
434 class DashboardSummaryAjaxView(View):
435 def get(self, request, **kwargs):
437 return float(sum(x))/len(x)
439 sites = getCDNOperatorData().values()
441 sites = [site for site in sites if site["numHPCSlivers"]>0]
443 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
444 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
445 average_cpu = int(avg( [site["load"] for site in sites] ))
447 result= {"total_slivers": total_slivers,
448 "total_bandwidth": total_bandwidth,
449 "average_cpu": average_cpu}
451 return HttpResponse(json.dumps(result), mimetype='application/javascript')
453 class DashboardAddOrRemoveSliverView(View):
454 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
456 def post(self, request, *args, **kwargs):
457 siteName = request.POST.get("site", None)
458 actionToDo = request.POST.get("actionToDo", "0")
460 siteList = [Site.objects.get(name=siteName)]
461 slice = Slice.objects.get(name="HyperCache")
463 if (actionToDo == "add"):
464 user_ip = request.GET.get("ip", get_ip(request))
465 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
466 elif (actionToDo == "rem"):
467 slice_decrease_slivers(request.user, siteList, slice, 1)
470 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
471 return HttpResponse('This is POST request ')
473 class DashboardAjaxView(View):
474 def get(self, request, **kwargs):
475 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
477 class DashboardAnalyticsAjaxView(View):
478 def get(self, request, name="hello_world", **kwargs):
479 if (name == "hpcSummary"):
480 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
481 elif (name == "hpcUserSite"):
482 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
483 elif (name == "hpcMap"):
484 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
485 elif (name == "bigquery"):
486 (mimetype, data) = DoPlanetStackAnalytics(request)
487 return HttpResponse(data, mimetype=mimetype)
489 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')