6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData(wait=False);
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
66 network = request.POST.get("network","0")
67 mountDataSets = request.POST.get("mountDataSets","0")
68 if (actionToDo == "add"):
69 serviceClass = ServiceClass.objects.get(name=serviceClass)
70 site = request.user.site
71 image = Image.objects.get(name=imageName)
72 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
74 return HttpResponse("Slice created")
76 class TenantUpdateSlice(View):
77 def post(self, request, *args, **kwargs):
\r
78 sliceName = request.POST.get("sliceName", "0")
\r
79 serviceClass = request.POST.get("serviceClass", "0")
\r
80 imageName = request.POST.get("imageName", "0")
\r
81 actionToDo = request.POST.get("actionToDo", "0")
\r
82 network = request.POST.get("network","0")
\r
83 dataSet = request.POST.get("dataSet","0")
\r
84 slice = Slice.objects.all()
\r
86 serviceClass = ServiceClass.objects.get(name=serviceClass)
\r
87 if(entry.name==sliceName):
\r
88 if (actionToDo == "update"):
\r
89 setattr(entry,'serviceClass',serviceClass)
\r
90 setattr(entry,'imagePreference',imageName)
\r
91 setattr(entry,'network',network)
\r
92 setattr(entry,'mountDataSets',dataSet)
\r
95 return HttpResponse("Slice updated")
\r
97 def update_slice(sliceName,**fields):
98 slice = Slice.objects.filter(name = sliceName)
\r
99 for (k,v) in fields.items():
\r
100 setattr(slice, k, v)
\r
104 def getTenantSliceInfo(user, tableFormat = False):
105 tenantSliceDetails = {}
106 tenantSliceData = getTenantInfo(user)
107 tenantServiceClassData = getServiceClassInfo(user)
109 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
110 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
112 tenantSliceDetails['userSliceInfo'] = tenantSliceData
113 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
114 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
115 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
116 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
117 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
118 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
119 return tenantSliceDetails
122 def getTenantInfo(user):
123 slices =Slice.objects.all()
126 sliceName = Slice.objects.get(id=entry.id).name
127 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
128 sliceServiceClass = entry.serviceClass.name
129 preferredImage = entry.imagePreference
130 sliceDataSet = entry.mountDataSets
131 sliceNetwork = entry.network
137 for sliver in slice.slivers.all():
138 if sliver.node.site.name in BLESSED_SITES:
139 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
140 sliceImage = sliver.image.name
141 sliceNode[str(sliver)] = sliver.name
142 numSliver = sum(sliceSite.values())
143 numSites = len(sliceSite)
144 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
147 def getTenantSitesInfo():
149 for entry in Site.objects.all():
150 if entry.name in BLESSED_SITES:
151 tenantSiteInfo.append({'siteName':entry.name})
152 return tenantSiteInfo
154 def userSliceTableFormatter(data):
161 def getServiceClassInfo(user):
162 serviceClassList = ServiceClass.objects.all()
164 for entry in serviceClassList:
165 sliceInfo.append({'serviceClass':entry.name})
168 def getImageInfo(user):
169 imageList = Image.objects.all()
170 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
172 for imageEntry in imageList:
173 imageInfo.append({'Image':imageEntry.name})
174 #imageInfo.append({'Image':imageEntry})
177 def getMountDataSets():
178 dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
180 for entry in dataSetList:
\r
181 dataSetInfo.append({'DataSet':entry})
\r
184 def getNetworkInfo(user):
185 #networkList = Network.objects.all()
186 networkList = ['Private Only','Private and Publicly Routable']
188 for networkEntry in networkList:
189 #networkInfo.append({'Network':networkEntry.name})
190 networkInfo.append({'Network':networkEntry})
193 def getDeploymentSites():
194 deploymentList = Deployment.objects.all()
196 for entry in deploymentList:
197 deploymentInfo.append({'DeploymentSite':entry.name})
198 return deploymentInfo
200 def getSliceInfo(user):
201 sliceList = Slice.objects.all()
202 slicePrivs = SlicePrivilege.objects.filter(user=user)
204 for entry in slicePrivs:
206 slicename = Slice.objects.get(id=entry.slice.id).name
207 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
208 sliverList=Sliver.objects.all()
210 for sliver in slice.slivers.all():
211 #sites_used['deploymentSites'] = sliver.node.deployment.name
212 # sites_used[sliver.image.name] = sliver.image.name
213 sites_used[sliver.node.site.name] = sliver.numberCores
214 sliceid = Slice.objects.get(id=entry.slice.id).id
216 sliverList = Sliver.objects.filter(slice=entry.slice.id)
219 if x.node.site not in siteList:
220 siteList[x.node.site] = 1
221 slivercount = len(sliverList)
222 sitecount = len(siteList)
224 traceback.print_exc()
228 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
229 'sitesUsed':sites_used,
230 'role': SliceRole.objects.get(id=entry.role.id).role,
231 'slivercount': slivercount,
232 'sitecount':sitecount})
236 def getCDNOperatorData(randomizeData = False, wait=True):
237 HPC_SLICE_NAME = "HyperCache"
239 bq = PlanetStackAnalytics()
241 rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
243 # wait=False on the first time the Dashboard is opened. This means we might
244 # not have any rows yet. The dashboard code polls every 30 seconds, so it
245 # will eventually pick them up.
248 rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
250 # dictionaryize the statistics rows by site name
253 stats_rows[row["site"]] = row
257 slice = Slice.objects.get(name=HPC_SLICE_NAME)
258 slice_slivers = list(slice.slivers.all())
261 for site in Site.objects.all():
262 # compute number of slivers allocated in the data model
263 allocated_slivers = 0
264 for sliver in slice_slivers:
265 if sliver.node.site == site:
266 allocated_slivers = allocated_slivers + 1
268 stats_row = stats_rows.get(site.name,{})
270 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
271 cpu=float(max_cpu)/100.0
272 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
274 # format it to what that CDN Operations View is expecting
275 new_row = {"lat": float(site.location.longitude),
276 "long": float(site.location.longitude),
277 "lat": float(site.location.latitude),
279 "numNodes": int(site.nodes.count()),
280 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
281 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
282 "siteUrl": str(site.site_url),
283 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
285 "hot": float(hotness)}
286 new_rows[str(site.name)] = new_row
288 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
289 for (k,v) in new_rows.items():
291 if v["numHPCSlivers"]==0:
292 for v2 in new_rows.values():
293 if (v!=v2) and (v2["numHPCSlivers"]>=0):
294 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
302 class SimulatorView(View):
303 def get(self, request, **kwargs):
304 sim = json.loads(file("/tmp/simulator.json","r").read())
305 text = "<html><head></head><body>"
306 text += "Iteration: %d<br>" % sim["iteration"]
307 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
308 text += "<table border=1>"
309 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
310 for site in sim["site_load"].values():
312 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
313 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
316 text += "</body></html>"
317 return HttpResponse(text)
319 class DashboardUserSiteView(View):
320 def get(self, request, **kwargs):
321 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
323 class TenantViewData(View):
324 def get(self, request, **kwargs):
325 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
327 def haversine(site_lat, site_lon, lat, lon):
329 if lat and lon and site_lat and site_lon:
330 site_lat = float(site_lat)
331 site_lon = float(site_lon)
335 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
336 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
341 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
342 # try to pick a site we're already using
343 has_slivers_here=False
345 for sliver in slice.slivers.all():
346 if sliver.node.site.name == site.name:
347 has_slivers_here=True
350 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
352 return (-has_slivers_here, d)
354 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
355 """ Returns list of sites, sorted from most favorable to least favorable """
359 client_geo = GeoIP().city(user_ip)
361 lat=float(client_geo["latitude"])
362 lon=float(client_geo["longitude"])
364 print "exception in geo code"
365 traceback.print_exc()
367 sites = Site.objects.all()
368 sites = [x for x in sites if x.name in BLESSED_SITES]
369 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
373 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
376 # let's compute how many slivers are in use in each node of each site
377 for site in siteList:
378 site.nodeList = list(site.nodes.all())
379 for node in site.nodeList:
381 for sliver in node.slivers.all():
382 if sliver.slice.id == slice.id:
383 node.sliverCount = node.sliverCount + 1
385 # Allocate slivers to nodes
386 # for now, assume we want to allocate all slivers from the same site
387 nodes = siteList[0].nodeList
389 # Sort the node list by number of slivers per node, then pick the
390 # node with the least number of slivers.
391 nodes = sorted(nodes, key=attrgetter("sliverCount"))
394 print "adding sliver at node", node.name, "of site", node.site.name
397 sliver = Sliver(name=node.name,
400 image = Image.objects.all()[0],
401 creator = User.objects.get(email=user),
402 deploymentNetwork=node.deployment,
406 node.sliverCount = node.sliverCount + 1
410 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
414 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
418 siteNames = [site.name for site in siteList]
422 for sliver in slice.slivers.all():
423 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
425 sliverList[sliver.name]=node.name
427 for key in sliverList:
429 sliver = Sliver.objects.filter(name=key)[0]
\r
431 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
433 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
437 class TenantDeleteSliceView(View):
438 def post(self,request):
\r
439 sliceName = request.POST.get("sliceName",None)
\r
440 slice = Slice.objects.get(name=sliceName)
\r
441 print slice, slice.id
\r
442 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
444 return HttpResponse("Slice deleted")
446 class TenantAddOrRemoveSliverView(View):
447 """ Add or remove slivers from a Slice
450 siteName - name of site. If not specified, PlanetStack will pick the
452 actionToDo - [add | rem]
453 count - number of slivers to add or remove
454 sliceName - name of slice
455 noAct - if set, no changes will be made to db, but result will still
456 show which sites would have been modified.
459 Dictionary of sites that were modified, and the count of nodes
460 that were added or removed at each site.
462 def post(self, request, *args, **kwargs):
463 siteName = request.POST.get("siteName", None)
464 actionToDo = request.POST.get("actionToDo", None)
465 count = int(request.POST.get("count","0"))
466 sliceName = request.POST.get("slice", None)
467 noAct = request.POST.get("noAct", False)
470 return HttpResponseServerError("No slice name given")
472 slice = Slice.objects.get(name=sliceName)
475 siteList = [Site.objects.get(name=siteName)]
479 if (actionToDo == "add"):
480 user_ip = request.GET.get("ip", get_ip(request))
481 if (siteList is None):
482 siteList = tenant_pick_sites(user, user_ip, slice, count)
484 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
485 elif (actionToDo == "rem"):
486 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
488 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
490 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
492 def get(self, request, *args, **kwargs):
493 request.POST = request.GET
494 return self.post(request, *args, **kwargs) # for testing REST in browser
495 #return HttpResponseServerError("GET is not supported")
497 class TenantPickSitesView(View):
498 """ primarily just for testing purposes """
499 def get(self, request, *args, **kwargs):
500 count = request.GET.get("count","0")
501 slice = request.GET.get("slice",None)
503 slice = Slice.objects.get(name=slice)
504 ip = request.GET.get("ip", get_ip(request))
505 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
506 sites = [x.name for x in sites]
507 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
509 class DashboardSummaryAjaxView(View):
510 def get(self, request, **kwargs):
512 return float(sum(x))/len(x)
514 sites = getCDNOperatorData().values()
516 sites = [site for site in sites if site["numHPCSlivers"]>0]
518 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
519 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
520 average_cpu = int(avg( [site["load"] for site in sites] ))
522 result= {"total_slivers": total_slivers,
523 "total_bandwidth": total_bandwidth,
524 "average_cpu": average_cpu}
526 return HttpResponse(json.dumps(result), mimetype='application/javascript')
528 class DashboardAddOrRemoveSliverView(View):
529 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
531 def post(self, request, *args, **kwargs):
532 siteName = request.POST.get("site", None)
533 actionToDo = request.POST.get("actionToDo", "0")
535 siteList = [Site.objects.get(name=siteName)]
536 slice = Slice.objects.get(name="HyperCache")
538 if (actionToDo == "add"):
539 user_ip = request.GET.get("ip", get_ip(request))
540 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
541 elif (actionToDo == "rem"):
542 slice_decrease_slivers(request.user, siteList, slice, 1)
545 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
546 return HttpResponse('This is POST request ')
548 class DashboardAjaxView(View):
549 def get(self, request, **kwargs):
550 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
552 class DashboardAnalyticsAjaxView(View):
553 def get(self, request, name="hello_world", **kwargs):
554 if (name == "hpcSummary"):
555 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
556 elif (name == "hpcUserSite"):
557 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
558 elif (name == "hpcMap"):
559 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
560 elif (name == "bigquery"):
561 (mimetype, data) = DoPlanetStackAnalytics(request)
562 return HttpResponse(data, mimetype=mimetype)
564 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')