6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from hpc.models import ContentProvider
12 from operator import attrgetter
13 from django.views.decorators.csrf import csrf_exempt
14 from django.http import HttpResponse, HttpResponseServerError
15 from django.core import urlresolvers
16 from django.contrib.gis.geoip import GeoIP
17 from ipware.ip import get_ip
21 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
23 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
24 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
26 sys.path.append("/opt/planetstack/hpc_wizard")
28 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
30 class DashboardWelcomeView(TemplateView):
31 template_name = 'admin/dashboard/welcome.html'
33 def get(self, request, *args, **kwargs):
34 context = self.get_context_data(**kwargs)
35 userDetails = getUserSliceInfo(request.user)
36 #context['site'] = userDetails['site']
38 context['userSliceInfo'] = userDetails['userSliceInfo']
39 context['cdnData'] = userDetails['cdnData']
40 context['cdnContentProviders'] = userDetails['cdnContentProviders']
41 return self.render_to_response(context=context)
43 def getUserSliceInfo(user, tableFormat = False):
46 userSliceData = getSliceInfo(user)
48 # pprint("******* GET USER SLICE INFO")
49 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
51 userDetails['userSliceInfo'] = userSliceData
52 userDetails['cdnData'] = getCDNOperatorData(wait=False)
53 userDetails['cdnContentProviders'] = getCDNContentProviderData()
56 class TenantCreateSlice(View):
57 def post(self, request, *args, **kwargs):
58 sliceName = request.POST.get("sliceName", "0")
59 serviceClass = request.POST.get("serviceClass", "0")
60 imageName = request.POST.get("imageName", "0")
61 actionToDo = request.POST.get("actionToDo", "0")
62 network = request.POST.get("network","0")
63 mountDataSets = request.POST.get("mountDataSets","0")
64 if (actionToDo == "add"):
65 serviceClass = ServiceClass.objects.get(name=serviceClass)
66 site = request.user.site
67 image = Image.objects.get(name=imageName)
68 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
70 return HttpResponse("Slice created")
72 class TenantUpdateSlice(View):
73 def post(self, request, *args, **kwargs):
\r
74 sliceName = request.POST.get("sliceName", "0")
\r
75 serviceClass = request.POST.get("serviceClass", "0")
\r
76 imageName = request.POST.get("imageName", "0")
\r
77 actionToDo = request.POST.get("actionToDo", "0")
\r
78 network = request.POST.get("network","0")
\r
79 dataSet = request.POST.get("dataSet","0")
\r
80 slice = Slice.objects.all()
\r
82 serviceClass = ServiceClass.objects.get(name=serviceClass)
\r
83 if(entry.name==sliceName):
\r
84 if (actionToDo == "update"):
\r
85 setattr(entry,'serviceClass',serviceClass)
\r
86 setattr(entry,'imagePreference',imageName)
\r
87 setattr(entry,'network',network)
\r
88 setattr(entry,'mountDataSets',dataSet)
\r
91 return HttpResponse("Slice updated")
\r
93 def update_slice(sliceName,**fields):
94 slice = Slice.objects.filter(name = sliceName)
\r
95 for (k,v) in fields.items():
\r
96 setattr(slice, k, v)
\r
100 def getTenantSliceInfo(user, tableFormat = False):
101 tenantSliceDetails = {}
102 tenantSliceData = getTenantInfo(user)
103 tenantServiceClassData = getServiceClassInfo(user)
105 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
106 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
108 tenantSliceDetails['userSliceInfo'] = tenantSliceData
109 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
110 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
111 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
112 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
113 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
114 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
115 return tenantSliceDetails
118 def getTenantInfo(user):
119 slices =Slice.objects.all()
122 sliceName = Slice.objects.get(id=entry.id).name
123 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
124 sliceServiceClass = entry.serviceClass.name
125 preferredImage = entry.imagePreference
126 sliceDataSet = entry.mountDataSets
127 sliceNetwork = entry.network
133 for sliver in slice.slivers.all():
134 if sliver.node.site.name in BLESSED_SITES:
135 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
136 sliceImage = sliver.image.name
137 sliceNode[str(sliver)] = sliver.node.name
138 numSliver = sum(sliceSite.values())
139 numSites = len(sliceSite)
140 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
143 def getTenantSitesInfo():
145 for entry in Site.objects.all():
146 if entry.name in BLESSED_SITES:
147 tenantSiteInfo.append({'siteName':entry.name})
148 return tenantSiteInfo
150 def userSliceTableFormatter(data):
157 def getServiceClassInfo(user):
158 serviceClassList = ServiceClass.objects.all()
160 for entry in serviceClassList:
161 sliceInfo.append({'serviceClass':entry.name})
164 def getImageInfo(user):
165 imageList = Image.objects.all()
166 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
168 for imageEntry in imageList:
169 imageInfo.append({'Image':imageEntry.name})
170 #imageInfo.append({'Image':imageEntry})
173 def getMountDataSets():
174 dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
176 for entry in dataSetList:
\r
177 dataSetInfo.append({'DataSet':entry})
\r
180 def getNetworkInfo(user):
181 #networkList = Network.objects.all()
182 networkList = ['Private Only','Private and Publicly Routable']
184 for networkEntry in networkList:
185 #networkInfo.append({'Network':networkEntry.name})
186 networkInfo.append({'Network':networkEntry})
189 def getDeploymentSites():
190 deploymentList = Deployment.objects.all()
192 for entry in deploymentList:
193 deploymentInfo.append({'DeploymentSite':entry.name})
194 return deploymentInfo
196 def getSliceInfo(user):
197 sliceList = Slice.objects.all()
198 slicePrivs = SlicePrivilege.objects.filter(user=user)
200 for entry in slicePrivs:
202 slicename = Slice.objects.get(id=entry.slice.id).name
203 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
204 sliverList=Sliver.objects.all()
206 for sliver in slice.slivers.all():
207 #sites_used['deploymentSites'] = sliver.node.deployment.name
208 # sites_used[sliver.image.name] = sliver.image.name
209 sites_used[sliver.node.site.name] = sliver.numberCores
210 sliceid = Slice.objects.get(id=entry.slice.id).id
212 sliverList = Sliver.objects.filter(slice=entry.slice.id)
215 if x.node.site not in siteList:
216 siteList[x.node.site] = 1
217 slivercount = len(sliverList)
218 sitecount = len(siteList)
220 traceback.print_exc()
224 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
225 'sitesUsed':sites_used,
226 'role': SliceRole.objects.get(id=entry.role.id).role,
227 'slivercount': slivercount,
228 'sitecount':sitecount})
232 def getCDNContentProviderData():
234 for dm_cp in ContentProvider.objects.all():
235 cp = {"name": dm_cp.name,
236 "account": dm_cp.account}
241 def getCDNOperatorData(randomizeData = False, wait=True):
242 HPC_SLICE_NAME = "HyperCache"
244 bq = PlanetStackAnalytics()
246 rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait) # why did we need %slice ??
248 # wait=False on the first time the Dashboard is opened. This means we might
249 # not have any rows yet. The dashboard code polls every 30 seconds, so it
250 # will eventually pick them up.
253 rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
255 # dictionaryize the statistics rows by site name
258 stats_rows[row["site"]] = row
262 slice = Slice.objects.filter(name=HPC_SLICE_NAME)
264 slice_slivers = list(slice[0].slivers.all())
269 for site in Site.objects.all():
270 # compute number of slivers allocated in the data model
271 allocated_slivers = 0
272 for sliver in slice_slivers:
273 if sliver.node.site == site:
274 allocated_slivers = allocated_slivers + 1
276 stats_row = stats_rows.get(site.name,{})
278 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
279 cpu=float(max_cpu)/100.0
280 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
282 # format it to what that CDN Operations View is expecting
283 new_row = {"lat": float(site.location.longitude),
284 "long": float(site.location.longitude),
285 "lat": float(site.location.latitude),
287 "numNodes": int(site.nodes.count()),
288 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
289 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
290 "siteUrl": str(site.site_url),
291 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
293 "hot": float(hotness)}
294 new_rows[str(site.name)] = new_row
296 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
297 for (k,v) in new_rows.items():
299 if v["numHPCSlivers"]==0:
300 for v2 in new_rows.values():
301 if (v!=v2) and (v2["numHPCSlivers"]>=0):
302 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
310 class SimulatorView(View):
311 def get(self, request, **kwargs):
312 sim = json.loads(file("/tmp/simulator.json","r").read())
313 text = "<html><head></head><body>"
314 text += "Iteration: %d<br>" % sim["iteration"]
315 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
316 text += "<table border=1>"
317 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
318 for site in sim["site_load"].values():
320 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
321 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
324 text += "</body></html>"
325 return HttpResponse(text)
327 class DashboardUserSiteView(View):
328 def get(self, request, **kwargs):
329 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
331 class TenantViewData(View):
332 def get(self, request, **kwargs):
333 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
335 def haversine(site_lat, site_lon, lat, lon):
337 if lat and lon and site_lat and site_lon:
338 site_lat = float(site_lat)
339 site_lon = float(site_lon)
343 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
344 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
349 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
350 # try to pick a site we're already using
351 has_slivers_here=False
353 for sliver in slice.slivers.all():
354 if sliver.node.site.name == site.name:
355 has_slivers_here=True
358 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
360 return (-has_slivers_here, d)
362 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
363 """ Returns list of sites, sorted from most favorable to least favorable """
367 client_geo = GeoIP().city(user_ip)
369 lat=float(client_geo["latitude"])
370 lon=float(client_geo["longitude"])
372 print "exception in geo code"
373 traceback.print_exc()
375 sites = Site.objects.all()
376 sites = [x for x in sites if x.name in BLESSED_SITES]
377 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
381 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
384 # let's compute how many slivers are in use in each node of each site
385 for site in siteList:
386 site.nodeList = list(site.nodes.all())
387 for node in site.nodeList:
389 for sliver in node.slivers.all():
390 if sliver.slice.id == slice.id:
391 node.sliverCount = node.sliverCount + 1
393 # Allocate slivers to nodes
394 # for now, assume we want to allocate all slivers from the same site
395 nodes = siteList[0].nodeList
397 # Sort the node list by number of slivers per node, then pick the
398 # node with the least number of slivers.
399 nodes = sorted(nodes, key=attrgetter("sliverCount"))
402 print "adding sliver at node", node.name, "of site", node.site.name
405 sliver = Sliver(name=node.name,
408 image = Image.objects.all()[0],
409 creator = User.objects.get(email=user),
410 deploymentNetwork=node.deployment,
414 node.sliverCount = node.sliverCount + 1
418 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
422 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
426 siteNames = [site.name for site in siteList]
430 for sliver in slice.slivers.all():
431 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
433 sliverList[sliver.name]=node.name
435 for key in sliverList:
437 sliver = Sliver.objects.filter(name=key)[0]
\r
439 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
441 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
445 class TenantDeleteSliceView(View):
446 def post(self,request):
\r
447 sliceName = request.POST.get("sliceName",None)
\r
448 slice = Slice.objects.get(name=sliceName)
\r
449 print slice, slice.id
\r
450 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
452 return HttpResponse("Slice deleted")
454 class TenantAddOrRemoveSliverView(View):
455 """ Add or remove slivers from a Slice
458 siteName - name of site. If not specified, PlanetStack will pick the
460 actionToDo - [add | rem]
461 count - number of slivers to add or remove
462 sliceName - name of slice
463 noAct - if set, no changes will be made to db, but result will still
464 show which sites would have been modified.
467 Dictionary of sites that were modified, and the count of nodes
468 that were added or removed at each site.
470 def post(self, request, *args, **kwargs):
471 siteName = request.POST.get("siteName", None)
472 actionToDo = request.POST.get("actionToDo", None)
473 count = int(request.POST.get("count","0"))
474 sliceName = request.POST.get("slice", None)
475 noAct = request.POST.get("noAct", False)
478 return HttpResponseServerError("No slice name given")
480 slice = Slice.objects.get(name=sliceName)
483 siteList = [Site.objects.get(name=siteName)]
487 if (actionToDo == "add"):
488 user_ip = request.GET.get("ip", get_ip(request))
489 if (siteList is None):
490 siteList = tenant_pick_sites(user, user_ip, slice, count)
492 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
493 elif (actionToDo == "rem"):
494 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
496 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
498 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
500 def get(self, request, *args, **kwargs):
501 request.POST = request.GET
502 return self.post(request, *args, **kwargs) # for testing REST in browser
503 #return HttpResponseServerError("GET is not supported")
505 class TenantPickSitesView(View):
506 """ primarily just for testing purposes """
507 def get(self, request, *args, **kwargs):
508 count = request.GET.get("count","0")
509 slice = request.GET.get("slice",None)
511 slice = Slice.objects.get(name=slice)
512 ip = request.GET.get("ip", get_ip(request))
513 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
514 sites = [x.name for x in sites]
515 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
517 class DashboardSummaryAjaxView(View):
518 def get(self, request, **kwargs):
520 return float(sum(x))/len(x)
522 sites = getCDNOperatorData().values()
524 sites = [site for site in sites if site["numHPCSlivers"]>0]
526 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
527 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
528 average_cpu = int(avg( [site["load"] for site in sites] ))
530 result= {"total_slivers": total_slivers,
531 "total_bandwidth": total_bandwidth,
532 "average_cpu": average_cpu}
534 return HttpResponse(json.dumps(result), mimetype='application/javascript')
536 class DashboardAddOrRemoveSliverView(View):
537 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
539 def post(self, request, *args, **kwargs):
540 siteName = request.POST.get("site", None)
541 actionToDo = request.POST.get("actionToDo", "0")
543 siteList = [Site.objects.get(name=siteName)]
544 slice = Slice.objects.get(name="HyperCache")
546 if (actionToDo == "add"):
547 user_ip = request.GET.get("ip", get_ip(request))
548 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
549 elif (actionToDo == "rem"):
550 slice_decrease_slivers(request.user, siteList, slice, 1)
553 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
554 return HttpResponse('This is POST request ')
556 class DashboardAjaxView(View):
557 def get(self, request, **kwargs):
558 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
560 class DashboardAnalyticsAjaxView(View):
561 def get(self, request, name="hello_world", **kwargs):
562 if (name == "hpcSummary"):
563 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
564 elif (name == "hpcUserSite"):
565 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
566 elif (name == "hpcMap"):
567 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
568 elif (name == "bigquery"):
569 (mimetype, data) = DoPlanetStackAnalytics(request)
570 return HttpResponse(data, mimetype=mimetype)
572 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')