6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from hpc.models import ContentProvider
12 from operator import attrgetter
13 from django.views.decorators.csrf import csrf_exempt
14 from django.http import HttpResponse, HttpResponseServerError
15 from django.core import urlresolvers
16 from django.contrib.gis.geoip import GeoIP
17 from ipware.ip import get_ip
21 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
23 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
24 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
26 sys.path.append("/opt/planetstack/hpc_wizard")
28 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
30 class DashboardWelcomeView(TemplateView):
31 template_name = 'admin/dashboard/welcome.html'
33 def get(self, request, *args, **kwargs):
34 context = self.get_context_data(**kwargs)
35 userDetails = getUserSliceInfo(request.user)
36 #context['site'] = userDetails['site']
38 context['userSliceInfo'] = userDetails['userSliceInfo']
39 context['cdnData'] = userDetails['cdnData']
40 context['cdnContentProviders'] = userDetails['cdnContentProviders']
41 return self.render_to_response(context=context)
43 def getUserSliceInfo(user, tableFormat = False):
46 userSliceData = getSliceInfo(user)
48 # pprint("******* GET USER SLICE INFO")
49 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
51 userDetails['userSliceInfo'] = userSliceData
52 userDetails['cdnData'] = getCDNOperatorData(wait=False)
53 userDetails['cdnContentProviders'] = getCDNContentProviderData()
56 class TenantCreateSlice(View):
57 def post(self, request, *args, **kwargs):
58 sliceName = request.POST.get("sliceName", "0")
59 serviceClass = request.POST.get("serviceClass", "0")
60 imageName = request.POST.get("imageName", "0")
61 actionToDo = request.POST.get("actionToDo", "0")
62 network = request.POST.get("network","0")
63 mountDataSets = request.POST.get("mountDataSets","0")
64 if (actionToDo == "add"):
65 serviceClass = ServiceClass.objects.get(name=serviceClass)
66 site = request.user.site
67 image = Image.objects.get(name=imageName)
68 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
70 return HttpResponse("Slice created")
72 class TenantUpdateSlice(View):
73 def post(self, request, *args, **kwargs):
\r
74 sliceName = request.POST.get("sliceName", "0")
\r
75 serviceClass = request.POST.get("serviceClass", "0")
\r
76 imageName = request.POST.get("imageName", "0")
\r
77 actionToDo = request.POST.get("actionToDo", "0")
\r
78 network = request.POST.get("network","0")
\r
79 dataSet = request.POST.get("dataSet","0")
\r
80 slice = Slice.objects.all()
\r
82 serviceClass = ServiceClass.objects.get(name=serviceClass)
\r
83 if(entry.name==sliceName):
\r
84 if (actionToDo == "update"):
\r
85 setattr(entry,'serviceClass',serviceClass)
\r
86 setattr(entry,'imagePreference',imageName)
\r
87 setattr(entry,'network',network)
\r
88 setattr(entry,'mountDataSets',dataSet)
\r
91 return HttpResponse("Slice updated")
\r
93 def update_slice(sliceName,**fields):
94 slice = Slice.objects.filter(name = sliceName)
\r
95 for (k,v) in fields.items():
\r
96 setattr(slice, k, v)
\r
100 def getTenantSliceInfo(user, tableFormat = False):
101 tenantSliceDetails = {}
102 tenantSliceData = getTenantInfo(user)
103 tenantServiceClassData = getServiceClassInfo(user)
105 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
106 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
108 tenantSliceDetails['userSliceInfo'] = tenantSliceData
109 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
110 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
111 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
112 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
113 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
114 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
115 return tenantSliceDetails
118 def getTenantInfo(user):
119 slices =Slice.objects.all()
122 sliceName = Slice.objects.get(id=entry.id).name
123 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
124 sliceServiceClass = entry.serviceClass.name
125 preferredImage = entry.imagePreference
126 sliceDataSet = entry.mountDataSets
127 sliceNetwork = entry.network
133 for sliver in slice.slivers.all():
134 if sliver.node.site.name in BLESSED_SITES:
135 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
136 sliceImage = sliver.image.name
137 sliceNode[str(sliver)] = sliver.node.name
138 numSliver = sum(sliceSite.values())
139 numSites = len(sliceSite)
140 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
143 def getTenantSitesInfo():
145 for entry in Site.objects.all():
146 if entry.name in BLESSED_SITES:
147 tenantSiteInfo.append({'siteName':entry.name})
148 return tenantSiteInfo
150 def userSliceTableFormatter(data):
157 def getServiceClassInfo(user):
158 serviceClassList = ServiceClass.objects.all()
160 for entry in serviceClassList:
161 sliceInfo.append({'serviceClass':entry.name})
164 def getImageInfo(user):
165 imageList = Image.objects.all()
166 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
168 for imageEntry in imageList:
169 imageInfo.append({'Image':imageEntry.name})
170 #imageInfo.append({'Image':imageEntry})
173 def getMountDataSets():
174 dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
176 for entry in dataSetList:
\r
177 dataSetInfo.append({'DataSet':entry})
\r
180 def getNetworkInfo(user):
181 #networkList = Network.objects.all()
182 networkList = ['Private Only','Private and Publicly Routable']
184 for networkEntry in networkList:
185 #networkInfo.append({'Network':networkEntry.name})
186 networkInfo.append({'Network':networkEntry})
189 def getDeploymentSites():
190 deploymentList = Deployment.objects.all()
192 for entry in deploymentList:
193 deploymentInfo.append({'DeploymentSite':entry.name})
194 return deploymentInfo
196 def getSliceInfo(user):
197 sliceList = Slice.objects.all()
198 slicePrivs = SlicePrivilege.objects.filter(user=user)
200 for entry in slicePrivs:
202 slicename = Slice.objects.get(id=entry.slice.id).name
203 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
204 sliverList=Sliver.objects.all()
206 for sliver in slice.slivers.all():
207 #sites_used['deploymentSites'] = sliver.node.deployment.name
208 # sites_used[sliver.image.name] = sliver.image.name
209 sites_used[sliver.node.site.name] = sliver.numberCores
210 sliceid = Slice.objects.get(id=entry.slice.id).id
212 sliverList = Sliver.objects.filter(slice=entry.slice.id)
215 if x.node.site not in siteList:
216 siteList[x.node.site] = 1
217 slivercount = len(sliverList)
218 sitecount = len(siteList)
220 traceback.print_exc()
224 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
225 'sitesUsed':sites_used,
226 'role': SliceRole.objects.get(id=entry.role.id).role,
227 'slivercount': slivercount,
228 'sitecount':sitecount})
232 def getCDNContentProviderData():
234 for dm_cp in ContentProvider.objects.all():
235 cp = {"name": dm_cp.name,
236 "account": dm_cp.account}
241 def getCDNOperatorData(randomizeData = False, wait=True):
242 HPC_SLICE_NAME = "HyperCache"
244 bq = PlanetStackAnalytics()
246 rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
248 # wait=False on the first time the Dashboard is opened. This means we might
249 # not have any rows yet. The dashboard code polls every 30 seconds, so it
250 # will eventually pick them up.
253 rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
255 # dictionaryize the statistics rows by site name
258 stats_rows[row["site"]] = row
262 slice = Slice.objects.get(name=HPC_SLICE_NAME)
263 slice_slivers = list(slice.slivers.all())
266 for site in Site.objects.all():
267 # compute number of slivers allocated in the data model
268 allocated_slivers = 0
269 for sliver in slice_slivers:
270 if sliver.node.site == site:
271 allocated_slivers = allocated_slivers + 1
273 stats_row = stats_rows.get(site.name,{})
275 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
276 cpu=float(max_cpu)/100.0
277 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
279 # format it to what that CDN Operations View is expecting
280 new_row = {"lat": float(site.location.longitude),
281 "long": float(site.location.longitude),
282 "lat": float(site.location.latitude),
284 "numNodes": int(site.nodes.count()),
285 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
286 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
287 "siteUrl": str(site.site_url),
288 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
290 "hot": float(hotness)}
291 new_rows[str(site.name)] = new_row
293 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
294 for (k,v) in new_rows.items():
296 if v["numHPCSlivers"]==0:
297 for v2 in new_rows.values():
298 if (v!=v2) and (v2["numHPCSlivers"]>=0):
299 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
307 class SimulatorView(View):
308 def get(self, request, **kwargs):
309 sim = json.loads(file("/tmp/simulator.json","r").read())
310 text = "<html><head></head><body>"
311 text += "Iteration: %d<br>" % sim["iteration"]
312 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
313 text += "<table border=1>"
314 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
315 for site in sim["site_load"].values():
317 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
318 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
321 text += "</body></html>"
322 return HttpResponse(text)
324 class DashboardUserSiteView(View):
325 def get(self, request, **kwargs):
326 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
328 class TenantViewData(View):
329 def get(self, request, **kwargs):
330 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
332 def haversine(site_lat, site_lon, lat, lon):
334 if lat and lon and site_lat and site_lon:
335 site_lat = float(site_lat)
336 site_lon = float(site_lon)
340 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
341 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
346 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
347 # try to pick a site we're already using
348 has_slivers_here=False
350 for sliver in slice.slivers.all():
351 if sliver.node.site.name == site.name:
352 has_slivers_here=True
355 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
357 return (-has_slivers_here, d)
359 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
360 """ Returns list of sites, sorted from most favorable to least favorable """
364 client_geo = GeoIP().city(user_ip)
366 lat=float(client_geo["latitude"])
367 lon=float(client_geo["longitude"])
369 print "exception in geo code"
370 traceback.print_exc()
372 sites = Site.objects.all()
373 sites = [x for x in sites if x.name in BLESSED_SITES]
374 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
378 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
381 # let's compute how many slivers are in use in each node of each site
382 for site in siteList:
383 site.nodeList = list(site.nodes.all())
384 for node in site.nodeList:
386 for sliver in node.slivers.all():
387 if sliver.slice.id == slice.id:
388 node.sliverCount = node.sliverCount + 1
390 # Allocate slivers to nodes
391 # for now, assume we want to allocate all slivers from the same site
392 nodes = siteList[0].nodeList
394 # Sort the node list by number of slivers per node, then pick the
395 # node with the least number of slivers.
396 nodes = sorted(nodes, key=attrgetter("sliverCount"))
399 print "adding sliver at node", node.name, "of site", node.site.name
402 sliver = Sliver(name=node.name,
405 image = Image.objects.all()[0],
406 creator = User.objects.get(email=user),
407 deploymentNetwork=node.deployment,
411 node.sliverCount = node.sliverCount + 1
415 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
419 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
423 siteNames = [site.name for site in siteList]
427 for sliver in slice.slivers.all():
428 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
430 sliverList[sliver.name]=node.name
432 for key in sliverList:
434 sliver = Sliver.objects.filter(name=key)[0]
\r
436 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
438 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
442 class TenantDeleteSliceView(View):
443 def post(self,request):
\r
444 sliceName = request.POST.get("sliceName",None)
\r
445 slice = Slice.objects.get(name=sliceName)
\r
446 print slice, slice.id
\r
447 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
449 return HttpResponse("Slice deleted")
451 class TenantAddOrRemoveSliverView(View):
452 """ Add or remove slivers from a Slice
455 siteName - name of site. If not specified, PlanetStack will pick the
457 actionToDo - [add | rem]
458 count - number of slivers to add or remove
459 sliceName - name of slice
460 noAct - if set, no changes will be made to db, but result will still
461 show which sites would have been modified.
464 Dictionary of sites that were modified, and the count of nodes
465 that were added or removed at each site.
467 def post(self, request, *args, **kwargs):
468 siteName = request.POST.get("siteName", None)
469 actionToDo = request.POST.get("actionToDo", None)
470 count = int(request.POST.get("count","0"))
471 sliceName = request.POST.get("slice", None)
472 noAct = request.POST.get("noAct", False)
475 return HttpResponseServerError("No slice name given")
477 slice = Slice.objects.get(name=sliceName)
480 siteList = [Site.objects.get(name=siteName)]
484 if (actionToDo == "add"):
485 user_ip = request.GET.get("ip", get_ip(request))
486 if (siteList is None):
487 siteList = tenant_pick_sites(user, user_ip, slice, count)
489 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
490 elif (actionToDo == "rem"):
491 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
493 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
495 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
497 def get(self, request, *args, **kwargs):
498 request.POST = request.GET
499 return self.post(request, *args, **kwargs) # for testing REST in browser
500 #return HttpResponseServerError("GET is not supported")
502 class TenantPickSitesView(View):
503 """ primarily just for testing purposes """
504 def get(self, request, *args, **kwargs):
505 count = request.GET.get("count","0")
506 slice = request.GET.get("slice",None)
508 slice = Slice.objects.get(name=slice)
509 ip = request.GET.get("ip", get_ip(request))
510 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
511 sites = [x.name for x in sites]
512 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
514 class DashboardSummaryAjaxView(View):
515 def get(self, request, **kwargs):
517 return float(sum(x))/len(x)
519 sites = getCDNOperatorData().values()
521 sites = [site for site in sites if site["numHPCSlivers"]>0]
523 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
524 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
525 average_cpu = int(avg( [site["load"] for site in sites] ))
527 result= {"total_slivers": total_slivers,
528 "total_bandwidth": total_bandwidth,
529 "average_cpu": average_cpu}
531 return HttpResponse(json.dumps(result), mimetype='application/javascript')
533 class DashboardAddOrRemoveSliverView(View):
534 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
536 def post(self, request, *args, **kwargs):
537 siteName = request.POST.get("site", None)
538 actionToDo = request.POST.get("actionToDo", "0")
540 siteList = [Site.objects.get(name=siteName)]
541 slice = Slice.objects.get(name="HyperCache")
543 if (actionToDo == "add"):
544 user_ip = request.GET.get("ip", get_ip(request))
545 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
546 elif (actionToDo == "rem"):
547 slice_decrease_slivers(request.user, siteList, slice, 1)
550 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
551 return HttpResponse('This is POST request ')
553 class DashboardAjaxView(View):
554 def get(self, request, **kwargs):
555 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
557 class DashboardAnalyticsAjaxView(View):
558 def get(self, request, name="hello_world", **kwargs):
559 if (name == "hpcSummary"):
560 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
561 elif (name == "hpcUserSite"):
562 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
563 elif (name == "hpcMap"):
564 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
565 elif (name == "bigquery"):
566 (mimetype, data) = DoPlanetStackAnalytics(request)
567 return HttpResponse(data, mimetype=mimetype)
569 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')