6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData(wait=False);
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
66 network = request.POST.get("network","0")
67 mountDataSets = request.POST.get("mountDataSets","0")
68 if (actionToDo == "add"):
69 serviceClass = ServiceClass.objects.get(name=serviceClass)
70 site = request.user.site
71 #image = Image.objects.get(name=imageName)
72 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName,mountDataSets=mountDataSets,network=network)
76 class TenantUpdateSlice(View):
77 def post(self, request, *args, **kwargs):
\r
78 sliceName = request.POST.get("sliceName", "0")
\r
79 serviceClass = request.POST.get("serviceClass", "0")
\r
80 imageName = request.POST.get("imageName", "0")
\r
81 actionToDo = request.POST.get("actionToDo", "0")
\r
82 network = request.POST.get("network","0")
\r
83 slice = Slice.objects.filter(name = sliceName)
\r
84 abc = ServiceClass.objects.get(name=serviceClass)
\r
85 if (actionToDo == "update"):
\r
86 # print getattr(slice,'serviceClass',abc)
\r
87 setattr(slice,'serviceClass',abc)
\r
88 #fields = {'serviceClass':ServiceClass.objects.get(name=serviceClass),
\r
89 # 'imagePreference':imageName,
\r
92 #update_slice(sliceName,**fields)
\r
93 return HttpResponse("Slice updated")
95 def update_slice(sliceName,**fields):
96 slice = Slice.objects.filter(name = sliceName)
\r
97 for (k,v) in fields.items():
\r
98 setattr(slice, k, v)
\r
102 def getTenantSliceInfo(user, tableFormat = False):
103 tenantSliceDetails = {}
104 tenantSliceData = getTenantInfo(user)
105 tenantServiceClassData = getServiceClassInfo(user)
107 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
108 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
110 tenantSliceDetails['userSliceInfo'] = tenantSliceData
111 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
112 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
113 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
114 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
115 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
116 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
117 return tenantSliceDetails
120 def getTenantInfo(user):
121 slices =Slice.objects.all()
124 sliceName = Slice.objects.get(id=entry.id).name
125 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
126 sliceServiceClass = entry.serviceClass.name
127 preferredImage = entry.imagePreference
128 sliceDataSet = entry.mountDataSets
129 sliceNetwork = entry.network
135 for sliver in slice.slivers.all():
136 if sliver.node.site.name in BLESSED_SITES:
137 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
138 sliceImage = sliver.image.name
139 sliceNode[sliver.instance_name] = sliver.name
140 numSliver = sum(sliceSite.values())
141 numSites = len(sliceSite)
142 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
145 def getTenantSitesInfo():
147 for entry in Site.objects.all():
148 if entry.name in BLESSED_SITES:
149 tenantSiteInfo.append({'siteName':entry.name})
150 return tenantSiteInfo
152 def userSliceTableFormatter(data):
159 def getServiceClassInfo(user):
160 serviceClassList = ServiceClass.objects.all()
162 for entry in serviceClassList:
163 sliceInfo.append({'serviceClass':entry.name})
166 def getImageInfo(user):
167 #imageList = Image.objects.all()
168 imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
170 for imageEntry in imageList:
171 #imageInfo.append({'Image':imageEntry.name})
172 imageInfo.append({'Image':imageEntry})
175 def getMountDataSets():
176 dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
178 for entry in dataSetList:
\r
179 dataSetInfo.append({'DataSet':entry})
\r
182 def getNetworkInfo(user):
183 #networkList = Network.objects.all()
184 networkList = ['Private Only','Private and Publicly Routable']
186 for networkEntry in networkList:
187 #networkInfo.append({'Network':networkEntry.name})
188 networkInfo.append({'Network':networkEntry})
191 def getDeploymentSites():
192 deploymentList = Deployment.objects.all()
194 for entry in deploymentList:
195 deploymentInfo.append({'DeploymentSite':entry.name})
196 return deploymentInfo
198 def getSliceInfo(user):
199 sliceList = Slice.objects.all()
200 slicePrivs = SlicePrivilege.objects.filter(user=user)
202 for entry in slicePrivs:
204 slicename = Slice.objects.get(id=entry.slice.id).name
205 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
206 sliverList=Sliver.objects.all()
208 for sliver in slice.slivers.all():
209 #sites_used['deploymentSites'] = sliver.node.deployment.name
210 # sites_used[sliver.image.name] = sliver.image.name
211 sites_used[sliver.node.site.name] = sliver.numberCores
212 sliceid = Slice.objects.get(id=entry.slice.id).id
214 sliverList = Sliver.objects.filter(slice=entry.slice.id)
217 if x.node.site not in siteList:
218 siteList[x.node.site] = 1
219 slivercount = len(sliverList)
220 sitecount = len(siteList)
222 traceback.print_exc()
226 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
227 'sitesUsed':sites_used,
228 'role': SliceRole.objects.get(id=entry.role.id).role,
229 'slivercount': slivercount,
230 'sitecount':sitecount})
234 def getCDNOperatorData(randomizeData = False, wait=True):
235 HPC_SLICE_NAME = "HyperCache"
237 bq = PlanetStackAnalytics()
239 rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
241 # wait=False on the first time the Dashboard is opened. This means we might
242 # not have any rows yet. The dashboard code polls every 30 seconds, so it
243 # will eventually pick them up.
246 rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
248 # dictionaryize the statistics rows by site name
251 stats_rows[row["site"]] = row
255 slice = Slice.objects.get(name=HPC_SLICE_NAME)
256 slice_slivers = list(slice.slivers.all())
259 for site in Site.objects.all():
260 # compute number of slivers allocated in the data model
261 allocated_slivers = 0
262 for sliver in slice_slivers:
263 if sliver.node.site == site:
264 allocated_slivers = allocated_slivers + 1
266 stats_row = stats_rows.get(site.name,{})
268 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
269 cpu=float(max_cpu)/100.0
270 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
272 # format it to what that CDN Operations View is expecting
273 new_row = {"lat": float(site.location.longitude),
274 "long": float(site.location.longitude),
275 "lat": float(site.location.latitude),
277 "numNodes": int(site.nodes.count()),
278 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
279 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
280 "siteUrl": str(site.site_url),
281 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
283 "hot": float(hotness)}
284 new_rows[str(site.name)] = new_row
286 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
287 for (k,v) in new_rows.items():
289 if v["numHPCSlivers"]==0:
290 for v2 in new_rows.values():
291 if (v!=v2) and (v2["numHPCSlivers"]>=0):
292 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
300 class SimulatorView(View):
301 def get(self, request, **kwargs):
302 sim = json.loads(file("/tmp/simulator.json","r").read())
303 text = "<html><head></head><body>"
304 text += "Iteration: %d<br>" % sim["iteration"]
305 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
306 text += "<table border=1>"
307 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
308 for site in sim["site_load"].values():
310 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
311 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
314 text += "</body></html>"
315 return HttpResponse(text)
317 class DashboardUserSiteView(View):
318 def get(self, request, **kwargs):
319 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
321 class TenantViewData(View):
322 def get(self, request, **kwargs):
323 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
325 def haversine(site_lat, site_lon, lat, lon):
326 site_lat = float(site_lat)
327 site_lon = float(site_lon)
331 if lat and lon and site_lat and site_lon:
333 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
334 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
339 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
340 # try to pick a site we're already using
341 has_slivers_here=False
343 for sliver in slice.slivers.all():
344 if sliver.node.site.name == site.name:
345 has_slivers_here=True
348 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
350 return (-has_slivers_here, d)
352 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
353 """ Returns list of sites, sorted from most favorable to least favorable """
357 client_geo = GeoIP().city(user_ip)
359 lat=float(client_geo["latitude"])
360 lon=float(client_geo["longitude"])
362 print "exception in geo code"
363 traceback.print_exc()
365 sites = Site.objects.all()
366 sites = [x for x in sites if x.name in BLESSED_SITES]
367 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
371 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
374 # let's compute how many slivers are in use in each node of each site
375 for site in siteList:
376 site.nodeList = list(site.nodes.all())
377 for node in site.nodeList:
379 for sliver in node.slivers.all():
380 if sliver.slice.id == slice.id:
381 node.sliverCount = node.sliverCount + 1
383 # Allocate slivers to nodes
384 # for now, assume we want to allocate all slivers from the same site
385 nodes = siteList[0].nodeList
387 # Sort the node list by number of slivers per node, then pick the
388 # node with the least number of slivers.
389 nodes = sorted(nodes, key=attrgetter("sliverCount"))
392 print "adding sliver at node", node.name, "of site", node.site.name
395 sliver = Sliver(name=node.name,
398 image = Image.objects.all()[0],
399 creator = User.objects.get(email=user),
400 deploymentNetwork=node.deployment,
404 node.sliverCount = node.sliverCount + 1
408 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
412 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
416 siteNames = [site.name for site in siteList]
420 for sliver in slice.slivers.all():
421 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
423 sliverList[sliver.name]=node.name
425 for key in sliverList:
427 sliver = Sliver.objects.filter(name=key)[0]
\r
429 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
431 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
435 class TenantDeleteSliceView(View):
436 def post(self,request):
\r
437 sliceName = request.POST.get("sliceName",None)
\r
438 slice = Slice.objects.get(name=sliceName)
\r
439 print slice, slice.id
\r
440 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
443 class TenantAddOrRemoveSliverView(View):
444 """ Add or remove slivers from a Slice
447 siteName - name of site. If not specified, PlanetStack will pick the
449 actionToDo - [add | rem]
450 count - number of slivers to add or remove
451 sliceName - name of slice
452 noAct - if set, no changes will be made to db, but result will still
453 show which sites would have been modified.
456 Dictionary of sites that were modified, and the count of nodes
457 that were added or removed at each site.
459 def post(self, request, *args, **kwargs):
460 siteName = request.POST.get("siteName", None)
461 actionToDo = request.POST.get("actionToDo", None)
462 count = int(request.POST.get("count","0"))
463 sliceName = request.POST.get("slice", None)
464 noAct = request.POST.get("noAct", False)
467 return HttpResponseServerError("No slice name given")
469 slice = Slice.objects.get(name=sliceName)
472 siteList = [Site.objects.get(name=siteName)]
476 if (actionToDo == "add"):
477 user_ip = request.GET.get("ip", get_ip(request))
478 if (siteList is None):
479 siteList = tenant_pick_sites(user, user_ip, slice, count)
481 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
482 elif (actionToDo == "rem"):
483 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
485 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
487 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
489 def get(self, request, *args, **kwargs):
490 request.POST = request.GET
491 return self.post(request, *args, **kwargs) # for testing REST in browser
492 #return HttpResponseServerError("GET is not supported")
494 class TenantPickSitesView(View):
495 """ primarily just for testing purposes """
496 def get(self, request, *args, **kwargs):
497 count = request.GET.get("count","0")
498 slice = request.GET.get("slice",None)
500 slice = Slice.objects.get(name=slice)
501 ip = request.GET.get("ip", get_ip(request))
502 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
503 sites = [x.name for x in sites]
504 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
506 class DashboardSummaryAjaxView(View):
507 def get(self, request, **kwargs):
509 return float(sum(x))/len(x)
511 sites = getCDNOperatorData().values()
513 sites = [site for site in sites if site["numHPCSlivers"]>0]
515 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
516 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
517 average_cpu = int(avg( [site["load"] for site in sites] ))
519 result= {"total_slivers": total_slivers,
520 "total_bandwidth": total_bandwidth,
521 "average_cpu": average_cpu}
523 return HttpResponse(json.dumps(result), mimetype='application/javascript')
525 class DashboardAddOrRemoveSliverView(View):
526 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
528 def post(self, request, *args, **kwargs):
529 siteName = request.POST.get("site", None)
530 actionToDo = request.POST.get("actionToDo", "0")
532 siteList = [Site.objects.get(name=siteName)]
533 slice = Slice.objects.get(name="HyperCache")
535 if (actionToDo == "add"):
536 user_ip = request.GET.get("ip", get_ip(request))
537 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
538 elif (actionToDo == "rem"):
539 slice_decrease_slivers(request.user, siteList, slice, 1)
542 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
543 return HttpResponse('This is POST request ')
545 class DashboardAjaxView(View):
546 def get(self, request, **kwargs):
547 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
549 class DashboardAnalyticsAjaxView(View):
550 def get(self, request, name="hello_world", **kwargs):
551 if (name == "hpcSummary"):
552 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
553 elif (name == "hpcUserSite"):
554 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
555 elif (name == "hpcMap"):
556 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
557 elif (name == "bigquery"):
558 (mimetype, data) = DoPlanetStackAnalytics(request)
559 return HttpResponse(data, mimetype=mimetype)
561 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')