6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 sys.path.append("/opt/planetstack/hpc_wizard")
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics
29 class DashboardWelcomeView(TemplateView):
30 template_name = 'admin/dashboard/welcome.html'
32 def get(self, request, *args, **kwargs):
33 context = self.get_context_data(**kwargs)
34 userDetails = getUserSliceInfo(request.user)
35 #context['site'] = userDetails['site']
37 context['userSliceInfo'] = userDetails['userSliceInfo']
38 context['cdnData'] = userDetails['cdnData']
39 return self.render_to_response(context=context)
41 def getUserSliceInfo(user, tableFormat = False):
44 # // site = Site.objects.filter(id=user.site.id)
46 # // site = Site.objects.filter(name="Princeton")
47 # // userDetails['sitename'] = site[0].name
48 # // userDetails['siteid'] = site[0].id
50 userSliceData = getSliceInfo(user)
52 # pprint("******* GET USER SLICE INFO")
53 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
55 userDetails['userSliceInfo'] = userSliceData
56 userDetails['cdnData'] = getCDNOperatorData();
57 # pprint( userDetails)
60 class TenantCreateSlice(View):
61 def post(self, request, *args, **kwargs):
62 sliceName = request.POST.get("sliceName", "0")
63 serviceClass = request.POST.get("serviceClass", "0")
64 imageName = request.POST.get("imageName", "0")
65 actionToDo = request.POST.get("actionToDo", "0")
67 if (actionToDo == "add"):
68 serviceClass = ServiceClass.objects.get(name=serviceClass)
69 site = request.user.site
70 #image = Image.objects.get(name=imageName)
71 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName)
76 def getTenantSliceInfo(user, tableFormat = False):
77 tenantSliceDetails = {}
78 tenantSliceData = getTenantInfo(user)
79 tenantServiceClassData = getServiceClassInfo(user)
81 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
82 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
84 tenantSliceDetails['userSliceInfo'] = tenantSliceData
85 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
86 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
87 tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
88 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
89 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
90 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
91 return tenantSliceDetails
94 def getTenantInfo(user):
95 slices =Slice.objects.all()
98 sliceName = Slice.objects.get(id=entry.id).name
99 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
100 sliceServiceClass = entry.serviceClass.name
101 preferredImage = entry.imagePreference
105 for sliver in slice.slivers.all():
106 numSliver +=sliver.numberCores
107 # sliceSite[sliver.deploymentNetwork.name] =sliceSite.get(sliver.deploymentNetwork.name,0) + 1
108 if sliver.node.site.name in BLESSED_SITES:
109 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
110 sliceImage = sliver.image.name
111 numSites = len(sliceSite)
112 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver})
115 def getTenantSitesInfo():
117 for entry in Site.objects.all():
118 if entry.name in BLESSED_SITES:
119 tenantSiteInfo.append({'siteName':entry.name})
120 return tenantSiteInfo
122 def userSliceTableFormatter(data):
129 def getServiceClassInfo(user):
130 serviceClassList = ServiceClass.objects.all()
132 for entry in serviceClassList:
133 sliceInfo.append({'serviceClass':entry.name})
136 def getImageInfo(user):
137 #imageList = Image.objects.all()
138 imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
140 for imageEntry in imageList:
141 #imageInfo.append({'Image':imageEntry.name})
142 imageInfo.append({'Image':imageEntry})
145 def getMountDataSets():
146 dataSetList = ['GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']
\r
148 for entry in dataSetList:
\r
149 dataSetInfo.append({'DataSet':entry})
\r
152 def getNetworkInfo(user):
153 #networkList = Network.objects.all()
154 networkList = ['Private Only','Private and Publicly Routable']
156 for networkEntry in networkList:
157 #networkInfo.append({'Network':networkEntry.name})
158 networkInfo.append({'Network':networkEntry})
161 def getDeploymentSites():
162 deploymentList = Deployment.objects.all()
164 for entry in deploymentList:
165 deploymentInfo.append({'DeploymentSite':entry.name})
166 return deploymentInfo
168 def getSliceInfo(user):
169 sliceList = Slice.objects.all()
170 slicePrivs = SlicePrivilege.objects.filter(user=user)
172 for entry in slicePrivs:
174 slicename = Slice.objects.get(id=entry.slice.id).name
175 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
176 sliverList=Sliver.objects.all()
178 for sliver in slice.slivers.all():
179 #sites_used['deploymentSites'] = sliver.node.deployment.name
180 # sites_used[sliver.image.name] = sliver.image.name
181 sites_used[sliver.node.site.name] = sliver.numberCores
182 sliceid = Slice.objects.get(id=entry.slice.id).id
184 sliverList = Sliver.objects.filter(slice=entry.slice.id)
187 if x.node.site not in siteList:
188 siteList[x.node.site] = 1
189 slivercount = len(sliverList)
190 sitecount = len(siteList)
192 traceback.print_exc()
196 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
197 'sitesUsed':sites_used,
198 'role': SliceRole.objects.get(id=entry.role.id).role,
199 'slivercount': slivercount,
200 'sitecount':sitecount})
204 def getCDNOperatorData(randomizeData = False):
205 bq = PlanetStackAnalytics()
207 #hpc_sliceNames = bq.service_to_sliceNames("HPC Service")
209 rows = bq.get_cached_query_results(bq.compose_latest_query())
211 #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames]
213 rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
215 bq.merge_datamodel_sites(rows)
219 new_row = {"lat": float(row.get("lat", 0)),
220 "long": float(row.get("long", 0)),
222 "numNodes": int(row.get("numNodes",0)),
223 "numHPCSlivers": int(row.get("count_hostname", 0)),
224 "siteUrl": str(row.get("url", "")),
225 "hot": float(row.get("hotness", 0.0)),
226 "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0),
227 "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))}
228 new_rows[str(row["site"])] = new_row
232 class SimulatorView(View):
233 def get(self, request, **kwargs):
234 sim = json.loads(file("/tmp/simulator.json","r").read())
235 text = "<html><head></head><body>"
236 text += "Iteration: %d<br>" % sim["iteration"]
237 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
238 text += "<table border=1>"
239 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
240 for site in sim["site_load"].values():
242 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
243 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
246 text += "</body></html>"
247 return HttpResponse(text)
249 class DashboardUserSiteView(View):
250 def get(self, request, **kwargs):
251 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
253 class TenantViewData(View):
254 def get(self, request, **kwargs):
255 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
257 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
258 # try to pick a site we're already using
259 has_slivers_here=False
261 for sliver in slice.slivers.all():
262 if sliver.node.site.name == site.name:
263 has_slivers_here=True
267 site_lat = site.location.latitude
268 site_lon = site.location.longitude
269 if lat and lon and site_lat and site_lon:
270 site_lat = float(site_lat)
271 site_lon = float(site_lon)
273 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
274 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
277 return (-has_slivers_here, d)
279 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
280 """ Returns list of sites, sorted from most favorable to least favorable """
284 client_geo = GeoIP().city(user_ip)
286 lat=float(client_geo["latitude"])
287 lon=float(client_geo["longitude"])
289 print "exception in geo code"
290 traceback.print_exc()
292 sites = Site.objects.all()
293 sites = [x for x in sites if x.name in BLESSED_SITES]
294 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
298 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
301 # let's compute how many slivers are in use in each node of each site
302 for site in siteList:
303 site.nodeList = list(site.nodes.all())
304 for node in site.nodeList:
306 for sliver in node.slivers.all():
307 if sliver.slice.id == slice.id:
308 node.sliverCount = node.sliverCount + 1
310 # Allocate slivers to nodes
311 # for now, assume we want to allocate all slivers from the same site
312 nodes = siteList[0].nodeList
314 # Sort the node list by number of slivers per node, then pick the
315 # node with the least number of slivers.
316 nodes = sorted(nodes, key=attrgetter("sliverCount"))
319 print "adding sliver at node", node.name, "of site", node.site.name
322 sliver = Sliver(name=node.name,
325 image = Image.objects.all()[0],
326 creator = User.objects.get(email=user),
327 deploymentNetwork=node.deployment,
331 node.sliverCount = node.sliverCount + 1
335 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
339 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
343 siteNames = [site.name for site in siteList]
347 for sliver in slice.slivers.all():
348 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
350 sliverList[sliver.name]=node.name
352 for key in sliverList:
354 sliver = Sliver.objects.filter(name=key)[0]
\r
356 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
358 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
362 class TenantDeleteSliceView(View):
363 def post(self,request):
\r
364 sliceName = request.POST.get("sliceName",None)
\r
365 slice = Slice.objects.get(name=sliceName)
\r
366 print slice, slice.id
\r
367 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
370 class TenantAddOrRemoveSliverView(View):
371 """ Add or remove slivers from a Slice
374 siteName - name of site. If not specified, PlanetStack will pick the
376 actionToDo - [add | rem]
377 count - number of slivers to add or remove
378 sliceName - name of slice
379 noAct - if set, no changes will be made to db, but result will still
380 show which sites would have been modified.
383 Dictionary of sites that were modified, and the count of nodes
384 that were added or removed at each site.
386 def post(self, request, *args, **kwargs):
387 siteName = request.POST.get("siteName", None)
388 actionToDo = request.POST.get("actionToDo", None)
389 count = int(request.POST.get("count","0"))
390 sliceName = request.POST.get("slice", None)
391 noAct = request.POST.get("noAct", False)
394 return HttpResponseServerError("No slice name given")
396 slice = Slice.objects.get(name=sliceName)
399 siteList = [Site.objects.get(name=siteName)]
403 if (actionToDo == "add"):
404 user_ip = request.GET.get("ip", get_ip(request))
405 if (siteList is None):
406 siteList = tenant_pick_sites(user, user_ip, slice, count)
408 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
409 elif (actionToDo == "rem"):
410 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
412 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
414 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
416 def get(self, request, *args, **kwargs):
417 request.POST = request.GET
418 return self.post(request, *args, **kwargs) # for testing REST in browser
419 #return HttpResponseServerError("GET is not supported")
421 class TenantPickSitesView(View):
422 """ primarily just for testing purposes """
423 def get(self, request, *args, **kwargs):
424 count = request.GET.get("count","0")
425 slice = request.GET.get("slice",None)
427 slice = Slice.objects.get(name=slice)
428 ip = request.GET.get("ip", get_ip(request))
429 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
430 sites = [x.name for x in sites]
431 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
433 class DashboardSummaryAjaxView(View):
434 def get(self, request, **kwargs):
436 return float(sum(x))/len(x)
438 sites = getCDNOperatorData().values()
440 sites = [site for site in sites if site["numHPCSlivers"]>0]
442 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
443 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
444 average_cpu = int(avg( [site["load"] for site in sites] ))
446 result= {"total_slivers": total_slivers,
447 "total_bandwidth": total_bandwidth,
448 "average_cpu": average_cpu}
450 return HttpResponse(json.dumps(result), mimetype='application/javascript')
452 class DashboardAddOrRemoveSliverView(View):
453 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
455 def post(self, request, *args, **kwargs):
456 siteName = request.POST.get("site", None)
457 actionToDo = request.POST.get("actionToDo", "0")
459 siteList = [Site.objects.get(name=siteName)]
460 slice = Slice.objects.get(name="HyperCache")
462 if (actionToDo == "add"):
463 user_ip = request.GET.get("ip", get_ip(request))
464 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
465 elif (actionToDo == "rem"):
466 slice_decrease_slivers(request.user, siteList, slice, 1)
469 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
470 return HttpResponse('This is POST request ')
472 class DashboardAjaxView(View):
473 def get(self, request, **kwargs):
474 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
476 class DashboardAnalyticsAjaxView(View):
477 def get(self, request, name="hello_world", **kwargs):
478 if (name == "hpcSummary"):
479 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
480 elif (name == "hpcUserSite"):
481 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
482 elif (name == "hpcMap"):
483 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
484 elif (name == "bigquery"):
485 (mimetype, data) = DoPlanetStackAnalytics(request)
486 return HttpResponse(data, mimetype=mimetype)
488 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')