6 from django.views.generic import TemplateView, View
8 from pprint import pprint
10 from syndicate.models import *
11 from core.models import *
12 from hpc.models import ContentProvider
13 from operator import attrgetter
14 from django import template
15 from django.views.decorators.csrf import csrf_exempt
16 from django.http import HttpResponse, HttpResponseServerError
17 from django.core import urlresolvers
18 from django.contrib.gis.geoip import GeoIP
19 from ipware.ip import get_ip
23 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
25 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
26 sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
28 sys.path.append("/opt/planetstack/hpc_wizard")
30 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
32 class DashboardWelcomeView(TemplateView):
33 template_name = 'admin/dashboard/welcome.html'
35 def get(self, request, *args, **kwargs):
36 context = self.get_context_data(**kwargs)
37 userDetails = getUserSliceInfo(request.user)
38 #context['site'] = userDetails['site']
40 context['userSliceInfo'] = userDetails['userSliceInfo']
41 context['cdnData'] = userDetails['cdnData']
42 context['cdnContentProviders'] = userDetails['cdnContentProviders']
43 return self.render_to_response(context=context)
45 class DashboardView(TemplateView):
46 head_template = r"""{% extends "admin/dashboard/dashboard_base.html" %}
47 {% load admin_static %}
51 tail_template = r"{% endblock %}"
53 def get(self, request, name="hpc_historical", *args, **kwargs):
54 context = self.get_context_data(**kwargs)
56 t = template.Template(self.head_template + open("/opt/planetstack/templates/admin/dashboard/%s.html" % name, "r").read() + self.tail_template)
58 userDetails = getUserSliceInfo(request.user)
59 #context['site'] = userDetails['site']
61 context['userSliceInfo'] = userDetails['userSliceInfo']
62 context['cdnData'] = userDetails['cdnData']
63 context['cdnContentProviders'] = userDetails['cdnContentProviders']
66 response_kwargs.setdefault('content_type', self.content_type)
67 return self.response_class(
\r
73 def getUserSliceInfo(user, tableFormat = False):
76 userSliceData = getSliceInfo(user)
78 # pprint("******* GET USER SLICE INFO")
79 userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
81 userDetails['userSliceInfo'] = userSliceData
82 userDetails['cdnData'] = getCDNOperatorData(wait=False)
83 userDetails['cdnContentProviders'] = getCDNContentProviderData()
86 class TenantCreateSlice(View):
87 def post(self, request, *args, **kwargs):
88 sliceName = request.POST.get("sliceName", "0")
89 serviceClass = request.POST.get("serviceClass", "0")
90 imageName = request.POST.get("imageName", "0")
91 actionToDo = request.POST.get("actionToDo", "0")
92 #network = request.POST.get("network","0")
93 mountDataSets = request.POST.get("mountDataSets","0")
94 if (actionToDo == "add"):
95 serviceClass = ServiceClass.objects.get(name=serviceClass)
96 site = request.user.site
97 image = Image.objects.get(name=imageName)
98 newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets)
100 return HttpResponse("Slice created")
102 class TenantUpdateSlice(View):
103 def post(self, request, *args, **kwargs):
\r
104 sliceName = request.POST.get("sliceName", "0")
\r
105 serviceClass = request.POST.get("serviceClass", "0")
\r
106 imageName = request.POST.get("imageName", "0")
\r
107 actionToDo = request.POST.get("actionToDo", "0")
\r
108 #network = request.POST.get("network","0")
\r
109 dataSet = request.POST.get("dataSet","0")
\r
110 slice = Slice.objects.all()
\r
111 for entry in slice:
\r
112 serviceClass = ServiceClass.objects.get(name=serviceClass)
\r
113 if(entry.name==sliceName):
\r
114 if (actionToDo == "update"):
\r
115 setattr(entry,'serviceClass',serviceClass)
\r
116 setattr(entry,'imagePreference',imageName)
\r
117 #setattr(entry,'network',network)
\r
118 setattr(entry,'mountDataSets',dataSet)
\r
121 return HttpResponse("Slice updated")
\r
123 def getTenantSliceInfo(user, tableFormat = False):
124 tenantSliceDetails = {}
125 tenantSliceData = getTenantInfo(user)
126 tenantServiceClassData = getServiceClassInfo(user)
128 tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
129 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
131 tenantSliceDetails['userSliceInfo'] = tenantSliceData
132 tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
133 tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
134 #tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
135 tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
136 tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
137 tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
138 tenantSliceDetails['publicKey'] = getPublicKey(user)
139 return tenantSliceDetails
141 def getTenantInfo(user):
142 slices =Slice.objects.all()
145 sliceName = Slice.objects.get(id=entry.id).name
146 slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
147 sliceServiceClass = entry.serviceClass.name
148 preferredImage = entry.imagePreference
149 sliceDataSet = entry.mountDataSets
150 #sliceNetwork = entry.network
156 for sliver in slice.slivers.all():
157 if sliver.node.site.name in BLESSED_SITES:
158 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
159 sliceImage = sliver.image.name
160 sliceNode[str(sliver)] = sliver.node.name
161 numSliver = sum(sliceSite.values())
162 numSites = len(sliceSite)
163 userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'instanceNodePair':sliceNode})
166 def getTenantSitesInfo():
168 for entry in Site.objects.all():
169 if entry.name in BLESSED_SITES:
170 tenantSiteInfo.append({'siteName':entry.name})
171 return tenantSiteInfo
173 def userSliceTableFormatter(data):
180 def getPublicKey(user):
181 users=User.objects.all()
\r
183 if (str(key.email)==str(user)):
\r
184 sshKey = key.public_key
\r
187 def getServiceClassInfo(user):
188 serviceClassList = ServiceClass.objects.all()
190 for entry in serviceClassList:
191 sliceInfo.append({'serviceClass':entry.name})
194 def getImageInfo(user):
195 imageList = Image.objects.all()
196 #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
198 for imageEntry in imageList:
199 imageInfo.append({'Image':imageEntry.name})
200 #imageInfo.append({'Image':imageEntry})
203 def createPrivateVolume(user, sliceName):
204 caps = Volume.CAP_READ_DATA | Volume.CAP_WRITE_DATA | Volume.CAP_HOST_DATA
205 getattr(Volume.default_gateway_caps,"read data") | \
206 getattr(Volume.default_gateway_caps,"write data") | \
207 getattr(Volume.default_gateway_caps,"host files")
208 v = Volume(name="private_" + sliceName, owner_id=user, description="private volume for %s" % sliceName, blocksize=61440, private=True, archive=False, default_gateway_caps = caps)
211 SYNDICATE_REPLICATE_PORTNUM = 1025
215 inuse[SYNDICATE_REPLICATE_PORTNUM] = True
216 for vs in VolumeSlice.objects.all():
217 inuse[vs.peer_portnum]=True
218 inuse[vs.replicate_portnum]=True
219 for network in Network.objects.all():
220 network_ports = [x.strip() for x in network.ports.split(",")]
221 for network_port in network_ports:
223 inuse[int(network_port)] = True
225 # in case someone has put a malformed port number in the list
227 for i in range(1025, 65535):
228 if not inuse.get(i,False):
232 def mountVolume(sliceName, volumeName, readWrite):
233 slice = Slice.objects.get(name=sliceName)
234 volume = Volume.objects.get(name=volumeName)
235 # choose some unused port numbers
236 flags = Volume.CAP_READ_DATA
238 flags = flags | Volume.CAP_WRITE_DATA
239 vs = VolumeSlice(volume_id = volume, slice_id = slice, gateway_caps=flags, peer_portnum = get_free_port(), replicate_portnum = SYNDICATE_REPLICATE_PORTNUM)
242 def hasPrivateVolume(sliceName):
243 slice = Slice.objects.get(name=sliceName)
244 for vs in VolumeSlice.objects.filter(slice_id=slice):
245 if vs.volume_id.private:
249 def getMountDataSets():
251 for volume in Volume.objects.all():
\r
252 if not volume.private:
\r
253 dataSetInfo.append({'DataSet': volume.name})
\r
257 def getNetworkInfo(user):
258 #networkList = Network.objects.all()
259 networkList = ['Private Only','Private and Publicly Routable']
261 for networkEntry in networkList:
262 #networkInfo.append({'Network':networkEntry.name})
263 networkInfo.append({'Network':networkEntry})
266 def getDeploymentSites():
267 deploymentList = Deployment.objects.all()
269 for entry in deploymentList:
270 deploymentInfo.append({'DeploymentSite':entry.name})
271 return deploymentInfo
273 def getSliceInfo(user):
274 sliceList = Slice.objects.all()
275 slicePrivs = SlicePrivilege.objects.filter(user=user)
277 for entry in slicePrivs:
279 slicename = Slice.objects.get(id=entry.slice.id).name
280 slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
281 sliverList=Sliver.objects.all()
283 for sliver in slice.slivers.all():
284 #sites_used['deploymentSites'] = sliver.node.deployment.name
285 # sites_used[sliver.image.name] = sliver.image.name
286 sites_used[sliver.node.site.name] = sliver.numberCores
287 sliceid = Slice.objects.get(id=entry.slice.id).id
289 sliverList = Sliver.objects.filter(slice=entry.slice.id)
292 if x.node.site not in siteList:
293 siteList[x.node.site] = 1
294 slivercount = len(sliverList)
295 sitecount = len(siteList)
297 traceback.print_exc()
301 userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
302 'sitesUsed':sites_used,
303 'role': SliceRole.objects.get(id=entry.role.id).role,
304 'slivercount': slivercount,
305 'sitecount':sitecount})
309 def getCDNContentProviderData():
311 for dm_cp in ContentProvider.objects.all():
312 cp = {"name": dm_cp.name,
313 "account": dm_cp.account}
318 def getCDNOperatorData(randomizeData = False, wait=True):
319 HPC_SLICE_NAME = "HyperCache"
321 bq = PlanetStackAnalytics()
323 rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
325 # wait=False on the first time the Dashboard is opened. This means we might
326 # not have any rows yet. The dashboard code polls every 30 seconds, so it
327 # will eventually pick them up.
330 rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
332 # dictionaryize the statistics rows by site name
335 stats_rows[row["site"]] = row
339 slice = Slice.objects.filter(name=HPC_SLICE_NAME)
341 slice_slivers = list(slice[0].slivers.all())
346 for site in Site.objects.all():
347 # compute number of slivers allocated in the data model
348 allocated_slivers = 0
349 for sliver in slice_slivers:
350 if sliver.node.site == site:
351 allocated_slivers = allocated_slivers + 1
353 stats_row = stats_rows.get(site.name,{})
355 max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
356 cpu=float(max_cpu)/100.0
357 hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
359 # format it to what that CDN Operations View is expecting
360 new_row = {"lat": float(site.location.longitude),
361 "long": float(site.location.longitude),
362 "lat": float(site.location.latitude),
364 "numNodes": int(site.nodes.count()),
365 "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics
366 "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model
367 "siteUrl": str(site.site_url),
368 "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
370 "hot": float(hotness)}
371 new_rows[str(site.name)] = new_row
373 # get rid of sites with 0 slivers that overlap other sites with >0 slivers
374 for (k,v) in new_rows.items():
376 if v["numHPCSlivers"]==0:
377 for v2 in new_rows.values():
378 if (v!=v2) and (v2["numHPCSlivers"]>=0):
379 d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
387 def getPageSummary(request):
388 slice = request.GET.get('slice', None)
389 site = request.GET.get('site', None)
390 node = request.GET.get('node', None)
392 class SimulatorView(View):
393 def get(self, request, **kwargs):
394 sim = json.loads(file("/tmp/simulator.json","r").read())
395 text = "<html><head></head><body>"
396 text += "Iteration: %d<br>" % sim["iteration"]
397 text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
398 text += "<table border=1>"
399 text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
400 for site in sim["site_load"].values():
402 text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
403 (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
406 text += "</body></html>"
407 return HttpResponse(text)
409 class DashboardUserSiteView(View):
410 def get(self, request, **kwargs):
411 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
413 class TenantViewData(View):
414 def get(self, request, **kwargs):
415 return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
417 def haversine(site_lat, site_lon, lat, lon):
419 if lat and lon and site_lat and site_lon:
420 site_lat = float(site_lat)
421 site_lon = float(site_lon)
425 a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
426 c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
431 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
432 # try to pick a site we're already using
433 has_slivers_here=False
435 for sliver in slice.slivers.all():
436 if sliver.node.site.name == site.name:
437 has_slivers_here=True
440 d = haversine(site.location.latitude, site.location.longitude, lat, lon)
442 return (-has_slivers_here, d)
444 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
445 """ Returns list of sites, sorted from most favorable to least favorable """
449 client_geo = GeoIP().city(user_ip)
451 lat=float(client_geo["latitude"])
452 lon=float(client_geo["longitude"])
454 print "exception in geo code"
455 traceback.print_exc()
457 sites = Site.objects.all()
458 sites = [x for x in sites if x.name in BLESSED_SITES]
459 sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
463 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
466 # let's compute how many slivers are in use in each node of each site
467 for site in siteList:
468 site.nodeList = list(site.nodes.all())
469 for node in site.nodeList:
471 for sliver in node.slivers.all():
472 if sliver.slice.id == slice.id:
473 node.sliverCount = node.sliverCount + 1
475 # Allocate slivers to nodes
476 # for now, assume we want to allocate all slivers from the same site
477 nodes = siteList[0].nodeList
479 # Sort the node list by number of slivers per node, then pick the
480 # node with the least number of slivers.
481 nodes = sorted(nodes, key=attrgetter("sliverCount"))
484 print "adding sliver at node", node.name, "of site", node.site.name
487 sliver = Sliver(name=node.name,
490 image = Image.objects.all()[0],
491 creator = User.objects.get(email=user),
492 deploymentNetwork=node.deployment,
496 node.sliverCount = node.sliverCount + 1
500 sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
504 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
508 siteNames = [site.name for site in siteList]
512 for sliver in slice.slivers.all():
513 if(not siteNames) or (sliver.node.site.name in siteNames):
\r
515 sliverList[sliver.name]=node.name
517 for key in sliverList:
519 sliver = Sliver.objects.filter(name=key)[0]
\r
521 print "deleting sliver",sliverList[key],"at node",sliver.node.name
\r
523 sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1
\r
527 class TenantDeleteSliceView(View):
528 def post(self,request):
\r
529 sliceName = request.POST.get("sliceName",None)
\r
530 slice = Slice.objects.get(name=sliceName)
\r
531 #print slice, slice.id
\r
532 sliceToDel=Slice(name=sliceName, id=slice.id)
\r
534 return HttpResponse("Slice deleted")
536 class TenantAddOrRemoveSliverView(View):
537 """ Add or remove slivers from a Slice
540 siteName - name of site. If not specified, PlanetStack will pick the
542 actionToDo - [add | rem]
543 count - number of slivers to add or remove
544 sliceName - name of slice
545 noAct - if set, no changes will be made to db, but result will still
546 show which sites would have been modified.
549 Dictionary of sites that were modified, and the count of nodes
550 that were added or removed at each site.
552 def post(self, request, *args, **kwargs):
553 siteName = request.POST.get("siteName", None)
554 actionToDo = request.POST.get("actionToDo", None)
555 count = int(request.POST.get("count","0"))
556 sliceName = request.POST.get("slice", None)
557 noAct = request.POST.get("noAct", False)
560 return HttpResponseServerError("No slice name given")
562 slice = Slice.objects.get(name=sliceName)
565 siteList = [Site.objects.get(name=siteName)]
569 if (actionToDo == "add"):
570 user_ip = request.GET.get("ip", get_ip(request))
571 if (siteList is None):
572 siteList = tenant_pick_sites(user, user_ip, slice, count)
574 sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
575 elif (actionToDo == "rem"):
576 sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
578 return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
580 return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
582 def get(self, request, *args, **kwargs):
583 request.POST = request.GET
584 return self.post(request, *args, **kwargs) # for testing REST in browser
585 #return HttpResponseServerError("GET is not supported")
587 class TenantPickSitesView(View):
588 """ primarily just for testing purposes """
589 def get(self, request, *args, **kwargs):
590 count = request.GET.get("count","0")
591 slice = request.GET.get("slice",None)
593 slice = Slice.objects.get(name=slice)
594 ip = request.GET.get("ip", get_ip(request))
595 sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
596 sites = [x.name for x in sites]
597 return HttpResponse(json.dumps(sites), mimetype='application/javascript')
599 class DashboardSummaryAjaxView(View):
600 def get(self, request, **kwargs):
604 return float(sum(x))/len(x)
606 sites = getCDNOperatorData().values()
608 sites = [site for site in sites if site["numHPCSlivers"]>0]
610 total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
611 total_bandwidth = sum( [site["bandwidth"] for site in sites] )
612 average_cpu = int(avg( [site["load"] for site in sites] ))
614 result= {"total_slivers": total_slivers,
615 "total_bandwidth": total_bandwidth,
616 "average_cpu": average_cpu}
618 return HttpResponse(json.dumps(result), mimetype='application/javascript')
620 class DashboardAddOrRemoveSliverView(View):
621 # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
623 def post(self, request, *args, **kwargs):
624 siteName = request.POST.get("site", None)
625 actionToDo = request.POST.get("actionToDo", "0")
627 siteList = [Site.objects.get(name=siteName)]
628 slice = Slice.objects.get(name="HyperCache")
630 if (actionToDo == "add"):
631 user_ip = request.GET.get("ip", get_ip(request))
632 slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
633 elif (actionToDo == "rem"):
634 slice_decrease_slivers(request.user, siteList, slice, 1)
637 print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
638 return HttpResponse('This is POST request ')
640 class DashboardAjaxView(View):
641 def get(self, request, **kwargs):
642 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
644 class DashboardAnalyticsAjaxView(View):
645 def get(self, request, name="hello_world", **kwargs):
646 if (name == "hpcSummary"):
647 return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
648 elif (name == "hpcUserSite"):
649 return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
650 elif (name == "hpcMap"):
651 return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
652 elif (name == "bigquery"):
653 (mimetype, data) = DoPlanetStackAnalytics(request)
654 return HttpResponse(data, mimetype=mimetype)
656 return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')