Merge branch 'master' of ssh://git.planet-lab.org/git/plstackapi
[plstackapi.git] / planetstack / core / plus / views.py
1 #views.py
2 import functools
3 import math
4 import os
5 import sys
6 from django.views.generic import TemplateView, View
7 import datetime
8 from pprint import pprint
9 import json
10 from core.models import *
11 from hpc.models import ContentProvider
12 from operator import attrgetter
13 from django.views.decorators.csrf import csrf_exempt
14 from django.http import HttpResponse, HttpResponseServerError
15 from django.core import urlresolvers
16 from django.contrib.gis.geoip import GeoIP
17 from ipware.ip import get_ip
18 import traceback
19 import socket
20
21 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
22
23 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
24     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
25 else:
26     sys.path.append("/opt/planetstack/hpc_wizard")
27 import hpc_wizard
28 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
29
30 class DashboardWelcomeView(TemplateView):
31     template_name = 'admin/dashboard/welcome.html'
32
33     def get(self, request, *args, **kwargs):
34         context = self.get_context_data(**kwargs)
35         userDetails = getUserSliceInfo(request.user)
36         #context['site'] = userDetails['site']
37
38         context['userSliceInfo'] = userDetails['userSliceInfo']
39         context['cdnData'] = userDetails['cdnData']
40         context['cdnContentProviders'] = userDetails['cdnContentProviders']
41         return self.render_to_response(context=context)
42
43 def getUserSliceInfo(user, tableFormat = False):
44         userDetails = {}
45
46         userSliceData = getSliceInfo(user)
47         if (tableFormat):
48 #            pprint("*******      GET USER SLICE INFO")
49             userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
50         else:
51             userDetails['userSliceInfo'] = userSliceData
52         userDetails['cdnData'] = getCDNOperatorData(wait=False)
53         userDetails['cdnContentProviders'] = getCDNContentProviderData()
54         return userDetails
55
56 class TenantCreateSlice(View):
57     def post(self, request, *args, **kwargs):
58         sliceName = request.POST.get("sliceName", "0")
59         serviceClass = request.POST.get("serviceClass", "0")
60         imageName = request.POST.get("imageName", "0")
61         actionToDo = request.POST.get("actionToDo", "0")
62         network = request.POST.get("network","0")
63         mountDataSets = request.POST.get("mountDataSets","0")
64         if (actionToDo == "add"):
65            serviceClass = ServiceClass.objects.get(name=serviceClass)
66            site = request.user.site
67            image = Image.objects.get(name=imageName)
68            newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
69            newSlice.save()
70         return HttpResponse("Slice created")
71
72 class TenantUpdateSlice(View):
73     def post(self, request, *args, **kwargs):\r
74         sliceName = request.POST.get("sliceName", "0")\r
75         serviceClass = request.POST.get("serviceClass", "0")\r
76         imageName = request.POST.get("imageName", "0")\r
77         actionToDo = request.POST.get("actionToDo", "0")\r
78         network = request.POST.get("network","0")\r
79         dataSet = request.POST.get("dataSet","0")\r
80         slice = Slice.objects.all()\r
81         for entry in slice:\r
82                 serviceClass = ServiceClass.objects.get(name=serviceClass)\r
83                 if(entry.name==sliceName):\r
84                          if (actionToDo == "update"):\r
85                                 setattr(entry,'serviceClass',serviceClass)\r
86                                 setattr(entry,'imagePreference',imageName)\r
87                                 setattr(entry,'network',network)\r
88                                 setattr(entry,'mountDataSets',dataSet)\r
89                                 entry.save()\r
90                                 break\r
91         return HttpResponse("Slice updated")\r
92 \r
93 def  update_slice(sliceName,**fields):
94          slice = Slice.objects.filter(name = sliceName)\r
95          for (k,v) in fields.items():\r
96                 setattr(slice, k, v)\r
97                 slice.save()\r
98          return slice
99
100 def getTenantSliceInfo(user, tableFormat = False):
101     tenantSliceDetails = {}
102     tenantSliceData = getTenantInfo(user)
103     tenantServiceClassData = getServiceClassInfo(user)
104     if (tableFormat):
105        tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
106        tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
107     else:
108        tenantSliceDetails['userSliceInfo'] = tenantSliceData
109     tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
110     tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
111     tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
112     tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
113     tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
114     tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
115     return tenantSliceDetails
116
117
118 def getTenantInfo(user):
119     slices =Slice.objects.all()
120     userSliceInfo = []
121     for entry in slices:
122        sliceName = Slice.objects.get(id=entry.id).name
123        slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
124        sliceServiceClass = entry.serviceClass.name
125        preferredImage =  entry.imagePreference
126        sliceDataSet = entry.mountDataSets
127        sliceNetwork = entry.network
128        numSliver = 0
129        sliceImage=""
130        sliceSite = {}
131        sliceNode = {}
132        sliceInstance= {}
133        for sliver in slice.slivers.all():
134             if sliver.node.site.name in BLESSED_SITES:
135                 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
136                 sliceImage = sliver.image.name
137                 sliceNode[str(sliver)] = sliver.node.name
138        numSliver = sum(sliceSite.values())
139        numSites = len(sliceSite)
140        userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
141     return userSliceInfo
142
143 def getTenantSitesInfo():
144         tenantSiteInfo=[]
145         for entry in Site.objects.all():
146             if entry.name in BLESSED_SITES:
147                  tenantSiteInfo.append({'siteName':entry.name})
148         return tenantSiteInfo
149
150 def userSliceTableFormatter(data):
151 #    pprint(data)
152     formattedData = {
153                      'rows' : data
154                     }
155     return formattedData
156
157 def getServiceClassInfo(user):
158     serviceClassList = ServiceClass.objects.all()
159     sliceInfo = []
160     for entry in serviceClassList:
161           sliceInfo.append({'serviceClass':entry.name})
162     return sliceInfo
163
164 def getImageInfo(user):
165     imageList = Image.objects.all()
166     #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
167     imageInfo = []
168     for imageEntry in imageList:
169           imageInfo.append({'Image':imageEntry.name})
170           #imageInfo.append({'Image':imageEntry})
171     return imageInfo
172
173 def getMountDataSets():
174         dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
175         dataSetInfo = []\r
176         for entry in dataSetList:\r
177                 dataSetInfo.append({'DataSet':entry})\r
178         return dataSetInfo
179
180 def getNetworkInfo(user):
181    #networkList = Network.objects.all()
182     networkList = ['Private Only','Private and Publicly Routable']
183     networkInfo = []
184     for networkEntry in networkList:
185           #networkInfo.append({'Network':networkEntry.name})
186           networkInfo.append({'Network':networkEntry})
187     return networkInfo
188
189 def getDeploymentSites():
190     deploymentList = Deployment.objects.all()
191     deploymentInfo = []
192     for entry in deploymentList:
193         deploymentInfo.append({'DeploymentSite':entry.name})
194     return deploymentInfo
195
196 def getSliceInfo(user):
197     sliceList = Slice.objects.all()
198     slicePrivs = SlicePrivilege.objects.filter(user=user)
199     userSliceInfo = []
200     for entry in slicePrivs:
201
202         slicename = Slice.objects.get(id=entry.slice.id).name
203         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
204         sliverList=Sliver.objects.all()
205         sites_used = {}
206         for sliver in slice.slivers.all():
207              #sites_used['deploymentSites'] = sliver.node.deployment.name
208              # sites_used[sliver.image.name] = sliver.image.name
209              sites_used[sliver.node.site.name] = sliver.numberCores
210         sliceid = Slice.objects.get(id=entry.slice.id).id
211         try:
212             sliverList = Sliver.objects.filter(slice=entry.slice.id)
213             siteList = {}
214             for x in sliverList:
215                if x.node.site not in siteList:
216                   siteList[x.node.site] = 1
217             slivercount = len(sliverList)
218             sitecount = len(siteList)
219         except:
220             traceback.print_exc()
221             slivercount = 0
222             sitecount = 0
223
224         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
225                               'sitesUsed':sites_used,
226                               'role': SliceRole.objects.get(id=entry.role.id).role,
227                               'slivercount': slivercount,
228                               'sitecount':sitecount})
229
230     return userSliceInfo
231
232 def getCDNContentProviderData():
233     cps = []
234     for dm_cp in ContentProvider.objects.all():
235         cp = {"name": dm_cp.name,
236               "account": dm_cp.account}
237         cps.append(cp)
238
239     return cps
240
241 def getCDNOperatorData(randomizeData = False, wait=True):
242     HPC_SLICE_NAME = "HyperCache"
243
244     bq = PlanetStackAnalytics()
245
246     rows = bq.get_cached_query_results(bq.compose_cached_query(), wait)
247
248     # wait=False on the first time the Dashboard is opened. This means we might
249     # not have any rows yet. The dashboard code polls every 30 seconds, so it
250     # will eventually pick them up.
251
252     if rows:
253         rows = bq.postprocess_results(rows, filter={"event": "hpc_heartbeat"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
254
255         # dictionaryize the statistics rows by site name
256         stats_rows = {}
257         for row in rows:
258             stats_rows[row["site"]] = row
259     else:
260         stats_rows = {}
261
262     slice = Slice.objects.filter(name=HPC_SLICE_NAME)
263     if slice:
264         slice_slivers = list(slice[0].slivers.all())
265     else:
266         slice_slivers = []
267
268     new_rows = {}
269     for site in Site.objects.all():
270         # compute number of slivers allocated in the data model
271         allocated_slivers = 0
272         for sliver in slice_slivers:
273             if sliver.node.site == site:
274                 allocated_slivers = allocated_slivers + 1
275
276         stats_row = stats_rows.get(site.name,{})
277
278         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
279         cpu=float(max_cpu)/100.0
280         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
281
282         # format it to what that CDN Operations View is expecting
283         new_row = {"lat": float(site.location.longitude),
284                "long": float(site.location.longitude),
285                "lat": float(site.location.latitude),
286                "health": 0,
287                "numNodes": int(site.nodes.count()),
288                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),     # measured number of slivers, from bigquery statistics
289                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
290                "siteUrl": str(site.site_url),
291                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
292                "load": max_cpu,
293                "hot": float(hotness)}
294         new_rows[str(site.name)] = new_row
295
296     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
297     for (k,v) in new_rows.items():
298         bad=False
299         if v["numHPCSlivers"]==0:
300             for v2 in new_rows.values():
301                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
302                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
303                     if d<100:
304                          bad=True
305             if bad:
306                 del new_rows[k]
307
308     return new_rows
309
310 class SimulatorView(View):
311     def get(self, request, **kwargs):
312         sim = json.loads(file("/tmp/simulator.json","r").read())
313         text = "<html><head></head><body>"
314         text += "Iteration: %d<br>" % sim["iteration"]
315         text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
316         text += "<table border=1>"
317         text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
318         for site in sim["site_load"].values():
319             text += "<tr>"
320             text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
321                         (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
322             text += "</tr>"
323         text += "</table>"
324         text += "</body></html>"
325         return HttpResponse(text)
326
327 class DashboardUserSiteView(View):
328     def get(self, request, **kwargs):
329         return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
330
331 class TenantViewData(View):
332     def get(self, request, **kwargs):
333         return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
334
335 def haversine(site_lat, site_lon, lat, lon):
336     d=0
337     if lat and lon and site_lat and site_lon:
338         site_lat = float(site_lat)
339         site_lon = float(site_lon)
340         lat = float(lat)
341         lon = float(lon)
342         R = 6378.1
343         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
344         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
345         d = R * c
346
347     return d
348
349 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
350     # try to pick a site we're already using
351     has_slivers_here=False
352     if slice:
353         for sliver in slice.slivers.all():
354             if sliver.node.site.name == site.name:
355                 has_slivers_here=True
356
357     # Haversine method
358     d = haversine(site.location.latitude, site.location.longitude, lat, lon)
359
360     return (-has_slivers_here, d)
361
362 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
363     """ Returns list of sites, sorted from most favorable to least favorable """
364     lat=None
365     lon=None
366     try:
367         client_geo = GeoIP().city(user_ip)
368         if client_geo:
369             lat=float(client_geo["latitude"])
370             lon=float(client_geo["longitude"])
371     except:
372         print "exception in geo code"
373         traceback.print_exc()
374
375     sites = Site.objects.all()
376     sites = [x for x in sites if x.name in BLESSED_SITES]
377     sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
378
379     return sites
380
381 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
382     sitesChanged = {}
383
384     # let's compute how many slivers are in use in each node of each site
385     for site in siteList:
386         site.nodeList = list(site.nodes.all())
387         for node in site.nodeList:
388             node.sliverCount = 0
389             for sliver in node.slivers.all():
390                  if sliver.slice.id == slice.id:
391                      node.sliverCount = node.sliverCount + 1
392
393     # Allocate slivers to nodes
394     # for now, assume we want to allocate all slivers from the same site
395     nodes = siteList[0].nodeList
396     while (count>0):
397         # Sort the node list by number of slivers per node, then pick the
398         # node with the least number of slivers.
399         nodes = sorted(nodes, key=attrgetter("sliverCount"))
400         node = nodes[0]
401
402         print "adding sliver at node", node.name, "of site", node.site.name
403
404         if not noAct:
405             sliver = Sliver(name=node.name,
406                         slice=slice,
407                         node=node,
408                         image = Image.objects.all()[0],
409                         creator = User.objects.get(email=user),
410                         deploymentNetwork=node.deployment,
411                         numberCores =1 )
412             sliver.save()
413
414         node.sliverCount = node.sliverCount + 1
415
416         count = count - 1
417
418         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
419
420     return sitesChanged
421
422 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
423     sitesChanged = {}
424     sliverList ={}
425     if siteList:
426         siteNames = [site.name for site in siteList]
427     else:
428         siteNames = None
429
430     for sliver in slice.slivers.all():
431         if(not siteNames) or (sliver.node.site.name in siteNames):\r
432                 node = sliver.node\r
433                 sliverList[sliver.name]=node.name
434
435     for key in sliverList:
436         if count>0:
437             sliver = Sliver.objects.filter(name=key)[0]\r
438             sliver.delete()\r
439             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
440             count=count-1\r
441             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
442 \r
443     return sitesChanged
444
445 class TenantDeleteSliceView(View):
446         def post(self,request):\r
447                 sliceName = request.POST.get("sliceName",None)\r
448                 slice = Slice.objects.get(name=sliceName)\r
449                 print slice, slice.id\r
450                 sliceToDel=Slice(name=sliceName, id=slice.id)\r
451                 sliceToDel.delete()
452                 return HttpResponse("Slice deleted")
453
454 class TenantAddOrRemoveSliverView(View):
455     """ Add or remove slivers from a Slice
456
457         Arguments:
458             siteName - name of site. If not specified, PlanetStack will pick the
459                        best site.,
460             actionToDo - [add | rem]
461             count - number of slivers to add or remove
462             sliceName - name of slice
463             noAct - if set, no changes will be made to db, but result will still
464                     show which sites would have been modified.
465
466         Returns:
467             Dictionary of sites that were modified, and the count of nodes
468             that were added or removed at each site.
469     """
470     def post(self, request, *args, **kwargs):
471         siteName = request.POST.get("siteName", None)
472         actionToDo = request.POST.get("actionToDo", None)
473         count = int(request.POST.get("count","0"))
474         sliceName = request.POST.get("slice", None)
475         noAct = request.POST.get("noAct", False)
476
477         if not sliceName:
478             return HttpResponseServerError("No slice name given")
479
480         slice = Slice.objects.get(name=sliceName)
481
482         if siteName:
483             siteList = [Site.objects.get(name=siteName)]
484         else:
485             siteList = None
486
487         if (actionToDo == "add"):
488             user_ip = request.GET.get("ip", get_ip(request))
489             if (siteList is None):
490                 siteList = tenant_pick_sites(user, user_ip, slice, count)
491
492             sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
493         elif (actionToDo == "rem"):
494             sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
495         else:
496             return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
497
498         return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
499
500     def get(self, request, *args, **kwargs):
501         request.POST = request.GET
502         return self.post(request, *args, **kwargs)  # for testing REST in browser
503         #return HttpResponseServerError("GET is not supported")
504
505 class TenantPickSitesView(View):
506     """ primarily just for testing purposes """
507     def get(self, request, *args, **kwargs):
508         count = request.GET.get("count","0")
509         slice = request.GET.get("slice",None)
510         if slice:
511             slice = Slice.objects.get(name=slice)
512         ip = request.GET.get("ip", get_ip(request))
513         sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
514         sites = [x.name for x in sites]
515         return HttpResponse(json.dumps(sites), mimetype='application/javascript')
516
517 class DashboardSummaryAjaxView(View):
518     def get(self, request, **kwargs):
519         def avg(x):
520             return float(sum(x))/len(x)
521
522         sites = getCDNOperatorData().values()
523
524         sites = [site for site in sites if site["numHPCSlivers"]>0]
525
526         total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
527         total_bandwidth = sum( [site["bandwidth"] for site in sites] )
528         average_cpu = int(avg( [site["load"] for site in sites] ))
529
530         result= {"total_slivers": total_slivers,
531                 "total_bandwidth": total_bandwidth,
532                 "average_cpu": average_cpu}
533
534         return HttpResponse(json.dumps(result), mimetype='application/javascript')
535
536 class DashboardAddOrRemoveSliverView(View):
537     # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
538
539     def post(self, request, *args, **kwargs):
540         siteName = request.POST.get("site", None)
541         actionToDo = request.POST.get("actionToDo", "0")
542
543         siteList = [Site.objects.get(name=siteName)]
544         slice = Slice.objects.get(name="HyperCache")
545
546         if (actionToDo == "add"):
547             user_ip = request.GET.get("ip", get_ip(request))
548             slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
549         elif (actionToDo == "rem"):
550             slice_decrease_slivers(request.user, siteList, slice, 1)
551
552         print '*' * 50
553         print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
554         return HttpResponse('This is POST request ')
555
556 class DashboardAjaxView(View):
557     def get(self, request, **kwargs):
558         return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
559
560 class DashboardAnalyticsAjaxView(View):
561     def get(self, request, name="hello_world", **kwargs):
562         if (name == "hpcSummary"):
563             return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
564         elif (name == "hpcUserSite"):
565             return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
566         elif (name == "hpcMap"):
567             return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
568         elif (name == "bigquery"):
569             (mimetype, data) = DoPlanetStackAnalytics(request)
570             return HttpResponse(data, mimetype=mimetype)
571         else:
572             return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
573