merge latest Amisha changes
[plstackapi.git] / planetstack / core / plus / views.py
1 #views.py
2 import functools
3 import math
4 import os
5 import sys
6 from django.views.generic import TemplateView, View
7 import datetime
8 from pprint import pprint
9 import json
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
17 import traceback
18 import socket
19
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
21
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
24 else:
25     sys.path.append("/opt/planetstack/hpc_wizard")
26 import hpc_wizard
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
28
29 class DashboardWelcomeView(TemplateView):
30     template_name = 'admin/dashboard/welcome.html'
31
32     def get(self, request, *args, **kwargs):
33         context = self.get_context_data(**kwargs)
34         userDetails = getUserSliceInfo(request.user)
35         #context['site'] = userDetails['site']
36
37         context['userSliceInfo'] = userDetails['userSliceInfo']
38         context['cdnData'] = userDetails['cdnData']
39         return self.render_to_response(context=context)
40
41 def getUserSliceInfo(user, tableFormat = False):
42         userDetails = {}
43 #        try:
44 # //           site = Site.objects.filter(id=user.site.id)
45 #  //      except:
46 #   //         site = Site.objects.filter(name="Princeton")
47 #    //    userDetails['sitename'] = site[0].name
48 #     //   userDetails['siteid'] = site[0].id
49
50         userSliceData = getSliceInfo(user)
51         if (tableFormat):
52 #            pprint("*******      GET USER SLICE INFO")
53             userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
54         else:
55             userDetails['userSliceInfo'] = userSliceData
56         userDetails['cdnData'] = getCDNOperatorData(wait=False);
57 #        pprint( userDetails)
58         return userDetails
59
60 class TenantCreateSlice(View):
61     def post(self, request, *args, **kwargs):
62         sliceName = request.POST.get("sliceName", "0")
63         serviceClass = request.POST.get("serviceClass", "0")
64         imageName = request.POST.get("imageName", "0")
65         actionToDo = request.POST.get("actionToDo", "0")
66         network = request.POST.get("network","0")
67         mountDataSets = request.POST.get("mountDataSets","0")
68         if (actionToDo == "add"):
69            serviceClass = ServiceClass.objects.get(name=serviceClass)
70            site = request.user.site
71            image = Image.objects.get(name=imageName)
72            newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=image,mountDataSets=mountDataSets,network=network)
73            newSlice.save()
74         return HttpResponse("Slice created")
75
76 class TenantUpdateSlice(View):
77     def post(self, request, *args, **kwargs):\r
78         sliceName = request.POST.get("sliceName", "0")\r
79         serviceClass = request.POST.get("serviceClass", "0")\r
80         imageName = request.POST.get("imageName", "0")\r
81         actionToDo = request.POST.get("actionToDo", "0")\r
82         network = request.POST.get("network","0")\r
83         dataSet = request.POST.get("dataSet","0")\r
84         slice = Slice.objects.all()\r
85         for entry in slice:\r
86                 serviceClass = ServiceClass.objects.get(name=serviceClass)\r
87                 if(entry.name==sliceName):\r
88                          if (actionToDo == "update"):\r
89                                 setattr(entry,'serviceClass',serviceClass)\r
90                                 setattr(entry,'imagePreference',imageName)\r
91                                 setattr(entry,'network',network)\r
92                                 setattr(entry,'mountDataSets',dataSet)\r
93                                 entry.save()\r
94                                 break\r
95         return HttpResponse("Slice updated")\r
96 \r
97 def  update_slice(sliceName,**fields):
98          slice = Slice.objects.filter(name = sliceName)\r
99          for (k,v) in fields.items():\r
100                 setattr(slice, k, v)\r
101                 slice.save()\r
102          return slice
103
104 def getTenantSliceInfo(user, tableFormat = False):
105     tenantSliceDetails = {}
106     tenantSliceData = getTenantInfo(user)
107     tenantServiceClassData = getServiceClassInfo(user)
108     if (tableFormat):
109        tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
110        tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
111     else:
112        tenantSliceDetails['userSliceInfo'] = tenantSliceData
113     tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
114     tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
115     tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
116     tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
117     tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
118     tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
119     return tenantSliceDetails
120
121
122 def getTenantInfo(user):
123     slices =Slice.objects.all()
124     userSliceInfo = []
125     for entry in slices:
126        sliceName = Slice.objects.get(id=entry.id).name
127        slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
128        sliceServiceClass = entry.serviceClass.name
129        preferredImage =  entry.imagePreference
130        sliceDataSet = entry.mountDataSets
131        sliceNetwork = entry.network
132        numSliver = 0
133        sliceImage=""
134        sliceSite = {}
135        sliceNode = {}
136        sliceInstance= {}
137        for sliver in slice.slivers.all():
138             if sliver.node.site.name in BLESSED_SITES:
139                 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
140                 sliceImage = sliver.image.name
141                 sliceNode[sliver.instance_name] = sliver.name
142        numSliver = sum(sliceSite.values())
143        numSites = len(sliceSite)
144        userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
145     return userSliceInfo
146
147 def getTenantSitesInfo():
148         tenantSiteInfo=[]
149         for entry in Site.objects.all():
150             if entry.name in BLESSED_SITES:
151                  tenantSiteInfo.append({'siteName':entry.name})
152         return tenantSiteInfo
153
154 def userSliceTableFormatter(data):
155 #    pprint(data)
156     formattedData = {
157                      'rows' : data
158                     }
159     return formattedData
160
161 def getServiceClassInfo(user):
162     serviceClassList = ServiceClass.objects.all()
163     sliceInfo = []
164     for entry in serviceClassList:
165           sliceInfo.append({'serviceClass':entry.name})
166     return sliceInfo
167
168 def getImageInfo(user):
169     imageList = Image.objects.all()
170     #imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
171     imageInfo = []
172     for imageEntry in imageList:
173           imageInfo.append({'Image':imageEntry.name})
174           #imageInfo.append({'Image':imageEntry})
175     return imageInfo
176
177 def getMountDataSets():
178         dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
179         dataSetInfo = []\r
180         for entry in dataSetList:\r
181                 dataSetInfo.append({'DataSet':entry})\r
182         return dataSetInfo
183
184 def getNetworkInfo(user):
185    #networkList = Network.objects.all()
186     networkList = ['Private Only','Private and Publicly Routable']
187     networkInfo = []
188     for networkEntry in networkList:
189           #networkInfo.append({'Network':networkEntry.name})
190           networkInfo.append({'Network':networkEntry})
191     return networkInfo
192
193 def getDeploymentSites():
194     deploymentList = Deployment.objects.all()
195     deploymentInfo = []
196     for entry in deploymentList:
197         deploymentInfo.append({'DeploymentSite':entry.name})
198     return deploymentInfo
199
200 def getSliceInfo(user):
201     sliceList = Slice.objects.all()
202     slicePrivs = SlicePrivilege.objects.filter(user=user)
203     userSliceInfo = []
204     for entry in slicePrivs:
205
206         slicename = Slice.objects.get(id=entry.slice.id).name
207         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
208         sliverList=Sliver.objects.all()
209         sites_used = {}
210         for sliver in slice.slivers.all():
211              #sites_used['deploymentSites'] = sliver.node.deployment.name
212              # sites_used[sliver.image.name] = sliver.image.name
213              sites_used[sliver.node.site.name] = sliver.numberCores
214         sliceid = Slice.objects.get(id=entry.slice.id).id
215         try:
216             sliverList = Sliver.objects.filter(slice=entry.slice.id)
217             siteList = {}
218             for x in sliverList:
219                if x.node.site not in siteList:
220                   siteList[x.node.site] = 1
221             slivercount = len(sliverList)
222             sitecount = len(siteList)
223         except:
224             traceback.print_exc()
225             slivercount = 0
226             sitecount = 0
227
228         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
229                               'sitesUsed':sites_used,
230                               'role': SliceRole.objects.get(id=entry.role.id).role,
231                               'slivercount': slivercount,
232                               'sitecount':sitecount})
233
234     return userSliceInfo
235
236 def getCDNOperatorData(randomizeData = False, wait=True):
237     HPC_SLICE_NAME = "HyperCache"
238
239     bq = PlanetStackAnalytics()
240
241     rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
242
243     # wait=False on the first time the Dashboard is opened. This means we might
244     # not have any rows yet. The dashboard code polls every 30 seconds, so it
245     # will eventually pick them up.
246
247     if rows:
248         rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
249
250         # dictionaryize the statistics rows by site name
251         stats_rows = {}
252         for row in rows:
253             stats_rows[row["site"]] = row
254     else:
255         stats_rows = {}
256
257     slice = Slice.objects.get(name=HPC_SLICE_NAME)
258     slice_slivers = list(slice.slivers.all())
259
260     new_rows = {}
261     for site in Site.objects.all():
262         # compute number of slivers allocated in the data model
263         allocated_slivers = 0
264         for sliver in slice_slivers:
265             if sliver.node.site == site:
266                 allocated_slivers = allocated_slivers + 1
267
268         stats_row = stats_rows.get(site.name,{})
269
270         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
271         cpu=float(max_cpu)/100.0
272         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
273
274         # format it to what that CDN Operations View is expecting
275         new_row = {"lat": float(site.location.longitude),
276                "long": float(site.location.longitude),
277                "lat": float(site.location.latitude),
278                "health": 0,
279                "numNodes": int(site.nodes.count()),
280                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),         # measured number of slivers, from bigquery statistics
281                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
282                "siteUrl": str(site.site_url),
283                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
284                "load": max_cpu,
285                "hot": float(hotness)}
286         new_rows[str(site.name)] = new_row
287
288     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
289     for (k,v) in new_rows.items():
290         bad=False
291         if v["numHPCSlivers"]==0:
292             for v2 in new_rows.values():
293                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
294                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
295                     if d<100:
296                          bad=True
297             if bad:
298                 del new_rows[k]
299
300     return new_rows
301
302 class SimulatorView(View):
303     def get(self, request, **kwargs):
304         sim = json.loads(file("/tmp/simulator.json","r").read())
305         text = "<html><head></head><body>"
306         text += "Iteration: %d<br>" % sim["iteration"]
307         text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
308         text += "<table border=1>"
309         text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
310         for site in sim["site_load"].values():
311             text += "<tr>"
312             text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
313                         (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
314             text += "</tr>"
315         text += "</table>"
316         text += "</body></html>"
317         return HttpResponse(text)
318
319 class DashboardUserSiteView(View):
320     def get(self, request, **kwargs):
321         return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
322
323 class TenantViewData(View):
324     def get(self, request, **kwargs):
325         return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
326
327 def haversine(site_lat, site_lon, lat, lon):
328     d=0
329     if lat and lon and site_lat and site_lon:
330         site_lat = float(site_lat)
331         site_lon = float(site_lon)
332         lat = float(lat)
333         lon = float(lon)
334         R = 6378.1
335         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
336         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
337         d = R * c
338
339     return d
340
341 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
342     # try to pick a site we're already using
343     has_slivers_here=False
344     if slice:
345         for sliver in slice.slivers.all():
346             if sliver.node.site.name == site.name:
347                 has_slivers_here=True
348
349     # Haversine method
350     d = haversine(site.location.latitude, site.location.longitude, lat, lon)
351
352     return (-has_slivers_here, d)
353
354 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
355     """ Returns list of sites, sorted from most favorable to least favorable """
356     lat=None
357     lon=None
358     try:
359         client_geo = GeoIP().city(user_ip)
360         if client_geo:
361             lat=float(client_geo["latitude"])
362             lon=float(client_geo["longitude"])
363     except:
364         print "exception in geo code"
365         traceback.print_exc()
366
367     sites = Site.objects.all()
368     sites = [x for x in sites if x.name in BLESSED_SITES]
369     sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
370
371     return sites
372
373 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
374     sitesChanged = {}
375
376     # let's compute how many slivers are in use in each node of each site
377     for site in siteList:
378         site.nodeList = list(site.nodes.all())
379         for node in site.nodeList:
380             node.sliverCount = 0
381             for sliver in node.slivers.all():
382                  if sliver.slice.id == slice.id:
383                      node.sliverCount = node.sliverCount + 1
384
385     # Allocate slivers to nodes
386     # for now, assume we want to allocate all slivers from the same site
387     nodes = siteList[0].nodeList
388     while (count>0):
389         # Sort the node list by number of slivers per node, then pick the
390         # node with the least number of slivers.
391         nodes = sorted(nodes, key=attrgetter("sliverCount"))
392         node = nodes[0]
393
394         print "adding sliver at node", node.name, "of site", node.site.name
395
396         if not noAct:
397             sliver = Sliver(name=node.name,
398                         slice=slice,
399                         node=node,
400                         image = Image.objects.all()[0],
401                         creator = User.objects.get(email=user),
402                         deploymentNetwork=node.deployment,
403                         numberCores =1 )
404             sliver.save()
405
406         node.sliverCount = node.sliverCount + 1
407
408         count = count - 1
409
410         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
411
412     return sitesChanged
413
414 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
415     sitesChanged = {}
416     sliverList ={}
417     if siteList:
418         siteNames = [site.name for site in siteList]
419     else:
420         siteNames = None
421
422     for sliver in slice.slivers.all():
423         if(not siteNames) or (sliver.node.site.name in siteNames):\r
424                 node = sliver.node\r
425                 sliverList[sliver.name]=node.name
426
427     for key in sliverList:
428         if count>0:
429             sliver = Sliver.objects.filter(name=key)[0]\r
430             sliver.delete()\r
431             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
432             count=count-1\r
433             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
434 \r
435     return sitesChanged
436
437 class TenantDeleteSliceView(View):
438         def post(self,request):\r
439                 sliceName = request.POST.get("sliceName",None)\r
440                 slice = Slice.objects.get(name=sliceName)\r
441                 print slice, slice.id\r
442                 sliceToDel=Slice(name=sliceName, id=slice.id)\r
443                 sliceToDel.delete()
444                 return HttpResponse("Slice deleted")
445
446 class TenantAddOrRemoveSliverView(View):
447     """ Add or remove slivers from a Slice
448
449         Arguments:
450             siteName - name of site. If not specified, PlanetStack will pick the
451                        best site.,
452             actionToDo - [add | rem]
453             count - number of slivers to add or remove
454             sliceName - name of slice
455             noAct - if set, no changes will be made to db, but result will still
456                     show which sites would have been modified.
457
458         Returns:
459             Dictionary of sites that were modified, and the count of nodes
460             that were added or removed at each site.
461     """
462     def post(self, request, *args, **kwargs):
463         siteName = request.POST.get("siteName", None)
464         actionToDo = request.POST.get("actionToDo", None)
465         count = int(request.POST.get("count","0"))
466         sliceName = request.POST.get("slice", None)
467         noAct = request.POST.get("noAct", False)
468
469         if not sliceName:
470             return HttpResponseServerError("No slice name given")
471
472         slice = Slice.objects.get(name=sliceName)
473
474         if siteName:
475             siteList = [Site.objects.get(name=siteName)]
476         else:
477             siteList = None
478
479         if (actionToDo == "add"):
480             user_ip = request.GET.get("ip", get_ip(request))
481             if (siteList is None):
482                 siteList = tenant_pick_sites(user, user_ip, slice, count)
483
484             sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
485         elif (actionToDo == "rem"):
486             sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
487         else:
488             return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
489
490         return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
491
492     def get(self, request, *args, **kwargs):
493         request.POST = request.GET
494         return self.post(request, *args, **kwargs)  # for testing REST in browser
495         #return HttpResponseServerError("GET is not supported")
496
497 class TenantPickSitesView(View):
498     """ primarily just for testing purposes """
499     def get(self, request, *args, **kwargs):
500         count = request.GET.get("count","0")
501         slice = request.GET.get("slice",None)
502         if slice:
503             slice = Slice.objects.get(name=slice)
504         ip = request.GET.get("ip", get_ip(request))
505         sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
506         sites = [x.name for x in sites]
507         return HttpResponse(json.dumps(sites), mimetype='application/javascript')
508
509 class DashboardSummaryAjaxView(View):
510     def get(self, request, **kwargs):
511         def avg(x):
512             return float(sum(x))/len(x)
513
514         sites = getCDNOperatorData().values()
515
516         sites = [site for site in sites if site["numHPCSlivers"]>0]
517
518         total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
519         total_bandwidth = sum( [site["bandwidth"] for site in sites] )
520         average_cpu = int(avg( [site["load"] for site in sites] ))
521
522         result= {"total_slivers": total_slivers,
523                 "total_bandwidth": total_bandwidth,
524                 "average_cpu": average_cpu}
525
526         return HttpResponse(json.dumps(result), mimetype='application/javascript')
527
528 class DashboardAddOrRemoveSliverView(View):
529     # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
530
531     def post(self, request, *args, **kwargs):
532         siteName = request.POST.get("site", None)
533         actionToDo = request.POST.get("actionToDo", "0")
534
535         siteList = [Site.objects.get(name=siteName)]
536         slice = Slice.objects.get(name="HyperCache")
537
538         if (actionToDo == "add"):
539             user_ip = request.GET.get("ip", get_ip(request))
540             slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
541         elif (actionToDo == "rem"):
542             slice_decrease_slivers(request.user, siteList, slice, 1)
543
544         print '*' * 50
545         print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
546         return HttpResponse('This is POST request ')
547
548 class DashboardAjaxView(View):
549     def get(self, request, **kwargs):
550         return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
551
552 class DashboardAnalyticsAjaxView(View):
553     def get(self, request, name="hello_world", **kwargs):
554         if (name == "hpcSummary"):
555             return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
556         elif (name == "hpcUserSite"):
557             return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
558         elif (name == "hpcMap"):
559             return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
560         elif (name == "bigquery"):
561             (mimetype, data) = DoPlanetStackAnalytics(request)
562             return HttpResponse(data, mimetype=mimetype)
563         else:
564             return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
565