Amisha's latest changes
[plstackapi.git] / planetstack / core / plus / views.py
1 #views.py
2 import functools
3 import math
4 import os
5 import sys
6 from django.views.generic import TemplateView, View
7 import datetime
8 from pprint import pprint
9 import json
10 from core.models import *
11 from operator import attrgetter
12 from django.views.decorators.csrf import csrf_exempt
13 from django.http import HttpResponse, HttpResponseServerError
14 from django.core import urlresolvers
15 from django.contrib.gis.geoip import GeoIP
16 from ipware.ip import get_ip
17 import traceback
18 import socket
19
20 BLESSED_SITES = ["Stanford", "Washington", "Princeton", "GeorgiaTech", "MaxPlanck"]
21
22 if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"):
23     sys.path.append("/home/smbaker/projects/vicci/cdn/bigquery")
24 else:
25     sys.path.append("/opt/planetstack/hpc_wizard")
26 import hpc_wizard
27 from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD
28
29 class DashboardWelcomeView(TemplateView):
30     template_name = 'admin/dashboard/welcome.html'
31
32     def get(self, request, *args, **kwargs):
33         context = self.get_context_data(**kwargs)
34         userDetails = getUserSliceInfo(request.user)
35         #context['site'] = userDetails['site']
36
37         context['userSliceInfo'] = userDetails['userSliceInfo']
38         context['cdnData'] = userDetails['cdnData']
39         return self.render_to_response(context=context)
40
41 def getUserSliceInfo(user, tableFormat = False):
42         userDetails = {}
43 #        try:
44 # //           site = Site.objects.filter(id=user.site.id)
45 #  //      except:
46 #   //         site = Site.objects.filter(name="Princeton")
47 #    //    userDetails['sitename'] = site[0].name
48 #     //   userDetails['siteid'] = site[0].id
49
50         userSliceData = getSliceInfo(user)
51         if (tableFormat):
52 #            pprint("*******      GET USER SLICE INFO")
53             userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData)
54         else:
55             userDetails['userSliceInfo'] = userSliceData
56         userDetails['cdnData'] = getCDNOperatorData(wait=False);
57 #        pprint( userDetails)
58         return userDetails
59
60 class TenantCreateSlice(View):
61     def post(self, request, *args, **kwargs):
62         sliceName = request.POST.get("sliceName", "0")
63         serviceClass = request.POST.get("serviceClass", "0")
64         imageName = request.POST.get("imageName", "0")
65         actionToDo = request.POST.get("actionToDo", "0")
66         network = request.POST.get("network","0")
67         mountDataSets = request.POST.get("mountDataSets","0")
68         if (actionToDo == "add"):
69            serviceClass = ServiceClass.objects.get(name=serviceClass)
70            site = request.user.site
71            #image = Image.objects.get(name=imageName)
72            newSlice = Slice(name=sliceName,serviceClass=serviceClass,site=site,imagePreference=imageName,mountDataSets=mountDataSets,network=network)
73            newSlice.save()
74         return newSlice
75
76 class TenantUpdateSlice(View):
77     def post(self, request, *args, **kwargs):\r
78         sliceName = request.POST.get("sliceName", "0")\r
79         serviceClass = request.POST.get("serviceClass", "0")\r
80         imageName = request.POST.get("imageName", "0")\r
81         actionToDo = request.POST.get("actionToDo", "0")\r
82         network = request.POST.get("network","0")\r
83         slice = Slice.objects.filter(name = sliceName)\r
84         abc = ServiceClass.objects.get(name=serviceClass)\r
85         if (actionToDo == "update"):\r
86         #       print getattr(slice,'serviceClass',abc)\r
87                 setattr(slice,'serviceClass',abc)\r
88         #fields = {'serviceClass':ServiceClass.objects.get(name=serviceClass),\r
89          #         'imagePreference':imageName,\r
90           #        'network':network\r
91            #      }\r
92         #update_slice(sliceName,**fields)\r
93         return HttpResponse("Slice updated")
94
95 def  update_slice(sliceName,**fields):
96          slice = Slice.objects.filter(name = sliceName)\r
97          for (k,v) in fields.items():\r
98                 setattr(slice, k, v)\r
99                 slice.save()\r
100          return slice
101
102 def getTenantSliceInfo(user, tableFormat = False):
103     tenantSliceDetails = {}
104     tenantSliceData = getTenantInfo(user)
105     tenantServiceClassData = getServiceClassInfo(user)
106     if (tableFormat):
107        tenantSliceDetails['userSliceInfo'] = userSliceTableFormatter(tenantSliceData)
108        tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
109     else:
110        tenantSliceDetails['userSliceInfo'] = tenantSliceData
111     tenantSliceDetails['sliceServiceClass']=userSliceTableFormatter(tenantServiceClassData)
112     tenantSliceDetails['image']=userSliceTableFormatter(getImageInfo(user))
113     tenantSliceDetails['network']=userSliceTableFormatter(getNetworkInfo(user))
114     tenantSliceDetails['deploymentSites']=userSliceTableFormatter(getDeploymentSites())
115     tenantSliceDetails['sites'] = userSliceTableFormatter(getTenantSitesInfo())
116     tenantSliceDetails['mountDataSets'] = userSliceTableFormatter(getMountDataSets())
117     return tenantSliceDetails
118
119
120 def getTenantInfo(user):
121     slices =Slice.objects.all()
122     userSliceInfo = []
123     for entry in slices:
124        sliceName = Slice.objects.get(id=entry.id).name
125        slice = Slice.objects.get(name=Slice.objects.get(id=entry.id).name)
126        sliceServiceClass = entry.serviceClass.name
127        preferredImage =  entry.imagePreference
128        sliceDataSet = entry.mountDataSets
129        sliceNetwork = entry.network
130        numSliver = 0
131        sliceImage=""
132        sliceSite = {}
133        sliceNode = {}
134        sliceInstance= {}
135        for sliver in slice.slivers.all():
136             if sliver.node.site.name in BLESSED_SITES:
137                 sliceSite[sliver.node.site.name] = sliceSite.get(sliver.node.site.name,0) + 1
138                 sliceImage = sliver.image.name
139                 sliceNode[sliver.instance_name] = sliver.name
140        numSliver = sum(sliceSite.values())
141        numSites = len(sliceSite)
142        userSliceInfo.append({'sliceName': sliceName,'sliceServiceClass': sliceServiceClass,'preferredImage':preferredImage,'numOfSites':numSites, 'sliceSite':sliceSite,'sliceImage':sliceImage,'numOfSlivers':numSliver,'sliceDataSet':sliceDataSet,'sliceNetwork':sliceNetwork, 'instanceNodePair':sliceNode})
143     return userSliceInfo
144
145 def getTenantSitesInfo():
146         tenantSiteInfo=[]
147         for entry in Site.objects.all():
148             if entry.name in BLESSED_SITES:
149                  tenantSiteInfo.append({'siteName':entry.name})
150         return tenantSiteInfo
151
152 def userSliceTableFormatter(data):
153 #    pprint(data)
154     formattedData = {
155                      'rows' : data
156                     }
157     return formattedData
158
159 def getServiceClassInfo(user):
160     serviceClassList = ServiceClass.objects.all()
161     sliceInfo = []
162     for entry in serviceClassList:
163           sliceInfo.append({'serviceClass':entry.name})
164     return sliceInfo
165
166 def getImageInfo(user):
167     #imageList = Image.objects.all()
168     imageList = ['Fedora 16 LXC rev 1.3','Hadoop','MPI']
169     imageInfo = []
170     for imageEntry in imageList:
171           #imageInfo.append({'Image':imageEntry.name})
172           imageInfo.append({'Image':imageEntry})
173     return imageInfo
174
175 def getMountDataSets():
176         dataSetList = ['------','GenBank','LSST','LHC','NOAA','Measurement Lab','Common Crawl']\r
177         dataSetInfo = []\r
178         for entry in dataSetList:\r
179                 dataSetInfo.append({'DataSet':entry})\r
180         return dataSetInfo
181
182 def getNetworkInfo(user):
183    #networkList = Network.objects.all()
184     networkList = ['Private Only','Private and Publicly Routable']
185     networkInfo = []
186     for networkEntry in networkList:
187           #networkInfo.append({'Network':networkEntry.name})
188           networkInfo.append({'Network':networkEntry})
189     return networkInfo
190
191 def getDeploymentSites():
192     deploymentList = Deployment.objects.all()
193     deploymentInfo = []
194     for entry in deploymentList:
195         deploymentInfo.append({'DeploymentSite':entry.name})
196     return deploymentInfo
197
198 def getSliceInfo(user):
199     sliceList = Slice.objects.all()
200     slicePrivs = SlicePrivilege.objects.filter(user=user)
201     userSliceInfo = []
202     for entry in slicePrivs:
203
204         slicename = Slice.objects.get(id=entry.slice.id).name
205         slice = Slice.objects.get(name=Slice.objects.get(id=entry.slice.id).name)
206         sliverList=Sliver.objects.all()
207         sites_used = {}
208         for sliver in slice.slivers.all():
209              #sites_used['deploymentSites'] = sliver.node.deployment.name
210              # sites_used[sliver.image.name] = sliver.image.name
211              sites_used[sliver.node.site.name] = sliver.numberCores
212         sliceid = Slice.objects.get(id=entry.slice.id).id
213         try:
214             sliverList = Sliver.objects.filter(slice=entry.slice.id)
215             siteList = {}
216             for x in sliverList:
217                if x.node.site not in siteList:
218                   siteList[x.node.site] = 1
219             slivercount = len(sliverList)
220             sitecount = len(siteList)
221         except:
222             traceback.print_exc()
223             slivercount = 0
224             sitecount = 0
225
226         userSliceInfo.append({'slicename': slicename, 'sliceid':sliceid,
227                               'sitesUsed':sites_used,
228                               'role': SliceRole.objects.get(id=entry.role.id).role,
229                               'slivercount': slivercount,
230                               'sitecount':sitecount})
231
232     return userSliceInfo
233
234 def getCDNOperatorData(randomizeData = False, wait=True):
235     HPC_SLICE_NAME = "HyperCache"
236
237     bq = PlanetStackAnalytics()
238
239     rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait)
240
241     # wait=False on the first time the Dashboard is opened. This means we might
242     # not have any rows yet. The dashboard code polls every 30 seconds, so it
243     # will eventually pick them up.
244
245     if rows:
246         rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80)
247
248         # dictionaryize the statistics rows by site name
249         stats_rows = {}
250         for row in rows:
251             stats_rows[row["site"]] = row
252     else:
253         stats_rows = {}
254
255     slice = Slice.objects.get(name=HPC_SLICE_NAME)
256     slice_slivers = list(slice.slivers.all())
257
258     new_rows = {}
259     for site in Site.objects.all():
260         # compute number of slivers allocated in the data model
261         allocated_slivers = 0
262         for sliver in slice_slivers:
263             if sliver.node.site == site:
264                 allocated_slivers = allocated_slivers + 1
265
266         stats_row = stats_rows.get(site.name,{})
267
268         max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0))
269         cpu=float(max_cpu)/100.0
270         hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD))
271
272         # format it to what that CDN Operations View is expecting
273         new_row = {"lat": float(site.location.longitude),
274                "long": float(site.location.longitude),
275                "lat": float(site.location.latitude),
276                "health": 0,
277                "numNodes": int(site.nodes.count()),
278                "activeHPCSlivers": int(stats_row.get("count_hostname", 0)),         # measured number of slivers, from bigquery statistics
279                "numHPCSlivers": allocated_slivers,                              # allocated number of slivers, from data model
280                "siteUrl": str(site.site_url),
281                "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0),
282                "load": max_cpu,
283                "hot": float(hotness)}
284         new_rows[str(site.name)] = new_row
285
286     # get rid of sites with 0 slivers that overlap other sites with >0 slivers
287     for (k,v) in new_rows.items():
288         bad=False
289         if v["numHPCSlivers"]==0:
290             for v2 in new_rows.values():
291                 if (v!=v2) and (v2["numHPCSlivers"]>=0):
292                     d = haversine(v["lat"],v["long"],v2["lat"],v2["long"])
293                     if d<100:
294                          bad=True
295             if bad:
296                 del new_rows[k]
297
298     return new_rows
299
300 class SimulatorView(View):
301     def get(self, request, **kwargs):
302         sim = json.loads(file("/tmp/simulator.json","r").read())
303         text = "<html><head></head><body>"
304         text += "Iteration: %d<br>" % sim["iteration"]
305         text += "Elapsed since report %d<br><br>" % sim["elapsed_since_report"]
306         text += "<table border=1>"
307         text += "<tr><th>site</th><th>trend</th><th>weight</th><th>bytes_sent</th><th>hot</th></tr>"
308         for site in sim["site_load"].values():
309             text += "<tr>"
310             text += "<td>%s</td><td>%0.2f</td><td>%0.2f</td><td>%d</td><td>%0.2f</td>" % \
311                         (site["name"], site["trend"], site["weight"], site["bytes_sent"], site["load_frac"])
312             text += "</tr>"
313         text += "</table>"
314         text += "</body></html>"
315         return HttpResponse(text)
316
317 class DashboardUserSiteView(View):
318     def get(self, request, **kwargs):
319         return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
320
321 class TenantViewData(View):
322     def get(self, request, **kwargs):
323         return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript')
324
325 def haversine(site_lat, site_lon, lat, lon):
326     site_lat = float(site_lat)
327     site_lon = float(site_lon)
328     lat = float(lat)
329     lon = float(lon)
330     d=0
331     if lat and lon and site_lat and site_lon:
332         R = 6378.1
333         a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2)
334         c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) )
335         d = R * c
336
337     return d
338
339 def siteSortKey(site, slice=None, count=None, lat=None, lon=None):
340     # try to pick a site we're already using
341     has_slivers_here=False
342     if slice:
343         for sliver in slice.slivers.all():
344             if sliver.node.site.name == site.name:
345                 has_slivers_here=True
346
347     # Haversine method
348     d = haversine(site.location.latitude, site.location.longitude, lat, lon)
349
350     return (-has_slivers_here, d)
351
352 def tenant_pick_sites(user, user_ip=None, slice=None, count=None):
353     """ Returns list of sites, sorted from most favorable to least favorable """
354     lat=None
355     lon=None
356     try:
357         client_geo = GeoIP().city(user_ip)
358         if client_geo:
359             lat=float(client_geo["latitude"])
360             lon=float(client_geo["longitude"])
361     except:
362         print "exception in geo code"
363         traceback.print_exc()
364
365     sites = Site.objects.all()
366     sites = [x for x in sites if x.name in BLESSED_SITES]
367     sites = sorted(sites, key=functools.partial(siteSortKey, slice=slice, count=count, lat=lat, lon=lon))
368
369     return sites
370
371 def slice_increase_slivers(user, user_ip, siteList, slice, count, noAct=False):
372     sitesChanged = {}
373
374     # let's compute how many slivers are in use in each node of each site
375     for site in siteList:
376         site.nodeList = list(site.nodes.all())
377         for node in site.nodeList:
378             node.sliverCount = 0
379             for sliver in node.slivers.all():
380                  if sliver.slice.id == slice.id:
381                      node.sliverCount = node.sliverCount + 1
382
383     # Allocate slivers to nodes
384     # for now, assume we want to allocate all slivers from the same site
385     nodes = siteList[0].nodeList
386     while (count>0):
387         # Sort the node list by number of slivers per node, then pick the
388         # node with the least number of slivers.
389         nodes = sorted(nodes, key=attrgetter("sliverCount"))
390         node = nodes[0]
391
392         print "adding sliver at node", node.name, "of site", node.site.name
393
394         if not noAct:
395             sliver = Sliver(name=node.name,
396                         slice=slice,
397                         node=node,
398                         image = Image.objects.all()[0],
399                         creator = User.objects.get(email=user),
400                         deploymentNetwork=node.deployment,
401                         numberCores =1 )
402             sliver.save()
403
404         node.sliverCount = node.sliverCount + 1
405
406         count = count - 1
407
408         sitesChanged[node.site.name] = sitesChanged.get(node.site.name,0) + 1
409
410     return sitesChanged
411
412 def slice_decrease_slivers(user, siteList, slice, count, noAct=False):
413     sitesChanged = {}
414     sliverList ={}
415     if siteList:
416         siteNames = [site.name for site in siteList]
417     else:
418         siteNames = None
419
420     for sliver in slice.slivers.all():
421         if(not siteNames) or (sliver.node.site.name in siteNames):\r
422                 node = sliver.node\r
423                 sliverList[sliver.name]=node.name
424
425     for key in sliverList:
426         if count>0:
427             sliver = Sliver.objects.filter(name=key)[0]\r
428             sliver.delete()\r
429             print "deleting sliver",sliverList[key],"at node",sliver.node.name\r
430             count=count-1\r
431             sitesChanged[sliver.node.site.name] = sitesChanged.get(sliver.node.site.name,0) - 1\r
432 \r
433     return sitesChanged
434
435 class TenantDeleteSliceView(View):
436         def post(self,request):\r
437                 sliceName = request.POST.get("sliceName",None)\r
438                 slice = Slice.objects.get(name=sliceName)\r
439                 print slice, slice.id\r
440                 sliceToDel=Slice(name=sliceName, id=slice.id)\r
441                 sliceToDel.delete()
442
443 class TenantAddOrRemoveSliverView(View):
444     """ Add or remove slivers from a Slice
445
446         Arguments:
447             siteName - name of site. If not specified, PlanetStack will pick the
448                        best site.,
449             actionToDo - [add | rem]
450             count - number of slivers to add or remove
451             sliceName - name of slice
452             noAct - if set, no changes will be made to db, but result will still
453                     show which sites would have been modified.
454
455         Returns:
456             Dictionary of sites that were modified, and the count of nodes
457             that were added or removed at each site.
458     """
459     def post(self, request, *args, **kwargs):
460         siteName = request.POST.get("siteName", None)
461         actionToDo = request.POST.get("actionToDo", None)
462         count = int(request.POST.get("count","0"))
463         sliceName = request.POST.get("slice", None)
464         noAct = request.POST.get("noAct", False)
465
466         if not sliceName:
467             return HttpResponseServerError("No slice name given")
468
469         slice = Slice.objects.get(name=sliceName)
470
471         if siteName:
472             siteList = [Site.objects.get(name=siteName)]
473         else:
474             siteList = None
475
476         if (actionToDo == "add"):
477             user_ip = request.GET.get("ip", get_ip(request))
478             if (siteList is None):
479                 siteList = tenant_pick_sites(user, user_ip, slice, count)
480
481             sitesChanged = slice_increase_slivers(request.user, user_ip, siteList, slice, count, noAct)
482         elif (actionToDo == "rem"):
483             sitesChanged = slice_decrease_slivers(request.user, siteList, slice, count, noAct)
484         else:
485             return HttpResponseServerError("Unknown actionToDo %s" % actionToDo)
486
487         return HttpResponse(json.dumps(sitesChanged), mimetype='application/javascript')
488
489     def get(self, request, *args, **kwargs):
490         request.POST = request.GET
491         return self.post(request, *args, **kwargs)  # for testing REST in browser
492         #return HttpResponseServerError("GET is not supported")
493
494 class TenantPickSitesView(View):
495     """ primarily just for testing purposes """
496     def get(self, request, *args, **kwargs):
497         count = request.GET.get("count","0")
498         slice = request.GET.get("slice",None)
499         if slice:
500             slice = Slice.objects.get(name=slice)
501         ip = request.GET.get("ip", get_ip(request))
502         sites = tenant_pick_sites(request.user, user_ip=ip, count=0, slice=slice)
503         sites = [x.name for x in sites]
504         return HttpResponse(json.dumps(sites), mimetype='application/javascript')
505
506 class DashboardSummaryAjaxView(View):
507     def get(self, request, **kwargs):
508         def avg(x):
509             return float(sum(x))/len(x)
510
511         sites = getCDNOperatorData().values()
512
513         sites = [site for site in sites if site["numHPCSlivers"]>0]
514
515         total_slivers = sum( [site["numHPCSlivers"] for site in sites] )
516         total_bandwidth = sum( [site["bandwidth"] for site in sites] )
517         average_cpu = int(avg( [site["load"] for site in sites] ))
518
519         result= {"total_slivers": total_slivers,
520                 "total_bandwidth": total_bandwidth,
521                 "average_cpu": average_cpu}
522
523         return HttpResponse(json.dumps(result), mimetype='application/javascript')
524
525 class DashboardAddOrRemoveSliverView(View):
526     # TODO: deprecate this view in favor of using TenantAddOrRemoveSliverView
527
528     def post(self, request, *args, **kwargs):
529         siteName = request.POST.get("site", None)
530         actionToDo = request.POST.get("actionToDo", "0")
531
532         siteList = [Site.objects.get(name=siteName)]
533         slice = Slice.objects.get(name="HyperCache")
534
535         if (actionToDo == "add"):
536             user_ip = request.GET.get("ip", get_ip(request))
537             slice_increase_slivers(request.user, user_ip, siteList, slice, 1)
538         elif (actionToDo == "rem"):
539             slice_decrease_slivers(request.user, siteList, slice, 1)
540
541         print '*' * 50
542         print 'Ask for site: ' + siteName + ' to ' + actionToDo + ' another HPC Sliver'
543         return HttpResponse('This is POST request ')
544
545 class DashboardAjaxView(View):
546     def get(self, request, **kwargs):
547         return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
548
549 class DashboardAnalyticsAjaxView(View):
550     def get(self, request, name="hello_world", **kwargs):
551         if (name == "hpcSummary"):
552             return HttpResponse(json.dumps(hpc_wizard.get_hpc_wizard().get_summary_for_view()), mimetype='application/javascript')
553         elif (name == "hpcUserSite"):
554             return HttpResponse(json.dumps(getUserSliceInfo(request.user, True)), mimetype='application/javascript')
555         elif (name == "hpcMap"):
556             return HttpResponse(json.dumps(getCDNOperatorData(True)), mimetype='application/javascript')
557         elif (name == "bigquery"):
558             (mimetype, data) = DoPlanetStackAnalytics(request)
559             return HttpResponse(data, mimetype=mimetype)
560         else:
561             return HttpResponse(json.dumps("Unknown"), mimetype='application/javascript')
562