From: Scott Baker Date: Tue, 22 Apr 2014 00:32:09 +0000 (-0700) Subject: include sites with no nodes in CDN operator data, do not wait on first CDN op page... X-Git-Url: http://git.onelab.eu/?a=commitdiff_plain;h=75095b6b5db8afd9628517d33eed7731967f3232;p=plstackapi.git include sites with no nodes in CDN operator data, do not wait on first CDN op page load, prevent sites with zero nodes from hiding other sites --- diff --git a/planetstack/core/plus/views.py b/planetstack/core/plus/views.py index 38a7a96..2b015ef 100644 --- a/planetstack/core/plus/views.py +++ b/planetstack/core/plus/views.py @@ -24,7 +24,7 @@ if os.path.exists("/home/smbaker/projects/vicci/cdn/bigquery"): else: sys.path.append("/opt/planetstack/hpc_wizard") import hpc_wizard -from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics +from planetstack_analytics import DoPlanetStackAnalytics, PlanetStackAnalytics, RED_LOAD, BLUE_LOAD class DashboardWelcomeView(TemplateView): template_name = 'admin/dashboard/welcome.html' @@ -53,7 +53,7 @@ def getUserSliceInfo(user, tableFormat = False): userDetails['userSliceInfo'] = userSliceTableFormatter(userSliceData) else: userDetails['userSliceInfo'] = userSliceData - userDetails['cdnData'] = getCDNOperatorData(); + userDetails['cdnData'] = getCDNOperatorData(wait=False); # pprint( userDetails) return userDetails @@ -201,32 +201,66 @@ def getSliceInfo(user): return userSliceInfo -def getCDNOperatorData(randomizeData = False): +def getCDNOperatorData(randomizeData = False, wait=True): + HPC_SLICE_NAME = "HyperCache" + bq = PlanetStackAnalytics() - #hpc_sliceNames = bq.service_to_sliceNames("HPC Service") + rows = bq.get_cached_query_results(bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]), wait) - rows = bq.get_cached_query_results(bq.compose_latest_query()) + # if wait==False, then we could have no rows yet - #rows = [x for x in rows if x.get("slice","") in hpc_sliceNames] + stats_rows = {} + if rows: + rows = bq.postprocess_results(rows, filter={"slice": HPC_SLICE_NAME}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80) - rows = bq.postprocess_results(rows, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time","site"], maxDeltaTime=80) + # dictionaryize the statistics rows by site name + stats_rows = {} + for row in rows: + stats_rows[row["site"]] = row - bq.merge_datamodel_sites(rows, slice="HyperCache") + slice = Slice.objects.get(name=HPC_SLICE_NAME) + slice_slivers = list(slice.slivers.all()) new_rows = {} - for row in rows: - new_row = {"lat": float(row.get("lat", 0)), - "long": float(row.get("long", 0)), + for site in Site.objects.all(): + # compute number of slivers allocated in the data model + allocated_slivers = 0 + for sliver in slice_slivers: + if sliver.node.site == site: + allocated_slivers = allocated_slivers + 1 + + stats_row = stats_rows.get(site.name,{}) + + max_cpu = stats_row.get("max_avg_cpu", stats_row.get("max_cpu",0)) + cpu=float(max_cpu)/100.0 + hotness = max(0.0, ((cpu*RED_LOAD) - BLUE_LOAD)/(RED_LOAD-BLUE_LOAD)) + + # format it to what that CDN Operations View is expecting + new_row = {"lat": float(site.location.longitude), + "long": float(site.location.longitude), + "lat": float(site.location.latitude), "health": 0, - "numNodes": int(row.get("numNodes",0)), - "activeHPCSlivers": int(row.get("count_hostname", 0)), - "numHPCSlivers": int(row.get("allocated_slivers", 0)), - "siteUrl": str(row.get("url", "")), - "hot": float(row.get("hotness", 0.0)), - "bandwidth": row.get("sum_computed_bytes_sent_div_elapsed",0), - "load": int(float(row.get("max_avg_cpu", row.get("max_cpu",0))))} - new_rows[str(row["site"])] = new_row + "numNodes": int(site.nodes.count()), + "activeHPCSlivers": int(stats_row.get("count_hostname", 0)), # measured number of slivers, from bigquery statistics + "numHPCSlivers": allocated_slivers, # allocated number of slivers, from data model + "siteUrl": str(site.site_url), + "bandwidth": stats_row.get("sum_computed_bytes_sent_div_elapsed",0), + "load": max_cpu, + "hot": float(hotness)} + new_rows[str(site.name)] = new_row + + # get rid of sites with 0 slivers that overlap other sites with >0 slivers + for (k,v) in new_rows.items(): + bad=False + if v["numHPCSlivers"]==0: + for v2 in new_rows.values(): + if (v!=v2) and (v2["numHPCSlivers"]>=0): + d = haversine(v["lat"],v["long"],v2["lat"],v2["long"]) + if d<100: + bad=True + if bad: + del new_rows[k] return new_rows @@ -255,6 +289,20 @@ class TenantViewData(View): def get(self, request, **kwargs): return HttpResponse(json.dumps(getTenantSliceInfo(request.user, True)), mimetype='application/javascript') +def haversine(site_lat, site_lon, lat, lon): + site_lat = float(site_lat) + site_lon = float(site_lon) + lat = float(lat) + lon = float(lon) + d=0 + if lat and lon and site_lat and site_lon: + R = 6378.1 + a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2) + c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) ) + d = R * c + + return d + def siteSortKey(site, slice=None, count=None, lat=None, lon=None): # try to pick a site we're already using has_slivers_here=False @@ -264,16 +312,7 @@ def siteSortKey(site, slice=None, count=None, lat=None, lon=None): has_slivers_here=True # Haversine method - d = 0 - site_lat = site.location.latitude - site_lon = site.location.longitude - if lat and lon and site_lat and site_lon: - site_lat = float(site_lat) - site_lon = float(site_lon) - R = 6378.1 - a = math.sin( math.radians((lat - site_lat)/2.0) )**2 + math.cos( math.radians(lat) )*math.cos( math.radians(site_lat) )*(math.sin( math.radians((lon - site_lon)/2.0 ) )**2) - c = 2 * math.atan2( math.sqrt(a), math.sqrt(1 - a) ) - d = R * c + d = haversine(site.location.latitude, site.location.longitude, lat, lon) return (-has_slivers_here, d) diff --git a/planetstack/hpc_wizard/planetstack_analytics.py b/planetstack/hpc_wizard/planetstack_analytics.py index 5287e2f..2afacb3 100644 --- a/planetstack/hpc_wizard/planetstack_analytics.py +++ b/planetstack/hpc_wizard/planetstack_analytics.py @@ -38,8 +38,9 @@ class PlanetStackAnalytics(BigQueryAnalytics): return [slice.name for slice in slices] - def compose_query(self, slice=None, site=None, node=None, service=None, timeBucket="60", avg=[], sum=[], count=[], computed=[], val=[], groupBy=["Time"], orderBy=["Time"], tableName="demoevents", latest=False): - tablePart = "[%s.%s@-3600000--1]" % ("vicci", tableName) + def compose_query(self, slice=None, site=None, node=None, service=None, timeBucket="60", avg=[], sum=[], count=[], computed=[], val=[], groupBy=["Time"], orderBy=["Time"], tableName="demoevents", latest=False, max_age=60*60): + max_age = max_age * 1000 + tablePart = "[%s.%s@-%d--1]" % ("vicci", tableName, max_age) fields = [] fieldNames = [] @@ -256,7 +257,7 @@ class PlanetStackAnalytics(BigQueryAnalytics): return base_query - def get_cached_query_results(self, q): + def get_cached_query_results(self, q, wait=True): global glo_cached_queries if q in glo_cached_queries: @@ -264,6 +265,9 @@ class PlanetStackAnalytics(BigQueryAnalytics): print "using cached query" return glo_cached_queries[q]["rows"] + if not wait: + return None + print "refreshing cached query" result = self.run_query(q) glo_cached_queries[q] = {"time": time.time(), "rows": result} @@ -389,24 +393,26 @@ def DoPlanetStackAnalytics(request): def main(): bq = PlanetStackAnalytics() - q = bq.compose_latest_query() + q = bq.compose_latest_query(groupByFields=["%hostname", "event", "%slice"]) results = bq.run_query(q) - results = bq.postprocess_results(results, - #filter={"site": "Princeton"}, - groupBy=["site"], - computed=["bytes_sent/elapsed"], - sum=["bytes_sent", "computed_bytes_sent_div_elapsed"], avg=["cpu"], - maxDeltaTime=60) + #results = bq.postprocess_results(results, + # filter={"slice": "HyperCache"}, + # groupBy=["site"], + # computed=["bytes_sent/elapsed"], + # sum=["bytes_sent", "computed_bytes_sent_div_elapsed"], avg=["cpu"], + # maxDeltaTime=60) + + results = bq.postprocess_results(results, filter={"slice": "HyperCache"}, maxi=["cpu"], count=["hostname"], computed=["bytes_sent/elapsed"], groupBy=["Time", "site"], maxDeltaTime=80) bq.dump_table(results) + sys.exit(0) + q=bq.compose_query(sum=["%bytes_sent"], avg=["%cpu"], latest=True, groupBy=["Time", "%site"]) print q bq.dump_table(bq.run_query(q)) - sys.exit(0) - q=bq.compose_query(avg=["%cpu","%bandwidth"], count=["%hostname"], slice="HyperCache") print q bq.dump_table(bq.run_query(q))