Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py (revision 2463) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py (working copy) @@ -59,7 +59,7 @@ url(r'^composer_config/(?P\w+)', composer_config), url(r'^composer_status/(?P\w+)', composer_status), url(r'^composer_query$', composer_query), - url(r'^llb/(?P\w+)$', handle_llb_stats_req), + url(r'^llb/(?P.*)$', handle_llb_stats_req), url(r'^log/(?P\w+)$', handle_log_location_app), url(r'^real_service$', real_service), url(r'^rs_block$', rs_block), Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (revision 2467) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (working copy) @@ -48,27 +48,27 @@ except ge.GenericError as e: raise e except ge.ANInternalException as exp: - return HttpResponse(json.dumps({ + return json.dumps({ 'parm_name': exp.parm_name, 'error': exp.error_code, 'message': exp.description, 'data': None - }), content_type='application/json', status=exp.error_code) + }) except Exception as e: - return HttpResponse(json.dumps({ + return json.dumps({ 'error': 500, 'message': "Request Composer_ui Error", 'data': str(e) - }), content_type='application/json') + }) else: if response_data['status'] == 200: - return HttpResponse(response_data['body'], content_type='application/json') + return response_data['body'] else: - return HttpResponse(json.dumps({ + return json.dumps({ 'error': response_data['status'], 'message': response_data['reason'], 'data': response_data['body'] - }), content_type='application/json') + }) finally: if http_client: http_client.close() @@ -76,7 +76,6 @@ @staticmethod def get_llb_stats_query_str(request): """Constructs the query for LLB stats""" - andebug('an.model.cli', 'Inside get_llb_stats_query_str') req_dict = { "url": LLBStatsDB.BASE_URL, "path": "db=composer&epoch=ms", @@ -84,21 +83,28 @@ "body": {}, "config": LLBStatsDB.CONTENT_TYPE } - andebug('an.model.cli', 'Request query: {}'.format(request)) query_string = ( 'SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) ' 'AS linkUsage, last(linkBandwidIn) * 8 AS received, last(linkBandwidOut) * 8 AS sent, ' 'last(linkConn) AS linkConn, last(linkHits) AS linkHits FROM llbStats WHERE time > now()-20s ' - 'GROUP BY agent_host, linkName, linkStatus;') + 'GROUP BY agent_host, linkName, linkStatus FILL(0);') - if 'query' in request and 'agent_host' in request['query']: - query_string = ('SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh,' - ' last(linkUsage) AS linkUsage, last(linkBandwidIn) * 8 AS received,' - ' last(linkBandwidOut) * 8 AS sent, last(linkConn) AS linkConn,' - ' last(linkHits) AS linkHits FROM llbStats WHERE agent_host =~ {}' - ' AND time > now()-20s GROUP BY agent_host, linkName, linkStatus;').format( - request['query']['agent_host']) + if 'query' in request: + query_params = request.get('query', {}) + agent_host = query_params.get('agent_host') + if agent_host: + query_string = ( + 'SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) AS linkUsage, ' + 'last(linkBandwidIn) * 8 AS received, last(linkBandwidOut) * 8 AS sent, last(linkConn) AS linkConn, ' + 'last(linkHits) AS linkHits FROM llbStats WHERE agent_host =~ \'{}\' ' + 'AND time > now()-20s GROUP BY agent_host, linkName, linkStatus FILL(0);' + ).format(agent_host) + else: + oper_log('error', 'system', + "Query must contain only agent_host details. Please specify agent_host in the query.") + raise ge.GenericError(400, + "Query must contain only agent_host details. Please specify agent_host in the query.") query_string = LLBStatsDB.format_query_string(query_string) req_dict["body"] = 'q=' + query_string @@ -107,7 +113,6 @@ @staticmethod def get_llb_monitoring_query_str(request): """Constructs the query for LLB monitoring data""" - andebug('an.model.cli', 'Inside get_llb_stats_query_str') req_dict = { "url": LLBStatsDB.BASE_URL, "path": "db=composer&epoch=ms", @@ -115,7 +120,6 @@ "body": {}, "config": LLBStatsDB.CONTENT_TYPE } - andebug('an.model.cli', 'Request query: {}'.format(request)) query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' 'linkHits) AS hits FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s; ' @@ -124,7 +128,7 @@ 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (select ' 'last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, last(linkBandwidIn) * 8 as ' 'received, last(linkBandwidOut) * 8 as sent from llbStats group by linkName, agent_host) ' - 'where time > now()-20s;') + 'where time > now()-20s FILL(0);') if 'query' in request and 'agent_host' in request['query']: query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' @@ -135,7 +139,7 @@ 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (' 'select last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, ' 'last(linkBandwidIn) * 8 as received, last(linkBandwidOut) * 8 as sent from llbStats GROUP' - 'BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s;' + 'BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s FILL(0);' ).format(request['query']['agent_host'], request['query']['agent_host'], request['query']['agent_host']) @@ -144,9 +148,8 @@ return req_dict @staticmethod - def get_llb_graphs_data_str(request): + def get_llb_historical_data_str(request): """Constructs the query for LLB monitoring data""" - andebug('an.model.cli', 'Inside get_llb_graphs_data_str') req_dict = { "url": LLBStatsDB.BASE_URL, "path": "db=composer&epoch=ms", @@ -154,7 +157,8 @@ "body": {}, "config": LLBStatsDB.CONTENT_TYPE } - if 'query' in request and 'from_time' in request['query'] and 'to_time' in request['query'] and 'link_name' in request['query']: + if 'query' in request and 'from_time' in request['query'] and 'to_time' in request['query'] and 'link_name' in \ + request['query']: from_time = request['query'].get('from_time', '') to_time = request['query'].get('to_time', '') link_name = request['query'].get('link_name', '') @@ -162,11 +166,7 @@ oper_log('error', 'system', "Query cannot be empty. Please specify from_time and to_time and link_name in the query.") raise ge.GenericError(400, "Query cannot be empty. Please specify from_time and to_time and link_name in " - "the query.") - - andebug('an.model.cli', 'From Time: {}'.format(from_time)) - andebug('an.model.cli', 'To Time: {}'.format(to_time)) - andebug('an.model.cli', 'Link Name: {}'.format(link_name)) + "the query.") # time_interval specifies the frequency at which data points should be generated. time_interval = LLBStatsDB.get_time_interval(from_time, to_time) @@ -178,26 +178,23 @@ # Ensure the to_time is properly stripped and lowercased for comparison. to_time = to_time.strip().lower() if isinstance(to_time, int): - andebug('an.model.cli', 'Inside now str if') query_string = ( "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} and time < {} and linkName = '{}' GROUP BY time({}), agent_host, linkStatus FILL(0);" + "llbStats WHERE time > {} and time < {} and linkName = '{}' GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" ).format(from_time, to_time, link_name, time_interval) else: - andebug('an.model.cli', 'Inside now str else') from_time = LLBStatsDB.get_time_in_influx_format(from_time) query_string = ( "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} and linkName = '{}' GROUP BY time({}), agent_host, linkStatus FILL(0);" + "llbStats WHERE time > {} and linkName = '{}' GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" ).format(from_time, link_name, time_interval) query_string = LLBStatsDB.format_query_string(query_string) - andebug('an.model.cli', 'Query String: {}'.format(query_string)) req_dict["body"] = 'q=' + query_string return req_dict @@ -213,14 +210,11 @@ try: from_time_in_int = int(from_time) # Try to convert to integer if it's in epoch ms format except ValueError: - andebug('an.model.cli', 'Invalid from_time format') oper_log('error', 'system', "From time is not specified in correct format.") raise ge.GenericError(400, "From time is not specified in correct format.") else: from_time_in_int = curr_time_in_ms - from_time_in_int - andebug('an.model.cli', 'From Time: {}'.format(from_time_in_int)) - if to_time == "now": to_time_in_int = curr_time_in_ms # If "now", use current time else: @@ -234,8 +228,6 @@ else: from_time_in_int = curr_time_in_ms - from_time_in_int - andebug('an.model.cli', 'To Time: {}'.format(to_time_in_int)) - # Calculate the absolute difference in time diff_time = abs(to_time_in_int - from_time_in_int) @@ -278,7 +270,6 @@ # Check if the given time string exists in the mappings if from_time in time_mappings: - andebug('an.model.cli', 'Matched time string: {}'.format(from_time)) return time_mappings[from_time] else: return -1 @@ -308,7 +299,6 @@ # Check if the given time string exists in the mappings if from_time in time_mappings: - andebug('an.model.cli', 'Matched time string: {}'.format(from_time)) return time_mappings[from_time] else: return -1 Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py (revision 2467) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py (working copy) @@ -3,16 +3,18 @@ from hive.db.llb_stats_queries import LLBStatsDB from hive.custom_exceptions import generic_exception as ge from cm.lib.libbasic_operation import oper_log +from hive.utils import andebug -def handle_llb_stats_req(request, app): +def handle_llb_stats_req(request, path=None): try: - if app == 'stats' or app is None and request.method == 'POST': - return get_llb_stats(request) - elif app == 'monitoring_data' and request.method == 'POST': + if path.startswith('stats') and request.method == 'POST': + if path == 'stats': + return get_llb_stats(request) + elif path == 'stats/historical': + return get_llb_historical_data(request) + elif path == 'monitoring_data' and request.method == 'POST': return get_monitoring_data(request) - elif app == 'graph_data' and request.method == 'POST': - return get_llb_graphs_data(request) else: return HttpResponse(json.dumps({ 'error': 405, @@ -31,18 +33,134 @@ req = json.loads(request.body) req_dict = LLBStatsDB.get_llb_stats_query_str(req) response = LLBStatsDB.execute_db_query(req_dict) - return response + + if not response: + return [] + + if isinstance(response, str): + response = json.loads(response) + + response_data = reformat_data_from_db(response, 'stats') + + if response_data: + return HttpResponse(json.dumps(response_data), content_type='application/json') + else: + return HttpResponse(json.dumps(response), content_type='application/json') def get_monitoring_data(request): req = json.loads(request.body) req_dict = LLBStatsDB.get_llb_monitoring_query_str(req) response = LLBStatsDB.execute_db_query(req_dict) - return response + if not response: + return [] + + if isinstance(response, str): + response = json.loads(response) + + response_data = reformat_data_from_db(response, 'monitoring_data') + + if response_data: + return HttpResponse(json.dumps(response_data), content_type='application/json') + else: + return HttpResponse(json.dumps(response), content_type='application/json') -def get_llb_graphs_data(request): +def get_llb_historical_data(request): req = json.loads(request.body) - req_dict = LLBStatsDB.get_llb_graphs_data_str(req) + req_dict = LLBStatsDB.get_llb_historical_data_str(req) response = LLBStatsDB.execute_db_query(req_dict) - return response + + # Handle cases where response is None or not JSON + if not response: + return [] + + if isinstance(response, str): + response = json.loads(response) + + response_data = reformat_data_from_db(response, 'historical') + + if response_data: + return HttpResponse(json.dumps(response_data), content_type='application/json') + else: + if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): + return HttpResponse(json.dumps({ + "error": "No valid data found" + }), content_type='application/json') + else: + return HttpResponse(json.dumps(response), content_type='application/json') + + +def reformat_data_from_db(response, data_type): + response_data = [] + results = response.get("results") + # Extract the relevant series from the results + for result in results: + if result is None: + pass + series = result.get("series") + if series: + for series_data in series: + series_obj = {} + columns = series_data.get("columns", []) + values = series_data.get("values", []) + tags = series_data.get("tags") + if tags: + series_obj.update(tags) # Add tag values directly to the response + attributes = list(map(str, columns)) + series_obj['attributes'] = attributes + + # Get column indices for easier lookup + col_indices = {col: idx for idx, col in enumerate(columns)} + + # Process each data point + units = [] + for value in values: + if data_type == 'historical': + unit = get_llb_historical_col_values(col_indices, value) + elif data_type == 'monitoring_data': + unit = get_llb_monitoring_col_values(col_indices, value) + elif data_type == 'stats': + unit = get_llb_stats_col_values(col_indices, value) + + units.append(unit) + + series_obj['values'] = units + response_data.append(series_obj) + + return response_data + + +def get_llb_historical_col_values(col_indices, value): + return [value[col_indices["time"]] if "time" in col_indices else 0, + round(value[col_indices["avgBandwidIn"]], + 2) if "avgBandwidIn" in col_indices else 0, + round(value[col_indices["avgBandwidOut"]], + 2) if "avgBandwidOut" in col_indices else 0, + round(value[col_indices["avgLinkUsage"]], + 2) if "avgLinkUsage" in col_indices else 0, + int(value[col_indices["avgLinkConn"]]) if "avgLinkConn" in col_indices else 0, + int(value[col_indices["avgLinkHits"]]) if "avgLinkHits" in col_indices else 0] + + +def get_llb_monitoring_col_values(col_indices, value): + return [value[col_indices["time"]] if "time" in col_indices else 0] + [ + value[col_indices[key]] for key in [ + "top_hits", "top_conn", "total_throughput", "received", "sent", "linkName", "agent_host" + ] if key in col_indices + ] + + +def get_llb_stats_col_values(col_indices, value): + return [value[col_indices["time"]] if "time" in col_indices else 0, + value[col_indices["linkGateway"]] if "linkGateway" in col_indices else 0, + round(value[col_indices["linkThresh"]], + 2) if "linkThresh" in col_indices else 0, + round(value[col_indices["linkUsage"]], + 2) if "linkUsage" in col_indices else 0, + round(value[col_indices["received"]], + 2) if "received" in col_indices else 0, + round(value[col_indices["sent"]], + 2) if "sent" in col_indices else 0, + int(value[col_indices["linkConn"]]) if "linkConn" in col_indices else 0, + int(value[col_indices["linkHits"]]) if "linkHits" in col_indices else 0]