Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (revision 2463) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (working copy) @@ -6,6 +6,7 @@ from djproject.an_settings import CMDATA from hive.custom_exceptions import generic_exception as ge from hive.utils import andebug +from cm.lib.libbasic_operation import oper_log class LLBStatsDB: @@ -44,6 +45,8 @@ 'reason': response.reason, 'body': response.read(), } + except ge.GenericError as e: + raise e except ge.ANInternalException as exp: return HttpResponse(json.dumps({ 'parm_name': exp.parm_name, @@ -81,18 +84,21 @@ "body": {}, "config": LLBStatsDB.CONTENT_TYPE } - query_string = ('SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) ' - 'AS linkUsage, last(linkBandwidIn) AS linkBandwidIn, last(linkBandwidOut) AS linkBandwidOut, ' - 'last(linkConn) AS linkConn, last(linkHits) AS linkHits FROM llbStats WHERE time > now()-20s ' - 'GROUP BY agent_host, linkName, linkStatus;') andebug('an.model.cli', 'Request query: {}'.format(request)) + query_string = ( + 'SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) ' + 'AS linkUsage, last(linkBandwidIn) * 8 AS received, last(linkBandwidOut) * 8 AS sent, ' + 'last(linkConn) AS linkConn, last(linkHits) AS linkHits FROM llbStats WHERE time > now()-20s ' + 'GROUP BY agent_host, linkName, linkStatus;') + if 'query' in request and 'agent_host' in request['query']: - query_string = ('SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) ' - 'AS linkUsage, last(linkBandwidIn) AS linkBandwidIn, last(linkBandwidOut) AS ' - 'linkBandwidOut, last(linkConn) AS linkConn, last(linkHits) AS linkHits FROM llbStats ' - 'WHERE agent_host =~ {} AND time > now()-20s GROUP BY agent_host, linkName, linkStatus;' - ).format(request['query']['agent_host']) + query_string = ('SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh,' + ' last(linkUsage) AS linkUsage, last(linkBandwidIn) * 8 AS received,' + ' last(linkBandwidOut) * 8 AS sent, last(linkConn) AS linkConn,' + ' last(linkHits) AS linkHits FROM llbStats WHERE agent_host =~ {}' + ' AND time > now()-20s GROUP BY agent_host, linkName, linkStatus;').format( + request['query']['agent_host']) query_string = LLBStatsDB.format_query_string(query_string) req_dict["body"] = 'q=' + query_string @@ -109,24 +115,227 @@ "body": {}, "config": LLBStatsDB.CONTENT_TYPE } - query_string = ('SELECT TOP(hits, 5), linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(linkHits) AS hits ' - 'FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s; ' - 'SELECT TOP(last, 5), linkName, agent_host FROM (SELECT LAST(linkConn) FROM llbStats GROUP BY ' - 'linkName, agent_host) WHERE time > now()-20s;') andebug('an.model.cli', 'Request query: {}'.format(request)) + query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' + 'linkHits) AS hits FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s; ' + 'SELECT TOP(conn, 5) as top_conn, linkName, agent_host FROM (SELECT LAST(linkConn) as conn ' + 'FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s;' + 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (select ' + 'last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, last(linkBandwidIn) * 8 as ' + 'received, last(linkBandwidOut) * 8 as sent from llbStats group by linkName, agent_host) ' + 'where time > now()-20s;') + if 'query' in request and 'agent_host' in request['query']: - query_string = ('SELECT TOP(hits, 5), linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(linkHits) AS ' - 'hits FROM llbStats GROUP BY linkName, agent_host) WHERE agent_host =~ {} and time > ' - 'now()-20s; SELECT TOP(last, 5), linkName, agent_host FROM (SELECT LAST(linkConn) FROM ' + query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' + 'linkHits) AS hits FROM llbStats GROUP BY linkName, agent_host) WHERE agent_host =~ {} ' + 'and time >now()-20s;' + 'SELECT TOP(last, 5) as top_conn, linkName, agent_host FROM (SELECT LAST(linkConn) FROM ' 'llbStats GROUP BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s;' - ).format(request['query']['agent_host'], request['query']['agent_host']) + 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (' + 'select last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, ' + 'last(linkBandwidIn) * 8 as received, last(linkBandwidOut) * 8 as sent from llbStats GROUP' + 'BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s;' + ).format(request['query']['agent_host'], request['query']['agent_host'], + request['query']['agent_host']) + + query_string = LLBStatsDB.format_query_string(query_string) + req_dict["body"] = 'q=' + query_string + return req_dict + + @staticmethod + def get_llb_graphs_data_str(request): + """Constructs the query for LLB monitoring data""" + andebug('an.model.cli', 'Inside get_llb_graphs_data_str') + req_dict = { + "url": LLBStatsDB.BASE_URL, + "path": "db=composer&epoch=ms", + "method": "POST", + "body": {}, + "config": LLBStatsDB.CONTENT_TYPE + } + if 'query' in request and 'from_time' in request['query'] and 'to_time' in request['query'] and 'link_name' in request['query']: + from_time = request['query'].get('from_time', '') + to_time = request['query'].get('to_time', '') + link_name = request['query'].get('link_name', '') + else: + oper_log('error', 'system', + "Query cannot be empty. Please specify from_time and to_time and link_name in the query.") + raise ge.GenericError(400, "Query cannot be empty. Please specify from_time and to_time and link_name in " + "the query.") + + andebug('an.model.cli', 'From Time: {}'.format(from_time)) + andebug('an.model.cli', 'To Time: {}'.format(to_time)) + andebug('an.model.cli', 'Link Name: {}'.format(link_name)) + + # time_interval specifies the frequency at which data points should be generated. + time_interval = LLBStatsDB.get_time_interval(from_time, to_time) + + if time_interval < 0: + oper_log('error', 'system', "Time Interval or Time Period is not specified properly.") + raise ge.GenericError(400, "Time Interval or Time Period not specified properly.") + + # Ensure the to_time is properly stripped and lowercased for comparison. + to_time = to_time.strip().lower() + if isinstance(to_time, int): + andebug('an.model.cli', 'Inside now str if') + + query_string = ( + "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " + "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " + "llbStats WHERE time > {} and time < {} and linkName = '{}' GROUP BY time({}), agent_host, linkStatus FILL(0);" + ).format(from_time, to_time, link_name, time_interval) + + + else: + andebug('an.model.cli', 'Inside now str else') + from_time = LLBStatsDB.get_time_in_influx_format(from_time) + query_string = ( + "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " + "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " + "llbStats WHERE time > {} and linkName = '{}' GROUP BY time({}), agent_host, linkStatus FILL(0);" + ).format(from_time, link_name, time_interval) query_string = LLBStatsDB.format_query_string(query_string) + andebug('an.model.cli', 'Query String: {}'.format(query_string)) req_dict["body"] = 'q=' + query_string + return req_dict @staticmethod + def get_time_interval(from_time, to_time): + curr_time_in_ms = LLBStatsDB.get_curr_sys_time() + + # Handle from_time (same logic as before) + from_time_in_int = LLBStatsDB.get_time_in_ms_for_str(from_time) + + if from_time_in_int == -1: + try: + from_time_in_int = int(from_time) # Try to convert to integer if it's in epoch ms format + except ValueError: + andebug('an.model.cli', 'Invalid from_time format') + oper_log('error', 'system', "From time is not specified in correct format.") + raise ge.GenericError(400, "From time is not specified in correct format.") + else: + from_time_in_int = curr_time_in_ms - from_time_in_int + + andebug('an.model.cli', 'From Time: {}'.format(from_time_in_int)) + + if to_time == "now": + to_time_in_int = curr_time_in_ms # If "now", use current time + else: + to_time_in_int = LLBStatsDB.get_time_in_ms_for_str(to_time) + if to_time_in_int == -1: + try: + to_time_in_int = int(to_time) # Try to convert to integer if it's in epoch ms format + except ValueError: + oper_log('error', 'system', "To time is not specified in correct format.") + raise ge.GenericError(400, "To time is not specified in correct format.") + else: + from_time_in_int = curr_time_in_ms - from_time_in_int + + andebug('an.model.cli', 'To Time: {}'.format(to_time_in_int)) + + # Calculate the absolute difference in time + diff_time = abs(to_time_in_int - from_time_in_int) + + # If there's a positive time difference, calculate the interval + if diff_time > 0: + diff_hours = diff_time / (1000 * 60 * 60) # Convert ms to hours + + if diff_hours > 24: + # More than 24 hours, divide by 96 for 15 minute intervals + return LLBStatsDB.convert_ms_to_influxdb_time(abs(diff_time / 96)) + else: + # Less than 24 hours, divide by 24 for hourly intervals + return LLBStatsDB.convert_ms_to_influxdb_time(abs(diff_time / 24)) + else: + # Return -1 for invalid or zero time difference + return -1 + + @staticmethod + def get_time_in_ms_for_str(from_time): + # Time in milliseconds for various time periods + time_mappings = { + "now-10m": 600000, # 10 minutes in ms + "now-15m": 900000, # 15 minutes in ms + "now-30m": 1800000, # 30 minutes in ms + "now-1h": 3600000, # 1 hour in ms + "now-3h": 10800000, # 3 hours in ms + "now-6h": 21600000, # 6 hours in ms + "now-12h": 43200000, # 12 hours in ms + "now-24h": 86400000, # 24 hours in ms + "now-2d": 172800000, # 2 days in ms + "now-7d": 604800000, # 7 days in ms + "now-30d": 2592000000, # 30 days in ms + "now-60d": 5184000000, # 60 days in ms + "now-90d": 7776000000, # 90 days in ms + "now-180d": 15552000000, # 180 days in ms + "now-365d": 31556952000, # 365 days in ms + "now-730d": 63113904000, # 730 days in ms + "now-1825d": 115680072000 # 1825 days in ms + } + + # Check if the given time string exists in the mappings + if from_time in time_mappings: + andebug('an.model.cli', 'Matched time string: {}'.format(from_time)) + return time_mappings[from_time] + else: + return -1 + + @staticmethod + def get_time_in_influx_format(from_time): + # Time in milliseconds for various time periods + time_mappings = { + "now-10m": "now()-10m", # 10 minutes in ms + "now-15m": "now()-15m", # 15 minutes in ms + "now-30m": "now()-30m", # 30 minutes in ms + "now-1h": "now()-1h", # 1 hour in ms + "now-3h": "now()-3h", # 3 hours in ms + "now-6h": "now()-6h", # 6 hours in ms + "now-12h": "now()-12h", # 12 hours in ms + "now-24h": "now()-24h", # 24 hours in ms + "now-2d": "now()-2d", # 2 days in ms + "now-7d": "now()-7d", # 7 days in ms + "now-30d": "now()-30d", # 30 days in ms + "now-60d": "now()-60d", # 60 days in ms + "now-90d": "now()-90d", # 90 days in ms + "now-180d": "now()-180d", # 180 days in ms + "now-365d": "now()-365d", # 365 days in ms + "now-730d": "now()-730d", # 730 days in ms + "now-1825d": "now()-1825d" # 1825 days in ms + } + + # Check if the given time string exists in the mappings + if from_time in time_mappings: + andebug('an.model.cli', 'Matched time string: {}'.format(from_time)) + return time_mappings[from_time] + else: + return -1 + + @staticmethod + def get_curr_sys_time(): + return int(time.time() * 1000) + + @staticmethod + def convert_ms_to_influxdb_time(ms): + # Convert milliseconds to seconds, minutes, hours, and days + seconds = ms // 1000 + minutes = seconds // 60 + hours = minutes // 60 + days = hours // 24 + + # Build the time string according to the largest possible unit + if days > 0: + return "{}d".format(days) # If days are more than 0, use days + elif hours > 0: + return "{}h".format(hours) # If hours are more than 0, use hours + elif minutes > 0: + return "{}m".format(minutes) # If minutes are more than 0, use minutes + else: + return "{}s".format(seconds) # Default to seconds + + @staticmethod def format_query_string(query): """Formats query string to ensure proper URL encoding""" return query.strip().replace(' ', "%20").replace('<', "%3C").replace('>', "%3E").replace(',', "%2C") \ Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py (revision 2463) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/llb_stats.py (working copy) @@ -1,18 +1,30 @@ from django.http import HttpResponse import json from hive.db.llb_stats_queries import LLBStatsDB +from hive.custom_exceptions import generic_exception as ge +from cm.lib.libbasic_operation import oper_log def handle_llb_stats_req(request, app): - if app == 'stats' or app is None and request.method == 'POST': - return get_llb_stats(request) - elif app == 'monitoring_data' and request.method == 'POST': - return get_monitoring_data(request) - else: - return HttpResponse(json.dumps({ - 'error': 405, - 'message': "Invalid HTTP method" - }), content_type='application/json') + try: + if app == 'stats' or app is None and request.method == 'POST': + return get_llb_stats(request) + elif app == 'monitoring_data' and request.method == 'POST': + return get_monitoring_data(request) + elif app == 'graph_data' and request.method == 'POST': + return get_llb_graphs_data(request) + else: + return HttpResponse(json.dumps({ + 'error': 405, + 'message': "Invalid HTTP method" + }), content_type='application/json') + except ge.GenericError as e: + oper_log('error', 'system', e.message) + return ge.handle_exception(e) + except Exception as e: + oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e)) + e.message = 'An Unexpected error occurred, details: {}'.format(e) + return ge.handle_exception(e) def get_llb_stats(request): @@ -26,4 +38,11 @@ req = json.loads(request.body) req_dict = LLBStatsDB.get_llb_monitoring_query_str(req) response = LLBStatsDB.execute_db_query(req_dict) - return response \ No newline at end of file + return response + + +def get_llb_graphs_data(request): + req = json.loads(request.body) + req_dict = LLBStatsDB.get_llb_graphs_data_str(req) + response = LLBStatsDB.execute_db_query(req_dict) + return response