Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py (working copy) @@ -18,7 +18,7 @@ from hive.controller.device_metrics import handle_device_metrics_req from hive.controller.slb_stats import handle_slb_stats_req from hive.controller.slb_vpn_stats import handle_ssl_vpn_stats_req -from hive.llb_stats import handle_llb_stats_req +from hive.controller.llb_stats import handle_llb_stats_req from hive.report.generate_report import handle_report_generation from hive.controller.backup_controller import handle_backup_req from hive.controller.restore_controller import handle_restore_req @@ -71,7 +71,7 @@ re_path(r'^apv/slb/(?P.*)$', handle_slb_stats_req), re_path(r'^device_metrics(?:/(?P.*))?$', handle_device_metrics_req), re_path(r'^ssl_vpn_stats(?:/(?P.*))?$', handle_ssl_vpn_stats_req), - re_path(r'^llb/(?P.*)$', handle_llb_stats_req), + re_path(r'^llb_stats(?:/(?P.*))?$', handle_llb_stats_req), re_path(r'^report/(?P.*)$', handle_report_generation), re_path(r'^log/(?P\w+)$', handle_log_location_app), re_path(r'^backup/(?P.*)$', handle_backup_req), Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/device_metrics.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/device_metrics.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/device_metrics.py (working copy) @@ -28,16 +28,12 @@ """ Fetch the device metrics data """ try: req_body = json.loads(request.body) - if 'query' in req_body and 'agent_host' not in req_body['query']: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") agent_host = req_body.get("agent_host", None) interval = req_body.get("interval", "1m") time_from = req_body.get("from", None) time_to = req_body.get("to", None) metric_name = req_body.get("metric_name", None) + json_response = fetch_device_metrics(agent_host, interval, metric_name, time_from, time_to) return JsonResponse(json_response) \ No newline at end of file @@ -45,4 +41,4 @@ except Exception as e: return JsonResponse({ "error": str(e) - }, status=500) + }, status=500) \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/slb_stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/slb_stats.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/slb_stats.py (working copy) @@ -29,21 +29,16 @@ def get_virtual_service_stats(request): """ Fetch data for SLB Virtual stats data """ try: - req = {} - req = json.loads(request.body) - if 'query' in req and 'agent_host' not in req['query']: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") + try: + req = json.loads(request.body) if request.body else {} + except json.JSONDecodeError: + req = {} - agent_host = req.get("agent_host") + agent_host = req.get("agent_host", None) server_id = req.get("server_id", None) interval = req.get("interval", "20s") - time_from = req.get("from") - time_to = req.get("to") - if not agent_host: - raise ge.GenericError(400, "agent_host field is required for Network Throughput stats") + time_from = req.get("from", None) + time_to = req.get("to", None) response = fetch_virtual_stats(agent_host, interval, req, server_id, time_from, time_to) \ No newline at end of file @@ -62,12 +57,6 @@ """ try: req_body = json.loads(request.body) - - if 'query' in req_body and 'agent_host' not in req_body['query']: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") agent_host = req_body.get("agent_host", None) server_id = req_body.get("server_id", None) interval = req_body.get("interval", "1m") \ No newline at end of file @@ -83,4 +72,4 @@ except Exception as e: return JsonResponse({ "error": str(e) - }, status=500) + }, status=500) \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/ssl_vpn_stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/ssl_vpn_stats.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/ssl_vpn_stats.py (working copy) @@ -27,11 +27,6 @@ """ Fetch the SSL VPN data """ try: req_body = json.loads(request.body) - if 'query' in req_body and 'agent_host' not in req_body['query']: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") agent_host = req_body.get("agent_host", None) interval = req_body.get("interval", "1m") vsite_id = req_body.get("vsite_id", None) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/device_metric_queries.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/device_metric_queries.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/device_metric_queries.py (working copy) @@ -1,6 +1,6 @@ - from hive.db.db_client import DBClient from hive.utils import andebug +from hive.db.utils import cast_fields class DeviceMetrics: \ No newline at end of file @@ -14,10 +14,10 @@ query = f""" SELECT agent_host, - (last(cpu_usage, time)) AS cpu, - (last(mem_usage, time)) AS mem, - (last(net_mem_usage, time)) AS net_mem, - (last(connections, time)) AS connections + COALESCE((last(cpu_usage, time)), 0) AS cpu, + COALESCE((last(mem_usage, time)), 0) AS mem, + COALESCE((last(net_mem_usage, time)), 0) AS net_mem, + COALESCE((last(connections, time)), 0) AS connections FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -25,8 +25,18 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "cpu": int, + "mem": int, + "net_mem": int, + "connections": int + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_throughput_data(interval, time_from, time_to): """ \ No newline at end of file @@ -65,8 +75,16 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "received": float, + "sent": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_cpu_usage_metrics(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -77,7 +95,7 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - AVG(cpu_usage) AS cpu + COALESCE(AVG(cpu_usage)::bigint, 0) AS cpu FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -86,8 +104,15 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "cpu_usage": int + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_mem_usage_metrics(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -98,8 +123,8 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - COALESCE(AVG(mem_usage), 0) AS memory_usage, - COALESCE(AVG(net_mem_usage), 0) AS network_mem_usage + COALESCE(AVG(mem_usage)::bigint, 0) AS memory_usage, + COALESCE(AVG(net_mem_usage)::bigint, 0) AS network_mem_usage FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -108,8 +133,16 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "memory_usage": int, + "network_mem_usage": int + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_disk_usage_metrics(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -120,17 +153,29 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - COALESCE(AVG(used) * 100.0 / NULLIF(AVG(size), 0), 0) AS disk_usage + COALESCE((AVG(used) * 100.0) / NULLIF(AVG(size), 0), 0) AS disk_usage FROM an_device_storage WHERE time >= {time_from} AND time <= {time_to} - AND agent_host = '{agent_host}' + AND agent_host = %s GROUP BY ts, agent_host ORDER BY ts; """ - req = {"query": query} - return DBClient.execute_query(req) + + # Pass parameters safely to avoid SQL injection + params = [agent_host] + + req = {"query": query, "params": params} + response = DBClient.execute_query(req) + # Cast disk_usage to int + field_casts = { + "disk_usage": int + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_network_throughput(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -170,8 +215,16 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "sent": float, + "received": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_ssl_connections(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -182,7 +235,7 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - SUM(total_openssl_conns) AS ssl_connection + COALESCE(SUM(total_openssl_conns), 0) AS ssl_connection FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -191,8 +244,15 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "ssl_connection": int + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_connections(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -203,7 +263,7 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - AVG(connections) AS connection + COALESCE(AVG(connections), 0) AS connection FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -212,8 +272,15 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "connection": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_requests(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -224,7 +291,7 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - AVG(requests) AS connection + COALESCE(AVG(requests), 0) AS requests FROM an_device_metrics WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -233,8 +300,15 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "requests": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + @staticmethod def get_ssl_core_utilization(agent_host, interval, time_from, time_to): """ \ No newline at end of file @@ -245,8 +319,8 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - AVG(ssl_ae_core_utilization) AS ssl_ae_core, - AVG(ssl_se_core_utilization) AS ssl_se_core + COALESCE(AVG(ssl_ae_core_utilization), 0) AS ssl_ae_core, + COALESCE(AVG(ssl_se_core_utilization), 0) AS ssl_se_core FROM an_device_performance WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -255,4 +329,12 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + + field_casts = { + "ssl_ae_core": float, + "ssl_se_core": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (working copy) @@ -1,361 +1,247 @@ -import hashlib -import http.client -import json -import time +from hive.db.db_client import DBClient +from hive.utils import andebug +from hive.db.utils import cast_fields -from cm.lib.libbasic_operation import oper_log -from djproject.an_settings import CMDATA -from hive.custom_exceptions import generic_exception as ge - class LLBStatsDB: """Class to handle all DB interactions for LLB Stats""" def __init__(self): pass - BASE_URL = "/v1/datasource/proxy/1/query" - CONTENT_TYPE = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'} - @staticmethod - def execute_db_query(req_dict): - """Executes a database query and returns a response""" - http_client = None + def get_llb_stats_data(agent_host, time_from, time_to): + """ + Returns the latest LLB stats data over the last interval (default 20s) + """ + andebug('an.model.cli', 'Inside get_llb_stats_data') - try: - req_payload = req_dict["body"] - timestamp = str(int(time.time())) - m = hashlib.md5() - m.update(CMDATA['COMPOSER_KEY'] + str(timestamp) + req_dict["url"] + CMDATA['COMPOSER_SECRET']) + query = f""" + SELECT + agent_host, + link_name, + link_status, + COALESCE(MAX(link_gateway::text), 'N/A') AS link_gateway, + COALESCE(MAX(link_thresh), 0) AS link_thresh, + COALESCE(MAX(link_usage), 0) AS link_usage, + ROUND(COALESCE(MAX(link_bandwid_in) * 8, 0), 2) AS received, + ROUND(COALESCE(MAX(link_bandwid_out) * 8, 0), 2) AS sent, + COALESCE(MAX(link_conn), 0) AS link_conn, + COALESCE(MAX(link_hits), 0) AS link_hits + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + """ - query_string = req_dict["path"] - if query_string == '': - query_string += "time=" + timestamp + "&sign=" + m.hexdigest() - else: - query_string += "&time=" + timestamp + "&sign=" + m.hexdigest() + if agent_host: + query += f" AND agent_host = '{agent_host}'" - http_client = http.client.HTTPConnection(CMDATA['COMPOSER_IP'], str(CMDATA['COMPOSER_PORT']), True, - timeout=15) - http_client.request(req_dict['method'], req_dict["url"] + "?" + query_string, req_payload, - req_dict['config']) + query += """ + GROUP BY agent_host, link_name, link_status + ORDER BY agent_host, link_name, link_status; + """ - response = http_client.getresponse() - response_data = { - 'status': response.status, - 'reason': response.reason, - 'body': response.read(), - } - except ge.GenericError as e: - raise e - except ge.ANInternalException as exp: - return json.dumps({ - 'parm_name': exp.parm_name, - 'error': exp.error_code, - 'message': exp.description, - 'data': None - }) - except Exception as e: - return json.dumps({ - 'error': 500, - 'message': "Request Composer_ui Error", - 'data': str(e) - }) - else: - if response_data['status'] == 200: - return response_data['body'] - else: - return json.dumps({ - 'error': response_data['status'], - 'message': response_data['reason'], - 'data': response_data['body'] - }) - finally: - if http_client: - http_client.close() + req = {"query": query} + andebug('an.model.cli', f'query : {query}') + response = DBClient.execute_query(req) + andebug('an.model.cli', f'response : {response}') + andebug('an.model.cli', f'response data : {response["data"]}') - @staticmethod - def get_llb_stats_data(agent_host): - """Returns the LLB stats data""" - req_dict = { - "url": LLBStatsDB.BASE_URL, - "path": "db=composer&epoch=ms", - "method": "POST", - "body": {}, - "config": LLBStatsDB.CONTENT_TYPE + field_casts = { + "link_thresh": int, + "link_usage": int, + "received": float, + "sent": float, + "link_conn": int, + "link_hits": int, } + response["data"] = cast_fields(response["data"], field_casts) + andebug('an.model.cli', f'response data after cast: {response["data"]}') - query_string = ( - 'SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) ' - 'AS linkUsage, last(linkBandwidIn) * 8 AS received, last(linkBandwidOut) * 8 AS sent, ' - 'last(linkConn) AS linkConn, last(linkHits) AS linkHits FROM llbStats WHERE time > now()-20s ' - 'GROUP BY agent_host, linkName, linkStatus FILL(0);') + return response + @staticmethod + def get_top_hits(agent_host, time_from, time_to): + andebug('an.model.cli', 'Inside get_top_hits') + query = f""" + SELECT + COALESCE((MAX(link_hits) - MIN(link_hits))::bigint, 0) AS hits, + link_name, + agent_host + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + """ if agent_host: - query_string = ( - 'SELECT LAST(linkGateway) AS linkGateway, last(linkThresh) AS linkThresh, last(linkUsage) AS linkUsage, ' - 'last(linkBandwidIn) * 8 AS received, last(linkBandwidOut) * 8 AS sent, last(linkConn) AS linkConn, ' - 'last(linkHits) AS linkHits FROM llbStats WHERE agent_host =~ \'{}\' ' - 'AND time > now()-20s GROUP BY agent_host, linkName, linkStatus FILL(0);' - ).format(agent_host) + query += f" AND agent_host = '{agent_host}'" + query += """ + GROUP BY link_name, agent_host + ORDER BY hits DESC + LIMIT 5; + """ + andebug('an.model.cli', f'query: {query}') + response = DBClient.execute_query({"query": query}) + andebug('an.model.cli', f'response: {response}') + andebug('an.model.cli', f'response data: {response["data"]}') - query_string = LLBStatsDB.format_query_string(query_string) - req_dict["body"] = 'q=' + query_string - response = LLBStatsDB.execute_db_query(req_dict) + field_casts = { + "hits": int + } + response["data"] = cast_fields(response["data"], field_casts) return response @staticmethod - def get_llb_monitoring_data(agent_host): - """Returns the LLB monitoring data""" - req_dict = { - "url": LLBStatsDB.BASE_URL, - "path": "db=composer&epoch=ms", - "method": "POST", - "body": {}, - "config": LLBStatsDB.CONTENT_TYPE - } - - query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' - 'linkHits) AS hits FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s; ' - 'SELECT TOP(conn, 5) as top_conn, linkName, agent_host FROM (SELECT LAST(linkConn) as conn ' - 'FROM llbStats GROUP BY linkName, agent_host) WHERE time > now()-20s;' - 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (select ' - 'last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, last(linkBandwidIn) * 8 as ' - 'received, last(linkBandwidOut) * 8 as sent from llbStats group by linkName, agent_host) ' - 'where time > now()-20s FILL(0);') - + def get_top_conn(agent_host, time_from, time_to): + andebug('an.model.cli', 'Inside get_top_conn') + query = f""" + SELECT sub.conn, sub.link_name, sub.agent_host + FROM ( + SELECT DISTINCT ON (link_name, agent_host) + COALESCE(link_conn::bigint, 0) AS conn, + link_name, + agent_host, + time + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + """ if agent_host: - query_string = ('SELECT TOP(hits, 5) as top_hits, linkName, agent_host FROM (SELECT MAX(linkHits)-MIN(' - 'linkHits) AS hits FROM llbStats GROUP BY linkName, agent_host) WHERE agent_host =~ {} ' - 'and time >now()-20s;' - 'SELECT TOP(last, 5) as top_conn, linkName, agent_host FROM (SELECT LAST(linkConn) FROM ' - 'llbStats GROUP BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s;' - 'SELECT TOP(total, 5) as total_throughput, received, sent, linkName, agent_host from (' - 'select last(linkBandwidIn) * 8 + last(linkBandwidOut) * 8 as total, ' - 'last(linkBandwidIn) * 8 as received, last(linkBandwidOut) * 8 as sent from llbStats GROUP' - 'BY linkName, agent_host) WHERE agent_host =~ {} and time > now()-20s FILL(0);' - ).format(agent_host, agent_host, agent_host) + query += f" AND agent_host = '{agent_host}'" + query += """ + ORDER BY link_name, agent_host, time DESC + ) sub + ORDER BY sub.conn DESC + LIMIT 5; + """ - query_string = LLBStatsDB.format_query_string(query_string) - req_dict["body"] = 'q=' + query_string - response = LLBStatsDB.execute_db_query(req_dict) + andebug('an.model.cli', f'query: {query}') + response = DBClient.execute_query({"query": query}) + andebug('an.model.cli', f'response: {response}') + andebug('an.model.cli', f'response data: {response["data"]}') - return response - - @staticmethod - def get_llb_historical_data(from_time, to_time, link_name): - """Returns the LLB monitoring data""" - req_dict = { - "url": LLBStatsDB.BASE_URL, - "path": "db=composer&epoch=ms", - "method": "POST", - "body": {}, - "config": LLBStatsDB.CONTENT_TYPE + field_casts = { + "conn": int } + response["data"] = cast_fields(response["data"], field_casts) + andebug('an.model.cli', f'response data after cast: {response["data"]}') - # time_interval specifies the frequency at which data points should be generated. - time_interval = LLBStatsDB.get_time_interval(from_time, to_time) + return response - if time_interval < 0: - oper_log('error', 'system', "Time Interval or Time Period is not specified properly.") - raise ge.GenericError(400, "Time Interval or Time Period not specified properly.") - - # Ensure the to_time is properly stripped and lowercased for comparison. - to_time = to_time.strip().lower() - - if link_name: - if isinstance(to_time, int): - query_string = ( - "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " - "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} and time < {} and linkName = '{}' GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" - ).format(from_time, to_time, link_name, time_interval) - else: - from_time = LLBStatsDB.get_time_in_influx_format(from_time) - query_string = ( - "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " - "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} and linkName = '{}' GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" - ).format(from_time, link_name, time_interval) - else: - if isinstance(to_time, int): - query_string = ( - "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " - "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} and time < {} GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" - ).format(from_time, to_time, time_interval) - else: - from_time = LLBStatsDB.get_time_in_influx_format(from_time) - query_string = ( - "SELECT mean(linkBandwidIn) AS avgBandwidIn, mean(linkBandwidOut) AS avgBandwidOut, " - "mean(linkUsage) AS avgLinkUsage, mean(linkConn) AS avgLinkConn, mean(linkHits) AS avgLinkHits FROM " - "llbStats WHERE time > {} GROUP BY time({}), agent_host, linkName, linkStatus FILL(0);" - ).format(from_time, time_interval) + @staticmethod + def get_top_throughput(agent_host, time_from, time_to): + query = f""" + SELECT (sub.received + sub.sent) AS total_throughput, + sub.received, + sub.sent, + sub.link_name, + sub.agent_host + FROM ( + SELECT DISTINCT ON (link_name, agent_host) + COALESCE((link_bandwid_in * 8)::numeric(20,2), 0) AS received, + COALESCE((link_bandwid_out * 8)::numeric(20,2), 0) AS sent, + link_name, + agent_host, + time + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + """ + if agent_host: + query += f" AND agent_host = '{agent_host}'" + query += """ + ORDER BY link_name, agent_host, time DESC + ) sub + ORDER BY total_throughput DESC + LIMIT 5; + """ + response = DBClient.execute_query({"query": query}) - query_string = LLBStatsDB.format_query_string(query_string) - req_dict["body"] = 'q=' + query_string - response = LLBStatsDB.execute_db_query(req_dict) - - return response - - @staticmethod - def get_llb_summary_data(from_time, to_time): - """Constructs the query for LLB monitoring data""" - req_dict = { - "url": LLBStatsDB.BASE_URL, - "path": "db=composer&epoch=ms", - "method": "POST", - "body": {}, - "config": LLBStatsDB.CONTENT_TYPE + field_casts = { + "total_throughput": float, + "received": float, + "sent": float } - - if isinstance(to_time, int): - query_string = ('select mean(linkHits) AS avgLinkHits, mean(conn) AS avgLinkConn, mean(received) AS ' - 'avgBandwidIn, mean(sent) AS avgBandwidOut, mean(linkUsage) AS avgLinkUsage ' - 'FROM llbStats WHERE time > {} and time < {} GROUP BY linkName;').format(from_time, - to_time) - else: - from_time = LLBStatsDB.get_time_in_influx_format(from_time) - query_string = ('select mean(linkHits) AS avgLinkHits, mean(conn) AS avgLinkConn, mean(received) AS ' - 'avgBandwidIn, mean(sent) AS avgBandwidOut, mean(linkUsage) AS avgLinkUsage ' - 'FROM llbStats WHERE time > {} GROUP BY linkName;').format(from_time) - - query_string = LLBStatsDB.format_query_string(query_string) - req_dict["body"] = 'q=' + query_string - response = LLBStatsDB.execute_db_query(req_dict) - + response["data"] = cast_fields(response["data"], field_casts) return response @staticmethod - def get_time_interval(from_time, to_time): - from_time_in_int, to_time_in_int = LLBStatsDB.get_time_range(from_time, to_time) + def get_llb_historical_data(time_from, time_to, link_name, interval="20s"): + """ + Returns historical LLB stats aggregated over time buckets. + """ + query = f""" + SELECT + time_bucket('{interval}', time) AS ts, + agent_host, + link_name, + link_status, + COALESCE(ROUND(AVG(link_bandwid_in)::numeric, 2), 0) AS avg_bandwid_in, + COALESCE(ROUND(AVG(link_bandwid_out)::numeric, 2), 0) AS avg_bandwid_out, + COALESCE(ROUND(AVG(link_usage)::numeric, 2), 0) AS avg_link_usage, + COALESCE(ROUND(AVG(link_conn)::numeric, 2), 0) AS avg_link_conn, + COALESCE(ROUND(AVG(link_hits)::numeric, 2), 0) AS avg_link_hits + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + """ - # Calculate the absolute difference in time - diff_time = abs(to_time_in_int - from_time_in_int) + if link_name: + query += f" AND link_name = '{link_name}'" - # If there's a positive time difference, calculate the interval - if diff_time > 0: - diff_hours = diff_time / (1000 * 60 * 60) # Convert ms to hours + query += """ + GROUP BY ts, agent_host, link_name, link_status + ORDER BY ts ASC; + """ - if diff_hours > 24: - # More than 24 hours, divide by 96 for 15 minute intervals - return LLBStatsDB.convert_ms_to_influxdb_time(abs(diff_time / 96)) - else: - # Less than 24 hours, divide by 24 for hourly intervals - return LLBStatsDB.convert_ms_to_influxdb_time(abs(diff_time / 24)) - else: - # Return -1 for invalid or zero time difference - return -1 + req = {"query": query} + andebug('an.model.cli', f'query: {query}') + response = DBClient.execute_query(req) - @staticmethod - def get_time_range(from_time, to_time): - curr_time_in_ms = LLBStatsDB.get_curr_sys_time() - # Handle from_time (same logic as before) - from_time_in_int = LLBStatsDB.get_time_in_ms_for_str(from_time) - if from_time_in_int == -1: - try: - from_time_in_int = int(from_time) # Try to convert to integer if it's in epoch ms format - except ValueError: - oper_log('error', 'system', "From time is not specified in correct format.") - raise ge.GenericError(400, "From time is not specified in correct format.") - else: - from_time_in_int = curr_time_in_ms - from_time_in_int - if to_time == "now": - to_time_in_int = curr_time_in_ms # If "now", use current time - else: - to_time_in_int = LLBStatsDB.get_time_in_ms_for_str(to_time) - if to_time_in_int == -1: - try: - to_time_in_int = int(to_time) # Try to convert to integer if it's in epoch ms format - except ValueError: - oper_log('error', 'system', "To time is not specified in correct format.") - raise ge.GenericError(400, "To time is not specified in correct format.") - else: - to_time_in_int = curr_time_in_ms - from_time_in_int - return from_time_in_int, to_time_in_int - - @staticmethod - def get_time_in_ms_for_str(from_time): - # Time in milliseconds for various time periods - time_mappings = { - "now-10m": 600000, # 10 minutes in ms - "now-15m": 900000, # 15 minutes in ms - "now-30m": 1800000, # 30 minutes in ms - "now-1h": 3600000, # 1 hour in ms - "now-3h": 10800000, # 3 hours in ms - "now-6h": 21600000, # 6 hours in ms - "now-12h": 43200000, # 12 hours in ms - "now-24h": 86400000, # 24 hours in ms - "now-2d": 172800000, # 2 days in ms - "now-7d": 604800000, # 7 days in ms - "now-30d": 2592000000, # 30 days in ms - "now-60d": 5184000000, # 60 days in ms - "now-90d": 7776000000, # 90 days in ms - "now-180d": 15552000000, # 180 days in ms - "now-365d": 31556952000, # 365 days in ms - "now-730d": 63113904000, # 730 days in ms - "now-1825d": 115680072000 # 1825 days in ms + field_casts = { + "avg_bandwid_in": float, + "avg_bandwid_out": float, + "avg_link_usage": float, + "avg_link_conn": float, + "avg_link_hits": float } + response["data"] = cast_fields(response["data"], field_casts) + return response - # Check if the given time string exists in the mappings - if from_time in time_mappings: - return time_mappings[from_time] - else: - return -1 - @staticmethod - def get_time_in_influx_format(from_time): - # Time in milliseconds for various time periods - time_mappings = { - "now-10m": "now()-10m", # 10 minutes in ms - "now-15m": "now()-15m", # 15 minutes in ms - "now-30m": "now()-30m", # 30 minutes in ms - "now-1h": "now()-1h", # 1 hour in ms - "now-3h": "now()-3h", # 3 hours in ms - "now-6h": "now()-6h", # 6 hours in ms - "now-12h": "now()-12h", # 12 hours in ms - "now-24h": "now()-24h", # 24 hours in ms - "now-2d": "now()-2d", # 2 days in ms - "now-7d": "now()-7d", # 7 days in ms - "now-30d": "now()-30d", # 30 days in ms - "now-60d": "now()-60d", # 60 days in ms - "now-90d": "now()-90d", # 90 days in ms - "now-180d": "now()-180d", # 180 days in ms - "now-365d": "now()-365d", # 365 days in ms - "now-730d": "now()-730d", # 730 days in ms - "now-1825d": "now()-1825d" # 1825 days in ms - } + def get_llb_summary_data(time_from, time_to): + andebug('an.model.cli', 'Inside get_llb_summary_data') + """ + Returns LLB summary stats aggregated over the full time range. + """ + query = f""" + SELECT + link_name, + COALESCE(CAST(AVG(link_hits) AS bigint), 0) AS avg_link_hits, + COALESCE(CAST(AVG(link_conn) AS bigint), 0) AS avg_link_conn, + COALESCE(ROUND(AVG(link_bandwid_in * 8)::numeric, 2), 0) AS avg_bandwid_in, + COALESCE(ROUND(AVG(link_bandwid_out * 8)::numeric, 2), 0) AS avg_bandwid_out, + COALESCE(ROUND(AVG(link_usage)::numeric, 2), 0) AS avg_link_usage + FROM apv_llb_stats + WHERE time >= {time_from} + AND time <= {time_to} + GROUP BY link_name + ORDER BY link_name; + """ - # Check if the given time string exists in the mappings - if from_time in time_mappings: - return time_mappings[from_time] - else: - return -1 + req = {"query": query} + andebug('an.model.cli', f'query: {query}') + response = DBClient.execute_query(req) + andebug('an.model.cli', f'response: {response}') + andebug('an.model.cli', f'response data: {response["data"]}') - @staticmethod - def get_curr_sys_time(): - return int(time.time() * 1000) - - @staticmethod - def convert_ms_to_influxdb_time(ms): - # Convert milliseconds to seconds, minutes, hours, and days - seconds = ms // 1000 - minutes = seconds // 60 - hours = minutes // 60 - days = hours // 24 - - # Build the time string according to the largest possible unit - if days > 0: - return "{}d".format(days) # If days are more than 0, use days - elif hours > 0: - return "{}h".format(hours) # If hours are more than 0, use hours - elif minutes > 0: - return "{}m".format(minutes) # If minutes are more than 0, use minutes - else: - return "{}s".format(seconds) # Default to seconds - - @staticmethod - def format_query_string(query): - """Formats query string to ensure proper URL encoding""" - return query.strip().replace(' ', "%20").replace('<', "%3C").replace('>', "%3E").replace(',', "%2C") \ - .replace(';', "%3B").replace('+', "%2B").replace('/', "%2F").replace('|', "%7C").replace('=', "%3D") + field_casts = { + "avg_link_hits": float, + "avg_link_conn": float, + "avg_bandwid_in": float, + "avg_bandwid_out": float, + "avg_link_usage": float + } + response["data"] = cast_fields(response["data"], field_casts) + andebug('an.model.cli', f'response data after cast: {response["data"]}') + return response \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/slb_stats_queries.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/slb_stats_queries.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/slb_stats_queries.py (working copy) @@ -1,6 +1,6 @@ from hive.db.db_client import DBClient -from hive.custom_exceptions import generic_exception as ge from hive.utils import andebug +from hive.db.utils import cast_fields class SLBStats: \ No newline at end of file @@ -9,60 +9,67 @@ pass @staticmethod - def get_virtual_service_stats(agent_host): + def get_virtual_service_stats(agent_host=None): + """ + Fetch latest virtual service stats from apv_virtual_stats + """ base_query = """ - WITH stats AS ( - SELECT - time, - agent_host, - serverid, - addr, - port, - protocol, - health_status, - ( - url_hits + hostname_hits + perstnt_cookie_hits + qos_cookie_hits + - default_hits + perstnt_url_hits + static_hits + qos_network_hits + - qos_url_hits + backup_hits + cache_hits + regex_hits + - rcookie_hits + icookie_hits + qos_client_port_hits + - qos_body_hits + header_hits + hash_url_hits + redirect_hits - ) AS total_hits, - conn_cnt, - in_byte_per_sec, - out_byte_per_sec - FROM apv_virtual_stats - WHERE time > now() - interval '20 seconds' - ) + WITH stats AS ( SELECT - max(total_hits) - min(total_hits) AS hits_diff, - (array_agg(conn_cnt ORDER BY time DESC))[1] AS last_conn_cnt, - (array_agg(in_byte_per_sec ORDER BY time DESC))[1] * 8 AS last_in_bps, - (array_agg(out_byte_per_sec ORDER BY time DESC))[1] * 8 AS last_out_bps, - serverid, + time, agent_host, + serverid, addr, port, protocol, - health_status - FROM stats + health_status, + ( + COALESCE(url_hits, 0) + COALESCE(hostname_hits, 0) + COALESCE(perstnt_cookie_hits, 0) + + COALESCE(qos_cookie_hits, 0) + COALESCE(default_hits, 0) + COALESCE(perstnt_url_hits, 0) + + COALESCE(static_hits, 0) + COALESCE(qos_network_hits, 0) + COALESCE(qos_url_hits, 0) + + COALESCE(backup_hits, 0) + COALESCE(cache_hits, 0) + COALESCE(regex_hits, 0) + + COALESCE(rcookie_hits, 0) + COALESCE(icookie_hits, 0) + COALESCE(qos_client_port_hits, 0) + + COALESCE(qos_body_hits, 0) + COALESCE(header_hits, 0) + COALESCE(hash_url_hits, 0) + + COALESCE(redirect_hits, 0) + ) AS total_hits, + COALESCE(conn_cnt, 0) AS conn_cnt, + COALESCE(in_byte_per_sec, 0) AS in_byte_per_sec, + COALESCE(out_byte_per_sec, 0) AS out_byte_per_sec + FROM apv_virtual_stats + WHERE time > now() - interval '20 seconds' + {agent_host_filter} + ) + SELECT + COALESCE(MAX(total_hits) - MIN(total_hits), 0) AS hits_diff, + COALESCE((ARRAY_AGG(conn_cnt ORDER BY time DESC))[1], 0) AS last_conn_cnt, + COALESCE((ARRAY_AGG(in_byte_per_sec ORDER BY time DESC))[1] * 8, 0) AS last_in, + COALESCE((ARRAY_AGG(out_byte_per_sec ORDER BY time DESC))[1] * 8, 0) AS last_out, + serverid, + agent_host, + addr, + port, + protocol, + health_status + FROM stats + GROUP BY serverid, agent_host, addr, port, protocol, health_status; """ - params = [] - if agent_host: - base_query += " WHERE agent_host ~ %s" - params.append(agent_host) + agent_host_filter = "AND agent_host ~ %s" if agent_host else "" + query = base_query.format(agent_host_filter=agent_host_filter) - base_query += " GROUP BY serverid, agent_host, addr, port, protocol, health_status;" + params = [agent_host] if agent_host else [] - req_dict = { - "query": base_query, - "params": params + response = DBClient.execute_query({"query": query, "params": params}) + + field_casts = { + "hits_diff": int, + "last_conn_cnt": int, + "last_in": float, + "last_out": float } + response["data"] = cast_fields(response["data"], field_casts) + return response - return DBClient.execute_query(req_dict) - - # --- Convert "from" / "to" --- - @staticmethod def get_network_throughput(agent_host, virtual_server_id, interval, time_from=None, time_to=None): \ No newline at end of file @@ -73,8 +80,8 @@ query = f""" SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, - avg(in_byte_per_sec) AS received, - avg(out_byte_per_sec) AS sent + COALESCE(avg(in_byte_per_sec), 0) AS received, + COALESCE(avg(out_byte_per_sec), 0) AS sent FROM apv_virtual_stats WHERE agent_host ~ %s AND serverid ~ %s \ No newline at end of file @@ -89,8 +96,15 @@ "params": [agent_host, virtual_server_id] } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "received": float, + "sent": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_total_hits(agent_host, virtual_server_id, interval, time_from, time_to): """ \ No newline at end of file @@ -133,8 +147,14 @@ "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "hit_diff": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_hits_distribution(agent_host, virtual_server_id, time_from, time_to): """ \ No newline at end of file @@ -177,8 +197,20 @@ "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "url_hits": int, + "hostname_hits": int, + "qos_cookie_hits": int, + "qos_network_hits": int, + "qos_url_hits": int, + "qos_client_port_hits": int, + "qos_body_hits": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_connection_count(agent_host, virtual_server_id, interval, time_from, time_to): """ \ No newline at end of file @@ -188,7 +220,7 @@ query = f""" SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, - COALESCE(SUM(conn_cnt), 0) AS connections + COALESCE(SUM(conn_cnt), 0) AS conn_cnt FROM apv_virtual_stats WHERE agent_host ~ %s AND serverid ~ %s \ No newline at end of file @@ -205,8 +237,14 @@ "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "conn_cnt": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_connections_per_sec(agent_host, virtual_server_id, interval, time_from, time_to): """ \ No newline at end of file @@ -233,8 +271,14 @@ "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "connections": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_real_service_stats(agent_host): """ \ No newline at end of file @@ -242,56 +286,64 @@ """ base_query = """ - WITH stats AS ( - SELECT - time, - agent_host, - real_server_id, - addr, - port, - protocol, - status, - rs_cnt_of_req, - rs_conn_per_sec, - rs_in_byte_per_sec, - rs_out_byte_per_sec - FROM apv_real_stats - WHERE time > now() - interval '20 seconds' - ) - SELECT - (array_agg(rs_cnt_of_req ORDER BY time DESC))[1] AS last_req_cnt, - (array_agg(rs_conn_per_sec ORDER BY time DESC))[1] AS last_conn_per_sec, - (array_agg(rs_in_byte_per_sec ORDER BY time DESC))[1] * 8 AS last_in_bps, - (array_agg(rs_out_byte_per_sec ORDER BY time DESC))[1] * 8 AS last_out_bps, - real_server_id, - agent_host, - addr, - port, - protocol, - status - FROM stats - """ + WITH stats AS ( + SELECT + time, + agent_host, + real_server_id, + addr, + port, + protocol, + status, + COALESCE(rs_cnt_of_req, 0) AS rs_cnt_of_req, + COALESCE(rs_conn_per_sec, 0) AS rs_conn_per_sec, + COALESCE(rs_in_byte_per_sec, 0) AS rs_in_byte_per_sec, + COALESCE(rs_out_byte_per_sec, 0) AS rs_out_byte_per_sec + FROM apv_real_stats + WHERE time > now() - interval '20 seconds' + {agent_host_filter} + ) + SELECT + COALESCE((array_agg(rs_cnt_of_req ORDER BY time DESC))[1], 0) AS last_req_cnt, + COALESCE((array_agg(rs_conn_per_sec ORDER BY time DESC))[1], 0) AS last_conn_per_sec, + COALESCE((array_agg(rs_in_byte_per_sec ORDER BY time DESC))[1] * 8, 0) AS last_in, + COALESCE((array_agg(rs_out_byte_per_sec ORDER BY time DESC))[1] * 8, 0) AS last_out, + real_server_id, + agent_host, + addr, + port, + protocol, + status + FROM stats + GROUP BY real_server_id, agent_host, addr, port, protocol, status; + """ - params = [] - if agent_host: - base_query += " WHERE agent_host ~ %s" - params.append(agent_host) + agent_host_filter = f"AND agent_host ~ %s" if agent_host else "" + query = base_query.format(agent_host_filter=agent_host_filter) - base_query += " GROUP BY real_server_id, agent_host, addr, port, protocol, status;" + params = [agent_host] if agent_host else [] req_dict = { - "query": base_query, + "query": query, "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "last_req_cnt": int, + "last_conn_per_sec": int, + "last_in": int, + "last_out": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_real_network_throughput(agent_host, real_server_id, interval, time_from=None, time_to=None): """ Fetch network throughput (Sent, Received) from apv_real_stats """ - query = f""" SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, \ No newline at end of file @@ -311,8 +363,15 @@ "params": [agent_host, real_server_id] } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "received": float, + "sent": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_outstanding_requests(agent_host, real_server_id, interval, time_from=None, time_to=None): """ \ No newline at end of file @@ -322,7 +381,7 @@ query = f""" SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, - sum(rs_cnt_of_req) AS request + COALESCE(sum(rs_cnt_of_req), 0) AS request FROM apv_real_stats WHERE agent_host ~ %s AND real_server_id ~ %s \ No newline at end of file @@ -337,8 +396,14 @@ "params": [agent_host, real_server_id] } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "request": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_real_stats_total_hits(agent_host, real_server_id, interval, time_from=None, time_to=None): """ \ No newline at end of file @@ -363,8 +428,14 @@ "params": [agent_host, real_server_id] } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "hits": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_real_connection_count(agent_host, real_server_id, interval, time_from, time_to): """ \ No newline at end of file @@ -374,7 +445,7 @@ query = f""" SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, - COALESCE(MAX(rs_conn_cnt)::INT, 0) AS connections_per_sec + COALESCE(MAX(rs_conn_cnt)::INT, 0) AS real_conn_cnt FROM apv_real_stats WHERE agent_host ~ %s AND real_server_id ~ %s \ No newline at end of file @@ -391,8 +462,14 @@ "params": params } - return DBClient.execute_query(req_dict) + response = DBClient.execute_query(req_dict) + field_casts = { + "real_conn_cnt": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_real_connections_per_sec(agent_host, real_server_id, interval, time_from=None, time_to=None): """ \ No newline at end of file @@ -416,4 +493,11 @@ "query": query, "params": [agent_host, real_server_id] } - return DBClient.execute_query(req_dict) + + response = DBClient.execute_query(req_dict) + + field_casts = { + "connections": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/ssl_vpn_stat_queries.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/ssl_vpn_stat_queries.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/ssl_vpn_stat_queries.py (working copy) @@ -1,5 +1,6 @@ from hive.db.db_client import DBClient from hive.utils import andebug +from hive.db.utils import cast_fields class SSLVpnStatQueries: \ No newline at end of file @@ -13,9 +14,6 @@ if agent_host: query = f""" SELECT DISTINCT ON (id, agent_host, ip) - agent_host, - id as vsite_name, - ip as vsite_ip, CAST(active_sessions AS BIGINT) FROM ag_virtual_site_stats WHERE agent_host = '{agent_host}' \ No newline at end of file @@ -25,9 +23,6 @@ else: query = f""" SELECT DISTINCT ON (id, agent_host, ip) - agent_host, - id as vsite_name, - ip as vsite_ip, CAST(active_sessions AS BIGINT) FROM ag_virtual_site_stats WHERE time > now() - interval '20 seconds' \ No newline at end of file @@ -35,8 +30,14 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "active_sessions": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_login_stats(agent_host, vsite_id, interval="1m", time_from="now() - interval '15 minutes'", time_to="now()"): \ No newline at end of file @@ -48,7 +49,7 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - id as vsite_name, + id, CAST(COALESCE(MAX(success_login) - MIN(success_login), 0) AS BIGINT) AS success, CAST(COALESCE(MAX(failure_login) - MIN(failure_login), 0) AS BIGINT) AS failure, CAST(COALESCE(MAX(error_login) - MIN(error_login), 0) AS BIGINT) AS error, \ No newline at end of file @@ -64,8 +65,18 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "success": int, + "failure": int, + "error": int, + "locked": int, + "rejected": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_net_throughput_stats(agent_host, vsite_id, interval="20s", \ No newline at end of file @@ -78,17 +89,19 @@ SELECT time_bucket_gapfill('{interval}', time, {time_from}, {time_to}) AS ts, agent_host, - id as vsite_name, + id, ROUND(COALESCE( (MAX("client_bytes_out") - MIN("client_bytes_out")) * 8 - / GREATEST(EXTRACT(EPOCH FROM (MAX(time) - MIN(time))), 1) + / GREATEST(EXTRACT(EPOCH FROM (MAX(time) - MIN(time))), 1), + 0 )::numeric, 2 - ) AS sent, + ) AS nw_sent, ROUND(COALESCE( (MAX("client_bytes_in") - MIN("client_bytes_in")) * 8 - / GREATEST(EXTRACT(EPOCH FROM (MAX(time) - MIN(time))), 1) + / GREATEST(EXTRACT(EPOCH FROM (MAX(time) - MIN(time))), 1), + 0 )::numeric, 2 - ) AS received + ) AS nw_recv FROM ag_virtual_site_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -98,8 +111,15 @@ ORDER BY ts; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "nw_sent": float, + "nw_recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_l3_tunnel_stats(agent_host, vsite_id, time_from="now() - interval '15 minutes'", time_to="now()"): """ \ No newline at end of file @@ -110,10 +130,10 @@ SELECT agent_host, id, - CAST((MAX(tunnels_open) - MIN(tunnels_open)) AS BIGINT) AS open, - CAST((MAX(tunnels_est) - MIN(tunnels_est)) AS BIGINT) AS est, - CAST((MAX(tunnels_rejected) - MIN(tunnels_rejected)) AS BIGINT) AS rejected, - CAST((MAX(tunnels_terminated) - MIN(tunnels_terminated)) AS BIGINT) AS terminated + COALESCE(CAST((MAX(tunnels_open) - MIN(tunnels_open)) AS BIGINT), 0) AS open, + COALESCE(CAST((MAX(tunnels_est) - MIN(tunnels_est)) AS BIGINT), 0) AS est, + COALESCE(CAST((MAX(tunnels_rejected) - MIN(tunnels_rejected)) AS BIGINT), 0) AS rejected, + COALESCE(CAST((MAX(tunnels_terminated) - MIN(tunnels_terminated)) AS BIGINT), 0) AS terminated FROM ag_vpn_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -123,13 +143,22 @@ ORDER BY agent_host, id; """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "open": int, + "est": int, + "rejected": int, + "terminated": int + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_l3_vpn_throughput(agent_host, vsite_id, interval="20s", time_from="now() - interval '15 minutes'", time_to="now()"): """ - Get VPN throughput (sent_bps, recv_bps) over time buckets + Get VPN throughput (sent, recv) over time buckets """ query = f""" \ No newline at end of file @@ -137,16 +166,16 @@ time_bucket('{interval}', time) AS ts, agent_host, id, - ROUND(( + COALESCE(ROUND(( (MAX(bytes_out) - MIN(bytes_out)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS sent, - ROUND(( + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS vpn_sent, + COALESCE(ROUND(( (MAX(bytes_in) - MIN(bytes_in)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS received + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS vpn_recv FROM ag_vpn_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -157,13 +186,20 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "vpn_sent": float, + "vpn_recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_l3_client_app_throughput(agent_host, vsite_id, interval="20s", time_from="now() - interval '15 minutes'", time_to="now()"): """ - Get VPN Client Application throughput (sent_bps, recv_bps) over time buckets + Get VPN Client Application throughput (sent, recv) over time buckets """ query = f""" \ No newline at end of file @@ -171,16 +207,16 @@ time_bucket('{interval}', time) AS ts, agent_host, id, - ROUND(( + COALESCE(ROUND(( (MAX(client_app_bytes_out) - MIN(client_app_bytes_out)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS sent, - ROUND(( + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS client_app_sent, + COALESCE(ROUND(( (MAX(client_app_bytes_in) - MIN(client_app_bytes_in)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS received + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS client_app_recv FROM ag_vpn_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -191,8 +227,15 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "client_app_sent": float, + "client_app_recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod def get_l7_request_status(agent_host, vsite_id, interval="20s", time_from="now() - interval '15 minutes'", \ No newline at end of file @@ -206,16 +249,16 @@ time_bucket('{interval}', time) AS ts, agent_host, id, - ROUND(( + COALESCE(ROUND(( (MAX(client_bytes_out) - MIN(client_bytes_out)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) ::numeric, 2) AS sent, - ROUND(( + COALESCE(ROUND(( (MAX(client_bytes_in) - MIN(client_bytes_in)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS received + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS recv FROM ag_web_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -226,42 +269,59 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "sent": float, + "recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response + @staticmethod - def get_l7_client_throughput(agent_host, vsite_id, interval="20s", - time_from="now() - interval '15 minutes'", + def get_l7_client_throughput(agent_host, vsite_id, interval="20s", time_from="now() - interval '15 hours'", time_to="now()"): """ - Get L7 Client Throughput (Sent bps, Received bps) over time buckets + Fetch client sent/recv bps from ag_web_stats """ query = f""" - SELECT - time_bucket('{interval}', time) AS ts, - agent_host, - id, - ROUND(( - (MAX(client_bytes_out) - MIN(client_bytes_out)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - )::numeric, 2) AS sent, - ROUND(( - (MAX(client_bytes_in) - MIN(client_bytes_in)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS received - FROM ag_web_stats - WHERE time >= {time_from} - AND time <= {time_to} - AND agent_host = '{agent_host}' - AND id = '{vsite_id}' - GROUP BY ts, agent_host, id - ORDER BY ts; - """ + SELECT + time_bucket('{interval}', time) AS ts, + agent_host, + id, + COALESCE( + ROUND( + ( (MAX(client_bytes_out) - MIN(client_bytes_out)) * 8 ) / + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))::numeric + , 2) + , 0) AS client_sent, + COALESCE( + ROUND( + ( (MAX(client_bytes_in) - MIN(client_bytes_in)) * 8 ) / + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))::numeric + , 2) + , 0) AS client_recv + FROM ag_web_stats + WHERE time >= {time_from} + AND time <= {time_to} + AND agent_host = '{agent_host}' + AND id = '{vsite_id}' + GROUP BY ts, agent_host, id + ORDER BY ts; + """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + field_casts = { + "client_sent": float, + "client_recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + + return response + def get_l7_server_throughput(agent_host, vsite_id, interval="20s", time_from="now() - interval '15 minutes'", time_to="now()"): \ No newline at end of file @@ -274,16 +334,16 @@ time_bucket('{interval}', time) AS ts, agent_host, id, - ROUND(( + COALESCE(ROUND(( (MAX(server_bytes_out) - MIN(server_bytes_out)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS sent, - ROUND(( + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS server_sent, + COALESCE(ROUND(( (MAX(server_bytes_in) - MIN(server_bytes_in)) * 8 / - GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time))) - ) - ::numeric, 2) AS received + GREATEST(1, EXTRACT(EPOCH FROM MAX(time) - MIN(time)))), + 0) + ::numeric, 2) AS server_recv FROM ag_web_stats WHERE time >= {time_from} AND time <= {time_to} \ No newline at end of file @@ -294,4 +354,11 @@ """ req = {"query": query} - return DBClient.execute_query(req) + response = DBClient.execute_query(req) + + field_casts = { + "server_sent": float, + "server_recv": float + } + response["data"] = cast_fields(response["data"], field_casts) + return response \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/utils.py (nonexistent) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/utils.py (working copy) @@ -0,0 +1,18 @@ +from hive.utils import andebug + + +def cast_fields(rows, field_casts): + """ Cast fields to appropriate cast specified in the fields_casts""" + + for row in rows: + for field, cast_type in field_casts.items(): + try: + if row.get(field) is not None: + row[field] = cast_type(row[field]) + else: + row[field] = 0 + + except (ValueError, TypeError): + # keep original if casting fails + pass + return rows Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/llb_stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py (nonexistent) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/llb_stats.py (working copy) @@ -0,0 +1,88 @@ +import json + +from django.http import JsonResponse + +from cm.lib.libbasic_operation import oper_log +from hive.custom_exceptions import generic_exception as ge +from hive.services.llb_stats_service import fetch_llb_stats, fetch_llb_monitoring_data, fetch_llb_historical_data, \ + fetch_llb_summary_data +from hive.utils import andebug + + +def handle_llb_stats_req(request, path=None): + try: + if request.method in 'POST': + return get_llb_stats(request) + else: + return JsonResponse(json.dumps({ + 'error': 405, + 'message': "Invalid HTTP method" + }), status=405, content_type='application/json') + except ge.GenericError as e: + oper_log('error', 'system', e.message) + return ge.handle_exception(e) + except Exception as e: + oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e)) + e.message = 'An Unexpected error occurred, details: {}'.format(e) + return ge.handle_exception(e) + + +def get_llb_stats(request): + """ Fetch data for LLB stats data """ + req = json.loads(request.body) + stat_name = req.get("stat_name", None) + + if stat_name == 'monitoring_data': + return get_monitoring_data(request) + elif stat_name == 'historical_data': + return get_historical_data(request) + elif stat_name == 'summary_data': + return get_summary_data(request) + else: + time_from = req.get("from", None) + time_to = req.get("to", None) + agent_host = req.get("agent_host", None) + return JsonResponse(fetch_llb_stats(agent_host, time_from, time_to)) + + +def get_monitoring_data(request): + """ Fetch data for LLB data for monitoring page """ + req = json.loads(request.body) + agent_host = req.get("agent_host", None) + time_from = req.get("from", None) + time_to = req.get("to", None) + + if agent_host is None: + oper_log('error', 'system', "agent_host details are not specified. Please provide value for agent_host param.") + raise ge.GenericError(400, "agent_host details are not specified. Please provide value for agent_host param.") + + return JsonResponse(fetch_llb_monitoring_data(agent_host, time_from, time_to)) + + +def get_historical_data(request): + """ Fetch data for LLB graphs for a particular time period""" + req = json.loads(request.body) + + time_from = req.get("from", None) + time_to = req.get("to", None) + interval = req.get("interval", None) + link_name = req.get("link_name", None) + + if time_from is None: + oper_log('error', 'system', "'from' time details are not specified. Please specify value for 'from' time param.") + raise ge.GenericError(400, "'from' time details are not specified. Please specify value for 'from' time param.") + elif time_to is None: + oper_log('error', 'system', "'to' time details are not specified. Please specify value for 'to' time param.") + raise ge.GenericError(400, "'to' time details are not specified. Please specify value for 'to' time param.") + + return JsonResponse(fetch_llb_historical_data(link_name, time_from, time_to, interval)) + + +def get_summary_data(request): + """ Fetch data for LLB graphs for a particular time period""" + req = json.loads(request.body) + + time_from = req.get("from", None) + time_to = req.get("to", None) + + return JsonResponse(fetch_llb_summary_data(time_from, time_to)) \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py (nonexistent) @@ -1,200 +0,0 @@ -import json - -from django.http import HttpResponse - -from cm.lib.libbasic_operation import oper_log -from hive.custom_exceptions import generic_exception as ge -from hive.db.llb_stats_queries import LLBStatsDB -from hive.utils import andebug - - -def handle_llb_stats_req(request, path=None): - try: - if path.startswith('stats') and request.method == 'POST': - if path == 'stats': - return get_llb_stats(request) - elif path == 'stats/historical': - return get_llb_historical_data(request) - return None - elif path == 'monitoring_data' and request.method == 'POST': - return get_monitoring_data(request) - else: - return HttpResponse(json.dumps({ - 'error': 405, - 'message': "Invalid HTTP method" - }), content_type='application/json') - except ge.GenericError as e: - oper_log('error', 'system', e.message) - return ge.handle_exception(e) - except Exception as e: - oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e)) - e.message = 'An Unexpected error occurred, details: {}'.format(e) - return ge.handle_exception(e) - - -def get_llb_stats(request): - """ Fetch data for LLB stats data """ - agent_host = None - req = json.loads(request.body) - if 'query' in req: - if 'agent_host' in req['query']: - agent_host = req['query']['agent_host'] - - else: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - - response = LLBStatsDB.get_llb_stats_data(agent_host) - return get_formatted_response(response, 'stats') - - -def get_monitoring_data(request): - """ Fetch data for LLB data for monitoring page """ - req = json.loads(request.body) - agent_host = None - if 'query' in req: - if 'agent_host' in req['query']: - agent_host = req['query']['agent_host'] - - else: - oper_log('error', 'system', "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - raise ge.GenericError(400, "Query must contain only agent_host details. Please specify agent_host in the " - "query.") - - response = LLBStatsDB.get_llb_monitoring_data(agent_host) - return get_formatted_response(response, 'monitoring_data') - - -def get_llb_historical_data(request): - """ Fetch data for LLB graphs for a particular time period""" - req = json.loads(request.body) - - if 'query' in req and 'from_time' in req['query'] and 'to_time' in req['query'] and 'link_name' in \ - req['query']: - from_time = req['query'].get('from_time', '') - to_time = req['query'].get('to_time', '') - link_name = req['query'].get('link_name', '') - else: - oper_log('error', 'system', - "Query cannot be empty. Please specify from_time and to_time and link_name in the query.") - raise ge.GenericError(400, "Query cannot be empty. Please specify from_time and to_time and link_name in " - "the query.") - - response = LLBStatsDB.get_llb_historical_data(from_time, to_time, link_name) - return get_formatted_response(response, 'historical') - - -def get_formatted_response(response, type): - if not response: - return [] - - if isinstance(response, str): - response = json.loads(response) - - response_data = reformat_data_from_db(response, type) - - if response_data: - return HttpResponse(json.dumps(response_data), content_type='application/json') - else: - if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): - return HttpResponse(json.dumps({ - "error": "No valid data found" - }), content_type='application/json') - else: - return HttpResponse(json.dumps(response), content_type='application/json') - - -def reformat_data_from_db(response, data_type): - response_data = [] - results = response.get("results") - # Extract the relevant series from the results - for result in results: - if result is None: - pass - series = result.get("series") - if series: - for series_data in series: - series_obj = {} - columns = series_data.get("columns", []) - values = series_data.get("values", []) - tags = series_data.get("tags") - if tags: - series_obj.update(tags) # Add tag values directly to the response - attributes = list(map(str, columns)) - series_obj['attributes'] = attributes - - # Get column indices for easier lookup - col_indices = {col: idx for idx, col in enumerate(columns)} - - # Process each data point - units = [] - unit = 0 - for value in values: - if data_type == 'historical': - unit = get_llb_historical_col_values(col_indices, value) - elif data_type == 'monitoring_data': - unit = get_llb_monitoring_col_values(col_indices, value) - elif data_type == 'stats': - unit = get_llb_stats_col_values(col_indices, value) - - units.append(unit) - - series_obj['values'] = units - response_data.append(series_obj) - - return response_data - - -def get_llb_historical_col_values(col_indices, value): - """ Get Column values LLB Historical """ - return [value[col_indices["time"]] if "time" in col_indices else 0, - round(value[col_indices["avgBandwidIn"]], - 2) if "avgBandwidIn" in col_indices else 0, - round(value[col_indices["avgBandwidOut"]], - 2) if "avgBandwidOut" in col_indices else 0, - round(value[col_indices["avgLinkUsage"]], - 2) if "avgLinkUsage" in col_indices else 0, - int(value[col_indices["avgLinkConn"]]) if "avgLinkConn" in col_indices else 0, - int(value[col_indices["avgLinkHits"]]) if "avgLinkHits" in col_indices else 0] - - -def get_llb_monitoring_col_values(col_indices, value): - """ Get Column values LLB Monitoring """ - return [value[col_indices["time"]] if "time" in col_indices else 0] + [ - value[col_indices[key]] for key in [ - "top_hits", "top_conn", "total_throughput", "received", "sent", "linkName", "agent_host" - ] if key in col_indices - ] - - -def get_llb_stats_col_values(col_indices, value): - """ Get Column values LLB Stats """ - andebug('an.model.cli', 'Inside get_llb_stats_col_values') - andebug('an.model.cli', 'value: {}'.format(value)) - andebug('an.model.cli', 'Time Value: {}'.format(str(value[col_indices["time"]]))) - andebug('an.model.cli', 'linkGateway: {}'.format(str(value[col_indices["linkGateway"]]))) - andebug('an.model.cli', 'linkThresh: {}'.format(str(round(value[col_indices["linkThresh"]], - 2)))) - andebug('an.model.cli', 'linkUsage: {}'.format(str(round(value[col_indices["linkUsage"]], - 2)))) - andebug('an.model.cli', 'received: {}'.format(str(round(value[col_indices["received"]], - 2)))) - andebug('an.model.cli', 'sent: {}'.format(str(round(value[col_indices["sent"]], - 2)))) - andebug('an.model.cli', 'linkConn: {}'.format(str(int(value[col_indices["linkConn"]])))) - andebug('an.model.cli', 'linkHits: {}'.format(str(int(value[col_indices["linkHits"]])))) - return [value[col_indices["time"]] if "time" in col_indices else 0, - value[col_indices["linkGateway"]] if "linkGateway" in col_indices else 0, - round(value[col_indices["linkThresh"]], - 2) if "linkThresh" in col_indices else 0, - round(value[col_indices["linkUsage"]], - 2) if "linkUsage" in col_indices else 0, - round(value[col_indices["received"]], - 2) if "received" in col_indices else 0, - round(value[col_indices["sent"]], - 2) if "sent" in col_indices else 0, - int(value[col_indices["linkConn"]]) if "linkConn" in col_indices else 0, - int(value[col_indices["linkHits"]]) if "linkHits" in col_indices else 0] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py (working copy) @@ -7,7 +7,7 @@ from cm.lib.libbasic_operation import oper_log from hive.custom_exceptions import generic_exception as ge from hive.db.llb_stats_queries import LLBStatsDB -from hive.llb_stats import reformat_data_from_db +# from hive.controller.llb_stats import reformat_data_from_db from hive.report.generate_pdf_report import Report REPORT_FILE_PATH = "/ca/webui/htdocs/new/src/hive/media/docs/" @@ -91,29 +91,29 @@ if isinstance(response, str): response = json.loads(response) - response_data = reformat_data_from_db(response, 'historical') + # response_data = reformat_data_from_db(response, 'historical') llb_dict = {} - if response_data: - for res_data in response_data: - data = res_data.get("values") - link_name = res_data.get("linkName") - for llb_point in data: - llb_dict[link_name] = { - 'time': llb_point[0], - 'avg_link_hits': llb_point[1], - 'avg_link_conn': llb_point[2], - 'avg_bandwidth_in': llb_point[3], - 'avg_bandwidth_out': llb_point[4], - 'avg_link_usage': llb_point[5], - } - else: - if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): - return HttpResponse(json.dumps({ - "error": "No valid data found" - }), content_type='application/json') - else: - return HttpResponse(json.dumps(response), content_type='application/json') + # if response_data: + # for res_data in response_data: + # data = res_data.get("values") + # link_name = res_data.get("linkName") + # for llb_point in data: + # llb_dict[link_name] = { + # 'time': llb_point[0], + # 'avg_link_hits': llb_point[1], + # 'avg_link_conn': llb_point[2], + # 'avg_bandwidth_in': llb_point[3], + # 'avg_bandwidth_out': llb_point[4], + # 'avg_link_usage': llb_point[5], + # } + # else: + # if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): + # return HttpResponse(json.dumps({ + # "error": "No valid data found" + # }), content_type='application/json') + # else: + # return HttpResponse(json.dumps(response), content_type='application/json') return llb_dict except Exception as e: @@ -139,36 +139,36 @@ if isinstance(response, str): response = json.loads(response) - response_data = reformat_data_from_db(response, 'historical') + # response_data = reformat_data_from_db(response, 'historical') llb_dict = {} - if response_data: - for res_data in response_data: - data = res_data.get("values") - link_name = res_data.get("linkName") - for llb_point in data: - time_list.append(llb_point[0]) - inbound_list.append(llb_point[1]) - outbound_list.append(llb_point[2]) - bandwidth_usage_list.append(llb_point[3]) - cc_list.append(llb_point[4]) - hit_list.append(llb_point[5]) - - llb_dict[link_name] = { - 'time': time_list, - 'inbound_list': inbound_list, - 'outbound_list': outbound_list, - 'bandwidth_usage_list': bandwidth_usage_list, - 'cc_list': cc_list, - 'hit_list': hit_list, - } - else: - if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): - return HttpResponse(json.dumps({ - "error": "No valid data found" - }), content_type='application/json') - else: - return HttpResponse(json.dumps(response), content_type='application/json') + # if response_data: + # for res_data in response_data: + # data = res_data.get("values") + # link_name = res_data.get("linkName") + # for llb_point in data: + # time_list.append(llb_point[0]) + # inbound_list.append(llb_point[1]) + # outbound_list.append(llb_point[2]) + # bandwidth_usage_list.append(llb_point[3]) + # cc_list.append(llb_point[4]) + # hit_list.append(llb_point[5]) + # + # llb_dict[link_name] = { + # 'time': time_list, + # 'inbound_list': inbound_list, + # 'outbound_list': outbound_list, + # 'bandwidth_usage_list': bandwidth_usage_list, + # 'cc_list': cc_list, + # 'hit_list': hit_list, + # } + # else: + # if 'results' in response and all(len(res) == 1 and 'statement_id' in res for res in response['results']): + # return HttpResponse(json.dumps({ + # "error": "No valid data found" + # }), content_type='application/json') + # else: + # return HttpResponse(json.dumps(response), content_type='application/json') return llb_dict except Exception as e: return HttpResponse(json.dumps({"error": "Error while fetching LLB data", Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/device_metrics_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/device_metrics_service.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/device_metrics_service.py (working copy) @@ -1,6 +1,6 @@ from hive.db.device_metric_queries import DeviceMetrics -from hive.db.utils import Utils from hive.services.utils import construct_json_response, convert_time +from hive.utils import andebug def fetch_device_metrics(agent_host, interval, metric_name, time_from, time_to): @@ -35,8 +35,8 @@ else: device_stats = DeviceMetrics.get_device_stats(time_from, time_to)['data'] net_throughput = DeviceMetrics.get_throughput_data(interval, time_from, time_to)['data'] - json_response = construct_json_response(json_response, { + json_response = { "device_stats": device_stats, "net_throughput": net_throughput - }) + } return json_response Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/llb_stats_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/llb_stats_service.py (nonexistent) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/llb_stats_service.py (working copy) @@ -0,0 +1,56 @@ +import json + +from hive.db.llb_stats_queries import LLBStatsDB +from hive.services.utils import convert_time +from hive.utils import andebug + + +def fetch_llb_stats(agent_host, time_from, time_to): + """ + Fetch LLB stats. + """ + time_from = convert_time(time_from, "now() - interval '15 minutes'") + time_to = convert_time(time_to, "now()") + + stats = LLBStatsDB.get_llb_stats_data(agent_host, time_from, time_to) + return stats # Direct dict/list, JsonResponse will handle JSON conversion + + +def fetch_llb_monitoring_data(agent_host, time_from, time_to): + """ + Fetch top_hits, top_conn, and top_throughput stats. + """ + time_from = convert_time(time_from, "now() - interval '1 hour'") + time_to = convert_time(time_to, "now()") + + top_hits = LLBStatsDB.get_top_hits(agent_host, time_from, time_to) + top_conn = LLBStatsDB.get_top_conn(agent_host, time_from, time_to) + top_throughput = LLBStatsDB.get_top_throughput(agent_host, time_from, time_to) + + return { + "top_hits": top_hits, + "top_conn": top_conn, + "top_throughput": top_throughput + } + + +def fetch_llb_historical_data(link_name, time_from, time_to, interval="20s"): + """ + Fetch historical LLB stats over time buckets. + """ + time_from = convert_time(time_from, "now() - interval '15 minutes'") + time_to = convert_time(time_to, "now()") + + historical = LLBStatsDB.get_llb_historical_data(time_from, time_to, link_name, interval) + return historical + + +def fetch_llb_summary_data(time_from, time_to): + """ + Fetch LLB Summary stats over time buckets. + """ + time_from = convert_time(time_from, "now() - interval '15 minutes'") + time_to = convert_time(time_to, "now()") + + historical = LLBStatsDB.get_llb_summary_data(time_from, time_to) + return historical \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/slb_stats_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/slb_stats_service.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/slb_stats_service.py (working copy) @@ -1,6 +1,6 @@ from hive.db.slb_stats_queries import SLBStats -from hive.db.utils import Utils -from hive.services.utils import convert_time +from hive.services.utils import construct_json_response, convert_time +from hive.utils import andebug def fetch_virtual_stats(agent_host, interval, req, server_id, time_from, time_to): \ No newline at end of file @@ -38,4 +38,4 @@ response = SLBStats.get_real_connections_per_sec(agent_host, server_id, interval, time_from, time_to) else: response = SLBStats.get_real_service_stats(agent_host) - return response + return response \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/ssl_vpn_stat_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/ssl_vpn_stat_service.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/ssl_vpn_stat_service.py (working copy) @@ -1,8 +1,12 @@ from hive.db.ssl_vpn_stat_queries import SSLVpnStatQueries from hive.services.utils import construct_json_response, convert_time +from hive.utils import andebug + + def fetch_ssl_vpn_stats(agent_host, vsite_id, interval, stat_name, time_from, time_to): time_from = convert_time(time_from, "now() - interval '15 minutes'") time_to = convert_time(time_to, "now()") + json_response = {} if stat_name == 'login_status': stats = SSLVpnStatQueries.get_login_stats(agent_host, vsite_id, interval, time_from, time_to) \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/utils.py (revision 2710) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/utils.py (working copy) @@ -33,7 +33,7 @@ }), content_type="application/json", status=500) -def construct_json_response(json_response, metrics): +def construct_json_response(metrics): json_response = { "data": metrics["data"] }