Index: /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh =================================================================== --- /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh (revision 39189) +++ /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh (working copy) @@ -73,6 +73,8 @@ then sh /var/crash/gslb_monitor.sh fi + # insert data into database of gslb statistics + /usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py gslbstatus #ORCH module if [ -f /var/crash/orch_monitor.sh ] && [ ${ORCH_INIT} -eq 1 ] then Index: /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py =================================================================== --- /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py (revision 39189) +++ /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py (working copy) @@ -5,6 +5,7 @@ C_PATH_TCP_STATUS_DB = "/var/crash/tcp_status/" C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/" C_PATH_SSL_STATUS_DB = "/var/crash/ssl_status/" +C_PATH_GSLB_STATUS_DB = "/var/crash/gslb_statistics/" def check_memory(db_path): if not os.path.isdir(db_path): @@ -36,3 +37,4 @@ check_memory(C_PATH_TCP_STATUS_DB) check_memory(C_PATH_TCP_SYN_DROP_DB) check_memory(C_PATH_SSL_STATUS_DB) + check_memory(C_PATH_GSLB_STATUS_DB) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/DatabaseService.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/DatabaseService.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/DatabaseService.py (working copy) @@ -0,0 +1,142 @@ +from hive.monitor_log.GSLBStatistics import ( + C_DB_HOST_STATISTICS, + C_DB_SERVICE_STATISTICS, + C_PATH_GSLB_STATUS_DB +) +import os +import sqlite3 +import time + +class DatabaseService(): + + def __init__(self): + pass + + def get_sql_with_time_range_for_top_host(self, time_range, num): + sql = """SELECT + host_name, + INCREMENT(host_hits) AS host_hits_all + FROM %s + WHERE time BETWEEN ? AND ? + GROUP BY host_name + ORDER BY host_hits_all DESC + LIMIT ? + """ % C_DB_HOST_STATISTICS + mapping = ( + time_range.ParseFrom(), + time_range.ParseTo(), + num + ) + return sql, mapping + + def get_sql_with_time_range_for_top_service(self, time_range, num): + sql = """SELECT + service_name, + INCREMENT(service_hits) AS service_hits_all + FROM %s + WHERE time BETWEEN ? AND ? + GROUP BY service_name + ORDER BY service_hits_all DESC + LIMIT ? + """ % C_DB_SERVICE_STATISTICS + mapping = ( + time_range.ParseFrom(), + time_range.ParseTo(), + num + ) + return sql, mapping + + def get_sql_with_time_range_for_host(self, time_range, name): + time_interval = time_range.Interval() + sql = """SELECT timef, MAX(host_hits) + FROM ( + SELECT + ((time / ?) * ?) AS timef, + host_hits, + host_name + FROM %s + ) AS sub + WHERE timef BETWEEN ? AND ? + AND host_name = ? + GROUP BY timef + """ % C_DB_HOST_STATISTICS + mapping = ( + time_interval, + time_interval, + time_range.ParseFrom(), + time_range.ParseTo(), + name + ) + return sql, mapping + + def get_sql_with_time_range_for_service(self, time_range, name): + time_interval = time_range.Interval() + sql = """SELECT timef, MAX(service_hits) + FROM ( + SELECT + ((time / ?) * ?) AS timef, + service_hits, + service_name + FROM %s + ) AS sub + WHERE timef BETWEEN ? AND ? + AND service_name = ? + GROUP BY timef + """ % C_DB_SERVICE_STATISTICS + mapping = ( + time_interval, + time_interval, + time_range.ParseFrom(), + time_range.ParseTo(), + name + ) + return sql, mapping + + def get_result_by_sql(self, db_list, sql, mapping): + if not os.path.isfile(C_PATH_GSLB_STATUS_DB + db_list): + return [] + conn = sqlite3.connect(C_PATH_GSLB_STATUS_DB + db_list) + conn.create_aggregate("INCREMENT", 1, CustomerIncrement) + db = conn.cursor() + try: + db.execute(sql, mapping) + except sqlite3.OperationalError as e: + if str(e).find("locked") != -1: + time.sleep(1) + try: + db.execute(sql, mapping) + except sqlite3.OperationalError as e: + if str(e).find("locked") != -1: + time.sleep(1) + try: + db.execute(sql, mapping) + except sqlite3.OperationalError as e: + return [] + except sqlite3.DatabaseError as e: + # file is encrypted or is not a database + return [] + + result = db.fetchall() + conn.close() + return result + + def orderByLast(self, elem): + return elem[-1] + +class CustomerIncrement(object): + def __init__(self): + self.prev = None + self.total = 0 + + def step(self, value): + value = int(value) + if self.prev is None: + pass # Skip sum up for the first row + elif value < self.prev: + self.total += value + else: + self.total += value - self.prev + self.prev = value + + def finalize(self): + return self.total Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/TopologyService.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/TopologyService.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/TopologyService.py (working copy) @@ -0,0 +1,260 @@ +from hive.monitor_log.GSLBStatistics import ( + HOST_STATISTICS, + SERVICE_STATISTICS, + MONITOR_COUNT +) +class CLI_RESPONSE(): + MSG = "msg" + POOL_NAME = "pool_name" + SERVICE_NAME = "service_name" + +class BASIC_COLUMN(): + TTL = "ttl" + FOWARD_ONLY = "forward_only" + +class POLICY_STATISTICS(): + POLICY_NAME = "Policy Name" + POOL_NAME = "Pool Name" + DEFAULT_POLICY = "Default Policy" + DEFAULT_POOL_NAME = "Default Pool" + RESORT_POLICY = "Resort Policy" + RESORT_POOL_NAME = "Resort Pool" + HOST_NAME = "Host Name" + +class POLICY_FIELDS(): + FIELD = "policies" + NAME = "name" + TYPE = "type" + POOL = "pool" + SERVICES = "services" + HOST = "host" + +class POLICY_TYPE(): + DEFAULT = 1 + REGION = 2 + RESORT = 3 + +class POOL_FIELDS(): + NAME = "name" + STATISTICS = "statistics" + +class TopologyService(): + + def __init__(self, session): + self.sess = session + def get_basic(self, hostname): + manager = self.sess.get_model_manager('apv', 'loadbalancing.gslb.host.Host') + get_host_data = manager.all() + output = {} + for host_obj in get_host_data: + if host_obj.name != hostname: + continue + output = { + BASIC_COLUMN.TTL: host_obj.ttl, + BASIC_COLUMN.FOWARD_ONLY: host_obj.forward_only + } + break + return output + def get_host_statistics(self, hostname): + manager = self.sess.get_model_manager('apv', 'loadbalancing.gslb.host.HostStatistics') + statistics = manager.perform_action("Filter", {"host_name": hostname}) + output = {} + if CLI_RESPONSE.MSG not in statistics: + return output + if not statistics[CLI_RESPONSE.MSG]: + return output + map = { + HOST_STATISTICS.HIT: HOST_STATISTICS.COL_HIT, + HOST_STATISTICS.DEFAULT_POOL: HOST_STATISTICS.COL_DEFAULT_POOL, + HOST_STATISTICS.DEFAULT_HIT: HOST_STATISTICS.COL_DEFAULT_HIT, + HOST_STATISTICS.RESORT_POOL: HOST_STATISTICS.COL_RESORT_POOL, + HOST_STATISTICS.RESORT_HIT: HOST_STATISTICS.COL_RESORT_HIT, + HOST_STATISTICS.REGION_COUNT: HOST_STATISTICS.COL_REGION_COUNT, + HOST_STATISTICS.REGION_HIT: HOST_STATISTICS.COL_REGION_HIT, + } + int_map = [ + HOST_STATISTICS.HIT, + HOST_STATISTICS.DEFAULT_HIT, + HOST_STATISTICS.RESORT_HIT, + HOST_STATISTICS.REGION_COUNT, + HOST_STATISTICS.REGION_HIT + ] + for each in statistics[CLI_RESPONSE.MSG].split("\n"): + if not each: + continue + sp = each.strip().split(":") + if len(sp) < 2: + continue + if sp[0] not in map: + continue + value = sp[1].strip() + if sp[0] in int_map: + value = int(value) + output[map[sp[0]]] = value + return output + def get_host_topology(self, hostname): + policy_pool_map = self.get_policy_pool_mapping(hostname) + pool_service_map = self.get_pool_service_mapping() + service_statistics_map = self.get_service_statistics_mapping() + service_bind_pool = self.get_services_data_bind_pools(pool_service_map, service_statistics_map) + return self.handle_output_of_policy(policy_pool_map, + service_bind_pool) + def get_policy_pool_mapping(self, hostname): + manager = self.sess.get_model_manager('apv', 'loadbalancing.gslb.policy.PolicyStatistics') + statistics = manager.perform_action("Filter_by_host_stats", {"host_name": hostname}) + result = {} + if CLI_RESPONSE.MSG not in statistics: + return result + if not statistics[CLI_RESPONSE.MSG]: + return result + current_policy = "" + type_map = { + POLICY_STATISTICS.POOL_NAME: POLICY_TYPE.REGION, + POLICY_STATISTICS.DEFAULT_POOL_NAME: POLICY_TYPE.DEFAULT, + POLICY_STATISTICS.RESORT_POOL_NAME: POLICY_TYPE.RESORT + } + for each in statistics[CLI_RESPONSE.MSG].split("\n"): + if not each: + continue + sp = each.strip().split(":") + if len(sp) < 2: + continue + if sp[0] == POLICY_STATISTICS.POLICY_NAME: + current_policy = sp[1].strip() + if current_policy not in result: + result[current_policy] = {} + if sp[0] == POLICY_STATISTICS.POOL_NAME \ + or sp[0] == POLICY_STATISTICS.DEFAULT_POOL_NAME \ + or sp[0] == POLICY_STATISTICS.RESORT_POOL_NAME: + if current_policy in result: + if POLICY_FIELDS.POOL not in result[current_policy]: + result[current_policy][POLICY_FIELDS.POOL] = {} + result[current_policy][POLICY_FIELDS.POOL][POLICY_FIELDS.NAME] = sp[1].strip() + result[current_policy][POLICY_FIELDS.POOL][POLICY_FIELDS.TYPE] = type_map[sp[0]] + result[current_policy][POLICY_FIELDS.POOL][POLICY_FIELDS.SERVICES] = [] + if sp[0] == POLICY_STATISTICS.HOST_NAME: + if current_policy in result: + if POLICY_FIELDS.POOL not in result[current_policy]: + result[current_policy][POLICY_FIELDS.POOL] = {} + result[current_policy][POLICY_FIELDS.POOL][POLICY_FIELDS.HOST] = sp[1].strip() + return result + def get_pool_service_mapping(self): + manager = self.sess.get_model_manager('apv', 'loadbalancing.gslb.pool.PoolMember') + get_pool_data = manager.all() + result = {} + for pool_obj in get_pool_data: + pool_name = pool_obj.pool[0][CLI_RESPONSE.POOL_NAME] + if pool_name not in result: + result[pool_name] = [] + result[pool_name].append(pool_obj.service[0][CLI_RESPONSE.SERVICE_NAME]) + return result + def get_service_statistics_mapping(self): + manager = self.sess.get_model_manager('apv', 'loadbalancing.gslb.service.ServiceStatistics') + get_service_data = manager.stats_all() + result = {} + if not get_service_data[0].statistics: + return result + service_map = { + SERVICE_STATISTICS.SERVICE_NAME: SERVICE_STATISTICS.COL_SERVICE_NAME, + SERVICE_STATISTICS.IP_ADDRESS: SERVICE_STATISTICS.COL_IP_ADDRESS, + SERVICE_STATISTICS.PORT: SERVICE_STATISTICS.COL_PORT, + SERVICE_STATISTICS.HEALTH: SERVICE_STATISTICS.COL_HEALTH, + SERVICE_STATISTICS.DISABLED: SERVICE_STATISTICS.COL_DISABLED, + SERVICE_STATISTICS.RELATION: SERVICE_STATISTICS.COL_RELATION, + SERVICE_STATISTICS.HITS: SERVICE_STATISTICS.COL_HITS, + SERVICE_STATISTICS.MONITOR_COUNT: SERVICE_STATISTICS.COL_MONITOR_COUNT, + } + monitor_map = { + MONITOR_COUNT.INSTANCE_ID: MONITOR_COUNT.COL_INSTANCE_ID, + MONITOR_COUNT.NAME: MONITOR_COUNT.COL_NAME, + MONITOR_COUNT.STATUS: MONITOR_COUNT.COL_STATUS, + MONITOR_COUNT.TYPE: MONITOR_COUNT.COL_TYPE, + MONITOR_COUNT.ADDRESS: MONITOR_COUNT.COL_ADDRESS, + MONITOR_COUNT.SOURCE: MONITOR_COUNT.COL_SOURCE, + MONITOR_COUNT.GATEWAY: MONITOR_COUNT.COL_GATEWAY, + } + int_map = [ + SERVICE_STATISTICS.HITS, + SERVICE_STATISTICS.MONITOR_COUNT, + SERVICE_STATISTICS.PORT, + MONITOR_COUNT.INSTANCE_ID, + ] + current_service = "" + for each in get_service_data[0].statistics.split("\n"): + if not each: + continue + sp = each.strip().split(":") + if len(sp) < 2: + continue + field = sp[0].strip() + value = sp[1].strip() + if field in int_map: + value = int(value) + if field == SERVICE_STATISTICS.SERVICE_NAME \ + and field not in result: + current_service = value + result[current_service] = {} + continue + if field in service_map: + service_field = service_map[field] + if current_service in result: + result[current_service][service_field] = value + if field in monitor_map: + monitor_field = monitor_map[field] + if current_service in result: + result[current_service][monitor_field] = value + return result + def get_services_data_bind_pools(self, pools, statistics): + result = {} + for pool, services in pools.items(): + if pool not in result: + result[pool] = [] + for service in services: + if service not in statistics: + continue + result[pool].append({ + POOL_FIELDS.NAME: service, + POOL_FIELDS.STATISTICS: statistics[service] + }) + return result + def get_hostname_and_topology_by_service(self, service_name): + pool_service_map = self.get_pool_service_mapping() + belong_pool = {} + for pool_name, services in pool_service_map.items(): + if service_name in services: + belong_pool[pool_name] = [service_name] + break + policy_pool_map = self.get_policy_pool_mapping("ALL") + belong_policy = {} + hostname = "" + for policy, pool in policy_pool_map.items(): + if POLICY_FIELDS.POOL not in pool \ + or not pool[POLICY_FIELDS.POOL]: + continue + if POLICY_FIELDS.NAME in pool[POLICY_FIELDS.POOL] \ + and pool[POLICY_FIELDS.POOL][POLICY_FIELDS.NAME] in belong_pool: + if POLICY_FIELDS.HOST in pool[POLICY_FIELDS.POOL]: + hostname = pool[POLICY_FIELDS.POOL][POLICY_FIELDS.HOST] + belong_policy[policy] = pool + service_statistics_map = self.get_service_statistics_mapping() + service_bind_pool = self.get_services_data_bind_pools(belong_pool, service_statistics_map) + return hostname, self.handle_output_of_policy(belong_policy, service_bind_pool) + def handle_output_of_policy(self, policy_pool_map, service_bind_pool): + output = { + POLICY_FIELDS.FIELD: [] + } + for policy, each in policy_pool_map.items(): + if POLICY_FIELDS.POOL not in each \ + and POOL_FIELDS.NAME not in each[POLICY_FIELDS.POOL]: + continue + pool_name = each[POLICY_FIELDS.POOL][POOL_FIELDS.NAME] + if pool_name in service_bind_pool \ + and POLICY_FIELDS.SERVICES in each[POLICY_FIELDS.POOL]: + each[POLICY_FIELDS.POOL][POLICY_FIELDS.SERVICES] = service_bind_pool[pool_name] + # no need to output this field + del each[POLICY_FIELDS.POOL][POLICY_FIELDS.HOST] + output[POLICY_FIELDS.FIELD].append({ + POLICY_FIELDS.NAME: policy, + POLICY_FIELDS.POOL: each[POLICY_FIELDS.POOL] + }) + return output Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/__init__.py (added) =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/__init__.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/__init__.py (revision 0) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/router.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/router.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/gslb_monitor/router.py (working copy) @@ -0,0 +1,220 @@ +from django.http import HttpResponse, Http404 +from hive.utils import get_current_session +import json +from apv.gslb_monitor.TopologyService import TopologyService +from hive.exceptions import BadRequestException +from hive.metric import TimeRange +from apv.gslb_monitor.DatabaseService import DatabaseService +import math + +C_RESPONSE_CODE_SUCCESS = 1 +C_RESPONSE_CODE_FAILED = 0 + +C_CONTENT_TYPE_JSON = 'application/json' + +C_ROW_LIMIT = 10 + +C_TYPE_HOST = 1 +C_TYPE_SERVICE = 2 + +class API_RESPONSE(): + RES = "result" + DATA = "data" + MSG = "msg" + + BASIC = "basic" + TOPOLOGY = "topology" + STATISTICS = "statictics" + +class API_METHOD(): + POST = 'POST' + GET = 'GET' + +class API_PARAM(): + HOST = "host" + SERVICE = "service" + +class TOP_CHART_PARAM(): + TYPE = 'type' + FROM = 'from' + TO = 'to' + COUNT = 'count' + +class HIT_CHART_PARAM(): + TYPE = 'type' + NAME = 'name' + FROM = 'from' + TO = 'to' + +def get_topology(request): + if request.method != API_METHOD.GET: + return HttpResponse(status=405) + if API_PARAM.HOST not in request.GET and API_PARAM.SERVICE not in request.GET: + raise Http404('Page not found') + if API_PARAM.HOST in request.GET and not request.GET[API_PARAM.HOST]: + raise Http404('Page not found') + if API_PARAM.SERVICE in request.GET and not request.GET[API_PARAM.SERVICE]: + raise Http404('Page not found') + session = get_current_session() + gslb_monitor_service = TopologyService(session) + if API_PARAM.HOST in request.GET: + hostname = request.GET[API_PARAM.HOST] + topology = gslb_monitor_service.get_host_topology(hostname) + elif API_PARAM.SERVICE in request.GET: + service_name = request.GET[API_PARAM.SERVICE] + hostname, topology = gslb_monitor_service.get_hostname_and_topology_by_service(service_name) + output = { + API_RESPONSE.RES: C_RESPONSE_CODE_SUCCESS, + API_RESPONSE.DATA: { + API_RESPONSE.BASIC: gslb_monitor_service.get_basic(hostname), + API_RESPONSE.TOPOLOGY: topology, + API_RESPONSE.STATISTICS: gslb_monitor_service.get_host_statistics(hostname) + }, + } + return HttpResponse(json.dumps(output), content_type=C_CONTENT_TYPE_JSON) + +def get_top_chart_data(request): + if request.method != API_METHOD.POST: + return HttpResponse(status=405) + param = json.loads(request.body) + if not isinstance(param, dict): + return HttpResponse(status=405) + ret = { + API_RESPONSE.RES: C_RESPONSE_CODE_FAILED, + API_RESPONSE.DATA: [], + API_RESPONSE.MSG: "" + } + valid_param = [ + TOP_CHART_PARAM.TYPE, + TOP_CHART_PARAM.FROM, + TOP_CHART_PARAM.TO, + ] + valid_type = [ + C_TYPE_HOST, + C_TYPE_SERVICE, + ] + for val in valid_param: + if val not in param: + return response_error_message(ret, "Please check the parameter") + if not param[val]: + return response_error_message(ret, "Please check the parameter") + if val == TOP_CHART_PARAM.TYPE \ + and int(param[val]) not in valid_type: + return response_error_message(ret, "Please check the parameter") + count = C_ROW_LIMIT + if TOP_CHART_PARAM.COUNT in param \ + and param[TOP_CHART_PARAM.COUNT] \ + and int(param[TOP_CHART_PARAM.COUNT]): + count = int(param[TOP_CHART_PARAM.COUNT]) + try: + time_range = TimeRange(param[TOP_CHART_PARAM.FROM], param[TOP_CHART_PARAM.TO]) + db_service = DatabaseService() + func_map = { + C_TYPE_HOST: (db_service.get_sql_with_time_range_for_top_host, [time_range, count]), + C_TYPE_SERVICE: (db_service.get_sql_with_time_range_for_top_service, [time_range, count]), + } + func, args = func_map[int(param[TOP_CHART_PARAM.TYPE])] + sql, mapping = func(*args) + need_db_list = time_range.FetchDBList() + temp = {} + for each in need_db_list: + result = db_service.get_result_by_sql(each, sql, mapping) + for row in result: + key = row[0] + value = row[1:] + + if key in temp.keys(): + for index in range(len(temp[key])): + if temp[key][index] is None: + temp[key][index] = value[index] + elif value[index] is not None: + temp[key][index] += value[index] + else: + temp[key] = list(value) + for key in temp.keys(): + ret[API_RESPONSE.DATA].append([key] + temp[key]) + tmp_cmd = sql.lower() + tmp_list = tmp_cmd.split("order by ") + if len(tmp_list) == 2: + if "desc" in tmp_list[1].split(): + reverse = True + else: + reverse = False + ret[API_RESPONSE.DATA].sort(key=db_service.orderByLast, reverse=reverse) + ret[API_RESPONSE.RES] = C_RESPONSE_CODE_SUCCESS + except BadRequestException: + return HttpResponse(status=400) + return HttpResponse(json.dumps(ret), content_type=C_CONTENT_TYPE_JSON) + +def get_hits_chart_data(request): + if request.method != API_METHOD.POST: + return HttpResponse(status=405) + param = json.loads(request.body) + if not isinstance(param, dict): + return HttpResponse(status=405) + ret = { + API_RESPONSE.RES: C_RESPONSE_CODE_FAILED, + API_RESPONSE.DATA: [], + API_RESPONSE.MSG: "" + } + valid_param = [ + HIT_CHART_PARAM.TYPE, + HIT_CHART_PARAM.NAME, + HIT_CHART_PARAM.FROM, + HIT_CHART_PARAM.TO, + ] + valid_type = [ + C_TYPE_HOST, + C_TYPE_SERVICE, + ] + for val in valid_param: + if val not in param: + return response_error_message(ret, "Please check the parameter") + if not param[val]: + return response_error_message(ret, "Please check the parameter") + if val == HIT_CHART_PARAM.TYPE \ + and int(param[val]) not in valid_type: + return response_error_message(ret, "Please check the parameter") + try: + time_range = TimeRange(param[TOP_CHART_PARAM.FROM], param[TOP_CHART_PARAM.TO]) + intervalStart = time_range.ParseFrom() * 1000 + fillInterval = time_range.Interval() * 1000 + exit = True + db_service = DatabaseService() + func_map = { + C_TYPE_HOST: (db_service.get_sql_with_time_range_for_host, + [time_range, param[HIT_CHART_PARAM.NAME]]), + C_TYPE_SERVICE: (db_service.get_sql_with_time_range_for_service, + [time_range, param[HIT_CHART_PARAM.NAME]]), + } + func, args = func_map[int(param[TOP_CHART_PARAM.TYPE])] + sql, mapping = func(*args) + need_db_list = time_range.FetchDBList() + for each in need_db_list: + result = db_service.get_result_by_sql(each, sql, mapping) + for meta in result: + if exit: + intervalStart = time_range.ParseFrom() * 1000 + exit = False + else: + intervalStart = ret[API_RESPONSE.DATA][-1][0] + fillInterval + intervalStart = math.floor( + intervalStart / fillInterval) * fillInterval + timestamp = meta[0] + templength = len(meta) - 1 + if timestamp - intervalStart > 60000: + while intervalStart < timestamp: + temp = [intervalStart] + for i in range(0, templength): + temp.append(None) + ret[API_RESPONSE.DATA].append(temp) + intervalStart += fillInterval + ret[API_RESPONSE.DATA].append(meta) + ret[API_RESPONSE.RES] = C_RESPONSE_CODE_SUCCESS + except BadRequestException: + return HttpResponse(status=400) + return HttpResponse(json.dumps(ret), content_type=C_CONTENT_TYPE_JSON) + +def response_error_message(ret, msg): + ret[API_RESPONSE.MSG] = msg + return HttpResponse(json.dumps(ret), content_type=C_CONTENT_TYPE_JSON) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/urls.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/urls.py (revision 39189) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/apv/urls.py (working copy) @@ -2,6 +2,12 @@ from apv.router import * from hive.report import generateReport from apv.company import * +from apv.gslb_monitor.router import ( + get_topology, + get_top_chart_data, + get_hits_chart_data +) +C_PATH_GSLB = "gslb" app_urlpatterns = patterns('', url(r'^%s/get_segment_user$' % PRODUCT_KEYWORD, get_segment_user), @@ -50,4 +56,7 @@ url(r'^%s/prometheus/Log/download' % PRODUCT_KEYWORD, prometheus_log_download), url(r'^%s/get_cloud_page$' % PRODUCT_KEYWORD, get_cloud_page), url(r'^%s/get_prometheus_page$' % PRODUCT_KEYWORD, get_prometheus_page), + url(r'^%s/%s/get_topology$' % (PRODUCT_KEYWORD, C_PATH_GSLB), get_topology), + url(r'^%s/%s/get_top_chart_data$' % (PRODUCT_KEYWORD, C_PATH_GSLB), get_top_chart_data), + url(r'^%s/%s/get_hits_chart_data$' % (PRODUCT_KEYWORD, C_PATH_GSLB), get_hits_chart_data), ) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/GSLBStatistics.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/GSLBStatistics.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/GSLBStatistics.py (working copy) @@ -0,0 +1,272 @@ +import commands +from DatabaseUtils import DatabaseUtils +import sqlite3 +import time +import os + +C_PATH_GSLB_STATUS_DB = "/var/crash/gslb_statistics/" + +C_DB_HOST_STATISTICS = "GSLBHostStatistics" +C_DB_SERVICE_STATISTICS = "GSLBServiceStatistics" + +class HOST_STATISTICS(): + NAME = "Host Name" + HIT = "Hit" + DEFAULT_POOL = "Default Pool" + DEFAULT_HIT = "Default Policy Hit" + RESORT_POOL = "Resort Pool" + RESORT_HIT = "Resort Policy Hit" + REGION_COUNT = "Region Policy Count" + REGION_HIT = "Region Policy Hit" + + COL_NAME = "host_name" + COL_HIT = "hit" + COL_DEFAULT_POOL = "default_pool" + COL_DEFAULT_HIT = "default_policy_hit" + COL_RESORT_POOL = "resort_pool" + COL_RESORT_HIT = "resort_policy_hit" + COL_REGION_COUNT = "region_policy_count" + COL_REGION_HIT = "region_hit" + +class SERVICE_STATISTICS(): + SERVICE_NAME = "Service Name" + IP_ADDRESS = "Ip Address" + PORT = "Port" + HEALTH = "Health" + DISABLED = "Disabled" + RELATION = "Relation" + HITS = "Hits" + MONITOR_COUNT = "Monitor Count" + + COL_SERVICE_NAME = "service_name" + COL_IP_ADDRESS = "ip_address" + COL_PORT = "port" + COL_HEALTH = "health" + COL_DISABLED = "disabled" + COL_RELATION = "relation" + COL_HITS = "hits" + COL_MONITOR_COUNT = "monitor_count" + +class MONITOR_COUNT(): + INSTANCE_ID = "Instance ID" + NAME = "Monitor" + STATUS = "Status" + TYPE = "Type" + ADDRESS = "Address" + SOURCE = "Source" + GATEWAY = "Gateway" + + COL_INSTANCE_ID = "instance_id" + COL_NAME = "name" + COL_STATUS = "status" + COL_TYPE = "type" + COL_ADDRESS = "address" + COL_SOURCE = "source" + COL_GATEWAY = "gateway" + +class GSLBStatistics(): + + def __init__(self): + self.dbUtils = DatabaseUtils() + + def insert_hostname(self): + data = self.get_hostname_data() + db_name = C_PATH_GSLB_STATUS_DB + self.dbUtils.get_date_for_db_name() + conn = sqlite3.connect(db_name) + db = conn.cursor() + try: + db.execute("""CREATE TABLE IF NOT EXISTS %s( + time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + host_name varchar(32) NOT NULL, + host_hits NUMERIC, + def_hits NUMERIC, + res_hits NUMERIC, + reg_cnts NUMERIC, + reg_hits NUMERIC); + """ % C_DB_HOST_STATISTICS) + now = int(time.time()) + if len(data): + for val in data.values(): + if HOST_STATISTICS.COL_NAME not in val \ + or not val[HOST_STATISTICS.COL_NAME]: + continue + db.execute("""INSERT INTO %s( + time, + host_name, + host_hits, + def_hits, + res_hits, + reg_cnts, + reg_hits + ) VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?) + """ % C_DB_HOST_STATISTICS, + (now, + val[HOST_STATISTICS.COL_NAME], + 0 if HOST_STATISTICS.COL_HIT not in val else val[HOST_STATISTICS.COL_HIT], + 0 if HOST_STATISTICS.COL_DEFAULT_HIT not in val else val[HOST_STATISTICS.COL_DEFAULT_HIT], + 0 if HOST_STATISTICS.COL_RESORT_HIT not in val else val[HOST_STATISTICS.COL_RESORT_HIT], + 0 if HOST_STATISTICS.COL_REGION_COUNT not in val else val[HOST_STATISTICS.COL_REGION_COUNT], + 0 if HOST_STATISTICS.COL_REGION_HIT not in val else val[HOST_STATISTICS.COL_REGION_HIT])) + except Exception as ex: + print("Cannot insert data into db %s" % C_DB_HOST_STATISTICS) + print(ex) + conn.commit() + conn.close() + + def insert_service(self): + data = self.get_service_data() + db_name = C_PATH_GSLB_STATUS_DB + self.dbUtils.get_date_for_db_name() + conn = sqlite3.connect(db_name) + db = conn.cursor() + try: + db.execute("""CREATE TABLE IF NOT EXISTS %s( + time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + service_name varchar(32) NOT NULL, + service_hits NUMERIC, + monitor_cnt NUMERIC); + """ % C_DB_SERVICE_STATISTICS) + now = int(time.time()) + if len(data): + for val in data.values(): + if SERVICE_STATISTICS.COL_SERVICE_NAME not in val \ + or not val[SERVICE_STATISTICS.COL_SERVICE_NAME]: + continue + db.execute("""INSERT INTO %s( + time, + service_name, + service_hits, + monitor_cnt + ) VALUES ( + ?, + ?, + ?, + ?) + """ % C_DB_SERVICE_STATISTICS, + (now, + val[SERVICE_STATISTICS.COL_SERVICE_NAME], + 0 if SERVICE_STATISTICS.COL_HITS not in val else val[SERVICE_STATISTICS.COL_HITS], + 0 if SERVICE_STATISTICS.COL_MONITOR_COUNT not in val else val[SERVICE_STATISTICS.COL_MONITOR_COUNT])) + except Exception as ex: + print("Cannot insert data into db %s" % C_DB_SERVICE_STATISTICS) + print(ex) + conn.commit() + conn.close() + + def get_hostname_data(self): + if not os.path.exists(C_PATH_GSLB_STATUS_DB): + os.mkdir(C_PATH_GSLB_STATUS_DB) + cmd = '/ca/bin/backend -c "show statistics sdns host"' + cmd = cmd.encode('utf-8') + chr(252) + (status, output) = commands.getstatusoutput(cmd) + if status != 0: + exit(1) + host_data = {} + map = { + HOST_STATISTICS.NAME: HOST_STATISTICS.COL_NAME, + HOST_STATISTICS.HIT: HOST_STATISTICS.COL_HIT, + HOST_STATISTICS.DEFAULT_POOL: HOST_STATISTICS.COL_DEFAULT_POOL, + HOST_STATISTICS.DEFAULT_HIT: HOST_STATISTICS.COL_DEFAULT_HIT, + HOST_STATISTICS.RESORT_POOL: HOST_STATISTICS.COL_RESORT_POOL, + HOST_STATISTICS.RESORT_HIT: HOST_STATISTICS.COL_RESORT_HIT, + HOST_STATISTICS.REGION_COUNT: HOST_STATISTICS.COL_REGION_COUNT, + HOST_STATISTICS.REGION_HIT: HOST_STATISTICS.COL_REGION_HIT, + } + int_map = [ + HOST_STATISTICS.HIT, + HOST_STATISTICS.DEFAULT_HIT, + HOST_STATISTICS.RESORT_HIT, + HOST_STATISTICS.REGION_COUNT, + HOST_STATISTICS.REGION_HIT + ] + current_host = "" + for each in output.split("\n"): + if not each: + continue + sp = each.strip().split(":") + if len(sp) < 2: + continue + if sp[0] == HOST_STATISTICS.NAME \ + and sp[1] not in host_data: + current_host = sp[1].strip() + host_data[current_host] = {} + if sp[0] not in map: + continue + value = sp[1].strip() + if sp[0] in int_map: + value = int(value) + host_data[current_host][map[sp[0]]] = value + return host_data + + def get_service_data(self): + if not os.path.exists(C_PATH_GSLB_STATUS_DB): + os.mkdir(C_PATH_GSLB_STATUS_DB) + cmd = '/ca/bin/backend -c "show statistics sdns service ip"' + cmd = cmd.encode('utf-8') + chr(252) + (status, output) = commands.getstatusoutput(cmd) + if status != 0: + exit(1) + service_data = {} + service_map = { + SERVICE_STATISTICS.SERVICE_NAME: SERVICE_STATISTICS.COL_SERVICE_NAME, + SERVICE_STATISTICS.IP_ADDRESS: SERVICE_STATISTICS.COL_IP_ADDRESS, + SERVICE_STATISTICS.PORT: SERVICE_STATISTICS.COL_PORT, + SERVICE_STATISTICS.HEALTH: SERVICE_STATISTICS.COL_HEALTH, + SERVICE_STATISTICS.DISABLED: SERVICE_STATISTICS.COL_DISABLED, + SERVICE_STATISTICS.RELATION: SERVICE_STATISTICS.COL_RELATION, + SERVICE_STATISTICS.HITS: SERVICE_STATISTICS.COL_HITS, + SERVICE_STATISTICS.MONITOR_COUNT: SERVICE_STATISTICS.COL_MONITOR_COUNT, + } + monitor_map = { + MONITOR_COUNT.INSTANCE_ID: MONITOR_COUNT.COL_INSTANCE_ID, + MONITOR_COUNT.NAME: MONITOR_COUNT.COL_NAME, + MONITOR_COUNT.STATUS: MONITOR_COUNT.COL_STATUS, + MONITOR_COUNT.TYPE: MONITOR_COUNT.COL_TYPE, + MONITOR_COUNT.ADDRESS: MONITOR_COUNT.COL_ADDRESS, + MONITOR_COUNT.SOURCE: MONITOR_COUNT.COL_SOURCE, + MONITOR_COUNT.GATEWAY: MONITOR_COUNT.COL_GATEWAY, + } + int_map = [ + SERVICE_STATISTICS.HITS, + SERVICE_STATISTICS.MONITOR_COUNT, + SERVICE_STATISTICS.PORT, + MONITOR_COUNT.INSTANCE_ID, + ] + current_service = "" + for each in output.split("\n"): + if not each: + continue + sp = each.strip().split(":") + if len(sp) < 2: + continue + field = sp[0].strip() + value = sp[1].strip() + if field in int_map: + value = int(value) + if field == SERVICE_STATISTICS.SERVICE_NAME \ + and field not in service_data: + current_service = value + service_data[current_service] = {} + if field in service_map: + service_field = service_map[field] + if current_service in service_data: + service_data[current_service][service_field] = value + if field in monitor_map: + monitor_field = monitor_map[field] + if current_service in service_data: + service_data[current_service][monitor_field] = value + return service_data + +def main(): + stat = GSLBStatistics() + stat.insert_hostname() + stat.insert_service() + +if __name__ == "__main__": + main() Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (revision 39189) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (working copy) @@ -6,6 +6,7 @@ import re from SLBRSGroup import SLBRSGroup from SystemInterface import SystemInterface +from GSLBStatistics import GSLBStatistics C_PATH_TCP_STATUS_DB = "/var/crash/tcp_status/" C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/" @@ -43,20 +44,19 @@ if len(sys.argv) < 1: exit(1) type = sys.argv[1] - if type not in ['tcpstatus', 'tcpsyndrop', 'sslstatus', 'systeminterface', 'slbrsgroup', 'tcpresetid']: + function_map = { + 'tcpstatus': (inset_db_tcp_status, []), + 'tcpsyndrop': (insert_db_tcp_syn_drop, []), + 'sslstatus': (insert_db_ssl_status, []), + 'systeminterface': (insert_db_system_interface_status, []), + 'slbrsgroup': (insert_db_slb_rs_group, []), + 'tcpresetid': (insert_db_tcp_reset_id, []), + 'gslbstatus': (insert_db_gslb_statistics, []), + } + if type not in function_map.keys(): exit(1) - if type == "tcpstatus": - inset_db_tcp_status() - elif type == "tcpsyndrop": - insert_db_tcp_syn_drop() - elif type == "sslstatus": - insert_db_ssl_status() - elif type == "systeminterface": - insert_db_system_interface_status() - elif type == "slbrsgroup": - insert_db_slb_rs_group() - elif type == "tcpresetid": - insert_db_tcp_reset_id() + func, args = function_map[type] + func(*args) def inset_db_tcp_status(): if not os.path.exists(C_PATH_TCP_STATUS_DB): @@ -463,6 +463,11 @@ return [each[0] for each in record] return record +def insert_db_gslb_statistics(): + handle = GSLBStatistics() + handle.insert_hostname() + handle.insert_service() + def get_date_for_db_name(): now = time.asctime( time.localtime(time.time()) ) now_date = datetime.strptime(now, '%a %b %d %H:%M:%S %Y')