Index: /branches/rel_apv_10_7/usr/click/bin/debug_monitor/slb_monitor.sh =================================================================== --- /branches/rel_apv_10_7/usr/click/bin/debug_monitor/slb_monitor.sh (revision 39370) +++ /branches/rel_apv_10_7/usr/click/bin/debug_monitor/slb_monitor.sh (working copy) @@ -34,3 +34,5 @@ # insert data into database of slb real service group /usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py slbrsgroup +# insert data into database of all slb real service +/usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py slbrealall Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/metric.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/metric.py (revision 39370) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/metric.py (working copy) @@ -1,472 +1,472 @@ -import json -import time -import os -import sqlite3 -import re -import math -from django.http import HttpResponse -from djproject.an_settings import GRAPH_DB_FILE_LIST -from django.utils.translation import ugettext_lazy as _ -from hive.exceptions import BadRequestException - - -def transDuration(duration): - try: - dur = int(duration[:-1]) - except ValueError: - raise BadRequestException - #dur = 0 - - if 's' in duration: - dur = dur * 1 - elif 'm' in duration: - dur = dur * 60 - elif 'h' in duration: - dur = dur * 60 * 60 - elif 'd' in duration: - dur = dur * 60 * 60 * 24 - elif 'w' in duration: - dur = dur * 60 * 60 * 24 * 7 - elif 'M' in duration: - dur = dur * 60 * 60 * 24 * 30 - elif 'y' in duration: - dur = dur * 60 * 60 * 24 * 365 - return dur - - -def roundInterval(interval): - if interval <= 12.5: - return 10 # 10s - # 17.5s - if interval <= 17.5: - return 15 # 15s - # 25s - if interval <= 25: - return 20 # 20s - # 45s - if interval <= 45: - return 30 # 30s - # 1.5m - if interval <= 90: - return 60 # 1m - # 3.5m - if interval <= 210: - return 120 # 2m - # 7.5m - if interval <= 450: - return 300 # 5m - # 12.5m - if interval <= 750: - return 600 # 10m - # 12.5m - if interval <= 1050: - return 900 # 15m - # 25m - if interval <= 1500: - return 1200 # 20m - # 45m - if interval <= 2700: - return 1800 # 30m - # 1.5h - if interval <= 5400: - return 3600 # 1h - # 2.5h - if interval <= 9000: - return 7200 # 2h - # 4.5h - if interval <= 16200: - return 10800 # 3h - # 9h - if interval <= 32400: - return 21600 # 6h - # 24h - if interval <= 86400: - return 43200 # 12h - return 86400 # 1d - - -class TimeRange(object): - def __init__(self, from_str, to_str): - self.from_str = from_str.replace(' ', '') - self.to_str = to_str.replace(' ', '') - self.now = time.time() - - def Interval(self): - interval = (self.ParseTo() - self.ParseFrom()) / 50 - rounded = roundInterval(interval) - if rounded < 10: - return 10 - return rounded - - def ParseFrom(self): - if 'now' in self.from_str: - fromRaw = self.from_str.replace("now-", "") - return int(time.time() - transDuration(fromRaw)) - - return int(self.from_str) - - def ParseTo(self): - if self.to_str == 'now': - return int(time.time()) - elif 'now' in self.to_str: - toRaw = self.to_str.replace("now-", "") - return int(time.time() - transDuration(toRaw)) - - return int(self.to_str) - - def FetchDBList(self): - TIMEFORMAT = '%Y-%m-%d %X' - start_time = time.strftime(TIMEFORMAT, - time.gmtime(self.ParseFrom())) - end_time = time.strftime(TIMEFORMAT, time.gmtime(self.ParseTo())) - - start_db = start_time.split(' ')[0] + '.db' - end_db = end_time.split(' ')[0] + '.db' - - all_db = filter(lambda e: e.endswith('.db') and e >= start_db and e <=end_db, - os.listdir(GRAPH_DB_FILE_LIST)) - all_db_list = sorted(all_db) - return all_db_list - - -def evaluateMacro(funcname, params, time_range): - - time_interval = time_range.Interval() - args = params.split(',') - - for i in range(0, len(args)): - args[i] = args[i].strip() - - if funcname == "__timeGroup": - return '{}/{}*{}*1000'.format(args[0], time_interval, time_interval) - elif funcname == "__timeFilter": - return "{} BETWEEN {} AND {}".format(args[0], time_range.ParseFrom(), - time_range.ParseTo()) - elif funcname == "__timeFilter2": - return "{} BETWEEN {} AND {}".format(args[0], time_range.ParseFrom(), - time_range.ParseTo()+time_interval) - elif funcname == "__timeGroup2": - if time_interval < 30: - time_interval = 30 - return '{}/{}*{}*1000'.format(args[0], time_interval, time_interval) - return "" - - -def replaceAllStringSubmatchFunc(sql, time_range): - - pattern = re.compile(r'\$([_a-zA-Z0-9]+)\(([^\)]*)\)') - - result = '' - lastIndex = 0 - - iter = pattern.finditer(sql) - for m in iter: - indexs = m.span() - - result += sql[lastIndex:indexs[0]] + evaluateMacro( - m.group(1), m.group(2), time_range) - lastIndex = indexs[1] - - return result + sql[lastIndex:] - -class CustomerIncrement: - def __init__(self): - self.pre = "NAN" - self.sum = 0 - def step(self, value): - if self.pre == "NAN": - self.sum = 0 - elif self.pre > int(value): - self.sum += int(value) - else: - self.sum += int(value) - self.pre - self.pre = int(value) - def finalize(self): - return self.sum - - -class CustomerLast: - def __init__(self): - self.value = None - - def step(self, value): - self.value = value - - def finalize(self): - return self.value - - -class CustomerRatio: - def __init__(self): - self.pre_value = None - self.start_time = None - self.end_time = None - self.total = 0 - - def step(self, time, value): - value = int(value) - if self.start_time is None: - self.start_time = time - self.pre_value = value - self.end_time = time - if value < self.pre_value: - self.total += value - else: - self.total += value - self.pre_value - self.pre_value = value - - def finalize(self): - if self.start_time and self.end_time: - idl_tm = self.end_time - self.start_time - if idl_tm > 0: - return self.total / idl_tm - else: - return 0 - return 0 - - -def query_database(each, cmd): - if os.path.isfile(GRAPH_DB_FILE_LIST + each): - - conn = sqlite3.connect(GRAPH_DB_FILE_LIST + each) - conn.create_aggregate("INCREMENT", 1, CustomerIncrement) - conn.create_aggregate("LAST", 1, CustomerLast) - conn.create_aggregate("RATIO", 2, CustomerRatio) - db = conn.cursor() - - try: - db.execute(cmd) - except sqlite3.OperationalError, e: - if str(e).find("locked") != -1: - time.sleep(1) - try: - db.execute(cmd) - except sqlite3.OperationalError, e: - if str(e).find("locked") != -1: - time.sleep(1) - try: - db.execute(cmd) - except sqlite3.OperationalError, e: - return [] - except sqlite3.DatabaseError, e: - # file is encrypted or is not a database - return [] - - result = db.fetchall() - conn.close() - return result - return [] - -def orderByLast(elem): - return elem[-1] - -def query_metric(res): - ret = {} - - time_range = TimeRange(res['from'], res['to']) - ret['data'] = [] - - if "index" in res and res["index"]: - need_db_list = ["storemon_1.db"] - else: - need_db_list = time_range.FetchDBList() - ret['db_list'] = need_db_list - - cmd = replaceAllStringSubmatchFunc(res['rawSql'], time_range) - #ret['cmd'] = cmd - - if res['format'] == "cluster": - # if format is cluster, and only support order by the last selected field. - # select , , from example_table where xxx order by limit N; - temp = {} - for each in need_db_list: - result = query_database(each, cmd) - - for row in result: - key = row[0] - value = row[1:] - - if key in temp.keys(): - for index in range(len(temp[key])): - if temp[key][index] is None: - temp[key][index] = value[index] - elif value[index] is not None: - temp[key][index] += value[index] - else: - temp[key] = list(value) - - for key in temp.keys(): - ret['data'].append([key] + temp[key]) - tmp_cmd = cmd.lower() - tmp_list = tmp_cmd.split("order by ") - if len(tmp_list) == 2: - if "desc" in tmp_list[1].split(): - reverse = True - else: - reverse = False - ret['data'].sort(key=orderByLast, reverse=reverse) - - elif res['format'] == "time_series": - intervalStart = time_range.ParseFrom() * 1000 - fillInterval = time_range.Interval() * 1000 - - exit = True - - for each in need_db_list: - result = query_database(each, cmd) - - for meta in result: - - if exit: - intervalStart = time_range.ParseFrom() * 1000 - exit = False - else: - intervalStart = ret['data'][-1][0] + fillInterval - - intervalStart = math.floor( - intervalStart / fillInterval) * fillInterval - - timestamp = meta[0] - templength = len(meta) - 1 - - if timestamp - intervalStart > 60000: - while intervalStart < timestamp: - temp = [intervalStart] - for i in range(0, templength): - temp.append(None) - ret['data'].append(temp) - intervalStart += fillInterval - - ret['data'].append(meta) - else: - for each in need_db_list: - result = query_database(each, cmd) - - for meta in result: - ret['data'].append(meta) - - return ret - - -def query_metrics(request): - if request.method == 'POST': - res = json.loads(request.body) - if isinstance(res, list): - for each in res: - for key in each.keys(): - if key not in ['from', 'to', 'index', 'rawSql', 'format']: - raise KeyError('wrong params') - elif isinstance(res, dict): - for key in res.keys(): - if key not in ['from', 'to', 'index', 'rawSql', 'format']: - raise KeyError('wrong params') - try: - if isinstance(res, list): - ret = [] - for meta in res: - ret.append(query_metric(meta)) - else: - ret = query_metric(res) - except BadRequestException: - response = HttpResponse() - response.status_code = 400 - return response - else: - return HttpResponse(status=405) - - return HttpResponse(json.dumps(ret), content_type='application/json') - -def query_statistics_llb_statistics(args): - p = os.popen("/ca/bin/webui_utils -a 4") - data = p.read() - p.close() - llb_link = json.loads(data) - llb_link_list = [] - if llb_link["result"]: - for data in llb_link["data"]: - llb_link_list.append(data['name']) - ret = {"data":[]} - if len(llb_link_list) > 0: - meta = { - "rawSql": "select $__timeGroup(time) as timef, name, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(usage), max(current_connection), max(hits), max(status) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name in ('%s') GROUP BY timef, name;" % ("','".join(llb_link_list)), - "format": "base_time_series", - "index": 1, - "from": args["from"], - "to": args["to"] - } - ret = query_metric(meta) - time_range = TimeRange(args['from'], args['to']) - intervalStart = time_range.ParseFrom() * 1000 - fillInterval = time_range.Interval() * 1000 - intervalStart = math.floor( - intervalStart / fillInterval) * fillInterval - link_interval_start = {} - for name in llb_link_list: - link_interval_start[name] = intervalStart - new_data = [] - for meta in ret['data']: - timestamp = meta[0] - name = meta[1] - templength = len(meta) - 2 - - if timestamp - link_interval_start[name] > 60000: - while link_interval_start[name] < timestamp: - temp = [link_interval_start[name], name] - for i in range(0, templength): - temp.append(None) - new_data.append(temp) - link_interval_start[name] += fillInterval - link_interval_start[name] = timestamp + fillInterval - new_data.append(meta) - ret['data'] = new_data - return ret - -def query_statistics(request): - if request.method == 'POST': - cmd_dict = { - "llb_total": { - "func": query_statistics_llb_statistics - }, - "llb_single": { - "rawSql": "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(thresh), max(current_connection), max(hits), avg(usage) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name = '%(link_name)s' GROUP BY timef;", - "format": "time_series", - "index": 1 - } - } - # post data format: - # { - # "from": "xxx", - # "to": "xxx", - # "idx": "xxxx", - # "": "xxxx", //optional - # .... - # } - - res = json.loads(request.body) - if "to" not in res: - return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') - if "from" not in res: - return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') - if "idx" in res: - idx = res['idx'] - if idx in cmd_dict: - if "func" in cmd_dict[idx]: - ret = cmd_dict[idx]["func"](res) - else: - try: - rawSql = cmd_dict[idx]['rawSql'] % (res) - except KeyError: - return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') - meta = { - "from": res["from"], - "to": res["to"], - "format": cmd_dict[idx]['format'], - "rawSql": rawSql, - "index": cmd_dict[idx]['index'], - } - ret = query_metric(meta) - return HttpResponse(json.dumps(ret), content_type='application/json') - else: - return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') - else: - return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') +import json +import time +import os +import sqlite3 +import re +import math +from django.http import HttpResponse +from djproject.an_settings import GRAPH_DB_FILE_LIST +from django.utils.translation import ugettext_lazy as _ +from hive.exceptions import BadRequestException + + +def transDuration(duration): + try: + dur = int(duration[:-1]) + except ValueError: + raise BadRequestException + #dur = 0 + + if 's' in duration: + dur = dur * 1 + elif 'm' in duration: + dur = dur * 60 + elif 'h' in duration: + dur = dur * 60 * 60 + elif 'd' in duration: + dur = dur * 60 * 60 * 24 + elif 'w' in duration: + dur = dur * 60 * 60 * 24 * 7 + elif 'M' in duration: + dur = dur * 60 * 60 * 24 * 30 + elif 'y' in duration: + dur = dur * 60 * 60 * 24 * 365 + return dur + + +def roundInterval(interval): + if interval <= 12.5: + return 10 # 10s + # 17.5s + if interval <= 17.5: + return 15 # 15s + # 25s + if interval <= 25: + return 20 # 20s + # 45s + if interval <= 45: + return 30 # 30s + # 1.5m + if interval <= 90: + return 60 # 1m + # 3.5m + if interval <= 210: + return 120 # 2m + # 7.5m + if interval <= 450: + return 300 # 5m + # 12.5m + if interval <= 750: + return 600 # 10m + # 12.5m + if interval <= 1050: + return 900 # 15m + # 25m + if interval <= 1500: + return 1200 # 20m + # 45m + if interval <= 2700: + return 1800 # 30m + # 1.5h + if interval <= 5400: + return 3600 # 1h + # 2.5h + if interval <= 9000: + return 7200 # 2h + # 4.5h + if interval <= 16200: + return 10800 # 3h + # 9h + if interval <= 32400: + return 21600 # 6h + # 24h + if interval <= 86400: + return 43200 # 12h + return 86400 # 1d + + +class TimeRange(object): + def __init__(self, from_str, to_str): + self.from_str = from_str.replace(' ', '') + self.to_str = to_str.replace(' ', '') + self.now = time.time() + + def Interval(self): + interval = (self.ParseTo() - self.ParseFrom()) / 50 + rounded = roundInterval(interval) + if rounded < 10: + return 10 + return rounded + + def ParseFrom(self): + if 'now' in self.from_str: + fromRaw = self.from_str.replace("now-", "") + return int(time.time() - transDuration(fromRaw)) + + return int(self.from_str) + + def ParseTo(self): + if self.to_str == 'now': + return int(time.time()) + elif 'now' in self.to_str: + toRaw = self.to_str.replace("now-", "") + return int(time.time() - transDuration(toRaw)) + + return int(self.to_str) + + def FetchDBList(self): + TIMEFORMAT = '%Y-%m-%d %X' + start_time = time.strftime(TIMEFORMAT, + time.gmtime(self.ParseFrom())) + end_time = time.strftime(TIMEFORMAT, time.gmtime(self.ParseTo())) + + start_db = start_time.split(' ')[0] + '.db' + end_db = end_time.split(' ')[0] + '.db' + + all_db = filter(lambda e: e.endswith('.db') and e >= start_db and e <=end_db, + os.listdir(GRAPH_DB_FILE_LIST)) + all_db_list = sorted(all_db) + return all_db_list + + +def evaluateMacro(funcname, params, time_range): + + time_interval = time_range.Interval() + args = params.split(',') + + for i in range(0, len(args)): + args[i] = args[i].strip() + + if funcname == "__timeGroup": + return '{}/{}*{}*1000'.format(args[0], time_interval, time_interval) + elif funcname == "__timeFilter": + return "{} BETWEEN {} AND {}".format(args[0], time_range.ParseFrom(), + time_range.ParseTo()) + elif funcname == "__timeFilter2": + return "{} BETWEEN {} AND {}".format(args[0], time_range.ParseFrom(), + time_range.ParseTo()+time_interval) + elif funcname == "__timeGroup2": + if time_interval < 30: + time_interval = 30 + return '{}/{}*{}*1000'.format(args[0], time_interval, time_interval) + return "" + + +def replaceAllStringSubmatchFunc(sql, time_range): + + pattern = re.compile(r'\$([_a-zA-Z0-9]+)\(([^\)]*)\)') + + result = '' + lastIndex = 0 + + iter = pattern.finditer(sql) + for m in iter: + indexs = m.span() + + result += sql[lastIndex:indexs[0]] + evaluateMacro( + m.group(1), m.group(2), time_range) + lastIndex = indexs[1] + + return result + sql[lastIndex:] + +class CustomerIncrement: + def __init__(self): + self.pre = "NAN" + self.sum = 0 + def step(self, value): + if self.pre == "NAN": + self.sum = 0 + elif self.pre > int(value): + self.sum += int(value) + else: + self.sum += int(value) - self.pre + self.pre = int(value) + def finalize(self): + return self.sum + + +class CustomerLast: + def __init__(self): + self.value = None + + def step(self, value): + self.value = value + + def finalize(self): + return self.value + + +class CustomerRatio: + def __init__(self): + self.pre_value = None + self.start_time = None + self.end_time = None + self.total = 0 + + def step(self, time, value): + value = int(value) + if self.start_time is None: + self.start_time = time + self.pre_value = value + self.end_time = time + if value < self.pre_value: + self.total += value + else: + self.total += value - self.pre_value + self.pre_value = value + + def finalize(self): + if self.start_time and self.end_time: + idl_tm = self.end_time - self.start_time + if idl_tm > 0: + return self.total / idl_tm + else: + return 0 + return 0 + + +def query_database(each, cmd): + if os.path.isfile(GRAPH_DB_FILE_LIST + each): + + conn = sqlite3.connect(GRAPH_DB_FILE_LIST + each) + conn.create_aggregate("INCREMENT", 1, CustomerIncrement) + conn.create_aggregate("LAST", 1, CustomerLast) + conn.create_aggregate("RATIO", 2, CustomerRatio) + db = conn.cursor() + + try: + db.execute(cmd) + except sqlite3.OperationalError, e: + if str(e).find("locked") != -1: + time.sleep(1) + try: + db.execute(cmd) + except sqlite3.OperationalError, e: + if str(e).find("locked") != -1: + time.sleep(1) + try: + db.execute(cmd) + except sqlite3.OperationalError, e: + return [] + except sqlite3.DatabaseError, e: + # file is encrypted or is not a database + return [] + + result = db.fetchall() + conn.close() + return result + return [] + +def orderByLast(elem): + return elem[-1] + +def query_metric(res): + ret = {} + + time_range = TimeRange(res['from'], res['to']) + ret['data'] = [] + + if "index" in res and res["index"]: + need_db_list = ["storemon_1.db"] + else: + need_db_list = time_range.FetchDBList() + ret['db_list'] = need_db_list + + cmd = replaceAllStringSubmatchFunc(res['rawSql'], time_range) + #ret['cmd'] = cmd + + if res['format'] == "cluster": + # if format is cluster, and only support order by the last selected field. + # select , , from example_table where xxx order by limit N; + temp = {} + for each in need_db_list: + result = query_database(each, cmd) + + for row in result: + key = row[0] + value = row[1:] + + if key in temp.keys(): + for index in range(len(temp[key])): + if temp[key][index] is None: + temp[key][index] = value[index] + elif value[index] is not None: + temp[key][index] += value[index] + else: + temp[key] = list(value) + + for key in temp.keys(): + ret['data'].append([key] + temp[key]) + tmp_cmd = cmd.lower() + tmp_list = tmp_cmd.split("order by ") + if len(tmp_list) == 2: + if "desc" in tmp_list[1].split(): + reverse = True + else: + reverse = False + ret['data'].sort(key=orderByLast, reverse=reverse) + + elif res['format'] == "time_series": + intervalStart = time_range.ParseFrom() * 1000 + fillInterval = time_range.Interval() * 1000 + + exit = True + + for each in need_db_list: + result = query_database(each, cmd) + + for meta in result: + + if exit: + intervalStart = time_range.ParseFrom() * 1000 + exit = False + else: + intervalStart = ret['data'][-1][0] + fillInterval + + intervalStart = math.floor( + intervalStart / fillInterval) * fillInterval + + timestamp = meta[0] + templength = len(meta) - 1 + + if timestamp - intervalStart > 60000: + while intervalStart < timestamp: + temp = [intervalStart] + for i in range(0, templength): + temp.append(None) + ret['data'].append(temp) + intervalStart += fillInterval + + ret['data'].append(meta) + else: + for each in need_db_list: + result = query_database(each, cmd) + + for meta in result: + ret['data'].append(meta) + + return ret + + +def query_metrics(request): + if request.method == 'POST': + res = json.loads(request.body) + if isinstance(res, list): + for each in res: + for key in each.keys(): + if key not in ['from', 'to', 'index', 'rawSql', 'format']: + raise KeyError('wrong params') + elif isinstance(res, dict): + for key in res.keys(): + if key not in ['from', 'to', 'index', 'rawSql', 'format']: + raise KeyError('wrong params') + try: + if isinstance(res, list): + ret = [] + for meta in res: + ret.append(query_metric(meta)) + else: + ret = query_metric(res) + except BadRequestException: + response = HttpResponse() + response.status_code = 400 + return response + else: + return HttpResponse(status=405) + + return HttpResponse(json.dumps(ret), content_type='application/json') + +def query_statistics_llb_statistics(args): + p = os.popen("/ca/bin/webui_utils -a 4") + data = p.read() + p.close() + llb_link = json.loads(data) + llb_link_list = [] + if llb_link["result"]: + for data in llb_link["data"]: + llb_link_list.append(data['name']) + ret = {"data":[]} + if len(llb_link_list) > 0: + meta = { + "rawSql": "select $__timeGroup(time) as timef, name, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(usage), max(current_connection), max(hits), max(status) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name in ('%s') GROUP BY timef, name;" % ("','".join(llb_link_list)), + "format": "base_time_series", + "index": 1, + "from": args["from"], + "to": args["to"] + } + ret = query_metric(meta) + time_range = TimeRange(args['from'], args['to']) + intervalStart = time_range.ParseFrom() * 1000 + fillInterval = time_range.Interval() * 1000 + intervalStart = math.floor( + intervalStart / fillInterval) * fillInterval + link_interval_start = {} + for name in llb_link_list: + link_interval_start[name] = intervalStart + new_data = [] + for meta in ret['data']: + timestamp = meta[0] + name = meta[1] + templength = len(meta) - 2 + + if timestamp - link_interval_start[name] > 60000: + while link_interval_start[name] < timestamp: + temp = [link_interval_start[name], name] + for i in range(0, templength): + temp.append(None) + new_data.append(temp) + link_interval_start[name] += fillInterval + link_interval_start[name] = timestamp + fillInterval + new_data.append(meta) + ret['data'] = new_data + return ret + +def query_statistics(request): + if request.method == 'POST': + cmd_dict = { + "llb_total": { + "func": query_statistics_llb_statistics + }, + "llb_single": { + "rawSql": "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(thresh), max(current_connection), max(hits), avg(usage) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name = '%(link_name)s' GROUP BY timef;", + "format": "time_series", + "index": 1 + } + } + # post data format: + # { + # "from": "xxx", + # "to": "xxx", + # "idx": "xxxx", + # "": "xxxx", //optional + # .... + # } + + res = json.loads(request.body) + if "to" not in res: + return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') + if "from" not in res: + return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') + if "idx" in res: + idx = res['idx'] + if idx in cmd_dict: + if "func" in cmd_dict[idx]: + ret = cmd_dict[idx]["func"](res) + else: + try: + rawSql = cmd_dict[idx]['rawSql'] % (res) + except KeyError: + return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') + meta = { + "from": res["from"], + "to": res["to"], + "format": cmd_dict[idx]['format'], + "rawSql": rawSql, + "index": cmd_dict[idx]['index'], + } + ret = query_metric(meta) + return HttpResponse(json.dumps(ret), content_type='application/json') + else: + return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') + else: + return HttpResponse(json.dumps({"result":-1, "msg":"Parameter error"}), content_type='application/json') Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/ItemController.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/ItemController.py (revision 39370) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/ItemController.py (working copy) @@ -363,6 +363,9 @@ "value": "rs_cc", "label": unicode(_("Concurrent Connections")), },{ + "value": "rs_acc", + "label": unicode(_("Average Connection Count")), + },{ "value": "rs_cps", "label": unicode(_("CPS")), },{ Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (revision 39370) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (working copy) @@ -7,6 +7,7 @@ from SLBRSGroup import SLBRSGroup from SystemInterface import SystemInterface from GSLBStatistics import GSLBStatistics +from SLBRSStatistics import SLBRSStatistics C_PATH_TCP_STATUS_DB = "/var/crash/tcp_status/" C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/" @@ -52,6 +53,7 @@ 'slbrsgroup': (insert_db_slb_rs_group, []), 'tcpresetid': (insert_db_tcp_reset_id, []), 'gslbstatus': (insert_db_gslb_statistics, []), + 'slbrealall': (insert_db_slb_real_all, []), } if type not in function_map.keys(): exit(1) @@ -468,6 +470,10 @@ handle.insert_hostname() handle.insert_service() +def insert_db_slb_real_all(): + handler = SLBRSStatistics() + handler.insert_data() + def get_date_for_db_name(): now = time.asctime( time.localtime(time.time()) ) now_date = datetime.strptime(now, '%a %b %d %H:%M:%S %Y') Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (revision 39370) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (working copy) @@ -1,7 +1,8 @@ +from datetime import datetime from pyecharts import Bar, Line, Page from pyecharts.chart import Chart from pyecharts.engine import create_default_environment -from reportlab.lib import colors +from reportlab.lib import colors from reportlab.lib.enums import TA_JUSTIFY from reportlab.lib.pagesizes import letter from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image, TableStyle @@ -342,6 +343,8 @@ def get_rs_data_sql(from_time, to_time, rs_name): time_list = [] cc_list = [] + acc_time_list = [] + acc_list = [] cps_list = [] request_list = [] inbound_list = [] @@ -349,14 +352,15 @@ average_in_list = [] average_out_list = [] average_response_list = [] - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_response_time), max(connection_count),max(reqcount) from GlobalRealServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %rs_name, + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_response_time), max(connection_count),max(reqcount) from GlobalRealServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %rs_name, 'to': to_time, 'from': from_time, 'format': 'time_series'} select_data = query_metric(select_dic) - errcode_dic = {'rawSql': 'select INCREMENT(resp_100), INCREMENT(resp_101), INCREMENT(resp_200), INCREMENT(resp_206), INCREMENT(resp_301), INCREMENT(resp_302), INCREMENT(resp_303), INCREMENT(resp_304), INCREMENT(resp_305), INCREMENT(resp_307), INCREMENT(resp_400), INCREMENT(resp_401), INCREMENT(resp_402), INCREMENT(resp_403), INCREMENT(resp_404), INCREMENT(resp_405), INCREMENT(resp_406), INCREMENT(resp_407), INCREMENT(resp_416), INCREMENT(resp_500), INCREMENT(resp_501), INCREMENT(resp_502), INCREMENT(resp_503), INCREMENT(resp_504), INCREMENT(resp_505), INCREMENT(resp_others) from RealServiceHTTPCodeStatistics WHERE $__timeFilter(time) and service_name = "%s" group by service_name;' %rs_name, 'to': to_time, 'from': from_time, 'format': 'table'} errcode_data = query_metric(errcode_dic) - + rs_acc_query = {'rawSql': 'select avg_conn_cnt, time from SLBRealAll', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + rs_acc_query_result = query_metric(rs_acc_query) data = select_data['data'] for each in data: time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) @@ -369,9 +373,14 @@ average_out_list.append(each[5]) average_response_list.append(value_toFixed(each[6],2)) + acc_time_list = [datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') for avg_conn_cnt, timestamp in rs_acc_query_result["data"]] + acc_list = [avg_conn_cnt for avg_conn_cnt, timestamp in rs_acc_query_result["data"]] + return { 'time': time_list, 'cc_list': cc_list, + 'acc_time': acc_time_list, + 'acc_list': acc_list, 'cps_list': cps_list, 'request_list': request_list, 'inbound_list': inbound_list, @@ -840,7 +849,6 @@ ptext = '' + unicode(_("SLB Real Service Status")) + '' story.append(Paragraph(ptext, styles[current_font])) story.append(Spacer(1, 12)) - for each_rs in slb_rs_list: rs_data_dic = get_rs_data_sql(from_time, to_time, each_rs) if 'rs_cc' in slb_rs_list[each_rs]: @@ -850,7 +858,15 @@ }] table_data = zip(rs_data_dic['time'], rs_data_dic['cc_list']) story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - + + if 'rs_acc' in slb_rs_list[each_rs]: + graph_item = [{ + "item_name" : unicode(_("Average Connection Count")), + "item_value" : rs_data_dic['acc_list'] + }] + table_data = zip(rs_data_dic['acc_time'], rs_data_dic['acc_list']) + story = Draw_Line_Chart(unicode(_("Average Connection Count"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['acc_time'], graph_item, table_data) + if 'rs_cps' in slb_rs_list[each_rs]: graph_item = [{ "item_name" : unicode(_("CPS")), @@ -858,7 +874,7 @@ }] table_data = zip(rs_data_dic['time'], rs_data_dic['cps_list']) story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - + if 'rs_request' in slb_rs_list[each_rs]: graph_item = [{ "item_name" : unicode(_("Outstanding Request Count")), @@ -866,16 +882,16 @@ }] table_data = zip(rs_data_dic['time'], rs_data_dic['request_list']) story = Draw_Line_Chart(unicode(_("Outstanding Request Count"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - + if 'rs_throughput' in slb_rs_list[each_rs]: graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(rs_data_dic['inbound_list'], rs_data_dic['outbound_list']) - + table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) story = Draw_Line_Chart(unicode(_("Throughput(bps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) - + if 'rs_pkg_rate' in slb_rs_list[each_rs]: graph_item, net_in_table, net_out_table, yaxis_name = package_rate_report(rs_data_dic['average_in_list'], rs_data_dic['average_out_list']) - + table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) story = Draw_Line_Chart(unicode(_("Package Rate(pps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) @@ -1482,6 +1498,10 @@ cw.writerow([unicode(_('Concurrent Count'))]) cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) cw.writerows(zip(rs_data_dic['time'], rs_data_dic['cc_list'])) + if 'rs_acc' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Average Connection Count'))]) + cw.writerow([unicode(_('Time')), unicode(_('Average Connection Count'))]) + cw.writerows(zip(rs_data_dic['acc_time'], rs_data_dic['acc_list'])) if 'rs_cps' in slb_rs_list[rs_name]: cw.writerow([unicode(_('CPS'))]) cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) @@ -1703,6 +1723,7 @@ return cw def generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list, ssl_total_list): sys_info = get_system_version() + with codecs.open(report_file_path+report_name, 'w', encoding="utf-8-sig") as csvfile: cw = csv.writer(csvfile) cw.writerow([unicode(_('Create Time')), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())]) @@ -1801,7 +1822,6 @@ story = [] story.append(im) story.append(t) - if system_list: story = generateSystemReport(from_time, to_time, system_list, env, styles, story) if slb_total_list: