Index: /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh =================================================================== --- /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh (revision 38774) +++ /branches/rel_apv_10_7/usr/click/bin/debug_monitor/monitor.sh (working copy) @@ -278,6 +278,9 @@ # insert data into database of tcp status /usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py tcpstatus +# insert data into database of tcp reset count + /usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py tcpresetid + #ssl info timestamp_gen $output_file SSL_HW=`/ca/bin/u_sysctl -n kern.ssl_initialized` Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/TCPResetCount.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/TCPResetCount.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/TCPResetCount.py (working copy) @@ -0,0 +1,92 @@ +from datetime import datetime, timedelta +import os +from DataController import DataController +import re +import time + +class TCPResetCount(): + C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/" + C_DB_TCP_RESET_COUNT = 'TCPResetCount' + C_TYPE_TOTAL = 'total' + + def __init__(self, from_time): + self.start_time = self.get_start_time(from_time) + now = time.asctime( time.localtime(time.time()) ) + self.end_time = datetime.strptime(now, '%a %b %d %H:%M:%S %Y') + def get_data_from_db(self, type): + """ + Get data from sqlite db by start time and type + :param string type: such as total or vs/rs + :param string name: such as reset id or name of a vs/rs + :return dict + example: + { + "time": [], + "count": [] + } + """ + ret = {} + if not os.path.exists(self.C_PATH_TCP_SYN_DROP_DB): + return ret + ret['data'] = [] + start_db = str(self.start_time).split(' ')[0] + '.db' + end_db = str(self.end_time).split(' ')[0] + '.db' + all_db = filter(lambda e: e.endswith('.db') and e >= start_db and e <=end_db, + os.listdir(self.C_PATH_TCP_SYN_DROP_DB)) + need_db_list = sorted(all_db) + ret['db_list'] = need_db_list + cmd = " ".join([ + "SELECT DATETIME(time, 'unixepoch') AS timef,", + "SUM(count),", + "name", + "FROM %s" % self.C_DB_TCP_RESET_COUNT, + "WHERE timef BETWEEN '%s' AND '%s'" % (self.start_time, self.end_time), + "AND type = '%s'" % type, + "GROUP BY timef, name;"]) + data_handle = DataController() + for each in need_db_list: + result = data_handle.query_database(self.C_PATH_TCP_SYN_DROP_DB, each, cmd) + if result: + ret['data'] = ret['data'] + result + return ret + + def get_start_time(self, from_time): + ''' + from_time is string type, which might be now-1h, now-2h, now-6h, now-1d, now-1w, or now-1M + return datetime object based on from_time + ''' + current_date_time = datetime.now() + period = from_time.split("-")[1] + match = re.match(r'(\d+)[hdwM]+', period) + period_number = int(match.group(1)) + if "h" in period: + return current_date_time - timedelta(hours=period_number) + elif "d" in period: + return current_date_time - timedelta(days=period_number) + elif "w" in period: + return current_date_time - timedelta(weeks=period_number) + elif "M" in period: + # there is no consideration for the condition 29, 30, 31 days in a month + return current_date_time - timedelta(days=30*period_number) + + def get_sql(self, type): + """ + Return dict that index is reset id or name of vs/rs + """ + type_list = {} + select_data = self.get_data_from_db(type) + data = select_data['data'] + for each in data: + if each[2] not in type_list: + type_list[each[2]] = { + "time": [], + "count_list": [] + } # ex: type_list['0x000e'] = {"time":[],"count_list":[]} + type_list[each[2]]["time"].append(each[0]) + type_list[each[2]]["count_list"].append(each[1]) + return type_list +if __name__=="__main__": + # example + tcpResetCnt = TCPResetCount("now-1h", gz_directory="./") + data = tcpResetCnt.get_data_from_db(tcpResetCnt.C_TYPE_TOTAL) + print(data) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (revision 38774) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (working copy) @@ -12,6 +12,7 @@ C_DB_TCP_STATUS = 'TCPStatusStatistics' C_DB_TCP_SYN_DROUP = 'TCPSynDropStatistics' C_DB_SSL_STATUS = 'SSLStatusStatistics' +C_DB_TCP_RESET_COUNT = 'TCPResetCount' C_TYPE_GLOBAL = "global" C_TYPE_TOTAL = 'total' @@ -87,7 +88,7 @@ if len(sys.argv) < 1: exit(1) type = sys.argv[1] - if type not in ['tcpstatus', 'tcpsyndrop', 'sslstatus', 'systeminterface', 'slbrsgroup']: + if type not in ['tcpstatus', 'tcpsyndrop', 'sslstatus', 'systeminterface', 'slbrsgroup', 'tcpresetid']: exit(1) if type == "tcpstatus": inset_db_tcp_status() @@ -99,6 +100,8 @@ insert_db_system_interface_status() elif type == "slbrsgroup": insert_db_slb_rs_group() + elif type == "tcpresetid": + insert_db_tcp_reset_id() def inset_db_tcp_status(): if not os.path.exists(C_PATH_TCP_STATUS_DB): @@ -633,6 +636,71 @@ conn2.commit() conn2.close() +def insert_db_tcp_reset_id(): + # because the data is from /ca/bin/anetstat -p tcp, we use same path + if not os.path.exists(C_PATH_TCP_SYN_DROP_DB): + os.mkdir(C_PATH_TCP_SYN_DROP_DB) + total_data = get_data_for_tcp_reset_id(False) + if not len(total_data): # if no data for total + exit(1) + db_name = C_PATH_TCP_SYN_DROP_DB + get_date_for_db_name() + conn = sqlite3.connect(db_name) + db = conn.cursor() + now = int(time.time()) + try: + db.execute(" ".join(["CREATE TABLE IF NOT EXISTS %s(" % C_DB_TCP_RESET_COUNT, + "time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,", + "type varchar(10) NOT NULL,", # type for total + "name varchar(32),", # rest id + "count varchar(18),", + "reason varchar(255));"])) + if len(total_data): + for entry in total_data: + if len(entry) < 3: + continue + db.execute(" ".join(['INSERT INTO %s(' % C_DB_TCP_RESET_COUNT, + 'time,', + 'type,', + 'name,', + 'count,', + 'reason', + ') VALUES (', + '?,', + '?,', + '?,', + '?,', + '?)']), + (now, + C_TYPE_TOTAL, + str(entry[0]).strip(), + str(entry[1]).strip(), + str(entry[2]).strip())) + except Exception as ex: + print("Cannot insert data into db %s" % C_DB_TCP_RESET_COUNT) + print(ex) + conn.commit() + conn.close() + +def get_data_for_tcp_reset_id(id_only): + """ + Use command to get tcp reset data and parse to list + :param boolean id_only: return list of reset id or not + :return list + """ + cmd = '/ca/bin/backend -c "show statistics reset"' + cmd = cmd.encode('utf-8') + chr(252) + (status, output) = commands.getstatusoutput(cmd) + if status != 0: + return [] + split_newline = output.splitlines() + target_content = "\n".join(split_newline[2:]) + record = re.findall(r"(?P[^\s]+)\s*(?P[^\s]+)\s*([^\n]+)", target_content, re.DOTALL) + if not record: + return [] + if id_only: + return [each[0] for each in record] + return record + def get_date_for_db_name(): now = time.asctime( time.localtime(time.time()) ) now_date = datetime.strptime(now, '%a %b %d %H:%M:%S %Y') Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (revision 38774) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (working copy) @@ -22,6 +22,8 @@ from hive.monitor_log.TCPStatistics import TCPStatistics from hive.monitor_log.SSLStatFacade import SSLStatFacade from hive.monitor_log.ChartController import ChartController +from hive.monitor_log.write_monitor_to_db import get_data_for_tcp_reset_id +from hive.monitor_log.TCPResetCount import TCPResetCount TIMEFORMAT='%Y-%m-%d %X' chart_path = "/var/crash/statmon/" @@ -1199,6 +1201,8 @@ tcp_syn_drop = TCPSYNDropStat(from_time) tcp_status_dic = tcp_status.get_sql(tcp_status.C_GLOBAL_INDEX) tcp_syn_drop_dic = tcp_syn_drop.get_sql(tcp_syn_drop.C_TYPE_TOTAL) + tcp_reset_cnt = TCPResetCount(from_time) + tcp_reset_cnt_dic = tcp_reset_cnt.get_sql(tcp_reset_cnt.C_TYPE_TOTAL) story.append(Spacer(1, 12)) ptext = '' + unicode(_("TCP Global Status")) + '' story.append(Paragraph(ptext, styles[current_font])) @@ -1223,6 +1227,21 @@ }] table_data = zip(tcp_syn_drop_dic['time'], tcp_syn_drop_dic[field + '_list']) story = Draw_Line_Chart(unicode(_(column)), env, styles, story, tcp_syn_drop_dic['time'], graph_item, table_data) + if "reset_count:" in field: + ptext = '' + unicode(_("TCP Reset Count")) + '' + story.append(Paragraph(ptext, styles[current_font])) + story.append(Spacer(1, 12)) + reset_ids = field.replace("reset_count:", "") + for reset_id in reset_ids.split(","): + # such as 0x000e + if reset_id not in tcp_reset_cnt_dic: + continue + graph_item = [{ + "item_name" : unicode(_("Count")), + "item_value" : tcp_reset_cnt_dic[reset_id]['count_list'] + }] + table_data = zip(tcp_reset_cnt_dic[reset_id]['time'], tcp_reset_cnt_dic[reset_id]['count_list']) + story = Draw_Line_Chart("Reset Id: " + reset_id, env, styles, story, tcp_reset_cnt_dic[reset_id]['time'], graph_item, table_data) return story def generateSSLTotalReport(from_time, ssl_list, env, styles, story): """ @@ -1602,6 +1621,8 @@ tcp_status_dic = tcp_status.get_sql(tcp_status.C_GLOBAL_INDEX) tcp_syn_drop = TCPSYNDropStat(from_time) tcp_syn_drop_dic = tcp_syn_drop.get_sql(tcp_syn_drop.C_TYPE_TOTAL) + tcp_reset_cnt = TCPResetCount(from_time) + tcp_reset_cnt_dic = tcp_reset_cnt.get_sql(tcp_reset_cnt.C_TYPE_TOTAL) for field in tcp_list: if field in list(tcp_status.C_COLUMN_MATCH.keys()): column = tcp_status.C_COLUMN_MATCH[field] # ex: LISTEN @@ -1615,6 +1636,14 @@ cw.writerow([unicode(_('Time')), unicode(_(column))]) if "time" in tcp_syn_drop_dic and (field + '_list') in tcp_syn_drop_dic: cw.writerows(zip(tcp_syn_drop_dic['time'], tcp_syn_drop_dic[field + '_list'])) + if "reset_count:" in field: + cw.writerow([unicode(_('TCP Reset Count'))]) + reset_ids = field.replace("reset_count:", "") + for reset_id in reset_ids.split(","): + # such as 0x000e + cw.writerow(["Reset ID: " + reset_id]) + cw.writerow([unicode(_('Time')), unicode(_('Count'))]) + cw.writerows(zip(tcp_reset_cnt_dic[reset_id]['time'], tcp_reset_cnt_dic[reset_id]['count_list'])) return cw def generateSSLTotalCSV(cw, from_time, ssl_list): """ @@ -1791,6 +1820,7 @@ llb_total_list = [] gslb_list = [] tcp_list = [] + reset_id_list = [] for each in monitor_type: if each[0] == 'system': @@ -1841,7 +1871,12 @@ if each[0] == 'gslb': gslb_list.append(each[1]) if each[0] == 'tcp': - tcp_list.append(each[1]) + if each[1] == "reset_count": + reset_id_list.append(each[2]) + else: + tcp_list.append(each[1]) + if len(reset_id_list): + tcp_list.append("reset_count:%s" % ",".join(reset_id_list)) os.popen("rm -rf /var/crash/statmon/report_chart_*") os.popen("rm -rf /ca/webui/htdocs/new/src/media/docs/Statistics_Report_*") intervalStart, intervalEnd = get_time_range(from_time, to_time) @@ -2208,6 +2243,27 @@ for each in list: each["value"] = "vs_" + each["value"] return list +def get_tcp_reset_count_list(): + """ + Get list of TCP Reset ID options + :return List + ex: [{ + "value": 'reset_count', + "label": 'Reset Count' + }] + """ + list = [{ + "value": 'reset_count', + "label": unicode(_('Reset Count')), + "children": [] + }] + reset_ids = get_data_for_tcp_reset_id(True) + for reset_id in reset_ids: + list[0]["children"].append({ + "value": reset_id, + "label": reset_id, + }) + return list def generateItems(request): item_list = [{ "value": 'system', @@ -2271,7 +2327,9 @@ }, { "value": 'tcp', "label": unicode(_('TCP')), - "children": get_statistics_tcp_list(is_vs=False) + get_tcp_syn_drop_list(is_vs=False) + "children": get_statistics_tcp_list(is_vs=False) + + get_tcp_syn_drop_list(is_vs=False) + + get_tcp_reset_count_list() }] return HttpResponse(json.dumps(item_list))