Index: /branches/rel_apv_10_7/tools/uosenv/postinstall.sh
===================================================================
--- /branches/rel_apv_10_7/tools/uosenv/postinstall.sh (revision 38551)
+++ /branches/rel_apv_10_7/tools/uosenv/postinstall.sh (working copy)
@@ -316,6 +316,11 @@
echo "*/5 * * * * root /usr/local/bin/python /ca/bin/webui_graphdb_clear.py >> /var/log/webui_graphdb_clear.log 2>&1" >>/etc/crontab
fi
+grep "clear_monitor_to_db.py" /etc/crontab >/dev/null
+if [ $? -ne 0 ]; then \
+ echo "*/5 * * * * root /usr/local/bin/python /ca/bin/clear_monitor_to_db.py >> /var/log/clear_monitor_to_db.log 2>&1" >>/etc/crontab
+fi
+
grep "free_cache.sh" /etc/crontab >/dev/null
if [ $? -ne 0 ]; then \
echo "*/30 * * * * root /ca/bin/free_cache.sh >> /var/log/free_cache.log 2>&1" >>/etc/crontab
Index: /branches/rel_apv_10_7/tools/update/ustacksystem.ks
===================================================================
--- /branches/rel_apv_10_7/tools/update/ustacksystem.ks (revision 38551)
+++ /branches/rel_apv_10_7/tools/update/ustacksystem.ks (working copy)
@@ -715,6 +715,11 @@
echo "*/5 * * * * root /usr/local/bin/python /ca/bin/webui_graphdb_clear.py >> /var/log/webui_graphdb_clear.log 2>&1" >>/etc/crontab
fi
+grep "clear_monitor_to_db.py" /etc/crontab >/dev/null
+if [ $? -ne 0 ]; then \
+ echo "*/5 * * * * root /usr/local/bin/python /ca/bin/clear_monitor_to_db.py >> /var/log/clear_monitor_to_db.log 2>&1" >>/etc/crontab
+fi
+
grep "webui_uploadfile_clear.py" /etc/crontab >/dev/null
if [ $? -ne 0 ]; then \
echo "*/60 * * * * root /usr/local/bin/python /ca/bin/webui_uploadfile_clear.py >> /var/log/webui_uploadfile_clear.log 2>&1" >>/etc/crontab
Index: /branches/rel_apv_10_7/usr/click/bin/debug_monitor/ssl_monitor.sh
===================================================================
--- /branches/rel_apv_10_7/usr/click/bin/debug_monitor/ssl_monitor.sh (revision 38551)
+++ /branches/rel_apv_10_7/usr/click/bin/debug_monitor/ssl_monitor.sh (working copy)
@@ -9,4 +9,7 @@
echo -e "\n" >> /var/crash/monitor.out0
echo "End of SSL debug info" >> /var/crash/monitor.out0
+
+# insert data into database of ssl status
+/usr/local/bin/python /ca/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py sslstatus
Index: /branches/rel_apv_10_7/usr/click/tools/Makefile
===================================================================
--- /branches/rel_apv_10_7/usr/click/tools/Makefile (revision 38551)
+++ /branches/rel_apv_10_7/usr/click/tools/Makefile (working copy)
@@ -6,7 +6,7 @@
SCRIPTS=click_upgrade.pl pre_upgrade_hooks.sh upgrade_hooks.sh config_upgrade.pl upgrade_check.py component_update.pl component_revert.pl bgp_telnet.pl start_waagent.py \
name_services.pl ca_crontab.sh ca_crontab_matchcmd.sh ca_crontab_ntp.sh ca_crontab_sync_time.sh build_config.py write_all_deferred.pl write_all_deferred.sh \
ssmtp_config.pl msmtp_config.pl syslog_wheel.sh comp_coredump.sh ssh-regenkey.sh sysv_click set_boot_menu.sh launcher hugepage_config.sh array_startup.sh \
- dnssec.sh convert_xls_to_txt apply_sip_callnum_diff snmpd snmpdp snmpd_uos snmpdp_uos snmpd_ft snmpdp_ft snmpd_ft_kylin snmpdp_ft_kylin ipv6_resolving.py webui_graphdb_clear.py External_URL_to_SLB_Config_Translator32_sign rest_build_config.py \
+ dnssec.sh convert_xls_to_txt apply_sip_callnum_diff snmpd snmpdp snmpd_uos snmpdp_uos snmpd_ft snmpdp_ft snmpd_ft_kylin snmpdp_ft_kylin ipv6_resolving.py webui_graphdb_clear.py clear_monitor_to_db.py External_URL_to_SLB_Config_Translator32_sign rest_build_config.py \
External_URL_to_SLB_Config_Translator64.zip External_URL_to_SLB_Config_Translator64_EN.zip free_cache.sh check_config.sh ssh_config.py install_patch.py webui_graphdb_update.py \
nic_check.sh show_netstat_port.py inc_sync.py webui_uploadfile_clear.py
SCRIPTSNAME_click_upgrade.pl=click_upgrade.pl
@@ -39,6 +39,7 @@
SCRIPTSNAME_dnssec.sh=dnssec.sh
SCRIPTSNAME_ipv6_resolving.py=ipv6_resolving.py
SCRIPTSNAME_webui_graphdb_clear.py=webui_graphdb_clear.py
+SCRIPTSNAME_clear_monitor_to_db.py=clear_monitor_to_db.py
SCRIPTSNAME_External_URL_to_SLB_Config_Translator32_sign=External_URL_to_SLB_Config_Translator32_sign
SCRIPTSNAME_External_URL_to_SLB_Config_Translator64.zip=External_URL_to_SLB_Config_Translator64.zip
SCRIPTSNAME_External_URL_to_SLB_Config_Translator64_EN.zip=External_URL_to_SLB_Config_Translator64_EN.zip
Index: /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py
===================================================================
--- /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py (revision 0)
+++ /branches/rel_apv_10_7/usr/click/tools/clear_monitor_to_db.py (working copy)
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+
+import commands, os
+
+C_PATH_TCP_STATUS_DB = "/var/crash/tcp_status/"
+C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/"
+C_PATH_SSL_STATUS_DB = "/var/crash/ssl_status/"
+
+def check_memory(db_path):
+ if not os.path.isdir(db_path):
+ return
+ var_size = commands.getoutput("df -k | grep ' /var$' | awk '{print $2}'")
+ monitordb_size = commands.getoutput("du -k %s | awk '{print $1}'" % db_path)
+ var_size_MB = commands.getoutput("df -m | grep ' /var$' | awk '{print $4}'")
+ db_percent = float(monitordb_size)*100/float(var_size)
+
+ print("The size of var is(KB) : %s" % var_size)
+ print("The size of monitor db is(KB) : %s" % monitordb_size)
+ print("The available size of var is(MB) : %s" % var_size_MB)
+ print("Percentage of monitor db use : %f" % db_percent)
+
+ db_list = filter(lambda e: e.endswith('.db'), os.listdir(db_path))
+ db_list = sorted(db_list)
+ if len(db_list) > 31:
+ print("The earliest db file " + db_list[0] + " is deleted.")
+ os.popen("rm -rf %s%s*" %(db_path, db_list[0]))
+ elif db_percent < 50 and float(var_size_MB) >= 1.0*1024:
+ # graphdb less than half of var/.
+ pass
+ else:
+ if len(db_list) > 0:
+ print("The space is not enough, delete the earliest db file " + db_list[0])
+ os.popen("rm -rf %s%s*" %(db_path, db_list[0]))
+
+if __name__ == '__main__':
+ check_memory(C_PATH_TCP_STATUS_DB)
+ check_memory(C_PATH_TCP_SYN_DROP_DB)
+ check_memory(C_PATH_SSL_STATUS_DB)
Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/DataController.py
===================================================================
--- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/DataController.py (revision 38551)
+++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/DataController.py (working copy)
@@ -1,6 +1,8 @@
import re
from datetime import datetime, timedelta
import os
+import time
+import sqlite3
class DataController():
def __init__(self):
@@ -64,4 +66,26 @@
self.fill_miss_data(res_list[-2][0], int(res_list[-2][2]), res_list[-1][0], int(res_list[-1][2]), cur_data_time),
self.fill_miss_data(res_list[-2][0], int(res_list[-2][3]), res_list[-1][0], int(res_list[-1][3]), cur_data_time),
))
- return res_list
\ No newline at end of file
+ return res_list
+ def query_database(self, db_path, each, cmd):
+ if os.path.isfile(db_path + each):
+ conn = sqlite3.connect(db_path + each)
+ db = conn.cursor()
+ try:
+ db.execute(cmd)
+ except sqlite3.OperationalError as e:
+ if str(e).find("locked") != -1:
+ time.sleep(1)
+ try:
+ db.execute(cmd)
+ except sqlite3.OperationalError as e:
+ if str(e).find("locked") != -1:
+ time.sleep(1)
+ try:
+ db.execute(cmd)
+ except sqlite3.OperationalError as e:
+ return []
+ result = db.fetchall()
+ conn.close()
+ return result
+ return []
\ No newline at end of file
Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/SSLStatFacade.py
===================================================================
--- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/SSLStatFacade.py (revision 38551)
+++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/SSLStatFacade.py (working copy)
@@ -1,15 +1,30 @@
import re
from datetime import datetime, timedelta
import os
-import gzip
-import shutil
from MonitorFileController import MonitorFileController
from DataController import DataController
+import time
class SSLStatFacade():
+ C_COLUMN_MATCH = {
+ 'connection_attempts': 'Connection Attempts',
+ 'successful_handshakes': 'Successful Handshakes',
+ 'active_connections': 'Active Connections',
+ 'alerts_total': 'Alerts Total',
+ 'alerts_sent': 'Alerts Sent',
+ 'alerts_received': 'Alerts Received',
+ 'warning_alerts': 'Warning Alerts',
+ 'fatal_alerts': 'Fatal Alerts'
+ }
+ C_PATH_SSL_STATUS_DB = "/var/crash/ssl_status/"
+ C_DB_SSL_STATUS = 'SSLStatusStatistics'
+ C_TYPE_TOTAL = 'total'
+
def __init__(self, from_time, gz_directory="/var/crash"):
self.gz_directory = gz_directory
self.start_time = self.get_start_time(from_time)
+ now = time.asctime( time.localtime(time.time()) )
+ self.end_time = datetime.strptime(now, '%a %b %d %H:%M:%S %Y')
self.directory_to_extract_to = os.path.join(os.getcwd(), "tmp", "sslmonitor")
self.format_data_list = None
def get_data(self):
@@ -79,9 +94,84 @@
if self.format_data_list==None:
self.get_data()
return [(data[0], data[3]) for data in self.format_data_list]
+ def get_data_from_db(self, type):
+ """
+ Get data from sqlite db by start time and type
+ :param string type: such as total or name of a virtual host
+ :return dict
+ example:
+ {
+ "time": [],
+ "connection_attempts": [],
+ ...
+ "fatal_alerts": []
+ }
+ """
+ ret = {}
+ if not os.path.exists(self.C_PATH_SSL_STATUS_DB):
+ return ret
+ ret['data'] = []
+ start_db = str(self.start_time).split(' ')[0] + '.db'
+ end_db = str(self.end_time).split(' ')[0] + '.db'
+ all_db = filter(lambda e: e.endswith('.db') and e >= start_db and e <=end_db,
+ os.listdir(self.C_PATH_SSL_STATUS_DB))
+ need_db_list = sorted(all_db)
+ ret['db_list'] = need_db_list
+ cmd = " ".join([
+ "SELECT DATETIME(time, 'unixepoch') AS timef,",
+ "SUM(connection_attempts),",
+ "SUM(successful_handshakes),",
+ "SUM(active_connections),",
+ "SUM(alerts_total),",
+ "SUM(alerts_sent),",
+ "SUM(alerts_received),",
+ "SUM(warning_alerts),",
+ "SUM(fatal_alerts)",
+ "FROM %s" % self.C_DB_SSL_STATUS,
+ "WHERE timef BETWEEN '%s' AND '%s'" % (self.start_time, self.end_time),
+ "AND type = '%s'" % type,
+ "GROUP BY timef;"])
+ data_handle = DataController()
+ for each in need_db_list:
+ result = data_handle.query_database(self.C_PATH_SSL_STATUS_DB, each, cmd)
+ if result:
+ ret['data'] = ret['data'] + result
+ return ret
+ def get_sql(self, type):
+ time_list = []
+ connection_attempts_list = []
+ successful_handshakes_list = []
+ active_connections_list = []
+ alerts_total_list = []
+ alerts_sent_list = []
+ alerts_received_list = []
+ warning_alerts_list = []
+ fatal_alerts_list = []
+ select_data = self.get_data_from_db(type)
+ data = select_data['data']
+ for each in data:
+ time_list.append(each[0])
+ connection_attempts_list.append(each[1])
+ successful_handshakes_list.append(each[2])
+ active_connections_list.append(each[3])
+ alerts_total_list.append(each[4])
+ alerts_sent_list.append(each[5])
+ alerts_received_list.append(each[6])
+ warning_alerts_list.append(each[7])
+ fatal_alerts_list.append(each[8])
+ return {
+ "time": time_list,
+ "connection_attempts_list": connection_attempts_list,
+ "successful_handshakes_list": successful_handshakes_list,
+ "active_connections_list": active_connections_list,
+ "alerts_total_list": alerts_total_list,
+ "alerts_sent_list": alerts_sent_list,
+ "alerts_received_list": alerts_received_list,
+ "warning_alerts_list": warning_alerts_list,
+ "fatal_alerts_list": fatal_alerts_list,
+ }
if __name__=="__main__":
# example
sslStat = SSLStatFacade("now-1h", gz_directory="/ca/webui/htdocs/new")
- connection_attempts_list = sslStat.get_connection_attempts_data()
- for e in connection_attempts_list:
- print(e)
+ data = sslStat.get_data_from_db(sslStat.C_TYPE_TOTAL)
+ print(data)
Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py
===================================================================
--- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (revision 0)
+++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/monitor_log/write_monitor_to_db.py (working copy)
@@ -0,0 +1,352 @@
+#!/usr/bin/python
+import sqlite3
+import commands, os, sys
+import time
+from datetime import datetime
+import re
+
+C_PATH_TCP_STATUS_DB = "/var/crash/tcp_status/"
+C_PATH_TCP_SYN_DROP_DB = "/var/crash/tcp_syn_drop/"
+C_PATH_SSL_STATUS_DB = "/var/crash/ssl_status/"
+
+C_DB_TCP_STATUS = 'TCPStatusStatistics'
+C_DB_TCP_SYN_DROUP = 'TCPSynDropStatistics'
+C_DB_SSL_STATUS = 'SSLStatusStatistics'
+
+C_TYPE_GLOBAL = "global"
+C_TYPE_TOTAL = 'total'
+
+C_PATTERN_TCP_SYN_DROP_MATCH = {
+ 'in_use_syn_drop': 'drop (\d+) syns by ours in use',
+ 'over_max_syn_drop': 'drop (\d+) syns by over max connection',
+ 'mss_err_syn_drop': 'drop (\d+) syns by get mss error',
+ 'mss_size_syn_drop': 'drop (\d+) syns by bad mss size',
+ 'eroute_err_syn_drop': 'drop (\d+) syns by eroute error',
+ 'system_stack_syn_route': 'route (\d+) syns to system stack'
+}
+
+C_COLUMN_SSL_STATUS_MATCH = {
+ 'connection_attempts': 'connection attempts',
+ 'successful_handshakes': 'successful handshakes',
+ 'active_connections': 'active connections',
+ 'alerts_total': 'alerts total',
+ 'alerts_sent': 'alerts sent',
+ 'alerts_received': 'alerts received',
+ 'warning_alerts': 'warning alerts',
+ 'fatal_alerts': 'fatal alerts'
+}
+
+def main():
+ if len(sys.argv) < 1:
+ exit(1)
+ type = sys.argv[1]
+ if type not in ['tcpstatus', 'tcpsyndrop', 'sslstatus']:
+ exit(1)
+ if type == "tcpstatus":
+ inset_db_tcp_status()
+ elif type == "tcpsyndrop":
+ insert_db_tcp_syn_drop()
+ elif type == "sslstatus":
+ insert_db_ssl_status()
+
+def inset_db_tcp_status():
+ if not os.path.exists(C_PATH_TCP_STATUS_DB):
+ os.mkdir(C_PATH_TCP_STATUS_DB)
+ cmd = '/ca/bin/backend -c "show statistics tcp all"'
+ cmd = cmd.encode('utf-8') + chr(252)
+ (status, output) = commands.getstatusoutput(cmd)
+ if status != 0:
+ exit(1)
+ grp_pattern = r"Global Statistics:\n\s+LISTEN:\s+(\d+)\s+\n\s+SYN_SENT:\s+(\d+)\s+\n\s+SYN_RCVD:\s+(\d+)\s+\n\s+ESTABLISHED:\s+(\d+)\s+\n\s+CLOSE_WAIT:\s+(\d+)\s+\n\s+FIN_WAIT_1:\s+(\d+)\s+\n\s+CLOSING:\s+(\d+)\s+\n\s+LAST_ACK:\s+(\d+)\s+\n\s+FIN_WAIT_2:\s+(\d+)\s+\n\s+TIME_WAIT:\s+(\d+)\s+\n"
+ vs_pattern = r"virtual service:\s+(\S+).*?LISTEN: (.*?)\n\s+SYN_SENT: (.*?)\n\s+SYN_RCVD: (.*?)\n\s+ESTABLISHED: (.*?)\n\s+CLOSE_WAIT: (.*?)\n\s+FIN_WAIT_1: (.*?)\n\s+CLOSING: (.*?)\n\s+LAST_ACK: (.*?)\n\s+FIN_WAIT_2: (.*?)\n\s+TIME_WAIT: (.*?)\n"
+ grp_data = []
+ if output:
+ grp_data = re.findall(grp_pattern, output, re.DOTALL)
+ vs_data = []
+ if output:
+ vs_data = re.findall(vs_pattern, output, re.DOTALL)
+ if len(grp_data) == 0 or len(vs_data) == 0:
+ exit(1)
+ db_name = C_PATH_TCP_STATUS_DB + get_date_for_db_name()
+ conn = sqlite3.connect(db_name)
+ db = conn.cursor()
+ try:
+ db.execute(" ".join(["CREATE TABLE IF NOT EXISTS %s(" % C_DB_TCP_STATUS,
+ "time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,",
+ "type varchar(32) NOT NULL,",
+ "listen varchar(18),",
+ "syn_sent varchar(18),",
+ "syn_rcvd varchar(18),",
+ "established varchar(18),",
+ "close_wait varchar(18),",
+ "fin_wait_1 varchar(18),",
+ "closing varchar(18),",
+ "last_ack varchar(18),",
+ "fin_wait_2 varchar(18),",
+ "time_wait varchar(18));"]))
+ now = int(time.time())
+ if len(grp_data):
+ db.execute(" ".join(['INSERT INTO %s(' % C_DB_TCP_STATUS,
+ 'time,',
+ 'type,',
+ 'listen,',
+ 'syn_sent,',
+ 'syn_rcvd,',
+ 'established,',
+ 'close_wait,',
+ 'fin_wait_1,',
+ 'closing,',
+ 'last_ack,',
+ 'fin_wait_2,',
+ 'time_wait',
+ ') VALUES (',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?)']),
+ (now,
+ C_TYPE_GLOBAL,
+ str(grp_data[0][0]).strip(),
+ str(grp_data[0][1]).strip(),
+ str(grp_data[0][2]).strip(),
+ str(grp_data[0][3]).strip(),
+ str(grp_data[0][4]).strip(),
+ str(grp_data[0][5]).strip(),
+ str(grp_data[0][6]).strip(),
+ str(grp_data[0][7]).strip(),
+ str(grp_data[0][8]).strip(),
+ str(grp_data[0][9]).strip()))
+ if len(vs_data):
+ for enrty in vs_data:
+ db.execute(" ".join(['INSERT INTO %s(' % C_DB_TCP_STATUS,
+ 'time,',
+ 'type,',
+ 'listen,',
+ 'syn_sent,',
+ 'syn_rcvd,',
+ 'established,',
+ 'close_wait,',
+ 'fin_wait_1,',
+ 'closing,',
+ 'last_ack,',
+ 'fin_wait_2,',
+ 'time_wait',
+ ') VALUES (',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?)']),
+ (now,
+ str(enrty[0]).strip(),
+ str(enrty[1]).strip(),
+ str(enrty[2]).strip(),
+ str(enrty[3]).strip(),
+ str(enrty[4]).strip(),
+ str(enrty[5]).strip(),
+ str(enrty[6]).strip(),
+ str(enrty[7]).strip(),
+ str(enrty[8]).strip(),
+ str(enrty[9]).strip(),
+ str(enrty[10]).strip()))
+ except Exception as ex:
+ print("Cannot insert data into db %s" % C_DB_TCP_STATUS)
+ print(ex)
+ conn.commit()
+ conn.close()
+
+def insert_db_tcp_syn_drop():
+ if not os.path.exists(C_PATH_TCP_SYN_DROP_DB):
+ os.mkdir(C_PATH_TCP_SYN_DROP_DB)
+ grp_cmd = '/ca/bin/anetstat -p tcp :'
+ (status, output) = commands.getstatusoutput(grp_cmd)
+ if status != 0:
+ exit(1)
+ grp_data = {}
+ if output:
+ for field, pattern in C_PATTERN_TCP_SYN_DROP_MATCH.items():
+ data = re.findall(pattern, output, re.DOTALL)
+ grp_data[field] = str(data[0]).strip()
+ db_name = C_PATH_TCP_SYN_DROP_DB + get_date_for_db_name()
+ conn = sqlite3.connect(db_name)
+ db = conn.cursor()
+ try:
+ db.execute(" ".join(["CREATE TABLE IF NOT EXISTS %s(" % C_DB_TCP_SYN_DROUP,
+ "time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,",
+ "type varchar(32) NOT NULL,",
+ "in_use_syn_drop varchar(18),",
+ "over_max_syn_drop varchar(18),",
+ "mss_err_syn_drop varchar(18),",
+ "mss_size_syn_drop varchar(18),",
+ "eroute_err_syn_drop varchar(18),",
+ "system_stack_syn_route varchar(18));"]))
+ now = int(time.time())
+ if grp_data:
+ db.execute(" ".join(['INSERT INTO %s(' % C_DB_TCP_SYN_DROUP,
+ 'time,',
+ 'type,',
+ 'in_use_syn_drop,',
+ 'over_max_syn_drop,',
+ 'mss_err_syn_drop,',
+ 'mss_size_syn_drop,',
+ 'eroute_err_syn_drop,',
+ 'system_stack_syn_route',
+ ') VALUES (',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?)']),
+ (now,
+ C_TYPE_TOTAL,
+ grp_data['in_use_syn_drop'],
+ grp_data['over_max_syn_drop'],
+ grp_data['mss_err_syn_drop'],
+ grp_data['mss_size_syn_drop'],
+ grp_data['eroute_err_syn_drop'],
+ grp_data['system_stack_syn_route']))
+ except Exception as ex:
+ print("Cannot insert data into db %s" % C_DB_TCP_SYN_DROUP)
+ print(ex)
+ conn.commit()
+ conn.close()
+
+def insert_db_ssl_status():
+ if not os.path.exists(C_PATH_SSL_STATUS_DB):
+ os.mkdir(C_PATH_SSL_STATUS_DB)
+ ttl_cmd = '/ca/bin/anetstat -p ssl :'
+ (status, output) = commands.getstatusoutput(ttl_cmd)
+ if status != 0:
+ exit(1)
+ ttl_data = {}
+ for field, keyword in C_COLUMN_SSL_STATUS_MATCH.items():
+ data = re.findall(r"ssl statistics:\s*Total:\s*[\s\S]*?(\d+)\s+" + keyword, output, re.DOTALL)
+ if len(data):
+ ttl_data[field] = str(data[0]).strip()
+ continue
+ ttl_data[field] = 0
+ each_cmd = '/ca/bin/backend -c "show statistics ssl"'
+ each_cmd = each_cmd.encode('utf-8') + chr(252)
+ (status, output) = commands.getstatusoutput(each_cmd)
+ if status != 0:
+ exit(1)
+ each_pattern = r"SSL Individual Statistics for \"(\S+)\"\n\s*Connection Attempts\s*: (.*?)\n\s*Successful Handshakes\s*: (.*?)\n\s*Active Connections\s*: (.*?)\n\s*Alerts Total\s*: (.*?)\n\s*Alerts Sent\s*: (.*?)\n\s*Alerts Received\s*: (.*?)\n\s*Warning Alerts\s*: (.*?)\n\s*Fatal Alerts\s*: (.*?)\n"
+ each_re = []
+ if output:
+ each_re = re.findall(each_pattern, output, re.DOTALL)
+
+ db_name = C_PATH_SSL_STATUS_DB + get_date_for_db_name()
+ conn = sqlite3.connect(db_name)
+ db = conn.cursor()
+ try:
+ db.execute(" ".join(["CREATE TABLE IF NOT EXISTS %s(" % C_DB_SSL_STATUS,
+ "time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,",
+ "type varchar(32) NOT NULL,",
+ "connection_attempts varchar(18),",
+ "successful_handshakes varchar(18),",
+ "active_connections varchar(18),",
+ "alerts_total varchar(18),",
+ "alerts_sent varchar(18),",
+ "alerts_received varchar(18),",
+ "warning_alerts varchar(18),",
+ "fatal_alerts varchar(18));"]))
+ now = int(time.time())
+ if ttl_data:
+ db.execute(" ".join(['INSERT INTO %s(' % C_DB_SSL_STATUS,
+ 'time,',
+ 'type,',
+ 'connection_attempts,',
+ 'successful_handshakes,',
+ 'active_connections,',
+ 'alerts_total,',
+ 'alerts_sent,',
+ 'alerts_received,',
+ 'warning_alerts,',
+ 'fatal_alerts',
+ ') VALUES (',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?)']),
+ (now,
+ C_TYPE_TOTAL,
+ ttl_data['connection_attempts'],
+ ttl_data['successful_handshakes'],
+ ttl_data['active_connections'],
+ ttl_data['alerts_total'],
+ ttl_data['alerts_sent'],
+ ttl_data['alerts_received'],
+ ttl_data['warning_alerts'],
+ ttl_data['fatal_alerts']))
+ if len(each_re):
+ for entry in each_re:
+ db.execute(" ".join(['INSERT INTO %s(' % C_DB_SSL_STATUS,
+ 'time,',
+ 'type,',
+ 'connection_attempts,',
+ 'successful_handshakes,',
+ 'active_connections,',
+ 'alerts_total,',
+ 'alerts_sent,',
+ 'alerts_received,',
+ 'warning_alerts,',
+ 'fatal_alerts',
+ ') VALUES (',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?,',
+ '?)']),
+ (now,
+ entry[0],
+ entry[1],
+ entry[2],
+ entry[3],
+ entry[4],
+ entry[5],
+ entry[6],
+ entry[7],
+ entry[8]))
+ except Exception as ex:
+ print("Cannot insert data into db %s" % C_DB_SSL_STATUS)
+ print(ex)
+ conn.commit()
+ conn.close()
+
+def get_date_for_db_name():
+ now = time.asctime( time.localtime(time.time()) )
+ now_date = datetime.strptime(now, '%a %b %d %H:%M:%S %Y')
+ db_name = now_date.strftime('%Y-%m-%d') + '.db'
+ return db_name
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py
===================================================================
--- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (revision 38551)
+++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (working copy)
@@ -20,6 +20,7 @@
import random, time, csv, codecs
from hive.monitor_log.TCPSYNDropStat import TCPSYNDropStat
from tcp_status.statistics import MonitorOutput
+from hive.monitor_log.SSLStatFacade import SSLStatFacade
from datetime import datetime
TIMEFORMAT='%Y-%m-%d %X'
@@ -864,6 +865,7 @@
ptext = '' + unicode(_("SSL Virtual Host Status")) + ''
story.append(Paragraph(ptext, styles["alibaba"]))
story.append(Spacer(1, 12))
+ ssl_status = SSLStatFacade(from_time)
for each_vh in slb_vh_dic:
ssl_vhost_dic = get_ssl_vh_sql(from_time, to_time,each_vh)
if "vh_cc" in slb_vh_dic[each_vh]:
@@ -887,31 +889,18 @@
table_data = zip(ssl_vhost_dic['time'], net_in_table, net_out_table)
story = Draw_Line_Chart(unicode(_("Throughput"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data, yaxis_name)
- if "mentioned_statistics" in slb_vh_dic[each_vh]:
- tcpSynDropStat = TCPSYNDropStat(from_time, gz_directory="/var/crash")
- ssl_data = tcpSynDropStat.get_data()
- time_list = [e[0] for e in ssl_data]
- connection_attempts_list = [e[1] for e in ssl_data]
- successful_handshakes_data = [e[2] for e in ssl_data]
- active_connections_data = [e[3] for e in ssl_data]
+ ssl_status_dic = ssl_status.get_sql(each_vh)
+ for filter in slb_vh_dic[each_vh]:
+ field = str(filter).replace("vh_", "")
+ if filter not in list(ssl_status.C_COLUMN_MATCH.keys()):
+ continue
+ column = ssl_status.C_COLUMN_MATCH[field] # ex: Connection Attempts
graph_item = [{
- "item_name" : unicode(_("connection attempts")),
- "table_item_name" : unicode(_("connection attempts")),
- "item_value" : connection_attempts_list
- },
- {
- "item_name" : unicode(_("successful handshakes")),
- "table_item_name" : unicode(_("successful handshakes")),
- "item_value" : successful_handshakes_data
- },
- {
- "item_name" : unicode(_("active connections")),
- "table_item_name" : unicode(_("active connections")),
- "item_value" : active_connections_data
- }
- ]
- table_data = zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data)
- story = Draw_Line_Chart(unicode(_("total ssl statistics")), env, styles, story, time_list, graph_item, table_data)
+ "item_name" : unicode(_(column)),
+ "item_value" : ssl_status_dic[field + '_list']
+ }]
+ table_data = zip(ssl_status_dic['time'], ssl_status_dic[field + '_list'])
+ story = Draw_Line_Chart(unicode(_(column))+"(%s)"%each_vh, env, styles, story, ssl_status_dic['time'], graph_item, table_data)
return story
def generateSSLRHostReport(from_time, to_time, slb_rh_dic, env, styles, story):
@@ -1182,6 +1171,36 @@
}]
story = Draw_Line_Chart(unicode(_(column)), env, styles, story, time_list, graph_item, table_data)
return story
+def generateSSLTotalReport(from_time, ssl_list, env, styles, story):
+ """
+ Get PDF data of SSL total statistics and filter columns
+ :param string from_time: start time, ex: now-1d
+ :param list ssl_list: needed columns
+ :param object env
+ :param object styles
+ :param object story: pdf object
+ """
+ story.append(Spacer(1, 12))
+ ptext = '' + unicode(_("SSL Total Statistics")) + ''
+ story.append(Paragraph(ptext, styles["alibaba"]))
+ story.append(Spacer(1, 12))
+
+ ssl_status = SSLStatFacade(from_time)
+ ssl_status_dic = ssl_status.get_sql(ssl_status.C_TYPE_TOTAL)
+ fields = list(ssl_status.C_COLUMN_MATCH.keys())
+ if "time" not in ssl_status_dic:
+ return story
+ for field in fields:
+ if field not in ssl_list:
+ continue
+ column = ssl_status.C_COLUMN_MATCH[field] # ex: Connection Attempts
+ graph_item = [{
+ "item_name" : unicode(_(column)),
+ "item_value" : ssl_status_dic[field + '_list']
+ }]
+ table_data = zip(ssl_status_dic['time'], ssl_status_dic[field + '_list'])
+ story = Draw_Line_Chart(unicode(_(column)), env, styles, story, ssl_status_dic['time'], graph_item, table_data)
+ return story
def generateSystemCSV(cw, from_time, to_time, end_time, start_time, system_list):
cw.writerow("")
cw.writerow([unicode(_('System Status'))])
@@ -1375,16 +1394,16 @@
cw.writerow([unicode(_('Throughput'))])
cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))])
cw.writerows(zip(ssl_vhost_dic['time'], value_format_1024_table(ssl_vhost_dic['net_in_list']), value_format_1024_table(ssl_vhost_dic['net_out_list'])))
- if 'mentioned_statistics' in slb_vh_list[vh_name]:
- tcpSynDropStat = TCPSYNDropStat(from_time, gz_directory="/var/crash")
- ssl_data = tcpSynDropStat.get_data()
- time_list = [e[0] for e in ssl_data]
- connection_attempts_list = [e[1] for e in ssl_data]
- successful_handshakes_data = [e[2] for e in ssl_data]
- active_connections_data = [e[3] for e in ssl_data]
- cw.writerow([unicode(_('total ssl statistics'))])
- cw.writerow([unicode(_('Time')), unicode(_('connection attempts')), unicode(_('successful handshakes')), unicode(_('active connections'))])
- cw.writerows(zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data))
+ ssl_status = SSLStatFacade(from_time)
+ ssl_status_dic = ssl_status.get_sql(vh_name)
+ for filter in slb_vh_list[vh_name]:
+ field = str(filter).replace("vh_", "")
+ if field not in list(ssl_status.C_COLUMN_MATCH.keys()):
+ continue
+ column = ssl_status.C_COLUMN_MATCH[field] # ex: Connection Attempts
+ cw.writerow([unicode(_(column))])
+ cw.writerow([unicode(_('Time')), unicode(_(column))])
+ cw.writerows(zip(ssl_status_dic['time'], ssl_status_dic[field + '_list']))
return cw
def generateSSLRHostCSV(cw, from_time, to_time, rh_name, slb_rh_list):
@@ -1512,7 +1531,28 @@
# {"2024-06-28 15:42:47": {"LISTEN": "0"}}
cw.writerow([date, each[column]])
return cw
-def generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list):
+def generateSSLTotalCSV(cw, from_time, ssl_list):
+ """
+ Get CSV data of SSL statistics and filter columns
+ :param object cw: csv object
+ :param string from_time: start time, ex: now-1d
+ :param list ssl_list: needed columns
+ """
+ cw.writerow([unicode(_('SSL Total Statistics'))])
+ ssl_status = SSLStatFacade(from_time)
+ fields = list(ssl_status.C_COLUMN_MATCH.keys())
+ ssl_status_dic = ssl_status.get_sql(ssl_status.C_TYPE_TOTAL)
+ if "time" not in ssl_status_dic:
+ return cw
+ for field in fields:
+ if field not in ssl_list:
+ continue
+ column = ssl_status.C_COLUMN_MATCH[field] # ex: Connection Attempts
+ cw.writerow([unicode(_(column))])
+ cw.writerow([unicode(_('Time')), unicode(_(column))])
+ cw.writerows(zip(ssl_status_dic['time'], ssl_status_dic[field + '_list']))
+ return cw
+def generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list, ssl_total_list):
sys_info = get_system_version()
is_tcp_status = False
is_tcp_group = True
@@ -1579,6 +1619,9 @@
if tcp_list:
cw.writerow("")
cw = generateTCPCSV(cw, get_tcp_status, tcp_list)
+ if ssl_total_list:
+ cw.writerow("")
+ cw = generateSSLTotalCSV(cw, from_time, ssl_total_list)
return
def myFirstPages(canvas, doc):
@@ -1598,7 +1641,7 @@
canvas.drawRightString(7.5 * inch, 0.75 * inch, unicode(_("Page %s")) % (doc.page))
canvas.restoreState()
-def generatePDFReport(pdf_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list):
+def generatePDFReport(pdf_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list, ssl_total_list):
sys_info = get_system_version()
is_tcp_status = False
@@ -1663,6 +1706,8 @@
story = generateGSLBReport(from_time, to_time, gslb_list, env, styles, story)
if tcp_list:
story = generateTCPReport(get_tcp_status, tcp_list, env, styles, story)
+ if ssl_total_list:
+ story = generateSSLTotalReport(from_time, ssl_total_list, env, styles, story)
doc.build(story,onFirstPage=myFirstPages, onLaterPages=myLaterPages)
return
@@ -1687,6 +1732,7 @@
slb_vh_dic = {} # ssl vhost service dic
slb_rh_dic = {} # ssl rhost service dic
slb_total_list = []
+ ssl_total_list = []
llb_list = []
llb_dic = {}
llb_total_list = []
@@ -1728,6 +1774,8 @@
slb_rh_dic[each[2]].append(each[3])
if each[1] == 'slb_total':
slb_total_list.append(each[2])
+ if each[1] == 'ssl_total_report':
+ ssl_total_list.append(each[2])
if each[0] == 'llb':
if each[2] == 'total_link':
llb_total_list.append(each[3])
@@ -1748,10 +1796,10 @@
start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalStart))
if report_type == 'csv':
report_name = "Statistics_Report_%s_%s.csv" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart)))
- generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list)
+ generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list, ssl_total_list)
if report_type == 'pdf':
report_name = "Statistics_Report_%s_%s.pdf" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart)))
- generatePDFReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list)
+ generatePDFReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list, ssl_total_list)
if os.path.exists(report_file_path+report_name):
return HttpResponse(json.dumps({"result":True, "filename": report_name}))
@@ -1905,17 +1953,8 @@
}, {
"value": 'vh_throughput',
"label": unicode(_('Throughput'))
- }]
+ }] + get_statistics_ssl_list(is_vh=True)
})
-
- vh_list.append({
- "value": "individual_virtual_host",
- "label": unicode(_("Individual virtual host")) ,
- "children": [{
- "value": 'mentioned_statistics',
- "label": unicode(_('mentioned statistics'))
- }]
- })
return vh_list
def get_ssl_real_list():
@@ -2036,6 +2075,49 @@
for each in list:
each["value"] = "vs_" + each["value"]
return list
+def get_statistics_ssl_list(is_vh):
+ """
+ Get list of SSL statistics options
+ :param bool is_vh: is for virtual host? False for total
+ :return List
+ Total ex: [{
+ "value": 'connection_attempts',
+ "label": 'Connection Attempts'
+ }]
+ Virtual Host ex: [{
+ "value": 'vs_connection_attempts',
+ "label": 'Connection Attempts'
+ }]
+ """
+ list = [{
+ "value": 'connection_attempts',
+ "label": unicode(_('Connection Attempts')),
+ }, {
+ "value": 'successful_handshakes',
+ "label": unicode(_('Successful Handshakes')),
+ }, {
+ "value": 'active_connections',
+ "label": unicode(_('Active Connections')),
+ }, {
+ "value": 'alerts_total',
+ "label": unicode(_('Alerts Total')),
+ }, {
+ "value": 'alerts_sent',
+ "label": unicode(_('Alerts Sent')),
+ }, {
+ "value": 'alerts_received',
+ "label": unicode(_('Alerts Received')),
+ }, {
+ "value": 'warning_alerts',
+ "label": unicode(_('Warning Alerts')),
+ }, {
+ "value": 'fatal_alerts',
+ "label": unicode(_('Fatal Alerts')),
+ }]
+ if is_vh:
+ for each in list:
+ each["value"] = "vh_" + each["value"]
+ return list
def generateItems(request):
item_list = [{
"value": 'system',
@@ -2076,6 +2158,10 @@
"value": 'rh_report',
"label": unicode(_('SSL Real Host')),
"children": []
+ }, {
+ "value": 'ssl_total_report',
+ "label": unicode(_('SSL Total')),
+ "children": get_statistics_ssl_list(is_vh=False)
}]
}, {
"value": 'llb',