Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (revision 38372) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/report.py (working copy) @@ -1,1900 +1,2091 @@ -from pyecharts import Bar, Line, Page -from pyecharts.chart import Chart -from pyecharts.engine import create_default_environment -from reportlab.lib import colors -from reportlab.lib.enums import TA_JUSTIFY -from reportlab.lib.pagesizes import letter -from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image, TableStyle -from reportlab.platypus.tables import Table -from reportlab.graphics.shapes import Drawing -from reportlab.graphics.shapes import Line as ReportLine -from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle -from reportlab.lib.units import inch, cm -from reportlab.pdfbase import pdfmetrics -from reportlab.pdfbase.ttfonts import TTFont -from django.utils.translation import ugettext_lazy as _ -from hive.metric import * -from hive.utils import andebug, get_current_session -from hive.model.legacycli import cli_parse, RegexParser, EasyParser, MATCHALL, MATCHONE -from hive.utils import local_equirement -import random, time, csv, codecs -from hive.monitor_log.SSLStatFacade import SSLStatFacade - -TIMEFORMAT='%Y-%m-%d %X' -chart_path = "/var/crash/statmon/" -report_file_path = "/ca/webui/htdocs/new/src/media/docs/" -font_file_path = "/ca/webui/htdocs/new/src/media/font/Alibaba/" - -current_lan = 'en' - -def generate_report_table_cover(data, col_num): - t = Table(data, colWidths=int(480/col_num), rowHeights=20) - - t.setStyle(TableStyle([('BACKGROUND', (0, 0), (-1, -1), colors.white), - ('ALIGN', (1,1), (-1,-1), 'LEFT'), - ('ALIGN', (0,0), (0, -1), 'RIGHT'), - ('FONT', (0,0), (-1, -1), 'alibaba'), - ])) - t.hAlign = "RIGHT" - return t - -def generate_report_table(data, col_num): - t = Table(data, colWidths=int(480/col_num), rowHeights=20) - data_len = len(data) - - for each in range(data_len): - if each % 2 == 0: - bg_color = colors.lightgrey - else: - bg_color = colors.white - - t.setStyle(TableStyle([('BACKGROUND', (0, each), (-1, each), bg_color), - ('FONT', (0,0), (-1, -1), 'alibaba'),])) - return t - -def get_time_range(from_time, to_time): - time_range = TimeRange(from_time, to_time) - intervalStart = time_range.ParseFrom() - intervalEnd = time_range.ParseTo() - return intervalStart, intervalEnd - -def Draw_a_Line(): - d = Drawing(480, 1) - d.add(ReportLine(0, 0, 480, 0, strokeColor=colors.lightgrey)) - return d - -def Draw_Line_Chart(chart_name, env, styles, story, time_list, chart_item, table_data, yaxis_name=""): - ptext = '%s' % chart_name - story.append(Paragraph(ptext, styles["alibaba"])) - d = Draw_a_Line() - story.append(d) - - table_header = [unicode(_('Time'))] - chart_ins = Line(height=370, is_animation=False) - for each in chart_item: - chart_ins.add(each['item_name'], time_list, each['item_value'], is_smooth=True,is_splitline_show=False, is_toolbox_show=False, yaxis_formatter=yaxis_name) - table_header.append(each['table_item_name']) if "table_item_name" in each else table_header.append(each['item_name']) - - random_value = random.randint(100000, 999999) - report_chart_name = 'report_chart_%s.png' % random_value - env.render_chart_to_file(chart_ins, path=chart_path+report_chart_name) - im = Image(chart_path+report_chart_name, 7*inch, 3*inch) - story.append(im) - - data = [tuple(table_header)] + table_data - t = generate_report_table(data, len(table_header)) - story.append(t) - story.append(Spacer(1, 12)) - - return story - -def Draw_Bar_Chart(chart_name, env, styles, story, xaxis_list, chart_item, table_data, xrotate): - ptext = '%s' % chart_name - story.append(Paragraph(ptext, styles["alibaba"])) - d = Draw_a_Line() - story.append(d) - - table_header = [chart_item['xaxis_name'], chart_item['yaxis_name']] - chart_ins = Bar(height=370, is_animation=False) - chart_ins.add(chart_item['xaxis_name'], xaxis_list, chart_item['item_value'], is_smooth=True,xaxis_interval=0, xaxis_rotate=xrotate,is_splitline_show=False, is_toolbox_show=False) - - random_value = random.randint(100000, 999999) - report_chart_name = 'report_chart_%s.png' % random_value - env.render_chart_to_file(chart_ins, path=chart_path+report_chart_name) - im = Image(chart_path+report_chart_name, 7*inch, 3*inch) - story.append(im) - - data = [tuple(table_header)] + table_data - t = generate_report_table(data, len(table_header)) - story.append(t) - story.append(Spacer(1, 12)) - - return story - -def yaxis_unit(max_value): - if max_value <= 1024: - return "" - if max_value > 1024 and max_value <= 1024*1024: - return "K" - if max_value > 1024*1024 and max_value <= 1024*1024*1024: - return "M" - if max_value > 1024*1024*1024 and max_value <= 1024*1024*1024*1024: - return "G" - if max_value > 1024*1024*1024*1024: - return "T" - -def value_format_1024(list, yaxis_name): - new_list = [] - for each in list: - if each is None: - new_list.append(None) - continue - if yaxis_name == "T": - new_list.append(each/(1024*1024*1024*1024)) - elif yaxis_name == "G": - new_list.append(each/(1024*1024*1024)) - elif yaxis_name == "M": - new_list.append(each/(1024*1024)) - elif yaxis_name == "K": - new_list.append(each/(1024)) - else: - new_list.append(each) - - return new_list - -def value_format_1024_table(list): - new_list = [] - for each in list: - if each is None: - new_list.append(None) - continue - if each > 1024*1024*1024*1024: - new_list.append("%s T"%value_toFixed(each/(1024.0*1024*1024*1024), 2)) - elif each > 1024*1024*1024 and each <= 1024*1024*1024*1024: - new_list.append("%s G"%value_toFixed(each/(1024.0*1024*1024), 2)) - elif each > 1024*1024 and each <= 1024*1024*1024: - new_list.append("%s M"% value_toFixed(each/(1024.0*1024), 2)) - elif each > 1024 and each <= 1024*1024: - new_list.append("%s K"%value_toFixed(each/(1024.0),2)) - else: - new_list.append(value_toFixed(each,2)) - - return new_list - -def value_toFixed(value, number): - if value is None: - return None - else: - if number == 0: - return int(round(float(value))) - return round(float(value), number) - -def inbound_outbound_report(inbound_list, outbound_list): - try: - yaxis_name = yaxis_unit(max(max(inbound_list),max(outbound_list))) - net_in_chart = value_format_1024(inbound_list, yaxis_name) - net_out_chart = value_format_1024(outbound_list, yaxis_name) - net_in_table = value_format_1024_table(inbound_list) - net_out_table = value_format_1024_table(outbound_list) - except ValueError: - yaxis_name = "" - net_in_chart = [] - net_out_chart = [] - net_in_table = [] - net_out_table = [] - - graph_item = [{ - "item_name" : unicode(_("Inbound")), - "table_item_name" : unicode(_("Inbound(bps)")), - "item_value" : net_in_chart - }, { - "item_name" : unicode(_("Outbound")), - "table_item_name" : unicode(_("Outbound(bps)")), - "item_value" : net_out_chart - }] - - return graph_item, net_in_table, net_out_table, yaxis_name - -def package_rate_report(inbound_list, outbound_list): - try: - yaxis_name = yaxis_unit(max(max(inbound_list),max(outbound_list))) - net_in_chart = value_format_1024(inbound_list, yaxis_name) - net_out_chart = value_format_1024(outbound_list, yaxis_name) - net_in_table = value_format_1024_table(inbound_list) - net_out_table = value_format_1024_table(outbound_list) - except ValueError: - yaxis_name = "" - net_in_chart = [] - net_out_chart = [] - net_in_table = [] - net_out_table = [] - graph_item = [{ - "item_name" : unicode(_("Average Package Rate In")), - "table_item_name" : unicode(_("Average Package Rate In(pps)")), - "item_value" : net_in_chart - }, { - "item_name" : unicode(_("Average Package Rate In")), - "table_item_name" : unicode(_("Average Package Rate In(pps)")), - "item_value" : net_out_chart - }] - - return graph_item, net_in_table, net_out_table,yaxis_name - -def get_system_run_status(system_list, system_data_dic): - run_status = "" - if "cpu_usage" in system_list: - if get_run_status(system_data_dic['cpu_usage']): - run_status += unicode(_("The average CPU usage exceeded 80%.")) - if "mem_usage" in system_list: - if get_run_status(system_data_dic['sys_mem']): - run_status += unicode(_("The average system memory usage exceeded 80%.")) - if get_run_status(system_data_dic['net_mem']): - run_status += unicode(_("The average network memory usage exceeded 80%.")) - if get_run_status(system_data_dic['conn_mem']): - run_status += unicode(_("The average connection memory usage exceeded 80%.")) - if "net_load" in system_list: - if check_network_thput(system_data_dic['net_in']): - run_status += unicode(_("The average throughput exceeded 80% of the licensed throughput limit. To ensure the normal delivery of your services, it is recommended to upgrade the license to a higher throughput.")) - return run_status - -def get_run_status(data): - for each in data: - if each and each > 80: - return True - else: - continue - return False - -def check_network_thput(data): - system_data = get_system_version() - if system_data['license_limit'] == None: - return False - value = system_data['license_limit'].split()[0] - unit_name = system_data['license_limit'].split()[1] - - if "M" in unit_name: - for each in data: - if each and each >= int(value) * 1024 * 1024 * 0.8: - return True - else: - continue - if "G" in unit_name: - for each in data: - if each and each >= int(value) * 1024 * 1024 * 1024 * 0.8: - return True - else: - continue - return False - -def get_system_version(): - sess = get_current_session() - sess.cli.set_enable() - cmd_output = sess.cli.cmd('show version') - result = cli_parse(cmd_output, [ - RegexParser('Host name : (.+?)\n', MATCHONE), - RegexParser('System CPU : (.+?)\n', MATCHONE), - RegexParser('System RAM : (.+?)\n', MATCHONE), - RegexParser('System up time : (.+?)\n', MATCHONE), - RegexParser('Model : (.+?)\n', MATCHONE), - RegexParser('Licensed Limits : .*Bandwidth(.*)\n', MATCHONE), - ]) - - return { - "host_name":result[0][0], - "system_cpu":result[1][0], - "system_ram":result[2][0], - "system_boot":result[3][0], - "system_model":result[4][0], - "license_limit":result[5][0].strip("()") if result[5] else None, - } - -def get_pdf_logo(): - if local_equirement(): - logo_path = "/ca/webui/htdocs/new/src/media/img/themes/new/login-logo-cn.png" - else: - logo_path = "/ca/webui/htdocs/new/src/media/img/themes/new/Array-Logo-en-2024.png" - return logo_path - -def get_system_data_sql(from_time, to_time): - time_list = [] - cpu_list = [] - sys_mem_list = [] - net_mem_list = [] - conn_mem_list = [] - cps_list = [] - net_time_list = [] - net_in_list = [] - net_out_list = [] - - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(cpu_usage), avg(system_memory_usage), avg(network_memory_usage), avg(connection_memory_usage), avg(cps) FROM SystemStatus where $__timeFilter(time) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - - select_dic_net = {'rawSql': 'select $__timeGroup(time) as timef, avg(throughput_in_bits), avg(throughput_out_bits) from InterfaceThroughputStatistics where $__timeFilter(time) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data_net = query_metric(select_dic_net) - data = select_data['data'] - data_net = select_data_net['data'] - - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cpu_list.append(value_toFixed(each[1], 2)) - sys_mem_list.append(value_toFixed(each[2], 0)) - net_mem_list.append(value_toFixed(each[3], 0)) - conn_mem_list.append(value_toFixed(each[4],0)) - cps_list.append(value_toFixed(each[5], 0)) - for each in data_net: - net_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - net_in_list.append(each[1]) - net_out_list.append(each[2]) - - return { - 'time': time_list, - 'net_time': net_time_list, - 'cpu_usage': cpu_list, - 'sys_mem': sys_mem_list, - 'net_mem': net_mem_list, - 'conn_mem': conn_mem_list, - 'cps': cps_list, - 'net_in': net_in_list, - 'net_out': net_out_list, - } - -def get_rs_data_sql(from_time, to_time, rs_name): - time_list = [] - cc_list = [] - cps_list = [] - request_list = [] - inbound_list = [] - outbound_list = [] - average_in_list = [] - average_out_list = [] - average_response_list = [] - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_response_time), max(connection_count),max(reqcount) from GlobalRealServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %rs_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - - errcode_dic = {'rawSql': 'select INCREMENT(resp_100), INCREMENT(resp_101), INCREMENT(resp_200), INCREMENT(resp_206), INCREMENT(resp_301), INCREMENT(resp_302), INCREMENT(resp_303), INCREMENT(resp_304), INCREMENT(resp_305), INCREMENT(resp_307), INCREMENT(resp_400), INCREMENT(resp_401), INCREMENT(resp_402), INCREMENT(resp_403), INCREMENT(resp_404), INCREMENT(resp_405), INCREMENT(resp_406), INCREMENT(resp_407), INCREMENT(resp_416), INCREMENT(resp_500), INCREMENT(resp_501), INCREMENT(resp_502), INCREMENT(resp_503), INCREMENT(resp_504), INCREMENT(resp_505), INCREMENT(resp_others) from RealServiceHTTPCodeStatistics WHERE $__timeFilter(time) and service_name = "%s" group by service_name;' %rs_name, - 'to': to_time, 'from': from_time, 'format': 'table'} - errcode_data = query_metric(errcode_dic) - - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cc_list.append(value_toFixed(each[3], 0)) - cps_list.append(value_toFixed(each[7], 0)) - request_list.append(value_toFixed(each[8], 0)) - inbound_list.append(each[1]) - outbound_list.append(each[2]) - average_in_list.append(each[4]) - average_out_list.append(each[5]) - average_response_list.append(value_toFixed(each[6],2)) - - return { - 'time': time_list, - 'cc_list': cc_list, - 'cps_list': cps_list, - 'request_list': request_list, - 'inbound_list': inbound_list, - 'outbound_list': outbound_list, - 'average_in_list': average_in_list, - 'average_out_list': average_out_list, - 'average_response_list': average_response_list, - 'err_list' : errcode_data['data'] - } - -def get_link_data_sql(from_time, to_time, link_name): - time_list = [] - cc_list = [] - bandwidth_list = [] - inbound_list = [] - outbound_list = [] - hit_list = [] - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(thresh), max(current_connection), max(hits), avg(usage) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name = "%s" GROUP BY timef;'%link_name, - 'to': to_time, 'from': from_time, 'index': 1, 'format': 'time_series'} - select_data = query_metric(select_dic) - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cc_list.append(value_toFixed(each[4],0)) - bandwidth_list.append(value_toFixed(each[6],2)) - inbound_list.append(each[1]) - outbound_list.append(each[2]) - hit_list.append(value_toFixed(each[5],0)) - - return { - 'time': time_list, - 'cc_list': cc_list, - 'inbound_list': inbound_list, - 'outbound_list': outbound_list, - 'hit_list': hit_list, - 'bandwidth_list': bandwidth_list - } - -def get_llb_total_sql(from_time, to_time, link_list): - time_list = [] - - map_dict ={} - for each in link_list: - map_dict[each] = { - 'time': [], - 'cc_list': [], - 'through_list': [], - 'hit_list': [], - 'bandwidth_list': [] - } - select_dic = {'rawSql': "select $__timeGroup(time) as timef, name, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(usage), max(current_connection), max(hits), max(status) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name in ('%s') GROUP BY timef, name;" % ("','".join(link_list)), - 'to': to_time, 'from': from_time, 'index': 1, 'format': 'base_time_series'} - select_data = query_metric(select_dic) - data = select_data['data'] - - for each in data: - time_stamp = time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000)) - if time_stamp not in time_list: - time_list.append(time_stamp) - map_dict[each[1]]["cc_list"].append(value_toFixed(each[5],0)) - map_dict[each[1]]["bandwidth_list"].append(value_toFixed(each[4],2)) - map_dict[each[1]]["through_list"].append(each[2]+each[3]) - map_dict[each[1]]["hit_list"].append(value_toFixed(each[6],0)) - - return map_dict, time_list - -def get_vs_data_sql(from_time, to_time, vs_name): - time_list = [] - time_cache_list = [] - time_comp_list = [] - cc_list = [] - cps_list = [] - inbound_list = [] - outbound_list = [] - average_in_list = [] - average_out_list = [] - average_rtt_list = [] - cache_list = [] - compress_data = [] - compress_all = [] - err_list = [] - - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_rtt), max(connection_count), avg(average_request_time), avg(average_response_time) from GlobalVirtualServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %vs_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cc_list.append(value_toFixed(each[3],0)) - cps_list.append(value_toFixed(each[7],0)) - inbound_list.append(each[1]) - outbound_list.append(each[2]) - average_in_list.append(each[4]) - average_out_list.append(each[5]) - average_rtt_list.append(value_toFixed(each[6],2)) - - select_dic_cache = {'rawSql': "select $__timeGroup(time) as timef, avg(cacheHitRatio) from PervsCacheStatistics where $__timeFilter(timef/1000) and pk = '{\"service_name\": \"%s\"}' GROUP BY timef;" %vs_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data_cache = query_metric(select_dic_cache) - - for each in select_data_cache['data']: - time_cache_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cache_list.append(value_toFixed(each[1], 2)) - - select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(compRatioComp), avg(compRatioAll) from PervsCompressStatistics where $__timeFilter(timef/1000) and service_name = '%s' GROUP BY timef" %vs_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - - - for each in select_data['data']: - time_comp_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - compress_data.append(value_toFixed(each[1],0)) - compress_all.append(value_toFixed(each[2],0)) - - errcode_dic = {'rawSql': 'select INCREMENT(resp_100), INCREMENT(resp_101), INCREMENT(resp_200), INCREMENT(resp_206), INCREMENT(resp_301), INCREMENT(resp_302), INCREMENT(resp_303), INCREMENT(resp_304), INCREMENT(resp_307), INCREMENT(resp_400), INCREMENT(resp_401), INCREMENT(resp_402), INCREMENT(resp_403), INCREMENT(resp_404), INCREMENT(resp_405), INCREMENT(resp_406), INCREMENT(resp_407), INCREMENT(resp_416), INCREMENT(resp_500), INCREMENT(resp_501), INCREMENT(resp_502), INCREMENT(resp_503), INCREMENT(resp_504), INCREMENT(resp_505), INCREMENT(resp_901), INCREMENT(resp_902), INCREMENT(resp_903), INCREMENT(resp_904), INCREMENT(resp_905), INCREMENT(resp_906), INCREMENT(resp_others) from VirtualServiceHTTPCodeStatistics WHERE $__timeFilter(time) and service_name = "%s" group by service_name;' %vs_name, - 'to': to_time, 'from': from_time, 'format': 'table'} - errcode_data = query_metric(errcode_dic) - err_list = errcode_data['data'] - - return { - "time": time_list, - "time_cache_list":time_cache_list, - "time_comp_list":time_comp_list, - "cc_list": cc_list, - "cps_list": cps_list, - "inbound_list": inbound_list, - "outbound_list": outbound_list, - "average_in_list": average_in_list, - "average_out_list": average_out_list, - "average_rtt_list": average_rtt_list, - "cache_list": cache_list, - "compress_data": compress_data, - "compress_all": compress_all, - "err_list": err_list, - } - -def get_ssl_vh_sql(from_time, to_time, vhost_name): - time_list = [] - cc_list = [] - cps_list = [] - net_in_list = [] - net_out_list = [] - - select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), max(connection_count) from ExSSLVirtualHostStatistics where $__timeFilter(timef/1000) and name = '%s' GROUP BY timef;"% vhost_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - net_in_list.append(each[1]) - net_out_list.append(each[2]) - cc_list.append(value_toFixed(each[3],0)) - cps_list.append(value_toFixed(each[4],0)) - - return { - "time": time_list, - "cc_list": cc_list, - "cps_list": cps_list, - "net_in_list": net_in_list, - "net_out_list": net_out_list, - } - -def get_ssl_rh_sql(from_time, to_time, rhost_name): - time_list = [] - cc_list = [] - cps_list = [] - net_in_list = [] - net_out_list = [] - - select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), max(connection_count) from ExSSLRealHostStatistics where $__timeFilter(timef/1000) and name = '%s' GROUP BY timef;"%rhost_name, - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - net_in_list.append(each[1]) - net_out_list.append(each[2]) - cc_list.append(value_toFixed(each[3],0)) - cps_list.append(value_toFixed(each[4],0)) - - return { - "time": time_list, - "cc_list": cc_list, - "cps_list": cps_list, - "net_in_list": net_in_list, - "net_out_list": net_out_list, - } -def get_slb_total_sql(from_time, to_time): - time_list = [] - compress_list = [] - cache_time_list = [] - ssl_time_list = [] - slb_time_list = [] - cache_list = [] - ssl_in_list = [] - ssl_out_list = [] - vhost_in_list = [] - vhost_out_list = [] - rhost_in_list = [] - rhost_out_list = [] - slb_in_list = [] - slb_out_list = [] - slb_cc_list = [] - slb_cps_list = [] - - compress_sql = {'rawSql': 'select $__timeGroup(time) as timef, avg(compRatioAll) from CompressionGlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - compress_data = query_metric(compress_sql) - cache_sql = {'rawSql': 'Select $__timeGroup(time) as timef, avg(cacheHitRatio) from CacheGlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - cache_data = query_metric(cache_sql) - ssl_sql = {'rawSql': 'Select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(average_bandwidth_in_vhost), avg(average_bandwidth_out_vhost), avg(average_bandwidth_in_rhost), avg(average_bandwidth_out_rhost) from GlobalSSLThroughStats where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - ssl_data = query_metric(ssl_sql) - slb_sql = {'rawSql': 'select $__timeGroup(time) as timef, avg(vsThroughputByteIn), avg(vsThroughputByteOut), max(vsTotalConnections), max(vsTotalConnectionsAll), max(clientConnectionPerSec) from GlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - slb_data = query_metric(slb_sql) - - for each in compress_data['data']: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - compress_list.append(value_toFixed(each[1],2)) - for each in cache_data['data']: - cache_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - cache_list.append(value_toFixed(each[1],2)) - for each in ssl_data['data']: - ssl_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - ssl_in_list.append(each[1]) - ssl_out_list.append(each[2]) - vhost_in_list.append(each[3]) - vhost_out_list.append(each[4]) - rhost_in_list.append(each[5]) - rhost_out_list.append(each[6]) - for each in slb_data['data']: - slb_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - slb_in_list.append(each[1]) - slb_out_list.append(each[2]) - slb_cc_list.append(value_toFixed(each[3], 0)) - slb_cps_list.append(value_toFixed(each[5], 0)) - - return { - "time": time_list, - "compress_list": compress_list, - "cache_time_list": cache_time_list, - "ssl_time_list": ssl_time_list, - "slb_time_list": slb_time_list, - "cache_list": cache_list, - "ssl_in_list": ssl_in_list, - "ssl_out_list": ssl_out_list, - "vhost_in_list": vhost_in_list, - "vhost_out_list": vhost_out_list, - "rhost_in_list": rhost_in_list, - "rhost_out_list": rhost_out_list, - "slb_in_list": slb_in_list, - "slb_out_list": slb_out_list, - "slb_cc_list": slb_cc_list, - "slb_cps_list": slb_cps_list, - } -def generateSystemReport(from_time, to_time, system_list, env, styles, story): - system_data_dic = get_system_data_sql(from_time, to_time) - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("System Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - if "cpu_usage" in system_list: - graph_item = [{ - "item_name" : unicode(_("CPU Usage")), - "table_item_name" : unicode(_("CPU Usage(%)")), - "item_value" : system_data_dic['cpu_usage'] - }] - table_data = zip(system_data_dic['time'], system_data_dic['cpu_usage']) - story = Draw_Line_Chart(unicode(_("CPU Usage(%)")), env, styles, story, system_data_dic['time'], graph_item, table_data) - - if "mem_usage" in system_list: - graph_item = [{ - "item_name" : unicode(_("System Memory")), - "item_value" : system_data_dic['sys_mem'] - }, { - "item_name" : unicode(_("Network Memory")), - "item_value" : system_data_dic['net_mem'] - }, { - "item_name" : unicode(_("Connection Memory")), - "item_value" : system_data_dic['conn_mem'] - }] - table_data = zip(system_data_dic['time'], system_data_dic['sys_mem'], system_data_dic['net_mem'], system_data_dic['conn_mem']) - story = Draw_Line_Chart(unicode(_("Memory Usage (%)")), env, styles, story, system_data_dic['time'], graph_item, table_data) - - if "net_load" in system_list: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(system_data_dic['net_in'], system_data_dic['net_out']) - - table_data = zip(system_data_dic['net_time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Network Load(bps)")), env, styles, story, system_data_dic['net_time'], graph_item, table_data, yaxis_name) - - if "conn_status" in system_list: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : system_data_dic['cps'] - }] - table_data = zip(system_data_dic['time'], system_data_dic['cps']) - story = Draw_Line_Chart(unicode(_("Connection Status")), env, styles, story, system_data_dic['time'], graph_item, table_data) - - run_status = get_system_run_status(system_list, system_data_dic) - intervalStart, intervalEnd = get_time_range(from_time, to_time) - end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalEnd)) - start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalStart)) - if 'cpu_usage' in system_list or 'mem_usage' in system_list or 'net_load' in system_list: - ptext = '' + unicode(_("Conclusion")) +'' - story.append(Paragraph(ptext, styles["alibaba"])) - d = Draw_a_Line() - story.append(d) - ptext = ''+ unicode(_('During the period from %s to %s, ')) % (start_time, end_time) - if not run_status: - ptext += unicode(_('the system was running in a healthy status.'))+'' - else: - ptext += unicode(_('the system encountered the following abnormal conditions:'))+'' - ptext += run_status - story.append(Paragraph(ptext, styles["alibaba"])) - - return story - -def generateSLBVSReport(from_time, to_time, slb_vs_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("SLB Virtual Service Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - for each_vs in slb_vs_list: - vs_data_dic = get_vs_data_sql(from_time, to_time, each_vs) - - if 'vs_cc' in slb_vs_list[each_vs]: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : vs_data_dic['cc_list'] - }] - table_data = zip(vs_data_dic['time'], vs_data_dic['cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) - - if 'vs_cps' in slb_vs_list[each_vs]: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : vs_data_dic['cps_list'] - }] - table_data = zip(vs_data_dic['time'], vs_data_dic['cps_list']) - story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) - - if 'vs_throughput' in slb_vs_list[each_vs]: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(vs_data_dic['inbound_list'], vs_data_dic['outbound_list']) - - table_data = zip(vs_data_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Throughput(bps)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data, yaxis_name) - - if 'vs_pkg_rate' in slb_vs_list[each_vs]: - graph_item, net_in_table, net_out_table, yaxis_name = package_rate_report(vs_data_dic['average_in_list'], vs_data_dic['average_out_list']) - - table_data = zip(vs_data_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Package Rate(pps)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data, yaxis_name) - - if 'vs_rtt' in slb_vs_list[each_vs]: - graph_item = [{ - "item_name" : unicode(_("Average Client Connection RTT")), - "item_value" : vs_data_dic['average_rtt_list'] - }] - table_data = zip(vs_data_dic['time'], vs_data_dic['average_rtt_list']) - story = Draw_Line_Chart(unicode(_("Average Client Connection RTT"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) - - if 'vs_cache_hit_radio' in slb_vs_list[each_vs]: - graph_item = graph_item = [{ - "item_name" : unicode(_("Cache Hit Ratio")), - "item_value" : vs_data_dic['cache_list'] - }] - table_data = zip(vs_data_dic['time_cache_list'], vs_data_dic['cache_list']) - story = Draw_Line_Chart(unicode(_("Cache Hit Ratio(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_cache_list'], graph_item, table_data) - - if 'vs_compression_radio_data' in slb_vs_list[each_vs]: - graph_item = graph_item = [{ - "item_name" : unicode(_("Compression Ratio for Compressible Data")), - "item_value" : vs_data_dic['compress_data'] - }] - table_data = zip(vs_data_dic['time_comp_list'], vs_data_dic['compress_data']) - story = Draw_Line_Chart(unicode(_("Compression Ratio for Compressible Data(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_comp_list'], graph_item, table_data) - if 'vs_compression_radio_all' in slb_vs_list[each_vs]: - graph_item = graph_item = [{ - "item_name" : unicode(_("Compression Ratio for All Data")), - "item_value" : vs_data_dic['compress_all'] - }] - table_data = zip(vs_data_dic['time_comp_list'], vs_data_dic['compress_all']) - story = Draw_Line_Chart(unicode(_("Compression Ratio for All Data(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_comp_list'], graph_item, table_data) - if "vs_response_code" in slb_vs_list[each_vs]: - err_data = vs_data_dic["err_list"][0] if vs_data_dic["err_list"] else [0]*31 - errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 901, 902, 903, 904, 905, 906, 'others'] - graph_item = { - "xaxis_name" : unicode(_("Response Code")), - "yaxis_name" : unicode(_("Hits")), - "item_value" : err_data - } - table_data = zip(errcode_list, err_data) - story = Draw_Bar_Chart(unicode(_("HTTP Response Code Statistics"))+"(%s)"%each_vs, env, styles, story, errcode_list, graph_item, table_data, 50) - - return story - -def generateSLBRSReport(from_time, to_time, slb_rs_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("SLB Real Service Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - for each_rs in slb_rs_list: - rs_data_dic = get_rs_data_sql(from_time, to_time, each_rs) - if 'rs_cc' in slb_rs_list[each_rs]: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : rs_data_dic['cc_list'] - }] - table_data = zip(rs_data_dic['time'], rs_data_dic['cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - - if 'rs_cps' in slb_rs_list[each_rs]: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : rs_data_dic['cps_list'] - }] - table_data = zip(rs_data_dic['time'], rs_data_dic['cps_list']) - story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - - if 'rs_request' in slb_rs_list[each_rs]: - graph_item = [{ - "item_name" : unicode(_("Outstanding Request Count")), - "item_value" : rs_data_dic['request_list'] - }] - table_data = zip(rs_data_dic['time'], rs_data_dic['request_list']) - story = Draw_Line_Chart(unicode(_("Outstanding Request Count"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - - if 'rs_throughput' in slb_rs_list[each_rs]: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(rs_data_dic['inbound_list'], rs_data_dic['outbound_list']) - - table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Throughput(bps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) - - if 'rs_pkg_rate' in slb_rs_list[each_rs]: - graph_item, net_in_table, net_out_table, yaxis_name = package_rate_report(rs_data_dic['average_in_list'], rs_data_dic['average_out_list']) - - table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Package Rate(pps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) - - if 'rs_response_time' in slb_rs_list[each_rs]: - graph_item = [{ - "item_name" : unicode(_("Average Response Time")), - "item_value" : rs_data_dic['average_response_list'] - }] - table_data = zip(rs_data_dic['time'], rs_data_dic['average_response_list']) - story = Draw_Line_Chart(unicode(_("Average Response Time"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) - - if 'rs_response_code' in slb_rs_list[each_rs]: - errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 305, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 'others'] - err_data = rs_data_dic['err_list'][0] if rs_data_dic['err_list'] else [0]*26 - graph_item = { - "xaxis_name" : unicode(_("Response Code")), - "yaxis_name" : unicode(_("Hits")), - "item_value" : err_data - } - table_data = zip(errcode_list, err_data) - story = Draw_Bar_Chart(unicode(_("HTTP Response Code Statistics"))+"(%s)"%each_rs, env, styles, story, errcode_list, graph_item, table_data, 30) - - return story - -def generateSSLVHostReport(from_time, to_time, slb_vh_dic, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("SSL Virtual Host Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - for each_vh in slb_vh_dic: - ssl_vhost_dic = get_ssl_vh_sql(from_time, to_time,each_vh) - if "vh_cc" in slb_vh_dic[each_vh]: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : ssl_vhost_dic['cc_list'] - }] - table_data = zip(ssl_vhost_dic['time'], ssl_vhost_dic['cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data) - - if "vh_cps" in slb_vh_dic[each_vh]: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : ssl_vhost_dic['cps_list'] - }] - table_data = zip(ssl_vhost_dic['time'], ssl_vhost_dic['cps_list']) - story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data) - - if "vh_throughput" in slb_vh_dic[each_vh]: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(ssl_vhost_dic['net_in_list'], ssl_vhost_dic['net_out_list']) - - table_data = zip(ssl_vhost_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Throughput"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data, yaxis_name) - if "mentioned_statistics" in slb_vh_dic[each_vh]: - sslStat = SSLStatFacade(from_time, gz_directory="/var/crash") - ssl_data = sslStat.get_data() - time_list = [e[0] for e in ssl_data] - connection_attempts_list = [e[1] for e in ssl_data] - successful_handshakes_data = [e[2] for e in ssl_data] - active_connections_data = [e[3] for e in ssl_data] - graph_item = [{ - "item_name" : unicode(_("connection attempts")), - "table_item_name" : unicode(_("connection attempts")), - "item_value" : connection_attempts_list - }, - { - "item_name" : unicode(_("successful handshakes")), - "table_item_name" : unicode(_("successful handshakes")), - "item_value" : successful_handshakes_data - }, - { - "item_name" : unicode(_("active connections")), - "table_item_name" : unicode(_("active connections")), - "item_value" : active_connections_data - } - ] - table_data = zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data) - story = Draw_Line_Chart(unicode(_("total ssl statistics")), env, styles, story, time_list, graph_item, table_data) - return story - -def generateSSLRHostReport(from_time, to_time, slb_rh_dic, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("SSL Real Host Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - for each_rh in slb_rh_dic: - ssl_rhost_dic = get_ssl_rh_sql(from_time, to_time,each_rh) - if "rh_cc" in slb_rh_dic[each_rh]: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : ssl_rhost_dic['cc_list'] - }] - table_data = zip(ssl_rhost_dic['time'], ssl_rhost_dic['cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data) - - if "rh_cps" in slb_rh_dic[each_rh]: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : ssl_rhost_dic['cps_list'] - }] - table_data = zip(ssl_rhost_dic['time'], ssl_rhost_dic['cps_list']) - story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data) - - if "rh_throughput" in slb_rh_dic[each_rh]: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(ssl_rhost_dic['net_in_list'], ssl_rhost_dic['net_out_list']) - - table_data = zip(ssl_rhost_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Throughput"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data, yaxis_name) - - return story - -def generateSLBTotalReport(from_time, to_time, slb_total_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("SLB Global Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - slb_total_dic = get_slb_total_sql(from_time, to_time) - if "total_throughput" in slb_total_list: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['slb_in_list'], slb_total_dic['slb_out_list']) - - table_data = zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("SLB Throughput(bps)")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data, yaxis_name) - - if "curr_conn" in slb_total_list: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : slb_total_dic['slb_cc_list'] - }] - table_data = zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data) - - if "cps" in slb_total_list: - graph_item = [{ - "item_name" : unicode(_("CPS")), - "item_value" : slb_total_dic['slb_cps_list'] - }] - table_data = zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cps_list']) - story = Draw_Line_Chart(unicode(_("CPS")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data) - - if "ssl_throughput" in slb_total_list: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['ssl_in_list'], slb_total_dic['ssl_out_list']) - - table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("SSL Total Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) - - if "ssl_vh_throughput" in slb_total_list: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['vhost_in_list'], slb_total_dic['vhost_out_list']) - - table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("SSL Virtual Host Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) - - if "ssl_rh_throughput" in slb_total_list: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['rhost_in_list'], slb_total_dic['rhost_out_list']) - - table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("SSL Real Host Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) - - if 'compress_ratio' in slb_total_list: - graph_item = [{ - "item_name" : unicode(_("Compression Ratio for All Data")), - "item_value" : slb_total_dic['compress_list'] - }] - table_data = zip(slb_total_dic['time'], slb_total_dic['compress_list']) - story = Draw_Line_Chart(unicode(_("Compression Ratio for All Data(%)")), env, styles, story, slb_total_dic['time'], graph_item, table_data) - - if 'cache_hit_radio' in slb_total_list: - graph_item = [{ - "item_name" : unicode(_("Cache Hits Ratio")), - "item_value" : slb_total_dic['cache_list'] - }] - table_data = zip(slb_total_dic['cache_time_list'], slb_total_dic['cache_list']) - story = Draw_Line_Chart(unicode(_("Cache Hit Ratio(%)")), env, styles, story, slb_total_dic['cache_time_list'], graph_item, table_data) - - return story - -def generateLLBTotalReport(from_time, to_time, llb_total_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("LLB Total Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - p = os.popen("/ca/bin/webui_utils -a 4") - #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' - llb_dic = json.loads(p.read()) - link_list = [] - for each in llb_dic["data"]: - link_list.append(each["name"]) - link_list = sorted(link_list) - - total_dic, time_list = get_llb_total_sql(from_time, to_time, link_list) - if "bandwidth_usage" in llb_total_list: - graph_item = [] - zip_list = [time_list] - for each in link_list: - graph_item.append({ - "item_name" : each, - "item_value" : total_dic[each]['bandwidth_list'] - }) - zip_list.append(total_dic[each]['bandwidth_list']) - table_data = zip(*zip_list) - story = Draw_Line_Chart(unicode(_("Bandwidth Usage")), env, styles, story, time_list, graph_item, table_data) - - if "link_throughput" in llb_total_list: - graph_item = [] - zip_list = [time_list] - max_list = [] - for each in link_list: - max_list.append(max(total_dic[each]['through_list'])) - yaxis_name = yaxis_unit(max(max_list)) - for each in link_list: - graph_item.append({ - "item_name" : each, - "item_value" : value_format_1024(total_dic[each]['through_list'],yaxis_name) - }) - zip_list.append(value_format_1024_table(total_dic[each]['through_list'])) - table_data = zip(*zip_list) - story = Draw_Line_Chart(unicode(_("Total Link Throughput(bps)")), env, styles, story, time_list, graph_item, table_data, yaxis_name) - - if "curr_conn" in llb_total_list: - graph_item = [] - zip_list = [time_list] - for each in link_list: - graph_item.append({ - "item_name" : each, - "item_value" : total_dic[each]['cc_list'] - }) - zip_list.append(total_dic[each]['cc_list']) - table_data = zip(*zip_list) - story = Draw_Line_Chart(unicode(_("Concurrent Connections")), env, styles, story, time_list, graph_item, table_data) - - if "total_hits" in llb_total_list: - graph_item = [] - zip_list = [time_list] - for each in link_list: - graph_item.append({ - "item_name" : each, - "item_value" : total_dic[each]['hit_list'] - }) - zip_list.append(total_dic[each]['hit_list']) - table_data = zip(*zip_list) - story = Draw_Line_Chart(unicode(_("Total Hits")), env, styles, story, time_list, graph_item, table_data) - - return story - -def generateLLBReport(from_time, to_time, llb_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = '' + unicode(_("LLB Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - for each_link in llb_list: - link_data_dic = get_link_data_sql(from_time, to_time, each_link) - if 'bandwidth_usage' in llb_list[each_link]: - graph_item = [{ - "item_name" : unicode(_("Bandwidth Usage")), - "item_value" : link_data_dic['bandwidth_list'] - }] - table_data = zip(link_data_dic['time'], link_data_dic['bandwidth_list']) - story = Draw_Line_Chart(unicode(_("Bandwidth Usage"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) - - if 'link_throughput' in llb_list[each_link]: - graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(link_data_dic['inbound_list'], link_data_dic['outbound_list']) - - table_data = zip(link_data_dic['time'], net_in_table, net_out_table) - story = Draw_Line_Chart(unicode(_("Link Throughput(bps)"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data, yaxis_name) - - if 'curr_conn' in llb_list[each_link]: - graph_item = [{ - "item_name" : unicode(_("Concurrent Connections")), - "item_value" : link_data_dic['cc_list'] - }] - table_data = zip(link_data_dic['time'], link_data_dic['cc_list']) - story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) - - if 'total_hits' in llb_list[each_link]: - graph_item = [{ - "item_name" : unicode(_("Hits")), - "item_value" : link_data_dic['hit_list'] - }] - table_data = zip(link_data_dic['time'], link_data_dic['hit_list']) - story = Draw_Line_Chart(unicode(_("Hits"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) - - return story - -def generateGSLBReport(from_time, to_time, gslb_list, env, styles, story): - story.append(Spacer(1, 12)) - ptext = ''+ unicode(_("GSLB Status")) + '' - story.append(Paragraph(ptext, styles["alibaba"])) - story.append(Spacer(1, 12)) - - time_list = [] - hit_list = [] - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, max(total_req) from QueryStatistics where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - data = select_data['data'] - for each in data: - time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) - hit_list.append(value_toFixed(each[1],0)) - - graph_item = [{ - "item_name" : unicode(_("Total Hits")), - "item_value" : hit_list - }] - table_data = zip(time_list, hit_list) - story = Draw_Line_Chart(unicode(_("Total Hits")), env, styles, story, time_list, graph_item, table_data) - - return story - -def generateSystemCSV(cw, from_time, to_time, end_time, start_time, system_list): - cw.writerow("") - cw.writerow([unicode(_('System Status'))]) - system_data_dic = get_system_data_sql(from_time, to_time) - - if 'cpu_usage' in system_list: - cw.writerow([unicode(_('CPU Usage(%)'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPU Usage'))]) - cw.writerows(zip(system_data_dic['time'], system_data_dic['cpu_usage'])) - if 'mem_usage' in system_list: - cw.writerow([unicode(_('Memory Usage (%)'))]) - cw.writerow([unicode(_('Time')), unicode(_('System Memory Usage')), unicode(_('Network Memory Usage')), unicode(_('Connection Memory Usage'))]) - cw.writerows(zip(system_data_dic['time'], system_data_dic['sys_mem'], system_data_dic['net_mem'], system_data_dic['conn_mem'])) - if 'conn_status' in system_list: - cw.writerow([unicode(_('Connection Status'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(system_data_dic['time'], system_data_dic['cps'])) - if 'net_load' in system_list: - net_in_table = value_format_1024_table(system_data_dic['net_in']) - net_out_table = value_format_1024_table(system_data_dic['net_out']) - cw.writerow([unicode(_('Network Load'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(system_data_dic['time'], net_in_table, net_out_table)) - run_status = get_system_run_status(system_list, system_data_dic) - if 'cpu_usage' in system_list or 'mem_usage' in system_list or 'net_load' in system_list: - cw.writerow([unicode(_("Conclusion"))]) - ptext = unicode(_('During the period from %s to %s, ')) % (start_time, end_time) - if not run_status: - ptext += unicode(_('the system was running in a healthy status.')) - else: - ptext += unicode(_('the system encountered the following abnormal conditions:')) - ptext += run_status - cw.writerow([ptext]) - return cw - -def generateSLBTotalCSV(cw, from_time, to_time, slb_total_list): - cw.writerow([unicode(_("SLB Global Status"))]) - slb_total_dic = get_slb_total_sql(from_time, to_time) - if 'total_throughput' in slb_total_list: - net_in_table = value_format_1024_table(slb_total_dic['slb_in_list']) - net_out_table = value_format_1024_table(slb_total_dic['slb_out_list']) - cw.writerow([unicode(_('SLB Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) - if 'curr_conn' in slb_total_list: - cw.writerow([unicode(_('Concurrent Connections'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cc_list'])) - if 'cps' in slb_total_list: - cw.writerow([unicode(_('CPS'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cps_list'])) - if 'ssl_throughput' in slb_total_list: - net_in_table = value_format_1024_table(slb_total_dic['ssl_in_list']) - net_out_table = value_format_1024_table(slb_total_dic['ssl_out_list']) - cw.writerow([unicode(_('SSL Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) - if 'ssl_vh_throughput' in slb_total_list: - net_in_table = value_format_1024_table(slb_total_dic['vhost_in_list']) - net_out_table = value_format_1024_table(slb_total_dic['vhost_out_list']) - cw.writerow([unicode(_('SSL Virtual Host Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) - if 'ssl_rh_throughput' in slb_total_list: - net_in_table = value_format_1024_table(slb_total_dic['rhost_in_list']) - net_out_table = value_format_1024_table(slb_total_dic['rhost_out_list']) - cw.writerow([unicode(_('SSL Real Host Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) - if 'cache_hit_radio' in slb_total_list: - cw.writerow([unicode(_('Cache Hit Ratio'))]) - cw.writerow([unicode(_('Time')), unicode(_('Cache Hit Ratio(%)'))]) - cw.writerows(zip(slb_total_dic['cache_time_list'], slb_total_dic['cache_list'])) - if 'compress_ratio' in slb_total_list: - cw.writerow([unicode(_('Compression Ratio for All Data'))]) - cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for All Data(%)'))]) - cw.writerows(zip(slb_total_dic['time'], slb_total_dic['compress_list'])) - return cw - -def generateSLBVSCSV(cw, from_time, to_time, vs_name, slb_vs_list): - cw.writerow([unicode(_("SLB Virtual Service Status"))+"(%s)"%vs_name]) - vs_data_dic = get_vs_data_sql(from_time, to_time, vs_name) - - if 'vs_cc' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('Concurrent Count'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cc_list'])) - if 'vs_cps' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('CPS'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cps_list'])) - if 'vs_throughput' in slb_vs_list[vs_name]: - net_in_table = value_format_1024_table(vs_data_dic['inbound_list']) - net_out_table = value_format_1024_table(vs_data_dic['outbound_list']) - cw.writerow([unicode(_('Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(vs_data_dic['time'], net_in_table, net_out_table)) - if 'vs_pkg_rate' in slb_vs_list[vs_name]: - net_in_table = value_format_1024_table(vs_data_dic['average_in_list']) - net_out_table = value_format_1024_table(vs_data_dic['average_out_list']) - cw.writerow([unicode(_('Package Rate'))]) - cw.writerow([unicode(_('Time')), unicode(_('Average Package Rate In(pps)')), unicode(_('Average Package Rate Out(pps)'))]) - cw.writerows(zip(vs_data_dic['time'], net_in_table, net_out_table)) - if 'vs_rtt' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('Average Client Connection RTT'))]) - cw.writerow([unicode(_('Time')), unicode(_('Average Client Connection RTT'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['average_rtt_list'])) - if "vs_response_code" in slb_vs_list[vs_name]: - err_data = vs_data_dic["err_list"][0] if vs_data_dic["err_list"] else [0]*31 - errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 901, 902, 903, 904, 905, 906, 'others'] - cw.writerow([unicode(_('HTTP Response Code Statistics'))]) - cw.writerow([unicode(_('Response Code')), unicode(_('Hits'))]) - cw.writerows(zip(errcode_list, err_data)) - if 'vs_cache_hit_radio' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('Cache Hit Ratio'))]) - cw.writerow([unicode(_('Time')), unicode(_('Cache Hit Ratio(%)'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cache_list'])) - if 'vs_compression_radio_data' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('Compression Ratio for Compressible Data'))]) - cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for Compressible Data(%)'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['compress_data'])) - if 'vs_compression_radio_all' in slb_vs_list[vs_name]: - cw.writerow([unicode(_('Compression Ratio for All Data'))]) - cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for All Data(%)'))]) - cw.writerows(zip(vs_data_dic['time'], vs_data_dic['compress_all'])) - return cw - -def generateSLBRSCSV(cw, from_time, to_time, rs_name, slb_rs_list): - cw.writerow([unicode(_("SLB Real Service Status"))+"(%s)"%rs_name]) - rs_data_dic = get_rs_data_sql(from_time, to_time, rs_name) - - if 'rs_cc' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('Concurrent Count'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) - cw.writerows(zip(rs_data_dic['time'], rs_data_dic['cc_list'])) - if 'rs_cps' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('CPS'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(rs_data_dic['time'], rs_data_dic['cps_list'])) - if 'rs_request' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('Outstanding Request Count'))]) - cw.writerow([unicode(_('Time')), unicode(_('Outstanding Request Count'))]) - cw.writerows(zip(rs_data_dic['time'], rs_data_dic['request_list'])) - if 'rs_throughput' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(rs_data_dic['time'], value_format_1024_table(rs_data_dic['inbound_list']), value_format_1024_table(rs_data_dic['outbound_list']))) - if 'rs_pkg_rate' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('Package Rate'))]) - cw.writerow([unicode(_('Time')), unicode(_('Average Package Rate In(pps)')), unicode(_('Average Package Rate Out(pps)'))]) - cw.writerows(zip(rs_data_dic['time'], value_format_1024_table(rs_data_dic['average_in_list']), value_format_1024_table(rs_data_dic['average_out_list']))) - if 'rs_response_time' in slb_rs_list[rs_name]: - cw.writerow([unicode(_('Average Response Time'))]) - cw.writerow([unicode(_('Time')), unicode(_('Average Response Time'))]) - cw.writerows(zip(rs_data_dic['time'], rs_data_dic['average_response_list'])) - if 'rs_response_code' in slb_rs_list[rs_name]: - errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 305, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 'others'] - err_data = rs_data_dic['err_list'][0] if rs_data_dic['err_list'] else [0]*26 - cw.writerow([unicode(_('HTTP Response Code Statistics'))]) - cw.writerow([unicode(_('Response Code')), unicode(_('Hits'))]) - cw.writerows(zip(errcode_list, err_data)) - return cw - -def generateSSLVHostCSV(cw, from_time, to_time, vh_name, slb_vh_list): - cw.writerow("") - cw.writerow([unicode(_('SSL Virtual Host Status'))+"(%s)"%vh_name]) - ssl_vhost_dic = get_ssl_vh_sql(from_time, to_time,vh_name) - - if 'vh_cc' in slb_vh_list[vh_name]: - cw.writerow([unicode(_('Concurrent Connections'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) - cw.writerows(zip(ssl_vhost_dic['time'], ssl_vhost_dic['cc_list'])) - if 'vh_cps' in slb_vh_list[vh_name]: - cw.writerow([unicode(_('CPS'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(ssl_vhost_dic['time'], ssl_vhost_dic['cps_list'])) - if 'vh_throughput' in slb_vh_list[vh_name]: - cw.writerow([unicode(_('Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(ssl_vhost_dic['time'], value_format_1024_table(ssl_vhost_dic['net_in_list']), value_format_1024_table(ssl_vhost_dic['net_out_list']))) - if 'mentioned_statistics' in slb_vh_list[vh_name]: - sslStat = SSLStatFacade(from_time, gz_directory="/var/crash") - ssl_data = sslStat.get_data() - time_list = [e[0] for e in ssl_data] - connection_attempts_list = [e[1] for e in ssl_data] - successful_handshakes_data = [e[2] for e in ssl_data] - active_connections_data = [e[3] for e in ssl_data] - cw.writerow([unicode(_('total ssl statistics'))]) - cw.writerow([unicode(_('Time')), unicode(_('connection attempts')), unicode(_('successful handshakes')), unicode(_('active connections'))]) - cw.writerows(zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data)) - return cw - -def generateSSLRHostCSV(cw, from_time, to_time, rh_name, slb_rh_list): - cw.writerow("") - cw.writerow([unicode(_('SSL Real Host Status'))+"(%s)"%rh_name]) - ssl_rhost_dic = get_ssl_rh_sql(from_time, to_time,rh_name) - - if 'rh_cc' in slb_rh_list[rh_name]: - cw.writerow([unicode(_('Concurrent Connections'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) - cw.writerows(zip(ssl_rhost_dic['time'], ssl_rhost_dic['cc_list'])) - if 'rh_cps' in slb_rh_list[rh_name]: - cw.writerow([unicode(_('CPS'))]) - cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) - cw.writerows(zip(ssl_rhost_dic['time'], ssl_rhost_dic['cps_list'])) - if 'rh_throughput' in slb_rh_list[rh_name]: - cw.writerow([unicode(_('Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(ssl_rhost_dic['time'], value_format_1024_table(ssl_rhost_dic['net_in_list']), value_format_1024_table(ssl_rhost_dic['net_out_list']))) - return cw - -def generateLLBTotalCSV(cw, from_time, to_time, llb_total_list): - cw.writerow([unicode(_("LLB Total Status"))]) - - p = os.popen("/ca/bin/webui_utils -a 4") - #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' - llb_dic = json.loads(p.read()) - link_list = [] - for each in llb_dic["data"]: - link_list.append(each["name"]) - link_list = sorted(link_list) - - total_dic, time_list = get_llb_total_sql(from_time, to_time, link_list) - if 'bandwidth_usage' in llb_total_list: - cw.writerow([unicode(_('Bandwidth Usage'))]) - csv_header = [unicode(_('Time'))] - csv_body = [time_list] - for each in link_list: - csv_header.append(each) - csv_body.append(total_dic[each]['bandwidth_list']) - cw.writerow(csv_header) - cw.writerows(zip(*csv_body)) - if 'link_throughput' in llb_total_list: - cw.writerow([unicode(_('Throughput'))]) - csv_header = [unicode(_('Time'))] - csv_body = [time_list] - for each in link_list: - csv_header.append(each) - csv_body.append(value_format_1024_table(total_dic[each]['through_list'])) - cw.writerow(csv_header) - cw.writerows(zip(*csv_body)) - if 'curr_conn' in llb_total_list: - cw.writerow([unicode(_('Concurrent Count'))]) - csv_header = [unicode(_('Time'))] - csv_body = [time_list] - for each in link_list: - csv_header.append(each) - csv_body.append(total_dic[each]['cc_list']) - cw.writerow(csv_header) - cw.writerows(zip(*csv_body)) - if 'total_hits' in llb_total_list: - cw.writerow([unicode(_('Total Hits'))]) - csv_header = [unicode(_('Time'))] - csv_body = [time_list] - for each in link_list: - csv_header.append(each) - csv_body.append(total_dic[each]['hit_list']) - cw.writerow(csv_header) - cw.writerows(zip(*csv_body)) - return cw - -def generateLLBCSV(cw, from_time, to_time, link_name, llb_list): - cw.writerow([unicode(_("LLB Status"))+"(%s)"%link_name]) - link_data_dic = get_link_data_sql(from_time, to_time, link_name) - - if 'bandwidth_usage' in llb_list[link_name]: - cw.writerow([unicode(_('Bandwidth Usage'))]) - cw.writerow([unicode(_('Time')), unicode(_('Bandwidth Usage'))]) - cw.writerows(zip(link_data_dic['time'], link_data_dic['bandwidth_list'])) - if 'link_throughput' in llb_list[link_name]: - cw.writerow([unicode(_('Throughput'))]) - cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) - cw.writerows(zip(link_data_dic['time'], value_format_1024_table(link_data_dic['inbound_list']), value_format_1024_table(link_data_dic['outbound_list']))) - if 'curr_conn' in llb_list[link_name]: - cw.writerow([unicode(_('Concurrent Count'))]) - cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) - cw.writerows(zip(link_data_dic['time'], link_data_dic['cc_list'])) - if 'total_hits' in llb_list[link_name]: - cw.writerow([unicode(_('Total Hits'))]) - cw.writerow([unicode(_('Time')), unicode(_('Total Hits'))]) - cw.writerows(zip(link_data_dic['time'], link_data_dic['hit_list'])) - return cw - -def generateGSLBRSCSV(cw, from_time, to_time): - select_dic = {'rawSql': 'select $__timeGroup(time) as timef, max(total_req) from QueryStatistics where $__timeFilter(timef/1000) GROUP BY timef;', - 'to': to_time, 'from': from_time, 'format': 'time_series'} - select_data = query_metric(select_dic) - cw.writerow([unicode(_('GSLB Status'))]) - cw.writerow([unicode(_('Total Hits'))]) - cw.writerow([unicode(_('Time')), unicode(_('Hits'))]) - for each in select_data['data']: - each = list(each) - each[0] = time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000)) - cw.writerow(each) - return cw - -def generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list): - sys_info = get_system_version() - with codecs.open(report_file_path+report_name, 'w', encoding="utf-8-sig") as csvfile: - cw = csv.writer(csvfile) - cw.writerow([unicode(_('Create Time')), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())]) - cw.writerow([unicode(_('Statistics From')), start_time]) - cw.writerow([unicode(_('Statistics To')), end_time]) - cw.writerow([unicode(_('Host Name')), sys_info['host_name']]) - cw.writerow([unicode(_('System CPU')), sys_info['system_cpu']]) - cw.writerow([unicode(_('System RAM')), sys_info['system_ram']]) - cw.writerow([unicode(_('Cumulative Running Time')), sys_info['system_boot']]) - cw.writerow([unicode(_('Model')), sys_info['system_model']]) - - if system_list: - cw = generateSystemCSV(cw, from_time, to_time, end_time, start_time, system_list) - if slb_total_list: - cw.writerow("") - cw = generateSLBTotalCSV(cw, from_time, to_time, slb_total_list) - if slb_vs_list: - cw.writerow("") - for each in slb_vs_list: - vs_name = each - cw = generateSLBVSCSV(cw, from_time, to_time, vs_name, slb_vs_list) - if slb_rs_list: - cw.writerow("") - for each in slb_rs_list: - rs_name = each - cw = generateSLBRSCSV(cw, from_time, to_time, rs_name, slb_rs_list) - if slb_vh_list: - cw.writerow("") - for each in slb_vh_list: - vh_name = each - cw = generateSSLVHostCSV(cw, from_time, to_time, vh_name, slb_vh_list) - if slb_rh_list: - cw.writerow("") - for each in slb_rh_list: - rh_name = each - cw = generateSSLRHostCSV(cw, from_time, to_time, rh_name, slb_rh_list) - if llb_total_list: - cw.writerow("") - cw = generateLLBTotalCSV(cw, from_time, to_time, llb_total_list) - if llb_list: - cw.writerow("") - for each in llb_list: - link_name = each - cw = generateLLBCSV(cw, from_time, to_time, link_name, llb_list) - if gslb_list: - cw.writerow("") - cw = generateGSLBRSCSV(cw, from_time, to_time) - - return - -def myFirstPages(canvas, doc): - canvas.saveState() - canvas.setFont('alibaba',10) - canvas.drawRightString(7.5 * inch, 0.75 * inch, unicode(_("Page %s")) % (doc.page)) - canvas.restoreState() - -def myLaterPages(canvas, doc): - styles=getSampleStyleSheet() - canvas.saveState() - p = Paragraph("" % get_pdf_logo(), styles["Heading1"]) - w, h = p.wrap(doc.width, doc.bottomMargin) - p.drawOn(canvas, doc.leftMargin, doc.topMargin + doc.height - 0.5*cm) - #canvas.line(doc.leftMargin, doc.bottomMargin+doc.height + 0.5*cm, doc.leftMargin+doc.width, doc.bottomMargin+doc.height + 0.5*cm) - canvas.setFont('alibaba',10) - canvas.drawRightString(7.5 * inch, 0.75 * inch, unicode(_("Page %s")) % (doc.page)) - canvas.restoreState() - -def generatePDFReport(pdf_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list): - sys_info = get_system_version() - doc = SimpleDocTemplate(report_file_path+pdf_name,pagesize=letter, - rightMargin=60,leftMargin=60, - topMargin=70,bottomMargin=55) - env = create_default_environment("png") - if current_lan == 'ja': - pdfmetrics.registerFont(TTFont('alibaba', font_file_path + 'AlibabaSansJP/AlibabaSansJP-Regular.ttf')) - else: - pdfmetrics.registerFont(TTFont('alibaba', 'Alibaba-PuHuiTi-Regular.ttf')) - - styles=getSampleStyleSheet() - styles.add(ParagraphStyle(fontName='alibaba', name='alibaba', alignment=TA_JUSTIFY)) - data = [[unicode(_("Create Time")), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())], - [unicode(_("Statistics From")), start_time], - [unicode(_("Statistics To")), end_time], - [unicode(_("Host Name")), sys_info['host_name']], - [unicode(_("System CPU")), sys_info['system_cpu']], - [unicode(_("System RAM")), sys_info['system_ram']], - [unicode(_("Model")), sys_info['system_model']], - [unicode(_("Cumulative Running Time")), sys_info['system_boot']]] - t = generate_report_table_cover(data, 3) - im = Image(get_pdf_logo(), 2*inch, 0.8*inch) - im.hAlign = 'LEFT' - story = [] - story.append(im) - story.append(t) - - if system_list: - story = generateSystemReport(from_time, to_time, system_list, env, styles, story) - if slb_total_list: - story = generateSLBTotalReport(from_time, to_time, slb_total_list, env, styles, story) - if slb_vs_list: - story = generateSLBVSReport(from_time, to_time, slb_vs_list, env, styles, story) - if slb_rs_list: - story = generateSLBRSReport(from_time, to_time, slb_rs_list, env, styles, story) - if slb_vh_list: - story = generateSSLVHostReport(from_time, to_time, slb_vh_list, env, styles, story) - if slb_rh_list: - story = generateSSLRHostReport(from_time, to_time, slb_rh_list, env, styles, story) - if llb_total_list: - story = generateLLBTotalReport(from_time, to_time, llb_total_list, env, styles, story) - if llb_list: - story = generateLLBReport(from_time, to_time, llb_list, env, styles, story) - if gslb_list: - story = generateGSLBReport(from_time, to_time, gslb_list, env, styles, story) - - doc.build(story,onFirstPage=myFirstPages, onLaterPages=myLaterPages) - - return - -def generateReport(request): - TIMEFORMAT='%Y-%m-%d %X' - post_data = request.raw_post_data - post_dict = json.loads(post_data) - from_time = post_dict['from'] - to_time = post_dict['to'] - monitor_type = post_dict['monitor_type'] - report_type = post_dict['report_type'] - global current_lan - current_lan = request.COOKIES['django_language'] - system_list = [] - slb_rn_list = [] # slb rs name list - slb_vn_list = [] # slb vs name list - slb_vs_list = {} # slb vs service dic - slb_rs_list = {} # slb rs service dic - slb_vh_list = [] # ssl vhost name list - slb_rh_list = [] # ssl rhost name list - slb_vh_dic = {} # ssl vhost service dic - slb_rh_dic = {} # ssl rhost service dic - slb_total_list = [] - llb_list = [] - llb_dic = {} - llb_total_list = [] - gslb_list = [] - - for each in monitor_type: - if each[0] == 'system': - system_list.append(each[1]) - if each[0] == 'slb': - if each[1] == 'vs_report': - if each[2] not in slb_vn_list: - slb_vn_list.append(each[2]) - slb_vs_list[each[2]] = [each[3]] - else: - slb_vs_list[each[2]].append(each[3]) - if each[1] == 'rs_report': - if each[2] not in slb_rn_list: - slb_rn_list.append(each[2]) - slb_rs_list[each[2]] = [each[3]] - else: - slb_rs_list[each[2]].append(each[3]) - if each[1] == 'vh_report': - if each[2] not in slb_vh_list: - slb_vh_list.append(each[2]) - slb_vh_dic[each[2]] = [each[3]] - else: - slb_vh_dic[each[2]].append(each[3]) - if each[1] == 'rh_report': - if each[2] not in slb_rh_list: - slb_rh_list.append(each[2]) - slb_rh_dic[each[2]] = [each[3]] - else: - slb_rh_dic[each[2]].append(each[3]) - if each[1] == 'slb_total': - slb_total_list.append(each[2]) - if each[0] == 'llb': - if each[2] == 'total_link': - llb_total_list.append(each[3]) - else: - if each[2] not in llb_list: - llb_list.append(each[2]) - llb_dic[each[2]] = [each[3]] - else: - llb_dic[each[2]].append(each[3]) - if each[0] == 'gslb': - gslb_list.append(each[1]) - os.popen("rm -rf /var/crash/statmon/report_chart_*") - os.popen("rm -rf /ca/webui/htdocs/new/src/media/docs/Statistics_Report_*") - intervalStart, intervalEnd = get_time_range(from_time, to_time) - end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalEnd)) - start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalStart)) - if report_type == 'csv': - report_name = "Statistics_Report_%s_%s.csv" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart))) - generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list) - if report_type == 'pdf': - report_name = "Statistics_Report_%s_%s.pdf" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart))) - generatePDFReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list) - - if os.path.exists(report_file_path+report_name): - return HttpResponse(json.dumps({"result":True, "filename": report_name})) - else: - return HttpResponse(json.dumps({"result":False})) - -def get_slb_total_list(): - return [{ - "value": 'total_throughput', - "label": unicode(_('Total Throughput')) - }, { - "value": 'curr_conn', - "label": unicode(_('Concurrent Connections')) - }, { - "value": 'cps', - "label": unicode(_('CPS')) - }, { - "value": 'ssl_throughput', - "label": unicode(_('SSL Total Throughput')) - }, { - "value": 'ssl_vh_throughput', - "label": unicode(_('SSL Virtual Host Throughput')) - }, { - "value": 'ssl_rh_throughput', - "label": unicode(_('SSL Real Host Throughput')) - }, { - "value": 'cache_hit_radio', - "label": unicode(_('Cache Hits Ratio')) - }, { - "value": 'compress_ratio', - "label": unicode(_('Compression Ratio for All Data')) - }] - -def get_slb_virtual_list(): - vs_list = [] - p = os.popen("/ca/bin/webui_utils -a 6") - #p = '{"result": 1, "data": [{"service_name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' - vs_dic = json.loads(p.read()) - - for each in vs_dic["data"]: - vs_name = each["service_name"] - vs_type = each["protocol"] - data = { - "value": vs_name, - "label": vs_name, - "children": [{ - # "value": "vs_policy_hits", - # "label": unicode(_("Policy Hits (Top 10)")), - # },{ - # "value": "vs_rs_load", - # "label": unicode(_("Real Service Load (Top 10)")), - # },{ - "value": "vs_cc", - "label": unicode(_("Concurrent Connections")), - },{ - "value": "vs_cps", - "label": unicode(_("CPS")), - },{ - "value": "vs_throughput", - "label": unicode(_("Throughput")), - },{ - "value": "vs_pkg_rate", - "label": unicode(_("Package Rate")), - },{ - "value": "vs_rtt", - "label": unicode(_("Average Client Connection RTT")), - }] - } - if vs_type in ['http', 'https']: - data["children"].extend([{ - "value": "vs_response_code", - "label": unicode(_("HTTP Response Code Statistics")), - },{ - "value": "vs_cache_hit_radio", - "label": unicode(_("Cache Hits Ratio")), - },{ - "value": "vs_compression_radio_data", - "label": unicode(_("Compression Ratio for Compressible Data")), - },{ - "value": "vs_compression_radio_all", - "label": unicode(_("Compression Ratio for All Data")), - }]) - vs_list.append(data) - return vs_list - -def get_slb_real_list(): - rs_list = [] - p = os.popen("/ca/bin/webui_utils -a 7") - #p = '{"result": 1, "data": [{"service_name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' - rs_dic = json.loads(p.read()) - for each in rs_dic["data"]: - rs_name = each["service_name"] - rs_type = each["protocol"] - data = { - "value": rs_name, - "label": rs_name, - "children": [{ - "value": "rs_cc", - "label": unicode(_("Concurrent Connections")), - },{ - "value": "rs_cps", - "label": unicode(_("CPS")), - },{ - "value": "rs_request", - "label": unicode(_("Outstanding Request Count")), - },{ - "value": "rs_throughput", - "label": unicode(_("Throughput")), - },{ - "value": "rs_pkg_rate", - "label": unicode(_("Package Rate")), - },{ - "value": "rs_response_time", - "label": unicode(_("Average Response time")), - }] - } - if rs_type in ['http', 'https']: - data['children'].append({ - "value": "rs_response_code", - "label": unicode(_("HTTP Response Code Statistics")) - }) - rs_list.append(data) - return rs_list - -def get_ssl_virtual_list(): - vh_list = [] - sess = get_current_session() - sess.cli.set_enable() - result = sess.cli.cmd('show ssl host', - EasyParser('ssl host virtual', ['?name', '?vs_name'])) - for each in result: - vh_name = each["name"] - vh_list.append({ - "value": vh_name, - "label": vh_name, - "children": [{ - "value": 'vh_cc', - "label": unicode(_('Concurrent Connections')) - }, { - "value": 'vh_cps', - "label": unicode(_('CPS')) - }, { - "value": 'vh_throughput', - "label": unicode(_('Throughput')) - }] - }) - vh_list.append({ - "value": "individual_virtual_host", - "label": unicode(_("Individual virtual host")) , - "children": [{ - "value": 'mentioned_statistics', - "label": unicode(_('mentioned statistics')) - }] - }) - return vh_list - -def get_ssl_real_list(): - rh_list = [] - sess = get_current_session() - sess.cli.set_enable() - result = sess.cli.cmd('show ssl host', - EasyParser('ssl host real', ['?name', '?vs_name'])) - for each in result: - rh_name = each['name'] - rh_list.append({ - "value": rh_name, - "label": rh_name, - "children": [{ - "value": 'rh_cc', - "label": unicode(_('Concurrent Connections')) - }, { - "value": 'rh_cps', - "label": unicode(_('CPS')) - }, { - "value": 'rh_throughput', - "label": unicode(_('Throughput')) - }] - }) - return rh_list - -def get_llb_link_route_list(): - link_list = [{ - "value": "total_link", - "label": unicode(_("Total")), - "children": [{ - "value": 'bandwidth_usage', - "label": unicode(_('Bandwidth Usage')), - }, { - "value": 'link_throughput', - "label": unicode(_('Total Link Throughput')), - }, { - "value": 'curr_conn', - "label": unicode(_('Concurrent Connections')), - }, { - "value": 'total_hits', - "label": unicode(_('Total Hits')), - }] - }] - p = os.popen("/ca/bin/webui_utils -a 4") - #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' - llb_dic = json.loads(p.read()) - for each in llb_dic["data"]: - link_name = each["name"] - link_list.append({ - "value": link_name, - "label": link_name, - "children": [{ - "value": 'bandwidth_usage', - "label": unicode(_('Bandwidth Usage')), - }, { - "value": 'link_throughput', - "label": unicode(_('Link Throughput')), - }, { - "value": 'curr_conn', - "label": unicode(_('Concurrent Connections')), - }, { - "value": 'total_hits', - "label": unicode(_('Total Hits')), - }] - }) - return link_list - -def generateItems(request): - item_list = [{ - "value": 'system', - "label": unicode(_('System')), - "children": [{ - "value": 'cpu_usage', - "label": unicode(_('CPU Usage')), - }, { - "value": 'mem_usage', - "label": unicode(_('Memory Usage')), - }, { - "value": 'net_load', - "label": unicode(_('Network Load')), - }, { - "value": 'conn_status', - "label": unicode(_('Connection Status')), - }] - }, { - "value": 'slb', - "label": unicode(_('SLB')), - "children": [{ - "value": 'slb_total', - "label": unicode(_('Global')), - "children": get_slb_total_list() - }, { - "value": 'vs_report', - "label": unicode(_('Virtual Service')), - "children": get_slb_virtual_list() - }, { - "value": 'rs_report', - "label": unicode(_('Real Service')), - "children": get_slb_real_list() - }, { - "value": 'vh_report', - "label": unicode(_('SSL Virtual Host')), - "children": get_ssl_virtual_list() - }, { - "value": 'rh_report', - "label": unicode(_('SSL Real Host')), - "children": get_ssl_real_list() - }] - }, { - "value": 'llb', - "label": unicode(_('LLB')), - "children": [{ - "value": "llb_link", - "label": unicode(_("LLB Link")), - "children": get_llb_link_route_list() - }] - }, { - "value": 'gslb', - "label": unicode(_('GSLB')), - "children": [{ - "value": 'total_hits', - "label": unicode(_('Total Host Hits')), - }] - }] - return HttpResponse(json.dumps(item_list)) +from pyecharts import Bar, Line, Page +from pyecharts.chart import Chart +from pyecharts.engine import create_default_environment +from reportlab.lib import colors +from reportlab.lib.enums import TA_JUSTIFY +from reportlab.lib.pagesizes import letter +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image, TableStyle +from reportlab.platypus.tables import Table +from reportlab.graphics.shapes import Drawing +from reportlab.graphics.shapes import Line as ReportLine +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.lib.units import inch, cm +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from django.utils.translation import ugettext_lazy as _ +from hive.metric import * +from hive.utils import andebug, get_current_session +from hive.model.legacycli import cli_parse, RegexParser, EasyParser, MATCHALL, MATCHONE +from hive.utils import local_equirement +import random, time, csv, codecs +from hive.monitor_log.SSLStatFacade import SSLStatFacade +from tcp_status.statistics import MonitorOutput +from datetime import datetime + +TIMEFORMAT='%Y-%m-%d %X' +chart_path = "/var/crash/statmon/" +report_file_path = "/ca/webui/htdocs/new/src/media/docs/" +font_file_path = "/ca/webui/htdocs/new/src/media/font/Alibaba/" + +current_lan = 'en' + +def generate_report_table_cover(data, col_num): + t = Table(data, colWidths=int(480/col_num), rowHeights=20) + + t.setStyle(TableStyle([('BACKGROUND', (0, 0), (-1, -1), colors.white), + ('ALIGN', (1,1), (-1,-1), 'LEFT'), + ('ALIGN', (0,0), (0, -1), 'RIGHT'), + ('FONT', (0,0), (-1, -1), 'alibaba'), + ])) + t.hAlign = "RIGHT" + return t + +def generate_report_table(data, col_num): + t = Table(data, colWidths=int(480/col_num), rowHeights=20) + data_len = len(data) + + for each in range(data_len): + if each % 2 == 0: + bg_color = colors.lightgrey + else: + bg_color = colors.white + + t.setStyle(TableStyle([('BACKGROUND', (0, each), (-1, each), bg_color), + ('FONT', (0,0), (-1, -1), 'alibaba'),])) + return t + +def get_time_range(from_time, to_time): + time_range = TimeRange(from_time, to_time) + intervalStart = time_range.ParseFrom() + intervalEnd = time_range.ParseTo() + return intervalStart, intervalEnd + +def Draw_a_Line(): + d = Drawing(480, 1) + d.add(ReportLine(0, 0, 480, 0, strokeColor=colors.lightgrey)) + return d + +def Draw_Line_Chart(chart_name, env, styles, story, time_list, chart_item, table_data, yaxis_name=""): + ptext = '%s' % chart_name + story.append(Paragraph(ptext, styles["alibaba"])) + d = Draw_a_Line() + story.append(d) + + table_header = [unicode(_('Time'))] + chart_ins = Line(height=370, is_animation=False) + for each in chart_item: + chart_ins.add(each['item_name'], time_list, each['item_value'], is_smooth=True,is_splitline_show=False, is_toolbox_show=False, yaxis_formatter=yaxis_name) + table_header.append(each['table_item_name']) if "table_item_name" in each else table_header.append(each['item_name']) + + random_value = random.randint(100000, 999999) + report_chart_name = 'report_chart_%s.png' % random_value + env.render_chart_to_file(chart_ins, path=chart_path+report_chart_name) + im = Image(chart_path+report_chart_name, 7*inch, 3*inch) + story.append(im) + + data = [tuple(table_header)] + table_data + t = generate_report_table(data, len(table_header)) + story.append(t) + story.append(Spacer(1, 12)) + + return story + +def Draw_Bar_Chart(chart_name, env, styles, story, xaxis_list, chart_item, table_data, xrotate): + ptext = '%s' % chart_name + story.append(Paragraph(ptext, styles["alibaba"])) + d = Draw_a_Line() + story.append(d) + + table_header = [chart_item['xaxis_name'], chart_item['yaxis_name']] + chart_ins = Bar(height=370, is_animation=False) + chart_ins.add(chart_item['xaxis_name'], xaxis_list, chart_item['item_value'], is_smooth=True,xaxis_interval=0, xaxis_rotate=xrotate,is_splitline_show=False, is_toolbox_show=False) + + random_value = random.randint(100000, 999999) + report_chart_name = 'report_chart_%s.png' % random_value + env.render_chart_to_file(chart_ins, path=chart_path+report_chart_name) + im = Image(chart_path+report_chart_name, 7*inch, 3*inch) + story.append(im) + + data = [tuple(table_header)] + table_data + t = generate_report_table(data, len(table_header)) + story.append(t) + story.append(Spacer(1, 12)) + + return story + +def yaxis_unit(max_value): + if max_value <= 1024: + return "" + if max_value > 1024 and max_value <= 1024*1024: + return "K" + if max_value > 1024*1024 and max_value <= 1024*1024*1024: + return "M" + if max_value > 1024*1024*1024 and max_value <= 1024*1024*1024*1024: + return "G" + if max_value > 1024*1024*1024*1024: + return "T" + +def value_format_1024(list, yaxis_name): + new_list = [] + for each in list: + if each is None: + new_list.append(None) + continue + if yaxis_name == "T": + new_list.append(each/(1024*1024*1024*1024)) + elif yaxis_name == "G": + new_list.append(each/(1024*1024*1024)) + elif yaxis_name == "M": + new_list.append(each/(1024*1024)) + elif yaxis_name == "K": + new_list.append(each/(1024)) + else: + new_list.append(each) + + return new_list + +def value_format_1024_table(list): + new_list = [] + for each in list: + if each is None: + new_list.append(None) + continue + if each > 1024*1024*1024*1024: + new_list.append("%s T"%value_toFixed(each/(1024.0*1024*1024*1024), 2)) + elif each > 1024*1024*1024 and each <= 1024*1024*1024*1024: + new_list.append("%s G"%value_toFixed(each/(1024.0*1024*1024), 2)) + elif each > 1024*1024 and each <= 1024*1024*1024: + new_list.append("%s M"% value_toFixed(each/(1024.0*1024), 2)) + elif each > 1024 and each <= 1024*1024: + new_list.append("%s K"%value_toFixed(each/(1024.0),2)) + else: + new_list.append(value_toFixed(each,2)) + + return new_list + +def value_toFixed(value, number): + if value is None: + return None + else: + if number == 0: + return int(round(float(value))) + return round(float(value), number) + +def inbound_outbound_report(inbound_list, outbound_list): + try: + yaxis_name = yaxis_unit(max(max(inbound_list),max(outbound_list))) + net_in_chart = value_format_1024(inbound_list, yaxis_name) + net_out_chart = value_format_1024(outbound_list, yaxis_name) + net_in_table = value_format_1024_table(inbound_list) + net_out_table = value_format_1024_table(outbound_list) + except ValueError: + yaxis_name = "" + net_in_chart = [] + net_out_chart = [] + net_in_table = [] + net_out_table = [] + + graph_item = [{ + "item_name" : unicode(_("Inbound")), + "table_item_name" : unicode(_("Inbound(bps)")), + "item_value" : net_in_chart + }, { + "item_name" : unicode(_("Outbound")), + "table_item_name" : unicode(_("Outbound(bps)")), + "item_value" : net_out_chart + }] + + return graph_item, net_in_table, net_out_table, yaxis_name + +def package_rate_report(inbound_list, outbound_list): + try: + yaxis_name = yaxis_unit(max(max(inbound_list),max(outbound_list))) + net_in_chart = value_format_1024(inbound_list, yaxis_name) + net_out_chart = value_format_1024(outbound_list, yaxis_name) + net_in_table = value_format_1024_table(inbound_list) + net_out_table = value_format_1024_table(outbound_list) + except ValueError: + yaxis_name = "" + net_in_chart = [] + net_out_chart = [] + net_in_table = [] + net_out_table = [] + graph_item = [{ + "item_name" : unicode(_("Average Package Rate In")), + "table_item_name" : unicode(_("Average Package Rate In(pps)")), + "item_value" : net_in_chart + }, { + "item_name" : unicode(_("Average Package Rate In")), + "table_item_name" : unicode(_("Average Package Rate In(pps)")), + "item_value" : net_out_chart + }] + + return graph_item, net_in_table, net_out_table,yaxis_name + +def get_system_run_status(system_list, system_data_dic): + run_status = "" + if "cpu_usage" in system_list: + if get_run_status(system_data_dic['cpu_usage']): + run_status += unicode(_("The average CPU usage exceeded 80%.")) + if "mem_usage" in system_list: + if get_run_status(system_data_dic['sys_mem']): + run_status += unicode(_("The average system memory usage exceeded 80%.")) + if get_run_status(system_data_dic['net_mem']): + run_status += unicode(_("The average network memory usage exceeded 80%.")) + if get_run_status(system_data_dic['conn_mem']): + run_status += unicode(_("The average connection memory usage exceeded 80%.")) + if "net_load" in system_list: + if check_network_thput(system_data_dic['net_in']): + run_status += unicode(_("The average throughput exceeded 80% of the licensed throughput limit. To ensure the normal delivery of your services, it is recommended to upgrade the license to a higher throughput.")) + return run_status + +def get_run_status(data): + for each in data: + if each and each > 80: + return True + else: + continue + return False + +def check_network_thput(data): + system_data = get_system_version() + if system_data['license_limit'] == None: + return False + value = system_data['license_limit'].split()[0] + unit_name = system_data['license_limit'].split()[1] + + if "M" in unit_name: + for each in data: + if each and each >= int(value) * 1024 * 1024 * 0.8: + return True + else: + continue + if "G" in unit_name: + for each in data: + if each and each >= int(value) * 1024 * 1024 * 1024 * 0.8: + return True + else: + continue + return False + +def get_system_version(): + sess = get_current_session() + sess.cli.set_enable() + cmd_output = sess.cli.cmd('show version') + result = cli_parse(cmd_output, [ + RegexParser('Host name : (.+?)\n', MATCHONE), + RegexParser('System CPU : (.+?)\n', MATCHONE), + RegexParser('System RAM : (.+?)\n', MATCHONE), + RegexParser('System up time : (.+?)\n', MATCHONE), + RegexParser('Model : (.+?)\n', MATCHONE), + RegexParser('Licensed Limits : .*Bandwidth(.*)\n', MATCHONE), + ]) + + return { + "host_name":result[0][0], + "system_cpu":result[1][0], + "system_ram":result[2][0], + "system_boot":result[3][0], + "system_model":result[4][0], + "license_limit":result[5][0].strip("()") if result[5] else None, + } + +def get_pdf_logo(): + if local_equirement(): + logo_path = "/ca/webui/htdocs/new/src/media/img/themes/new/login-logo-cn.png" + else: + logo_path = "/ca/webui/htdocs/new/src/media/img/themes/new/Array-Logo-en-2024.png" + return logo_path + +def get_system_data_sql(from_time, to_time): + time_list = [] + cpu_list = [] + sys_mem_list = [] + net_mem_list = [] + conn_mem_list = [] + cps_list = [] + net_time_list = [] + net_in_list = [] + net_out_list = [] + + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(cpu_usage), avg(system_memory_usage), avg(network_memory_usage), avg(connection_memory_usage), avg(cps) FROM SystemStatus where $__timeFilter(time) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + + select_dic_net = {'rawSql': 'select $__timeGroup(time) as timef, avg(throughput_in_bits), avg(throughput_out_bits) from InterfaceThroughputStatistics where $__timeFilter(time) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data_net = query_metric(select_dic_net) + data = select_data['data'] + data_net = select_data_net['data'] + + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cpu_list.append(value_toFixed(each[1], 2)) + sys_mem_list.append(value_toFixed(each[2], 0)) + net_mem_list.append(value_toFixed(each[3], 0)) + conn_mem_list.append(value_toFixed(each[4],0)) + cps_list.append(value_toFixed(each[5], 0)) + for each in data_net: + net_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + net_in_list.append(each[1]) + net_out_list.append(each[2]) + + return { + 'time': time_list, + 'net_time': net_time_list, + 'cpu_usage': cpu_list, + 'sys_mem': sys_mem_list, + 'net_mem': net_mem_list, + 'conn_mem': conn_mem_list, + 'cps': cps_list, + 'net_in': net_in_list, + 'net_out': net_out_list, + } + +def get_rs_data_sql(from_time, to_time, rs_name): + time_list = [] + cc_list = [] + cps_list = [] + request_list = [] + inbound_list = [] + outbound_list = [] + average_in_list = [] + average_out_list = [] + average_response_list = [] + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_response_time), max(connection_count),max(reqcount) from GlobalRealServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %rs_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + + errcode_dic = {'rawSql': 'select INCREMENT(resp_100), INCREMENT(resp_101), INCREMENT(resp_200), INCREMENT(resp_206), INCREMENT(resp_301), INCREMENT(resp_302), INCREMENT(resp_303), INCREMENT(resp_304), INCREMENT(resp_305), INCREMENT(resp_307), INCREMENT(resp_400), INCREMENT(resp_401), INCREMENT(resp_402), INCREMENT(resp_403), INCREMENT(resp_404), INCREMENT(resp_405), INCREMENT(resp_406), INCREMENT(resp_407), INCREMENT(resp_416), INCREMENT(resp_500), INCREMENT(resp_501), INCREMENT(resp_502), INCREMENT(resp_503), INCREMENT(resp_504), INCREMENT(resp_505), INCREMENT(resp_others) from RealServiceHTTPCodeStatistics WHERE $__timeFilter(time) and service_name = "%s" group by service_name;' %rs_name, + 'to': to_time, 'from': from_time, 'format': 'table'} + errcode_data = query_metric(errcode_dic) + + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cc_list.append(value_toFixed(each[3], 0)) + cps_list.append(value_toFixed(each[7], 0)) + request_list.append(value_toFixed(each[8], 0)) + inbound_list.append(each[1]) + outbound_list.append(each[2]) + average_in_list.append(each[4]) + average_out_list.append(each[5]) + average_response_list.append(value_toFixed(each[6],2)) + + return { + 'time': time_list, + 'cc_list': cc_list, + 'cps_list': cps_list, + 'request_list': request_list, + 'inbound_list': inbound_list, + 'outbound_list': outbound_list, + 'average_in_list': average_in_list, + 'average_out_list': average_out_list, + 'average_response_list': average_response_list, + 'err_list' : errcode_data['data'] + } + +def get_link_data_sql(from_time, to_time, link_name): + time_list = [] + cc_list = [] + bandwidth_list = [] + inbound_list = [] + outbound_list = [] + hit_list = [] + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(thresh), max(current_connection), max(hits), avg(usage) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name = "%s" GROUP BY timef;'%link_name, + 'to': to_time, 'from': from_time, 'index': 1, 'format': 'time_series'} + select_data = query_metric(select_dic) + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cc_list.append(value_toFixed(each[4],0)) + bandwidth_list.append(value_toFixed(each[6],2)) + inbound_list.append(each[1]) + outbound_list.append(each[2]) + hit_list.append(value_toFixed(each[5],0)) + + return { + 'time': time_list, + 'cc_list': cc_list, + 'inbound_list': inbound_list, + 'outbound_list': outbound_list, + 'hit_list': hit_list, + 'bandwidth_list': bandwidth_list + } + +def get_llb_total_sql(from_time, to_time, link_list): + time_list = [] + + map_dict ={} + for each in link_list: + map_dict[each] = { + 'time': [], + 'cc_list': [], + 'through_list': [], + 'hit_list': [], + 'bandwidth_list': [] + } + select_dic = {'rawSql': "select $__timeGroup(time) as timef, name, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(usage), max(current_connection), max(hits), max(status) from ExLLBLinkStatistics where $__timeFilter(timef/1000) and name in ('%s') GROUP BY timef, name;" % ("','".join(link_list)), + 'to': to_time, 'from': from_time, 'index': 1, 'format': 'base_time_series'} + select_data = query_metric(select_dic) + data = select_data['data'] + + for each in data: + time_stamp = time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000)) + if time_stamp not in time_list: + time_list.append(time_stamp) + map_dict[each[1]]["cc_list"].append(value_toFixed(each[5],0)) + map_dict[each[1]]["bandwidth_list"].append(value_toFixed(each[4],2)) + map_dict[each[1]]["through_list"].append(each[2]+each[3]) + map_dict[each[1]]["hit_list"].append(value_toFixed(each[6],0)) + + return map_dict, time_list + +def get_vs_data_sql(from_time, to_time, vs_name): + time_list = [] + time_cache_list = [] + time_comp_list = [] + cc_list = [] + cps_list = [] + inbound_list = [] + outbound_list = [] + average_in_list = [] + average_out_list = [] + average_rtt_list = [] + cache_list = [] + compress_data = [] + compress_all = [] + err_list = [] + + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), avg(average_packets_in), avg(average_packets_out), avg(average_rtt), max(connection_count), avg(average_request_time), avg(average_response_time) from GlobalVirtualServiceStats where $__timeFilter(timef/1000) and service_name = "%s" GROUP BY timef;' %vs_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cc_list.append(value_toFixed(each[3],0)) + cps_list.append(value_toFixed(each[7],0)) + inbound_list.append(each[1]) + outbound_list.append(each[2]) + average_in_list.append(each[4]) + average_out_list.append(each[5]) + average_rtt_list.append(value_toFixed(each[6],2)) + + select_dic_cache = {'rawSql': "select $__timeGroup(time) as timef, avg(cacheHitRatio) from PervsCacheStatistics where $__timeFilter(timef/1000) and pk = '{\"service_name\": \"%s\"}' GROUP BY timef;" %vs_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data_cache = query_metric(select_dic_cache) + + for each in select_data_cache['data']: + time_cache_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cache_list.append(value_toFixed(each[1], 2)) + + select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(compRatioComp), avg(compRatioAll) from PervsCompressStatistics where $__timeFilter(timef/1000) and service_name = '%s' GROUP BY timef" %vs_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + + + for each in select_data['data']: + time_comp_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + compress_data.append(value_toFixed(each[1],0)) + compress_all.append(value_toFixed(each[2],0)) + + errcode_dic = {'rawSql': 'select INCREMENT(resp_100), INCREMENT(resp_101), INCREMENT(resp_200), INCREMENT(resp_206), INCREMENT(resp_301), INCREMENT(resp_302), INCREMENT(resp_303), INCREMENT(resp_304), INCREMENT(resp_307), INCREMENT(resp_400), INCREMENT(resp_401), INCREMENT(resp_402), INCREMENT(resp_403), INCREMENT(resp_404), INCREMENT(resp_405), INCREMENT(resp_406), INCREMENT(resp_407), INCREMENT(resp_416), INCREMENT(resp_500), INCREMENT(resp_501), INCREMENT(resp_502), INCREMENT(resp_503), INCREMENT(resp_504), INCREMENT(resp_505), INCREMENT(resp_901), INCREMENT(resp_902), INCREMENT(resp_903), INCREMENT(resp_904), INCREMENT(resp_905), INCREMENT(resp_906), INCREMENT(resp_others) from VirtualServiceHTTPCodeStatistics WHERE $__timeFilter(time) and service_name = "%s" group by service_name;' %vs_name, + 'to': to_time, 'from': from_time, 'format': 'table'} + errcode_data = query_metric(errcode_dic) + err_list = errcode_data['data'] + + return { + "time": time_list, + "time_cache_list":time_cache_list, + "time_comp_list":time_comp_list, + "cc_list": cc_list, + "cps_list": cps_list, + "inbound_list": inbound_list, + "outbound_list": outbound_list, + "average_in_list": average_in_list, + "average_out_list": average_out_list, + "average_rtt_list": average_rtt_list, + "cache_list": cache_list, + "compress_data": compress_data, + "compress_all": compress_all, + "err_list": err_list, + } + +def get_ssl_vh_sql(from_time, to_time, vhost_name): + time_list = [] + cc_list = [] + cps_list = [] + net_in_list = [] + net_out_list = [] + + select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), max(connection_count) from ExSSLVirtualHostStatistics where $__timeFilter(timef/1000) and name = '%s' GROUP BY timef;"% vhost_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + net_in_list.append(each[1]) + net_out_list.append(each[2]) + cc_list.append(value_toFixed(each[3],0)) + cps_list.append(value_toFixed(each[4],0)) + + return { + "time": time_list, + "cc_list": cc_list, + "cps_list": cps_list, + "net_in_list": net_in_list, + "net_out_list": net_out_list, + } + +def get_ssl_rh_sql(from_time, to_time, rhost_name): + time_list = [] + cc_list = [] + cps_list = [] + net_in_list = [] + net_out_list = [] + + select_dic = {'rawSql': "select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), max(current_connection_count), max(connection_count) from ExSSLRealHostStatistics where $__timeFilter(timef/1000) and name = '%s' GROUP BY timef;"%rhost_name, + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + net_in_list.append(each[1]) + net_out_list.append(each[2]) + cc_list.append(value_toFixed(each[3],0)) + cps_list.append(value_toFixed(each[4],0)) + + return { + "time": time_list, + "cc_list": cc_list, + "cps_list": cps_list, + "net_in_list": net_in_list, + "net_out_list": net_out_list, + } +def get_slb_total_sql(from_time, to_time): + time_list = [] + compress_list = [] + cache_time_list = [] + ssl_time_list = [] + slb_time_list = [] + cache_list = [] + ssl_in_list = [] + ssl_out_list = [] + vhost_in_list = [] + vhost_out_list = [] + rhost_in_list = [] + rhost_out_list = [] + slb_in_list = [] + slb_out_list = [] + slb_cc_list = [] + slb_cps_list = [] + + compress_sql = {'rawSql': 'select $__timeGroup(time) as timef, avg(compRatioAll) from CompressionGlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + compress_data = query_metric(compress_sql) + cache_sql = {'rawSql': 'Select $__timeGroup(time) as timef, avg(cacheHitRatio) from CacheGlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + cache_data = query_metric(cache_sql) + ssl_sql = {'rawSql': 'Select $__timeGroup(time) as timef, avg(average_bandwidth_in), avg(average_bandwidth_out), avg(average_bandwidth_in_vhost), avg(average_bandwidth_out_vhost), avg(average_bandwidth_in_rhost), avg(average_bandwidth_out_rhost) from GlobalSSLThroughStats where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + ssl_data = query_metric(ssl_sql) + slb_sql = {'rawSql': 'select $__timeGroup(time) as timef, avg(vsThroughputByteIn), avg(vsThroughputByteOut), max(vsTotalConnections), max(vsTotalConnectionsAll), max(clientConnectionPerSec) from GlobalStatistics where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + slb_data = query_metric(slb_sql) + + for each in compress_data['data']: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + compress_list.append(value_toFixed(each[1],2)) + for each in cache_data['data']: + cache_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + cache_list.append(value_toFixed(each[1],2)) + for each in ssl_data['data']: + ssl_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + ssl_in_list.append(each[1]) + ssl_out_list.append(each[2]) + vhost_in_list.append(each[3]) + vhost_out_list.append(each[4]) + rhost_in_list.append(each[5]) + rhost_out_list.append(each[6]) + for each in slb_data['data']: + slb_time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + slb_in_list.append(each[1]) + slb_out_list.append(each[2]) + slb_cc_list.append(value_toFixed(each[3], 0)) + slb_cps_list.append(value_toFixed(each[5], 0)) + + return { + "time": time_list, + "compress_list": compress_list, + "cache_time_list": cache_time_list, + "ssl_time_list": ssl_time_list, + "slb_time_list": slb_time_list, + "cache_list": cache_list, + "ssl_in_list": ssl_in_list, + "ssl_out_list": ssl_out_list, + "vhost_in_list": vhost_in_list, + "vhost_out_list": vhost_out_list, + "rhost_in_list": rhost_in_list, + "rhost_out_list": rhost_out_list, + "slb_in_list": slb_in_list, + "slb_out_list": slb_out_list, + "slb_cc_list": slb_cc_list, + "slb_cps_list": slb_cps_list, + } +def generateSystemReport(from_time, to_time, system_list, env, styles, story): + system_data_dic = get_system_data_sql(from_time, to_time) + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("System Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + if "cpu_usage" in system_list: + graph_item = [{ + "item_name" : unicode(_("CPU Usage")), + "table_item_name" : unicode(_("CPU Usage(%)")), + "item_value" : system_data_dic['cpu_usage'] + }] + table_data = zip(system_data_dic['time'], system_data_dic['cpu_usage']) + story = Draw_Line_Chart(unicode(_("CPU Usage(%)")), env, styles, story, system_data_dic['time'], graph_item, table_data) + + if "mem_usage" in system_list: + graph_item = [{ + "item_name" : unicode(_("System Memory")), + "item_value" : system_data_dic['sys_mem'] + }, { + "item_name" : unicode(_("Network Memory")), + "item_value" : system_data_dic['net_mem'] + }, { + "item_name" : unicode(_("Connection Memory")), + "item_value" : system_data_dic['conn_mem'] + }] + table_data = zip(system_data_dic['time'], system_data_dic['sys_mem'], system_data_dic['net_mem'], system_data_dic['conn_mem']) + story = Draw_Line_Chart(unicode(_("Memory Usage (%)")), env, styles, story, system_data_dic['time'], graph_item, table_data) + + if "net_load" in system_list: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(system_data_dic['net_in'], system_data_dic['net_out']) + + table_data = zip(system_data_dic['net_time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Network Load(bps)")), env, styles, story, system_data_dic['net_time'], graph_item, table_data, yaxis_name) + + if "conn_status" in system_list: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : system_data_dic['cps'] + }] + table_data = zip(system_data_dic['time'], system_data_dic['cps']) + story = Draw_Line_Chart(unicode(_("Connection Status")), env, styles, story, system_data_dic['time'], graph_item, table_data) + + run_status = get_system_run_status(system_list, system_data_dic) + intervalStart, intervalEnd = get_time_range(from_time, to_time) + end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalEnd)) + start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalStart)) + if 'cpu_usage' in system_list or 'mem_usage' in system_list or 'net_load' in system_list: + ptext = '' + unicode(_("Conclusion")) +'' + story.append(Paragraph(ptext, styles["alibaba"])) + d = Draw_a_Line() + story.append(d) + ptext = ''+ unicode(_('During the period from %s to %s, ')) % (start_time, end_time) + if not run_status: + ptext += unicode(_('the system was running in a healthy status.'))+'' + else: + ptext += unicode(_('the system encountered the following abnormal conditions:'))+'' + ptext += run_status + story.append(Paragraph(ptext, styles["alibaba"])) + + return story + +def generateSLBVSReport(from_time, to_time, slb_vs_list, env, styles, story, tcp_status): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("SLB Virtual Service Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + for each_vs in slb_vs_list: + vs_data_dic = get_vs_data_sql(from_time, to_time, each_vs) + + if 'vs_cc' in slb_vs_list[each_vs]: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : vs_data_dic['cc_list'] + }] + table_data = zip(vs_data_dic['time'], vs_data_dic['cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) + + if 'vs_cps' in slb_vs_list[each_vs]: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : vs_data_dic['cps_list'] + }] + table_data = zip(vs_data_dic['time'], vs_data_dic['cps_list']) + story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) + + if 'vs_throughput' in slb_vs_list[each_vs]: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(vs_data_dic['inbound_list'], vs_data_dic['outbound_list']) + + table_data = zip(vs_data_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Throughput(bps)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data, yaxis_name) + + if 'vs_pkg_rate' in slb_vs_list[each_vs]: + graph_item, net_in_table, net_out_table, yaxis_name = package_rate_report(vs_data_dic['average_in_list'], vs_data_dic['average_out_list']) + + table_data = zip(vs_data_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Package Rate(pps)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data, yaxis_name) + + if 'vs_rtt' in slb_vs_list[each_vs]: + graph_item = [{ + "item_name" : unicode(_("Average Client Connection RTT")), + "item_value" : vs_data_dic['average_rtt_list'] + }] + table_data = zip(vs_data_dic['time'], vs_data_dic['average_rtt_list']) + story = Draw_Line_Chart(unicode(_("Average Client Connection RTT"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time'], graph_item, table_data) + + if 'vs_cache_hit_radio' in slb_vs_list[each_vs]: + graph_item = graph_item = [{ + "item_name" : unicode(_("Cache Hit Ratio")), + "item_value" : vs_data_dic['cache_list'] + }] + table_data = zip(vs_data_dic['time_cache_list'], vs_data_dic['cache_list']) + story = Draw_Line_Chart(unicode(_("Cache Hit Ratio(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_cache_list'], graph_item, table_data) + + if 'vs_compression_radio_data' in slb_vs_list[each_vs]: + graph_item = graph_item = [{ + "item_name" : unicode(_("Compression Ratio for Compressible Data")), + "item_value" : vs_data_dic['compress_data'] + }] + table_data = zip(vs_data_dic['time_comp_list'], vs_data_dic['compress_data']) + story = Draw_Line_Chart(unicode(_("Compression Ratio for Compressible Data(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_comp_list'], graph_item, table_data) + if 'vs_compression_radio_all' in slb_vs_list[each_vs]: + graph_item = graph_item = [{ + "item_name" : unicode(_("Compression Ratio for All Data")), + "item_value" : vs_data_dic['compress_all'] + }] + table_data = zip(vs_data_dic['time_comp_list'], vs_data_dic['compress_all']) + story = Draw_Line_Chart(unicode(_("Compression Ratio for All Data(%)"))+"(%s)"%each_vs, env, styles, story, vs_data_dic['time_comp_list'], graph_item, table_data) + if "vs_response_code" in slb_vs_list[each_vs]: + err_data = vs_data_dic["err_list"][0] if vs_data_dic["err_list"] else [0]*31 + errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 901, 902, 903, 904, 905, 906, 'others'] + graph_item = { + "xaxis_name" : unicode(_("Response Code")), + "yaxis_name" : unicode(_("Hits")), + "item_value" : err_data + } + table_data = zip(errcode_list, err_data) + story = Draw_Bar_Chart(unicode(_("HTTP Response Code Statistics"))+"(%s)"%each_vs, env, styles, story, errcode_list, graph_item, table_data, 50) + + if 'vs_tcp' in slb_vs_list[each_vs] and slb_vs_list[each_vs]['vs_tcp'] and each_vs in tcp_status: + mo = MonitorOutput() + for column in mo.convert_list_to_column(slb_vs_list[each_vs]['vs_tcp']): + # ex: LISTEN + item_value = [] + table_data = [] + time_list = [] + for each in tcp_status[each_vs]: + # each = ["2024-06-28 15:42:47", {"LISTEN": "0"}] + date = each[0] + value = each[1][column] + item_value.append(value) # ex: 2 + table_data.append([date, value]) # ex: ["2024-06-27 16:12:00", 2] + time_list.append(date) # ex: "2024-06-27 16:12:00" + graph_item = [{ + "item_name" : unicode(_(column)), # ex: LISTEN + "item_value" : item_value + }] + story = Draw_Line_Chart(unicode(_(column))+"(%s)"%each_vs, env, styles, story, time_list, graph_item, table_data) + + return story + +def generateSLBRSReport(from_time, to_time, slb_rs_list, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("SLB Real Service Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + for each_rs in slb_rs_list: + rs_data_dic = get_rs_data_sql(from_time, to_time, each_rs) + if 'rs_cc' in slb_rs_list[each_rs]: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : rs_data_dic['cc_list'] + }] + table_data = zip(rs_data_dic['time'], rs_data_dic['cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) + + if 'rs_cps' in slb_rs_list[each_rs]: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : rs_data_dic['cps_list'] + }] + table_data = zip(rs_data_dic['time'], rs_data_dic['cps_list']) + story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) + + if 'rs_request' in slb_rs_list[each_rs]: + graph_item = [{ + "item_name" : unicode(_("Outstanding Request Count")), + "item_value" : rs_data_dic['request_list'] + }] + table_data = zip(rs_data_dic['time'], rs_data_dic['request_list']) + story = Draw_Line_Chart(unicode(_("Outstanding Request Count"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) + + if 'rs_throughput' in slb_rs_list[each_rs]: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(rs_data_dic['inbound_list'], rs_data_dic['outbound_list']) + + table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Throughput(bps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) + + if 'rs_pkg_rate' in slb_rs_list[each_rs]: + graph_item, net_in_table, net_out_table, yaxis_name = package_rate_report(rs_data_dic['average_in_list'], rs_data_dic['average_out_list']) + + table_data = zip(rs_data_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Package Rate(pps)"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data, yaxis_name) + + if 'rs_response_time' in slb_rs_list[each_rs]: + graph_item = [{ + "item_name" : unicode(_("Average Response Time")), + "item_value" : rs_data_dic['average_response_list'] + }] + table_data = zip(rs_data_dic['time'], rs_data_dic['average_response_list']) + story = Draw_Line_Chart(unicode(_("Average Response Time"))+"(%s)"%each_rs, env, styles, story, rs_data_dic['time'], graph_item, table_data) + + if 'rs_response_code' in slb_rs_list[each_rs]: + errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 305, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 'others'] + err_data = rs_data_dic['err_list'][0] if rs_data_dic['err_list'] else [0]*26 + graph_item = { + "xaxis_name" : unicode(_("Response Code")), + "yaxis_name" : unicode(_("Hits")), + "item_value" : err_data + } + table_data = zip(errcode_list, err_data) + story = Draw_Bar_Chart(unicode(_("HTTP Response Code Statistics"))+"(%s)"%each_rs, env, styles, story, errcode_list, graph_item, table_data, 30) + + return story + +def generateSSLVHostReport(from_time, to_time, slb_vh_dic, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("SSL Virtual Host Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + for each_vh in slb_vh_dic: + ssl_vhost_dic = get_ssl_vh_sql(from_time, to_time,each_vh) + if "vh_cc" in slb_vh_dic[each_vh]: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : ssl_vhost_dic['cc_list'] + }] + table_data = zip(ssl_vhost_dic['time'], ssl_vhost_dic['cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data) + + if "vh_cps" in slb_vh_dic[each_vh]: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : ssl_vhost_dic['cps_list'] + }] + table_data = zip(ssl_vhost_dic['time'], ssl_vhost_dic['cps_list']) + story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data) + + if "vh_throughput" in slb_vh_dic[each_vh]: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(ssl_vhost_dic['net_in_list'], ssl_vhost_dic['net_out_list']) + + table_data = zip(ssl_vhost_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Throughput"))+"(%s)"%each_vh, env, styles, story, ssl_vhost_dic['time'], graph_item, table_data, yaxis_name) + if "mentioned_statistics" in slb_vh_dic[each_vh]: + sslStat = SSLStatFacade(from_time, gz_directory="/var/crash") + ssl_data = sslStat.get_data() + time_list = [e[0] for e in ssl_data] + connection_attempts_list = [e[1] for e in ssl_data] + successful_handshakes_data = [e[2] for e in ssl_data] + active_connections_data = [e[3] for e in ssl_data] + graph_item = [{ + "item_name" : unicode(_("connection attempts")), + "table_item_name" : unicode(_("connection attempts")), + "item_value" : connection_attempts_list + }, + { + "item_name" : unicode(_("successful handshakes")), + "table_item_name" : unicode(_("successful handshakes")), + "item_value" : successful_handshakes_data + }, + { + "item_name" : unicode(_("active connections")), + "table_item_name" : unicode(_("active connections")), + "item_value" : active_connections_data + } + ] + table_data = zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data) + story = Draw_Line_Chart(unicode(_("total ssl statistics")), env, styles, story, time_list, graph_item, table_data) + return story + +def generateSSLRHostReport(from_time, to_time, slb_rh_dic, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("SSL Real Host Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + for each_rh in slb_rh_dic: + ssl_rhost_dic = get_ssl_rh_sql(from_time, to_time,each_rh) + if "rh_cc" in slb_rh_dic[each_rh]: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : ssl_rhost_dic['cc_list'] + }] + table_data = zip(ssl_rhost_dic['time'], ssl_rhost_dic['cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data) + + if "rh_cps" in slb_rh_dic[each_rh]: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : ssl_rhost_dic['cps_list'] + }] + table_data = zip(ssl_rhost_dic['time'], ssl_rhost_dic['cps_list']) + story = Draw_Line_Chart(unicode(_("CPS"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data) + + if "rh_throughput" in slb_rh_dic[each_rh]: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(ssl_rhost_dic['net_in_list'], ssl_rhost_dic['net_out_list']) + + table_data = zip(ssl_rhost_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Throughput"))+"(%s)"%each_rh, env, styles, story, ssl_rhost_dic['time'], graph_item, table_data, yaxis_name) + + return story + +def generateSLBTotalReport(from_time, to_time, slb_total_list, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("SLB Global Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + slb_total_dic = get_slb_total_sql(from_time, to_time) + if "total_throughput" in slb_total_list: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['slb_in_list'], slb_total_dic['slb_out_list']) + + table_data = zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("SLB Throughput(bps)")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data, yaxis_name) + + if "curr_conn" in slb_total_list: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : slb_total_dic['slb_cc_list'] + }] + table_data = zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data) + + if "cps" in slb_total_list: + graph_item = [{ + "item_name" : unicode(_("CPS")), + "item_value" : slb_total_dic['slb_cps_list'] + }] + table_data = zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cps_list']) + story = Draw_Line_Chart(unicode(_("CPS")), env, styles, story, slb_total_dic['slb_time_list'], graph_item, table_data) + + if "ssl_throughput" in slb_total_list: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['ssl_in_list'], slb_total_dic['ssl_out_list']) + + table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("SSL Total Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) + + if "ssl_vh_throughput" in slb_total_list: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['vhost_in_list'], slb_total_dic['vhost_out_list']) + + table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("SSL Virtual Host Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) + + if "ssl_rh_throughput" in slb_total_list: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(slb_total_dic['rhost_in_list'], slb_total_dic['rhost_out_list']) + + table_data = zip(slb_total_dic['ssl_time_list'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("SSL Real Host Throughput(bps)")), env, styles, story, slb_total_dic['ssl_time_list'], graph_item, table_data, yaxis_name) + + if 'compress_ratio' in slb_total_list: + graph_item = [{ + "item_name" : unicode(_("Compression Ratio for All Data")), + "item_value" : slb_total_dic['compress_list'] + }] + table_data = zip(slb_total_dic['time'], slb_total_dic['compress_list']) + story = Draw_Line_Chart(unicode(_("Compression Ratio for All Data(%)")), env, styles, story, slb_total_dic['time'], graph_item, table_data) + + if 'cache_hit_radio' in slb_total_list: + graph_item = [{ + "item_name" : unicode(_("Cache Hits Ratio")), + "item_value" : slb_total_dic['cache_list'] + }] + table_data = zip(slb_total_dic['cache_time_list'], slb_total_dic['cache_list']) + story = Draw_Line_Chart(unicode(_("Cache Hit Ratio(%)")), env, styles, story, slb_total_dic['cache_time_list'], graph_item, table_data) + + return story + +def generateLLBTotalReport(from_time, to_time, llb_total_list, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("LLB Total Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + p = os.popen("/ca/bin/webui_utils -a 4") + #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' + llb_dic = json.loads(p.read()) + link_list = [] + for each in llb_dic["data"]: + link_list.append(each["name"]) + link_list = sorted(link_list) + + total_dic, time_list = get_llb_total_sql(from_time, to_time, link_list) + if "bandwidth_usage" in llb_total_list: + graph_item = [] + zip_list = [time_list] + for each in link_list: + graph_item.append({ + "item_name" : each, + "item_value" : total_dic[each]['bandwidth_list'] + }) + zip_list.append(total_dic[each]['bandwidth_list']) + table_data = zip(*zip_list) + story = Draw_Line_Chart(unicode(_("Bandwidth Usage")), env, styles, story, time_list, graph_item, table_data) + + if "link_throughput" in llb_total_list: + graph_item = [] + zip_list = [time_list] + max_list = [] + for each in link_list: + max_list.append(max(total_dic[each]['through_list'])) + yaxis_name = yaxis_unit(max(max_list)) + for each in link_list: + graph_item.append({ + "item_name" : each, + "item_value" : value_format_1024(total_dic[each]['through_list'],yaxis_name) + }) + zip_list.append(value_format_1024_table(total_dic[each]['through_list'])) + table_data = zip(*zip_list) + story = Draw_Line_Chart(unicode(_("Total Link Throughput(bps)")), env, styles, story, time_list, graph_item, table_data, yaxis_name) + + if "curr_conn" in llb_total_list: + graph_item = [] + zip_list = [time_list] + for each in link_list: + graph_item.append({ + "item_name" : each, + "item_value" : total_dic[each]['cc_list'] + }) + zip_list.append(total_dic[each]['cc_list']) + table_data = zip(*zip_list) + story = Draw_Line_Chart(unicode(_("Concurrent Connections")), env, styles, story, time_list, graph_item, table_data) + + if "total_hits" in llb_total_list: + graph_item = [] + zip_list = [time_list] + for each in link_list: + graph_item.append({ + "item_name" : each, + "item_value" : total_dic[each]['hit_list'] + }) + zip_list.append(total_dic[each]['hit_list']) + table_data = zip(*zip_list) + story = Draw_Line_Chart(unicode(_("Total Hits")), env, styles, story, time_list, graph_item, table_data) + + return story + +def generateLLBReport(from_time, to_time, llb_list, env, styles, story): + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("LLB Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + for each_link in llb_list: + link_data_dic = get_link_data_sql(from_time, to_time, each_link) + if 'bandwidth_usage' in llb_list[each_link]: + graph_item = [{ + "item_name" : unicode(_("Bandwidth Usage")), + "item_value" : link_data_dic['bandwidth_list'] + }] + table_data = zip(link_data_dic['time'], link_data_dic['bandwidth_list']) + story = Draw_Line_Chart(unicode(_("Bandwidth Usage"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) + + if 'link_throughput' in llb_list[each_link]: + graph_item, net_in_table, net_out_table, yaxis_name = inbound_outbound_report(link_data_dic['inbound_list'], link_data_dic['outbound_list']) + + table_data = zip(link_data_dic['time'], net_in_table, net_out_table) + story = Draw_Line_Chart(unicode(_("Link Throughput(bps)"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data, yaxis_name) + + if 'curr_conn' in llb_list[each_link]: + graph_item = [{ + "item_name" : unicode(_("Concurrent Connections")), + "item_value" : link_data_dic['cc_list'] + }] + table_data = zip(link_data_dic['time'], link_data_dic['cc_list']) + story = Draw_Line_Chart(unicode(_("Concurrent Connections"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) + + if 'total_hits' in llb_list[each_link]: + graph_item = [{ + "item_name" : unicode(_("Hits")), + "item_value" : link_data_dic['hit_list'] + }] + table_data = zip(link_data_dic['time'], link_data_dic['hit_list']) + story = Draw_Line_Chart(unicode(_("Hits"))+"(%s)"%each_link, env, styles, story, link_data_dic['time'], graph_item, table_data) + + return story + +def generateGSLBReport(from_time, to_time, gslb_list, env, styles, story): + story.append(Spacer(1, 12)) + ptext = ''+ unicode(_("GSLB Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + time_list = [] + hit_list = [] + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, max(total_req) from QueryStatistics where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + data = select_data['data'] + for each in data: + time_list.append(time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000))) + hit_list.append(value_toFixed(each[1],0)) + + graph_item = [{ + "item_name" : unicode(_("Total Hits")), + "item_value" : hit_list + }] + table_data = zip(time_list, hit_list) + story = Draw_Line_Chart(unicode(_("Total Hits")), env, styles, story, time_list, graph_item, table_data) + + return story + +def generateTCPReport(tcp_status, tcp_list, env, styles, story): + """ + Get PDF data of tcp status and filter columns + :param dict tcp_status: statistics of tcp status + :param list tcp_list: needed columns + :param object env + :param object styles + :param object story: pdf object + """ + mo = MonitorOutput() + story.append(Spacer(1, 12)) + ptext = '' + unicode(_("TCP Global Status")) + '' + story.append(Paragraph(ptext, styles["alibaba"])) + story.append(Spacer(1, 12)) + + if not MonitorOutput.C_GLOBAL_INDEX in tcp_status: + return story + + for column in mo.convert_list_to_column(tcp_list): + # ex: LISTEN + item_value = [] + table_data = [] + time_list = [] + for each in tcp_status[MonitorOutput.C_GLOBAL_INDEX]: + # each = ["2024-06-28 15:42:47", {"LISTEN": "0"}] + date = each[0] + value = each[1][column] + item_value.append(value) # ex: 2 + table_data.append([date, value]) # ex: ["2024-06-27 16:12:00", 2] + time_list.append(date) # ex: "2024-06-27 16:12:00" + graph_item = [{ + "item_name" : unicode(_(column)), # ex: LISTEN + "item_value" : item_value + }] + story = Draw_Line_Chart(unicode(_(column)), env, styles, story, time_list, graph_item, table_data) + return story +def generateSystemCSV(cw, from_time, to_time, end_time, start_time, system_list): + cw.writerow("") + cw.writerow([unicode(_('System Status'))]) + system_data_dic = get_system_data_sql(from_time, to_time) + + if 'cpu_usage' in system_list: + cw.writerow([unicode(_('CPU Usage(%)'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPU Usage'))]) + cw.writerows(zip(system_data_dic['time'], system_data_dic['cpu_usage'])) + if 'mem_usage' in system_list: + cw.writerow([unicode(_('Memory Usage (%)'))]) + cw.writerow([unicode(_('Time')), unicode(_('System Memory Usage')), unicode(_('Network Memory Usage')), unicode(_('Connection Memory Usage'))]) + cw.writerows(zip(system_data_dic['time'], system_data_dic['sys_mem'], system_data_dic['net_mem'], system_data_dic['conn_mem'])) + if 'conn_status' in system_list: + cw.writerow([unicode(_('Connection Status'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(system_data_dic['time'], system_data_dic['cps'])) + if 'net_load' in system_list: + net_in_table = value_format_1024_table(system_data_dic['net_in']) + net_out_table = value_format_1024_table(system_data_dic['net_out']) + cw.writerow([unicode(_('Network Load'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(system_data_dic['time'], net_in_table, net_out_table)) + run_status = get_system_run_status(system_list, system_data_dic) + if 'cpu_usage' in system_list or 'mem_usage' in system_list or 'net_load' in system_list: + cw.writerow([unicode(_("Conclusion"))]) + ptext = unicode(_('During the period from %s to %s, ')) % (start_time, end_time) + if not run_status: + ptext += unicode(_('the system was running in a healthy status.')) + else: + ptext += unicode(_('the system encountered the following abnormal conditions:')) + ptext += run_status + cw.writerow([ptext]) + return cw + +def generateSLBTotalCSV(cw, from_time, to_time, slb_total_list): + cw.writerow([unicode(_("SLB Global Status"))]) + slb_total_dic = get_slb_total_sql(from_time, to_time) + if 'total_throughput' in slb_total_list: + net_in_table = value_format_1024_table(slb_total_dic['slb_in_list']) + net_out_table = value_format_1024_table(slb_total_dic['slb_out_list']) + cw.writerow([unicode(_('SLB Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) + if 'curr_conn' in slb_total_list: + cw.writerow([unicode(_('Concurrent Connections'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cc_list'])) + if 'cps' in slb_total_list: + cw.writerow([unicode(_('CPS'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], slb_total_dic['slb_cps_list'])) + if 'ssl_throughput' in slb_total_list: + net_in_table = value_format_1024_table(slb_total_dic['ssl_in_list']) + net_out_table = value_format_1024_table(slb_total_dic['ssl_out_list']) + cw.writerow([unicode(_('SSL Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) + if 'ssl_vh_throughput' in slb_total_list: + net_in_table = value_format_1024_table(slb_total_dic['vhost_in_list']) + net_out_table = value_format_1024_table(slb_total_dic['vhost_out_list']) + cw.writerow([unicode(_('SSL Virtual Host Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) + if 'ssl_rh_throughput' in slb_total_list: + net_in_table = value_format_1024_table(slb_total_dic['rhost_in_list']) + net_out_table = value_format_1024_table(slb_total_dic['rhost_out_list']) + cw.writerow([unicode(_('SSL Real Host Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(slb_total_dic['slb_time_list'], net_in_table, net_out_table)) + if 'cache_hit_radio' in slb_total_list: + cw.writerow([unicode(_('Cache Hit Ratio'))]) + cw.writerow([unicode(_('Time')), unicode(_('Cache Hit Ratio(%)'))]) + cw.writerows(zip(slb_total_dic['cache_time_list'], slb_total_dic['cache_list'])) + if 'compress_ratio' in slb_total_list: + cw.writerow([unicode(_('Compression Ratio for All Data'))]) + cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for All Data(%)'))]) + cw.writerows(zip(slb_total_dic['time'], slb_total_dic['compress_list'])) + return cw + +def generateSLBVSCSV(cw, from_time, to_time, vs_name, slb_vs_list, tcp_status): + cw.writerow([unicode(_("SLB Virtual Service Status"))+"(%s)"%vs_name]) + vs_data_dic = get_vs_data_sql(from_time, to_time, vs_name) + + if 'vs_cc' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('Concurrent Count'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cc_list'])) + if 'vs_cps' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('CPS'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cps_list'])) + if 'vs_throughput' in slb_vs_list[vs_name]: + net_in_table = value_format_1024_table(vs_data_dic['inbound_list']) + net_out_table = value_format_1024_table(vs_data_dic['outbound_list']) + cw.writerow([unicode(_('Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(vs_data_dic['time'], net_in_table, net_out_table)) + if 'vs_pkg_rate' in slb_vs_list[vs_name]: + net_in_table = value_format_1024_table(vs_data_dic['average_in_list']) + net_out_table = value_format_1024_table(vs_data_dic['average_out_list']) + cw.writerow([unicode(_('Package Rate'))]) + cw.writerow([unicode(_('Time')), unicode(_('Average Package Rate In(pps)')), unicode(_('Average Package Rate Out(pps)'))]) + cw.writerows(zip(vs_data_dic['time'], net_in_table, net_out_table)) + if 'vs_rtt' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('Average Client Connection RTT'))]) + cw.writerow([unicode(_('Time')), unicode(_('Average Client Connection RTT'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['average_rtt_list'])) + if "vs_response_code" in slb_vs_list[vs_name]: + err_data = vs_data_dic["err_list"][0] if vs_data_dic["err_list"] else [0]*31 + errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 901, 902, 903, 904, 905, 906, 'others'] + cw.writerow([unicode(_('HTTP Response Code Statistics'))]) + cw.writerow([unicode(_('Response Code')), unicode(_('Hits'))]) + cw.writerows(zip(errcode_list, err_data)) + if 'vs_cache_hit_radio' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('Cache Hit Ratio'))]) + cw.writerow([unicode(_('Time')), unicode(_('Cache Hit Ratio(%)'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['cache_list'])) + if 'vs_compression_radio_data' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('Compression Ratio for Compressible Data'))]) + cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for Compressible Data(%)'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['compress_data'])) + if 'vs_compression_radio_all' in slb_vs_list[vs_name]: + cw.writerow([unicode(_('Compression Ratio for All Data'))]) + cw.writerow([unicode(_('Time')), unicode(_('Compression Ratio for All Data(%)'))]) + cw.writerows(zip(vs_data_dic['time'], vs_data_dic['compress_all'])) + if 'vs_tcp' in slb_vs_list[vs_name] and slb_vs_list[vs_name]['vs_tcp'] and vs_name in tcp_status: + cw.writerow([unicode(_('TCP Statistics of Virtual Service: %s') % vs_name)]) + mo = MonitorOutput() + for column in mo.convert_list_to_column(slb_vs_list[vs_name]['vs_tcp']): + # ex: LISTEN + cw.writerow([unicode(_(column))]) + # ex: Time, LISTEN + cw.writerow([unicode(_('Time')), unicode(_(column))]) + for each in tcp_status[vs_name]: + # each = ["2024-06-28 15:42:47", {"LISTEN": "0"}] + cw.writerow([each[0], each[1][column]]) + + return cw + +def generateSLBRSCSV(cw, from_time, to_time, rs_name, slb_rs_list): + print("generateSLBVSCSV") + cw.writerow([unicode(_("SLB Real Service Status"))+"(%s)"%rs_name]) + rs_data_dic = get_rs_data_sql(from_time, to_time, rs_name) + + if 'rs_cc' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Concurrent Count'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) + cw.writerows(zip(rs_data_dic['time'], rs_data_dic['cc_list'])) + if 'rs_cps' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('CPS'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(rs_data_dic['time'], rs_data_dic['cps_list'])) + if 'rs_request' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Outstanding Request Count'))]) + cw.writerow([unicode(_('Time')), unicode(_('Outstanding Request Count'))]) + cw.writerows(zip(rs_data_dic['time'], rs_data_dic['request_list'])) + if 'rs_throughput' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(rs_data_dic['time'], value_format_1024_table(rs_data_dic['inbound_list']), value_format_1024_table(rs_data_dic['outbound_list']))) + if 'rs_pkg_rate' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Package Rate'))]) + cw.writerow([unicode(_('Time')), unicode(_('Average Package Rate In(pps)')), unicode(_('Average Package Rate Out(pps)'))]) + cw.writerows(zip(rs_data_dic['time'], value_format_1024_table(rs_data_dic['average_in_list']), value_format_1024_table(rs_data_dic['average_out_list']))) + if 'rs_response_time' in slb_rs_list[rs_name]: + cw.writerow([unicode(_('Average Response Time'))]) + cw.writerow([unicode(_('Time')), unicode(_('Average Response Time'))]) + cw.writerows(zip(rs_data_dic['time'], rs_data_dic['average_response_list'])) + if 'rs_response_code' in slb_rs_list[rs_name]: + errcode_list = [100, 101, 200, 206, 301, 302, 303, 304, 305, 307, 400, 401, 402, 403, 404, 405, 406, 407, 416, 500, 501, 502, 503, 504, 505, 'others'] + err_data = rs_data_dic['err_list'][0] if rs_data_dic['err_list'] else [0]*26 + cw.writerow([unicode(_('HTTP Response Code Statistics'))]) + cw.writerow([unicode(_('Response Code')), unicode(_('Hits'))]) + cw.writerows(zip(errcode_list, err_data)) + return cw + +def generateSSLVHostCSV(cw, from_time, to_time, vh_name, slb_vh_list): + cw.writerow("") + cw.writerow([unicode(_('SSL Virtual Host Status'))+"(%s)"%vh_name]) + ssl_vhost_dic = get_ssl_vh_sql(from_time, to_time,vh_name) + + if 'vh_cc' in slb_vh_list[vh_name]: + cw.writerow([unicode(_('Concurrent Connections'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) + cw.writerows(zip(ssl_vhost_dic['time'], ssl_vhost_dic['cc_list'])) + if 'vh_cps' in slb_vh_list[vh_name]: + cw.writerow([unicode(_('CPS'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(ssl_vhost_dic['time'], ssl_vhost_dic['cps_list'])) + if 'vh_throughput' in slb_vh_list[vh_name]: + cw.writerow([unicode(_('Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(ssl_vhost_dic['time'], value_format_1024_table(ssl_vhost_dic['net_in_list']), value_format_1024_table(ssl_vhost_dic['net_out_list']))) + if 'mentioned_statistics' in slb_vh_list[vh_name]: + sslStat = SSLStatFacade(from_time, gz_directory="/var/crash") + ssl_data = sslStat.get_data() + time_list = [e[0] for e in ssl_data] + connection_attempts_list = [e[1] for e in ssl_data] + successful_handshakes_data = [e[2] for e in ssl_data] + active_connections_data = [e[3] for e in ssl_data] + cw.writerow([unicode(_('total ssl statistics'))]) + cw.writerow([unicode(_('Time')), unicode(_('connection attempts')), unicode(_('successful handshakes')), unicode(_('active connections'))]) + cw.writerows(zip(time_list, connection_attempts_list, successful_handshakes_data, active_connections_data)) + return cw + +def generateSSLRHostCSV(cw, from_time, to_time, rh_name, slb_rh_list): + cw.writerow("") + cw.writerow([unicode(_('SSL Real Host Status'))+"(%s)"%rh_name]) + ssl_rhost_dic = get_ssl_rh_sql(from_time, to_time,rh_name) + + if 'rh_cc' in slb_rh_list[rh_name]: + cw.writerow([unicode(_('Concurrent Connections'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Connections'))]) + cw.writerows(zip(ssl_rhost_dic['time'], ssl_rhost_dic['cc_list'])) + if 'rh_cps' in slb_rh_list[rh_name]: + cw.writerow([unicode(_('CPS'))]) + cw.writerow([unicode(_('Time')), unicode(_('CPS'))]) + cw.writerows(zip(ssl_rhost_dic['time'], ssl_rhost_dic['cps_list'])) + if 'rh_throughput' in slb_rh_list[rh_name]: + cw.writerow([unicode(_('Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(ssl_rhost_dic['time'], value_format_1024_table(ssl_rhost_dic['net_in_list']), value_format_1024_table(ssl_rhost_dic['net_out_list']))) + return cw + +def generateLLBTotalCSV(cw, from_time, to_time, llb_total_list): + cw.writerow([unicode(_("LLB Total Status"))]) + + p = os.popen("/ca/bin/webui_utils -a 4") + #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' + llb_dic = json.loads(p.read()) + link_list = [] + for each in llb_dic["data"]: + link_list.append(each["name"]) + link_list = sorted(link_list) + + total_dic, time_list = get_llb_total_sql(from_time, to_time, link_list) + if 'bandwidth_usage' in llb_total_list: + cw.writerow([unicode(_('Bandwidth Usage'))]) + csv_header = [unicode(_('Time'))] + csv_body = [time_list] + for each in link_list: + csv_header.append(each) + csv_body.append(total_dic[each]['bandwidth_list']) + cw.writerow(csv_header) + cw.writerows(zip(*csv_body)) + if 'link_throughput' in llb_total_list: + cw.writerow([unicode(_('Throughput'))]) + csv_header = [unicode(_('Time'))] + csv_body = [time_list] + for each in link_list: + csv_header.append(each) + csv_body.append(value_format_1024_table(total_dic[each]['through_list'])) + cw.writerow(csv_header) + cw.writerows(zip(*csv_body)) + if 'curr_conn' in llb_total_list: + cw.writerow([unicode(_('Concurrent Count'))]) + csv_header = [unicode(_('Time'))] + csv_body = [time_list] + for each in link_list: + csv_header.append(each) + csv_body.append(total_dic[each]['cc_list']) + cw.writerow(csv_header) + cw.writerows(zip(*csv_body)) + if 'total_hits' in llb_total_list: + cw.writerow([unicode(_('Total Hits'))]) + csv_header = [unicode(_('Time'))] + csv_body = [time_list] + for each in link_list: + csv_header.append(each) + csv_body.append(total_dic[each]['hit_list']) + cw.writerow(csv_header) + cw.writerows(zip(*csv_body)) + return cw + +def generateLLBCSV(cw, from_time, to_time, link_name, llb_list): + cw.writerow([unicode(_("LLB Status"))+"(%s)"%link_name]) + link_data_dic = get_link_data_sql(from_time, to_time, link_name) + + if 'bandwidth_usage' in llb_list[link_name]: + cw.writerow([unicode(_('Bandwidth Usage'))]) + cw.writerow([unicode(_('Time')), unicode(_('Bandwidth Usage'))]) + cw.writerows(zip(link_data_dic['time'], link_data_dic['bandwidth_list'])) + if 'link_throughput' in llb_list[link_name]: + cw.writerow([unicode(_('Throughput'))]) + cw.writerow([unicode(_('Time')), unicode(_('Inbound(bps)')), unicode(_('Outbound(bps)'))]) + cw.writerows(zip(link_data_dic['time'], value_format_1024_table(link_data_dic['inbound_list']), value_format_1024_table(link_data_dic['outbound_list']))) + if 'curr_conn' in llb_list[link_name]: + cw.writerow([unicode(_('Concurrent Count'))]) + cw.writerow([unicode(_('Time')), unicode(_('Concurrent Count'))]) + cw.writerows(zip(link_data_dic['time'], link_data_dic['cc_list'])) + if 'total_hits' in llb_list[link_name]: + cw.writerow([unicode(_('Total Hits'))]) + cw.writerow([unicode(_('Time')), unicode(_('Total Hits'))]) + cw.writerows(zip(link_data_dic['time'], link_data_dic['hit_list'])) + return cw + +def generateGSLBRSCSV(cw, from_time, to_time): + select_dic = {'rawSql': 'select $__timeGroup(time) as timef, max(total_req) from QueryStatistics where $__timeFilter(timef/1000) GROUP BY timef;', + 'to': to_time, 'from': from_time, 'format': 'time_series'} + select_data = query_metric(select_dic) + cw.writerow([unicode(_('GSLB Status'))]) + cw.writerow([unicode(_('Total Hits'))]) + cw.writerow([unicode(_('Time')), unicode(_('Hits'))]) + for each in select_data['data']: + each = list(each) + each[0] = time.strftime(TIMEFORMAT, time.localtime(int(each[0])/1000)) + cw.writerow(each) + return cw +def generateTCPCSV(cw, tcp_status, tcp_list): + """ + Get CSV data of global tcp status and filter columns + :param object cw: csv object + :param dict tcp_status: statistics of tcp status + :param list tcp_list: needed columns + """ + mo = MonitorOutput() + cw.writerow([unicode(_('TCP Global Statistics'))]) + if not MonitorOutput.C_GLOBAL_INDEX in tcp_status: + return cw + for column in mo.convert_list_to_column(tcp_list): + # ex: LISTEN + cw.writerow([unicode(_(column))]) + # ex: Time, LISTEN + cw.writerow([unicode(_('Time')), unicode(_(column))]) + for each in tcp_status[MonitorOutput.C_GLOBAL_INDEX]: + # each = ["2024-06-28 15:42:47", {"LISTEN": "0"}] + cw.writerow([each[0], each[1][column]]) + return cw +def generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list): + print("generateCSVReport") + sys_info = get_system_version() + is_tcp_status = False + if slb_vs_list: + for vsname in slb_vs_list: + if 'vs_tcp' in slb_vs_list[vsname]: + is_tcp_status = True + get_tcp_status = {} + if is_tcp_status or tcp_list: + mo = MonitorOutput() + tr = TimeRange(from_time, to_time) + start_time = tr.ParseFrom() + end_time = tr.ParseTo() + # get tcp status of all + get_tcp_status = mo.get(datetime.fromtimestamp(start_time), datetime.fromtimestamp(end_time)) + with codecs.open(report_file_path+report_name, 'w', encoding="utf-8-sig") as csvfile: + cw = csv.writer(csvfile) + cw.writerow([unicode(_('Create Time')), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())]) + cw.writerow([unicode(_('Statistics From')), start_time]) + cw.writerow([unicode(_('Statistics To')), end_time]) + cw.writerow([unicode(_('Host Name')), sys_info['host_name']]) + cw.writerow([unicode(_('System CPU')), sys_info['system_cpu']]) + cw.writerow([unicode(_('System RAM')), sys_info['system_ram']]) + cw.writerow([unicode(_('Cumulative Running Time')), sys_info['system_boot']]) + cw.writerow([unicode(_('Model')), sys_info['system_model']]) + + if system_list: + cw = generateSystemCSV(cw, from_time, to_time, end_time, start_time, system_list) + if slb_total_list: + cw.writerow("") + cw = generateSLBTotalCSV(cw, from_time, to_time, slb_total_list) + if slb_vs_list: + cw.writerow("") + for each in slb_vs_list: + vs_name = each + cw = generateSLBVSCSV(cw, from_time, to_time, vs_name, slb_vs_list, get_tcp_status) + if slb_rs_list: + cw.writerow("") + for each in slb_rs_list: + rs_name = each + cw = generateSLBRSCSV(cw, from_time, to_time, rs_name, slb_rs_list) + if slb_vh_list: + cw.writerow("") + for each in slb_vh_list: + vh_name = each + cw = generateSSLVHostCSV(cw, from_time, to_time, vh_name, slb_vh_list) + if slb_rh_list: + cw.writerow("") + for each in slb_rh_list: + rh_name = each + cw = generateSSLRHostCSV(cw, from_time, to_time, rh_name, slb_rh_list) + if llb_total_list: + cw.writerow("") + cw = generateLLBTotalCSV(cw, from_time, to_time, llb_total_list) + if llb_list: + cw.writerow("") + for each in llb_list: + link_name = each + cw = generateLLBCSV(cw, from_time, to_time, link_name, llb_list) + if gslb_list: + cw.writerow("") + cw = generateGSLBRSCSV(cw, from_time, to_time) + if tcp_list: + cw.writerow("") + cw = generateTCPCSV(cw, get_tcp_status, tcp_list) + return + +def myFirstPages(canvas, doc): + canvas.saveState() + canvas.setFont('alibaba',10) + canvas.drawRightString(7.5 * inch, 0.75 * inch, unicode(_("Page %s")) % (doc.page)) + canvas.restoreState() + +def myLaterPages(canvas, doc): + styles=getSampleStyleSheet() + canvas.saveState() + p = Paragraph("" % get_pdf_logo(), styles["Heading1"]) + w, h = p.wrap(doc.width, doc.bottomMargin) + p.drawOn(canvas, doc.leftMargin, doc.topMargin + doc.height - 0.5*cm) + #canvas.line(doc.leftMargin, doc.bottomMargin+doc.height + 0.5*cm, doc.leftMargin+doc.width, doc.bottomMargin+doc.height + 0.5*cm) + canvas.setFont('alibaba',10) + canvas.drawRightString(7.5 * inch, 0.75 * inch, unicode(_("Page %s")) % (doc.page)) + canvas.restoreState() + +def generatePDFReport(pdf_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_list, slb_rh_list, slb_total_list, llb_list, llb_total_list, gslb_list, tcp_list): + sys_info = get_system_version() + + is_tcp_status = False + if slb_vs_list: + for vsname in slb_vs_list: + if 'vs_tcp' in slb_vs_list[vsname]: + is_tcp_status = True + get_tcp_status = {} + if is_tcp_status or tcp_list: + mo = MonitorOutput() + tr = TimeRange(from_time, to_time) + start_time = tr.ParseFrom() + end_time = tr.ParseTo() + # get tcp status of all + get_tcp_status = mo.get(datetime.fromtimestamp(start_time), datetime.fromtimestamp(end_time)) + + doc = SimpleDocTemplate(report_file_path+pdf_name,pagesize=letter, + rightMargin=60,leftMargin=60, + topMargin=70,bottomMargin=55) + env = create_default_environment("png") + if current_lan == 'ja': + pdfmetrics.registerFont(TTFont('alibaba', font_file_path + 'AlibabaSansJP/AlibabaSansJP-Regular.ttf')) + else: + pdfmetrics.registerFont(TTFont('alibaba', 'Alibaba-PuHuiTi-Regular.ttf')) + + styles=getSampleStyleSheet() + styles.add(ParagraphStyle(fontName='alibaba', name='alibaba', alignment=TA_JUSTIFY)) + data = [[unicode(_("Create Time")), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())], + [unicode(_("Statistics From")), start_time], + [unicode(_("Statistics To")), end_time], + [unicode(_("Host Name")), sys_info['host_name']], + [unicode(_("System CPU")), sys_info['system_cpu']], + [unicode(_("System RAM")), sys_info['system_ram']], + [unicode(_("Model")), sys_info['system_model']], + [unicode(_("Cumulative Running Time")), sys_info['system_boot']]] + t = generate_report_table_cover(data, 3) + im = Image(get_pdf_logo(), 2*inch, 0.8*inch) + im.hAlign = 'LEFT' + story = [] + story.append(im) + story.append(t) + + if system_list: + story = generateSystemReport(from_time, to_time, system_list, env, styles, story) + if slb_total_list: + story = generateSLBTotalReport(from_time, to_time, slb_total_list, env, styles, story) + if slb_vs_list: + story = generateSLBVSReport(from_time, to_time, slb_vs_list, env, styles, story, get_tcp_status) + if slb_rs_list: + story = generateSLBRSReport(from_time, to_time, slb_rs_list, env, styles, story) + if slb_vh_list: + story = generateSSLVHostReport(from_time, to_time, slb_vh_list, env, styles, story) + if slb_rh_list: + story = generateSSLRHostReport(from_time, to_time, slb_rh_list, env, styles, story) + if llb_total_list: + story = generateLLBTotalReport(from_time, to_time, llb_total_list, env, styles, story) + if llb_list: + story = generateLLBReport(from_time, to_time, llb_list, env, styles, story) + if gslb_list: + story = generateGSLBReport(from_time, to_time, gslb_list, env, styles, story) + if tcp_list: + story = generateTCPReport(get_tcp_status, tcp_list, env, styles, story) + doc.build(story,onFirstPage=myFirstPages, onLaterPages=myLaterPages) + + return + +def generateReport(request): + TIMEFORMAT='%Y-%m-%d %X' + post_data = request.raw_post_data + post_dict = json.loads(post_data) + from_time = post_dict['from'] + to_time = post_dict['to'] + monitor_type = post_dict['monitor_type'] + report_type = post_dict['report_type'] + global current_lan + current_lan = request.COOKIES['django_language'] + system_list = [] + slb_rn_list = [] # slb rs name list + slb_vn_list = [] # slb vs name list + slb_vs_list = {} # slb vs service dic + slb_rs_list = {} # slb rs service dic + slb_vh_list = [] # ssl vhost name list + slb_rh_list = [] # ssl rhost name list + slb_vh_dic = {} # ssl vhost service dic + slb_rh_dic = {} # ssl rhost service dic + slb_total_list = [] + llb_list = [] + llb_dic = {} + llb_total_list = [] + gslb_list = [] + tcp_list = [] + + for each in monitor_type: + if each[0] == 'system': + system_list.append(each[1]) + if each[0] == 'slb': + if each[1] == 'vs_report': + if each[2] not in slb_vn_list: + slb_vn_list.append(each[2]) + slb_vs_list[each[2]] = {} + # slb vs maybe in 5 layers, + # so changed it as slb_vs_list[3] is module: + # and slb_vs_list[4] is list of tcp statistics + if each[3] not in slb_vs_list[each[2]]: + slb_vs_list[each[2]][each[3]] = [] + if 'vs_tcp' in each[3] and len(each) > 4: + slb_vs_list[each[2]][each[3]].append(each[4]) + if each[1] == 'rs_report': + if each[2] not in slb_rn_list: + slb_rn_list.append(each[2]) + slb_rs_list[each[2]] = [each[3]] + else: + slb_rs_list[each[2]].append(each[3]) + if each[1] == 'vh_report': + if each[2] not in slb_vh_list: + slb_vh_list.append(each[2]) + slb_vh_dic[each[2]] = [each[3]] + else: + slb_vh_dic[each[2]].append(each[3]) + if each[1] == 'rh_report': + if each[2] not in slb_rh_list: + slb_rh_list.append(each[2]) + slb_rh_dic[each[2]] = [each[3]] + else: + slb_rh_dic[each[2]].append(each[3]) + if each[1] == 'slb_total': + slb_total_list.append(each[2]) + if each[0] == 'llb': + if each[2] == 'total_link': + llb_total_list.append(each[3]) + else: + if each[2] not in llb_list: + llb_list.append(each[2]) + llb_dic[each[2]] = [each[3]] + else: + llb_dic[each[2]].append(each[3]) + if each[0] == 'gslb': + gslb_list.append(each[1]) + if each[0] == 'tcp': + tcp_list.append(each[1]) + os.popen("rm -rf /var/crash/statmon/report_chart_*") + os.popen("rm -rf /ca/webui/htdocs/new/src/media/docs/Statistics_Report_*") + intervalStart, intervalEnd = get_time_range(from_time, to_time) + end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalEnd)) + start_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(intervalStart)) + print("report_type", report_type) + if report_type == 'csv': + report_name = "Statistics_Report_%s_%s.csv" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart))) + generateCSVReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list) + if report_type == 'pdf': + report_name = "Statistics_Report_%s_%s.pdf" % (time.strftime("%Y%m%d%H%M", time.localtime(intervalEnd)), time.strftime("%Y%m%d%H%M", time.localtime(intervalStart))) + generatePDFReport(report_name, from_time, to_time, end_time, start_time, system_list, slb_vs_list, slb_rs_list, slb_vh_dic, slb_rh_dic, slb_total_list, llb_dic, llb_total_list, gslb_list, tcp_list) + + if os.path.exists(report_file_path+report_name): + return HttpResponse(json.dumps({"result":True, "filename": report_name})) + else: + return HttpResponse(json.dumps({"result":False})) + +def get_slb_total_list(): + return [{ + "value": 'total_throughput', + "label": unicode(_('Total Throughput')) + }, { + "value": 'curr_conn', + "label": unicode(_('Concurrent Connections')) + }, { + "value": 'cps', + "label": unicode(_('CPS')) + }, { + "value": 'ssl_throughput', + "label": unicode(_('SSL Total Throughput')) + }, { + "value": 'ssl_vh_throughput', + "label": unicode(_('SSL Virtual Host Throughput')) + }, { + "value": 'ssl_rh_throughput', + "label": unicode(_('SSL Real Host Throughput')) + }, { + "value": 'cache_hit_radio', + "label": unicode(_('Cache Hits Ratio')) + }, { + "value": 'compress_ratio', + "label": unicode(_('Compression Ratio for All Data')) + }] + +def get_slb_virtual_list(): + vs_list = [] + p = os.popen("/ca/bin/webui_utils -a 6") + #p = '{"result": 1, "data": [{"service_name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' + vs_dic = json.loads(p.read()) + + for each in vs_dic["data"]: + vs_name = each["service_name"] + vs_type = each["protocol"] + data = { + "value": vs_name, + "label": vs_name, + "children": [{ + # "value": "vs_policy_hits", + # "label": unicode(_("Policy Hits (Top 10)")), + # },{ + # "value": "vs_rs_load", + # "label": unicode(_("Real Service Load (Top 10)")), + # },{ + "value": "vs_cc", + "label": unicode(_("Concurrent Connections")), + },{ + "value": "vs_cps", + "label": unicode(_("CPS")), + },{ + "value": "vs_throughput", + "label": unicode(_("Throughput")), + },{ + "value": "vs_pkg_rate", + "label": unicode(_("Package Rate")), + },{ + "value": "vs_rtt", + "label": unicode(_("Average Client Connection RTT")), + }] + } + if vs_type in ['http', 'https']: + data["children"].extend([{ + "value": "vs_response_code", + "label": unicode(_("HTTP Response Code Statistics")), + },{ + "value": "vs_cache_hit_radio", + "label": unicode(_("Cache Hits Ratio")), + },{ + "value": "vs_compression_radio_data", + "label": unicode(_("Compression Ratio for Compressible Data")), + },{ + "value": "vs_compression_radio_all", + "label": unicode(_("Compression Ratio for All Data")), + }]) + data["children"].extend([{ + "value": 'vs_tcp', + "label": unicode(_('TCP')), + "children": get_statistics_tcp_list(is_vs=True) + }]) + vs_list.append(data) + return vs_list + +def get_slb_real_list(): + rs_list = [] + p = os.popen("/ca/bin/webui_utils -a 7") + #p = '{"result": 1, "data": [{"service_name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' + rs_dic = json.loads(p.read()) + for each in rs_dic["data"]: + rs_name = each["service_name"] + rs_type = each["protocol"] + data = { + "value": rs_name, + "label": rs_name, + "children": [{ + "value": "rs_cc", + "label": unicode(_("Concurrent Connections")), + },{ + "value": "rs_cps", + "label": unicode(_("CPS")), + },{ + "value": "rs_request", + "label": unicode(_("Outstanding Request Count")), + },{ + "value": "rs_throughput", + "label": unicode(_("Throughput")), + },{ + "value": "rs_pkg_rate", + "label": unicode(_("Package Rate")), + },{ + "value": "rs_response_time", + "label": unicode(_("Average Response time")), + }] + } + if rs_type in ['http', 'https']: + data['children'].append({ + "value": "rs_response_code", + "label": unicode(_("HTTP Response Code Statistics")) + }) + rs_list.append(data) + return rs_list + +def get_ssl_virtual_list(): + vh_list = [] + sess = get_current_session() + sess.cli.set_enable() + result = sess.cli.cmd('show ssl host', + EasyParser('ssl host virtual', ['?name', '?vs_name'])) + for each in result: + vh_name = each["name"] + vh_list.append({ + "value": vh_name, + "label": vh_name, + "children": [{ + "value": 'vh_cc', + "label": unicode(_('Concurrent Connections')) + }, { + "value": 'vh_cps', + "label": unicode(_('CPS')) + }, { + "value": 'vh_throughput', + "label": unicode(_('Throughput')) + }] + }) + + vh_list.append({ + "value": "individual_virtual_host", + "label": unicode(_("Individual virtual host")) , + "children": [{ + "value": 'mentioned_statistics', + "label": unicode(_('mentioned statistics')) + }] + }) + return vh_list + +def get_ssl_real_list(): + rh_list = [] + sess = get_current_session() + sess.cli.set_enable() + result = sess.cli.cmd('show ssl host', + EasyParser('ssl host real', ['?name', '?vs_name'])) + for each in result: + rh_name = each['name'] + rh_list.append({ + "value": rh_name, + "label": rh_name, + "children": [{ + "value": 'rh_cc', + "label": unicode(_('Concurrent Connections')) + }, { + "value": 'rh_cps', + "label": unicode(_('CPS')) + }, { + "value": 'rh_throughput', + "label": unicode(_('Throughput')) + }] + }) + return rh_list + +def get_llb_link_route_list(): + link_list = [{ + "value": "total_link", + "label": unicode(_("Total")), + "children": [{ + "value": 'bandwidth_usage', + "label": unicode(_('Bandwidth Usage')), + }, { + "value": 'link_throughput', + "label": unicode(_('Total Link Throughput')), + }, { + "value": 'curr_conn', + "label": unicode(_('Concurrent Connections')), + }, { + "value": 'total_hits', + "label": unicode(_('Total Hits')), + }] + }] + p = os.popen("/ca/bin/webui_utils -a 4") + #p = '{"result": 1, "data": [{"name": "r1", "protocol": "http", "enable": 1, "updown_status": 0,"ip": "1.1.1.1", "port": 80, "max_conn": 0, "health_relation": "and", "recovery_time": 0, "warm_up_time": 0, "max_cps": 0, "soft_bandwidth": 0, "hard_bandwidth": 0,"keepdip": 0, "hc_up": 3, "hc_down": 3,"server_conn_reuse": 1, "server_persist": 1, "check_type": "icmp", "total_hits": 0, "total_bytes_in": 0, "total_bytes_out": 0, "current_connection_count": 0, "cps": 0, "average_response_time": 10}]}' + llb_dic = json.loads(p.read()) + for each in llb_dic["data"]: + link_name = each["name"] + link_list.append({ + "value": link_name, + "label": link_name, + "children": [{ + "value": 'bandwidth_usage', + "label": unicode(_('Bandwidth Usage')), + }, { + "value": 'link_throughput', + "label": unicode(_('Link Throughput')), + }, { + "value": 'curr_conn', + "label": unicode(_('Concurrent Connections')), + }, { + "value": 'total_hits', + "label": unicode(_('Total Hits')), + }] + }) + return link_list + + +def get_statistics_tcp_list(is_vs): + """ + Get list of statistics tcp options + :param bool is_vs: is for virtual service? False for global + :return List + Global ex: [{ + "value": 'listen', + "label": 'LISTEN' + }] + Virtual Service ex: [{ + "value": 'vs_listen', + "label": 'LISTEN' + }] + """ + list = [{ + "value": 'listen', + "label": unicode(_('LISTEN')), + }, { + "value": 'syn_sent', + "label": unicode(_('SYN_SENT')), + }, { + "value": 'syn_rcvd', + "label": unicode(_('SYN_RCVD')), + }, { + "value": 'established', + "label": unicode(_('ESTABLISHED')), + }, { + "value": 'close_wait', + "label": unicode(_('CLOSE_WAIT')), + }, { + "value": 'fin_wait_1', + "label": unicode(_('FIN_WAIT_1')), + }, { + "value": 'closing', + "label": unicode(_('CLOSING')), + }, { + "value": 'last_ack', + "label": unicode(_('LAST_ACK')), + }, { + "value": 'fin_wait_2', + "label": unicode(_('FIN_WAIT_2')), + }, { + "value": 'time_wait', + "label": unicode(_('TIME_WAIT')), + }] + if is_vs: + for each in list: + each["value"] = "vs_" + each["value"] + return list +def generateItems(request): + item_list = [{ + "value": 'system', + "label": unicode(_('System')), + "children": [{ + "value": 'cpu_usage', + "label": unicode(_('CPU Usage')), + }, { + "value": 'mem_usage', + "label": unicode(_('Memory Usage')), + }, { + "value": 'net_load', + "label": unicode(_('Network Load')), + }, { + "value": 'conn_status', + "label": unicode(_('Connection Status')), + }] + }, { + "value": 'slb', + "label": unicode(_('SLB')), + "children": [{ + "value": 'slb_total', + "label": unicode(_('Global')), + "children": get_slb_total_list() + }, { + "value": 'vs_report', + "label": unicode(_('Virtual Service')), + "children": get_slb_virtual_list() + }, { + "value": 'rs_report', + "label": unicode(_('Real Service')), + "children": get_slb_real_list() + }, { + "value": 'vh_report', + "label": unicode(_('SSL Virtual Host')), + "children": get_ssl_virtual_list() + }, { + "value": 'rh_report', + "label": unicode(_('SSL Real Host')), + "children": get_ssl_real_list() + }] + }, { + "value": 'llb', + "label": unicode(_('LLB')), + "children": [{ + "value": "llb_link", + "label": unicode(_("LLB Link")), + "children": get_llb_link_route_list() + }] + }, { + "value": 'gslb', + "label": unicode(_('GSLB')), + "children": [{ + "value": 'total_hits', + "label": unicode(_('Total Host Hits')), + }] + }, { + "value": 'tcp', + "label": unicode(_('TCP')), + "children": get_statistics_tcp_list(is_vs=False) + }] + return HttpResponse(json.dumps(item_list)) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/__init__.py (added) =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/__init__.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/__init__.py (revision 0) Index: /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/statistics.py =================================================================== --- /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/statistics.py (revision 0) +++ /branches/rel_apv_10_7/usr/click/webui/htdocs/new/src/hive/tcp_status/statistics.py (working copy) @@ -0,0 +1,269 @@ +""" +This file is for parsing data from monitor.out* +and for the part of tcp statistics +Author: shuinvy@arraynetworks.net +""" +from datetime import datetime +import time +import os +from hive.utils import andebug +from django.utils.translation import ugettext_lazy as _ + +class MonitorOutput: + """ + Class for parsing data from monitor.out* + """ + C_FILE_LOCATION = "/var/crash/" + C_PREFIX = "monitor.out" + C_FILE_EXT = ".gz" + C_INIT_FILE = "/var/crash/monitor.out0" + C_TCP_COMMAND = "show statistics tcp all" + C_GLOBAL_PATTERN = "Global Statistics" + C_GLOBAL_INDEX = "global" + C_VIRTUAL_SERVICE_PATTERN = "virtual service" + C_EACH_FIELDS = [ + 'LISTEN', + 'SYN_SENT', + 'SYN_RCVD', + 'ESTABLISHED', + 'CLOSE_WAIT', + 'FIN_WAIT_1', + 'CLOSING', + 'LAST_ACK', + 'FIN_WAIT_2', + 'TIME_WAIT' + ] + C_COLUMN_MATCH = { + 'LISTEN': 'listen', + 'SYN_SENT': 'syn_sent', + 'SYN_RCVD': 'syn_rcvd', + 'ESTABLISHED': 'established', + 'CLOSE_WAIT': 'close_wait', + 'FIN_WAIT_1': 'fin_wait_1', + 'CLOSING': 'closing', + 'LAST_ACK': 'last_ack', + 'FIN_WAIT_2': 'fin_wait_2', + 'TIME_WAIT': 'time_wait' + } + + def __init__(self): + """ + :param string type: global or vs(virtual service) + """ + + def get(self, start_time_obj, end_time_obj): + """ + Get monitor data(filter by date range) + It will sort time asc + :param datetime start_time_obj: start time + :param datetime end_time_obj: end time + :return dict + global example: { + 'global': [{ + '2024-06-24 07:12:43': { + "ESTABLISHED": "0", + "SYN_RCVD": "0", + "LAST_ACK": "0", + "FIN_WAIT_2": "1", + "FIN_WAIT_1": "0", + "TIME_WAIT": "0", + "CLOSING": "0", + "SYN_SENT": "0", + "CLOSE_WAIT": "1", + "LISTEN": "0" + } + }] + } + vs example: { + 'v1': [{ + '2024-06-24 07:12:43': { + "ESTABLISHED": "0", + "SYN_RCVD": "0", + "LAST_ACK": "0", + "FIN_WAIT_2": "1", + "FIN_WAIT_1": "0", + "TIME_WAIT": "0", + "CLOSING": "0", + "SYN_SENT": "0", + "CLOSE_WAIT": "1", + "LISTEN": "0" + } + }] + } + """ + # check is original file exist + check_init = os.path.isfile(self.C_INIT_FILE) + if not check_init: + # no statistics data + return {} + out0 = self.read_file(self.C_INIT_FILE, start_time_obj, end_time_obj) + final = out0.copy() + """ + To get the list of logs, we should make new folder + and copy file into the new folder + then unzip the file in the folder + get data from the file + and remove the folder(and the file inside it) + """ + for item in os.listdir(self.C_FILE_LOCATION): + if os.path.isfile(os.path.join(self.C_FILE_LOCATION, item)) and self.C_PREFIX in item and item.endswith(self.C_FILE_EXT): + # get modify time of the file + ti_m = os.path.getmtime(self.C_FILE_LOCATION + item) + m_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ti_m)) + m_time_obj = datetime.strptime(m_time, "%Y-%m-%d %H:%M:%S") + if m_time_obj < start_time_obj or m_time_obj > end_time_obj: + # not in the date interval + continue + folder_name = item.replace(self.C_FILE_EXT, "") + # make folder for the gz files, because it will be removed after you unzip it. + os.mkdir(self.C_FILE_LOCATION + folder_name) + os.system("cp %s %s" % (self.C_FILE_LOCATION + item, self.C_FILE_LOCATION + folder_name)) + copyed_gz_path = self.C_FILE_LOCATION + folder_name + "/" + item + os.system("gunzip %s" % copyed_gz_path) + unzip_file = self.C_FILE_LOCATION + folder_name + "/" + folder_name + # get data from unzip file of monitor + outN = self.read_file(unzip_file, start_time_obj, end_time_obj) + # remove unzip file and folder we created + os.remove(unzip_file) + os.rmdir(self.C_FILE_LOCATION + folder_name) + # merge data with same key(each virtual service name or 'global' in each monitor files) + final = self.merge_by_key(final, outN) + output = {} + for key in final.keys(): + output[key] = sorted(final[key].items(), key = lambda x:datetime.strptime(x[0], '%Y-%m-%d %H:%M:%S'), reverse=False) + return output + + def read_file(self, file_path, start_time_obj, end_time_obj): + """ + Get statistics data from file based on type + :param string file_path: the full path of the file + :param time start_time_obj: start time (time object) + :param time end_time_obj: start time (time object) + :return dict + global example: { + 'global': { + '2024-06-24 07:12:43': { + "ESTABLISHED": "0", + "SYN_RCVD": "0", + "LAST_ACK": "0", + "FIN_WAIT_2": "1", + "FIN_WAIT_1": "0", + "TIME_WAIT": "0", + "CLOSING": "0", + "SYN_SENT": "0", + "CLOSE_WAIT": "1", + "LISTEN": "0" + } + } + } + vs example: { + 'v1': { + '2024-06-24 07:12:43': { + "ESTABLISHED": "0", + "SYN_RCVD": "0", + "LAST_ACK": "0", + "FIN_WAIT_2": "1", + "FIN_WAIT_1": "0", + "TIME_WAIT": "0", + "CLOSING": "0", + "SYN_SENT": "0", + "CLOSE_WAIT": "1", + "LISTEN": "0" + } + } + } + """ + try: + fp = open(file_path) + lines = fp.readlines() + is_start_line = False + index_list = [] # add line index of file to be passed + final_list = {} + for ind, line in enumerate(lines): + # find the place the commnad line is started + if self.C_TCP_COMMAND in line: + is_start_line = True + date_line = self.get_date_str(lines[ind-1]) # date_line always before command line + date_obj = datetime.strptime(date_line, "%a %b %d %H:%M:%S %Z %Y") + date_format = str(date_obj) + if is_start_line: + if ind in index_list: + # already be handled because the field is required, + # so skip these idnex + continue + if self.C_GLOBAL_PATTERN in line or self.C_VIRTUAL_SERVICE_PATTERN in line: + section_list = {} + # if length = 10, that is range(1, 11) + for start_ind in range(1, len(self.C_EACH_FIELDS) + 1): + next_index = ind + start_ind + the_pair_of_next = self.get_pair(lines[next_index]) + if the_pair_of_next['key'] in self.C_EACH_FIELDS: + section_list[the_pair_of_next['key']] = the_pair_of_next['value'] + index_list.append(next_index) + if date_obj < start_time_obj or date_obj > end_time_obj: + # not in the time interval + continue + if self.C_GLOBAL_PATTERN in line: + each_key = self.C_GLOBAL_INDEX + else: + each_key = self.get_virtual_service_name(line) + if not each_key in final_list: + final_list[each_key] = {} + #final_list[each_key][date_format] = section_list + final_list[each_key].update({date_format: section_list}) + if '/Ao=' in line.encode("base64"): + # end of show statistics tcp parts + is_start_line = False + except Exception as ex: + andebug("hive.debug", "Cannot open monitor file: %s" % ex) + return {} + finally: + fp.close() + return final_list + + def get_pair(self, line): + # line is " LISTEN: 0" + tmp = line.split(":") + return { + 'key': str(tmp[0]).strip(), + 'value': str(tmp[1]).strip() + } + + def get_date_str(self, line): + # line is "[Sun Jun 24 02:12:43 GMT 2024]" + tmp = str(line).strip() + if not '[' in tmp or not ']' in tmp: + return tmp + return tmp.split('[')[1].split(']')[0] + + def get_virtual_service_name(self, line): + # line is " virtual service: tcp1" + tmp = str(line).strip() + tmp2 = tmp.split(":") + return str(tmp2[1]).strip() + + def merge_by_key(self, dict1, dict2): + dict1tmp = dict1 + dict2tmp = dict2 + # choose dict2 as base if key count of dict2 is much than dict1 + if len(dict2) > len(dict1): + dict1tmp = dict2 + dict2tmp = dict1 + for key in dict1tmp.keys(): + if dict2tmp and not key in dict2tmp: + continue + if dict2tmp and dict2tmp[key]: + dict1tmp[key].update(dict2tmp[key]) + return dict1tmp + + def convert_list_to_column(self, list): + output = [] + reverse_map = {v: k for k, v in self.C_COLUMN_MATCH.items()} + for value in list: + value = value.replace("vs_", "") + # value: listen + if not value in reverse_map: + continue + # append(LISTEN) + output.append(reverse_map[value]) + return output