Index: /branches/amp_4_0/scripts/install_postgresql.sh =================================================================== --- /branches/amp_4_0/scripts/install_postgresql.sh (nonexistent) +++ /branches/amp_4_0/scripts/install_postgresql.sh (working copy) @@ -0,0 +1,58 @@ +if rpm -qa | grep postgresql10 > /dev/null; +then + echo "postgresql exists" + +else + echo "install postgresql" + #yum install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm + #yum install postgresql96 postgresql96-server + #/usr/pgsql-9.6/bin/postgresql96-setup initdb + yum install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm + yum install postgresql10 + yum install postgresql10-server + /usr/pgsql-10/bin/postgresql-10-setup initdb +fi +sed -i -e 's|host all all 127.0.0.1/32 ident|host all all 127.0.0.1/32 trust|' /var/lib/pgsql/10/data/pg_hba.conf +systemctl enable postgresql-10 +systemctl start postgresql-10 +if sudo -Hiu postgres /usr/pgsql-10/bin/psql -lqt | cut -d \| -f 1 | grep -w cm; then + echo "database exists" +else + echo "create database" + sudo -Hiu postgres /usr/pgsql-10/bin/createdb cm +fi + + +if rpm -qa | grep timescaledb > /dev/null; +then + echo "timescaledb exists" +else + yum install https://timescalereleases.blob.core.windows.net/rpm/timescaledb-0.11.0-postgresql-10-0.x86_64.rpm + echo "shared_preload_libraries = 'timescaledb'" >> /var/lib/pgsql/10/data/postgresql.conf + systemctl restart postgresql-10 + /usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -d cm --command 'CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;' +fi + +if rpm -qa | grep postgresql10-contrib > /dev/null; +then + echo "postgresql10-contrib exists" +else + yum install postgresql10-contrib-10.5 +fi + +if rpm -qa | grep postgresql10-plpython-10.5 > /dev/null; +then + echo "postgresql10-plpython exists" +else + yum install postgresql10-plpython-10.5 +fi + +#rpm -ivh apache-madlib-1.15-bin-Linux.rpm +if rpm -qa | grep madlib > /dev/null; +then + /usr/local/madlib/bin/madpack -p postgres -c postgres@127.0.0.1:5432/cm install +else + echo "Please install madlib first" +fi + + Property changes on: scripts/install_postgresql.sh ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/company.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/company.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/company.py (working copy) @@ -1,14 +1,21 @@ # encoding: utf-8 +import datetime -COMPANY_COPYRIGHT = {'en':'2019 Beijing Array Networks Co., Ltd. All Rights reserved.','zh-cn':'2019 北京华耀科技有限公司. 版权所有.','zh-tw':'2019 北京華耀科技有限公司. 版權所有.','ja':'2019 Beijing Array Networks Co., Ltd. All Rights reserved.'} +current_year = datetime.datetime.now().year + +COMPANY_COPYRIGHT = {'en': f'{current_year} Array Networks, Inc. All rights reserved.', + 'zh-cn': f'{current_year} Array Networks, Inc. 保留所有权利。', + 'zh-tw': f'{current_year} Array Networks, Inc. 保留所有權利。', + 'ja': f'{current_year} Array Networks, Inc. 無断複写・転載を禁じます。' + } COMPANY_TRADEMASK = 'SpeedCoreTM' COMPANY_NAME = 'Array Networks' -COMPANY_NAME_DICT = {'en':COMPANY_NAME,'zh-cn':COMPANY_NAME,'zh-tw':COMPANY_NAME,'ja':COMPANY_NAME} -COMPANY_NAME_FULL = 'Beijing Array Networks Co., Ltd.' +COMPANY_NAME_DICT = {'en': COMPANY_NAME, 'zh-cn': COMPANY_NAME, 'zh-tw': COMPANY_NAME, 'ja': COMPANY_NAME} +COMPANY_NAME_FULL = 'ARRAY NETWORKS, INC.' COMPANY_EMAIL = 'support@arraynetworks.net' -COMPANY_PHONE = '877-992-7729 (877-99-ARRAY)' +COMPANY_PHONE = '1-877-992-7729 (1-877-99-ARRAY)' PRODUCT_NAME = 'AMP' PRODUCT_KEYWORD = 'amp' OS_NAME = 'ArrayOS' COMPANY_SNMP_MIBOID = '7564' -COMPANY_KEYWORD = 'Array' \ No newline at end of file +COMPANY_KEYWORD = 'Array' Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/AG_CLI_TABLE.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/AG_CLI_TABLE.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/AG_CLI_TABLE.py (working copy) @@ -1,39 +1,55 @@ -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE import re -AG_CLI_TABLE = { - "vsite" : { - "vpn" : [ - {"parser" : 'vpn (?Pon|off)', "cmd" : ''}, - {"parser" : 'vpn backend keepalive (?Pon|off)', "cmd" : ''}, - {"parser" : 'vpn clientisolate (?Pon|off)', "cmd" : ''}, - {"parser" : 'vpn clientinfocollect (?Pon|off)', "cmd" : ''}, - {"parser" : 'vpn resource group "(?P.*?)"', "cmd" : 'no vpn resource group %(group_name)s'}, - {"parser" : 'vpn resource groupitem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)', "cmd" : 'no vpn resource groupitem network %(group_name)s %(resource)s'}, - {"parser" : 'vpn resource groupexcludeditem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)', "cmd" : 'no vpn resource groupexcludeditem network %(group_name)s %(resource)s '}, - {"parser" : 'vpn netpool name "(?P.*?)"', "cmd" : 'no vpn netpool name %(netpool_name)s'}, - {"parser" : 'vpn netpool iprange dynamic "(?P.*?)" (?P[0-9|\.|:|a-f|A-F]+) (?P[0-9|\.|:|a-f|A-F]+) "(?P.*?)"', "cmd" : 'no vpn netpool iprange dynamic %(netpool_name)s %(low_ip)s %(high_ip)s %(unit_name)s'} - ], - "acl" : [ - {"parser" : 'acl resourcegroup network "(?P.*?)" "(?P.*?)"', "cmd" : 'no acl resourcegroup %(group_name)s'}, - {"parser" : 'acl resource "(?P.*?)" "(?P.*?)"', "cmd" : 'no acl resource %(group_name)s %(resource)s'}, - {"parser" : 'acl rule "(?P.*?)" "(?P.*?)" "(?PPERMIT|DENY)" (?P[0-9]+) "(?PR|U|G)"', "cmd" : 'no acl rule %(target_name)s %(group_name)s %(target_type)s'} - ], - "role" : [ - {"parser" : 'role name "(?P.*?)" "(?P.*?)" (?P[0-9]+)', "cmd" : 'no role name %(role_name)s'}, - {"parser" : 'role qualification "(?P.*?)" "(?P.*?)" "(?P.*?)"', "cmd" : 'no role qualification %(role_name)s %(qualification_name)s'}, - {"parser" : 'role condition "(?P.*?)" "(?P.*?)" "(?P.*?)"', "cmd" : 'no role condition %(role_name)s %(qualification_name)s %(condition)s'}, - {"parser" : 'role resource netpool "(?P.*?)" "(?P.*?)"', "cmd" : 'no role resource netpool %(role_name)s %(netpool_name)s'}, - {"parser" : 'role resource vpnresourcegroup "(?P.*?)" "(?P.*?)"', "cmd" : 'no role resource vpnresourcegroup %(role_name)s %(vpn_group_name)s'} - ] - }, - "global" : { - "interface" : [ - {"parser" : 'interface mtu (?P.*?) (?P[0-9]+)', "cmd" : ''} - ] - } +from hive.model.legacycli import cli_parse, RegexParser, MATCHONE + +AG_CLI_TABLE = { + "vsite": { + "vpn": [ + {"parser": 'vpn (?Pon|off)', "cmd": ''}, + {"parser": 'vpn backend keepalive (?Pon|off)', "cmd": ''}, + {"parser": 'vpn clientisolate (?Pon|off)', "cmd": ''}, + {"parser": 'vpn clientinfocollect (?Pon|off)', "cmd": ''}, + {"parser": 'vpn resource group "(?P.*?)"', "cmd": 'no vpn resource group %(group_name)s'}, + {"parser": 'vpn resource groupitem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)', + "cmd": 'no vpn resource groupitem network %(group_name)s %(resource)s'}, + { + "parser": 'vpn resource groupexcludeditem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)', + "cmd": 'no vpn resource groupexcludeditem network %(group_name)s %(resource)s '}, + {"parser": 'vpn netpool name "(?P.*?)"', "cmd": 'no vpn netpool name %(netpool_name)s'}, + { + "parser": 'vpn netpool iprange dynamic "(?P.*?)" (?P[0-9|\.|:|a-f|A-F]+) (?P[0-9|\.|:|a-f|A-F]+) "(?P.*?)"', + "cmd": 'no vpn netpool iprange dynamic %(netpool_name)s %(low_ip)s %(high_ip)s %(unit_name)s'} + ], + "acl": [ + {"parser": 'acl resourcegroup network "(?P.*?)" "(?P.*?)"', + "cmd": 'no acl resourcegroup %(group_name)s'}, + {"parser": 'acl resource "(?P.*?)" "(?P.*?)"', + "cmd": 'no acl resource %(group_name)s %(resource)s'}, + { + "parser": 'acl rule "(?P.*?)" "(?P.*?)" "(?PPERMIT|DENY)" (?P[0-9]+) "(?PR|U|G)"', + "cmd": 'no acl rule %(target_name)s %(group_name)s %(target_type)s'} + ], + "role": [ + {"parser": 'role name "(?P.*?)" "(?P.*?)" (?P[0-9]+)', + "cmd": 'no role name %(role_name)s'}, + {"parser": 'role qualification "(?P.*?)" "(?P.*?)" "(?P.*?)"', + "cmd": 'no role qualification %(role_name)s %(qualification_name)s'}, + {"parser": 'role condition "(?P.*?)" "(?P.*?)" "(?P.*?)"', + "cmd": 'no role condition %(role_name)s %(qualification_name)s %(condition)s'}, + {"parser": 'role resource netpool "(?P.*?)" "(?P.*?)"', + "cmd": 'no role resource netpool %(role_name)s %(netpool_name)s'}, + {"parser": 'role resource vpnresourcegroup "(?P.*?)" "(?P.*?)"', + "cmd": 'no role resource vpnresourcegroup %(role_name)s %(vpn_group_name)s'} + ] + }, + "global": { + "interface": [ + {"parser": 'interface mtu (?P.*?) (?P[0-9]+)', "cmd": ''} + ] + } } + def get_delete_cli(cli_str, vsite_name): module = cli_str.split(' ')[0] if module not in AG_CLI_TABLE[vsite_name]: @@ -42,4 +58,4 @@ result = cli_parse(cli_str, RegexParser(item['parser'], MATCHONE, reflags=re.S)) if result: return (item['cmd'] % result) - return 'Error' \ No newline at end of file + return 'Error' Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/activation_server.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/activation_server.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/activation_server.py (working copy) @@ -1,22 +1,26 @@ +import json + +from django.utils.translation import gettext_lazy as _ + from cm.lib.communication import send_http_request from hive.exceptions import ModelQueryException -from django.utils.translation import gettext_lazy as _ -from hive.model.legacycli import CLICmdException,CLICmdError,CLICmdWarning,CLICmdInfo -import json -__=_ +from hive.model.legacycli import CLICmdError +__ = _ + ServerIP = 'localhost' -#ServerIP = '10.8.11.21' -class ActivationServer(object): + +class ActivationServer(object): @classmethod def check_license(cls, license): try: - response_data = send_http_request('POST', '/cm/check_license', license, 'json', 'http', ServerIP, '8001', 10) + response_data = send_http_request('POST', '/cm/check_license', license, 'json', 'http', ServerIP, '8001', + 10) except Exception as e: if 'timed out' in str(e): raise ModelQueryException(CLICmdError(__('Check Timeout!'))) - raise ModelQueryException(CLICmdError(__('Check Failed! Reason: %s'%str(e)))) + raise ModelQueryException(CLICmdError(__('Check Failed! Reason: %s' % str(e)))) # response_data: {'status': 200, 'body': '{\n\t"status":\ttrue,\n\t"msg":\t""\n}', 'reason': 'OK'} if response_data['status'] != 200: raise ModelQueryException(CLICmdError(__('Check Failed! Something Wrong With Activation Server.'))) @@ -31,7 +35,7 @@ @classmethod def activate_device(cls, data): - #data: {'license_name': u'license1', 'device_id': [u'192.168.4.71']} + # data: {'license_name': u'license1', 'device_id': [u'192.168.4.71']} try: response_data = send_http_request('POST', '/cm/activate_device', data, 'json', 'http', ServerIP, '8001', 10) except Exception as e: @@ -40,7 +44,7 @@ raise ModelQueryException(CLICmdError(__('Activation Failed!'))) if response_data['status'] != 200: raise ModelQueryException(CLICmdError(__('Activation Failed! Something Wrong With Activation Server.'))) - + if response_data["body"]: body = json.loads(response_data["body"]) if body: @@ -48,13 +52,15 @@ raise ModelQueryException(CLICmdError(__(body["msg"]))) else: return body['data'] + return None + return None - @classmethod def deactivate_device(cls, data): - #data: {'device_id': [], 'licence_name': licence_name} + # data: {'device_id': [], 'licence_name': licence_name} try: - response_data = send_http_request('POST', '/cm/deactivate_device', data, 'json', 'http', ServerIP, '8001', 80) + response_data = send_http_request('POST', '/cm/deactivate_device', data, 'json', 'http', ServerIP, '8001', + 80) except Exception as e: if 'timed out' in str(e): raise ModelQueryException(CLICmdError(__('Deactivation Timeout!'))) @@ -69,7 +75,8 @@ raise ModelQueryException(CLICmdError(__(body["msg"]))) else: return body['data'] - + return None + return None @classmethod def get_license_info(cls, data=[]): @@ -89,15 +96,17 @@ @classmethod def delete_license_info(cls, data, import_mode=False): - #data: {'license_name': ['license1', 'license2', 'licnese3']} + # data: {'license_name': ['license1', 'license2', 'licnese3']} try: - response_data = send_http_request('POST', '/cm/delete_license_info', data, 'json', 'http', ServerIP, '8001', 10) + response_data = send_http_request('POST', '/cm/delete_license_info', data, 'json', 'http', ServerIP, '8001', + 10) except Exception as e: if 'timed out' in str(e): raise ModelQueryException(CLICmdError(__('Delete license info timeout!'))) raise ModelQueryException(CLICmdError(__('Delete license info Failed!'))) if response_data['status'] != 200: - raise ModelQueryException(CLICmdError(__('Delete license info Failed! Something Wrong With Activation Server.'))) + raise ModelQueryException( + CLICmdError(__('Delete license info Failed! Something Wrong With Activation Server.'))) if response_data["body"]: body = json.loads(response_data["body"]) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py (working copy) @@ -1,17 +1,22 @@ import http.client -import json -import dicttoxml -from xml.dom.minidom import parseString -import pycurl, threading +import threading from io import StringIO +from ipaddress import ip_address, IPv4Address +from xml.dom.minidom import parseString + +import dicttoxml +import pycurl + from cm.lib.postgres_db import DB from djproject.an_settings import * -from ipaddress import ip_address, IPv4Address -"""According to the device_type, modify the url /rest/device_type/system/version to - /rest/apv/system/version or /rest/ag/[global | vsite_name]/system/version """ -def modify_url(url, device_type, vsite_name = None): +According to the device_type, modify the url /rest/device_type/system/version to +/rest/apv/system/version or /rest/ag/[global | vsite_name]/system/version +""" + + +def modify_url(url, device_type, vsite_name=None): _type = None new_url = None # query device type from string @@ -48,6 +53,7 @@ new_url = '/'.join(url_list) return str(new_url) + class ThreadLock(object): def __init__(self): self.data = {} @@ -73,13 +79,18 @@ if device_ip in self.data: del self.data[device_ip] + def check_ipv4(ip): return True if type(ip_address(str(ip))) is IPv4Address else False -#L = threading.Lock() + +# L = threading.Lock() L = ThreadLock() -def send_https_rest_request(method, url, params, ip, restapi_port, restapi_username, restapi_password, proto='https', cookie='', save_cookie=False, lock=True): - authorization = 'Basic %s'%(('%s:%s'%(restapi_username, restapi_password)).encode('base64').replace('\n','')) + + +def send_https_rest_request(method, url, params, ip, restapi_port, restapi_username, restapi_password, proto='https', + cookie='', save_cookie=False, lock=True): + authorization = 'Basic %s' % (('%s:%s' % (restapi_username, restapi_password)).encode('base64').replace('\n', '')) buffer = StringIO() curl = pycurl.Curl() curl.setopt(pycurl.CONNECTTIMEOUT, 5) @@ -96,10 +107,10 @@ curl.setopt(pycurl.POSTFIELDS, params) # curl.setopt(pycurl.COOKIEFILE, "ComposerUISESS=f1f713c9e000f5d3f280adbd124df4f5; Path=/; Max-Age=86400") if save_cookie: - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE - curl.setopt(pycurl.HEADER, 1) #don't write http header to response - curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s'%authorization, 'Expect:']) - curl.setopt(pycurl.URL, "http://%s:%d%s"%(str(ip), int(restapi_port), url)) + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE + curl.setopt(pycurl.HEADER, 1) # don't write http header to response + curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s' % authorization, 'Expect:']) + curl.setopt(pycurl.URL, "http://%s:%d%s" % (str(ip), int(restapi_port), url)) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if cookie: curl.setopt(pycurl.COOKIE, cookie) @@ -131,10 +142,10 @@ if proto == 'https': curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE - curl.setopt(pycurl.HEADER, 0) #don't write http header to response - curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s'%authorization, 'Expect:']) - curl.setopt(pycurl.URL, "https://%s:%d%s"%(str(ip), int(restapi_port), url)) + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE + curl.setopt(pycurl.HEADER, 0) # don't write http header to response + curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s' % authorization, 'Expect:']) + curl.setopt(pycurl.URL, "https://%s:%d%s" % (str(ip), int(restapi_port), url)) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if cookie: curl.setopt(pycurl.COOKIE, cookie) @@ -144,7 +155,7 @@ curl.perform() data = { 'status': curl.getinfo(pycurl.RESPONSE_CODE), - #'reason': response.reason, + # 'reason': response.reason, 'body': buffer.getvalue(), } curl.close() @@ -157,10 +168,10 @@ # curl.setopt(pycurl.SSL_VERIFYPEER, 0) # curl.setopt(pycurl.SSL_VERIFYHOST, 0) - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE - curl.setopt(pycurl.HEADER, 0) #don't write http header to response - curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s'%authorization, 'Expect:']) - curl_url = "http://%s:%d%s"%(str(ip), int(restapi_port), url) + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE + curl.setopt(pycurl.HEADER, 0) # don't write http header to response + curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s' % authorization, 'Expect:']) + curl_url = "http://%s:%d%s" % (str(ip), int(restapi_port), url) curl.setopt(pycurl.URL, curl_url) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if cookie: @@ -181,6 +192,7 @@ lock.release() return data + def send_https_xmlrpc_request(method, url, params, ip, xmlrpc_port, proto='https'): if not check_ipv4(ip): ip = "[%s]" % ip @@ -193,26 +205,26 @@ if proto == 'https': curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE curl.setopt(pycurl.HEADER, 0) - curl.setopt(pycurl.HTTPHEADER, ['Content-Type: text/xml', 'Expect:']) - curl.setopt(pycurl.URL, "https://%s:%d%s"%(str(ip), int(xmlrpc_port), url)) + curl.setopt(pycurl.HTTPHEADER, ['Content-Type: text/xml', 'Expect:']) + curl.setopt(pycurl.URL, "https://%s:%d%s" % (str(ip), int(xmlrpc_port), url)) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) try: curl.perform() data = { 'status': curl.getinfo(pycurl.RESPONSE_CODE), - #'reason': response.reason, + # 'reason': response.reason, 'body': buffer.getvalue(), } curl.close() except Exception as e: raise NameError(str(e)) else: - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE curl.setopt(pycurl.HEADER, 0) - curl.setopt(pycurl.HTTPHEADER, ['Content-Type: text/xml', 'Expect:']) - curl_url = "http://%s:%d%s"%(str(ip), int(xmlrpc_port), url) + curl.setopt(pycurl.HTTPHEADER, ['Content-Type: text/xml', 'Expect:']) + curl_url = "http://%s:%d%s" % (str(ip), int(xmlrpc_port), url) curl.setopt(pycurl.URL, curl_url) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) try: @@ -226,8 +238,9 @@ raise NameError(str(e)) return data + def send_http_request(method, url, data, format, proto, ip, port, timeout=20): - headers = {"Accept": "application/%s"%format} + headers = {"Accept": "application/%s" % format} if format == 'json': params = json.dumps(data) else: @@ -251,15 +264,17 @@ httpClient.close() return data -#following is added by zhangjw, for avx using -def call_restapi(method, url, params, ip, restapi_port, restapi_username, restapi_password, proto='http', cookie='', save_cookie=False): - authorization = 'Basic %s'%(('%s:%s'%(restapi_username, restapi_password)).encode('base64').replace('\n','')) + +# following is added by zhangjw, for avx using +def call_restapi(method, url, params, ip, restapi_port, restapi_username, restapi_password, proto='http', cookie='', + save_cookie=False): + authorization = 'Basic %s' % (('%s:%s' % (restapi_username, restapi_password)).encode('base64').replace('\n', '')) if not check_ipv4(ip): ip = "[%s]" % ip buffer = StringIO() curl = pycurl.Curl() - curl.setopt(pycurl.VERBOSE, 1) #DEBUG MODE - curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s'%authorization, 'Expect: ']) + curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE + curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s' % authorization, 'Expect: ']) curl.setopt(pycurl.CONNECTTIMEOUT, 5) if method == 'POST': curl.setopt(pycurl.POST, 1) @@ -268,8 +283,8 @@ curl.setopt(pycurl.CUSTOMREQUEST, "PUT") curl.setopt(pycurl.POSTFIELDS, params) if save_cookie: - curl.setopt(pycurl.HEADER, 1) #write http header to response - curl.setopt(pycurl.URL, str("http://%s:%d%s"%(ip, restapi_port, url))) + curl.setopt(pycurl.HEADER, 1) # write http header to response + curl.setopt(pycurl.URL, str("http://%s:%d%s" % (ip, restapi_port, url))) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if cookie: curl.setopt(pycurl.COOKIE, cookie) @@ -290,13 +305,13 @@ curl.close() return data else: - curl.setopt(pycurl.HEADER, 0) #don't write http header to response + curl.setopt(pycurl.HEADER, 0) # don't write http header to response if proto == 'https': curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) - curl.setopt(pycurl.URL, str("https://%s:%d%s"%(ip, restapi_port, url))) + curl.setopt(pycurl.URL, str("https://%s:%d%s" % (ip, restapi_port, url))) else: - curl.setopt(pycurl.URL, str("http://%s:%d%s"%(ip, restapi_port, url))) + curl.setopt(pycurl.URL, str("http://%s:%d%s" % (ip, restapi_port, url))) curl.setopt(pycurl.WRITEFUNCTION, buffer.write) if cookie: curl.setopt(pycurl.COOKIE, cookie) @@ -306,4 +321,4 @@ 'body': buffer.getvalue(), } curl.close() - return data \ No newline at end of file + return data Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/composer.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/composer.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/composer.py (working copy) @@ -1,8 +1,9 @@ -import pycurl -import urllib import json from io import StringIO +import pycurl + + def get_composer_ui_data(method, composer_ui_url, post_data=None): buffer = StringIO() curl = pycurl.Curl() @@ -15,10 +16,10 @@ curl.setopt(pycurl.COOKIE, 'ComposerUICookie=f1f713c9e000f5d3f280adbd124df4f5') curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) - #curl.setopt(pycurl.VERBOSE, 1L) #DEBUG MODE - curl.setopt(pycurl.HEADER, 0) #don't write http header to response - curl.setopt(pycurl.URL, 'http://127.0.0.1:3000%s'%composer_ui_url) - #curl.setopt(pycurl.URL, 'http://10.8.11.253:3000%s'%composer_ui_url) + # curl.setopt(pycurl.VERBOSE, 1L) #DEBUG MODE + curl.setopt(pycurl.HEADER, 0) # don't write http header to response + curl.setopt(pycurl.URL, 'http://127.0.0.1:3000%s' % composer_ui_url) + # curl.setopt(pycurl.URL, 'http://10.8.11.253:3000%s'%composer_ui_url) curl.setopt(pycurl.WRITEDATA, buffer) curl.perform() data = { @@ -26,4 +27,4 @@ 'body': buffer.getvalue(), } curl.close() - return data \ No newline at end of file + return data Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/console.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/console.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/console.py (working copy) @@ -1,10 +1,16 @@ -#!/usr/bin/python -import array, time, glob, optparse, random, re -import socket, os, sys, pty, signal, select, gzip -import commands, threading, fcntl, termios, struct, pwd +import array +import fcntl import logging +import os +import pty +import select +import signal +import struct +import termios +import threading +import time + from djproject.an_settings import LEGACY_CLI_AGENT_IP -from hive.utils import andebug logger = logging.getLogger('hive.debug') @@ -13,1006 +19,1110 @@ except ImportError: import StringIO + class Terminal: - def __init__(self, w, h): - self.w = w - self.h = h - self.vt100_charset_graph = [ - 0x25ca, 0x2026, 0x2022, 0x3f, - 0xb6, 0x3f, 0xb0, 0xb1, - 0x3f, 0x3f, 0x2b, 0x2b, - 0x2b, 0x2b, 0x2b, 0xaf, - 0x2014, 0x2014, 0x2014, 0x5f, - 0x2b, 0x2b, 0x2b, 0x2b, - 0x7c, 0x2264, 0x2265, 0xb6, - 0x2260, 0xa3, 0xb7, 0x7f - ] - self.vt100_esc = { - '#8': self.esc_DECALN, - '(A': self.esc_G0_0, - '(B': self.esc_G0_1, - '(0': self.esc_G0_2, - '(1': self.esc_G0_3, - '(2': self.esc_G0_4, - ')A': self.esc_G1_0, - ')B': self.esc_G1_1, - ')0': self.esc_G1_2, - ')1': self.esc_G1_3, - ')2': self.esc_G1_4, - '7': self.esc_DECSC, - '8': self.esc_DECRC, - '=': self.esc_DECKPAM, - '>': self.esc_DECKPNM, - 'D': self.esc_IND, - 'E': self.esc_NEL, - 'H': self.esc_HTS, - 'M': self.esc_RI, - 'N': self.esc_SS2, - 'O': self.esc_SS3, - 'P': self.esc_DCS, - 'X': self.esc_SOS, - 'Z': self.esc_DECID, - '[': self.esc_CSI, - '\\': self.esc_ST, - ']': self.esc_OSC, - '^': self.esc_PM, - '_': self.esc_APC, - 'c': self.reset_hard, - } - self.vt100_csi = { - '@': self.csi_ICH, - 'A': self.csi_CUU, - 'B': self.csi_CUD, - 'C': self.csi_CUF, - 'D': self.csi_CUB, - 'E': self.csi_CNL, - 'F': self.csi_CPL, - 'G': self.csi_CHA, - 'H': self.csi_CUP, - 'I': self.csi_CHT, - 'J': self.csi_ED, - 'K': self.csi_EL, - 'L': self.csi_IL, - 'M': self.csi_DL, - 'P': self.csi_DCH, - 'S': self.csi_SU, - 'T': self.csi_SD, - 'W': self.csi_CTC, - 'X': self.csi_ECH, - 'Z': self.csi_CBT, - '`': self.csi_HPA, - 'a': self.csi_HPR, - 'b': self.csi_REP, - 'c': self.csi_DA, - 'd': self.csi_VPA, - 'e': self.csi_VPR, - 'f': self.csi_HVP, - 'g': self.csi_TBC, - 'h': self.csi_SM, - 'l': self.csi_RM, - 'm': self.csi_SGR, - 'n': self.csi_DSR, - 'r': self.csi_DECSTBM, - 's': self.csi_SCP, - 'u': self.csi_RCP, - 'x': self.csi_DECREQTPARM, - '!p': self.csi_DECSTR, - } - self.vt100_keyfilter_ansikeys = { - '~':'~', - 'A':'\x1b[A', - 'B':'\x1b[B', - 'C':'\x1b[C', - 'D':'\x1b[D', - 'F':'\x1b[F', - 'H':'\x1b[H', - '1':'\x1b[5~', - '2':'\x1b[6~', - '3':'\x1b[2~', - '4':'\x1b[3~', - 'a':'\x1bOP', - 'b':'\x1bOQ', - 'c':'\x1bOR', - 'd':'\x1bOS', - 'e':'\x1b[15~', - 'f':'\x1b[17~', - 'g':'\x1b[18~', - 'h':'\x1b[19~', - 'i':'\x1b[20~', - 'j':'\x1b[21~', - 'k':'\x1b[23~', - 'l':'\x1b[24~', - } - self.vt100_keyfilter_appkeys = { - '~':'~', - 'A':'\x1bOA', - 'B':'\x1bOB', - 'C':'\x1bOC', - 'D':'\x1bOD', - 'F':'\x1bOF', - 'H':'\x1bOH', - '1':'\x1b[5~', - '2':'\x1b[6~', - '3':'\x1b[2~', - '4':'\x1b[3~', - 'a':'\x1bOP', - 'b':'\x1bOQ', - 'c':'\x1bOR', - 'd':'\x1bOS', - 'e':'\x1b[15~', - 'f':'\x1b[17~', - 'g':'\x1b[18~', - 'h':'\x1b[19~', - 'i':'\x1b[20~', - 'j':'\x1b[21~', - 'k':'\x1b[23~', - 'l':'\x1b[24~', - } - self.reset_hard() + def __init__(self, w, h): + self.w = w + self.h = h + self.vt100_charset_graph = [ + 0x25ca, 0x2026, 0x2022, 0x3f, + 0xb6, 0x3f, 0xb0, 0xb1, + 0x3f, 0x3f, 0x2b, 0x2b, + 0x2b, 0x2b, 0x2b, 0xaf, + 0x2014, 0x2014, 0x2014, 0x5f, + 0x2b, 0x2b, 0x2b, 0x2b, + 0x7c, 0x2264, 0x2265, 0xb6, + 0x2260, 0xa3, 0xb7, 0x7f + ] + self.vt100_esc = { + '#8': self.esc_DECALN, + '(A': self.esc_G0_0, + '(B': self.esc_G0_1, + '(0': self.esc_G0_2, + '(1': self.esc_G0_3, + '(2': self.esc_G0_4, + ')A': self.esc_G1_0, + ')B': self.esc_G1_1, + ')0': self.esc_G1_2, + ')1': self.esc_G1_3, + ')2': self.esc_G1_4, + '7': self.esc_DECSC, + '8': self.esc_DECRC, + '=': self.esc_DECKPAM, + '>': self.esc_DECKPNM, + 'D': self.esc_IND, + 'E': self.esc_NEL, + 'H': self.esc_HTS, + 'M': self.esc_RI, + 'N': self.esc_SS2, + 'O': self.esc_SS3, + 'P': self.esc_DCS, + 'X': self.esc_SOS, + 'Z': self.esc_DECID, + '[': self.esc_CSI, + '\\': self.esc_ST, + ']': self.esc_OSC, + '^': self.esc_PM, + '_': self.esc_APC, + 'c': self.reset_hard, + } + self.vt100_csi = { + '@': self.csi_ICH, + 'A': self.csi_CUU, + 'B': self.csi_CUD, + 'C': self.csi_CUF, + 'D': self.csi_CUB, + 'E': self.csi_CNL, + 'F': self.csi_CPL, + 'G': self.csi_CHA, + 'H': self.csi_CUP, + 'I': self.csi_CHT, + 'J': self.csi_ED, + 'K': self.csi_EL, + 'L': self.csi_IL, + 'M': self.csi_DL, + 'P': self.csi_DCH, + 'S': self.csi_SU, + 'T': self.csi_SD, + 'W': self.csi_CTC, + 'X': self.csi_ECH, + 'Z': self.csi_CBT, + '`': self.csi_HPA, + 'a': self.csi_HPR, + 'b': self.csi_REP, + 'c': self.csi_DA, + 'd': self.csi_VPA, + 'e': self.csi_VPR, + 'f': self.csi_HVP, + 'g': self.csi_TBC, + 'h': self.csi_SM, + 'l': self.csi_RM, + 'm': self.csi_SGR, + 'n': self.csi_DSR, + 'r': self.csi_DECSTBM, + 's': self.csi_SCP, + 'u': self.csi_RCP, + 'x': self.csi_DECREQTPARM, + '!p': self.csi_DECSTR, + } + self.vt100_keyfilter_ansikeys = { + '~': '~', + 'A': '\x1b[A', + 'B': '\x1b[B', + 'C': '\x1b[C', + 'D': '\x1b[D', + 'F': '\x1b[F', + 'H': '\x1b[H', + '1': '\x1b[5~', + '2': '\x1b[6~', + '3': '\x1b[2~', + '4': '\x1b[3~', + 'a': '\x1bOP', + 'b': '\x1bOQ', + 'c': '\x1bOR', + 'd': '\x1bOS', + 'e': '\x1b[15~', + 'f': '\x1b[17~', + 'g': '\x1b[18~', + 'h': '\x1b[19~', + 'i': '\x1b[20~', + 'j': '\x1b[21~', + 'k': '\x1b[23~', + 'l': '\x1b[24~', + } + self.vt100_keyfilter_appkeys = { + '~': '~', + 'A': '\x1bOA', + 'B': '\x1bOB', + 'C': '\x1bOC', + 'D': '\x1bOD', + 'F': '\x1bOF', + 'H': '\x1bOH', + '1': '\x1b[5~', + '2': '\x1b[6~', + '3': '\x1b[2~', + '4': '\x1b[3~', + 'a': '\x1bOP', + 'b': '\x1bOQ', + 'c': '\x1bOR', + 'd': '\x1bOS', + 'e': '\x1b[15~', + 'f': '\x1b[17~', + 'g': '\x1b[18~', + 'h': '\x1b[19~', + 'i': '\x1b[20~', + 'j': '\x1b[21~', + 'k': '\x1b[23~', + 'l': '\x1b[24~', + } + self.reset_hard() + + # Reset functions + def reset_hard(self): + # Attribute mask: 0x0XFB0000 + # X: Bit 0 - Underlined + # Bit 1 - Negative + # Bit 2 - Concealed + # F: Foreground + # B: Background + self.attr = 0x00fe0000 + # UTF-8 decoder + self.utf8_units_count = 0 + self.utf8_units_received = 0 + self.utf8_char = 0 + # Key filter + self.vt100_keyfilter_escape = False + # Last char + self.vt100_lastchar = 0 + # Control sequences + self.vt100_parse_len = 0 + self.vt100_parse_state = "" + self.vt100_parse_func = "" + self.vt100_parse_param = "" + # Buffers + self.vt100_out = "" + # Caches + self.dump_cache = "" + # Invoke other resets + self.reset_screen() + self.reset_soft() + + def reset_soft(self): + # Attribute mask: 0x0XFB0000 + # X: Bit 0 - Underlined + # Bit 1 - Negative + # Bit 2 - Concealed + # F: Foreground + # B: Background + self.attr = 0x00fe0000 + # Scroll parameters + self.scroll_area_y0 = 0 + self.scroll_area_y1 = self.h + # Character sets + self.vt100_charset_is_single_shift = False + self.vt100_charset_is_graphical = False + self.vt100_charset_g_sel = 0 + self.vt100_charset_g = [0, 0] + # Modes + self.vt100_mode_insert = False + self.vt100_mode_lfnewline = False + self.vt100_mode_cursorkey = False + self.vt100_mode_column_switch = False + self.vt100_mode_inverse = False + self.vt100_mode_origin = False + self.vt100_mode_autowrap = True + self.vt100_mode_cursor = True + self.vt100_mode_alt_screen = False + self.vt100_mode_backspace = False + # Init DECSC state + self.esc_DECSC() + self.vt100_saved2 = self.vt100_saved + self.esc_DECSC() + + def reset_screen(self): + # Screen + self.screen = array.array('i', [self.attr | 0x20] * self.w * self.h) + self.screen2 = array.array('i', [self.attr | 0x20] * self.w * self.h) + # Scroll parameters + self.scroll_area_y0 = 0 + self.scroll_area_y1 = self.h + # Cursor position + self.cx = 0 + self.cy = 0 + # Tab stops + self.tab_stops = range(0, self.w, 8) + + # UTF-8 functions + def utf8_decode(self, d): + o = '' + for c in d: + char = ord(c) + if self.utf8_units_count != self.utf8_units_received: + self.utf8_units_received += 1 + if (char & 0xc0) == 0x80: + self.utf8_char = (self.utf8_char << 6) | (char & 0x3f) + if self.utf8_units_count == self.utf8_units_received: + if self.utf8_char < 0x10000: + o += chr(self.utf8_char) + self.utf8_units_count = self.utf8_units_received = 0 + else: + o += '?' + while self.utf8_units_received: + o += '?' + self.utf8_units_received -= 1 + self.utf8_units_count = 0 + else: + if (char & 0x80) == 0x00: + o += c + elif (char & 0xe0) == 0xc0: + self.utf8_units_count = 1 + self.utf8_char = char & 0x1f + elif (char & 0xf0) == 0xe0: + self.utf8_units_count = 2 + self.utf8_char = char & 0x0f + elif (char & 0xf8) == 0xf0: + self.utf8_units_count = 3 + self.utf8_char = char & 0x07 + else: + o += '?' + return o + + def utf8_charwidth(self, char): + if char >= 0x2e80: + return 2 + else: + return 1 + + # Low-level terminal functions + def peek(self, y0, x0, y1, x1): + return self.screen[self.w * y0 + x0:self.w * (y1 - 1) + x1] + + def poke(self, y, x, s): + pos = self.w * y + x + self.screen[pos:pos + len(s)] = s + + def fill(self, y0, x0, y1, x1, char): + n = self.w * (y1 - y0 - 1) + (x1 - x0) + self.poke(y0, x0, array.array('i', [char] * n)) + + def clear(self, y0, x0, y1, x1): + self.fill(y0, x0, y1, x1, self.attr | 0x20) + + # Scrolling functions + def scroll_area_up(self, y0, y1, n=1): + n = min(y1 - y0, n) + self.poke(y0, 0, self.peek(y0 + n, 0, y1, self.w)) + self.clear(y1 - n, 0, y1, self.w) + + def scroll_area_down(self, y0, y1, n=1): + n = min(y1 - y0, n) + self.poke(y0 + n, 0, self.peek(y0, 0, y1 - n, self.w)) + self.clear(y0, 0, y0 + n, self.w) + + def scroll_area_set(self, y0, y1): + y0 = max(0, min(self.h - 1, y0)) + y1 = max(1, min(self.h, y1)) + if y1 > y0: + self.scroll_area_y0 = y0 + self.scroll_area_y1 = y1 + + def scroll_line_right(self, y, x, n=1): + if x < self.w: + n = min(self.w - self.cx, n) + self.poke(y, x + n, self.peek(y, x, y + 1, self.w - n)) + self.clear(y, x, y + 1, x + n) + + def scroll_line_left(self, y, x, n=1): + if x < self.w: + n = min(self.w - self.cx, n) + self.poke(y, x, self.peek(y, x + n, y + 1, self.w)) + self.clear(y, self.w - n, y + 1, self.w) + + # Cursor functions + def cursor_line_width(self, next_char): + wx = self.utf8_charwidth(next_char) + lx = 0 + for x in range(min(self.cx, self.w)): + char = self.peek(self.cy, x, self.cy + 1, x + 1)[0] & 0xffff + wx += self.utf8_charwidth(char) + lx += 1 + return wx, lx + + def cursor_up(self, n=1): + self.cy = max(self.scroll_area_y0, self.cy - n) + + def cursor_down(self, n=1): + self.cy = min(self.scroll_area_y1 - 1, self.cy + n) + + def cursor_left(self, n=1): + self.cx = max(0, self.cx - n) + + def cursor_right(self, n=1): + self.cx = min(self.w - 1, self.cx + n) + + def cursor_set_x(self, x): + self.cx = max(0, x) + + def cursor_set_y(self, y): + self.cy = max(0, min(self.h - 1, y)) + + def cursor_set(self, y, x): + self.cursor_set_x(x) + self.cursor_set_y(y) + + # Dumb terminal + def ctrl_BS(self): + delta_y, cx = divmod(self.cx - 1, self.w) + cy = max(self.scroll_area_y0, self.cy + delta_y) + self.cursor_set(cy, cx) + + def ctrl_HT(self, n=1): + if n > 0 and self.cx >= self.w: + return + if n <= 0 and self.cx == 0: + return + ts = 0 + for i in range(len(self.tab_stops)): + if self.cx >= self.tab_stops[i]: + ts = i + ts += n + if ts < len(self.tab_stops) and ts >= 0: + self.cursor_set_x(self.tab_stops[ts]) + else: + self.cursor_set_x(self.w - 1) + + def ctrl_LF(self): + if self.vt100_mode_lfnewline: + self.ctrl_CR() + if self.cy == self.scroll_area_y1 - 1: + self.scroll_area_up(self.scroll_area_y0, self.scroll_area_y1) + else: + self.cursor_down() + + def ctrl_CR(self): + self.cursor_set_x(0) + + def dumb_write(self, char): + if char < 32: + if char == 8: + self.ctrl_BS() + elif char == 9: + self.ctrl_HT() + elif char >= 10 and char <= 12: + self.ctrl_LF() + elif char == 13: + self.ctrl_CR() + return True + return False + + def dumb_echo(self, char): + # Check right bound + wx, cx = self.cursor_line_width(char) + # Newline + if wx > self.w: + if self.vt100_mode_autowrap: + self.ctrl_CR() + self.ctrl_LF() + else: + self.cx = cx - 1 + if self.vt100_mode_insert: + self.scroll_line_right(self.cy, self.cx) + if self.vt100_charset_is_single_shift: + self.vt100_charset_is_single_shift = False + elif self.vt100_charset_is_graphical and (char & 0xffe0) == 0x0060: + char = self.vt100_charset_graph[char - 0x60] + self.poke(self.cy, self.cx, array.array('i', [self.attr | char])) + self.cursor_set_x(self.cx + 1) + + # VT100 CTRL, ESC, CSI handlers + def vt100_charset_update(self): + self.vt100_charset_is_graphical = ( + self.vt100_charset_g[self.vt100_charset_g_sel] == 2) + + def vt100_charset_set(self, g): + # Invoke active character set + self.vt100_charset_g_sel = g + self.vt100_charset_update() + + def vt100_charset_select(self, g, charset): + # Select charset + self.vt100_charset_g[g] = charset + self.vt100_charset_update() + + def vt100_setmode(self, p, state): + # Set VT100 mode + p = self.vt100_parse_params(p, [], False) + for m in p: + if m == '4': + # Insertion replacement mode + self.vt100_mode_insert = state + elif m == '20': + # Linefeed/new line mode + self.vt100_mode_lfnewline = state + elif m == '?1': + # Cursor key mode + self.vt100_mode_cursorkey = state + elif m == '?3': + # Column mode + if self.vt100_mode_column_switch: + if state: + self.w = 132 + else: + self.w = 80 + self.reset_screen() + elif m == '?5': + # Screen mode + self.vt100_mode_inverse = state + elif m == '?6': + # Region origin mode + self.vt100_mode_origin = state + if state: + self.cursor_set(self.scroll_area_y0, 0) + else: + self.cursor_set(0, 0) + elif m == '?7': + # Autowrap mode + self.vt100_mode_autowrap = state + elif m == '?25': + # Text cursor enable mode + self.vt100_mode_cursor = state + elif m == '?40': + # Column switch control + self.vt100_mode_column_switch = state + elif m == '?47': + # Alternate screen mode + if ((state and not self.vt100_mode_alt_screen) or + (not state and self.vt100_mode_alt_screen)): + self.screen, self.screen2 = self.screen2, self.screen + self.vt100_saved, self.vt100_saved2 = self.vt100_saved2, self.vt100_saved + self.vt100_mode_alt_screen = state + elif m == '?67': + # Backspace/delete + self.vt100_mode_backspace = state + + def ctrl_SO(self): + # Shift out + self.vt100_charset_set(1) + + def ctrl_SI(self): + # Shift in + self.vt100_charset_set(0) + + def esc_CSI(self): + # CSI start sequence + self.vt100_parse_reset('csi') + + def esc_DECALN(self): + # Screen alignment display + self.fill(0, 0, self.h, self.w, 0x00fe0045) + + def esc_G0_0(self): + self.vt100_charset_select(0, 0) + + def esc_G0_1(self): + self.vt100_charset_select(0, 1) + + def esc_G0_2(self): + self.vt100_charset_select(0, 2) + + def esc_G0_3(self): + self.vt100_charset_select(0, 3) + + def esc_G0_4(self): + self.vt100_charset_select(0, 4) + + def esc_G1_0(self): + self.vt100_charset_select(1, 0) + + def esc_G1_1(self): + self.vt100_charset_select(1, 1) + + def esc_G1_2(self): + self.vt100_charset_select(1, 2) + + def esc_G1_3(self): + self.vt100_charset_select(1, 3) + + def esc_G1_4(self): + self.vt100_charset_select(1, 4) + + def esc_DECSC(self): + # Store cursor + self.vt100_saved = {} + self.vt100_saved['cx'] = self.cx + self.vt100_saved['cy'] = self.cy + self.vt100_saved['attr'] = self.attr + self.vt100_saved['charset_g_sel'] = self.vt100_charset_g_sel + self.vt100_saved['charset_g'] = self.vt100_charset_g[:] + self.vt100_saved['mode_autowrap'] = self.vt100_mode_autowrap + self.vt100_saved['mode_origin'] = self.vt100_mode_origin + + def esc_DECRC(self): + # Retore cursor + self.cx = self.vt100_saved['cx'] + self.cy = self.vt100_saved['cy'] + self.attr = self.vt100_saved['attr'] + self.vt100_charset_g_sel = self.vt100_saved['charset_g_sel'] + self.vt100_charset_g = self.vt100_saved['charset_g'][:] + self.vt100_charset_update() + self.vt100_mode_autowrap = self.vt100_saved['mode_autowrap'] + self.vt100_mode_origin = self.vt100_saved['mode_origin'] + + def esc_DECKPAM(self): + # Application keypad mode + pass + + def esc_DECKPNM(self): + # Numeric keypad mode + pass + + def esc_IND(self): + # Index + self.ctrl_LF() + + def esc_NEL(self): + # Next line + self.ctrl_CR() + self.ctrl_LF() + + def esc_HTS(self): + # Character tabulation set + self.csi_CTC('0') + + def esc_RI(self): + # Reverse line feed + if self.cy == self.scroll_area_y0: + self.scroll_area_down(self.scroll_area_y0, self.scroll_area_y1) + else: + self.cursor_up() + + def esc_SS2(self): + # Single-shift two + self.vt100_charset_is_single_shift = True + + def esc_SS3(self): + # Single-shift three + self.vt100_charset_is_single_shift = True + + def esc_DCS(self): + # Device control string + self.vt100_parse_reset('str') + + def esc_SOS(self): + # Start of string + self.vt100_parse_reset('str') + + def esc_DECID(self): + # Identify terminal + self.csi_DA('0') + + def esc_ST(self): + # String terminator + pass + + def esc_OSC(self): + # Operating system command + self.vt100_parse_reset('str') + + def esc_PM(self): + # Privacy message + self.vt100_parse_reset('str') + + def esc_APC(self): + # Application program command + self.vt100_parse_reset('str') + + def csi_ICH(self, p): + # Insert character + p = self.vt100_parse_params(p, [1]) + self.scroll_line_right(self.cy, self.cx, p[0]) + + def csi_CUU(self, p): + # Cursor up + p = self.vt100_parse_params(p, [1]) + self.cursor_up(max(1, p[0])) + + def csi_CUD(self, p): + # Cursor down + p = self.vt100_parse_params(p, [1]) + self.cursor_down(max(1, p[0])) + + def csi_CUF(self, p): + # Cursor right + p = self.vt100_parse_params(p, [1]) + self.cursor_right(max(1, p[0])) + + def csi_CUB(self, p): + # Cursor left + p = self.vt100_parse_params(p, [1]) + self.cursor_left(max(1, p[0])) + + def csi_CNL(self, p): + # Cursor next line + self.csi_CUD(p) + self.ctrl_CR() + + def csi_CPL(self, p): + # Cursor preceding line + self.csi_CUU(p) + self.ctrl_CR() + + def csi_CHA(self, p): + # Cursor character absolute + p = self.vt100_parse_params(p, [1]) + self.cursor_set_x(p[0] - 1) + + def csi_CUP(self, p): + # Set cursor position + p = self.vt100_parse_params(p, [1, 1]) + if self.vt100_mode_origin: + self.cursor_set(self.scroll_area_y0 + p[0] - 1, p[1] - 1) + else: + self.cursor_set(p[0] - 1, p[1] - 1) + + def csi_CHT(self, p): + # Cursor forward tabulation + p = self.vt100_parse_params(p, [1]) + self.ctrl_HT(max(1, p[0])) + + def csi_ED(self, p): + # Erase in display + p = self.vt100_parse_params(p, ['0'], False) + if p[0] == '0': + self.clear(self.cy, self.cx, self.h, self.w) + elif p[0] == '1': + self.clear(0, 0, self.cy + 1, self.cx + 1) + elif p[0] == '2': + self.clear(0, 0, self.h, self.w) + + def csi_EL(self, p): + # Erase in line + p = self.vt100_parse_params(p, ['0'], False) + if p[0] == '0': + self.clear(self.cy, self.cx, self.cy + 1, self.w) + elif p[0] == '1': + self.clear(self.cy, 0, self.cy + 1, self.cx + 1) + elif p[0] == '2': + self.clear(self.cy, 0, self.cy + 1, self.w) + + def csi_IL(self, p): + # Insert line + p = self.vt100_parse_params(p, [1]) + if (self.cy >= self.scroll_area_y0 and self.cy < self.scroll_area_y1): + self.scroll_area_down(self.cy, self.scroll_area_y1, max(1, p[0])) + + def csi_DL(self, p): + # Delete line + p = self.vt100_parse_params(p, [1]) + if (self.cy >= self.scroll_area_y0 and self.cy < self.scroll_area_y1): + self.scroll_area_up(self.cy, self.scroll_area_y1, max(1, p[0])) + + def csi_DCH(self, p): + # Delete characters + p = self.vt100_parse_params(p, [1]) + self.scroll_line_left(self.cy, self.cx, max(1, p[0])) + + def csi_SU(self, p): + # Scroll up + p = self.vt100_parse_params(p, [1]) + self.scroll_area_up(self.scroll_area_y0, self.scroll_area_y1, max(1, p[0])) + + def csi_SD(self, p): + # Scroll down + p = self.vt100_parse_params(p, [1]) + self.scroll_area_down(self.scroll_area_y0, self.scroll_area_y1, max(1, p[0])) + + def csi_CTC(self, p): + # Cursor tabulation control + p = self.vt100_parse_params(p, ['0'], False) + for m in p: + if m == '0': + try: + ts = self.tab_stops.index(self.cx) + except ValueError: + tab_stops = self.tab_stops + tab_stops.append(self.cx) + tab_stops.sort() + self.tab_stops = tab_stops + elif m == '2': + try: + self.tab_stops.remove(self.cx) + except ValueError: + pass + elif m == '5': + self.tab_stops = [0] + + def csi_ECH(self, p): + # Erase character + p = self.vt100_parse_params(p, [1]) + n = min(self.w - self.cx, max(1, p[0])) + self.clear(self.cy, self.cx, self.cy + 1, self.cx + n); + + def csi_CBT(self, p): + # Cursor backward tabulation + p = self.vt100_parse_params(p, [1]) + self.ctrl_HT(1 - max(1, p[0])) + + def csi_HPA(self, p): + # Character position absolute + p = self.vt100_parse_params(p, [1]) + self.cursor_set_x(p[0] - 1) + + def csi_HPR(self, p): + # Character position forward + self.csi_CUF(p) + + def csi_REP(self, p): + # Repeat + p = self.vt100_parse_params(p, [1]) + if self.vt100_lastchar < 32: + return + n = min(2000, max(1, p[0])) + while n: + self.dumb_echo(self.vt100_lastchar) + n -= 1 + self.vt100_lastchar = 0 + + def csi_DA(self, p): + # Device attributes + p = self.vt100_parse_params(p, ['0'], False) + if p[0] == '0': + self.vt100_out = "\x1b[?1;2c" + elif p[0] == '>0' or p[0] == '>': + self.vt100_out = "\x1b[>0;184;0c" + + def csi_VPA(self, p): + # Line position absolute + p = self.vt100_parse_params(p, [1]) + self.cursor_set_y(p[0] - 1) + + def csi_VPR(self, p): + # Line position forward + self.csi_CUD(p) + + def csi_HVP(self, p): + # Character and line position + self.csi_CUP(p) + + def csi_TBC(self, p): + # Tabulation clear + p = self.vt100_parse_params(p, ['0'], False) + if p[0] == '0': + self.csi_CTC('2') + elif p[0] == '3': + self.csi_CTC('5') + + def csi_SM(self, p): + # Set mode + self.vt100_setmode(p, True) + + def csi_RM(self, p): + # Reset mode + self.vt100_setmode(p, False) + + def csi_SGR(self, p): + # Select graphic rendition + p = self.vt100_parse_params(p, [0]) + for m in p: + if m == 0: + # Reset + self.attr = 0x00fe0000 + elif m == 4: + # Underlined + self.attr |= 0x01000000 + elif m == 7: + # Negative + self.attr |= 0x02000000 + elif m == 8: + # Concealed + self.attr |= 0x04000000 + elif m == 24: + # Not underlined + self.attr &= 0x7eff0000 + elif m == 27: + # Positive + self.attr &= 0x7dff0000 + elif m == 28: + # Revealed + self.attr &= 0x7bff0000 + elif m >= 30 and m <= 37: + # Foreground + self.attr = (self.attr & 0x7f0f0000) | ((m - 30) << 20) + elif m == 39: + # Default fg color + self.attr = (self.attr & 0x7f0f0000) | 0x00f00000 + elif m >= 40 and m <= 47: + # Background + self.attr = (self.attr & 0x7ff00000) | ((m - 40) << 16) + elif m == 49: + # Default bg color + self.attr = (self.attr & 0x7ff00000) | 0x000e0000 + + def csi_DSR(self, p): + # Device status report + p = self.vt100_parse_params(p, ['0'], False) + if p[0] == '5': + self.vt100_out = "\x1b[0n" + elif p[0] == '6': + x = self.cx + 1 + y = self.cy + 1 + self.vt100_out = '\x1b[%d;%dR' % (y, x) + elif p[0] == '7': + self.vt100_out = 'WebShell' + elif p[0] == '8': + self.vt100_out = '1.0.0' + elif p[0] == '?6': + x = self.cx + 1 + y = self.cy + 1 + self.vt100_out = '\x1b[?%d;%dR' % (y, x) + elif p[0] == '?15': + self.vt100_out = '\x1b[?13n' + elif p[0] == '?25': + self.vt100_out = '\x1b[?20n' + elif p[0] == '?26': + self.vt100_out = '\x1b[?27;1n' + elif p[0] == '?53': + self.vt100_out = '\x1b[?53n' + + def csi_DECSTBM(self, p): + # Set top and bottom margins + p = self.vt100_parse_params(p, [1, self.h]) + self.scroll_area_set(p[0] - 1, p[1]) + if self.vt100_mode_origin: + self.cursor_set(self.scroll_area_y0, 0) + else: + self.cursor_set(0, 0) + + def csi_SCP(self, p): + # Save cursor position + self.vt100_saved_cx = self.cx + self.vt100_saved_cy = self.cy + + def csi_RCP(self, p): + # Restore cursor position + self.cx = self.vt100_saved_cx + self.cy = self.vt100_saved_cy + + def csi_DECREQTPARM(self, p): + # Request terminal parameters + p = self.vt100_parse_params(p, [], False) + if p[0] == '0': + self.vt100_out = "\x1b[2;1;1;112;112;1;0x" + elif p[0] == '1': + self.vt100_out = "\x1b[3;1;1;112;112;1;0x" + + def csi_DECSTR(self, p): + # Soft terminal reset + self.reset_soft() + + # VT100 Parser + def vt100_parse_params(self, p, d, to_int=True): + # Process parameters (params p with defaults d) + # Add prefix to all parameters + prefix = '' + if len(p) > 0: + if p[0] >= '<' and p[0] <= '?': + prefix = p[0] + p = p[1:] + p = p.split(';') + else: + p = '' + # Process parameters + n = max(len(p), len(d)) + o = [] + for i in range(n): + value_def = False + if i < len(p): + value = prefix + p[i] + value_def = True + if to_int: + try: + value = int(value) + except ValueError: + value_def = False + if (not value_def) and i < len(d): + value = d[i] + o.append(value) + return o + + def vt100_parse_reset(self, vt100_parse_state=""): + self.vt100_parse_state = vt100_parse_state + self.vt100_parse_len = 0 + self.vt100_parse_func = "" + self.vt100_parse_param = "" + + def vt100_parse_process(self): + if self.vt100_parse_state == 'esc': + # ESC mode + f = self.vt100_parse_func + try: + self.vt100_esc[f]() + except KeyError: + pass + if self.vt100_parse_state == 'esc': + self.vt100_parse_reset() + else: + # CSI mode + f = self.vt100_parse_func + p = self.vt100_parse_param + try: + self.vt100_csi[f](p) + except KeyError: + pass + if self.vt100_parse_state == 'csi': + self.vt100_parse_reset() + + def vt100_write(self, char): + if char < 32: + if char == 27: + self.vt100_parse_reset('esc') + return True + elif char == 14: + self.ctrl_SO() + elif char == 15: + self.ctrl_SI() + elif (char & 0xffe0) == 0x0080: + self.vt100_parse_reset('esc') + self.vt100_parse_func = chr(char - 0x40) + self.vt100_parse_process() + return True + + if self.vt100_parse_state: + if self.vt100_parse_state == 'str': + if char >= 32: + return True + self.vt100_parse_reset() + else: + if char < 32: + if char == 24 or char == 26: + self.vt100_parse_reset() + return True + else: + self.vt100_parse_len += 1 + if self.vt100_parse_len > 32: + self.vt100_parse_reset() + else: + char_msb = char & 0xf0 + if char_msb == 0x20: + # Intermediate bytes (added to function) + self.vt100_parse_func += chr(char) + elif char_msb == 0x30 and self.vt100_parse_state == 'csi': + # Parameter byte + self.vt100_parse_param += chr(char) + else: + # Function byte + self.vt100_parse_func += chr(char) + self.vt100_parse_process() + return True + self.vt100_lastchar = char + return False + + # External interface + def set_size(self, w, h): + if w < 2 or w > 256 or h < 2 or h > 256: + return False + self.w = w + self.h = h + # reset() + return True + + def read(self): + d = self.vt100_out + self.vt100_out = "" + return d + + def write(self, d): + d = self.utf8_decode(d) + for c in d: + char = ord(c) + if self.vt100_write(char): + continue + if self.dumb_write(char): + continue + if char <= 0xffff: + self.dumb_echo(char) + return True + + def pipe(self, d): + o = '' + for c in d: + char = ord(c) + if self.vt100_keyfilter_escape: + self.vt100_keyfilter_escape = False + try: + if self.vt100_mode_cursorkey: + o += self.vt100_keyfilter_appkeys[c] + else: + o += self.vt100_keyfilter_ansikeys[c] + except KeyError: + pass + elif c == '~': + self.vt100_keyfilter_escape = True + elif char == 127: + if self.vt100_mode_backspace: + o += chr(8) + else: + o += chr(127) + else: + o += c + if self.vt100_mode_lfnewline and char == 13: + o += chr(10) + return o - # Reset functions - def reset_hard(self): - # Attribute mask: 0x0XFB0000 - # X: Bit 0 - Underlined - # Bit 1 - Negative - # Bit 2 - Concealed - # F: Foreground - # B: Background - self.attr = 0x00fe0000 - # UTF-8 decoder - self.utf8_units_count = 0 - self.utf8_units_received = 0 - self.utf8_char = 0 - # Key filter - self.vt100_keyfilter_escape = False - # Last char - self.vt100_lastchar = 0 - # Control sequences - self.vt100_parse_len = 0 - self.vt100_parse_state = "" - self.vt100_parse_func = "" - self.vt100_parse_param = "" - # Buffers - self.vt100_out = "" - # Caches - self.dump_cache = "" - # Invoke other resets - self.reset_screen() - self.reset_soft() - def reset_soft(self): - # Attribute mask: 0x0XFB0000 - # X: Bit 0 - Underlined - # Bit 1 - Negative - # Bit 2 - Concealed - # F: Foreground - # B: Background - self.attr = 0x00fe0000 - # Scroll parameters - self.scroll_area_y0 = 0 - self.scroll_area_y1 = self.h - # Character sets - self.vt100_charset_is_single_shift = False - self.vt100_charset_is_graphical = False - self.vt100_charset_g_sel = 0 - self.vt100_charset_g = [0, 0] - # Modes - self.vt100_mode_insert = False - self.vt100_mode_lfnewline = False - self.vt100_mode_cursorkey = False - self.vt100_mode_column_switch = False - self.vt100_mode_inverse = False - self.vt100_mode_origin = False - self.vt100_mode_autowrap = True - self.vt100_mode_cursor = True - self.vt100_mode_alt_screen = False - self.vt100_mode_backspace = False - # Init DECSC state - self.esc_DECSC() - self.vt100_saved2 = self.vt100_saved - self.esc_DECSC() - def reset_screen(self): - # Screen - self.screen = array.array('i', [self.attr | 0x20] * self.w * self.h) - self.screen2 = array.array('i', [self.attr | 0x20] * self.w * self.h) - # Scroll parameters - self.scroll_area_y0 = 0 - self.scroll_area_y1 = self.h - # Cursor position - self.cx = 0 - self.cy = 0 - # Tab stops - self.tab_stops = range(0, self.w, 8) - - # UTF-8 functions - def utf8_decode(self, d): - o = '' - for c in d: - char = ord(c) - if self.utf8_units_count != self.utf8_units_received: - self.utf8_units_received += 1 - if (char & 0xc0) == 0x80: - self.utf8_char = (self.utf8_char << 6) | (char & 0x3f) - if self.utf8_units_count == self.utf8_units_received: - if self.utf8_char<0x10000: - o += chr(self.utf8_char) - self.utf8_units_count = self.utf8_units_received = 0 - else: - o += '?' - while self.utf8_units_received: - o += '?' - self.utf8_units_received -= 1 - self.utf8_units_count = 0 - else: - if (char & 0x80) == 0x00: - o += c - elif (char & 0xe0) == 0xc0: - self.utf8_units_count = 1 - self.utf8_char = char & 0x1f - elif (char & 0xf0) == 0xe0: - self.utf8_units_count = 2 - self.utf8_char = char & 0x0f - elif (char & 0xf8) == 0xf0: - self.utf8_units_count = 3 - self.utf8_char = char & 0x07 - else: - o += '?' - return o - def utf8_charwidth(self, char): - if char >= 0x2e80: - return 2 - else: - return 1 - - # Low-level terminal functions - def peek(self, y0, x0, y1, x1): - return self.screen[self.w * y0 + x0:self.w * (y1 - 1) + x1] - def poke(self, y, x, s): - pos = self.w * y + x - self.screen[pos:pos + len(s)] = s - def fill(self, y0, x0, y1, x1, char): - n = self.w * (y1 - y0 - 1) + (x1 - x0) - self.poke(y0, x0, array.array('i', [char] * n)) - def clear(self, y0, x0, y1, x1): - self.fill(y0, x0, y1, x1, self.attr | 0x20) - - # Scrolling functions - def scroll_area_up(self, y0, y1, n = 1): - n = min(y1-y0, n) - self.poke(y0, 0, self.peek(y0 + n, 0, y1, self.w)) - self.clear(y1-n, 0, y1, self.w) - def scroll_area_down(self, y0, y1, n = 1): - n = min(y1-y0, n) - self.poke(y0 + n, 0, self.peek(y0, 0, y1-n, self.w)) - self.clear(y0, 0, y0 + n, self.w) - def scroll_area_set(self, y0, y1): - y0 = max(0, min(self.h-1, y0)) - y1 = max(1, min(self.h, y1)) - if y1 > y0: - self.scroll_area_y0 = y0 - self.scroll_area_y1 = y1 - def scroll_line_right(self, y, x, n = 1): - if x < self.w: - n = min(self.w-self.cx, n) - self.poke(y, x + n, self.peek(y, x, y + 1, self.w - n)) - self.clear(y, x, y + 1, x + n) - def scroll_line_left(self, y, x, n = 1): - if x < self.w: - n = min(self.w - self.cx, n) - self.poke(y, x, self.peek(y, x + n, y + 1, self.w)) - self.clear(y, self.w - n, y + 1, self.w) + def dump(self): + dump = u"" + attr_ = -1 + cx, cy = min(self.cx, self.w - 1), self.cy + for y in range(0, self.h): + wx = 0 + for x in range(0, self.w): + d = self.screen[y * self.w + x] + char = d & 0xffff + attr = d >> 16 + # Cursor + if cy == y and cx == x and self.vt100_mode_cursor: + attr = attr & 0xfff0 | 0x000c + # Attributes + if attr != attr_: + if attr_ != -1: + dump += u'' + bg = attr & 0x000f + fg = (attr & 0x00f0) >> 4 + # Inverse + inv = attr & 0x0200 + inv2 = self.vt100_mode_inverse + if (inv and not inv2) or (inv2 and not inv): + fg, bg = bg, fg + # Concealed + if attr & 0x0400: + fg = 0xc + # Underline + if attr & 0x0100: + ul = ' ul' + else: + ul = '' + dump += u'' % (fg, bg, ul) + attr_ = attr + # Escape HTML characters + if char == 38: + dump += '&' + elif char == 60: + dump += '<' + elif char == 62: + dump += '>' + else: + wx += self.utf8_charwidth(char) + if wx <= self.w: + dump += chr(char) + dump += "\n" + # Encode in UTF-8 + dump = dump.encode('utf-8') + dump += '' + # Cache dump + if self.dump_cache == dump: + return '' + else: + self.dump_cache = dump + return '' % cy + dump - # Cursor functions - def cursor_line_width(self, next_char): - wx = self.utf8_charwidth(next_char) - lx = 0 - for x in range(min(self.cx, self.w)): - char = self.peek(self.cy, x, self.cy + 1, x + 1)[0] & 0xffff - wx += self.utf8_charwidth(char) - lx += 1 - return wx, lx - def cursor_up(self, n = 1): - self.cy = max(self.scroll_area_y0, self.cy - n) - def cursor_down(self, n = 1): - self.cy = min(self.scroll_area_y1 - 1, self.cy + n) - def cursor_left(self, n = 1): - self.cx = max(0, self.cx - n) - def cursor_right(self, n = 1): - self.cx = min(self.w - 1, self.cx + n) - def cursor_set_x(self, x): - self.cx = max(0, x) - def cursor_set_y(self, y): - self.cy = max(0, min(self.h - 1, y)) - def cursor_set(self, y, x): - self.cursor_set_x(x) - self.cursor_set_y(y) - - # Dumb terminal - def ctrl_BS(self): - delta_y, cx = divmod(self.cx - 1, self.w) - cy = max(self.scroll_area_y0, self.cy + delta_y) - self.cursor_set(cy, cx) - def ctrl_HT(self, n = 1): - if n > 0 and self.cx >= self.w: - return - if n <= 0 and self.cx == 0: - return - ts = 0 - for i in range(len(self.tab_stops)): - if self.cx >= self.tab_stops[i]: - ts = i - ts += n - if ts < len(self.tab_stops) and ts >= 0: - self.cursor_set_x(self.tab_stops[ts]) - else: - self.cursor_set_x(self.w - 1) - def ctrl_LF(self): - if self.vt100_mode_lfnewline: - self.ctrl_CR() - if self.cy == self.scroll_area_y1 - 1: - self.scroll_area_up(self.scroll_area_y0, self.scroll_area_y1) - else: - self.cursor_down() - def ctrl_CR(self): - self.cursor_set_x(0) - def dumb_write(self, char): - if char < 32: - if char == 8: - self.ctrl_BS() - elif char == 9: - self.ctrl_HT() - elif char >= 10 and char <= 12: - self.ctrl_LF() - elif char == 13: - self.ctrl_CR() - return True - return False - def dumb_echo(self, char): - # Check right bound - wx, cx = self.cursor_line_width(char) - # Newline - if wx > self.w: - if self.vt100_mode_autowrap: - self.ctrl_CR() - self.ctrl_LF() - else: - self.cx = cx - 1 - if self.vt100_mode_insert: - self.scroll_line_right(self.cy, self.cx) - if self.vt100_charset_is_single_shift: - self.vt100_charset_is_single_shift = False - elif self.vt100_charset_is_graphical and (char & 0xffe0) == 0x0060: - char = self.vt100_charset_graph[char - 0x60] - self.poke(self.cy, self.cx, array.array('i', [self.attr | char])) - self.cursor_set_x(self.cx + 1) - - # VT100 CTRL, ESC, CSI handlers - def vt100_charset_update(self): - self.vt100_charset_is_graphical = ( - self.vt100_charset_g[self.vt100_charset_g_sel] == 2) - def vt100_charset_set(self, g): - # Invoke active character set - self.vt100_charset_g_sel = g - self.vt100_charset_update() - def vt100_charset_select(self, g, charset): - # Select charset - self.vt100_charset_g[g] = charset - self.vt100_charset_update() - def vt100_setmode(self, p, state): - # Set VT100 mode - p = self.vt100_parse_params(p, [], False) - for m in p: - if m == '4': - # Insertion replacement mode - self.vt100_mode_insert = state - elif m == '20': - # Linefeed/new line mode - self.vt100_mode_lfnewline = state - elif m == '?1': - # Cursor key mode - self.vt100_mode_cursorkey = state - elif m == '?3': - # Column mode - if self.vt100_mode_column_switch: - if state: - self.w = 132 - else: - self.w = 80 - self.reset_screen() - elif m == '?5': - # Screen mode - self.vt100_mode_inverse = state - elif m == '?6': - # Region origin mode - self.vt100_mode_origin = state - if state: - self.cursor_set(self.scroll_area_y0, 0) - else: - self.cursor_set(0, 0) - elif m == '?7': - # Autowrap mode - self.vt100_mode_autowrap = state - elif m == '?25': - # Text cursor enable mode - self.vt100_mode_cursor = state - elif m == '?40': - # Column switch control - self.vt100_mode_column_switch = state - elif m == '?47': - # Alternate screen mode - if ((state and not self.vt100_mode_alt_screen) or - (not state and self.vt100_mode_alt_screen)): - self.screen, self.screen2 = self.screen2, self.screen - self.vt100_saved, self.vt100_saved2 = self.vt100_saved2, self.vt100_saved - self.vt100_mode_alt_screen = state - elif m == '?67': - # Backspace/delete - self.vt100_mode_backspace = state - def ctrl_SO(self): - # Shift out - self.vt100_charset_set(1) - def ctrl_SI(self): - # Shift in - self.vt100_charset_set(0) - def esc_CSI(self): - # CSI start sequence - self.vt100_parse_reset('csi') - def esc_DECALN(self): - # Screen alignment display - self.fill(0, 0, self.h, self.w, 0x00fe0045) - def esc_G0_0(self): - self.vt100_charset_select(0, 0) - def esc_G0_1(self): - self.vt100_charset_select(0, 1) - def esc_G0_2(self): - self.vt100_charset_select(0, 2) - def esc_G0_3(self): - self.vt100_charset_select(0, 3) - def esc_G0_4(self): - self.vt100_charset_select(0, 4) - def esc_G1_0(self): - self.vt100_charset_select(1, 0) - def esc_G1_1(self): - self.vt100_charset_select(1, 1) - def esc_G1_2(self): - self.vt100_charset_select(1, 2) - def esc_G1_3(self): - self.vt100_charset_select(1, 3) - def esc_G1_4(self): - self.vt100_charset_select(1, 4) - def esc_DECSC(self): - # Store cursor - self.vt100_saved = {} - self.vt100_saved['cx'] = self.cx - self.vt100_saved['cy'] = self.cy - self.vt100_saved['attr'] = self.attr - self.vt100_saved['charset_g_sel'] = self.vt100_charset_g_sel - self.vt100_saved['charset_g'] = self.vt100_charset_g[:] - self.vt100_saved['mode_autowrap'] = self.vt100_mode_autowrap - self.vt100_saved['mode_origin'] = self.vt100_mode_origin - def esc_DECRC(self): - # Retore cursor - self.cx = self.vt100_saved['cx'] - self.cy = self.vt100_saved['cy'] - self.attr = self.vt100_saved['attr'] - self.vt100_charset_g_sel = self.vt100_saved['charset_g_sel'] - self.vt100_charset_g = self.vt100_saved['charset_g'][:] - self.vt100_charset_update() - self.vt100_mode_autowrap = self.vt100_saved['mode_autowrap'] - self.vt100_mode_origin = self.vt100_saved['mode_origin'] - def esc_DECKPAM(self): - # Application keypad mode - pass - def esc_DECKPNM(self): - # Numeric keypad mode - pass - def esc_IND(self): - # Index - self.ctrl_LF() - def esc_NEL(self): - # Next line - self.ctrl_CR() - self.ctrl_LF() - def esc_HTS(self): - # Character tabulation set - self.csi_CTC('0') - def esc_RI(self): - # Reverse line feed - if self.cy == self.scroll_area_y0: - self.scroll_area_down(self.scroll_area_y0, self.scroll_area_y1) - else: - self.cursor_up() - def esc_SS2(self): - # Single-shift two - self.vt100_charset_is_single_shift = True - def esc_SS3(self): - # Single-shift three - self.vt100_charset_is_single_shift = True - def esc_DCS(self): - # Device control string - self.vt100_parse_reset('str') - def esc_SOS(self): - # Start of string - self.vt100_parse_reset('str') - def esc_DECID(self): - # Identify terminal - self.csi_DA('0') - def esc_ST(self): - # String terminator - pass - def esc_OSC(self): - # Operating system command - self.vt100_parse_reset('str') - def esc_PM(self): - # Privacy message - self.vt100_parse_reset('str') - def esc_APC(self): - # Application program command - self.vt100_parse_reset('str') - def csi_ICH(self, p): - # Insert character - p = self.vt100_parse_params(p, [1]) - self.scroll_line_right(self.cy, self.cx, p[0]) - def csi_CUU(self, p): - # Cursor up - p = self.vt100_parse_params(p, [1]) - self.cursor_up(max(1, p[0])) - def csi_CUD(self, p): - # Cursor down - p = self.vt100_parse_params(p, [1]) - self.cursor_down(max(1, p[0])) - def csi_CUF(self, p): - # Cursor right - p = self.vt100_parse_params(p, [1]) - self.cursor_right(max(1, p[0])) - def csi_CUB(self, p): - # Cursor left - p = self.vt100_parse_params(p, [1]) - self.cursor_left(max(1, p[0])) - def csi_CNL(self, p): - # Cursor next line - self.csi_CUD(p) - self.ctrl_CR() - def csi_CPL(self, p): - # Cursor preceding line - self.csi_CUU(p) - self.ctrl_CR() - def csi_CHA(self, p): - # Cursor character absolute - p = self.vt100_parse_params(p, [1]) - self.cursor_set_x(p[0] - 1) - def csi_CUP(self, p): - # Set cursor position - p = self.vt100_parse_params(p, [1, 1]) - if self.vt100_mode_origin: - self.cursor_set(self.scroll_area_y0 + p[0] - 1, p[1] - 1) - else: - self.cursor_set(p[0] - 1, p[1] - 1) - def csi_CHT(self, p): - # Cursor forward tabulation - p = self.vt100_parse_params(p, [1]) - self.ctrl_HT(max(1, p[0])) - def csi_ED(self, p): - # Erase in display - p = self.vt100_parse_params(p, ['0'], False) - if p[0] == '0': - self.clear(self.cy, self.cx, self.h, self.w) - elif p[0] == '1': - self.clear(0, 0, self.cy + 1, self.cx + 1) - elif p[0] == '2': - self.clear(0, 0, self.h, self.w) - def csi_EL(self, p): - # Erase in line - p = self.vt100_parse_params(p, ['0'], False) - if p[0] == '0': - self.clear(self.cy, self.cx, self.cy + 1, self.w) - elif p[0] == '1': - self.clear(self.cy, 0, self.cy + 1, self.cx + 1) - elif p[0] == '2': - self.clear(self.cy, 0, self.cy + 1, self.w) - def csi_IL(self, p): - # Insert line - p = self.vt100_parse_params(p, [1]) - if (self.cy >= self.scroll_area_y0 and self.cy < self.scroll_area_y1): - self.scroll_area_down(self.cy, self.scroll_area_y1, max(1, p[0])) - def csi_DL(self, p): - # Delete line - p = self.vt100_parse_params(p, [1]) - if (self.cy >= self.scroll_area_y0 and self.cy < self.scroll_area_y1): - self.scroll_area_up(self.cy, self.scroll_area_y1, max(1, p[0])) - def csi_DCH(self, p): - # Delete characters - p = self.vt100_parse_params(p, [1]) - self.scroll_line_left(self.cy, self.cx, max(1, p[0])) - def csi_SU(self, p): - # Scroll up - p = self.vt100_parse_params(p, [1]) - self.scroll_area_up(self.scroll_area_y0, self.scroll_area_y1, max(1, p[0])) - def csi_SD(self, p): - # Scroll down - p = self.vt100_parse_params(p, [1]) - self.scroll_area_down(self.scroll_area_y0, self.scroll_area_y1, max(1, p[0])) - def csi_CTC(self, p): - # Cursor tabulation control - p = self.vt100_parse_params(p, ['0'], False) - for m in p: - if m == '0': - try: - ts = self.tab_stops.index(self.cx) - except ValueError: - tab_stops = self.tab_stops - tab_stops.append(self.cx) - tab_stops.sort() - self.tab_stops = tab_stops - elif m == '2': - try: - self.tab_stops.remove(self.cx) - except ValueError: - pass - elif m == '5': - self.tab_stops = [0] - def csi_ECH(self, p): - # Erase character - p = self.vt100_parse_params(p, [1]) - n = min(self.w - self.cx, max(1, p[0])) - self.clear(self.cy, self.cx, self.cy + 1, self.cx + n); - def csi_CBT(self, p): - # Cursor backward tabulation - p = self.vt100_parse_params(p, [1]) - self.ctrl_HT(1 - max(1, p[0])) - def csi_HPA(self, p): - # Character position absolute - p = self.vt100_parse_params(p, [1]) - self.cursor_set_x(p[0] - 1) - def csi_HPR(self, p): - # Character position forward - self.csi_CUF(p) - def csi_REP(self, p): - # Repeat - p = self.vt100_parse_params(p, [1]) - if self.vt100_lastchar < 32: - return - n = min(2000, max(1, p[0])) - while n: - self.dumb_echo(self.vt100_lastchar) - n -= 1 - self.vt100_lastchar = 0 - def csi_DA(self, p): - # Device attributes - p = self.vt100_parse_params(p, ['0'], False) - if p[0] == '0': - self.vt100_out = "\x1b[?1;2c" - elif p[0] == '>0' or p[0] == '>': - self.vt100_out = "\x1b[>0;184;0c" - def csi_VPA(self, p): - # Line position absolute - p = self.vt100_parse_params(p, [1]) - self.cursor_set_y(p[0] - 1) - def csi_VPR(self, p): - # Line position forward - self.csi_CUD(p) - def csi_HVP(self, p): - # Character and line position - self.csi_CUP(p) - def csi_TBC(self, p): - # Tabulation clear - p = self.vt100_parse_params(p, ['0'], False) - if p[0] == '0': - self.csi_CTC('2') - elif p[0] == '3': - self.csi_CTC('5') - def csi_SM(self, p): - # Set mode - self.vt100_setmode(p, True) - def csi_RM(self, p): - # Reset mode - self.vt100_setmode(p, False) - def csi_SGR(self, p): - # Select graphic rendition - p = self.vt100_parse_params(p, [0]) - for m in p: - if m == 0: - # Reset - self.attr = 0x00fe0000 - elif m == 4: - # Underlined - self.attr |= 0x01000000 - elif m == 7: - # Negative - self.attr |= 0x02000000 - elif m == 8: - # Concealed - self.attr |= 0x04000000 - elif m == 24: - # Not underlined - self.attr &= 0x7eff0000 - elif m == 27: - # Positive - self.attr &= 0x7dff0000 - elif m == 28: - # Revealed - self.attr &= 0x7bff0000 - elif m >= 30 and m <= 37: - # Foreground - self.attr = (self.attr & 0x7f0f0000) | ((m - 30) << 20) - elif m == 39: - # Default fg color - self.attr = (self.attr & 0x7f0f0000) | 0x00f00000 - elif m >= 40 and m <= 47: - # Background - self.attr = (self.attr & 0x7ff00000) | ((m - 40) << 16) - elif m == 49: - # Default bg color - self.attr = (self.attr & 0x7ff00000) | 0x000e0000 - def csi_DSR(self, p): - # Device status report - p = self.vt100_parse_params(p, ['0'], False) - if p[0] == '5': - self.vt100_out = "\x1b[0n" - elif p[0] == '6': - x = self.cx + 1 - y = self.cy + 1 - self.vt100_out = '\x1b[%d;%dR' % (y, x) - elif p[0] == '7': - self.vt100_out = 'WebShell' - elif p[0] == '8': - self.vt100_out = '1.0.0' - elif p[0] == '?6': - x = self.cx + 1 - y = self.cy + 1 - self.vt100_out = '\x1b[?%d;%dR' % (y, x) - elif p[0] == '?15': - self.vt100_out = '\x1b[?13n' - elif p[0] == '?25': - self.vt100_out = '\x1b[?20n' - elif p[0] == '?26': - self.vt100_out = '\x1b[?27;1n' - elif p[0] == '?53': - self.vt100_out = '\x1b[?53n' - def csi_DECSTBM(self, p): - # Set top and bottom margins - p = self.vt100_parse_params(p, [1, self.h]) - self.scroll_area_set(p[0] - 1, p[1]) - if self.vt100_mode_origin: - self.cursor_set(self.scroll_area_y0, 0) - else: - self.cursor_set(0, 0) - def csi_SCP(self, p): - # Save cursor position - self.vt100_saved_cx = self.cx - self.vt100_saved_cy = self.cy - def csi_RCP(self, p): - # Restore cursor position - self.cx = self.vt100_saved_cx - self.cy = self.vt100_saved_cy - def csi_DECREQTPARM(self, p): - # Request terminal parameters - p = self.vt100_parse_params(p, [], False) - if p[0] == '0': - self.vt100_out = "\x1b[2;1;1;112;112;1;0x" - elif p[0] == '1': - self.vt100_out = "\x1b[3;1;1;112;112;1;0x" - def csi_DECSTR(self, p): - # Soft terminal reset - self.reset_soft() - - # VT100 Parser - def vt100_parse_params(self, p, d, to_int = True): - # Process parameters (params p with defaults d) - # Add prefix to all parameters - prefix = '' - if len(p) > 0: - if p[0] >= '<' and p[0] <= '?': - prefix = p[0] - p = p[1:] - p = p.split(';') - else: - p = '' - # Process parameters - n = max(len(p), len(d)) - o = [] - for i in range(n): - value_def = False - if i < len(p): - value = prefix + p[i] - value_def = True - if to_int: - try: - value = int(value) - except ValueError: - value_def = False - if (not value_def) and i < len(d): - value = d[i] - o.append(value) - return o - def vt100_parse_reset(self, vt100_parse_state = ""): - self.vt100_parse_state = vt100_parse_state - self.vt100_parse_len = 0 - self.vt100_parse_func = "" - self.vt100_parse_param = "" - def vt100_parse_process(self): - if self.vt100_parse_state == 'esc': - # ESC mode - f = self.vt100_parse_func - try: - self.vt100_esc[f]() - except KeyError: - pass - if self.vt100_parse_state == 'esc': - self.vt100_parse_reset() - else: - # CSI mode - f = self.vt100_parse_func - p = self.vt100_parse_param - try: - self.vt100_csi[f](p) - except KeyError: - pass - if self.vt100_parse_state == 'csi': - self.vt100_parse_reset() - def vt100_write(self, char): - if char < 32: - if char == 27: - self.vt100_parse_reset('esc') - return True - elif char == 14: - self.ctrl_SO() - elif char == 15: - self.ctrl_SI() - elif (char & 0xffe0) == 0x0080: - self.vt100_parse_reset('esc') - self.vt100_parse_func = chr(char - 0x40) - self.vt100_parse_process() - return True - if self.vt100_parse_state: - if self.vt100_parse_state == 'str': - if char >= 32: - return True - self.vt100_parse_reset() - else: - if char < 32: - if char == 24 or char == 26: - self.vt100_parse_reset() - return True - else: - self.vt100_parse_len += 1 - if self.vt100_parse_len > 32: - self.vt100_parse_reset() - else: - char_msb = char & 0xf0 - if char_msb == 0x20: - # Intermediate bytes (added to function) - self.vt100_parse_func += chr(char) - elif char_msb == 0x30 and self.vt100_parse_state == 'csi': - # Parameter byte - self.vt100_parse_param += chr(char) - else: - # Function byte - self.vt100_parse_func += chr(char) - self.vt100_parse_process() - return True - self.vt100_lastchar = char - return False +class SynchronizedMethod: + def __init__(self, lock, orig): + self.lock = lock + self.orig = orig - # External interface - def set_size(self, w, h): - if w < 2 or w > 256 or h < 2 or h > 256: - return False - self.w = w - self.h = h - # reset() - return True - def read(self): - d = self.vt100_out - self.vt100_out = "" - return d - def write(self, d): - d = self.utf8_decode(d) - for c in d: - char = ord(c) - if self.vt100_write(char): - continue - if self.dumb_write(char): - continue - if char <= 0xffff: - self.dumb_echo(char) - return True - def pipe(self, d): - o = '' - for c in d: - char = ord(c) - if self.vt100_keyfilter_escape: - self.vt100_keyfilter_escape = False - try: - if self.vt100_mode_cursorkey: - o += self.vt100_keyfilter_appkeys[c] - else: - o += self.vt100_keyfilter_ansikeys[c] - except KeyError: - pass - elif c == '~': - self.vt100_keyfilter_escape = True - elif char == 127: - if self.vt100_mode_backspace: - o += chr(8) - else: - o += chr(127) - else: - o += c - if self.vt100_mode_lfnewline and char == 13: - o += chr(10) - return o - def dump(self): - dump = u"" - attr_ = -1 - cx, cy = min(self.cx, self.w - 1), self.cy - for y in range(0, self.h): - wx = 0 - for x in range(0, self.w): - d = self.screen[y * self.w + x] - char = d & 0xffff - attr = d >> 16 - # Cursor - if cy == y and cx == x and self.vt100_mode_cursor: - attr = attr & 0xfff0 | 0x000c - # Attributes - if attr != attr_: - if attr_ != -1: - dump += u'' - bg = attr & 0x000f - fg = (attr & 0x00f0) >> 4 - # Inverse - inv = attr & 0x0200 - inv2 = self.vt100_mode_inverse - if (inv and not inv2) or (inv2 and not inv): - fg, bg = bg, fg - # Concealed - if attr & 0x0400: - fg = 0xc - # Underline - if attr & 0x0100: - ul = ' ul' - else: - ul = '' - dump += u'' % (fg, bg, ul) - attr_ = attr - # Escape HTML characters - if char == 38: - dump += '&' - elif char == 60: - dump += '<' - elif char == 62: - dump += '>' - else: - wx += self.utf8_charwidth(char) - if wx <= self.w: - dump += chr(char) - dump += "\n" - # Encode in UTF-8 - dump = dump.encode('utf-8') - dump += '' - # Cache dump - if self.dump_cache == dump: - return '' - else: - self.dump_cache = dump - return '' % cy + dump + def __call__(self, *l): + self.lock.acquire() + try: + r = self.orig(*l) + finally: + self.lock.release() + pass + return r -class SynchronizedMethod: - def __init__(self, lock, orig): - self.lock = lock - self.orig = orig - def __call__(self, *l): - self.lock.acquire() - try: - r = self.orig(*l) - finally: - self.lock.release() - pass - return r class Multiplex: - def __init__(self, env_term = 'xterm-color'): + def __init__(self, env_term='xterm-color'): self.session = {} self.env_term = env_term # Synchronize methods @@ -1020,28 +1130,30 @@ for name in ['proc_keepalive', 'proc_read', 'proc_write', 'proc_dump']: orig = getattr(self, name) setattr(self, name, SynchronizedMethod(self.lock, orig)) - self.thread = threading.Thread(target = self.poll_proc_thread) + self.thread = threading.Thread(target=self.poll_proc_thread) self.thread.start() + def proc_keepalive(self, sid, w, h): # Update terminal size if self.session[sid]['w'] != w or self.session[sid]['h'] != h: try: - fcntl.ioctl(self.session[sid]['fd'], - struct.unpack('i', struct.pack('I', termios.TIOCSWINSZ) )[0], - struct.pack("HHHH", h, w, 0, 0) ) + fcntl.ioctl(self.session[sid]['fd'], + struct.unpack('i', struct.pack('I', termios.TIOCSWINSZ))[0], + struct.pack("HHHH", h, w, 0, 0)) except (IOError, OSError) as e: - logger.error('TIOCSWINSZ error: ' + str(e) ) + logger.error('TIOCSWINSZ error: ' + str(e)) pass self.session[sid]['term'].set_size(w, h) self.session[sid]['w'] = w self.session[sid]['h'] = h return True + def proc_spawn(self, sid, username, password, w, h, ip): # Fork new process try: pid, fd = pty.fork() except (IOError, OSError) as e: - logger.error('fork error: ' + str(e) ) + logger.error('fork error: ' + str(e)) return False if pid == 0: # Safe way to make it work under BSD and Linux @@ -1065,11 +1177,11 @@ os.putenv('PATH', os.environ['PATH']) os.putenv('LANG', ls[0] + '.UTF-8') os.execv(command[0], command) - + except (IOError, OSError): pass os._exit(0) - else: + else: while True: output = os.read(fd, 1024).strip() logger.info(output) @@ -1081,7 +1193,7 @@ elif 'are you sure you want to continue connecting (yes/no)?' in lower: os.write(fd, 'yes\n') elif 'company privacy warning' in lower: - pass # This is an understood message + pass # This is an understood message elif 'warning: rsa key found for host' in lower: os.write(fd, 'yes\n') elif 'warning: permanently added' in lower: @@ -1108,24 +1220,26 @@ self.session[sid]['fd'] = fd fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK) try: - fcntl.ioctl(fd, - struct.unpack('i', struct.pack('I', termios.TIOCSWINSZ) )[0], - struct.pack("HHHH", h, w, 0, 0)) + fcntl.ioctl(fd, + struct.unpack('i', struct.pack('I', termios.TIOCSWINSZ))[0], + struct.pack("HHHH", h, w, 0, 0)) except (IOError, OSError) as e: - logger.error('TIOCSWINSZ exception: ' + str(e) ) + logger.error('TIOCSWINSZ exception: ' + str(e)) pass return True + def proc_waitfordeath(self, sid): try: os.close(self.session[sid]['fd']) except (KeyError, IOError, OSError) as e: - logger.info('proc_waitfordeath close exception: ' + str(e) ) + logger.info('proc_waitfordeath close exception: ' + str(e)) pass try: os.waitpid(self.session[sid]['pid'], 0) except (KeyError, IOError, OSError): - pass + pass return True + # Read from process def proc_read(self, sid): if sid not in self.session: @@ -1153,9 +1267,10 @@ try: os.write(fd, d) except (IOError, OSError) as e: - logger.error('Read response error: ' + str(e) ) + logger.error('Read response error: ' + str(e)) return False return True + # Write to process def proc_write(self, sid, d): if sid not in self.session: @@ -1166,15 +1281,17 @@ fd = self.session[sid]['fd'] os.write(fd, d) except (IOError, OSError) as e: - logger.error('proc_write error: ' + str(e) ) + logger.error('proc_write error: ' + str(e)) return False return True + # Dump terminal output def proc_dump(self, sid): if sid not in self.session: - logger.error('proc_dump session missed: ' + sid ) + logger.error('proc_dump session missed: ' + sid) return False return self.session[sid]['term'].dump() + # Poll and read all process's outputs def poll_proc_thread(self): while True: @@ -1186,10 +1303,10 @@ try: i, o, e = select.select(fds, [], [], 1.0) except (IOError, OSError) as e: - logger.error('poll_proc_thread exception: ' + str(e) ) + logger.error('poll_proc_thread exception: ' + str(e)) i = [] except Exception as e: - logger.error('poll_proc_thread unknown exception: ' + str(e) ) + logger.error('poll_proc_thread unknown exception: ' + str(e)) for fd in i: sid = fd2sid[fd] self.proc_read(sid) @@ -1197,6 +1314,7 @@ time.sleep(0.002) return False + class SingletonType(type): def __call__(cls, *args, **kwargs): try: @@ -1205,52 +1323,57 @@ cls.__instance = super(SingletonType, cls).__call__(*args, **kwargs) return cls.__instance + class WebConsoleRequestHandler: __metaclass__ = SingletonType + def __init__(self): self.multiplex = Multiplex() + def handle_get(self, vmname, w, h, k, pid): if not vmname in self.multiplex.session: logger.error('console disconnected: %s' % vmname) return 'Error: Web console disconnected.' if self.multiplex.session[vmname]['pid'] != int(pid): - logger.info('Console occupied by another user: %s, %d' % (vmname, self.multiplex.session[vmname]['pid']) ) + logger.info('Console occupied by another user: %s, %d' % (vmname, self.multiplex.session[vmname]['pid'])) return 'Error: Web console is occupied by another admin user.' try: if self.multiplex.proc_keepalive(vmname, w, h): if k: - #logger.info('proc_write: %s' % k) + # logger.info('proc_write: %s' % k) self.multiplex.proc_write(vmname, k) time.sleep(0.002) d = self.multiplex.proc_dump(vmname) - #logger.info(d) + # logger.info(d) return '' + d else: logger.error('Console disconnected: %s' % vmname) return 'Error: Web console disconnected.' except (KeyError, ValueError, IndexError): - logger.error('Invalid console parameters') - return 'Error: Web console disconnected.' - def create_console(self, vmname, username, password, width, height, ip = LEGACY_CLI_AGENT_IP): + logger.error('Invalid console parameters') + return 'Error: Web console disconnected.' + + def create_console(self, vmname, username, password, width, height, ip=LEGACY_CLI_AGENT_IP): if vmname in self.multiplex.session: if 'pid' in self.multiplex.session[vmname]: - logger.info('Kill console parent process: %s, %d' % (vmname, self.multiplex.session[vmname]['pid']) ) + logger.info('Kill console parent process: %s, %d' % (vmname, self.multiplex.session[vmname]['pid'])) try: os.kill(self.multiplex.session[vmname]['pid'], signal.SIGKILL) except OSError as e: logger.info('Exception on killing console parent process: ' + str(e)) self.multiplex.proc_waitfordeath(vmname) del self.multiplex.session[vmname] - #self._kill_process_by_cmd_line('virsh console ' + vmname) - #if get_domain_status(vmname) != 'running': + # self._kill_process_by_cmd_line('virsh console ' + vmname) + # if get_domain_status(vmname) != 'running': # logger.info('VM not running: %s' % vmname) # return 0 if not self.multiplex.proc_spawn(vmname, username, password, width, height, ip): logger.error('Console spawn failed: %s' % vmname) return 0 else: - logger.info('Console spawned: %s, %d' % (vmname, self.multiplex.session[vmname]['pid']) ) + logger.info('Console spawned: %s, %d' % (vmname, self.multiplex.session[vmname]['pid'])) return self.multiplex.session[vmname]['pid'] + def _kill_process_by_cmd_line(self, pstring): for line in os.popen("ps ax | grep \"" + pstring + "\" | grep -v grep"): fields = line.split() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/db/influxdb/acquire.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/db/influxdb/acquire.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/db/influxdb/acquire.py (working copy) @@ -1,79 +1,81 @@ from influxdb import InfluxDBClient -from hive.utils import andebug + from djproject.an_settings import * + def query_sql(sql, raw=False): - client = InfluxDBClient(CMDATA['INFLUXDB_IP'], CMDATA['INFLUXDB_DEFAULT_PORT'], CMDATA['INFLUXDB_USER'], CMDATA['INFLUXDB_PASSWORD'], CMDATA['INFLUXDB_DATABASES'], timeout=10) - # client = InfluxDBClient('10.8.8.4', 8086, 'admin', 'admin', 'composer', timeout=10) - try: - rs = client.query(sql) - if raw: - result = rs.raw["series"] - else: - result = list(rs.get_points()) - data = {"state": True, "data": result} - return data - except Exception as e: - if hasattr(e.message, "message"): - return {"state": False, "data": ["Cannot connect the influxdb."]} - else: - return {"state": False, "data": ["Database query error."]} - # return {"state": False, "data": [e.content]} + client = InfluxDBClient(CMDATA['INFLUXDB_IP'], CMDATA['INFLUXDB_DEFAULT_PORT'], CMDATA['INFLUXDB_USER'], + CMDATA['INFLUXDB_PASSWORD'], CMDATA['INFLUXDB_DATABASES'], timeout=10) + # client = InfluxDBClient('10.8.8.4', 8086, 'admin', 'admin', 'composer', timeout=10) + try: + rs = client.query(sql) + if raw: + result = rs.raw["series"] + else: + result = list(rs.get_points()) + data = {"state": True, "data": result} + return data + except Exception as e: + if hasattr(e.message, "message"): + return {"state": False, "data": ["Cannot connect the influxdb."]} + else: + return {"state": False, "data": ["Database query error."]} + # return {"state": False, "data": [e.content]} def execute_sql(sql_params_dict, paging=None, raw=False): - count = 0 - sql = "select %s from %s" % ((",").join(sql_params_dict["select"]), sql_params_dict["from"]) - if "where" in sql_params_dict: - sql += " where %s" % (" and ").join(sql_params_dict["where"]) - if "time" in sql_params_dict: - if sql_params_dict["time"]: - for index, item in enumerate(sql_params_dict["time"]): - if type(item) == int: - sql_params_dict["time"][index] = str(item) + "s" - if len(sql_params_dict["time"]) == 1: - sql += " and time > " + sql_params_dict["time"][0] - else: - sql += " and time > %s and time < %s" % (sql_params_dict["time"][0], sql_params_dict["time"][1]) - if "time" in sql_params_dict and "where" not in sql_params_dict: - if sql_params_dict["time"]: - for index, item in enumerate(sql_params_dict["time"]): + count = 0 + sql = "select %s from %s" % ((",").join(sql_params_dict["select"]), sql_params_dict["from"]) + if "where" in sql_params_dict: + sql += " where %s" % (" and ").join(sql_params_dict["where"]) + if "time" in sql_params_dict: + if sql_params_dict["time"]: + for index, item in enumerate(sql_params_dict["time"]): if type(item) == int: - sql_params_dict["time"][index] = str(item) + "s" - if len(sql_params_dict["time"]) == 1: - sql += " where time > %s" % (sql_params_dict["time"][0]) - else: - sql += " where time > %s and time < %s" % (sql_params_dict["time"][0], sql_params_dict["time"][1]) - if paging: - get_field_key_sql = "show field keys from %s" % sql_params_dict["from"] - table_keys_response = query_sql(get_field_key_sql) - if table_keys_response["state"]: - if len(sql_params_dict["select"]) > 1 or sql_params_dict["select"][0] == "*": - field_key = table_keys_response["data"][0]["fieldKey"] - count_sql = "select count(%s) from %s" % (field_key, sql.split("from ")[1]) - else: - count_sql = "select count(%s) from %s" % (sql_params_dict["select"][0], sql.split("from ")[1]) - if query_sql(count_sql)["data"]: - count = query_sql(count_sql)["data"][0]["count"] - if not count: - count = 0 - else: + sql_params_dict["time"][index] = str(item) + "s" + if len(sql_params_dict["time"]) == 1: + sql += " and time > " + sql_params_dict["time"][0] + else: + sql += " and time > %s and time < %s" % (sql_params_dict["time"][0], sql_params_dict["time"][1]) + if "time" in sql_params_dict and "where" not in sql_params_dict: + if sql_params_dict["time"]: + for index, item in enumerate(sql_params_dict["time"]): + if type(item) == int: + sql_params_dict["time"][index] = str(item) + "s" + if len(sql_params_dict["time"]) == 1: + sql += " where time > %s" % (sql_params_dict["time"][0]) + else: + sql += " where time > %s and time < %s" % (sql_params_dict["time"][0], sql_params_dict["time"][1]) + if paging: + get_field_key_sql = "show field keys from %s" % sql_params_dict["from"] + table_keys_response = query_sql(get_field_key_sql) + if table_keys_response["state"]: + if len(sql_params_dict["select"]) > 1 or sql_params_dict["select"][0] == "*": + field_key = table_keys_response["data"][0]["fieldKey"] + count_sql = "select count(%s) from %s" % (field_key, sql.split("from ")[1]) + else: + count_sql = "select count(%s) from %s" % (sql_params_dict["select"][0], sql.split("from ")[1]) + if query_sql(count_sql)["data"]: + count = query_sql(count_sql)["data"][0]["count"] + if not count: count = 0 - - if "group_by" in sql_params_dict: - sql += " group by %s" % (sql_params_dict["group_by"]) + else: + count = 0 - if "order_by" in sql_params_dict: - sql += " order by %s" % (sql_params_dict["order_by"]) - if "limit" in sql_params_dict: - sql += " limit %s" % (sql_params_dict["limit"]) + if "group_by" in sql_params_dict: + sql += " group by %s" % (sql_params_dict["group_by"]) - if paging: - if "each_page_num" in paging and "pages" in paging: - sql += " limit %s offset %s" % (paging["each_page_num"], paging["pages"]) - sql_response = query_sql(sql, raw) - if paging: - response = {"total": count, "rows": sql_response["data"]} - else: - response = {"state": sql_response["state"], "search_result": sql_response["data"]} - return response \ No newline at end of file + if "order_by" in sql_params_dict: + sql += " order by %s" % (sql_params_dict["order_by"]) + if "limit" in sql_params_dict: + sql += " limit %s" % (sql_params_dict["limit"]) + + if paging: + if "each_page_num" in paging and "pages" in paging: + sql += " limit %s offset %s" % (paging["each_page_num"], paging["pages"]) + sql_response = query_sql(sql, raw) + if paging: + response = {"total": count, "rows": sql_response["data"]} + else: + response = {"state": sql_response["state"], "search_result": sql_response["data"]} + return response Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/elk.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/elk.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/elk.py (working copy) @@ -1,5 +1,8 @@ +import logging +import time + from elasticsearch import Elasticsearch, client -import json, logging, time + from djproject.an_settings import * logger = logging.getLogger('hive.debug') @@ -20,6 +23,7 @@ } }""" +# ToDo: Update with ELK authentication. class ELK(object): def __init__(self, host, port): @@ -31,11 +35,11 @@ return self.conn def put_template(self, name): - #body = json.loads(TEMPLATE_BODY % name) + # body = json.loads(TEMPLATE_BODY % name) body = json.loads(TEMPLATE_BODY) try: clt = client.IndicesClient(self.conn) - #clt.put_template(name=name, body=body) + # clt.put_template(name=name, body=body) clt.create(index=name, body=body) except Exception as e: logger.error('Create %s template failed! %s ' % (name, str(e))) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_ftp_server.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_ftp_server.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_ftp_server.py (working copy) @@ -1,10 +1,15 @@ -from pyftpdlib.authorizers import DummyAuthorizer -from pyftpdlib.handlers import FTPHandler -from pyftpdlib.servers import FTPServer, MultiprocessFTPServer -import logging, os +import logging import logging.handlers +import os +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import MultiprocessFTPServer + CM_UpdateBuilds_PATH = r'/ca/package' +FTP_LOG = '/var/log/ftpd.log' + + class FTP_Server(object): def __init__(self): if not os.path.exists(CM_UpdateBuilds_PATH): @@ -16,16 +21,17 @@ try: server = MultiprocessFTPServer(("0.0.0.0", 9993), handler) except Exception as e: - log_file = open('/var/log/ftpd.log', 'a+') + log_file = open(FTP_LOG, 'a+') log_file.write(str(e)) log_file.close() return - + logger = logging.getLogger('pyftpdlib') - logging.basicConfig(filename='/var/log/ftpd.log', level=logging.INFO) - loghandler = logging.handlers.RotatingFileHandler('/var/log/ftpd.log', maxBytes=1024*1024*10, backupCount=2) + logging.basicConfig(filename=FTP_LOG, level=logging.INFO) + loghandler = logging.handlers.RotatingFileHandler(FTP_LOG, maxBytes=1024 * 1024 * 10, backupCount=2) logger.addHandler(loghandler) - + server.serve_forever() + ftpserver = FTP_Server() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_tftp_server.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_tftp_server.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/init_tftp_server.py (working copy) @@ -1,9 +1,13 @@ -import logging, os +import logging import logging.handlers +import os + import tftpy CM_CONFIGBACKUP_PATH = r'/ca/config/backup' +TFTP_LOG = '/var/log/tftpd.log' + class TFTP_Server(object): def __init__(self): if not os.path.exists(CM_CONFIGBACKUP_PATH): @@ -11,22 +15,22 @@ try: server = tftpy.TftpServer(CM_CONFIGBACKUP_PATH) except Exception as e: - log_file = open('/var/log/tftpd.log', 'a+') + log_file = open(TFTP_LOG, 'a+') log_file.write(str(e)) log_file.close() return logger = logging.getLogger('tftpy') - logging.basicConfig(filename='/var/log/tftpd.log', level=logging.INFO) - loghandler = logging.handlers.RotatingFileHandler('/var/log/tftpd.log', maxBytes=1024*1024*10, backupCount=2) + logging.basicConfig(filename=TFTP_LOG, level=logging.INFO) + loghandler = logging.handlers.RotatingFileHandler(TFTP_LOG, maxBytes=1024 * 1024 * 10, backupCount=2) logger.addHandler(loghandler) try: server.listen('0.0.0.0', 69) except Exception as e: - log_file = open('/var/log/tftpd.log', 'a+') + log_file = open(TFTP_LOG, 'a+') log_file.write(str(e)) log_file.close() TFTP_Server() - + tftpserver = TFTP_Server() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/install_postgresql.sh =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/install_postgresql.sh (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/install_postgresql.sh (nonexistent) @@ -1,58 +0,0 @@ -if rpm -qa | grep postgresql10 > /dev/null; -then - echo "postgresql exists" - -else - echo "install postgresql" - #yum install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-centos96-9.6-3.noarch.rpm - #yum install postgresql96 postgresql96-server - #/usr/pgsql-9.6/bin/postgresql96-setup initdb - yum install https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-centos10-10-2.noarch.rpm - yum install postgresql10 - yum install postgresql10-server - /usr/pgsql-10/bin/postgresql-10-setup initdb -fi -sed -i -e 's|host all all 127.0.0.1/32 ident|host all all 127.0.0.1/32 trust|' /var/lib/pgsql/10/data/pg_hba.conf -systemctl enable postgresql-10 -systemctl start postgresql-10 -if sudo -Hiu postgres /usr/pgsql-10/bin/psql -lqt | cut -d \| -f 1 | grep -w cm; then - echo "database exists" -else - echo "create database" - sudo -Hiu postgres /usr/pgsql-10/bin/createdb cm -fi - - -if rpm -qa | grep timescaledb > /dev/null; -then - echo "timescaledb exists" -else - yum install https://timescalereleases.blob.core.windows.net/rpm/timescaledb-0.11.0-postgresql-10-0.x86_64.rpm - echo "shared_preload_libraries = 'timescaledb'" >> /var/lib/pgsql/10/data/postgresql.conf - systemctl restart postgresql-10 - /usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -d cm --command 'CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;' -fi - -if rpm -qa | grep postgresql10-contrib > /dev/null; -then - echo "postgresql10-contrib exists" -else - yum install postgresql10-contrib-10.5 -fi - -if rpm -qa | grep postgresql10-plpython-10.5 > /dev/null; -then - echo "postgresql10-plpython exists" -else - yum install postgresql10-plpython-10.5 -fi - -#rpm -ivh apache-madlib-1.15-bin-Linux.rpm -if rpm -qa | grep madlib > /dev/null; -then - /usr/local/madlib/bin/madpack -p postgres -c postgres@127.0.0.1:5432/cm install -else - echo "Please install madlib first" -fi - - Property changes on: src/webui/webui/htdocs/new/src/cm/lib/install_postgresql.sh ___________________________________________________________________ Deleted: svn:executable ## -1 +0,0 ## -* \ No newline at end of property Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py (working copy) @@ -1,15 +1,17 @@ -#for task -from cm.lib.communication import send_https_rest_request, modify_url, send_https_xmlrpc_request -from cm.lib.postgres_db import DB -import json, logging, socket, fcntl, struct, hashlib, os -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE, EasyParser +# for task +import fcntl +import hashlib +import struct import subprocess import time -from hive.utils import andebug, get_current_session, get_device_type import xml.etree.ElementTree as ET +from io import IOBase + +from cm.lib.communication import send_https_rest_request, send_https_xmlrpc_request from djproject.an_settings import * from hive.log_utils import * -from io import IOBase +from hive.model.legacycli import cli_parse +from hive.utils import get_current_session, get_device_type logger = logging.getLogger('hive.debug') @@ -21,7 +23,8 @@ if device_info['protocol'] == 'xmlrpc': params = build_xml_for_rpc(cli, device_info, mode) try: - rest_response_data = send_https_xmlrpc_request('POST', '/cgi-bin/xmlrpc_server', params, device_info['ip_address'], device_info['restapi_port']) + rest_response_data = send_https_xmlrpc_request('POST', '/cgi-bin/xmlrpc_server', params, + device_info['ip_address'], device_info['restapi_port']) except Exception as e: logger.error('Send xmlrpc to device:%s failed, %s' % (device_info['ip_address'], str(e))) return {'message': "requestfailed", 'data': str(e)} @@ -42,7 +45,9 @@ new_url = '/rest/%s/cli_extend' % _type data = {'cmd': cli} try: - rest_response_data = send_https_rest_request('POST', new_url, json.dumps(data), device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, json.dumps(data), device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: logger.error('Send command to device:%s failed, %s' % (device_info['ip_address'], str(e))) return {'message': "requestfailed", 'data': str(e)} @@ -64,11 +69,13 @@ def send_command_to_device(device_info, url, running_params, method='POST'): try: - rest_response_data = send_https_rest_request(method, url, running_params, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request(method, url, running_params, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: if 'timed out' in str(e): return 'time out' - #print 'send command:', str(e) + # print 'send command:', str(e) logger.error('Send command to device:%s failed, %s' % (device_info['ip_address'], str(e))) return 'failed' if rest_response_data['status'] == 200: @@ -85,10 +92,15 @@ else: return "failed" + def send_xmlrpc_to_device(device_info, url, running_params, method='POST', mode='config'): params = build_xml_for_rpc(running_params, device_info, mode) try: - rest_response_data = send_https_xmlrpc_request(method, "/cgi-bin/xmlrpc_server", params, device_info['ip_address'], device_info['xmlrpc_port'] if "xmlrpc_port" in device_info else device_info["restapi_port"], device_info["xmlrpc_protocol"] if "xmlrpc_protocol" in device_info else "https") + rest_response_data = send_https_xmlrpc_request(method, "/cgi-bin/xmlrpc_server", params, + device_info['ip_address'], + device_info['xmlrpc_port'] if "xmlrpc_port" in device_info else + device_info["restapi_port"], device_info[ + "xmlrpc_protocol"] if "xmlrpc_protocol" in device_info else "https") except Exception as e: if 'timed out' in str(e): return 'failed' @@ -107,6 +119,7 @@ else: return "failed" + def _filter_cli_list(content, mode): cli_list = [] tmp_list = [] @@ -116,14 +129,16 @@ tmp_list.append(cli) if mode == "config_with_input": for i in range(0, len(tmp_list) / 2): - cli_list.append(tmp_list[i * 2] + "\n" + tmp_list[i + 1]) + cli_list.append(tmp_list[i * 2] + "\n" + tmp_list[i + 1]) else: cli_list = tmp_list return cli_list + def _concat_cli_and_output(cli_list, output_list, device_info): result = [] - if device_info['type'].lower() in VPN_TYPE_LIST and (cli_list[0].startswith("switch") or cli_list[0].startswith("sw")): + if device_info['type'].lower() in VPN_TYPE_LIST and ( + cli_list[0].startswith("switch") or cli_list[0].startswith("sw")): cli_list = cli_list[1:] output_list = output_list[1:] for index, value in enumerate(cli_list): @@ -135,6 +150,7 @@ # return ("\n").join(result) return json.dumps(result) + def build_xml_for_rpc(content, device_info, mode='config'): root = ET.Element("methodCall") @@ -211,6 +227,7 @@ root_string = '' + root_string return root_string + def parse_xml_from_rpc(response): xmlp = ET.XMLParser(encoding="iso-8859-1") root = ET.fromstring(response.strip(), parser=xmlp) @@ -232,18 +249,19 @@ return result - def get_rest_info_from_device(device_name): db = DB.get_connected_db() fetchall_sql = '''SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, name, enable_password FROM device where name='%s' ''' % device_name data = db.fetchall(fetchall_sql) db.close() if not data: - return [None,] - key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', 'enable_password'] + return [None, ] + key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', + 'enable_password'] result = [dict(zip(key, each)) for each in data] return result + def get_ip_address(ifname): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) return socket.inet_ntoa(fcntl.ioctl( @@ -252,6 +270,7 @@ struct.pack('256s', ifname[:15]) )[20:24]) + def md5sum(fname): def read_chunks(fh): fh.seek(0) @@ -261,6 +280,7 @@ chunk = fh.read(8192) else: fh.seek(0) + m = hashlib.md5() if isinstance(fname, str) and os.path.exists(fname): @@ -275,9 +295,11 @@ return m.hexdigest() + def escapeshellarg(cmd_str): ret = '"' - esacpe_chars = ['#', '&', ';', '`', '|', '*', '?', '~', '<', '>', '^', '(', ')', '[', ']', '{', '}', '$', '\\', ',', '"'] + esacpe_chars = ['#', '&', ';', '`', '|', '*', '?', '~', '<', '>', '^', '(', ')', '[', ']', '{', '}', '$', '\\', ',', + '"'] for each in cmd_str: if each in esacpe_chars: ret += '\\' @@ -285,6 +307,7 @@ ret += '"' return ret + def cmd_direct(username, cmd, *args): cmd = cmd.strip() f_eop = chr(252) @@ -301,6 +324,7 @@ else: return cli_parse(output, list(args)) + def oper_log(level, module, operation, username=""): if log_check(level): ip = "localhost" @@ -312,13 +336,16 @@ else: username = "system" s_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) - insert_sql = "insert into operation_log(username, client_ip, time, level, module, operation) values('%s', '%s', '%s', '%s', '%s', '%s')" % (username, ip, s_time, level, module, operation) + insert_sql = "insert into operation_log(username, client_ip, time, level, module, operation) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + username, ip, s_time, level, module, operation) db = DB.get_connected_db() db.execute_sql(insert_sql) db.close() - log_msg = "%s level=%s, username=%s, client_ip=%s, module=%s, message=\"%s\"" %(s_time, level, username, ip, module, operation) + log_msg = "%s level=%s, username=%s, client_ip=%s, module=%s, message=\"%s\"" % (s_time, level, username, ip, + module, operation) send_log_to_hosts(log_msg) + def rest_log(level, username, ip, method, url, post_data, result): if log_check(level): s_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) @@ -327,13 +354,16 @@ request_msg += " %s" % post_data result_msg = "success" if result["result"] else "failed, %s" % json.dumps(result["msg"]) operation = "Process RESTful API request: %s, result: %s" % (request_msg, result_msg) - insert_sql = "insert into operation_log(username, client_ip, time, level, module, operation) values('%s', '%s', '%s', '%s', '%s', '%s')" % (username, ip, s_time, level, "API", operation) + insert_sql = "insert into operation_log(username, client_ip, time, level, module, operation) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + username, ip, s_time, level, "API", operation) db = DB.get_connected_db() db.execute_sql(insert_sql) db.close() - log_msg = "%s level=%s, username=%s, client_ip=%s, module=API, message=\"%s\"" %(s_time, level, username, ip, operation) + log_msg = "%s level=%s, username=%s, client_ip=%s, module=API, message=\"%s\"" % (s_time, level, username, ip, + operation) send_log_to_hosts(log_msg) + def query_all_default_tmpkey(): db = DB.get_connected_db() fetchall_sql = "SELECT key, default_value FROM config_template" @@ -344,17 +374,23 @@ return result + def insert_default_tmpkey(device_ip): default_tmpkey_list = query_all_default_tmpkey() db = DB.get_connected_db() for each in default_tmpkey_list: - insert_sql = "INSERT INTO device_config_template(device_id, key, value) VALUES('%s', '%s', '%s')" % (device_ip, each["key"], each["default_value"]) + insert_sql = "INSERT INTO device_config_template(device_id, key, value) VALUES('%s', '%s', '%s')" % (device_ip, + each[ + "key"], + each[ + "default_value"]) db.execute_sql(insert_sql) db.close() return + def delete_all_tmpkey(device_ip): try: db = DB.get_connected_db() @@ -367,6 +403,7 @@ return + def get_template_content(file_path, device_ip): db = DB.get_connected_db() fetchall_sql = "SELECT key, value FROM device_config_template WHERE device_id = '%s'" % device_ip @@ -384,4 +421,3 @@ file_content = file_content.replace("%%" + each["key"] + "%%", each["value"]) return file_content - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libconfig.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libconfig.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libconfig.py (working copy) @@ -1,19 +1,24 @@ -#for configuration file -import os, json, shutil, time, logging -from cm.lib.libbasic_operation import send_command_to_device, md5sum, send_xmlrpc_to_device, get_template_content, send_cli_to_device +# for configuration file +import logging +import shutil +import time + +from cm.lib.communication import send_https_rest_request +from cm.lib.libbasic_operation import send_command_to_device, md5sum, send_xmlrpc_to_device, get_template_content, \ + send_cli_to_device from cm.lib.parse_configfile import TarFiles -from hive.model.query import mark_expire_all -from hive.model.loading import get_model -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL from cm.lib.postgres_db import DB -from cm.lib.communication import send_https_rest_request -from hive.utils import andebug, get_device_type, standard_model_type from djproject.an_settings import * +from hive.model.legacycli import cli_parse, RegexParser, MATCHALL +from hive.model.loading import get_model +from hive.model.query import mark_expire_all +from hive.utils import get_device_type logger = logging.getLogger('hive.debug') engineer_cmd_list = ["ssl import pem"] + class Backup_APV(object): def __init__(self, device_info, file_path, cli_url, cli_cmd): self.device_info = device_info @@ -22,13 +27,13 @@ self.cli_url = cli_url def get_config_from_device(self): - data = {'cmd':self.cli_cmd} + data = {'cmd': self.cli_cmd} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return [False, 'Get config file from device<%s> failed.' % self.device_info['name']] - return [True, ret] - + return [True, ret] + def write_config(self, content): try: with open(self.file_path, 'w') as f: @@ -36,24 +41,25 @@ except IOError: return 'Write file failed.' return '' - + def run(self): try: content = self.get_config_from_device() if not content[0]: return content[1] - ret = self.write_config(content[1]) + ret = self.write_config(content[1]) except Exception as e: logger.error('Backup %s:%s' % (self.device_info['name'], str(e))) - return 'Backup internal error.' + return 'Backup internal error.' return '' + class Backup_Custom(object): def __init__(self, config_content, task_name, file_path): self.config_content = config_content self.task_name = task_name - self.file_path = file_path - + self.file_path = file_path + def write_config(self, content): try: with open(self.file_path, 'w') as f: @@ -61,18 +67,19 @@ except IOError: return 'Write file failed.' return '' - + def run(self): try: - ret = self.write_config(self.config_content) + ret = self.write_config(self.config_content) if ret: - logger.error('Customize file%s: %s' % (task_name, ret)) + logger.error('Customize file%s: %s' % (self.task_name, ret)) return ret except Exception as e: - logger.error('Customize file %s: %s' % (task_name, str(e))) - return 'Backup internal error.' + logger.error('Customize file %s: %s' % (self.task_name, str(e))) + return 'Backup internal error.' return '' - + + class Recover_APV(object): def __init__(self, file_path, cli_url, device_info, file_type, xmlrpc_status=False): self.cli_url = cli_url @@ -80,6 +87,7 @@ self.device_info = device_info self.xmlrpc_status = xmlrpc_status self.file_type = file_type + def sync_device(self): eng_flag = False if not os.path.exists(self.file_path): @@ -93,7 +101,7 @@ except Exception as e: logger.error('Recover %s:%s' % (self.device_info['name'], str(e))) return [0, 'Read config file failed.', eng_flag] - + if not content: return [0, 'This is config file has nothing', eng_flag] for each in engineer_cmd_list: @@ -109,6 +117,7 @@ if ret == 'failed': return [0, 'Sync config file failed.', eng_flag] return [1, ret, eng_flag] + def run(self): try: ret = self.sync_device() @@ -116,8 +125,10 @@ return ret except Exception as e: logger.error('Recover %s:%s' % (self.device_info['name'], str(e))) - return [0, 'Recover internal error.', eng_flag] - return + return [0, 'Recover internal error.', False] + return + + class Backup_AG(object): def __init__(self, device_info, virtual_file_path, cli_url, check_config_url, real_file_path, cli_cmd): self.device_info = device_info @@ -138,24 +149,24 @@ if self.device_info["protocol"] == "xmlrpc": ret = send_xmlrpc_to_device(self.device_info, self.cli_url, self.cli_cmd['write_config']) else: - data = {'cmd' : self.cli_cmd['write_config']} + data = {'cmd': self.cli_cmd['write_config']} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return 'Write config file from device failed.' return '' - + def export_config_file(self): if self.device_info["protocol"] == "xmlrpc": ret = send_xmlrpc_to_device(self.device_info, self.cli_url, self.cli_cmd['export_config']) else: - data = {'cmd' : self.cli_cmd['export_config']} + data = {'cmd': self.cli_cmd['export_config']} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return 'Export config file from device failed.' return '' - + def check_local_config_file(self): if not os.path.exists(self.virtual_file_path): return "Export config file from device failed." @@ -165,7 +176,11 @@ if self.device_info["protocol"] == "xmlrpc": rest_response_data = send_xmlrpc_to_device(self.device_info, self.cli_url, "show config file") else: - rest_response_data = send_https_rest_request('GET', self.check_config_url, '', self.device_info['ip_address'], self.device_info['restapi_port'], self.device_info['restapi_username'], self.device_info['restapi_password']) + rest_response_data = send_https_rest_request('GET', self.check_config_url, '', + self.device_info['ip_address'], + self.device_info['restapi_port'], + self.device_info['restapi_username'], + self.device_info['restapi_password']) except Exception as e: return "Can not connect the device." else: @@ -177,7 +192,7 @@ for each in config_file_list: if file_name in each: items = each.split() - device_file['size'] =items[0] + device_file['size'] = items[0] device_file['md5'] = items[-1] break except: @@ -196,14 +211,14 @@ return 'Get config file info from device failed.' else: return 'Can not get the config file info from device.' - + if 'size' not in device_file: return 'Can not find the config file info from device.' tmp_file_size = 0 check_file_times = 3 pre_file_size = 0 while True: - time.sleep(3) + time.sleep(3) file_size = os.path.getsize(self.virtual_file_path) if device_file['size'] > file_size: if pre_file_size != file_size: @@ -224,23 +239,24 @@ return "Export config file from device failed, MD5 not match." except Exception as e: return "Export config file from device failed." - return '' - + return '' + def parse_config_file(self): - #makedir target_path + # makedir target_path target_path = '.'.join(self.real_file_path.split('.')[0:-1]) if os.path.exists(target_path): shutil.rmtree(target_path) os.makedirs(target_path) tar = TarFiles() - ret = tar.extract_tar(os.path.dirname(self.real_file_path), self.real_file_name, '.'.join(self.real_file_name.split('.')[0:-1])) + ret = tar.extract_tar(os.path.dirname(self.real_file_path), self.real_file_name, + '.'.join(self.real_file_name.split('.')[0:-1])) vsite_list = [] file_name_base = '.'.join(self.real_file_name.split('.')[0:-1]) for root, dir, files in os.walk(target_path): for file in files: - fullpath=os.path.join(root, file) + fullpath = os.path.join(root, file) path_list = fullpath.split('/') if 'vsites' in path_list and path_list[path_list.index('vsites') + 1] not in vsite_list: vsite_list.append(path_list[path_list.index('vsites') + 1]) @@ -251,25 +267,25 @@ insert_sql = "INSERT INTO tar_file(name, vsite_name) values('%s', '%s')" % (file_name_base, vsite) db.execute_sql(insert_sql) db.close() - mark_expire_all(get_model('cm', ['configuration', 'config_file', 'TarFile'])) - + mark_expire_all(get_model('cm', ['configuration', 'config_file', 'TarFile'])) + def remove_config_file_from_device(self): if self.virtual_file_path and os.path.exists(self.virtual_file_path): os.remove(self.virtual_file_path) - #remove the config from device + # remove the config from device if self.device_info["protocol"] == "xmlrpc": ret = send_xmlrpc_to_device(self.device_info, self.cli_url, self.cli_cmd['remove_config']) else: - data = {'cmd' : self.cli_cmd['remove_config']} + data = {'cmd': self.cli_cmd['remove_config']} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return False - return True - + return True + def save_show_running_all(self): - data = {'cmd' : 'show running all'} + data = {'cmd': 'show running all'} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': @@ -281,7 +297,7 @@ except IOError: return "Save running config file failed." return '' - + def run(self): if self.device_info["protocol"] == "xmlrpc": ret = self.write_net_config_all() @@ -303,17 +319,17 @@ # ag restapi will timeout in 10 mins, but export will continue doing after that. # so do not return error, continue do check_local_config_file result = self.export_config_file() - + ret = self.check_local_config_file() if ret: return ret - #ret = self.save_show_running_all() - #if ret: + # ret = self.save_show_running_all() + # if ret: # return ret - + os.rename(self.virtual_file_path, self.real_file_path) - + if self.device_info['type'].lower() in VPN_TYPE_LIST: self.parse_config_file() except Exception as e: @@ -322,9 +338,11 @@ finally: self.remove_config_file_from_device() return '' - + + class Recover_AG(object): - def __init__(self, real_file_path, virtual_file_path, device_info, cli_cmd, cli_url, check_config_url, batch_cli_url, xmlrpc_status=False): + def __init__(self, real_file_path, virtual_file_path, device_info, cli_cmd, cli_url, check_config_url, + batch_cli_url, xmlrpc_status=False): self.real_file_path = real_file_path self.virtual_file_path = virtual_file_path self.device_info = device_info @@ -333,27 +351,29 @@ self.check_config_url = check_config_url self.batch_cli_url = batch_cli_url self.xmlrpc_status = xmlrpc_status - + def import_config(self): if self.device_info["protocol"] == "xmlrpc": ret = send_xmlrpc_to_device(self.device_info, self.cli_url, self.cli_cmd['import_config']) else: - data = {'cmd' : self.cli_cmd['import_config']} + data = {'cmd': self.cli_cmd['import_config']} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return 'Import config file from ACM failed.' - return '' + return '' + def sync_config(self): if self.device_info["protocol"] == "xmlrpc": ret = send_xmlrpc_to_device(self.device_info, self.cli_url, self.cli_cmd['sync_config']) else: - running_params = json.dumps({'config':self.cli_cmd['sync_config']}) + running_params = json.dumps({'config': self.cli_cmd['sync_config']}) ret = send_command_to_device(self.device_info, self.batch_cli_url, running_params) if ret == 'failed': return [0, 'Recover: Sync to device failed.'] - return [1, ret] + return [1, ret] + def check_config_recover(self): failed_time = 60 new_url = '/rest/ag/global/system/HostSettings' @@ -363,7 +383,10 @@ if self.device_info["protocol"] == "xmlrpc": rest_response_data = send_xmlrpc_to_device(self.device_info, new_url, 'show host') else: - rest_response_data = send_https_rest_request('GET', new_url, '', self.device_info['ip_address'], self.device_info['restapi_port'], self.device_info['restapi_username'], self.device_info['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.device_info['ip_address'], + self.device_info['restapi_port'], + self.device_info['restapi_username'], + self.device_info['restapi_password']) except Exception as e: failed_time -= 1 else: @@ -373,29 +396,36 @@ else: if rest_response_data['status'] == 200: return '' - failed_time -= 1 + failed_time -= 1 return 'Can not connect the device.' + def remove_config_file_from_device(self): if self.virtual_file_path and os.path.exists(self.virtual_file_path): os.remove(self.virtual_file_path) - #remove the config from device - data = {'cmd' : self.cli_cmd['remove_config']} + # remove the config from device + data = {'cmd': self.cli_cmd['remove_config']} running_params = json.dumps(data) ret = send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return False - return True - def check_device_config_file(self, file_size, file_md5, file_name): + return True + + def check_device_config_file(self, file_size, file_md5, file_name): tmp_file_size = 0 check_file_times = 3 while check_file_times: try: time.sleep(3) if self.device_info["protocol"] == "xmlrpc": - rest_response_data = send_xmlrpc_to_device(self.device_info, self.check_config_url, "show config file") + rest_response_data = send_xmlrpc_to_device(self.device_info, self.check_config_url, + "show config file") else: - rest_response_data = send_https_rest_request('GET', self.check_config_url, '', self.device_info['ip_address'], self.device_info['restapi_port'], self.device_info['restapi_username'], self.device_info['restapi_password']) + rest_response_data = send_https_rest_request('GET', self.check_config_url, '', + self.device_info['ip_address'], + self.device_info['restapi_port'], + self.device_info['restapi_username'], + self.device_info['restapi_password']) except Exception as e: return "Can't connect the device." else: @@ -427,13 +457,14 @@ if tmp_file_size == 0: check_file_times -= 1 except: - #check_file_times -= 1 + # check_file_times -= 1 return 'Put the config file failed.' else: return 'Can not find the config file.' if not check_file_times: return 'Put config file failed.' - return '' + return '' + def run(self): shutil.copy(self.real_file_path, self.virtual_file_path) if self.device_info["protocol"] == "xmlrpc": @@ -457,27 +488,28 @@ ret = self.import_config() if ret: return [0, ret] - + ret = self.check_device_config_file(file_size, file_md5, file_name) if ret: return [0, ret] - + ret = self.sync_config() if not ret[0]: return ret else: res = ret[1] - + ret = self.check_config_recover() if ret: return [0, ret] except Exception as e: logger.error('Recover %s:%s' % (self.device_info['name'], str(e))) - return [0, 'Recover internal error.'] + return [0, 'Recover internal error.'] finally: self.remove_config_file_from_device() - return [1, res] + return [1, res] + class Apply_AG(object): def __init__(self, real_file_path, device_info, cli_url, xmlrpc_status=False): self.real_file_path = real_file_path @@ -494,7 +526,9 @@ elif response_data['message'] == 'failed': return [False, 'Get Config from device failed!'] - vsite_parser = RegexParser('virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', MATCHALL) + vsite_parser = RegexParser( + 'virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', + MATCHALL) if self.device_info['protocol'] == 'xmlrpc': output = cli_parse(response_data['data'], vsite_parser) else: @@ -502,7 +536,7 @@ for item in output: ret.append(item["vsite_name"]) return [True, ret] - + def sync_config(self): if not os.path.exists(self.real_file_path): return [0, 'Can not find the file.'] @@ -511,7 +545,7 @@ content = f.read() except IOError: return [0, 'Read config file failed.'] - + if not content: return [0, 'This is config file has nothing'] if self.device_info['scope'] == 'global': @@ -544,13 +578,14 @@ else: for each in self.switch_list: cmd = { - "config":"sw %s\n%s\n" % (each, content) + "config": "sw %s\n%s\n" % (each, content) } running_params = json.dumps(cmd) ret += send_command_to_device(self.device_info, self.cli_url, running_params) if ret == 'failed': return [0, 'Sync to device failed.'] return [1, ret] + def run(self): try: ret = self.sync_config() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libmonitor.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libmonitor.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libmonitor.py (working copy) @@ -1,9 +1,12 @@ -from cm.lib.postgres_db import DB -import datetime, logging, json +import datetime import http.client as httplib +import json +import logging + +from cm.lib.libbasic_operation import cmd_direct +from cm.lib.postgres_db import DB from cm.lib.send_email import SendMail from hive.model.legacycli import EasyParser -from cm.lib.libbasic_operation import cmd_direct from hive.model.loading import get_model logger = logging.getLogger('hive.debug') @@ -38,21 +41,23 @@ ] }]""" -#{"error":0,"data":{"legend":["snmp_scada_node.time"],"series":[{"data":[["2018-01-08T07:26:36.655007196Z","10.8.11.35",6,0,0]],"name":"snmp_scada_node.time"}]}} + +# {"error":0,"data":{"legend":["snmp_scada_node.time"],"series":[{"data":[["2018-01-08T07:26:36.655007196Z","10.8.11.35",6,0,0]],"name":"snmp_scada_node.time"}]}} def send_command_to_composer(scheme, url, params): - headers = {"Accept":"application/json", "Cookie":"ComposerUICookie=f1f713c9e000f5d3f280adbd124df4f5"} + headers = {"Accept": "application/json", "Cookie": "ComposerUICookie=f1f713c9e000f5d3f280adbd124df4f5"} try: httpClient = httplib.HTTPConnection('127.0.0.1', 3000, True, timeout=20) - #httpClient.request('POST', '/composer/graph/xyAxis', json.dumps(SELECT_INFO), headers) + # httpClient.request('POST', '/composer/graph/xyAxis', json.dumps(SELECT_INFO), headers) httpClient.request(scheme, url, params, headers) except Exception as e: return 'Falied' response = httpClient.getresponse() result = json.loads(response.read()) httpClient.close() - #return json.loads(response.read())['data']['series'][0]['data'] + # return json.loads(response.read())['data']['series'][0]['data'] return result['data']['series'][0]['data'] + def parse_datetime(data): for item in data: str_time = item[0].split('.')[0] @@ -60,6 +65,7 @@ return data + def get_cpu_usage(ip_addr, interval): parms = {} parms['select'] = """ "time", "cpu_usage" """ @@ -134,6 +140,7 @@ data = parse_datetime(data) return data + def get_mem_usage(ip_addr, interval): parms = {} parms['select'] = """ "time", "mem_usage" """ @@ -149,6 +156,7 @@ data = parse_datetime(data) return data + def get_inoctets_usage(ip_addr, interval): parms = {} parms['select'] = """ "inoctets" """ @@ -164,6 +172,7 @@ data = parse_datetime(data) return data + def get_outoctets_usage(ip_addr, interval): parms = {} parms['select'] = """ "outoctets" """ @@ -179,31 +188,33 @@ data = parse_datetime(data) return data + monitor_type = { - #device_type - 1 : {"name":"cpu_usage", "func": get_cpu_usage}, - 2 : {"name":"mem_usage", "func": get_mem_usage}, - 3 : {"name":"inoctets_usage", "func": get_inoctets_usage}, - 4 : {"name":"outoctets_usage", "func": get_outoctets_usage}, + # device_type + 1: {"name": "cpu_usage", "func": get_cpu_usage}, + 2: {"name": "mem_usage", "func": get_mem_usage}, + 3: {"name": "inoctets_usage", "func": get_inoctets_usage}, + 4: {"name": "outoctets_usage", "func": get_outoctets_usage}, - #sslvpn - 1000 : {"name":"sslvpn"}, + # sslvpn + 1000: {"name": "sslvpn"}, - #slb - 2000 : {"name":"slb"}, + # slb + 2000: {"name": "slb"}, - #ha - 3000 : {"name":"ha_switch"}, + # ha + 3000: {"name": "ha_switch"}, - #ssl_cert - 4000 : {"name":"ssl_cert"}, + # ssl_cert + 4000: {"name": "ssl_cert"}, - #custom - 5000 : {"name":"custom"} + # custom + 5000: {"name": "custom"} } + class Decision(object): - #def __init__(self, ip_list, condition): + # def __init__(self, ip_list, condition): def __init__(self, target_type=None, target_list=None, continued_time=None, inhibit_time=None, interval_time=None): ''' self.ip_list = ip_list @@ -220,7 +231,7 @@ data = send_command_to_composer('POST', self.composer_url, params) if not data: return None - return {'cpu_usage':data[0][2], 'mem_usage':data[0][3]} + return {'cpu_usage': data[0][2], 'mem_usage': data[0][3]} def get_target_info(self): db = DB.get_connected_db() @@ -228,7 +239,7 @@ if self.target_type == 1 or self.target_type == 2: pass else: - #return {"ip":"device_name"} + # return {"ip":"device_name"} for device in self.target_list: select_sql = "select ip_address from device where name='%s';" % device data = db.fetchone(select_sql) @@ -239,8 +250,8 @@ return result def deal_condition(self, data, condition, target_status, ip, monitor_id): - #target_status : {"ip1":{"alert_status":1, "start_time":time}} - #data {"cpu_usage":[[time, value], ...], "mem_usage":[], ...} + # target_status : {"ip1":{"alert_status":1, "start_time":time}} + # data {"cpu_usage":[[time, value], ...], "mem_usage":[], ...} alert_flag = 0 condition_keys = data.keys() end_time = datetime.timedelta() @@ -261,9 +272,10 @@ for index in range(0, min_len): real_condition = condition["alias"] - #temp_condition = {} # {"type_name": "condition"} + # temp_condition = {} # {"type_name": "condition"} for item in condition_keys: - temp_condition = " %d %s %d " % (data[item][index][1], condition[item]["operator"], condition[item]["value"]) + temp_condition = " %d %s %d " % (data[item][index][1], condition[item]["operator"], + condition[item]["value"]) real_condition = real_condition.replace(item, str(eval(temp_condition))) data_message = "%s:%s; " % (item, str(data[item][index][1])) if eval(real_condition): @@ -275,51 +287,57 @@ if datetime_delta.seconds > self.continued_time: alert_status = 2 alert_flag = 1 - #TODO - result.append({"start_time":start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status":alert_status}) + # TODO + result.append( + {"start_time": start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status": alert_status}) elif alert_status == 2: datetime_delta = data[item][index][0] - start_time if self.inhibit_time and datetime_delta.seconds > self.inhibit_time: alert_flag = 1 - #TODO - result.append({"start_time":start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status":alert_status}) + # TODO + result.append( + {"start_time": start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status": alert_status}) elif not self.inhibit_time and not alert_flag: alert_flag = 1 - #TODO - result.append({"start_time":start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status":alert_status}) + # TODO + result.append( + {"start_time": start_time.strftime('%Y-%m-%d %H:%M:%S'), "alert_status": alert_status}) else: if alert_status == 2: end_time = data[item][index][0].strftime('%Y-%m-%d %H:%M:%S') - #TODO + # TODO if result: result[-1]["end_time"] = end_time result[-1]["alert_status"] = 0 else: self.modify_monitor_msg(monitor_id, end_time, 0) - #result.append() + # result.append() alert_status = 0 if not target_status: target_status = {} if start_time: - target_status[ip] = {"alert_status":alert_status, "start_time":start_time.strftime('%Y-%m-%d %H:%M:%S')} + target_status[ip] = {"alert_status": alert_status, "start_time": start_time.strftime('%Y-%m-%d %H:%M:%S')} self.modify_monitor(monitor_id, target_status) return result def modify_monitor(self, monitor_id, target_status): - update_sql = """ update monitor set extend = extend || '{"target_status":%s}'::jsonb where id=%d """ % (json.dumps(target_status), monitor_id) + update_sql = """ update monitor set extend = extend || '{"target_status":%s}'::jsonb where id=%d """ % ( + json.dumps(target_status), monitor_id) db = DB.get_connected_db() db.execute_sql(update_sql) db.close() - + def modify_monitor_msg(self, monitor_id, end_time, alert_status): - update_sql = """ update monitor_msg set extend = extend || '{"end_time":"%s", "alert_status":%d}'::jsonb where monitor_id=%d and (extend->>'alert_status')::int=2 """ % (end_time, alert_status, monitor_id) + update_sql = """ update monitor_msg set extend = extend || '{"end_time":"%s", "alert_status":%d}'::jsonb where monitor_id=%d and (extend->>'alert_status')::int=2 """ % ( + end_time, alert_status, monitor_id) db = DB.get_connected_db() db.execute_sql(update_sql) db.close() - + def check_repeater_monitor_msg(self, monitor_id, alert_status): - select_sql = "select id from monitor_msg where monitor_id=%d and deal_status=0 and (extend->>'alert_status')::int = %d;" % (monitor_id, alert_status) + select_sql = "select id from monitor_msg where monitor_id=%d and deal_status=0 and (extend->>'alert_status')::int = %d;" % ( + monitor_id, alert_status) db = DB.get_connected_db() data = db.fetchone(select_sql) db.close() @@ -327,8 +345,6 @@ return False return True - - def determine(self, monitor_info): result = [] global monitor_type @@ -340,9 +356,9 @@ logger.error('Get %s data from composer failed!' % ip) continue """ - #real_condition = monitor_info["condition"]["alias"] - #data_message = "" - data_list = {} # {"cpu_usage":[], "mem_usage":[]} + # real_condition = monitor_info["condition"]["alias"] + # data_message = "" + data_list = {} # {"cpu_usage":[], "mem_usage":[]} for type_id in monitor_info["type_id"]: td = int(type_id) if td > 5000: @@ -358,7 +374,7 @@ data_list[monitor_type[td]["name"]] = monitor_data if not monitor_data: break - #target_status = monitor_info["target_status"][ip] if ip in monitor_info["target_status"] else None + # target_status = monitor_info["target_status"][ip] if ip in monitor_info["target_status"] else None nonetype_flag = 0 for item in data_list.keys(): if not data_list[item]: @@ -366,12 +382,15 @@ break if nonetype_flag: continue - alert_list = self.deal_condition(data_list, monitor_info["condition"], monitor_info["target_status"], ip, monitor_info["id"]) + alert_list = self.deal_condition(data_list, monitor_info["condition"], monitor_info["target_status"], ip, + monitor_info["id"]) for alert in alert_list: end_time = alert["end_time"] if "end_time" in alert else "" - result.append({ip:target_info[ip], "data_message":{"start_time":alert['start_time'], "alert_status":alert['alert_status'], "end_time":end_time}}) - #TODO + result.append({ip: target_info[ip], "data_message": {"start_time": alert['start_time'], + "alert_status": alert['alert_status'], + "end_time": end_time}}) + # TODO """ try: if eval(real_condition): @@ -398,7 +417,7 @@ ''' def send_mail(self, recive_list, message): - #get email server config + # get email server config db = DB.get_connected_db() select_sql = "select settings from global_settings where name='email_settings';" data = db.fetchone(select_sql) @@ -409,7 +428,8 @@ return -1 db.close() - sm = SendMail(server_info['smtpserver'], int(server_info['smtpport']), server_info['address'], server_info['password']) + sm = SendMail(server_info['smtpserver'], int(server_info['smtpport']), server_info['address'], + server_info['password']) ret = sm.send_mail_text("Alarm", message, recive_list) if ret: logger.error('send mail failed %s' % ret) @@ -427,12 +447,11 @@ email_list.append(item['email']) continue return email_list - def alert_msg(self, dec_list, monitor_info): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') email_list = self.get_user_email(monitor_info['assign']) - #email_list = user_info['email'] + # email_list = user_info['email'] db = DB.get_connected_db() if self.target_type == 1 or self.target_type == 2: @@ -444,22 +463,29 @@ target_name = item[ip] data_message = item['data_message'] end_time = data_message['end_time'] if "end_time" in data_message else "" - message = "The target device:%s has triggered this alarm. \n start_time:%s end_time:%s " % (ip, data_message["start_time"], end_time if end_time else "now") + message = "The target device:%s has triggered this alarm. \n start_time:%s end_time:%s " % (ip, + data_message[ + "start_time"], + end_time if end_time else "now") comment = "%s\n%s" % (message, monitor_info['message']) data = { - "monitor_id":monitor_info['id'], - "create_time":now, - "deal_status":0, - "target_type":self.target_type, - "target_name":target_name, - "extend":json.dumps({"comment":comment, "start_time":data_message['start_time'], "end_time":end_time, "alert_status":data_message['alert_status']}) - } - if data_message['alert_status'] != 2 or (data_message['alert_status'] == 2 and self.check_repeater_monitor_msg(monitor_info['id'], data_message['alert_status'])): - + "monitor_id": monitor_info['id'], + "create_time": now, + "deal_status": 0, + "target_type": self.target_type, + "target_name": target_name, + "extend": json.dumps( + {"comment": comment, "start_time": data_message['start_time'], "end_time": end_time, + "alert_status": data_message['alert_status']}) + } + if data_message['alert_status'] != 2 or ( + data_message['alert_status'] == 2 and self.check_repeater_monitor_msg(monitor_info['id'], + data_message[ + 'alert_status'])): insert_sql = "insert into monitor_msg(monitor_id, create_time, deal_status, target_type, target_name, extend) values('%(monitor_id)d', '%(create_time)s', '%(deal_status)d', '%(target_type)d', '%(target_name)s', '%(extend)s')" % data db.execute_sql(insert_sql) - #send email or send message + # send email or send message if monitor_info['level'] < 2: self.send_mail(email_list, comment) @@ -473,32 +499,31 @@ def determine(self, monitor_info): service = get_model('cm', ['device_mgmt', 'service', 'Services']) service_manage = service.get_manager(None) - #{"target_type":0|1|2, "target_name":"values", "end_time":""} + # {"target_type":0|1|2, "target_name":"values", "end_time":""} result = [] - slb_cert = service_manage._perform_CheckSSLCert({"service_type":"SLB", "service_name":""}) + slb_cert = service_manage._perform_CheckSSLCert({"service_type": "SLB", "service_name": ""}) now = datetime.datetime.now() for item in slb_cert["result"]: - #'vhost': u'vhost2', 'vs_name': 'ftps1', 'end': u'Jun 8 15:58:53 2018', 'device_list': '10.4.72.2', 'service_name': '10.4.72.2-ftps1', 'before': u'Jun 7 15:58:53 2018'} - if 'end' in item: - end_time = datetime.datetime.strptime(item['end'], "%b %d %H:%M:%S %Y") + # 'vhost': u'vhost2', 'vs_name': 'ftps1', 'end': u'Jun 8 15:58:53 2018', 'device_list': '10.4.72.2', 'service_name': '10.4.72.2-ftps1', 'before': u'Jun 7 15:58:53 2018'} + if 'end' in item: + end_time = datetime.datetime.strptime(item['end'], "%b %d %H:%M:%S %Y") delta_time = end_time - now if delta_time.days < 30 and delta_time.days > 0: - result.append({"target_type":2, "target_name":item['service_name'], "end_time":item['end']}) + result.append({"target_type": 2, "target_name": item['service_name'], "end_time": item['end']}) if delta_time.days < 0: - result.append({"target_type":2, "target_name":item['service_name']}) + result.append({"target_type": 2, "target_name": item['service_name']}) - sslvpn_cert = service_manage._perform_CheckSSLCert({"service_type":"SSLVPN", "service_name":""}) - #{'service_name': '192.168.2.140-xutao', 'vs_name': 'xutao', 'end': u'Sep 20 06:41:03 2025', 'device_list': '192.168.2.140', 'before': u'Jul 4 06:41:03 2017'} + sslvpn_cert = service_manage._perform_CheckSSLCert({"service_type": "SSLVPN", "service_name": ""}) + # {'service_name': '192.168.2.140-xutao', 'vs_name': 'xutao', 'end': u'Sep 20 06:41:03 2025', 'device_list': '192.168.2.140', 'before': u'Jul 4 06:41:03 2017'} for item in sslvpn_cert['result']: - if 'end' in item: - end_time = datetime.datetime.strptime(item['end'], "%b %d %H:%M:%S %Y") + if 'end' in item: + end_time = datetime.datetime.strptime(item['end'], "%b %d %H:%M:%S %Y") delta_time = end_time - now if delta_time.days < 30 and delta_time.days > 0: - result.append({"target_type":1, "target_name":item['service_name'], "end_time":item['end']}) + result.append({"target_type": 1, "target_name": item['service_name'], "end_time": item['end']}) if delta_time.days < 0: - result.append({"target_type":1, "target_name":item['service_name']}) + result.append({"target_type": 1, "target_name": item['service_name']}) return result - def alert_msg(self, dec_list, monitor_info): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -513,25 +538,21 @@ else: message += "It has expired. \n" - comment = "%s%s\n" % (message, monitor_info['message']) + comment = "%s%s\n" % (message, monitor_info['message']) data = { - "monitor_id":monitor_info['id'], - "create_time":now, - "deal_status":0, - "target_type":item['target_type'], - "target_name":item['target_name'], - "extend":json.dumps({"comment":comment}) + "monitor_id": monitor_info['id'], + "create_time": now, + "deal_status": 0, + "target_type": item['target_type'], + "target_name": item['target_name'], + "extend": json.dumps({"comment": comment}) } message_list.append(comment) insert_sql = "insert into monitor_msg(monitor_id, create_time, deal_status, target_type, target_name, extend) values('%(monitor_id)d', '%(create_time)s', '%(deal_status)d', '%(target_type)d', '%(target_name)s', '%(extend)s')" % data db.execute_sql(insert_sql) - - db.close() + db.close() if monitor_info['level'] < 2: self.send_mail(email_list, '\n'.join(message_list)) return - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/parse_configfile.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/parse_configfile.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/parse_configfile.py (working copy) @@ -1,13 +1,17 @@ -import os, re, uuid -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE +import re +import uuid + from cm.lib.postgres_db import DB -from hive.model.query import mark_expire_all -from hive.model.loading import get_model from djproject.an_settings import * +from hive.model.legacycli import cli_parse, RegexParser, MATCHALL +from hive.model.loading import get_model +from hive.model.query import mark_expire_all + class TarFiles(object): def __init__(self): pass + def create_tar(self, dir_path, file_name): uid = uuid.uuid1() if file_name and dir_path: @@ -22,10 +26,12 @@ if rest != 0: return rest return 0 + def extract_tar(self, dir_path, file_name, dir_name): uid = uuid.uuid1() if file_name and dir_path and dir_name: - cmd = "cd %s && mkdir -p %s && cp %s %s && tar xf %s -C %s && rm %s" % (dir_path, dir_name, file_name, uid, uid, dir_name, uid) + cmd = "cd %s && mkdir -p %s && cp %s %s && tar xf %s -C %s && rm %s" % (dir_path, dir_name, file_name, uid, + uid, dir_name, uid) else: return -1 rest = os.system(cmd) @@ -33,7 +39,8 @@ return rest return 0 -#get vsite config from the device config file, and write to vsite file + +# get vsite config from the device config file, and write to vsite file def while_parse(config_file, pattern, vsite_name): line = config_file.readline() if vsite_name: @@ -53,6 +60,8 @@ if vsite_name: vsite_file.close() return None + + def parse_vsite_from_configfile(file_path, device_name): if not os.path.exists(file_path): return @@ -75,7 +84,8 @@ vsite_filepath = vsite_path + '/' + device_name + '-' + vsite_name config_file.close() -#parse vsite config from the device config file, then insert to database + +# parse vsite config from the device config file, then insert to database def get_specific_info_from_config(config_file, start_line, end_line, filter_str=[]): line = config_file.readline() specific_info = '' @@ -85,7 +95,7 @@ if line == end_line: break if flag: - #if filter_str and line.find(filter_str) != -1: + # if filter_str and line.find(filter_str) != -1: if filter_str: for item in filter_str: if re.match(item, line): @@ -98,12 +108,14 @@ line = config_file.readline() return specific_info + + def write_specific_info_to_config(file_path, start_line, end_line, config_info): if not os.path.exists(file_path): return ('Can not find the file(%s)!' % os.path.basename(file_path)) try: config_file = open(file_path, 'r') - config_file_bak = open(file_path+'_bak', 'w') + config_file_bak = open(file_path + '_bak', 'w') except IOError: return ('Open file failed!') line = config_file.readline() @@ -120,8 +132,10 @@ line = config_file.readline() config_file.close() config_file_bak.close() - os.rename(file_path+'_bak', file_path) + os.rename(file_path + '_bak', file_path) return None + + def delete_specific_info_from_config(file_path, start_line, end_line, specific_info): if not os.path.exists(file_path): return ('Can not find the file(%s)!' % os.path.basename(file_path)) @@ -135,6 +149,8 @@ if ret: return ret return None + + def insert_vsite(vsite_config, device_name): db = DB.get_connected_db() delete_sql = "DELETE FROM vsite_list where device_name='%s'" % (device_name) @@ -145,12 +161,13 @@ vsite['vsite_domain'] = vsite['vsite_domain'] if 'vsite_domain' in vsite else "" vsite['vsite_ip'] = vsite['vsite_ip'] if 'vsite_ip' in vsite else "" save_sql = "INSERT INTO vsite_list(vs_name, device_name, site_FQDM, ip, site_type, description, parent_site) values('%(vsite_name)s', '%(device_name)s', '%(vsite_domain)s', '%(vsite_ip)s', '%(vsite_type)s', '%(vsite_des)s', '%(vsite_parent)s')" % vsite - #select_sql = "SELECT id from vsite_list where vs_name='%s' and device_name='%s'" % (vsite['vsite_name'], device_name) + # select_sql = "SELECT id from vsite_list where vs_name='%s' and device_name='%s'" % (vsite['vsite_name'], device_name) db.execute_sql(save_sql) mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'VsiteConfig'])) db.close() return + def parse_vsite_config_from_configfile(file_path, device_name): if not os.path.exists(file_path): return @@ -159,12 +176,16 @@ except IOError: return - vsite_config = get_specific_info_from_config(config_file, "#virtual site global configuration\n", "#session global configuration\n") + vsite_config = get_specific_info_from_config(config_file, "#virtual site global configuration\n", + "#session global configuration\n") config_file.close() - rtn = cli_parse(vsite_config,[ - RegexParser('virtual site name "(?P.*?)" "(?P.*?)" "(?P.*?)" "(?P.*?)"', MATCHALL), - RegexParser('virtual site ip "(?P.*?)" (?P[0-9|\.|:|\w]+) (?P[0-9]+)', MATCHALL), - RegexParser('virtual site domain "(?P.*?)" "(?P.*?)"', MATCHALL)]) + rtn = cli_parse(vsite_config, [ + RegexParser( + 'virtual site name "(?P.*?)" "(?P.*?)" "(?P.*?)" "(?P.*?)"', + MATCHALL), + RegexParser('virtual site ip "(?P.*?)" (?P[0-9|\.|:|\w]+) (?P[0-9]+)', + MATCHALL), + RegexParser('virtual site domain "(?P.*?)" "(?P.*?)"', MATCHALL)]) new_vsite_config = [] for item in rtn: for vs in item: @@ -180,7 +201,8 @@ new_vsite_config.append(vs) return insert_vsite(new_vsite_config, device_name) -#just for vs _filter + +# just for vs _filter def parse_specific_config(asso_data, start_line, end_line, regex_str, regex_flag): vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + asso_data['device_name'] + '-' + asso_data['vs_name'] @@ -191,9 +213,9 @@ except IOError: return ('Open file(%s) failed!' % asso_data['vs_name']) vpn_config = get_specific_info_from_config(vs_file, start_line, end_line) - rtn = cli_parse(vpn_config,[#role name "bug_role" "bug_role" 1 - RegexParser(regex_str, regex_flag), - ]) + rtn = cli_parse(vpn_config, [ # role name "bug_role" "bug_role" 1 + RegexParser(regex_str, regex_flag), + ]) vs_file.close() return rtn @@ -230,7 +252,7 @@ continue config_file_bak.write(vsite_line) vsite_line = vsite_file.readline() - + if line: config_file_bak.write(end_line) line = config_file.readline() @@ -242,6 +264,8 @@ config_file_bak.close() vsite_file.close() os.rename(device_path_bak, device_path) + + def remove_vsite_from_config(vsite_name, device_name): device_path = DEFAULT_CONFIG_FILE_PATH + 'device/' + device_name device_path_bak = DEFAULT_CONFIG_FILE_PATH + 'device/' + device_name + '_bak' @@ -254,8 +278,8 @@ specific_line = "# Virtual site '%s' configurations\n" % vsite_name end_line = "####################################\n" line = config_file.readline() - - flag = True + + flag = True while line: if not flag and line == end_line: break @@ -274,6 +298,8 @@ config_file.close() config_file_bak.close() os.rename(device_path_bak, device_path) + + def add_vsite_to_config(vsite_name, device_name): vsite_path = DEFAULT_VSITE_PATH + device_name + '-' + vsite_name device_path = DEFAULT_CONFIG_FILE_PATH + 'device/' + device_name @@ -303,10 +329,11 @@ return +# just for vs _insert insert_or_delete: True-insert False delete +INSERT = True +DELETE = False -#just for vs _insert insert_or_delete: True-insert False delete -INSERT=True -DELETE=False + def change_config_file(asso_data, specific_line, cli_str, insert_or_delete): vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + asso_data['device_name'] + '-' + asso_data['vs_name'] @@ -318,7 +345,7 @@ except IOError: return ('Open file(%s) failed!' % os.path.basename(src_vs)) try: - dest_vs = open(vs_file_bak, 'w') + dest_vs = open(vs_file_bak, 'w') except IOError: return ('Open file(%s) failed!' % os.path.basename(dest_vs)) @@ -338,13 +365,15 @@ os.rename(vs_file_bak, vs_file_path) - #sync to device config file + # sync to device config file sync_vsite_to_config(asso_data['vs_name'], asso_data['device_name']) return + + # #just for vs _delete # def delete_config_from_file(asso_data, regex_str): -#parse diff config +# parse diff config def parse_diff_config(diff_list, result_dict): original = [] for item in diff_list.splitlines(): @@ -356,4 +385,4 @@ result_dict['add'].append(item) original.append(item) result_dict['original'] = ("\n").join(original) - #print result_dict + # print result_dict Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py (working copy) @@ -1,45 +1,33 @@ -# python sqlite -# Author : zhangjw -# Create : 2016-02-22 -# Version: 1.0 -# DB-API interface for SQLite databases - import psycopg2 +AN_DB_NAME = "cm" +AN_USER = "amp_admin" +AN_USER_PASSWORD = "Array@123$" + class DB(object): def __init__(self, conn): self.conn = conn - # Enabling Foreign Key Support - # cu = self.conn.cursor() - # cu.execute('PRAGMA foreign_keys = ON;') - # self.conn.commit() - # cu.close() @classmethod def get_connected_db(cls, path=None): - # conn = psycopg2.connect(database="cm", user="postgres", password="", host="10.23.0.142", port="5432") - conn = psycopg2.connect(database="cm", - user="postgres", - password="", + conn = psycopg2.connect(database=AN_DB_NAME, + user=AN_USER, + password=AN_USER_PASSWORD, host="127.0.0.1", port="5432") - if conn: - return DB(conn) - else: - return None + return DB(conn) if conn else None def get_cursor(self): return self.conn.cursor() - def execute_sql(self, sql): + def execute_sql(self, sql, params=None): assert sql - cu = self.get_cursor() - cu.execute(sql) - self.conn.commit() - cu.close() + with self.conn.cursor() as cu: + cu.execute(sql, params or ()) + self.conn.commit() - def execute_sql_ingnore_exception(self, sql): + def execute_sql_ignore_exception(self, sql): try: self.execute_sql(sql) except Exception as e: @@ -47,564 +35,31 @@ self.conn.rollback() def close(self): - self.conn.close() + if self.conn: + self.conn.close() + self.conn = None - def fetchall(self, sql): + def fetchall(self, sql, params=None): assert sql - cu = self.get_cursor() - cu.execute(sql) - result = cu.fetchall() - cu.close() - return result + with self.conn.cursor() as cu: + cu.execute(sql, params or ()) + return cu.fetchall() def fetchone(self, sql): assert sql - cu = self.get_cursor() - cu.execute(sql) - result = cu.fetchone() - cu.close() - return result + with self.conn.cursor() as cu: + cu.execute(sql) + return cu.fetchone() def drop_table(self, table): assert table sql = 'DROP TABLE IF EXISTS ' + table - cu = self.get_cursor() - cu.execute(sql) - self.conn.commit() - cu.close() + self.execute_sql(sql) def execute_sql_with_data(self, sql, data): assert sql assert data - cu = self.get_cursor() - for d in data: - cu.execute(sql, d) + with self.conn.cursor() as cu: + for d in data: + cu.execute(sql, d) self.conn.commit() - cu.close() - - # following functions just be called when CM package being installed or be called to create data for testing - # def create_table_device(self): - # print('Creating table device...') - # create_table_sql = '''CREATE TABLE IF NOT EXISTS `device` ( - # `id` varchar(50) NOT NULL, - # `zone` varchar(10) NOT NULL, - # `ip` varchar(20) NOT NULL, - # `restapi_port` INTEGER DEFAULT 9997, - # `restapi_username` varchar(20) DEFAULT NULL, - # `restapi_password` varchar(20) DEFAULT NULL, - # `connection` varchar(10) NOT NULL DEFAULT 'unconnected', - # `status` varchar(10) NOT NULL DEFAULT 'new', - # `version` varchar(5000) DEFAULT NULL, - # `license_name` varchar(10) DEFAULT NULL, - # PRIMARY KEY (`id`), - # FOREIGN KEY (license_name) REFERENCES license(name) ON DELETE SET NULL, - # FOREIGN KEY (zone) REFERENCES proxy_cm(name) ON DELETE CASCADE - # )''' - # self.execute_sql(create_table_sql) - def create_table_device(self): - print('Creating table device...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS device ( - id varchar(64) NOT NULL, - zone varchar(32) NOT NULL, - name varchar(32) NOT NULL UNIQUE, - ip_address varchar(64) NOT NULL, - restapi_port INTEGER DEFAULT 9997, - restapi_username varchar(16) DEFAULT NULL, - restapi_password varchar(256) DEFAULT NULL, - console_username varchar(16) DEFAULT NULL, - console_password varchar(256) DEFAULT NULL, - connection varchar(16) NOT NULL DEFAULT 'unconnected', - status varchar(16) NOT NULL DEFAULT 'new', - version varchar(8192) DEFAULT NULL, - license_key varchar(128) DEFAULT NULL, - gateway_domain varchar(128) DEFAULT NULL, - location varchar(128) NOT NULL, - firewall_ip varchar(64) NOT NULL, - intranet_ip varchar(64) NOT NULL, - type varchar(16) NOT NULL, - extend_fields varchar(8192) DEFAULT NULL, - own varchar(64) DEFAULT NULL, - log_enable INTEGER DEFAULT 0, - snmp_enable INTEGER DEFAULT 0, - webui_port INTEGER DEFAULT 8888, - device_group varchar(64) NOT NULL, - extend jsonb, - PRIMARY KEY (id), - FOREIGN KEY (zone) REFERENCES proxy_cm(name) ON DELETE CASCADE, - FOREIGN KEY (device_group) REFERENCES device_group(name) ON DELETE CASCADE - ); - ALTER TABLE device DROP COLUMN IF EXISTS snmp_enable; - ALTER TABLE device DROP COLUMN IF EXISTS snmp_general; - ALTER TABLE device ADD COLUMN IF NOT EXISTS snmp_general varchar(256) NOT NULL DEFAULT '{"snmp_enable": false}'; - ALTER TABLE device ADD COLUMN IF NOT EXISTS enable_password varchar(32) DEFAULT NULL; - ALTER TABLE device ADD COLUMN IF NOT EXISTS create_time varchar(40) DEFAULT (now()); - ALTER TABLE device ADD COLUMN IF NOT EXISTS protocol varchar(16) DEFAULT 'restapi'; - ''' - self.execute_sql(create_table_sql) - - def create_table_ip_pool(self): - print('Creating table ip_pool...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS ip_pool ( - id serial primary key , - ip_pool_range varchar(128) NOT NULL, - device_id varchar(64) NOT NULL REFERENCES device(id) ON DELETE CASCADE, - extend jsonb - )''' - self.execute_sql(create_table_sql) - # FOREIGN KEY (device_id) REFERENCES device(id) ON DELETE CASCADE - - def create_table_update(self): - print('Creating table update_list...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS update_list ( - id serial primary key , - app_name varchar(32) NOT NULL, - build_version varchar(64) NOT NULL, - file_size INTEGER DEFAULT 0, - md5_value varchar(64) DEFAULT NULL, - download_link varchar(128) DEFAULT NULL, - location varchar(128) DEFAULT NULL, - extend jsonb - )''' - self.execute_sql(create_table_sql) - - def create_table_service(self): - print('Creating table service...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS service( - id serial primary key, - name varchar(64), - device_type varchar(32) NOT NULL, - service_name varchar(128) NOT NULL, - service_type varchar(32) NOT NULL, - extend jsonb - )''' - self.execute_sql(create_table_sql) - - def create_table_task(self): - print('Creating table task...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS task( - id serial primary key , - name varchar(256) NOT NULL, - type varchar(16) NOT NULL, - trigger varchar(16) NOT NULL, - state varchar(16) NOT NULL, - next_run_time varchar(64) NOT NULL, - description text, - failed_times integer default 3, - scheduler varchar(1024) default NULL, - custom_fields varchar(1024) default NULL, - result_field varchar(1024) default NULL, - device_list varchar(1024) default NULL, - extend jsonb - ); - ALTER TABLE task ALTER COLUMN scheduler TYPE VARCHAR(10240); - ''' - self.execute_sql(create_table_sql) - - def create_table_config_template(self): - print('Creating table config_template...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS config_template( - key varchar(128) primary key, - default_value varchar(128) NOT NULL, - description varchar(256) - ); - ''' - self.execute_sql(create_table_sql) - - def create_table_device_config_template(self): - print('Creating table device_config_template...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS device_config_template( - id serial NOT NULL, - device_id varchar(64) NOT NULL, - key varchar(64) NOT NULL, - value varchar(64) NOT NULL, - PRIMARY KEY (id) - ); - ''' - self.execute_sql(create_table_sql) - - def create_table_vsite_list(self): - print('Creating table vsite_list') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS vsite_list( - id serial primary key , - vs_name varchar(64) NOT NULL, - device_name varchar(32) NOT NULL, - site_FQDM varchar(128) DEFAULT NULL, - ip varchar(128) NOT NULL, - site_type varchar(32) NOT NULL, - description varchar(128) DEFAULT NULL, - parent_site varchar(64) DEFAULT NULL, - extend jsonb - ) ''' - self.execute_sql(create_table_sql) - - def create_table_device_group(self): - print('Creating table device_group...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS device_group ( - name character varying(128) COLLATE pg_catalog."default" NOT NULL, - CONSTRAINT device_group_pkey PRIMARY KEY (name) - );''' - self.execute_sql(create_table_sql) - - def create_table_ha_cluster(self): - print('Creating table ha_cluster...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS ha_cluster( - id serial primary key , - name varchar(64) NOT NULL, - device_type varchar(16) default NULL, - extend_fields varchar(2048) default null - ) ''' - self.execute_sql(create_table_sql) - - def create_table_hc_device(self): - print('Creating table hc_device...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS hc_device( - g_id serial references ha_cluster(id) on delete cascade, - d_id varchar(64) references device(id) on delete cascade, - unique(d_id, g_id) - )''' - self.execute_sql(create_table_sql) - - def create_table_file_type(self): - print('Creating table file_type...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS file_type( - name varchar(32) primary key, - extend jsonb - );''' - self.execute_sql(create_table_sql) - save_sql = '''INSERT INTO file_type(name) values ('device')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('command')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('diff')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('system')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('backup')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('customize')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('template')''' - self.execute_sql_ingnore_exception(save_sql) - save_sql = '''INSERT INTO file_type(name) values ('vs')''' - self.execute_sql_ingnore_exception(save_sql) - - def create_table_file(self): - print('Creating table file_list...') - # when type is backup or device , use 'modify_time' to save device name. - create_table_sql = ''' CREATE TABLE IF NOT EXISTS file_list( - name varchar(128) primary key, - create_time varchar(64) NOT NULL, - modify_time varchar(64) DEFAULT NULL, - type varchar(32) NOT NULL, - device_type varchar(32) DEFAULT NULL, - comment text, - FOREIGN KEY (type) REFERENCES file_type(name) ON DELETE CASCADE, - extend jsonb - );''' - self.execute_sql(create_table_sql) - - def create_table_tar_file(self): - print('Creating table tar file...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS tar_file( - id serial primary key , - name varchar(64) NOT NULL, - vsite_name varchar(64) DEFAULT NULL, - extend jsonb - ) ''' - self.execute_sql(create_table_sql) - - def create_table_proxy_cm(self): - print('Creating table proxy_cm...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS proxy_cm ( - name varchar(32) NOT NULL, - ip varchar(64) NOT NULL, - PRIMARY KEY (name), - extend jsonb - )''' - self.execute_sql(create_table_sql) - save_sql = '''INSERT INTO proxy_cm(name, ip) values ('default', '0.0.0.0')''' - self.execute_sql_ingnore_exception(save_sql) - - def create_table_zone(self): - print('Creating table zone...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS zone ( - name varchar(32) NOT NULL, - PRIMARY KEY (name) - )''' - self.execute_sql(create_table_sql) - - def create_table_cm_role(self): - print('Creating table role...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS cm_role ( - name varchar(64) primary key, - description varchar(32) NOT NULL, - priority integer DEFAULT 1, - status integer DEFAULT 0, - time varchar(64) NOT NULL, - device_list varchar(3328) DEFAULT NULL, - extend jsonb - ) - ''' - self.execute_sql(create_table_sql) - - def create_table_cm_acl_resource(self): - print('Creating table acl resource...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS cm_acl_resource ( - id serial primary key , - group_name varchar(64) NOT NULL, - resource varchar(512) NOT NULL, - description varchar(32) NOT NULL, - status integer DEFAULT 0, - time varchar(64) NOT NULL, - device_list varchar(3328) DEFAULT NULL, - extend jsonb - ) - ''' - self.execute_sql(create_table_sql) - - def create_table_cm_vpn_resource(self): - print('Creating table vpn resource...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS cm_vpn_resource ( - id serial primary key , - group_name varchar(64) NOT NULL, - resource varchar(128) NOT NULL, - type integer default 1, - group_type varchar(32) DEFAULT 'included', - description varchar(32) NOT NULL, - status integer DEFAULT 0, - time varchar(64) NOT NULL, - device_list varchar(3328) DEFAULT NULL, - extend jsonb - ) - ''' - self.execute_sql(create_table_sql) - - def create_table_audit_user(self): - print('Creating table audit_user...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS audit_user( - id serial primary key , - user_name varchar(16) NOT NULL, - vsite_name varchar(64) NOT NULL, - host_name varchar(64) NOT NULL, - extend jsonb - ); - ALTER TABLE audit_user ADD COLUMN IF NOT EXISTS lasted timestamp with time zone; - ''' - self.execute_sql(create_table_sql) - - def create_table_system_user(self): - print('Creating table system_user...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS system_user( - name_name varchar(64) primary key, - level varchar(16) NOT NULL, - phone_num varchar(64) default NULL, - email varchar(64) default NULL, - extend jsonb - ) - ''' - self.execute_sql(create_table_sql) - - def create_table_extension_log(self): - print('Creating table extension log...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS ext_log ( - name varchar(64) NOT NULL, - action varchar(20) NOT NULL, - time varchar(30) NOT NULL, - result varchar(20) NOT NULL - ); - ''' - add_column_sql = ''' - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name='ext_log' AND column_name='id' - ) THEN - ALTER TABLE ext_log ADD COLUMN id BIGSERIAL; - END IF; - END - $$; - ''' - add_pk_sql = ''' - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.table_constraints - WHERE table_name='ext_log' AND constraint_type='PRIMARY KEY' - ) THEN - ALTER TABLE ext_log ADD CONSTRAINT ext_log_pkey PRIMARY KEY (id); - END IF; - END - $$; - ''' - - self.execute_sql(create_table_sql) - self.execute_sql(add_column_sql) - self.execute_sql(add_pk_sql) - - def create_table_user_authorization(self): - print('Creating table user_authorization...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS user_authorization ( - id serial primary key, - username varchar(128) NOT NULL UNIQUE, - auth varchar(8192) DEFAULT NULL - ); - ALTER TABLE user_authorization ADD COLUMN IF NOT EXISTS root INTEGER DEFAULT 0; - ALTER TABLE user_authorization ADD COLUMN IF NOT EXISTS role_id INTEGER; - ALTER TABLE user_authorization ADD COLUMN IF NOT EXISTS user_type varchar(32); - ALTER TABLE user_authorization DROP CONSTRAINT IF EXISTS role_foreign_key; - ALTER TABLE user_authorization ADD CONSTRAINT role_foreign_key FOREIGN KEY - (role_id) REFERENCES role (id); - UPDATE user_authorization set user_type='Super Admin' where id=1; - ''' - self.execute_sql(create_table_sql) - - def create_table_operation_log(self): - print('Creating table operation_log...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS operation_log ( - id serial primary key, - username varchar(128) NOT NULL, - client_ip varchar(64) NOT NULL, - time varchar(30) NOT NULL, - level varchar(20), - module varchar(20), - operation varchar(1024) - )''' - self.execute_sql(create_table_sql) - - def create_table_schedule_backup_all(self): - print('Creating table schedule_backup_all...') - create_table_sql = '''CREATE TABLE IF NOT EXISTS schedule_backup_all ( - task_name varchar(128) NOT NULL UNIQUE, - device_name varchar(128) NOT NULL, - interval INTEGER NOT NULL, - unit varchar(10) NOT NULL - )''' - self.execute_sql(create_table_sql) - - def create_log_settings_table(self): - print('Creating table log_settings...') - create_table_sql = ''' CREATE TABLE IF NOT EXISTS log_settings( - enable integer default 1, - level INTEGER default 2 - )''' - self.execute_sql(create_table_sql) - - def create_log_host_table(self): - print('Creating table log_host...') - # protocol: 0 - TCP, 1 - UDP - create_table_sql = ''' CREATE TABLE IF NOT EXISTS log_host( - host varchar(128), - port integer NOT NULL, - protocol integer default 1, - PRIMARY KEY (host, port, protocol) - )''' - self.execute_sql(create_table_sql) - - def create_table_role(self): - print('Creating role table...') - create_table_query = ''' - CREATE TABLE IF NOT EXISTS role( - id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ), - role_name varchar(128), - created_at timestamp without time zone, - CONSTRAINT role_primary_key PRIMARY KEY (id) - ) - ''' - self.execute_sql(create_table_query) - - def create_table_role_device_group(self): - print('Creating role_device_group table...') - create_table_query = ''' - CREATE TABLE IF NOT EXISTS role_device_group ( - id serial primary key, - role_id integer, - device_group_name varchar(128), - created_at timestamp without time zone, - FOREIGN KEY (role_id) REFERENCES role (id) ON DELETE CASCADE, - FOREIGN KEY (device_group_name) REFERENCES device_group(name) ON DELETE CASCADE - );''' - self.execute_sql(create_table_query) - - def create_table_backup_schedule(self): - print('Creating table backup_schedule...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS backup_schedule ( - id SERIAL PRIMARY KEY, - frequency varchar(64) NOT NULL, - time TIME NOT NULL, - day_of_the_week integer DEFAULT NULL, - day_of_the_month integer DEFAULT NULL, - month integer DEFAULT NULL CHECK (month BETWEEN 1 AND 12), - destination varchar(64) DEFAULT NULL - ); - ''' - self.execute_sql(create_table_sql) - - def create_table_remote_storage(self): - print('Creating table remote_storage...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS remote_storage ( - id SERIAL PRIMARY KEY, - ip varchar(64) NOT NULL, - username varchar(64) NOT NULL, - password varchar(64) NOT NULL, - path varchar(64) DEFAULT NULL - ); - ''' - self.execute_sql(create_table_sql) - - def create_table_backups(self): - print('Creating table backups...') - create_table_sql = ''' - CREATE TABLE IF NOT EXISTS backups ( - id SERIAL PRIMARY KEY, - filename varchar(64) NOT NULL, - status varchar(64) DEFAULT NULL, - destination varchar(64) DEFAULT NULL, - time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - ''' - self.execute_sql(create_table_sql) - - -if __name__ == '__main__': - db = DB.get_connected_db() - db.create_table_cm_role() - db.create_table_zone() - db.create_table_cm_acl_resource() - db.create_table_cm_vpn_resource() - db.create_table_device_group() - db.create_table_ha_cluster() - db.create_table_vsite_list() - db.create_table_tar_file() - db.create_table_proxy_cm() - db.create_table_device() - db.create_table_hc_device() - db.create_table_service() - db.create_table_ip_pool() - db.create_table_update() - db.create_table_task() - db.create_table_file_type() - db.create_table_file() - db.create_table_audit_user() - db.create_table_extension_log() - db.create_table_role() - db.create_table_user_authorization() - db.create_table_operation_log() - db.create_table_schedule_backup_all() - db.create_log_settings_table() - db.create_log_host_table() - db.create_table_config_template() - db.create_table_device_config_template() - db.create_table_role_device_group() - db.create_table_backup_schedule() - db.create_table_remote_storage() - db.create_table_backups() - db.close() \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/send_email.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/send_email.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/send_email.py (working copy) @@ -1,8 +1,9 @@ +import smtplib +from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.mime.application import MIMEApplication -import smtplib + class SendMail(object): def __init__(self, smtpserver, smtpport, smtpsender, smtppassword): self.smtpserver = smtpserver.strip() @@ -47,7 +48,7 @@ if ret: return ret msg = MIMEText(body_text, "plain", "utf-8") - #msg = MIMEText(body_text, "html", "utf-8") + # msg = MIMEText(body_text, "html", "utf-8") msg['Subject'] = subject msg['From'] = self.smtpsender msg['To'] = ','.join(receiver) @@ -59,7 +60,7 @@ ret = self.login() if ret: return ret - + msg = MIMEMultipart() msg.attach(MIMEText(body_text, "plain", "utf-8")) msg['Subject'] = subject @@ -73,37 +74,37 @@ return self.send_email(receiver, msg) + def send_emails(smtpserver, smtpport, sender, password, receivers, subject, body_text, file_path, filename): - msg = MIMEMultipart() - msg['Subject'] = subject - msg['From'] = sender - msg['To'] = receivers + msg = MIMEMultipart() + msg['Subject'] = subject + msg['From'] = sender + msg['To'] = receivers - #Mail Text Body - puretext = MIMEText(body_text) - msg.attach(puretext) + # Mail Text Body + puretext = MIMEText(body_text) + msg.attach(puretext) - #Attachment - attachment_part = MIMEApplication(open(file_path+'/%s'%filename, 'rb').read()) - attachment_part.add_header('Content-Disposition', 'attachment', filename=filename) - msg.attach(attachment_part) + # Attachment + attachment_part = MIMEApplication(open(file_path + '/%s' % filename, 'rb').read()) + attachment_part.add_header('Content-Disposition', 'attachment', filename=filename) + msg.attach(attachment_part) - - #Send Emails - try: - client = smtplib.SMTP() - client.connect(smtpserver, smtpport) - client.starttls() - #client.set_debuglevel(1) - client.login(sender, password) - client.sendmail(sender, receivers, msg.as_string()) - client.quit() - print('Send Emails successfully') - except smtplib.SMTPRecipientsRefused: - print('Recipient refused') - except smtplib.SMTPAuthenticationError: - print('Auth error') - except smtplib.SMTPSenderRefused: - print('Sender refused') - except smtplib.SMTPException as e: - print(e.message) + # Send Emails + try: + client = smtplib.SMTP() + client.connect(smtpserver, smtpport) + client.starttls() + # client.set_debuglevel(1) + client.login(sender, password) + client.sendmail(sender, receivers, msg.as_string()) + client.quit() + print('Send Emails successfully') + except smtplib.SMTPRecipientsRefused: + print('Recipient refused') + except smtplib.SMTPAuthenticationError: + print('Auth error') + except smtplib.SMTPSenderRefused: + print('Sender refused') + except smtplib.SMTPException as e: + print(e.message) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py (working copy) @@ -1,30 +1,32 @@ -#from cm.lib.sqlite_db import DB -from cm.lib.postgres_db import DB -from hive.imports.model import * -from hive.utils import andebug, get_device_type, standard_model_type -from apscheduler.schedulers.background import BackgroundScheduler +import datetime +import difflib +import logging.handlers +import shutil +from importlib import reload + +import requests from apscheduler.events import * -from cm.conf import NETWORK_CARD +from apscheduler.schedulers.background import BackgroundScheduler + +from cm.lib.communication import send_https_rest_request, modify_url, call_restapi +from cm.lib.libbasic_operation import oper_log, send_cli_to_device +from cm.lib.libbasic_operation import send_command_to_device, get_rest_info_from_device, get_ip_address, \ + send_xmlrpc_to_device +from cm.lib.libconfig import Backup_AG, Backup_APV, Recover_AG, Recover_APV, Backup_Custom, Apply_AG +from cm.lib.libmonitor import Decision, SSLCertDecision +from cm.lib.parse_configfile import parse_vsite_from_configfile, parse_vsite_config_from_configfile, parse_diff_config, \ + TarFiles +from cm.lib.postgres_db import DB from djproject.settings import FILE_UPLOAD_DIR, CONFIG_UPLOAD_STORE_DIR -import hashlib, json, time, logging, logging.handlers, threading, re -import os, shutil, requests, datetime, difflib -from hive.model.query import mark_expire_all +from hive.imports.model import * from hive.model.loading import get_model -from cm.lib.communication import send_https_rest_request, modify_url, call_restapi -from cm.lib.libbasic_operation import send_command_to_device, get_rest_info_from_device, get_ip_address, md5sum, send_xmlrpc_to_device -import hashlib, uuid, tarfile -from cm.lib.parse_configfile import parse_vsite_from_configfile, DEFAULT_CONFIG_FILE_PATH, parse_vsite_config_from_configfile, parse_diff_config, TarFiles +from hive.model.query import mark_expire_all from hive.notification import send_notification -from cm.lib.libmonitor import Decision, SSLCertDecision -from cm.lib.libconfig import Backup_AG, Backup_APV, Recover_AG, Recover_APV, Backup_Custom, Apply_AG -#from cm.models.device_mgmt.device import get_rest_info_from_device -#from cm.models.configuration.update import update_updatelist -from djproject.an_settings import * -import socket -from cm.lib.libbasic_operation import oper_log, send_cli_to_device +from hive.utils import get_device_type logger = logging.getLogger('hive.debug') + def append_job_description(name, description): db = DB.get_connected_db() select_sql = "SELECT description FROM task WHERE name='%s'" % name @@ -36,19 +38,27 @@ db.close() + def remove_job_from_database(name): db = DB.get_connected_db() del_sql = "DELETE FROM task WHERE name='%s' " % name db.execute_sql(del_sql) db.close() -def add_job_into_database(name, type, description, trigger, state, next_run_time, scheduler, failed_times=3, device_list=''): + +def add_job_into_database(name, type, description, trigger, state, next_run_time, scheduler, failed_times=3, + device_list=''): db = DB.get_connected_db() - save_sql = "INSERT INTO task(name, type, description, trigger, state, next_run_time, scheduler, failed_times, device_list) values('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%d', '%s') " % (name, type, description, trigger, state, next_run_time, scheduler, failed_times, device_list) + save_sql = "INSERT INTO task(name, type, description, trigger, state, next_run_time, scheduler, failed_times, device_list) values('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%d', '%s') " % ( + name, type, description, trigger, state, next_run_time, scheduler, failed_times, device_list) db.execute_sql(save_sql) db.close() - oper_log('info', 'system', 'Add a new task. Task name: %s, task type: %s, task description: %s, task trigger: %s.' % (name, type, description, trigger)) + oper_log('info', 'system', + 'Add a new task. Task name: %s, task type: %s, task description: %s, task trigger: %s.' % (name, type, + description, + trigger)) + def update_job_state(name, state): db = DB.get_connected_db() update_sql = "UPDATE task SET state='%s' WHERE name='%s'" % (state, name) @@ -56,12 +66,14 @@ db.close() oper_log('info', 'system', 'Update task state. Task name: %s, task result: %s.' % (name, state)) + def update_job_runtime(name, time): db = DB.get_connected_db() update_sql = "UPDATE task SET next_run_time='%s' WHERE name='%s'" % (time, name) db.execute_sql(update_sql) db.close() + def update_job_result_field(name, value, username): db = DB.get_connected_db() update_sql = "UPDATE task SET result_field='%s' WHERE name='%s'" % (value, name) @@ -69,24 +81,28 @@ db.close() oper_log('info', 'system', 'Update task state. Task name: %s, task result: %s.' % (name, value)) + def update_job_failed(name, failed_times): db = DB.get_connected_db() update_sql = "UPDATE task SET failed_times='%d' WHERE name='%s'" % (failed_times, name) db.execute_sql(update_sql) - db.close() + db.close() + def get_job_failed(name): db = DB.get_connected_db() select_sql = "SELECT failed_times FROM task WHERE name='%s'" % name data = db.fetchall(select_sql) return data[0][0] + def update_job_custom_field(name, values): db = DB.get_connected_db() update_sql = "UPDATE task SET custom_fields='%s' WHERE name='%s'" % (values, name) db.execute_sql(update_sql) - db.close() + db.close() + def add_description(task_name, values): description = values + ' ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' append_job_description(task_name, description) @@ -95,6 +111,8 @@ ''' start upload job ''' + + def update_updatelist(file_size, id, path): file_path = CONFIG_UPLOAD_STORE_DIR + os.path.basename(path) update_sql = "UPDATE update_list SET file_size='%d', location='%s' WHERE id='%d'" % (file_size, file_path, id) @@ -102,6 +120,8 @@ db.execute_sql(update_sql) db.close() mark_expire_all(get_model('cm', ['configuration', 'update', 'Update'])) + + def get_file_md5(filename): myhash = hashlib.md5() f = open(filename, 'rb') @@ -111,8 +131,10 @@ break myhash.update(b) f.close() - file_md5 = myhash.hexdigest() - return file_md5 + file_md5 = myhash.hexdigest() + return file_md5 + + def compare_md5(file_md5, id): select_sql = "SELECT md5_value FROM update_list WHERE id='%d'" % id db = DB.get_connected_db() @@ -124,6 +146,8 @@ return True else: return False + + def mv_file(path): file_name = os.path.basename(path) file_path = CONFIG_UPLOAD_STORE_DIR + file_name @@ -131,12 +155,13 @@ os.remove(file_path) shutil.move(path, file_path) + def get_file_from_url(get_url): file_name = get_url.split('/')[-1] - get_params = {'filename':file_name} + get_params = {'filename': file_name} full_path = FILE_UPLOAD_DIR + file_name try: - rt = requests.get(url = get_url, stream = True, params = get_params) + rt = requests.get(url=get_url, stream=True, params=get_params) except: return None if not os.path.exists(FILE_UPLOAD_DIR): @@ -146,6 +171,8 @@ f.write(chunk) f.flush() return full_path + + def upload_job(update_list_id, name, tmp_url): description = "The execution time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' append_job_description(name, description) @@ -153,7 +180,7 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) path = get_file_from_url(tmp_url) if not path: - #raise ModelQueryException(CLICmdError(__('Getting file failed, please check your url!'))) + # raise ModelQueryException(CLICmdError(__('Getting file failed, please check your url!'))) append_job_description(name, "Getting file failed, please check your url!\n") update_job_state(name, 'failed') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) @@ -164,7 +191,7 @@ mv_file(path) update_updatelist(file_size, update_list_id, path) else: - #raise ModelQueryException(CLICmdError(__('Md5 error, please check the file!'))) + # raise ModelQueryException(CLICmdError(__('Md5 error, please check the file!'))) append_job_description(name, "Md5 error, please check your url!\n") update_job_state(name, 'failed') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) @@ -176,6 +203,8 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) send_notification('success', 'Task <%s> has done.' % name, model='image') return + + ''' end upload job ''' @@ -183,6 +212,8 @@ ''' start update job ''' + + def update_failed(description, task_name): add_description(task_name, description) update_job_state(task_name, 'failed') @@ -196,12 +227,14 @@ db.close() return data.split('/')[-1] if data else None -def check_package_from_device(update_list_id, filename, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info): + +def check_package_from_device(update_list_id, filename, task_name, ip_address, restapi_port, restapi_username, + restapi_password, device_type, device_info): db = DB.get_connected_db() fetchall_sql = '''SELECT md5_value, file_size FROM update_list where id='%d' ''' % update_list_id data = db.fetchall(fetchall_sql) db.close() - #4 | image3 | Rel_APV_10_3_1_9 | 515895165 | fdce12389937be070d2da9d058423a34 | http://10.23.0.4/10.3.1/ArrayOS-Rel_APV_10_3_1_9.array | /ca/package/ArrayOS-Rel_APV_10_3_1_9.array | + # 4 | image3 | Rel_APV_10_3_1_9 | 515895165 | fdce12389937be070d2da9d058423a34 | http://10.23.0.4/10.3.1/ArrayOS-Rel_APV_10_3_1_9.array | /ca/package/ArrayOS-Rel_APV_10_3_1_9.array | md5_value = data[0][0] file_size = data[0][1] @@ -216,7 +249,8 @@ time.sleep(3) if device_info['protocol'] == "restapi": new_url = modify_url('/rest/device_type/cli_extend', device_type) - rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, + restapi_username, restapi_password) else: rest_response_data = send_xmlrpc_to_device(device_info, '', cli_cmd) except Exception as e: @@ -240,10 +274,9 @@ add_description(task_name, description) return False - if contents.split('\n')[1]: file_info = contents.strip().split('\n')[1].split(' ') - #apv: ['515895165', '', '', '', 'Dec', '28', '2019', '05:49:18', 'ArrayOS-Rel_APV_10_3_1_9.array', '', 'fdce12389937be070d2da9d058423a34', '', '/var/crash/package/ArrayOS-Rel_APV_10_3_1_9.array'] + # apv: ['515895165', '', '', '', 'Dec', '28', '2019', '05:49:18', 'ArrayOS-Rel_APV_10_3_1_9.array', '', 'fdce12389937be070d2da9d058423a34', '', '/var/crash/package/ArrayOS-Rel_APV_10_3_1_9.array'] else: description = 'Error, device download the build failed.' add_description(task_name, description) @@ -275,26 +308,31 @@ return True + def update_custom_field(task_name, step, args): custom_field = {} - custom_field['update'] = {'step':step, 'args':args} + custom_field['update'] = {'step': step, 'args': args} custom_str = json.dumps(custom_field) update_job_custom_field(task_name, custom_str) -def waiting_for_device(version, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info): + +def waiting_for_device(version, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, + device_info): # description = 'Waiting for AG response: ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' # append_job_description(task_name, description) add_description(task_name, 'Waiting for device response: ') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - update_custom_field(task_name, 'waiting_for_device', [version, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info]) + update_custom_field(task_name, 'waiting_for_device', + [version, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, + device_info]) - time.sleep(60*2) + time.sleep(60 * 2) if isinstance(device_type, list): deviceType = device_type[0] if device_type[0].lower() in WAF_TYPE_LIST and device_type[1]: - time.sleep(60*6) + time.sleep(60 * 6) else: deviceType = device_type description = '' @@ -305,7 +343,8 @@ try: if device_info['protocol'] == 'restapi': new_url = modify_url('/rest/device_type/system/SystemInfo/version', deviceType) - rest_response_data = send_https_rest_request('GET', new_url, '', ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('GET', new_url, '', ip_address, restapi_port, + restapi_username, restapi_password) else: rest_response_data = send_xmlrpc_to_device(device_info, '', 'show version') except Exception as e: @@ -328,8 +367,8 @@ continue version_ag = rest_body['SystemInfo']['version'] rtn = cli_parse(version_ag, [ - RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), - ]) + RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), + ]) new_version = rtn[0][0] else: time.sleep(10) @@ -338,7 +377,7 @@ else: try: version_ag = json.loads(rest_response_data)[0]["output"] - rtn = cli_parse(version_ag, [ RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE)]) + rtn = cli_parse(version_ag, [RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE)]) new_version = rtn[0][0] except Exception as e: time.sleep(10) @@ -349,7 +388,7 @@ db.execute_sql(update_sql) db.close() break - #hide following code because the case "upgrade with build of current version will fail" + # hide following code because the case "upgrade with build of current version will fail" # if new_version != version: # db = DB.get_connected_db() # update_sql = "UPDATE device SET version='%s' WHERE ip_address='%s'" % (version_ag, ip_address) @@ -362,7 +401,7 @@ # break if not failed_time: - #add_description(task_name, description) + # add_description(task_name, description) update_failed(description, task_name) return False add_description(task_name, 'System update successful! ') @@ -372,17 +411,21 @@ return True -def system_update_device(file_name, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info): + +def system_update_device(file_name, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_info): # description = 'Send command to AG: ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' # append_job_description(task_name, description) add_description(task_name, 'Send command to Device: ') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - update_custom_field(task_name, 'system_update_device', [file_name, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info]) + update_custom_field(task_name, 'system_update_device', + [file_name, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_info]) if isinstance(device_type, list): deviceType = device_type[0] else: deviceType = device_type - + failed_time = get_job_failed(task_name) description = '' version = '' @@ -391,7 +434,8 @@ time.sleep(1) if device_info['protocol'] == 'restapi': new_url = modify_url('/rest/device_type/system/SystemInfo/version', deviceType) - rest_response_data = send_https_rest_request('GET', new_url, '', ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('GET', new_url, '', ip_address, restapi_port, + restapi_username, restapi_password) else: rest_response_data = send_xmlrpc_to_device(device_info, '', 'show version') except Exception as e: @@ -406,8 +450,8 @@ rest_body = json.loads(rest_response_data['body']) version = rest_body['SystemInfo']['version'] rtn = cli_parse(version, [ - RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), - ]) + RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), + ]) version = rtn[0][0] break else: @@ -423,8 +467,8 @@ if not failed_time: update_failed(description, task_name) return False - #add_description(task_name, description) - #return False + # add_description(task_name, description) + # return False description = '' update_cmd_str = 'system update "/var/crash/package/%s" \nYES\n' % file_name if isinstance(device_type, list): @@ -440,13 +484,16 @@ time.sleep(1) if device_info['protocol'] == 'restapi': new_url = modify_url('/rest/device_type/cli_extend', deviceType) - rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, + restapi_username, restapi_password) else: - rest_response_data = send_xmlrpc_to_device(device_info, '', 'system update "/var/crash/package/%s"\nYES' % file_name, mode="config_with_input") + rest_response_data = send_xmlrpc_to_device(device_info, '', + 'system update "/var/crash/package/%s"\nYES' % file_name, + mode="config_with_input") except Exception as e: failed_time -= 1 if 'timed out' in str(e) or 'Empty reply from server' in str(e): - #description = 'Error, please check the ip address and make sure the device have already turned on the RESTful API service! ' + # description = 'Error, please check the ip address and make sure the device have already turned on the RESTful API service! ' break description = 'Error, can not connect the device! ' continue @@ -468,26 +515,31 @@ # print 'contents', contents break else: - #APV timeout + # APV timeout break - #failed_time -= 1 + # failed_time -= 1 else: if rest_response_data: break if not failed_time: update_failed(description, task_name) - #add_description(task_name, description) + # add_description(task_name, description) return False - return waiting_for_device(version, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info) + return waiting_for_device(version, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_info) -def upload_to_device(update_list_id, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_name): + +def upload_to_device(update_list_id, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_name): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') add_description(task_name, 'Upload build to Device: ') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - update_custom_field(task_name, 'upload_to_device', [update_list_id, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_name]) + update_custom_field(task_name, 'upload_to_device', + [update_list_id, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_name]) if isinstance(device_type, list): deviceType = device_type[0] @@ -506,7 +558,7 @@ reload(cm.conf) network_card = cm.conf.NETWORK_CARD try: - local_ip = get_ip_address(network_card) + local_ip = get_ip_address(network_card) except Exception as e: description = 'Local network settings are abnormal. Please configuare management interface by CLI "system management intertface ".' add_description(task_name, description) @@ -514,18 +566,19 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return False device_info = get_rest_info_from_device(device_name)[0] - file_path = 'ftp://array:admin@%s:9993/%s' % (local_ip, filename) #maybe you have a better idea + file_path = 'ftp://array:admin@%s:9993/%s' % (local_ip, filename) # maybe you have a better idea running_params = 'system package "%s"' % file_path if deviceType.lower() in VPN_TYPE_LIST: running_params = json.dumps({"config": running_params}) new_url = modify_url('/rest/device_type/batch_cli', deviceType) try: if device_info['protocol'] == 'restapi': - rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, + restapi_username, restapi_password) else: rest_response_data = send_xmlrpc_to_device(device_info, '', 'system package "%s"' % file_path) except Exception as e: - #mybe timeout, do not return fialed here, continue do check_package_from_device + # mybe timeout, do not return fialed here, continue do check_package_from_device pass else: if device_info['protocol'] == 'restapi': @@ -546,9 +599,11 @@ fetchall_sql = '''SELECT location FROM update_list where id='%d' ''' % update_list_id location = db.fetchall(fetchall_sql)[0][0] - new_location = location[:-6] + ".array" if location.endswith(".click") else location[:-6] + ".click" + new_location = location[:-6] + ".array" if location.endswith(".click") else location[ + :-6] + ".click" # update this new location for `update` db and move this upgrade file with new suffix. - update_location_sql = "UPDATE update_list SET location = '%s' where id = '%d'" % (new_location, update_list_id) + update_location_sql = "UPDATE update_list SET location = '%s' where id = '%d'" % ( + new_location, update_list_id) db.execute_sql(update_location_sql) db.close() @@ -567,9 +622,11 @@ file_path = 'ftp://array:admin@%s:9993/%s' % (local_ip, filename) running_params = 'system package "%s"' % file_path try: - rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, restapi_port, restapi_username, restapi_password) + rest_response_data = send_https_rest_request('POST', new_url, running_params, ip_address, + restapi_port, restapi_username, + restapi_password) except Exception as e: - #maybe timeout, do not return fialed here, continue do check_package_from_device + # maybe timeout, do not return fialed here, continue do check_package_from_device pass else: if rest_response_data['status'] == 200: @@ -586,7 +643,7 @@ elif not check_upload_response(task_name, contents): return False else: - if rest_response_data != "failed" and rest_response_data != "time out": + if rest_response_data != "failed" and rest_response_data != "time out": contents = None try: contents = json.loads(rest_response_data)[0]["output"] @@ -601,12 +658,15 @@ else: config_failed('Failed to upload build to device.', task_name) return False - if not check_package_from_device(update_list_id, filename, task_name, ip_address, restapi_port, restapi_username, restapi_password, deviceType, device_info): + if not check_package_from_device(update_list_id, filename, task_name, ip_address, restapi_port, restapi_username, + restapi_password, deviceType, device_info): update_job_state(task_name, 'failed') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - return - return system_update_device(filename, task_name, ip_address, restapi_port, restapi_username, restapi_password, device_type, device_info) - + return + return system_update_device(filename, task_name, ip_address, restapi_port, restapi_username, restapi_password, + device_type, device_info) + + def check_upload_response(task_name, contents): if contents.find('Invalid package name') != -1: description = 'Error, invalid package name;' @@ -633,7 +693,8 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return False return True - + + ''' end update job ''' @@ -641,11 +702,14 @@ ''' start config jobs ''' + + def config_failed(description, task_name): add_description(task_name, description) update_job_state(task_name, 'failed') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) + def config_batch_cli(task_name, device_info, file_path): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') @@ -658,8 +722,8 @@ return try: if device_info['type'].lower() in VPN_TYPE_LIST: - sync_config = json.dumps({'config':file_command.read()}) - elif device_info['type'].lower() in ADC_TYPE_LIST: + sync_config = json.dumps({'config': file_command.read()}) + elif device_info['type'].lower() in ADC_TYPE_LIST: sync_config = file_command.read() except: config_failed("The file(%s) has special character." % os.path.basename(file_path), task_name) @@ -669,7 +733,9 @@ try: new_url = modify_url('/rest/device_type/batch_cli', device_info['type']) - rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: config_failed("Can not connect device!", task_name) return @@ -694,7 +760,7 @@ else: config_failed(config_result, task_name) return - elif device_info['type'].lower() in ADC_TYPE_LIST: + elif device_info['type'].lower() in ADC_TYPE_LIST: if not config_result: add_description(task_name, config_result.replace('\\', '')) update_job_state(task_name, 'done') @@ -704,35 +770,39 @@ config_failed(config_result, task_name) return + def refresh_device(device_list): - #refresh device config + # refresh device config for device in device_list: device_info = get_rest_info_from_device(device) - #device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] + # device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] _type = get_device_type(device_info[0]['type']) if device_info[0]['type'].lower() in VPN_TYPE_LIST: - data = {'cmd' : 'show running all'} + data = {'cmd': 'show running all'} new_url = '/rest/%s/global/cli_extend' % _type elif device_info[0]['type'].lower() in ADC_TYPE_LIST: - data = {'cmd' : 'show running'} + data = {'cmd': 'show running'} new_url = '/rest/%s/cli_extend' % _type running_params = json.dumps(data) - + try: - rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info[0]['ip_address'], device_info[0]['restapi_port'], device_info[0]['restapi_username'], device_info[0]['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info[0]['ip_address'], + device_info[0]['restapi_port'], + device_info[0]['restapi_username'], + device_info[0]['restapi_password']) except: continue if rest_response_data['status'] == 200: config_result = json.loads(rest_response_data['body'])['contents'] else: continue - + device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') if not os.path.isdir(device_path): os.makedirs(device_path) - #create_time = datetime.datetime.now().strftime('%Y-%m-%d') - #file_name = device_info[0]['name'] + '_' + datetime.datetime.now().strftime('%Y-%m-%d') + # create_time = datetime.datetime.now().strftime('%Y-%m-%d') + # file_name = device_info[0]['name'] + '_' + datetime.datetime.now().strftime('%Y-%m-%d') file_name = device_info[0]['name'] try: config_file = open(device_path + '/' + file_name, 'w') @@ -743,28 +813,30 @@ create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, device_type, device_id) values('%s', '%s', '%s', '%s', '%s', '%s') on conflict (name) do update set modify_time=excluded.modify_time" % (file_name, create_time, create_time, 'device', device_info[0]['type'], device_info[0]['id']) - #delete_sql = "DELETE FROM file_list where name='%s' " % file_name + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, device_type, device_id) values('%s', '%s', '%s', '%s', '%s', '%s') on conflict (name) do update set modify_time=excluded.modify_time" % ( + file_name, create_time, create_time, 'device', device_info[0]['type'], device_info[0]['id']) + # delete_sql = "DELETE FROM file_list where name='%s' " % file_name db = DB.get_connected_db() - #db.execute_sql(delete_sql) + # db.execute_sql(delete_sql) db.execute_sql(insert_sql) db.close() - + if device_info[0]['type'].lower() in ADC_TYPE_LIST: return - #refresh vsite file + # refresh vsite file device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') if not os.path.isdir(device_path): return for device in device_list: device_file_path = device_path + '/' + device - #parse vsite name, ip... from global config + # parse vsite name, ip... from global config parse_vsite_config_from_configfile(device_file_path, device) - #parse all vsite config from 'show running all', and write to vsite file + # parse all vsite config from 'show running all', and write to vsite file parse_vsite_from_configfile(device_file_path, device) return + def write_diff_file(task_name, check_result, result_file_path, task_type, username): if task_type == 'interval' and os.path.exists(result_file_path): try: @@ -787,24 +859,27 @@ create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % (os.path.basename(result_file_path), create_time, create_time, 'diff') + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % ( + os.path.basename(result_file_path), create_time, create_time, 'diff') delete_sql = "DELETE FROM file_list where name='%s' " % os.path.basename(result_file_path) db.execute_sql(delete_sql) db.execute_sql(insert_sql) db.close() - update_job_result_field(task_name, json.dumps({"config_check":{os.path.basename(result_file_path):"diff"}}), username) + update_job_result_field(task_name, json.dumps({"config_check": {os.path.basename(result_file_path): "diff"}}), + username) mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) return True + def config_compliance_check_all(task_name, check_info, file_name, file_type, result_file_path, task_type, username): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - diff_result = {create_time:{}} + diff_result = {create_time: {}} check_result = diff_result[create_time] device_name = check_info['device'] @@ -819,9 +894,9 @@ except IOError: config_failed("Get global config from AMP falied.", task_name) return - #For vpn customize config, just diff with show runnning all - running_params = json.dumps({'cmd':'show running all'}) - new_url = modify_url('/rest/device_type/cli_extend', device_info['type']) + # For vpn customize config, just diff with show runnning all + running_params = json.dumps({'cmd': 'show running all'}) + new_url = modify_url('/rest/device_type/cli_extend', device_info['type']) check_result[device_name]["device_ip"] = device_info["ip_address"] content = send_command_to_device(device_info, new_url, running_params) @@ -832,16 +907,16 @@ d = difflib.unified_diff(sorted(file_content), sorted(content.splitlines())) d_list = '\n'.join(list(d)) - check_result[device_name]['global'] = {"add":[], "delete":[]} + check_result[device_name]['global'] = {"add": [], "delete": []} parse_diff_config(d_list, check_result[device_name]['global']) else: - running_params = json.dumps({'cmd':'show running'}) + running_params = json.dumps({'cmd': 'show running'}) new_url = modify_url('/rest/device_type/cli_extend', device_info['type']) if device_info['type'].lower() in VPN_TYPE_LIST: - running_params = json.dumps({'config':'show running'}) + running_params = json.dumps({'config': 'show running'}) new_url = modify_url('/rest/device_type/batch_cli', device_info['type']) - #get global config from AMP + # get global config from AMP file_path = DEFAULT_CONFIG_FILE_PATH + file_type + '/' + file_name try: file_global = open(file_path + '/ca.conf', 'r') @@ -850,7 +925,7 @@ return file_content = file_global.read().splitlines() file_global.close() - elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: + elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: file_path = DEFAULT_CONFIG_FILE_PATH + file_type + '/' + file_name try: with open(file_path, 'r') as f: @@ -879,11 +954,11 @@ d = difflib.unified_diff(sorted(file_content), sorted(content.splitlines())) d_list = '\n'.join(list(d)) - check_result[device_name]['global'] = {"add":[], "delete":[]} + check_result[device_name]['global'] = {"add": [], "delete": []} parse_diff_config(d_list, check_result[device_name]['global']) if device_info['type'].lower() in VPN_TYPE_LIST: - #get vsite config from AMP and device + # get vsite config from AMP and device db = DB.get_connected_db() fetchall_sql = "SELECT * FROM tar_file where name='%s'" % file_name data = db.fetchall(fetchall_sql) @@ -902,7 +977,7 @@ file_vsite.close() if device_info['protocol'] == 'restapi': - running_params = json.dumps({'config':'show running', 'vsite_name':item['vsite_name']}) + running_params = json.dumps({'config': 'show running', 'vsite_name': item['vsite_name']}) new_url = '/rest/%s/global/batch_cli' % get_device_type(device_info['type']) content = send_command_to_device(device_info, new_url, running_params) else: @@ -917,9 +992,9 @@ d = difflib.unified_diff(sorted(file_content), sorted(content.splitlines())) d_list = '\n'.join(list(d)) - check_result[device_name][item['vsite_name']] = {"add":[], "delete":[]} + check_result[device_name][item['vsite_name']] = {"add": [], "delete": []} parse_diff_config(d_list, check_result[device_name][item['vsite_name']]) - + if task_type == 'interval': return write_diff_file(task_name, diff_result, result_file_path, task_type, username) else: @@ -934,6 +1009,7 @@ mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) send_notification('success', 'Task <%s> has done.' % task_name, model='config') + def config_compliance_check_custom(task_name, check_info, options, file_type, diff_file_path, task_type, username): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') @@ -941,7 +1017,7 @@ check_name = options["source_device"] + "-" + options["target_device"] create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - diff_result = {create_time:{}} + diff_result = {create_time: {}} check_result = diff_result[create_time] check_result[check_name] = {} @@ -954,7 +1030,8 @@ with open(file_path, 'r') as f: source_config_content = f.read().splitlines() except IOError: - check_result[check_name]["error_msg"] = "Failed to read the source device <%s> config." % options["source_device"] + check_result[check_name]["error_msg"] = "Failed to read the source device <%s> config." % options[ + "source_device"] # query the target device config. try: if options["target_config"].endswith(".all_cfg_tar"): @@ -964,7 +1041,8 @@ with open(file_path, 'r') as f: target_config_content = f.read().splitlines() except IOError: - check_result[check_name]["error_msg"] = "Failed to read the target device <%s> config." % options["target_device"] + check_result[check_name]["error_msg"] = "Failed to read the target device <%s> config." % options[ + "target_device"] # diff viste if have same name viste. if options["source_config"].endswith(".all_cfg_tar") and options["target_config"].endswith(".all_cfg_tar"): @@ -994,7 +1072,6 @@ check_result[check_name]['global'] = {"add": [], "delete": [], "original": None} parse_diff_config(d_list, check_result[check_name]['global']) - res = write_diff_file(task_name, diff_result, diff_file_path, task_type, username) if not res: return @@ -1005,6 +1082,7 @@ mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) send_notification('success', 'Task <%s> has done.' % task_name, model='config') + def config_summary_diff(task_name, target_list, task_name_list, diff_file_path, task_file_list, source, source_file): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') @@ -1059,10 +1137,11 @@ result_file.write(json.dumps(result)) db = DB.get_connected_db() - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % (os.path.basename(diff_file_path), create_time, create_time, 'diff') + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % ( + os.path.basename(diff_file_path), create_time, create_time, 'diff') db.execute_sql(insert_sql) db.close() - + add_description(task_name, "Successful!") update_job_state(task_name, 'done') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) @@ -1070,21 +1149,19 @@ send_notification('success', 'Task <%s> has done.' % task_name, model='config') - - - # module_splite_line = { # "vpn" : {"start_line":"#VPN configuration\n", "end_line":"#aaa configuration\n", "cmd":"show vpn config"}, # "acl" : {"start_line":"#acl configuration\n", "end_line":"#url policy configuration\n", "cmd":"show acl config"}, # "role" : {"start_line":"#role configuration\n", "end_line":"#acl configuration\n", "cmd":"show role config"} # } -def config_compliance_check(task_name, check_info, file_name, file_type, result_file_path, module_name, task_type, username): +def config_compliance_check(task_name, check_info, file_name, file_type, result_file_path, module_name, task_type, + username): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - + create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - diff_result = {create_time:{}} + diff_result = {create_time: {}} check_result = diff_result[create_time] device = check_info['device'] @@ -1097,16 +1174,16 @@ if check_info["is_global"]: vsite_name = 'global' config_path = "%s/ca.conf" % file_path - running_params = json.dumps({"config":"show running %s" % module_name}) + running_params = json.dumps({"config": "show running %s" % module_name}) else: vsite_name = check_info['vsite_name'] config_path = '%s/vsites/%s/ca.conf.%s' % (file_path, vsite_name, vsite_name) - running_params = json.dumps({"config":"show running %s" % module_name, "vsite_name":vsite_name}) + running_params = json.dumps({"config": "show running %s" % module_name, "vsite_name": vsite_name}) if file_type == 'customize': config_path = file_path - #awk_config = "%s/%s" % (os.getcwd(), "cm/lib/get_conf_by_key.awk") - #cmd_value = "cat %s | awk -v key='%s' -f %s" % (config_path, module_name, awk_config) + # awk_config = "%s/%s" % (os.getcwd(), "cm/lib/get_conf_by_key.awk") + # cmd_value = "cat %s | awk -v key='%s' -f %s" % (config_path, module_name, awk_config) cmd_value = "cat %s | awk -v key='%s' -f %s" % (config_path, module_name, CM_AWK_CONF) try: @@ -1116,7 +1193,7 @@ return module_config = cm_vsite_file.read().splitlines() cm_vsite_file.close() - elif device_info['type'].lower() in ADC_TYPE_LIST: + elif device_info['type'].lower() in ADC_TYPE_LIST: file_path = DEFAULT_CONFIG_FILE_PATH + file_type + '/' + file_name cmd_value = "cat %s | awk -v key='%s' -f %s" % (file_path, module_name, CM_AWK_CONF) @@ -1127,10 +1204,10 @@ return module_config = cm_vsite_file.read().splitlines() cm_vsite_file.close() - running_params = json.dumps({"cmd":"show running %s" % module_name}) + running_params = json.dumps({"cmd": "show running %s" % module_name}) vsite_name = 'global' - #get vsite config from device + # get vsite config from device check_result[device]["device_ip"] = device_info["ip_address"] if device_info['protocol'] == 'restapi': @@ -1140,7 +1217,8 @@ if check_info["is_global"]: content = send_xmlrpc_to_device(device_info, new_url, "show running %s" % module_name) else: - content = send_xmlrpc_to_device(device_info, new_url, "sw %s\nshow running %s" % (check_info['vsite_name'], module_name)) + content = send_xmlrpc_to_device(device_info, new_url, + "sw %s\nshow running %s" % (check_info['vsite_name'], module_name)) if content == "failed": config_failed("Can not connect the device.", task_name) @@ -1154,7 +1232,7 @@ d = difflib.unified_diff(sorted(module_config), sorted(content.splitlines())) d_list = '\n'.join(list(d)) - check_result[device][vsite_name] = {"add":[], "delete":[]} + check_result[device][vsite_name] = {"add": [], "delete": []} parse_diff_config(d_list, check_result[device][vsite_name]) if task_type == 'interval': return write_diff_file(task_name, diff_result, result_file_path, task_type, username) @@ -1162,42 +1240,44 @@ res = write_diff_file(task_name, diff_result, result_file_path, task_type, username) if not res: config_failed(res, task_name) - return + return add_description(task_name, "Successful!") update_job_state(task_name, 'done') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - #mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) + # mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) send_notification('success', 'Task <%s> has done.' % task_name, model='config') + def config_backup(task_name, device_info, comment, file_num=0): - #device_info, virtual_file_path, cli_url, check_config_url, real_file_path, cli_cmd + # device_info, virtual_file_path, cli_url, check_config_url, real_file_path, cli_cmd add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - + try: - #refresh_device([device_info['name'],]) - #device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') - #if not os.path.isdir(device_path): + # refresh_device([device_info['name'],]) + # device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') + # if not os.path.isdir(device_path): # os.makedirs(device_path) - - cli_url = modify_url('/rest/device_type/cli_extend', device_info['type']) + + cli_url = modify_url('/rest/device_type/cli_extend', device_info['type']) if device_info['type'].lower() in VPN_TYPE_LIST: uid = uuid.uuid1() cm = __import__("cm") reload(cm.conf) network_card = cm.conf.NETWORK_CARD - #get config file from device + # get config file from device try: local_ip = get_ip_address(network_card) except Exception as e: description = 'Local network settings are abnormal. Please configuare management interface by CLI "system management intertface ".' config_failed(description, task_name) return -1 - - cli_cmd = {} + + cli_cmd = {} virtual_file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + '%s.all_cfg_tar' % uid - real_file_name = device_info['name'] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + '.all_cfg_tar' + real_file_name = device_info['name'] + '-' + datetime.datetime.now().strftime( + '%Y_%m_%d_%H:%M:%S') + '.all_cfg_tar' real_file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + real_file_name cli_cmd['export_config'] = 'config export all tftp %s %s' % (local_ip, uid) cli_cmd['write_config'] = 'write file all %s' % uid @@ -1210,29 +1290,30 @@ config_failed(ret, task_name) return -1 # copy config to device path - #sys_cmd = "rm -rf %s/%s && cp -r %s %s/%s" %(device_path, device_info['name'], real_file_path[0:-12], device_path, device_info['name']) - #os.popen(sys_cmd).read() - #parse_vsite_config_from_configfile('%s/%s/ca.conf' %(device_path, device_info['name']), device_info['name']) - elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: + # sys_cmd = "rm -rf %s/%s && cp -r %s %s/%s" %(device_path, device_info['name'], real_file_path[0:-12], device_path, device_info['name']) + # os.popen(sys_cmd).read() + # parse_vsite_config_from_configfile('%s/%s/ca.conf' %(device_path, device_info['name']), device_info['name']) + elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: real_file_name = device_info['name'] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + '.cfg' back_file_dir = DEFAULT_CONFIG_FILE_PATH + 'backup/' if not os.path.exists(back_file_dir): os.makedirs(back_file_dir) - real_file_path = back_file_dir + real_file_name + real_file_path = back_file_dir + real_file_name bu = Backup_APV(device_info, real_file_path, cli_url, 'show running') - + ret = bu.run() if ret: config_failed(ret, task_name) return -1 # copy config to device path - #sys_cmd = "rm -rf %s/%s && cp %s %s/%s" %(device_path, device_info['name'], real_file_path[0:-4], device_path, device_info['name']) - #os.popen(sys_cmd).read() + # sys_cmd = "rm -rf %s/%s && cp %s %s/%s" %(device_path, device_info['name'], real_file_path[0:-4], device_path, device_info['name']) + # os.popen(sys_cmd).read() db = DB.get_connected_db() create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # for backup config file, use 'modify_time' to save device name - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (real_file_name, create_time, device_info['name'], 'backup', comment, device_info['type']) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + real_file_name, create_time, device_info['name'], 'backup', comment, device_info['type']) db.execute_sql(insert_sql) db.close() @@ -1242,16 +1323,17 @@ mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) send_notification('success', 'Task <%s> has done.' % task_name, model='config') except Exception as e: - config_failed("Internal Error: %s" %e, task_name) + config_failed("Internal Error: %s" % e, task_name) return -1 return 0 + def get_recent_file(device_name): db = DB.get_connected_db() fetchall_sql = "SELECT name FROM file_list where type='backup' and name like '%" + device_name + "%'" data = db.fetchall(fetchall_sql) - key = ['name',] - backup_list = [dict(zip(key, each)) for each in data] + key = ['name', ] + backup_list = [dict(zip(key, each)) for each in data] db.close() device_backup_list = [] @@ -1265,6 +1347,7 @@ return sorted(device_backup_list)[-1] + def _reorder_show_slb_summary(msg): config_list = msg.split('\n') vs_list = [] @@ -1299,8 +1382,9 @@ if cf not in policy_list: policy_list.append(cf) continue - return "\n".join(vs_list+vlink_list+rs_list+group_list+member_list+policy_list) + return "\n".join(vs_list + vlink_list + rs_list + group_list + member_list + policy_list) + def clone_apv_vs(pk_list, file_name, comment, options): device_info = get_rest_info_from_device(pk_list['name'])[0] vs_name = options["vs"] @@ -1309,12 +1393,14 @@ if not os.path.isdir(file_path): os.makedirs(file_path) - data = {"cmd" : 'show slb summary "%s"' % vs_name} + data = {"cmd": 'show slb summary "%s"' % vs_name} new_url = '/rest/%s/cli_extend' % get_device_type(device_info['type'].lower()) running_params = json.dumps(data) try: - rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) except: return "Can not connect device!" @@ -1329,7 +1415,8 @@ with open(dest_file_path, 'w') as f: f.write(config_result) - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (dest_file_name, create_time, pk_list['name'], 'vs', comment, device_info['type']) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + dest_file_name, create_time, pk_list['name'], 'vs', comment, device_info['type']) delete_sql = "DELETE FROM file_list where name='%s' " % file_name db = DB.get_connected_db() db.execute_sql(delete_sql) @@ -1337,6 +1424,7 @@ db.close() return "" + def clone_apv(pk_list, file_name, comment, device_type=None): file_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'system') dest_file_name = file_name + '.cfg' @@ -1346,17 +1434,20 @@ if pk_list['file_type'] == 'device': # in amp 2.1, will clone the last backup config file. # in amp 3.0, change to clone the device running config. - #src_file_name = get_recent_file(pk_list['name']) - #src_file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + src_file_name + # src_file_name = get_recent_file(pk_list['name']) + # src_file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + src_file_name device_info = get_rest_info_from_device(pk_list['name']) - #device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] + # device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] _type = get_device_type(device_info[0]['type']) - data = {'cmd' : 'show running'} + data = {'cmd': 'show running'} new_url = '/rest/%s/cli_extend' % _type running_params = json.dumps(data) - + try: - rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info[0]['ip_address'], device_info[0]['restapi_port'], device_info[0]['restapi_username'], device_info[0]['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info[0]['ip_address'], + device_info[0]['restapi_port'], + device_info[0]['restapi_username'], + device_info[0]['restapi_password']) except: return "Send request to device failed." if rest_response_data['status'] == 200: @@ -1374,16 +1465,18 @@ src_file_path = DEFAULT_CONFIG_FILE_PATH + pk_list['file_type'] + '/' + src_file_name if not os.path.exists(src_file_path): return "The file[%s] is not found!" % src_file_path - + shutil.copyfile(src_file_path, dest_file_path) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (dest_file_name, create_time, pk_list['name'], 'system', comment, device_type) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + dest_file_name, create_time, pk_list['name'], 'system', comment, device_type) db = DB.get_connected_db() db.execute_sql(insert_sql) db.close() return "" + def local_clone_vs(pk_list, file_name, comment, device_type=""): file_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vs') if not os.path.isdir(file_path): @@ -1408,11 +1501,13 @@ dest_file_name = file_name + '.cfg' dest_file_path = DEFAULT_CONFIG_FILE_PATH + 'vs/' + dest_file_name shutil.copyfile(src_file_path, dest_file_path) - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (dest_file_name, create_time, src_file_name, 'vs', comment, device_type) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + dest_file_name, create_time, src_file_name, 'vs', comment, device_type) db.execute_sql(insert_sql) db.close() return "" + def clone_ag(pk_list, file_name, comment, options): file_type = pk_list["file_type"] db = DB.get_connected_db() @@ -1424,14 +1519,14 @@ cm = __import__("cm") reload(cm.conf) network_card = cm.conf.NETWORK_CARD - #get config file from device + # get config file from device try: local_ip = get_ip_address(network_card) except Exception as e: return 'Local network settings are abnormal. Please configuare management interface by CLI "system management intertface ".' device_info = get_rest_info_from_device(pk_list['name'])[0] cli_url = modify_url('/rest/device_type/cli_extend', device_info['type']) - cli_cmd = {} + cli_cmd = {} virtual_file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + '%s.all_cfg_tar' % uid cli_cmd['export_config'] = 'config export all tftp %s %s' % (local_ip, uid) cli_cmd['write_config'] = 'write file all %s' % uid @@ -1441,7 +1536,7 @@ bu = Backup_AG(device_info, virtual_file_path, cli_url, check_config_url, target_all_path, cli_cmd) ret = bu.run() if ret: - return ret + return ret else: clone_file_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, pk_list['file_type']) clone_file_name = pk_list['name'] @@ -1451,7 +1546,7 @@ return "The file[%s] is not found!" % clone_file_name create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - + target_all_path = target_path + '.all_cfg_tar' file_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'system') if not os.path.isdir(file_path): @@ -1459,14 +1554,14 @@ sys_cmd = "cp -r %s/%s %s" % (clone_file_path, clone_file_name, target_path) os.popen(sys_cmd).read() tar = TarFiles() - ret = tar.create_tar(target_path, file_name+".all_cfg_tar") + ret = tar.create_tar(target_path, file_name + ".all_cfg_tar") if ret: - return "create tar failed, errno %d" % ret + return "create tar failed, errno %d" % ret vsite_list = [] for root, dir, files in os.walk(target_path): for file in files: - fullpath=os.path.join(root, file) + fullpath = os.path.join(root, file) path_list = fullpath.split('/') if 'vsites' in path_list and path_list[path_list.index('vsites') + 1] not in vsite_list: vsite_list.append(path_list[path_list.index('vsites') + 1]) @@ -1476,12 +1571,14 @@ insert_sql = "INSERT INTO tar_file(name, vsite_name) values('%s', '%s')" % (file_name, vsite) db.execute_sql(insert_sql) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (file_name + '.all_cfg_tar', create_time, pk_list['name'], 'system', comment, pk_list["device_type"]) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + file_name + '.all_cfg_tar', create_time, pk_list['name'], 'system', comment, pk_list["device_type"]) db.execute_sql(insert_sql) db.close() mark_expire_all(get_model('cm', ['configuration', 'config_file', 'TarFile'])) return "" + def config_clone(task_name, options): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') @@ -1493,9 +1590,9 @@ device_type = pk_list["device_type"] if device_type.lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: - new_file_name = file_name+'.cfg' + new_file_name = file_name + '.cfg' elif device_type.lower() in VPN_TYPE_LIST: - new_file_name = file_name+'.all_cfg_tar' + new_file_name = file_name + '.all_cfg_tar' else: new_file_name = file_name select_sql = "SELECT type from file_list where name='%s' " % new_file_name @@ -1542,18 +1639,19 @@ send_notification('success', 'Task <%s> has done.' % task_name, model='config') return + def config_custom(task_name, config_content, device_type, comment): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - + real_file_name = task_name back_file_dir = DEFAULT_CONFIG_FILE_PATH + 'customize/' if not os.path.exists(back_file_dir): os.makedirs(back_file_dir) real_file_path = back_file_dir + real_file_name bu = Backup_Custom(config_content, task_name, real_file_path) - + ret = bu.run() if ret: config_failed(ret, task_name) @@ -1564,6 +1662,7 @@ mark_expire_all(get_model('cm', ['configuration', 'config_file', 'ConfigFile'])) send_notification('success', 'Task <%s> has done.' % task_name, model='config') + def _check_xmlrpc_status(device_info): if device_info["protocol"] == "xmlrpc": return {"status": True, "protocol": "https", "port": device_info["restapi_port"]} @@ -1571,7 +1670,7 @@ _type = get_device_type(device_info['type']) cli_url = modify_url('/rest/device_type/cli_extend', _type) - data = {'cmd' : 'show xmlrpc'} + data = {'cmd': 'show xmlrpc'} xmlrpc_params = json.dumps(data) content = send_command_to_device(device_info, cli_url, xmlrpc_params) @@ -1588,12 +1687,14 @@ return {"status": True, "protocol": res[0]["protocol"], "port": res[1]["port"]} else: # The output of APV version before 10.4 is like 'XMLRPC is ON using port 9999'. - res = cli_parse(content, RegexParser('XMLRPC is (?PON|OFF) using (?Phttp|https), port (?P[0-9]+)', MATCHONE)) + res = cli_parse(content, RegexParser( + 'XMLRPC is (?PON|OFF) using (?Phttp|https), port (?P[0-9]+)', MATCHONE)) if res and res["status"] == "ON": return {"status": True, "protocol": res["protocol"], "port": res["port"]} else: return {"status": False} + def apply_config_backup(device_name, device_info, description): date = datetime.datetime.now() + datetime.timedelta(seconds=1) @@ -1607,7 +1708,8 @@ new_schedule['args'] = [backup_task_name, device_info, description] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(backup_task_name, 'config_backup', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) + add_job_into_database(backup_task_name, 'config_backup', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) res = config_backup(backup_task_name, device_info, 'Backup configuration before applying.') @@ -1615,6 +1717,7 @@ return res + def config_recover(task_name, device_info, file_type, file_name, backup=False, device_name=None): add_description(task_name, "The execution time: ") update_job_state(task_name, 'ongoing') @@ -1658,8 +1761,9 @@ cli_cmd['remove_config'] = 'no config %s \nYES' % (uid) check_config_url = '/rest/%s/global/admintools/config_mgmt/SavedAllFile' % _type batch_cli_url = '/rest/%s/global/batch_cli' % _type - rcv = Recover_AG(real_file_path, virtual_file_path, device_info, cli_cmd, cli_url, check_config_url, batch_cli_url, xmlprc_status["status"]) - elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: + rcv = Recover_AG(real_file_path, virtual_file_path, device_info, cli_cmd, cli_url, check_config_url, + batch_cli_url, xmlprc_status["status"]) + elif device_info['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: if xmlprc_status["status"]: batch_cli_url = '/cgi-bin/xmlrpc_server' device_info["xmlrpc_port"] = xmlprc_status["port"] @@ -1692,9 +1796,12 @@ apply_config_backup(device_name, device_info, 'Backup configuration after applying.') return + def config_init_device(task_name, device_info, file_type, file_name, backup=False, device_name=None): config_recover(task_name, device_info, file_type, file_name, backup, device_name) return + + ''' end config jobs ''' @@ -1703,6 +1810,7 @@ start sync ''' + def update_cm_role_status(status, pk_dict): ''' status: 0-new 1-success 2-failed pk_dict: {'table':, 'pk_field':} @@ -1721,6 +1829,8 @@ db.close() mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'role', 'CMRoleGroup'])) return + + def update_cm_role_devicelist(device_list, pk_dict, action, failed_device=None): table = pk_dict.pop('table') flag = 0 @@ -1735,7 +1845,7 @@ db = DB.get_connected_db() select_sql = "SELECT device_list from %s where %s" % (table, condition) data = db.fetchall(select_sql) - result=[each[0] for each in data] + result = [each[0] for each in data] if result[0]: device = str(result[0], 'utf-8').split(',') else: @@ -1755,25 +1865,29 @@ mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'acl', 'CMACLResource'])) mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'vpn', 'CMVPNResource'])) - return + return + + def remove_resource_device_list(device_list, role_name, database): db = DB.get_connected_db() - #select id, device_list from cm_vpn_resource where group_name=role_name + # select id, device_list from cm_vpn_resource where group_name=role_name select_sql = "SELECT id, device_list from %s where group_name='%s'" % (database, role_name) data = db.fetchall(select_sql) key = ['id', 'device_list'] result = [dict(zip(key, each)) for each in data] - #get difference between device_list and device_list of database + # get difference between device_list and device_list of database for item in result: result_list = list(set(str(item['device_list'], 'utf-8').split(',')).difference(set(device_list))) - #update cm_vpn_resource set device_list where id=<> + # update cm_vpn_resource set device_list where id=<> update_sql = "UPDATE %s SET device_list='%s' where id=%d" % (database, ','.join(result_list), item['id']) db.execute_sql(update_sql) db.close() return + + # def sync_to_device(device_list, url, running_params, pk_dict, action, task_name=None): # if task_name: # add_description(task_name, "The execution time: ") @@ -1844,9 +1958,12 @@ if not device_info: continue try: - rest_response_data = send_https_rest_request('POST', url, params['params'], device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', url, params['params'], device_info['ip_address'], + device_info['restapi_port'], + device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: - #description = 'Error, please check the ip address and make sure the device have already turned on the RESTful API service!' + # description = 'Error, please check the ip address and make sure the device have already turned on the RESTful API service!' description += 'Can not connect the device %s[%s].\n' % (device, device_info['ip_address']) else: if rest_response_data['status'] == 200: @@ -1872,6 +1989,8 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return '' + + ''' end sync ''' @@ -1879,29 +1998,32 @@ ''' start monitor_alert ''' + + def get_monitor_info(monitor_id): db = DB.get_connected_db() select_sql = "select level, extend, interval from monitor where id=%d" % monitor_id data = db.fetchone(select_sql) db.close() result = { - "id" : monitor_id, - "level" : data[0], - "condition" : data[1]["condition"], - "message" : data[1]["message"], - "assign" : data[1]["assign"], - "type_id" : data[1]["type_id"], + "id": monitor_id, + "level": data[0], + "condition": data[1]["condition"], + "message": data[1]["message"], + "assign": data[1]["assign"], + "type_id": data[1]["type_id"], "target_status": data[1]["target_status"] if "target_status" in data[1] else None, - "continued" : data[2] - } + "continued": data[2] + } return result + def get_global_settings(): global_settings = {} select_interval_sql = "select name, settings from global_settings where name='alert_interval';" select_inhibit_sql = "select name, settings from global_settings where name='alert_inhibit';" db = DB.get_connected_db() - data = db.fetchone(select_interval_sql) + data = db.fetchone(select_interval_sql) if not data: global_settings["interval"] = 5 else: @@ -1913,9 +2035,10 @@ global_settings["inhibit"] = int(data[1]) db.close() return global_settings - -#def monitor_alert(ip_list, monitor_id, condition, msg): -#def monitor_alert(target_type, target_list, monitor_id, user_info): + + +# def monitor_alert(ip_list, monitor_id, condition, msg): +# def monitor_alert(target_type, target_list, monitor_id, user_info): def monitor_alert(target_type, target_list, monitor_id): monitor_info = get_monitor_info(monitor_id) global_settings = get_global_settings() @@ -1923,34 +2046,36 @@ if len(monitor_info['type_id']) == 1 and int(monitor_info['type_id'][0]) == 4000: decistion = SSLCertDecision() else: - decistion = Decision(target_type, target_list, monitor_info["continued"] * 60, global_settings["inhibit"] * 60, global_settings['interval']) - + decistion = Decision(target_type, target_list, monitor_info["continued"] * 60, global_settings["inhibit"] * 60, + global_settings['interval']) + dec_list = decistion.determine(monitor_info) - #decistion.alert_msg(dec_list, monitor_info, user_info) + # decistion.alert_msg(dec_list, monitor_info, user_info) if dec_list: decistion.alert_msg(dec_list, monitor_info) mark_expire_all(get_model('cm', ['monitor', 'MonitorMsg'])) + ''' end monitor_alert ''' Scheduler_func_list = { - 'upload_job' : upload_job, - 'upload_to_device' : upload_to_device, - 'system_update_device' : system_update_device, - 'waiting_for_device' : waiting_for_device, - 'config_init_device' : config_init_device, - 'config_batch_cli' : config_batch_cli, - 'config_compliance_check' : config_compliance_check, - 'config_compliance_check_all' : config_compliance_check_all, - 'config_backup' : config_backup, - 'config_recover' : config_recover, - 'sync_to_device' : sync_to_device, - 'monitor_alert' : monitor_alert, + 'upload_job': upload_job, + 'upload_to_device': upload_to_device, + 'system_update_device': system_update_device, + 'waiting_for_device': waiting_for_device, + 'config_init_device': config_init_device, + 'config_batch_cli': config_batch_cli, + 'config_compliance_check': config_compliance_check, + 'config_compliance_check_all': config_compliance_check_all, + 'config_backup': config_backup, + 'config_recover': config_recover, + 'sync_to_device': sync_to_device, + 'monitor_alert': monitor_alert, 'config_custom': config_custom, - 'config_clone' : config_clone, + 'config_clone': config_clone, 'config_compliance_check_custom': config_compliance_check_custom } @@ -1958,24 +2083,28 @@ start check device status ''' + def set_device_version(device_id, version): db = DB.get_connected_db() update_sql = "UPDATE device SET version='%s' WHERE id='%s'" % (version, device_id) db.execute_sql(update_sql) db.close() + def set_device_status(device_id, status): db = DB.get_connected_db() update_sql = "UPDATE device SET connection='%s' WHERE id='%s'" % (status, device_id) db.execute_sql(update_sql) db.close() + def set_device_snmp(device_id, snmp_general): db = DB.get_connected_db() update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" % (snmp_general, device_id) db.execute_sql(update_sql) db.close() + class DeviceStatusThread(threading.Thread): def __init__(self, data): super(DeviceStatusThread, self).__init__() @@ -1993,11 +2122,15 @@ break else: failed_time -= 1 - logger.error('Get device<%s>(%s) version failed. %s' % (self.data['name'], self.data['ip'], response_data['data'])) + logger.error('Get device<%s>(%s) version failed. %s' % (self.data['name'], self.data['ip'], + response_data['data'])) else: try: new_url = modify_url('/rest/device_type/system/SystemInfo/version', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: failed_time -= 1 logger.error('Get device<%s>(%s) version failed. %s' % (self.data['name'], self.data['ip'], str(e))) @@ -2015,7 +2148,8 @@ set_device_status(self.data['id'], 'unconnected') else: if os.path.exists(os.path.join(extension_path, "monitoring")): - if self.data['protocol'] == 'xmlrpc' or "nsae" in self.data['type'].lower() or "netopti" in self.data['type'].lower(): + if self.data['protocol'] == 'xmlrpc' or "nsae" in self.data['type'].lower() or "netopti" in self.data[ + 'type'].lower(): response_data = send_cli_to_device(self.data, "show snmp", mode='enable') if response_data['message'] == 'success': try: @@ -2040,16 +2174,24 @@ snmp_general["v3user"] = result[6]['v3user'].replace('\"', '') set_device_snmp(self.data['id'], json.dumps(snmp_general)) except Exception as e: - logger.error('Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], str(e))) else: - logger.error('Get device<%s>(%s) SNMPGeneral failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMPGeneral failed. %s' % (self.data['name'], self.data['ip'], str(e))) else: if self.data['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/general', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/general', + self.data['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: - logger.error('Get device<%s>(%s) SNMPGeneral failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMPGeneral failed. %s' % (self.data['name'], self.data['ip'], + str(e))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -2067,24 +2209,33 @@ if 'v3_user' in rest_body['SNMPGeneral']: for user in rest_body['SNMPGeneral']['v3_user']: if 'username' in user and 'auth_password' in user and 'level' in user: - v3user = user['username']+' '+user['auth_password']+' '+user['level'] + v3user = user['username'] + ' ' + user['auth_password'] + ' ' + \ + user['level'] if 'priv_password' in user: - v3user += ' '+user['priv_password'] + v3user += ' ' + user['priv_password'] snmp_general['v3user'] = v3user break set_device_snmp(self.data['id'], json.dumps(snmp_general)) except Exception as e: - logger.error('Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], + str(e))) elif self.data['type'].lower() in VPN_TYPE_LIST: # Failed to query the snmp general info from FieldGroup `general` in current AG versions. # It should add more judgment when the bug is fixed in new AG version. But the old version # will always query the snmp info follow below. snmp_general = {} try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_version', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_version', + self.data['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: - logger.error('Get device<%s>(%s) SNMP version failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMP version failed. %s' % (self.data['name'], self.data['ip'], + str(e))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -2093,13 +2244,21 @@ if 'snmp_version' in rest_body['SNMPGeneral']: snmp_general['snmp_version'] = rest_body['SNMPGeneral']['snmp_version'] except Exception as e: - logger.error('Set device<%s>(%s) snmp version failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error('Set device<%s>(%s) snmp version failed. %s' % (self.data['name'], + self.data['ip'], + str(e))) try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_enable', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_enable', + self.data['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: - logger.error('Get device<%s>(%s) SNMP enable failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMP enable failed. %s' % (self.data['name'], self.data['ip'], + str(e))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -2108,13 +2267,21 @@ if 'snmp_enable' in rest_body['SNMPGeneral']: snmp_general['snmp_enable'] = rest_body['SNMPGeneral']['snmp_enable'] except Exception as e: - logger.error('Set device<%s>(%s) snmp enable failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error('Set device<%s>(%s) snmp enable failed. %s' % (self.data['name'], + self.data['ip'], + str(e))) try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/community', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/community', + self.data['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: - logger.error('Get device<%s>(%s) SNMP community failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMP community failed. %s' % (self.data['name'], self.data['ip'], + str(e))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -2123,12 +2290,20 @@ if 'community' in rest_body['SNMPGeneral']: snmp_general['community'] = rest_body['SNMPGeneral']['community'] except Exception as e: - logger.error('Set device<%s>(%s) snmp community failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error('Set device<%s>(%s) snmp community failed. %s' % (self.data['name'], + self.data['ip'], + str(e))) try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/v3_user', self.data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/v3_user', + self.data['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', self.data['ip'], + self.data['restapi_port'], + self.data['restapi_username'], + self.data['restapi_password']) except Exception as e: - logger.error('Get device<%s>(%s) SNMP v3_user failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Get device<%s>(%s) SNMP v3_user failed. %s' % (self.data['name'], self.data['ip'], + str(e))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -2137,35 +2312,45 @@ if 'v3_user' in rest_body['SNMPGeneral']: for user in rest_body['SNMPGeneral']['v3_user']: if 'username' in user and 'auth_password' in user and 'level' in user: - v3user = user['username']+' '+user['auth_password']+' '+user['level'] + v3user = user['username'] + ' ' + user['auth_password'] + ' ' + \ + user['level'] if 'priv_password' in user: - v3user += ' '+user['priv_password'] + v3user += ' ' + user['priv_password'] snmp_general['v3user'] = v3user break except Exception as e: - logger.error('Set device<%s>(%s) snmp v3_user failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error('Set device<%s>(%s) snmp v3_user failed. %s' % (self.data['name'], + self.data['ip'], + str(e))) try: set_device_snmp(self.data['id'], json.dumps(snmp_general)) except Exception as e: - logger.error('Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], str(e))) + logger.error( + 'Set device<%s>(%s) snmp failed. %s' % (self.data['name'], self.data['ip'], str(e))) + def check_device_status(): try: - db = DB.get_connected_db() + db = DB.get_connected_db() select_sql = "SELECT id, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, type, enable_password, connection FROM device" data = db.fetchall(select_sql) db.close() except Exception as e: # select from db failed, mostly because avx extension is not installed. return - key = ['id', 'name', 'ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password', 'connection'] + key = ['id', 'name', 'ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', + 'enable_password', 'connection'] result = [dict(zip(key, each)) for each in data] for each in result: t = DeviceStatusThread(each) t.start() + + ''' end check device status ''' + + def get_device_ip(device_list): db = DB.get_connected_db() device_ip = [] @@ -2179,18 +2364,21 @@ db.close() return device_ip + def set_host_version(host_id, version): db = DB.get_connected_db() update_sql = "UPDATE host SET version='%s' WHERE id='%s'" % (version, host_id) db.execute_sql(update_sql) db.close() + def set_host_connection(host_id, connection): db = DB.get_connected_db() update_sql = "UPDATE host SET connection='%s' WHERE id='%s'" % (connection, host_id) db.execute_sql(update_sql) db.close() + def set_host_sys_status(host_id, data): regex = re.compile(r'\\(?![/u"])') data['cpu_usage'] = json.loads(regex.sub(r"\\\\", data['cpu_usage']).replace("'", '"')) @@ -2207,12 +2395,14 @@ db.execute_sql(update_sql) db.close() + def set_host_vm_number(host_id, number): db = DB.get_connected_db() update_sql = "UPDATE host SET vm_number=%s WHERE id='%s'" % (number, host_id) db.execute_sql(update_sql) db.close() + class HostInfoThread(threading.Thread): def __init__(self, data): super(HostInfoThread, self).__init__() @@ -2220,7 +2410,9 @@ def run(self): try: - rest_response_data = call_restapi('GET', '/rest/avx/system/SystemInfo/version', '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password'], 'https') + rest_response_data = call_restapi('GET', '/rest/avx/system/SystemInfo/version', '', self.data['ip'], + self.data['restapi_port'], self.data['restapi_username'], + self.data['restapi_password'], 'https') except Exception as e: logger.error('Get host<%s> version failed. %s' % (self.data['ip'], str(e))) set_host_connection(self.data['id'], 'Unconnected') @@ -2231,7 +2423,10 @@ set_host_version(self.data['id'], json.loads(rest_response_data['body'])['SystemInfo']['version']) # get cpu/memory/disk data try: - rest_response_data = call_restapi('GET', '/rest/avx/system/SystemStatus/status', '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password'], 'https') + rest_response_data = call_restapi('GET', '/rest/avx/system/SystemStatus/status', '', + self.data['ip'], self.data['restapi_port'], + self.data['restapi_username'], self.data['restapi_password'], + 'https') except Exception as e: logger.error('Get host<%s> cpu/memory/disk usage failed. %s' % (self.data['ip'], str(e))) mark_expire_all(get_model('cm', ['virtualization', 'Host'])) @@ -2239,9 +2434,12 @@ if rest_response_data['status'] == 200: set_host_sys_status(self.data['id'], json.loads(rest_response_data['body'])['SystemStatus']) mark_expire_all(get_model('cm', ['virtualization', 'Host'])) - #get vm numbers + # get vm numbers try: - rest_response_data = call_restapi('GET', '/rest/avx/va/instance/VAInstance/_get_list_data', '', self.data['ip'], self.data['restapi_port'], self.data['restapi_username'], self.data['restapi_password'], 'https') + rest_response_data = call_restapi('GET', '/rest/avx/va/instance/VAInstance/_get_list_data', '', + self.data['ip'], self.data['restapi_port'], + self.data['restapi_username'], self.data['restapi_password'], + 'https') except Exception as e: logger.error('Get host<%s> vm numbers failed. %s' % (self.data['ip'], str(e))) mark_expire_all(get_model('cm', ['virtualization', 'Host'])) @@ -2254,7 +2452,7 @@ def check_host_status(): try: - db = DB.get_connected_db() + db = DB.get_connected_db() select_sql = "SELECT id, name, ip, restapi_port, restapi_username, restapi_password, connection FROM host" data = db.fetchall(select_sql) db.close() @@ -2277,7 +2475,7 @@ class TaskScheduler(object): def __init__(self): self.data = {} - #start a task to check devices's connection every 600 seconds + # start a task to check devices's connection every 600 seconds sched = BackgroundScheduler() sched.add_job(check_host_status, 'interval', seconds=30) sched.add_job(check_device_status, 'interval', seconds=30) @@ -2285,13 +2483,13 @@ # check_device_status() - #init log + # init log self.init_log() - #when GenReportSchedules object init, we should restart the schedule by configuration in db + # when GenReportSchedules object init, we should restart the schedule by configuration in db db = DB.get_connected_db() sql = '''SELECT name, type, trigger, next_run_time, scheduler, state, custom_fields FROM task''' data = db.fetchall(sql) - #db.close() + # db.close() key = ['name', 'type', 'trigger', 'next_run_time', 'scheduler', 'state', 'custom_fields'] result = [dict(zip(key, each)) for each in data] for item in result: @@ -2303,17 +2501,23 @@ unit = item['next_run_time'].split()[1] interval = int(item['next_run_time'].split()[0]) if unit == "seconds": - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], seconds=interval, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], seconds=interval, + args=new_scheduler['args'], id=item['name']) elif unit == "minutes": - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], minutes=interval, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], minutes=interval, + args=new_scheduler['args'], id=item['name']) elif unit == "hours": - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], hours=interval, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], hours=interval, + args=new_scheduler['args'], id=item['name']) elif unit == "days": - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], days=interval, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], days=interval, + args=new_scheduler['args'], id=item['name']) elif unit == "weeks": - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], weeks=interval, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], weeks=interval, + args=new_scheduler['args'], id=item['name']) else: - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], hours=int(item['next_run_time']), args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], + hours=int(item['next_run_time']), args=new_scheduler['args'], id=item['name']) else: date = datetime.datetime.strptime(item['next_run_time'], '%Y-%m-%d %H:%M:%S') now = datetime.datetime.now() @@ -2323,7 +2527,8 @@ else: run_time = now + datetime.timedelta(seconds=3) sched = BackgroundScheduler() - sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], next_run_time=run_time, args=new_scheduler['args'], id=item['name']) + sched.add_job(Scheduler_func_list[new_scheduler['func']], item['trigger'], next_run_time=run_time, + args=new_scheduler['args'], id=item['name']) # if self.get_schedule(item['name']): # self.remove(item['name']) self.add(item['name'], sched) @@ -2338,7 +2543,9 @@ func_str = new_scheduler['func'] args_scheduler = new_scheduler['args'] sched = BackgroundScheduler() - sched.add_job(Scheduler_func_list[func_str], item['trigger'], next_run_time=now + datetime.timedelta(seconds=3), args=args_scheduler, id=item['name']) + sched.add_job(Scheduler_func_list[func_str], item['trigger'], + next_run_time=now + datetime.timedelta(seconds=3), args=args_scheduler, + id=item['name']) self.add(item['name'], sched) else: continue @@ -2352,28 +2559,27 @@ logger.setLevel(logging.INFO) fmt = "%(asctime)s - %(name)s - %(filename)s[line:%(lineno)d] - %(levelname)s - %(message)s" log_format = logging.Formatter(fmt) - loghandler = logging.handlers.RotatingFileHandler('/var/log/schedule.log', maxBytes=1024 * 1024 * 100, backupCount=3) + loghandler = logging.handlers.RotatingFileHandler('/var/log/schedule.log', maxBytes=1024 * 1024 * 100, + backupCount=3) loghandler.setFormatter(log_format) logger.addHandler(loghandler) - def get_schedule(self, title): if title in self.data: - return self.data[title] + return self.data[title] return None def add(self, title, schedule): if title not in self.data: self.data[title] = schedule - #we should start the schedule after adding + # we should start the schedule after adding send_notification('info', 'Task <%s> is starting.' % title) schedule.add_listener(scheduler_error_listener, EVENT_JOB_ERROR) schedule.start() - def remove(self, title): if title in self.data: - #we should shutdown the schedule before removing + # we should shutdown the schedule before removing self.data[title].shutdown(wait=False) del self.data[title] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py (working copy) @@ -1,18 +1,22 @@ -from hive.imports.model import * -from django.utils.translation import ugettext_lazy as _ -from cm.lib.postgres_db import DB -from cm.models.audit.user.user_body import * -import json +import datetime + +import requests +from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ + +from cm.lib.communication import send_https_rest_request, modify_url from cm.lib.elk import get_elk from cm.lib.libbasic_operation import get_rest_info_from_device, send_command_to_device -from hive.utils import andebug, get_device_type, standard_model_type +from cm.lib.postgres_db import DB +from cm.models.audit.user.user_body import * from djproject.an_settings import * -from cm.lib.communication import send_https_rest_request, modify_url -from django.db.models.query import QuerySet -import requests -import datetime -__=_ +from hive.imports.model import * +from hive.utils import get_device_type +__ = _ + + +# ToDo: Fix the syntax error in the following code. class AuditUser(ANModel): id = IntegerField(primary_key=True, configurable=False) user_name = CharField(verbose_name='User Name', length='1..16') @@ -64,7 +68,7 @@ CharField(verbose_name='Interval Time', name='interval'), CharField(verbose_name='Device list', name='device_list'), CharField(verbose_name='Condition', name='condition', optional=True) - ) + ) class LogVirtualServiceDetail(Action): verbose_name = _('Virtual Service Detail Log') @@ -107,12 +111,12 @@ CharField(verbose_name='Site Name', name='site'), CharField(verbose_name='Host Name', name='host'), EnumField(name='type', verbose_name='Detail Type', default='l3', values=( - ('l3', 'L3'), - ('l7', 'L7'), - ('acl', 'ACL') - )) + ('l3', 'L3'), + ('l7', 'L7'), + ('acl', 'ACL') + )) ) - + class ServerDetail(Action): verbose_name = _('Server Detail') action_name = _('ServerDetail') @@ -122,9 +126,9 @@ option_fields = ( CharField(verbose_name='Server IP', name='server_ip'), EnumField(name='server_type', verbose_name='Server Type', default='l3', values=( - ('l3', 'L3'), - ('l7', 'L7') - )) + ('l3', 'L3'), + ('l7', 'L7') + )) ) class UserLogStatistic(Action): @@ -140,7 +144,7 @@ CharField(verbose_name='Interval Time', name='interval'), CharField(verbose_name='Device Ip', name='device_ip'), CharField(verbose_name='Condition', name='condition', optional=True) - ) + ) class LogRanking(Action): verbose_name = _('Log Ranking') @@ -153,14 +157,14 @@ CharField(verbose_name='End Time', name='end_time'), IntegerField(verbose_name='Top Number', name='top_num', default=10), EnumField(verbose_name='Top Type', name='top_type', values=( - ('traffic_l3', 'Traffic L3'), - ('traffic_l7', 'Traffic L7'), - ('using_time', 'Using Times'), - ('deny_access', 'Deny Access'), - ('duration', 'Duration'), - ('server_l3', 'Server L3'), - ('server_l7', 'Server L7') - )), + ('traffic_l3', 'Traffic L3'), + ('traffic_l7', 'Traffic L7'), + ('using_time', 'Using Times'), + ('deny_access', 'Deny Access'), + ('duration', 'Duration'), + ('server_l3', 'Server L3'), + ('server_l7', 'Server L7') + )), CharField(verbose_name='Server IP', name='server_ip', default="") ) @@ -175,7 +179,7 @@ CharField(verbose_name='End Time', name='to'), CharField(verbose_name='Server IP', name='server_ip', optional=True), CharField(verbose_name='Interval Time', name='interval', optional=True) - ) + ) class VSStatistic(Action): verbose_name = _('VS Statistic') @@ -190,7 +194,7 @@ CharField(verbose_name='Device list', name='device_list'), CharField(verbose_name='Interval Time', name='interval'), CharField(verbose_name='Server IP', name='server_ip', optional=True) - ) + ) class URLStatistic(Action): verbose_name = _('URL Statistic') @@ -204,13 +208,13 @@ CharField(verbose_name='Virtual Service', name='vs'), CharField(verbose_name='Device list', name='device_list'), EnumField(verbose_name='Url Type', name='url_type', optional=True, values=( - ('document', 'Document'), - ('audio', 'Audio'), - ('video', 'Video'), - ('image', 'Image'), - ('software', 'Software') - )) - ) + ('document', 'Document'), + ('audio', 'Audio'), + ('video', 'Video'), + ('image', 'Image'), + ('software', 'Software') + )) + ) class ClearUserData(Action): verbose_name = _('Clear User Data') @@ -225,14 +229,13 @@ BooleanField(verbose_name='clear_account', name='clear_account') ) - class Manager(CLIManager): def _get_query_set(self): db = DB.get_connected_db() fetchall_sql = "SELECT id, user_name, vsite_name, host_name FROM audit_user;" data = db.fetchall(fetchall_sql) key = ['id', 'user_name', 'vsite_name', 'host_name'] - audit_result = [dict(zip(key, each)) for each in data] + audit_result = [dict(zip(key, each)) for each in data] fetchall_sql = "SELECT name, version from device;" data = db.fetchall(fetchall_sql) @@ -240,13 +243,13 @@ device_result = [dict(zip(key, each)) for each in data] db.close() - + device_info = {} for device_each in device_result: replace_result = device_each['version'].replace('\\n', '\n') rtn = cli_parse(replace_result, [ - RegexParser('Host name : (.+?)\n', MATCHONE), - ]) + RegexParser('Host name : (.+?)\n', MATCHONE), + ]) device_hostname = rtn[0][0] device_info[device_hostname] = device_each['name'] for audit_each in audit_result: @@ -254,11 +257,12 @@ audit_each['device_name'] = device_info[audit_each['host_name']] continue - return QuerySet(self._model, audit_result) + return QuerySet(self._model, audit_result) def _insert(self, instance): db = DB.get_connected_db() - insert_sql = "INSERT INTO audit_user(user_name, vsite_name, host_name) values('%s', '%s', '%s')" % (instance.user_name, instance.vsite_name, instance.host_name) + insert_sql = "INSERT INTO audit_user(user_name, vsite_name, host_name) values('%s', '%s', '%s')" % ( + instance.user_name, instance.vsite_name, instance.host_name) db.execute_sql(insert_sql) db.close() return @@ -280,12 +284,16 @@ log_result = {} hardware_id = options.has_key('hardware_id') if hardware_id: - body_text = body_hardwareid_login % (options['user_name'], options['hardware_id'], options['host'], options['site'], options['begin_time'], options['end_time']) + body_text = body_hardwareid_login % (options['user_name'], options['hardware_id'], options['host'], + options['site'], options['begin_time'], options['end_time']) else: - body_text = body_login % (options['user_name'], options['host'], options['site'], options['begin_time'], options['end_time']) + body_text = body_login % (options['user_name'], options['host'], options['site'], options['begin_time'], + options['end_time']) try: es = get_elk() - res = es._search(device_type="VPN_TYPE", doc_type="syslog", body=json.loads(body_text), _source=["time", "logid", "useragent", "sessionid", "aaa_method", "l3responselength", "l7requestlength", "l7responselength", "l3requestlength"]) + res = es._search(device_type="VPN_TYPE", doc_type="syslog", body=json.loads(body_text), + _source=["time", "logid", "useragent", "sessionid", "aaa_method", "l3responselength", + "l7requestlength", "l7responselength", "l3requestlength"]) except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) if not res: @@ -308,7 +316,8 @@ if source['sessionid'] in log_result: if 'login_time' in log_result[source['sessionid']] and 'login_time' in source: - source['login_time'] = log_result[source['sessionid']]['login_time'] if log_result[source['sessionid']]['login_time'] < source['login_time'] else source['login_time'] + source['login_time'] = log_result[source['sessionid']]['login_time'] if \ + log_result[source['sessionid']]['login_time'] < source['login_time'] else source['login_time'] log_result[source['sessionid']].update(source) else: log_result[source['sessionid']] = source @@ -322,14 +331,18 @@ l3_icmp_logid = ["200064009", "200064010"] if options['type'] == 'l3': - body_text = body_detail_l3 % (options['session_id'], options['user_name'], options['host'], options['site']) - source = ["server", "src", "sport", "serverport", "clientip", "logid", "responselength", "requestlength", "time"] - #source = ["server", "serverport", "clientip", "logid", "responselength", "requestlength", "time"] + body_text = body_detail_l3 % (options['session_id'], options['user_name'], options['host'], + options['site']) + source = ["server", "src", "sport", "serverport", "clientip", "logid", "responselength", + "requestlength", "time"] + # source = ["server", "serverport", "clientip", "logid", "responselength", "requestlength", "time"] elif options['type'] == 'l7': - body_text = body_detail_l7 % (options['session_id'], options['user_name'], options['host'], options['site']) + body_text = body_detail_l7 % (options['session_id'], options['user_name'], options['host'], + options['site']) source = ["serverip", "logid", "url", "httpstatuscode", "time"] elif options['type'] == 'acl': - body_text = body_detail_acl % (options['session_id'], options['user_name'], options['host'], options['site']) + body_text = body_detail_acl % (options['session_id'], options['user_name'], options['host'], + options['site']) source = ["logid", "url", "serverip", "serverport", "time"] try: @@ -351,14 +364,14 @@ if self.blacklist('serverip', 'localhost', source): continue - #l3 + # l3 if source['logid'] in l3_tcp_logid: source['type'] = 'tcp' elif source['logid'] in l3_udp_logid: source['type'] = 'udp' elif source['logid'] in l3_icmp_logid: source['type'] = 'icmp' - #acl + # acl elif source['logid'] in ['200052020', '200050007']: if 'url' not in source: source['url'] = '' @@ -369,9 +382,8 @@ source['url'] = '%s%s' % (source['serverip'], source['url']) del source['serverip'] - log_result.append(source) - return {'result':log_result} + return {'result': log_result} def get_traffic(self, body_text): try: @@ -380,14 +392,20 @@ except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) if not res: - raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) + raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) l3_traffic = res['aggregations'] - agg_result = [item.update({"site":site_list['key'], "host":host_list['key']}) for host_list in l3_traffic['host_list']['buckets'] for site_list in host_list['vsite_list']['buckets'] for item in site_list['user_list']['buckets']] - return [item for host_list in l3_traffic['host_list']['buckets'] for site_list in host_list['vsite_list']['buckets'] for item in site_list['user_list']['buckets']] + agg_result = [item.update({"site": site_list['key'], "host": host_list['key']}) for host_list in + l3_traffic['host_list']['buckets'] for site_list in host_list['vsite_list']['buckets'] for + item in site_list['user_list']['buckets']] + return [item for host_list in l3_traffic['host_list']['buckets'] for site_list in + host_list['vsite_list']['buckets'] for item in site_list['user_list']['buckets']] + def get_user_list(self, body_text): agg_result = self.get_traffic(body_text) - result = [{'site':item['site'], 'host':item['host'], 'user':item['key'], 'count':item['doc_count']} for item in agg_result] - return sorted(result, key=lambda x:x['count'], reverse=True) + result = [{'site': item['site'], 'host': item['host'], 'user': item['key'], 'count': item['doc_count']} for + item in agg_result] + return sorted(result, key=lambda x: x['count'], reverse=True) + def get_server_list(self, body_text): try: es = get_elk() @@ -395,23 +413,27 @@ except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) if not res: - raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) + raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) agg_result = res['aggregations']['server_list']['buckets'] - return [{item['key']:item['doc_count']} for item in agg_result] + return [{item['key']: item['doc_count']} for item in agg_result] + def get_duration_list(self, body_text): agg_result = self.get_traffic(body_text) - ret = sorted(agg_result, key = lambda x:x['duration']['value'], reverse=True) - return [{'site':item['site'], 'host':item['host'], 'user':item['key'], 'count':item['duration']['value']} for item in ret] + ret = sorted(agg_result, key=lambda x: x['duration']['value'], reverse=True) + return [ + {'site': item['site'], 'host': item['host'], 'user': item['key'], 'count': item['duration']['value']} + for item in ret] + def _perform_LogRanking(self, options): log_result = [] if options['top_type'] == 'traffic_l3': body_text = body_top_l3_traffic % (options['begin_time'], options['end_time']) ret = self.get_traffic(body_text) - ret = sorted(ret, key=lambda x:x['send']['value'], reverse=True) + ret = sorted(ret, key=lambda x: x['send']['value'], reverse=True) elif options['top_type'] == 'traffic_l7': body_text = body_top_l7_traffic % (options['begin_time'], options['end_time']) ret = self.get_traffic(body_text) - ret = sorted(ret, key=lambda x:x['send']['value'], reverse=True) + ret = sorted(ret, key=lambda x: x['send']['value'], reverse=True) elif options['top_type'] == 'using_time': body_text = body_top_using_time % (options['begin_time'], options['end_time']) ret = self.get_user_list(body_text) @@ -434,18 +456,18 @@ for i in range(0, int(options['top_num'])): if i < len(ret): log_result.append(ret[i]) - return {'result':log_result} - #return log_result + return {'result': log_result} + # return log_result def _perform_ServerDetail(self, options): - #log_result = [] + # log_result = [] if options['server_type'] == 'l3': body_text = body_server_detail_l3 % options['server_ip'] elif options['server_type'] == 'l7': body_text = body_server_detail_l7 % options['server_ip'] ret = self.get_traffic(body_text) - return {'result':ret} + return {'result': ret} def _perform_DeleteLog(self, options): body_text = body_delete_by_query_time % (options["begin_time"], options["end_time"]) @@ -453,7 +475,7 @@ es = get_elk() result = es._delete_by_query(body_text, "_all", "syslog") if result: - raise ModelQueryException(CLICmdError(__('Delete Log failed.'))) + raise ModelQueryException(CLICmdError(__('Delete Log failed.'))) return def get_virtual_service_log(self, options, op_type, device_type=None): @@ -466,8 +488,8 @@ parms["condition"] += condition_list[each] % condition[each] parms["fw"] = options["fw"] parms["from"] = options["begin_time"] - parms["to"] = options["end_time"] - + parms["to"] = options["end_time"] + es = get_elk() if op_type == "list": parms["interval"] = options["interval"] @@ -475,25 +497,31 @@ res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text)) if not res: raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) - result = res["aggregations"]["log_over_time"]["buckets"] + result = res["aggregations"]["log_over_time"]["buckets"] for item in result: if item["doc_count"] == 0: - item.update({"key_as_string":datetime.datetime.strptime(item["key_as_string"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%Y-%m-%d %H:%M:%S')}) + item.update({"key_as_string": datetime.datetime.strptime(item["key_as_string"], + "%Y-%m-%dT%H:%M:%S.%fZ").strftime( + '%Y-%m-%d %H:%M:%S')}) elif op_type == "detail": body_text = body_query % parms - res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), _source=["time","src","serverip","useragent","httpmethod","httpstatuscode","duration"]) + res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), + _source=["time", "src", "serverip", "useragent", "httpmethod", "httpstatuscode", + "duration"]) if not res: raise ModelQueryException(CLICmdError(__("Please send log to AMP first."))) - #data = res['hits']['hits'] + # data = res['hits']['hits'] result = [] for each in res['hits']['hits']: result.append(each["_source"]) - return {"result":result} + return {"result": result} def get_virtual_service_IP(self, device, virtual_service): - parse = [RegexParser('slb virtual (?P.*?) "%s" (?P[0-9|\.|:|a-f|A-F]+) (?P.*?)' % virtual_service, MATCHALL),] - cmd = {'cmd':'show slb virtual all'} + parse = [RegexParser( + 'slb virtual (?P.*?) "%s" (?P[0-9|\.|:|a-f|A-F]+) (?P.*?)' % virtual_service, + MATCHALL), ] + cmd = {'cmd': 'show slb virtual all'} device_info = get_rest_info_from_device(device)[0] url = '/rest/%s/cli_extend' % get_device_type(device_info["type"]) content = send_command_to_device(device_info, url, json.dumps(cmd)) @@ -501,14 +529,13 @@ return None result = cli_parse(content, parse) return result[0][0]['virtual_ip'] - def _perform_LogVirtualService(self, options): device = options['device_list'].split(',')[0] device_type = get_rest_info_from_device(device)[0]["type"] virtual_ip = options['vs_ip'] if not virtual_ip: - raise ModelQueryException(CLICmdError(__('Get data failed.'))) + raise ModelQueryException(CLICmdError(__('Get data failed.'))) options['fw'] = virtual_ip return self.get_virtual_service_log(options, "list", device_type) @@ -517,7 +544,7 @@ device_type = get_rest_info_from_device(device)[0]["type"] virtual_ip = options['vs_ip'] if not virtual_ip: - raise ModelQueryException(CLICmdError(__('Get data failed.'))) + raise ModelQueryException(CLICmdError(__('Get data failed.'))) options['fw'] = virtual_ip return self.get_virtual_service_log(options, "detail", device_type) @@ -549,7 +576,7 @@ if "user" in condition: user_detail_flag = 1 parms["size"] = 9999 - + for each in condition: if each in condition_list: parms["condition"] += condition_list[each] % condition[each] @@ -563,12 +590,15 @@ try: es = get_elk() if user_detail_flag: - response = es._search_index(index, doc_type="syslog", body=json.loads(body_text), _source=["time", "src", "httpmethod", "httpstatuscode", "l7requestlength", "l7responselength", "serverip", "duration", "ua_browser"]) + response = es._search_index(index, doc_type="syslog", body=json.loads(body_text), + _source=["time", "src", "httpmethod", "httpstatuscode", + "l7requestlength", "l7responselength", "serverip", "duration", + "ua_browser"]) else: response = es._search_index(index, doc_type="syslog", body=json.loads(body_text)) except Exception as e: raise ModelQueryException(CLICmdError(__(str(e)))) - + result = {} result["http_method"] = response["aggregations"]["http_method"]["buckets"] result["http_status_code"] = response["aggregations"]["http_status_code"]["buckets"] @@ -587,27 +617,29 @@ result["data"] = [item["_source"] for item in response["hits"]["hits"]] for item in result["data"]: if item["src"] in result["ip_region"]: - continue + continue result["ip_region"][item["src"]] = self._get_ip_region(item["src"]) return result def _perform_L7Statistic(self, options): if options["server_ip"]: body_text = body_server_detail_l7_date_histogram % options - source = ["time", "src", "httpmethod", "httpstatuscode", "l7requestlength", "l7responselength", "serverip", "duration"] + source = ["time", "src", "httpmethod", "httpstatuscode", "l7requestlength", "l7responselength", + "serverip", "duration"] else: body_text = body_top_server_l7 % (options["from"], options["to"], "") source = [] try: es = get_elk() - res = es._search(device_type="VPN_TYPE", doc_type="syslog", body=json.loads(body_text), _source=source, sort=["time"]) + res = es._search(device_type="VPN_TYPE", doc_type="syslog", body=json.loads(body_text), _source=source, + sort=["time"]) except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) if options["server_ip"]: result = {} - result["data"] = [ item["_source"] for item in res["hits"]["hits"]] + result["data"] = [item["_source"] for item in res["hits"]["hits"]] result["date_histogram"] = res["aggregations"]["log_over_time"]["buckets"] return result else: @@ -618,7 +650,7 @@ device_type = get_rest_info_from_device(device)[0]["type"] virtual_ip = self.get_virtual_service_IP(device, options['vs']) if not virtual_ip: - raise ModelQueryException(CLICmdError(__('Get data failed.'))) + raise ModelQueryException(CLICmdError(__('Get data failed.'))) options['fw'] = virtual_ip parms = {} src_detail_flag = 0 @@ -636,20 +668,22 @@ parms["condition"] += condition_list[each] % condition[each] parms["fw"] = options["fw"] parms["from"] = options["begin_time"] - parms["to"] = options["end_time"] + parms["to"] = options["end_time"] parms["interval"] = options["interval"] body_text = body_src_statistics % parms - #print(body_text) + # print(body_text) try: es = get_elk() if src_detail_flag: - response = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), _source = ["time", "httpstatuscode", "requestlength", "responselength", "useragent", "serverip", "duration", "src"]) + response = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), + _source=["time", "httpstatuscode", "requestlength", "responselength", + "useragent", "serverip", "duration", "src"]) else: response = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text)) except Exception as e: - raise ModelQueryException(CLICmdError(__(str(e)))) - + raise ModelQueryException(CLICmdError(__(str(e)))) + result = {} if response: result["http_method"] = response["aggregations"]["http_method"]["buckets"] @@ -669,7 +703,7 @@ result["data"] = [item["_source"] for item in response["hits"]["hits"]] for item in result["data"]: if item["src"] in result["ip_region"]: - continue + continue result["ip_region"][item["src"]] = self._get_ip_region(item["src"]) return result @@ -678,19 +712,21 @@ device_type = get_rest_info_from_device(device)[0]["type"] virtual_ip = self.get_virtual_service_IP(device, options['vs']) if not virtual_ip: - raise ModelQueryException(CLICmdError(__('Get data failed.'))) - options['fw'] = virtual_ip - + raise ModelQueryException(CLICmdError(__('Get data failed.'))) + options['fw'] = virtual_ip + if options["server_ip"]: body_text = body_server_detail_virtual_service % options - source = ["time", "src", "httpmethod", "httpstatuscode", "requestlength", "responselength", "serverip", "duration"] + source = ["time", "src", "httpmethod", "httpstatuscode", "requestlength", "responselength", "serverip", + "duration"] else: body_text = body_top_server % (options["fw"], options["from"], options["to"]) source = [] try: es = get_elk() - res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), _source=source, sort=["time"]) + res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text), _source=source, + sort=["time"]) except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) @@ -698,18 +734,18 @@ if res: if options["server_ip"]: result = {} - result["data"] = [ item["_source"] for item in res["hits"]["hits"]] + result["data"] = [item["_source"] for item in res["hits"]["hits"]] result["date_histogram"] = res["aggregations"]["log_over_time"]["buckets"] else: - result = {"result": res["aggregations"]["server_list"]["buckets"]} - return result + result = {"result": res["aggregations"]["server_list"]["buckets"]} + return result def _perform_URLStatistic(self, options): device = options['device_list'].split(',')[0] device_type = get_rest_info_from_device(device)[0]["type"] virtual_ip = self.get_virtual_service_IP(device, options['vs']) if not virtual_ip: - raise ModelQueryException(CLICmdError(__('Get data failed.'))) + raise ModelQueryException(CLICmdError(__('Get data failed.'))) options['fw'] = virtual_ip options['condition'] = "" if 'url_type' in options: @@ -721,7 +757,7 @@ es = get_elk() res = es._search(device_type=device_type, doc_type="syslog", body=json.loads(body_text)) except Exception as e: - raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) + raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) result = [] if res: @@ -730,15 +766,18 @@ item = {"real_ip": serverip["key"]} item.update(url) result.append(item) - return {"result":result} + return {"result": result} def send_commands_to_device(self, device_name, cli_list): - #return None + # return None device_info = get_rest_info_from_device(device_name)[0] - sync_config = json.dumps({'config':'\n'.join(cli_list)}) + sync_config = json.dumps({'config': '\n'.join(cli_list)}) try: new_url = modify_url('/rest/device_type/batch_cli', device_info['type']) - rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], + device_info['restapi_port'], + device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: return "connection error" if rest_response_data['status'] == 200: @@ -750,7 +789,7 @@ return "success" def _perform_ClearUserData(self, options): - result={"result":True, "result_msg":[]} + result = {"result": True, "result_msg": []} user_dict = options["id"] es = get_elk() db = DB.get_connected_db() @@ -758,7 +797,7 @@ host_name = user_dict[device_name]["host_name"] if options["clear_hardware_id"] or options["clear_account"]: for vs_name in user_dict[device_name]["vs_dict"]: - cli_list=[] + cli_list = [] user_list = user_dict[device_name]["vs_dict"][vs_name] cli_list.append("switch \"%s\"" % vs_name) for user_name in user_list: @@ -769,7 +808,8 @@ ret = self.send_commands_to_device(device_name, cli_list) if ret == "connection error": result["result"] = False - result["result_msg"].append("Error: Connect to device %s failed. Log clearing for this device is terminated." % device_name) + result["result_msg"].append( + "Error: Connect to device %s failed. Log clearing for this device is terminated." % device_name) break if ret == "process error": result["result"] = False @@ -780,7 +820,8 @@ msg += "and remove localDB accounts " elif options["clear_account"]: msg += "remove localDB accounts " - msg += "on device %s virtual site %s. Log clearing for this virtual site is terminated." % (device_name, vs_name) + msg += "on device %s virtual site %s. Log clearing for this virtual site is terminated." % ( + device_name, vs_name) result["result_msg"].append(msg) continue if ret != "success": @@ -796,22 +837,26 @@ vs_name_str = cmd[8:-1] result["result"] = False is_error = True - msg = "Error: Virtual site %s is not configured on device %s. User Data clearing for this virtual site is terminated." %(vs_name_str, device_name) + msg = "Error: Virtual site %s is not configured on device %s. User Data clearing for this virtual site is terminated." % ( + vs_name_str, device_name) result["result_msg"].append(msg) break elif cmd[0:37] == "clear localdb hardwareid rule all all": result["result"] = False is_error = True user_name = cmd[39:-1] - msg = "Error: Failed to clear hardwareid rules for user %s on device %s virtual site %s, error message: %s Log clearing for this user is terminated." %(user_name, device_name, vs_name, cmd_rst) + msg = "Error: Failed to clear hardwareid rules for user %s on device %s virtual site %s, error message: %s Log clearing for this user is terminated." % ( + user_name, device_name, vs_name, cmd_rst) elif cmd[0:18] == "no localdb account": user_name = cmd[20:-1] if cmd_rst[0:28] == "Cannot find the account name": - msg = "Warning: User %s on device %s virtual site %s is not a localDB account" %(user_name, device_name, vs_name) + msg = "Warning: User %s on device %s virtual site %s is not a localDB account" % ( + user_name, device_name, vs_name) else: result["result"] = False is_error = True - msg = "Error: Failed to remove localDB account %s on device %s virtual site %s, error message: %s Log clearing for this user is terminated." %(user_name, device_name, vs_name, cmd_rst) + msg = "Error: Failed to remove localDB account %s on device %s virtual site %s, error message: %s Log clearing for this user is terminated." % ( + user_name, device_name, vs_name, cmd_rst) result["result_msg"].append(msg) if is_error: continue @@ -822,7 +867,9 @@ rt = es._delete_by_query(body_text, "_all", "") if rt: result["result"] = False - result["result_msg"].append("Error: Delete log failed for user %s on device %s virtual site %s." %(user_name, device_name, vs_name)) + result["result_msg"].append( + "Error: Delete log failed for user %s on device %s virtual site %s." % ( + user_name, device_name, vs_name)) elif options["clear_log"]: for vs_name in user_dict[device_name]["vs_dict"]: user_list = user_dict[device_name]["vs_dict"][vs_name] @@ -832,8 +879,9 @@ rt = es._delete_by_query(body_text, "_all", "") if rt: result["result"] = False - result["result_msg"].append("Error: Delete log failed for user %s on device %s virtual site %s." %(user_name, device_name, vs_name)) + result["result_msg"].append( + "Error: Delete log failed for user %s on device %s virtual site %s." % (user_name, + device_name, + vs_name)) db.close() return result - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py (working copy) @@ -1,21 +1,28 @@ -from hive.imports.model import * +import copy +import datetime +import shutil +from importlib import reload + +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from cm.lib.AG_CLI_TABLE import get_delete_cli +from cm.lib.communication import send_https_rest_request, modify_url +from cm.lib.libbasic_operation import get_ip_address +from cm.lib.parse_configfile import parse_vsite_from_configfile, parse_vsite_config_from_configfile from cm.lib.postgres_db import DB -from djproject.settings import FILE_UPLOAD_DIR, CONFIG_UPLOAD_STORE_DIR -import os, shutil, requests, datetime, json, tarfile -from cm.models.tasking import GLOBAL_TASK +from cm.lib.task_scheduler import config_init_device, add_job_into_database, remove_job_from_database, config_batch_cli, \ + config_compliance_check, config_backup, config_custom, config_recover, config_compliance_check_all, \ + config_compliance_check_custom, config_clone, config_summary_diff from cm.models.device_mgmt.device import get_rest_info_from_device -from apscheduler.schedulers.background import BackgroundScheduler -from cm.lib.communication import send_https_rest_request, modify_url -from cm.lib.parse_configfile import parse_vsite_from_configfile, DEFAULT_CONFIG_FILE_PATH, parse_vsite_config_from_configfile, TarFiles -from cm.lib.task_scheduler import config_init_device, add_job_into_database, remove_job_from_database, config_batch_cli, config_compliance_check, config_backup, config_custom, config_recover, config_compliance_check_all, config_compliance_check_custom, config_clone, config_summary_diff -from cm.lib.AG_CLI_TABLE import get_delete_cli +from cm.models.tasking import GLOBAL_TASK from djproject.an_settings import * -from cm.lib.libbasic_operation import get_ip_address +from djproject.settings import CONFIG_UPLOAD_STORE_DIR +from hive.imports.model import * from hive.utils import get_current_session -from django.db.models.query import QuerySet -import copy -__=_ + +__ = _ """ class FileType(ANModel): name = CharField(verbose_name='File Type', primary_key=True, length='1..32') @@ -56,9 +63,11 @@ self._model._meta.mark_delay_query(each) return QuerySet(self._model, result) """ + + class ConfigFile(ANModel): name = CharField(verbose_name='File Name', primary_key=True, length='1..128') - #asso_file_type = AssoField2(verbose_name='File Type', primary_key=True, tgt='configuration.config_file.FileType.asso', + # asso_file_type = AssoField2(verbose_name='File Type', primary_key=True, tgt='configuration.config_file.FileType.asso', # mul='*', pos='right') file_type = CharField(verbose_name='File Type', length='1..32', primary_key=True) device_type = CharField(verbose_name='Device Type', length='1..32', optional=True) @@ -78,22 +87,22 @@ verbose_name = _('Sync From Device') action_name = _('Sync') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False config_change = True instance_mul = '0' option_fields = ( - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), ) class Clone_file(Action): verbose_name = _('Clone File') action_name = _('Clone') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -102,7 +111,7 @@ option_fields = ( CharField(name='file_name', verbose_name='File Name', length='1..32'), TextField(name='comment', verbose_name='Comment', optional=True, length='1..4196'), - BooleanField(name="option",verbose_name=_('Option'), default=True, display=('all', 'vs')), + BooleanField(name="option", verbose_name=_('Option'), default=True, display=('all', 'vs')), CharField(name="vs", verbose_name="Virtaul Service", optional=True), ) @@ -110,7 +119,7 @@ verbose_name = _('Initialize Device') action_name = _('Init') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -118,25 +127,30 @@ instance_mul = '1' option_fields = ( UnionField(name='devices', verbose_name='devices', optional=True, fields={ - 'group':DynamicMultiEnumField(verbose_name='Device Group', values=['__value:---Please Select---', '__field:cm.device_mgmt.device_group.DeviceGroup.name']), - 'device':DynamicMultiEnumField(verbose_name='Device', values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']) - }), + 'group': DynamicMultiEnumField(verbose_name='Device Group', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device_group.DeviceGroup.name']), + 'device': DynamicMultiEnumField(verbose_name='Device', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device.Device.name']) + }), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), ('loop', 'Loop') )), EnumField(verbose_name='Scope', optional=True, default='global', name='scope', values=( ('global', 'Global'), ('vs', 'VS') )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), - CharField(name='loop_time', verbose_name='Loop time(Seconds)', condition=ValueCondition('option', ['loop'])), - BooleanField(name='backup', verbose_name='Backup Before And After Application', optional=True, default=False), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), + CharField(name='loop_time', verbose_name='Loop time(Seconds)', + condition=ValueCondition('option', ['loop'])), + BooleanField(name='backup', verbose_name='Backup Before And After Application', optional=True, + default=False), ) class Compliance_check(Action): verbose_name = _('Compliance_check') action_name = _('Check') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -144,16 +158,19 @@ instance_mul = '1' option_fields = ( DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), + values=['__field:cm.device_mgmt.device.Device.name']), EnumField(name='module', verbose_name='Module', values=( ('all', 'All'), ('custom', 'Custom'))), - CharField(name='custom', verbose_name='Custom Value', optional=True, length='1..128', condition=ValueCondition('module', ['custom'])), - CharField(name='vsite_name', verbose_name='Vsite Name', length='1..64', condition=ValueCondition('module', ['custom'])), + CharField(name='custom', verbose_name='Custom Value', optional=True, length='1..128', + condition=ValueCondition('module', ['custom'])), + CharField(name='vsite_name', verbose_name='Vsite Name', length='1..64', + condition=ValueCondition('module', ['custom'])), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), ('loop', 'Loop') )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), CharField(name='loop_time', verbose_name='Loop time(Seconds)', condition=ValueCondition('option', ['loop'])) ) @@ -161,29 +178,33 @@ verbose_name = _('Compliance_check') action_name = _('Check') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False config_change = True instance_mul = '1' option_fields = ( - DynamicMultiEnumField(name='source_device', verbose_name='Source Device', values=['__field:cm.device_mgmt.device.Device.name']), + DynamicMultiEnumField(name='source_device', verbose_name='Source Device', + values=['__field:cm.device_mgmt.device.Device.name']), CharField(name='source_config', verbose_name='Source Config'), - DynamicMultiEnumField(name='target_device', verbose_name='Target Device', values=['__field:cm.device_mgmt.device.Device.name'], optional=True), + DynamicMultiEnumField(name='target_device', verbose_name='Target Device', + values=['__field:cm.device_mgmt.device.Device.name'], optional=True), CharField(name='target_config', verbose_name='Target Config', optional=True), CharField(name='target_type', verbose_name='Target Type', optional=True, default='device'), UnionField(name='devices', verbose_name='devices', optional=True, fields={ - 'group':DynamicMultiEnumField(verbose_name='device_group', values=['__value:---Please Select---', '__field:cm.device_mgmt.device_group.DeviceGroup.name']), - 'device':DynamicMultiEnumField(verbose_name='devices', values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']) - }), + 'group': DynamicMultiEnumField(verbose_name='device_group', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device_group.DeviceGroup.name']), + 'device': DynamicMultiEnumField(verbose_name='devices', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device.Device.name']) + }), ) class Batch_CLI(Action): verbose_name = _('Batch commands') action_name = _('Sync') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -191,18 +212,19 @@ instance_mul = '1' option_fields = ( DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), + values=['__field:cm.device_mgmt.device.Device.name']), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), ) - + class Refresh(Action): verbose_name = _('Refresh') action_name = _('Refresh') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -213,7 +235,7 @@ verbose_name = _('Sync Diff') action_name = _('Sync_Diff') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -226,7 +248,7 @@ verbose_name = _('Backup') action_name = _('Backup') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -240,7 +262,7 @@ verbose_name = _('Onekey Backup') action_name = _('Onekey Backup') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -248,24 +270,26 @@ instance_mul = '0' option_fields = ( UnionField(name='devices', verbose_name='Devices', optional=True, fields={ - 'group':DynamicMultiEnumField(verbose_name='Device Group', values=['__field:cm.device_mgmt.device_group.DeviceGroup.name']), - 'device':DynamicMultiEnumField(verbose_name='Device', values=['__value:ALL', '__field:cm.device_mgmt.device.Device.name']) - }), + 'group': DynamicMultiEnumField(verbose_name='Device Group', + values=['__field:cm.device_mgmt.device_group.DeviceGroup.name']), + 'device': DynamicMultiEnumField(verbose_name='Device', + values=['__value:ALL', '__field:cm.device_mgmt.device.Device.name']) + }), IntegerField(name='interval', verbose_name='Interval', optional=True), EnumField(name='unit', verbose_name='Unit', optional=True, default='days', values=( ('seconds', 'Seconds'), ('minutes', 'Minutes'), - ('hours', 'Hours'), - ('days', 'Days'), - ('weeks', 'Weeks')) - ) + ('hours', 'Hours'), + ('days', 'Days'), + ('weeks', 'Weeks')) + ) ) class Onekey_Compliance_Check(Action): verbose_name = _('Onekey Compliance Check') action_name = _('Onekey Compliance Check') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -275,12 +299,15 @@ EnumField(name='module', verbose_name='Module', values=( ('all', 'All'), ('custom', 'Custom'))), - CharField(name='custom', verbose_name='Custom Value', optional=True, length='1..128', condition=ValueCondition('module', ['custom'])), - CharField(name='vsite_name', verbose_name='Vsite Name', length='1..64', condition=ValueCondition('module', ['custom'])), + CharField(name='custom', verbose_name='Custom Value', optional=True, length='1..128', + condition=ValueCondition('module', ['custom'])), + CharField(name='vsite_name', verbose_name='Vsite Name', length='1..64', + condition=ValueCondition('module', ['custom'])), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), ('loop', 'Loop') )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), CharField(name='loop_time', verbose_name='Loop time(Seconds)', condition=ValueCondition('option', ['loop'])) ) @@ -288,14 +315,15 @@ verbose_name = _('Recover') action_name = _('Recover') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False config_change = True instance_mul = '1' option_fields = ( - BooleanField(name='backup', verbose_name='Backup Before And After Application', optional=True, default=False), + BooleanField(name='backup', verbose_name='Backup Before And After Application', optional=True, + default=False), ) class Manager(CLIManager): @@ -312,7 +340,7 @@ del each['type'] #self._model._meta.mark_delay_query(each) """ - res = sorted(result, key=lambda x: x["create_time"], reverse=True) + res = sorted(result, key=lambda x: x["create_time"], reverse=True) return QuerySet(self._model, res) def _update_comment(self, instance): @@ -337,7 +365,7 @@ db.close() raise ModelQueryException(CLICmdError(__('The file name has existed!'))) db.close() - + if not os.path.exists(file_path): os.makedirs(file_path) if file_type == "customize" or file_type == "template": @@ -361,7 +389,8 @@ raise ModelQueryException(CLICmdError(__('This cli did not include ##URL##.'))) if filename == data['name']: - raise ModelQueryException(CLICmdError(__('This configuration file name can not be same with the upload file name.'))) + raise ModelQueryException( + CLICmdError(__('This configuration file name can not be same with the upload file name.'))) config_dir_path = os.path.join(CONFIG_UPLOAD_STORE_DIR, "upload") if not os.path.exists(config_dir_path): @@ -379,19 +408,22 @@ try: local_ip = get_ip_address(network_card) except Exception as e: - raise ModelQueryException(CLICmdError(__('Local network settings are abnormal. Please configuare management interface by CLI "system management intertface ".'))) + raise ModelQueryException(CLICmdError( + __('Local network settings are abnormal. Please configuare management interface by CLI "system management intertface ".'))) cli_url = '"ftp://array:admin@%s:9993/upload/%s/%s"' % (local_ip, task_name, filename) - cli_content = cli_content.replace('"'+replace_str+'"', cli_url) + cli_content = cli_content.replace('"' + replace_str + '"', cli_url) cli_content = cli_content.replace(replace_str, cli_url) new_schedule = {} description = "Customzied config file create time: " + create_time + '\n' new_schedule['func'] = 'config_custom' - new_schedule['args'] = [task_name, cli_content, data['device_type'], "Add a new customzied config file."] + new_schedule['args'] = [task_name, cli_content, data['device_type'], + "Add a new customzied config file."] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_custom', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list="") + add_job_into_database(task_name, 'config_custom', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list="") sched = BackgroundScheduler() sched.add_job(config_custom, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -400,7 +432,8 @@ else: os.mknod(file_path + '/' + data['name']) - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % (data['name'] + ".cfg", create_time, create_time, file_type, data['comment'], data['device_type']) + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + data['name'] + ".cfg", create_time, create_time, file_type, data['comment'], data['device_type']) db = DB.get_connected_db() db.execute_sql(insert_sql) db.close() @@ -431,12 +464,15 @@ return def send_commands_to_device(self, device_name, cli_list): - #return None + # return None device_info = get_rest_info_from_device(device_name)[0] - sync_config = json.dumps({'config':'\n'.join(cli_list)}) + sync_config = json.dumps({'config': '\n'.join(cli_list)}) try: new_url = modify_url('/rest/device_type/batch_cli', device_info['type']) - rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, sync_config, device_info['ip_address'], + device_info['restapi_port'], + device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: return "Can not connect the device:%s" % device_name if rest_response_data['status'] == 200: @@ -471,8 +507,9 @@ if result and result != 'Error': result_cli.append(result) elif result == 'Error': - command_result[device_name] += '%s:%s' % (add_cli[1:], 'Sorry ACM can not execute this command.\n') - #print result_cli + command_result[device_name] += '%s:%s' % (add_cli[1:], + 'Sorry ACM can not execute this command.\n') + # print result_cli ret = self.send_commands_to_device(device_name, result_cli) if ret: command_result[device_name] += ret @@ -493,7 +530,8 @@ new_schedule['func'] = 'config_backup' new_schedule['args'] = [task_name, device_info, options['comment']] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_backup', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) + add_job_into_database(task_name, 'config_backup', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) sched = BackgroundScheduler() sched.add_job(config_backup, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -539,12 +577,14 @@ GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} - description = " Backup config create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = " Backup config create time: " + datetime.datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + '\n' if trigger == "interval": description = "Scheduler" + description # insert this interval backup into schedule_backup_all table. db = DB.get_connected_db() - insert_sql = "INSERT INTO schedule_backup_all(task_name, device_name, interval, unit) values('%s', '%s', %d, '%s') " % (task_name, item, options["interval"], options["unit"]) + insert_sql = "INSERT INTO schedule_backup_all(task_name, device_name, interval, unit) values('%s', '%s', %d, '%s') " % ( + task_name, item, options["interval"], options["unit"]) db.execute_sql(insert_sql) db.close() @@ -555,33 +595,41 @@ new_schedule["unit"] = options["unit"] new_schedule_str = json.dumps(new_schedule) if trigger == "interval": - add_job_into_database(task_name, 'config_backup', description, trigger, 'waiting', str(options["interval"]) + " " + options["unit"], new_schedule_str, device_list=item) + add_job_into_database(task_name, 'config_backup', description, trigger, 'waiting', + str(options["interval"]) + " " + options["unit"], new_schedule_str, + device_list=item) else: - add_job_into_database(task_name, 'config_backup', description, trigger, 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=item) + add_job_into_database(task_name, 'config_backup', description, trigger, 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=item) sched = BackgroundScheduler() if trigger == "interval": if options["unit"] == "seconds": - sched.add_job(config_backup, trigger, seconds=options["interval"], args=new_schedule['args'], id=task_name) + sched.add_job(config_backup, trigger, seconds=options["interval"], args=new_schedule['args'], + id=task_name) elif options["unit"] == "minutes": - sched.add_job(config_backup, trigger, minutes=options["interval"], args=new_schedule['args'], id=task_name) + sched.add_job(config_backup, trigger, minutes=options["interval"], args=new_schedule['args'], + id=task_name) elif options["unit"] == "hours": - sched.add_job(config_backup, trigger, hours=options["interval"], args=new_schedule['args'], id=task_name) + sched.add_job(config_backup, trigger, hours=options["interval"], args=new_schedule['args'], + id=task_name) elif options["unit"] == "days": - sched.add_job(config_backup, trigger, days=options["interval"], args=new_schedule['args'], id=task_name) + sched.add_job(config_backup, trigger, days=options["interval"], args=new_schedule['args'], + id=task_name) elif options["unit"] == "weeks": - sched.add_job(config_backup, trigger, weeks=options["interval"], args=new_schedule['args'], id=task_name) + sched.add_job(config_backup, trigger, weeks=options["interval"], args=new_schedule['args'], + id=task_name) else: sched.add_job(config_backup, trigger, next_run_time=date, args=new_schedule['args'], id=task_name) GLOBAL_TASK.add(task_name, sched) - mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - return + mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) + return def _perform_Recover(self, options): pk_list = options['__pk_list'][0] backup = True if "backup" in options and options["backup"] else False file_type = pk_list['file_type'] - #pk_list {u'file_type': u'backup', u'name': u'10.8.2.55-2017_08_04_17:35:20.all_cfg_tar'} + # pk_list {u'file_type': u'backup', u'name': u'10.8.2.55-2017_08_04_17:35:20.all_cfg_tar'} if file_type != 'backup': raise ModelQueryException(CLICmdError(__("Sorry, only backup file can do this!"))) @@ -597,7 +645,8 @@ file_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + pk_list['name'] if not os.path.exists(file_path): - raise ModelQueryException(CLICmdError(__("Sorry, can not find this file(%s)." % os.path.basename(file_path)))) + raise ModelQueryException( + CLICmdError(__("Sorry, can not find this file(%s)." % os.path.basename(file_path)))) device_info = get_rest_info_from_device(device_name)[0] if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) @@ -607,7 +656,8 @@ new_schedule['func'] = 'config_recover' new_schedule['args'] = [task_name, device_info, file_type, pk_list['name'], backup, device_name] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_recover', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) + add_job_into_database(task_name, 'config_recover', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_name) sched = BackgroundScheduler() sched.add_job(config_recover, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -616,63 +666,67 @@ return def _perform_Refresh(self, options): - #configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'qintianzhu'}]} - #configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'\u5a01\u9707\u5929_2017-05-25'}]} - #configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'\u5a01\u9707\u5929'}, {u'file_type': u'device', u'name': u'\u64ce\u5929\u67f1'}]} - #print "configuration refresh", options + # configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'qintianzhu'}]} + # configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'\u5a01\u9707\u5929_2017-05-25'}]} + # configuration refresh {u'__pk_list': [{u'file_type': u'device', u'name': u'\u5a01\u9707\u5929'}, {u'file_type': u'device', u'name': u'\u64ce\u5929\u67f1'}]} + # print "configuration refresh", options pk_list = options['__pk_list'] - device_list = {'device':[]} + device_list = {'device': []} for data in pk_list: if data['file_type'] != "device": raise ModelQueryException(CLICmdError(__("Sorry, you can only refresh the device type file!"))) device_list['device'].append(data['name']) self._perform_Sync_from_device(device_list) - #refresh vsite file + # refresh vsite file device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') if not os.path.isdir(device_path): return for device in device_list['device']: file_path = device_path + '/' + device - #parse vsite name, ip... from global config + # parse vsite name, ip... from global config parse_vsite_config_from_configfile(file_path, device) - #parse all vsite config from 'show running all', and write to vsite file + # parse all vsite config from 'show running all', and write to vsite file parse_vsite_from_configfile(file_path, device) return def _perform_Sync_from_device(self, options): - #sync from device {u'device': [u'\u5a01\u9707\u5929', u'\u64ce\u5929\u67f1'], - #perform sync from device {u'device': [u'\u64ce\u5929\u67f1'], u'__pk_list': []} - #print "perform sync from device", options + # sync from device {u'device': [u'\u5a01\u9707\u5929', u'\u64ce\u5929\u67f1'], + # perform sync from device {u'device': [u'\u64ce\u5929\u67f1'], u'__pk_list': []} + # print "perform sync from device", options for device in options['device']: device_info = get_rest_info_from_device(device) _type = get_device_type(device_info[0]['type']) if not _type: raise ModelQueryException(CLICmdError(__("Cannot query the device url from device type."))) - #device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] + # device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] if device_info[0]['type'].lower() in VPN_TYPE_LIST: - data = {'cmd' : 'show running all'} + data = {'cmd': 'show running all'} new_url = '/rest/%s/global/cli_extend' % _type elif device_info[0]['type'].lower() in ADC_TYPE_LIST: - data = {'cmd' : 'show running'} + data = {'cmd': 'show running'} new_url = '/rest/%s/cli_extend' % _type running_params = json.dumps(data) - + try: - rest_response_data = send_https_rest_request('POST', new_url, running_params, device_info[0]['ip_address'], device_info[0]['restapi_port'], device_info[0]['restapi_username'], device_info[0]['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, running_params, + device_info[0]['ip_address'], + device_info[0]['restapi_port'], + device_info[0]['restapi_username'], + device_info[0]['restapi_password']) except: raise ModelQueryException(CLICmdError(__("Can not connect!"))) if rest_response_data['status'] == 200: config_result = json.loads(rest_response_data['body'])['contents'] else: raise ModelQueryException(CLICmdError(__("The request is not found!"))) - + device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') if not os.path.isdir(device_path): os.makedirs(device_path) - #create_time = datetime.datetime.now().strftime('%Y-%m-%d') - #file_name = device_info[0]['name'] + '_' + datetime.datetime.now().strftime('%Y-%m-%d') + # create_time = datetime.datetime.now().strftime('%Y-%m-%d') + # file_name = device_info[0]['name'] + '_' + datetime.datetime.now().strftime('%Y-%m-%d') file_name = device_info[0]['name'] try: config_file = open(device_path + '/' + file_name, 'w') @@ -683,7 +737,8 @@ create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % (file_name, create_time, create_time, 'device') + insert_sql = "INSERT INTO file_list(name, create_time, modify_time, type) values('%s', '%s', '%s', '%s') " % ( + file_name, create_time, create_time, 'device') delete_sql = "DELETE FROM file_list where name='%s' " % file_name db = DB.get_connected_db() db.execute_sql(delete_sql) @@ -695,8 +750,8 @@ db = DB.get_connected_db() fetchall_sql = "SELECT name FROM file_list where type='backup' and name like '%" + device_name + "%'" data = db.fetchall(fetchall_sql) - key = ['name',] - backup_list = [dict(zip(key, each)) for each in data] + key = ['name', ] + backup_list = [dict(zip(key, each)) for each in data] db.close() device_backup_list = [] @@ -722,7 +777,10 @@ new_schedule['func'] = 'config_clone' new_schedule['args'] = [task_name, options] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_clone', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=pk_list['name'][:-4] if pk_list['name'][-4:] == '.cfg' else pk_list['name']) + add_job_into_database(task_name, 'config_clone', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=pk_list['name'][:-4] if pk_list['name'][-4:] == '.cfg' else pk_list[ + 'name']) sched = BackgroundScheduler() sched.add_job(config_clone, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -762,8 +820,9 @@ else: device_list = options['devices'].values()[0] - #options {u'device': u'\u64ce\u5929\u67f1', u'__pk_list': [{u'asso_file_type': [{u'_asso_idx': 0, u'name': u'test'}], u'name': u'\u6d4b\u8bd5'}]} - if pk_list['file_type'] != 'system' and pk_list['file_type'] != 'customize' and pk_list['file_type'] != 'vs' and pk_list['file_type'] != 'template': + # options {u'device': u'\u64ce\u5929\u67f1', u'__pk_list': [{u'asso_file_type': [{u'_asso_idx': 0, u'name': u'test'}], u'name': u'\u6d4b\u8bd5'}]} + if pk_list['file_type'] != 'system' and pk_list['file_type'] != 'customize' and pk_list[ + 'file_type'] != 'vs' and pk_list['file_type'] != 'template': raise ModelQueryException(CLICmdError(__("Sorry, only the system file can do this!"))) file_name = pk_list['name'] file_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, pk_list['file_type']) @@ -773,12 +832,14 @@ if options['option'] == 'immediate': date = date_now + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < date_now: - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), date_now.strftime('%Y-%m-%d %H:%M:%S'))))) - + if date < date_now: + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + date_now.strftime('%Y-%m-%d %H:%M:%S'))))) + for device in device_list: if type(device) == list: device_str = device[0] @@ -798,9 +859,11 @@ new_schedule = {} description = " Initialize device create time: " + date_now.strftime('%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'config_init_device' - new_schedule['args'] = [task_name, device_info, pk_list['file_type'], file_name, options['backup'], device_str] + new_schedule['args'] = [task_name, device_info, pk_list['file_type'], file_name, options['backup'], + device_str] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_init', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_str) + add_job_into_database(task_name, 'config_init', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device_str) sched = BackgroundScheduler() sched.add_job(config_init_device, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -808,17 +871,20 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return - def check_config_from_device(self, file_name, diff_file_path, task_name, options, file_type, source_deivce=None): + def check_config_from_device(self, file_name, diff_file_path, task_name, options, file_type, + source_deivce=None): if not os.path.exists(os.path.dirname(diff_file_path)): os.mkdir(os.path.dirname(diff_file_path)) if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if options['option'] != 'loop' and date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + if options['option'] != 'loop' and date < datetime.datetime.now(): + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) @@ -845,34 +911,47 @@ else: check_info['is_global'] = True - description = " Compliance check create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = " Compliance check create time: " + datetime.datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + '\n' if options['module'] == 'all': new_schedule['func'] = 'config_compliance_check_all' - new_schedule['args'] = [task_name, check_info, file_name, file_type, diff_file_path, task_type, get_current_session().username] + new_schedule['args'] = [task_name, check_info, file_name, file_type, diff_file_path, task_type, + get_current_session().username] else: new_schedule['func'] = 'config_compliance_check' - new_schedule['args'] = [task_name, check_info, file_name, file_type, diff_file_path, options['custom'], task_type, get_current_session().username] + new_schedule['args'] = [task_name, check_info, file_name, file_type, diff_file_path, options['custom'], + task_type, get_current_session().username] new_schedule_str = json.dumps(new_schedule) - #add_job_into_database(task_name, 'config', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str) + # add_job_into_database(task_name, 'config', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str) sched = BackgroundScheduler() if options['module'] == 'all': if options['option'] == 'loop': - add_job_into_database(task_name, 'config_check', description, 'interval', 'waiting', options['loop_time'], new_schedule_str, device_list=options["device"]) - sched.add_job(config_compliance_check_all, 'interval', hours=int(options['loop_time']), args=new_schedule['args'], id=task_name) + add_job_into_database(task_name, 'config_check', description, 'interval', 'waiting', + options['loop_time'], new_schedule_str, device_list=options["device"]) + sched.add_job(config_compliance_check_all, 'interval', hours=int(options['loop_time']), + args=new_schedule['args'], id=task_name) else: - add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=options["device"]) - sched.add_job(config_compliance_check_all, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) + add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=options["device"]) + sched.add_job(config_compliance_check_all, 'date', next_run_time=date, args=new_schedule['args'], + id=task_name) else: if options['option'] == 'loop': - add_job_into_database(task_name, 'config_check', description, 'interval', 'waiting', options['loop_time'], new_schedule_str, device_list=options["device"]) - sched.add_job(config_compliance_check, 'interval', hours=int(options['loop_time']), args=new_schedule['args'], id=task_name) + add_job_into_database(task_name, 'config_check', description, 'interval', 'waiting', + options['loop_time'], new_schedule_str, device_list=options["device"]) + sched.add_job(config_compliance_check, 'interval', hours=int(options['loop_time']), + args=new_schedule['args'], id=task_name) else: - add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=options["device"]) - sched.add_job(config_compliance_check, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) + add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=options["device"]) + sched.add_job(config_compliance_check, 'date', next_run_time=date, args=new_schedule['args'], + id=task_name) GLOBAL_TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - return + return def _perform_Compliance_check(self, options): pk_list = options['__pk_list'][0] @@ -880,7 +959,8 @@ raise ModelQueryException(CLICmdError(__("Sorry you can not do it!"))) # fetch the source device which cloned. - fetch_sql = "select modify_time from file_list where name = '%s' and type = '%s' and device_type = '%s'" % (pk_list['name'], pk_list['file_type'], pk_list['device_type']) + fetch_sql = "select modify_time from file_list where name = '%s' and type = '%s' and device_type = '%s'" % ( + pk_list['name'], pk_list['file_type'], pk_list['device_type']) db = DB.get_connected_db() source_deivce = db.fetchall(fetch_sql)[0][0] db.close() @@ -891,13 +971,17 @@ else: file_name = pk_list['name'] - diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, file_name, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) + diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, file_name, + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) if pk_list['file_type'] == 'customize': - task_name = pk_list['name'].split('.cfg')[0] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + task_name = pk_list['name'].split('.cfg')[0] + '-' + datetime.datetime.now().strftime( + '%Y_%m_%d_%H:%M:%S') else: - task_name = pk_list['name'].split('.all_cfg_tar')[0] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + task_name = pk_list['name'].split('.all_cfg_tar')[0] + '-' + datetime.datetime.now().strftime( + '%Y_%m_%d_%H:%M:%S') - return self.check_config_from_device(file_name, diff_file_path, task_name, options, pk_list['file_type'], None if ":" in source_deivce and "-" in source_deivce else source_deivce) + return self.check_config_from_device(file_name, diff_file_path, task_name, options, pk_list['file_type'], + None if ":" in source_deivce and "-" in source_deivce else source_deivce) def _perform_Custom_Compliance_check(self, options): file_type = "backup" @@ -919,7 +1003,8 @@ each_device_list = [each[0] for each in data if each[0]] if not len(each_device_list): db.close() - raise ModelQueryException(CLICmdError(__('Please add devices to the device group: %s.' % group))) + raise ModelQueryException( + CLICmdError(__('Please add devices to the device group: %s.' % group))) device_list += each_device_list db.close() else: @@ -933,7 +1018,8 @@ target_list.append({'target': each, 'target_file': data[0][0]}) else: db.close() - raise ModelQueryException(CLICmdError(__('Device (%s) has no backup file, please backup first.' % each))) + raise ModelQueryException( + CLICmdError(__('Device (%s) has no backup file, please backup first.' % each))) db.close() else: target_list = [{'target': options["target_device"], 'target_file': options["target_config"]}] @@ -953,7 +1039,8 @@ 'target_file': each["target_file"] } diff_file_name = options["source_device"] + "-" + options["target_device"] - diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, diff_file_name, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) + diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, diff_file_name, + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) task_file_list.append({ "diff_file_path": diff_file_path, "target_device": each["target"], @@ -971,20 +1058,25 @@ remove_job_from_database(task_name) new_schedule = {} - description = "Check compliance between %s and %s create time: " % (options["source_device"], options["target_device"]) + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = "Check compliance between %s and %s create time: " % (options["source_device"], options[ + "target_device"]) + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'config_compliance_check_custom' - new_schedule['args'] = [task_name, check_info, options, file_type, diff_file_path, "date", get_current_session().username] + new_schedule['args'] = [task_name, check_info, options, file_type, diff_file_path, "date", + get_current_session().username] new_schedule_str = json.dumps(new_schedule) sched = BackgroundScheduler() - add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) - sched.add_job(config_compliance_check_custom, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) + add_job_into_database(task_name, 'config_check', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) + sched.add_job(config_compliance_check_custom, 'date', next_run_time=date, args=new_schedule['args'], + id=task_name) GLOBAL_TASK.add(task_name, sched) if target_type == 'batch': diff_file_name = 'summary_diff' - diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, diff_file_name, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) + diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, diff_file_name, + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) task_name = diff_file_name + "-" + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') if not os.path.exists(os.path.dirname(diff_file_path)): os.mkdir(os.path.dirname(diff_file_path)) @@ -994,19 +1086,21 @@ remove_job_from_database(task_name) new_schedule = {} - description = "Summary diff, source device is : %s" % source + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = "Summary diff, source device is : %s" % source + datetime.datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'config_summary_diff' - new_schedule['args'] = [task_name, target_list, task_name_list, diff_file_path, task_file_list, source, source_file] + new_schedule['args'] = [task_name, target_list, task_name_list, diff_file_path, task_file_list, source, + source_file] new_schedule_str = json.dumps(new_schedule) sched = BackgroundScheduler() - add_job_into_database(task_name, 'summary_diff', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) + add_job_into_database(task_name, 'summary_diff', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) sched.add_job(config_summary_diff, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) GLOBAL_TASK.add(task_name, sched) - mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - return + return def _perform_Onekey_Compliance_Check(self, options): for pk_list in options['__pk_list']: @@ -1017,11 +1111,14 @@ file_name = pk_list['name'] opt = copy.copy(options) opt["device"] = pk_list["device_name"] - diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, file_name, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) + diff_file_path = '%sdiff/diff-%s-%s' % (DEFAULT_CONFIG_FILE_PATH, file_name, + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) if pk_list['file_type'] == 'customize': - task_name = pk_list['name'].split('.cfg')[0] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + task_name = pk_list['name'].split('.cfg')[0] + '-' + datetime.datetime.now().strftime( + '%Y_%m_%d_%H:%M:%S') else: - task_name = pk_list['name'].split('.all_cfg_tar')[0] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') + task_name = pk_list['name'].split('.all_cfg_tar')[0] + '-' + datetime.datetime.now().strftime( + '%Y_%m_%d_%H:%M:%S') self.check_config_from_device(file_name, diff_file_path, task_name, opt, pk_list['file_type']) return @@ -1037,11 +1134,13 @@ if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + if date < datetime.datetime.now(): + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) for device in options['device']: time.sleep(1) date_now = datetime.datetime.now() @@ -1051,19 +1150,23 @@ GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} - description = " Send batch commands to device create time: " + date_now.strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = " Send batch commands to device create time: " + date_now.strftime( + '%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'config_batch_cli' new_schedule['args'] = [task_name, device_info, file_path] new_schedule_str = json.dumps(new_schedule) real_date = date + datetime.timedelta(seconds=1) - add_job_into_database(task_name, 'sync_any', description, 'date', 'waiting', real_date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device) + add_job_into_database(task_name, 'sync_any', description, 'date', 'waiting', + real_date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device) sched = BackgroundScheduler() - sched.add_job(config_batch_cli, 'date', next_run_time=real_date, args=new_schedule['args'], id=task_name) + sched.add_job(config_batch_cli, 'date', next_run_time=real_date, args=new_schedule['args'], + id=task_name) GLOBAL_TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return + class TarFile(ANModel): id = IntegerField(verbose_name='ID', primary_key=True, configurable=False) name = CharField(verbose_name='File Name', primary_key=True, length='1..64') @@ -1086,8 +1189,9 @@ del each['type'] #self._model._meta.mark_delay_query(each) """ - return QuerySet(self._model, result) + return QuerySet(self._model, result) + class TemplateFile(ANModel): name = CharField(verbose_name='File Name', primary_key=True, length='1..64') device_type = CharField(verbose_name='Device Type', length='1..64') @@ -1113,4 +1217,4 @@ del each['type'] #self._model._meta.mark_delay_query(each) """ - return QuerySet(self._model, result) + return QuerySet(self._model, result) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py (working copy) @@ -1,21 +1,24 @@ -from hive.imports.model import * +import datetime + +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -#from cm.lib.sqlite_db import DB + from cm.lib.postgres_db import DB -from djproject.settings import FILE_UPLOAD_DIR -import hashlib, time -import os, shutil, requests, datetime, json -from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, get_file_md5, compare_md5, mv_file, upload_job, update_updatelist, upload_to_device +from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, get_file_md5, compare_md5, mv_file, \ + upload_job, update_updatelist, upload_to_device from cm.models.tasking import GLOBAL_TASK -from apscheduler.schedulers.background import BackgroundScheduler -from django.db.models.query import QuerySet -__=_ +from djproject.settings import FILE_UPLOAD_DIR +from hive.imports.model import * +__ = _ + + class Update(ANModel): id = IntegerField(primary_key=True, configurable=False) app_name = CharField(verbose_name='APP Name', length='1..32', primary_key=True) build_version = CharField(verbose_name='Build Version', length='1..64', primary_key=True) - #file_size = IntegerField(verbose_name='File Size', optional=True, configurable=False) + # file_size = IntegerField(verbose_name='File Size', optional=True, configurable=False) file_size = CharField(verbose_name='File Size', optional=True, configurable=False) md5_value = CharField(verbose_name='MD5', length='1..64', editable=True) download_link = CharField(verbose_name='Download Link', optional=True, length='1..128', configurable=False) @@ -24,15 +27,15 @@ class Meta: verbose_name = _('Upgrade') show_im_export_button = False - list_config_options={ - 'columns':[ - {'name':'app_name'}, - {'name':'build_version'}, - {'name':'md5_value'}, - {'name':'file_size'}, - {'name':'download_link'}, - {'name':'location'} - ]} + list_config_options = { + 'columns': [ + {'name': 'app_name'}, + {'name': 'build_version'}, + {'name': 'md5_value'}, + {'name': 'file_size'}, + {'name': 'download_link'}, + {'name': 'location'} + ]} class Upload(Action): verbose_name = _('Build Upload') @@ -43,10 +46,12 @@ instance_mul = '1' option_fields = ( UnionField(name='using', verbose_name='Using', optional=True, fields={ - 'local':ImportLocalFileField(verbose_name=_('Local File'), button_text=__('Choose Update File'), maxFileSize=500000000, acceptFileSuffix='^.*?\\.(array|click)$'), - 'URL':ImportURLField(verbose_name=_('URL')) + 'local': ImportLocalFileField(verbose_name=_('Local File'), button_text=__('Choose Update File'), + maxFileSize=500000000, acceptFileSuffix='^.*?\\.(array|click)$'), + 'URL': ImportURLField(verbose_name=_('URL')) }), ) + class System_update(Action): verbose_name = _('Update') action_name = _('Update') @@ -58,16 +63,18 @@ # DynamicMultiEnumField(name='device', verbose_name='devices', # values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']), UnionField(name='devices', verbose_name='devices', optional=True, fields={ - 'group':DynamicMultiEnumField(verbose_name='device_group', values=['__value:---Please Select---', '__field:cm.device_mgmt.device_group.DeviceGroup.name']), - 'device':DynamicMultiEnumField(verbose_name='devices', values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']) - }), + 'group': DynamicMultiEnumField(verbose_name='device_group', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device_group.DeviceGroup.name']), + 'device': DynamicMultiEnumField(verbose_name='devices', values=['__value:---Please Select---', + '__field:cm.device_mgmt.device.Device.name']) + }), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), BooleanField(verbose_name='Backup Database', name='backup_db', default=False, optional=True), ) - class Manager(CLIManager): def _get_query_set(self): @@ -81,13 +88,14 @@ file_size = item['file_size'] if file_size < 1024: item['file_size'] = str(file_size) - elif file_size < 1024*1024: - item['file_size'] = str(file_size/1024) + 'KB' - elif file_size < 1024*1024*1024: - item['file_size'] = str(file_size/1024/1024) + 'MB' + elif file_size < 1024 * 1024: + item['file_size'] = str(file_size / 1024) + 'KB' + elif file_size < 1024 * 1024 * 1024: + item['file_size'] = str(file_size / 1024 / 1024) + 'MB' else: - item['file_size'] = str(file_size/1024/1024/1024) + 'GB' + item['file_size'] = str(file_size / 1024 / 1024 / 1024) + 'GB' return QuerySet(self._model, result) + def _update_md5_value(self, instance): data = instance.get_field_dict() update_sql = "UPDATE update_list SET md5_value='%s' WHERE id='%d'" % (data['md5_value'], data['id']) @@ -100,10 +108,12 @@ db.execute_sql(update_sql) db.close() return + def _insert(self, instance): data = instance.get_field_dict() db = DB.get_connected_db() - save_sql = "INSERT INTO update_list(app_name, build_version, md5_value) values('%s', '%s', '%s')" % (data['app_name'], data['build_version'], data['md5_value']) + save_sql = "INSERT INTO update_list(app_name, build_version, md5_value) values('%s', '%s', '%s')" % ( + data['app_name'], data['build_version'], data['md5_value']) count_sql = "select count(id) from update_list;" select_sql = "SELECT id FROM update_list WHERE md5_value='%s'" % (data['md5_value']) @@ -119,12 +129,14 @@ db.execute_sql(save_sql) db.close() return + def _delete_instance(self, instance): data = instance.get_field_dict() db = DB.get_connected_db() - #check the status of upload - select_sql = "SELECT state FROM task where name='%s' and type='upload'" % (data['app_name']+'-'+data['build_version']) + # check the status of upload + select_sql = "SELECT state FROM task where name='%s' and type='upload'" % ( + data['app_name'] + '-' + data['build_version']) result = db.fetchall(select_sql) if result and (result[0][0] == 'ongoing' or result[0][0] == 'waiting'): raise ModelQueryException(CLICmdError(__('This build is uploading.'))) @@ -139,6 +151,7 @@ db.close() return + def _perform_Upload(self, options): def update_download_link(id, url): update_sql = "UPDATE update_list SET download_link='%s' WHERE id='%d'" % (url, id) @@ -146,8 +159,9 @@ db.execute_sql(update_sql) db.close() mark_expire_all(Update) - #print 'action update ', options, options['using'].values()[0], options['__pk_list'][0]['id'] - #print options['using'] + + # print 'action update ', options, options['using'].values()[0], options['__pk_list'][0]['id'] + # print options['using'] if 'local' in options['using']: tmp_file = options['using'].values()[0] tmp_file = FILE_UPLOAD_DIR + os.path.basename(tmp_file) @@ -163,10 +177,11 @@ update_download_link(int(options['__pk_list'][0]['id']), tmp_url) - #name = 'upload-' + options['__pk_list'][0]['app_name'] + '-' + options['__pk_list'][0]['build_version'] - date_now = datetime.datetime.now() #.strftime('%Y-%m-%d %H:%M:%S') + # name = 'upload-' + options['__pk_list'][0]['app_name'] + '-' + options['__pk_list'][0]['build_version'] + date_now = datetime.datetime.now() # .strftime('%Y-%m-%d %H:%M:%S') date = date_now + datetime.timedelta(seconds=3) - name = options['__pk_list'][0]['app_name'] + '-' + options['__pk_list'][0]['build_version'] + '-' + date_now.strftime('%Y_%m_%d_%H:%M:%S') + name = options['__pk_list'][0]['app_name'] + '-' + options['__pk_list'][0][ + 'build_version'] + '-' + date_now.strftime('%Y_%m_%d_%H:%M:%S') if GLOBAL_TASK.get_schedule(name): GLOBAL_TASK.remove(name) @@ -177,13 +192,16 @@ new_schedule['func'] = 'upload_job' new_schedule['args'] = [int(options['__pk_list'][0]['id']), name, tmp_url] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(name, 'upload', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str) + add_job_into_database(name, 'upload', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str) sched = BackgroundScheduler() - sched.add_job(upload_job, 'date', next_run_time=date, args=[int(options['__pk_list'][0]['id']), name, tmp_url], id=name) + sched.add_job(upload_job, 'date', next_run_time=date, + args=[int(options['__pk_list'][0]['id']), name, tmp_url], id=name) GLOBAL_TASK.add(name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return + def _perform_System_update(self, options): def get_data_from_device(device_name): db = DB.get_connected_db() @@ -193,12 +211,13 @@ key = ['ip_address', 'restapi_port', 'restapi_username', 'restapi_password', 'type'] result = [dict(zip(key, each)) for each in data] return result + pk_list = options['__pk_list'][0] - #print 'options', options - #options {u'__pk_list': [{u'build_version': u'9.4.0.138.7', u'app_name': u'AG', u'id': 1}], + # print 'options', options + # options {u'__pk_list': [{u'build_version': u'9.4.0.138.7', u'app_name': u'AG', u'id': 1}], # u'devices': {u'group': [u'BeiJing']}, u'option': u'immediate'} # u'devices': {u'device': [u'10.8.2.55', u'10.8.2.54']} - #devices = options['devices'] + # devices = options['devices'] device_list = [] if 'group' in options['devices']: group_list = options['devices'].values()[0] @@ -214,11 +233,12 @@ db.close() elif 'device' in options['devices']: device_list = options['devices'].values()[0] - + for device in device_list: time.sleep(1) data_now = datetime.datetime.now() - task_name = pk_list['app_name'] + '-' + pk_list['build_version'] + '-' + data_now.strftime('%Y_%m_%d_%H:%M:%S') + task_name = pk_list['app_name'] + '-' + pk_list['build_version'] + '-' + data_now.strftime( + '%Y_%m_%d_%H:%M:%S') device_info = get_data_from_device(device)[0] if not device_info: continue @@ -226,26 +246,32 @@ GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) - #data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - + # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + if options['option'] == 'immediate': date = data_now + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < data_now: - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), data_now.strftime('%Y-%m-%d %H:%M:%S'))))) - + if date < data_now: + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + data_now.strftime('%Y-%m-%d %H:%M:%S'))))) + new_schedule = {} description = " UPDATE create time: " + data_now.strftime('%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'upload_to_device' - new_schedule['args'] = [int(pk_list['id']), task_name, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password'], [device_info['type'],options['backup_db']], device] + new_schedule['args'] = [int(pk_list['id']), task_name, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password'], [device_info['type'], options['backup_db']], + device] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'update', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device) + add_job_into_database(task_name, 'update', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=device) sched = BackgroundScheduler() sched.add_job(upload_to_device, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) GLOBAL_TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - return + return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py (working copy) @@ -1,21 +1,25 @@ -from hive.imports.model import * +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from cm.lib.communication import send_https_rest_request +from cm.lib.parse_configfile import parse_specific_config, \ + change_config_file, INSERT, DELETE, add_vsite_to_config, remove_vsite_from_config from cm.lib.postgres_db import DB -from cm.lib.communication import send_https_rest_request, modify_url -from cm.lib.parse_configfile import DEFAULT_CONFIG_FILE_PATH, DEFAULT_VSITE_PATH, parse_specific_config, change_config_file, INSERT, DELETE, add_vsite_to_config, remove_vsite_from_config -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE from cm.models.device_mgmt.device import get_rest_info_from_device -import os -import json +from hive.imports.model import * +from hive.model.legacycli import MATCHALL from hive.utils import get_current_session -from django.db.models.query import QuerySet -__=_ +__ = _ + + def send_command_device(device_info, url, running_params): try: - rest_response_data = send_https_rest_request('POST', url, running_params, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', url, running_params, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: - #print "Connect device failed", str(e) + # print "Connect device failed", str(e) return 'failed' if rest_response_data['status'] == 200: contents = json.loads(rest_response_data['body'])['contents'] @@ -23,6 +27,8 @@ return contents else: return "failed" + + def vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete): device_name = asso_data['device_name'] vsite_name = asso_data['vs_name'] @@ -31,12 +37,15 @@ if not device_info: return "Please input a correct device_name" - #sw_str = 'sw %s\n' % vsite_name - #data = {'cmd': sw_str + cli_str} - data = {'cmd':cli_str, 'vsite_name':vsite_name} + # sw_str = 'sw %s\n' % vsite_name + # data = {'cmd': sw_str + cli_str} + data = {'cmd': cli_str, 'vsite_name': vsite_name} running_params = json.dumps(data) try: - rest_response_data = send_https_rest_request('POST', '/rest/ag/global/cli_extend', running_params, device_info[0]['ip_address'], device_info[0]['restapi_port'], device_info[0]['restapi_username'], device_info[0]['restapi_password']) + rest_response_data = send_https_rest_request('POST', '/rest/ag/global/cli_extend', running_params, + device_info[0]['ip_address'], device_info[0]['restapi_port'], + device_info[0]['restapi_username'], + device_info[0]['restapi_password']) except: return "Connect device failed" if rest_response_data['status'] == 200: @@ -51,6 +60,7 @@ return ret return None + def CM_check(new_url, running_params_list, parse_list, src_str): result_diff = {} for params in running_params_list: @@ -61,10 +71,13 @@ device_info = get_rest_info_from_device(device)[0] for i in range(0, len(params['params'])): try: - rest_response_data = send_https_rest_request('POST', new_url, params['params'][i], device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, params['params'][i], + device_info['ip_address'], device_info['restapi_port'], + device_info['restapi_username'], + device_info['restapi_password']) except Exception as e: error_flag = '1' - error_msg.update({error_flag:'unconnected'}) + error_msg.update({error_flag: 'unconnected'}) break ''' for i in range(0, len(src_str)): @@ -81,11 +94,11 @@ if type(contents) != list: error_flag = '3' - error_msg.update({error_flag:contents}) + error_msg.update({error_flag: contents}) break else: error_flag = '2' - error_msg.update({error_flag:'Get resource failed.'}) + error_msg.update({error_flag: 'Get resource failed.'}) break ''' for i in range(0, len(src_str)): @@ -95,8 +108,8 @@ continue ''' - #{u'resourcegroup': u'array', u'cmdid': u'vpn resource groupitem network', u'type': 1, u'netresource': u'10.8.2.50/255.255.255.0:0-65535'} - #{u'resourcegroup': u'array', u'cmdid': u'vpn resource groupexcludeditem network', u'type': 0, u'netresource': u'10.8.2.50/255.255.255.0:0-65535'} + # {u'resourcegroup': u'array', u'cmdid': u'vpn resource groupitem network', u'type': 1, u'netresource': u'10.8.2.50/255.255.255.0:0-65535'} + # {u'resourcegroup': u'array', u'cmdid': u'vpn resource groupexcludeditem network', u'type': 0, u'netresource': u'10.8.2.50/255.255.255.0:0-65535'} for item in contents: dest_str.append(parse_list[i] % item) @@ -104,18 +117,19 @@ for i in range(0, len(src_str)): if src_str[i] not in result_diff: result_diff[src_str[i]] = {} - result_diff[src_str[i]].update({device:error_msg[error_flag]}) + result_diff[src_str[i]].update({device: error_msg[error_flag]}) continue for i in range(0, len(src_str)): if src_str[i] not in result_diff: result_diff[src_str[i]] = {} if src_str[i] in dest_str: - result_diff[src_str[i]].update({device:True}) + result_diff[src_str[i]].update({device: True}) else: - result_diff[src_str[i]].update({device:False}) + result_diff[src_str[i]].update({device: False}) return result_diff + class VsiteConfig(ANModel): vs_name = CharField(verbose_name='Vsite Name', primary_key=True, length='1..64') device_name = CharField(verbose_name='Device Name', primary_key=True, length='1..32') @@ -124,176 +138,196 @@ site_type = CharField(verbose_name='Site Type', configurable=False, default='exclusive', length='1..32') description = TextField(verbose_name='Vsite Description', optional=True, length='1..128') parent_site = CharField(verbose_name='Parent Site', configurable=False, optional=True, default='', length='1..64') - + ssl_group = FieldGroup(verbose_name='SSL', writable=False, fields={ - 'country_code' : CharField(verbose_name='Country Code', length='1..2', optional=True), - 'Province' : CharField(verbose_name='State/Province', length='1..64', optional=True), - 'city' : CharField(verbose_name='City/Locality', length='1..64', optional=True), - 'organization' : CharField(verbose_name='Organization', length='1..64', optional=True), - 'unit' : CharField(verbose_name='Organizational Unit', length='1..64', optional=True), - 'email' : CharField(verbose_name='Administrator\'s Email', length='1..64', optional=True) - }) - + 'country_code': CharField(verbose_name='Country Code', length='1..2', optional=True), + 'Province': CharField(verbose_name='State/Province', length='1..64', optional=True), + 'city': CharField(verbose_name='City/Locality', length='1..64', optional=True), + 'organization': CharField(verbose_name='Organization', length='1..64', optional=True), + 'unit': CharField(verbose_name='Organizational Unit', length='1..64', optional=True), + 'email': CharField(verbose_name='Administrator\'s Email', length='1..64', optional=True) + }) + vpn_resource_group = FieldGroup(writable=False, verbose_name='VPN Resource Group', level=BASIC, fields={ - 'asso_vpn_resource_group' : AssoField2(verbose_name='Resource Group', - tgt='configuration.vsite_config.vpn.VPNResourceGroup.asso', mul='1', pos='left', optional=True), - }) + 'asso_vpn_resource_group': AssoField2(verbose_name='Resource Group', + tgt='configuration.vsite_config.vpn.VPNResourceGroup.asso', mul='1', + pos='left', optional=True), + }) vpn_network_item = FieldGroup(writable=False, verbose_name='VPN Network Resource Item', level=BASIC, fields={ - 'asso_vpn_network_item' : AssoField2(verbose_name='VPN Network Resource Item', - tgt='configuration.vsite_config.vpn.NetGroupItem.asso', mul='1', pos='left', optional=True), - }) - vpn_network_exclude_item = FieldGroup(writable=False, verbose_name='VPN Network Resource Exclude Item', level=BASIC, fields={ - 'asso_vpn_network_exclude_item' : AssoField2(verbose_name='VPN Network Exclude Item', - tgt='configuration.vsite_config.vpn.NetGroupExcludedItem.asso', mul='1', pos='left', optional=True), - }) + 'asso_vpn_network_item': AssoField2(verbose_name='VPN Network Resource Item', + tgt='configuration.vsite_config.vpn.NetGroupItem.asso', mul='1', pos='left', + optional=True), + }) + vpn_network_exclude_item = FieldGroup(writable=False, verbose_name='VPN Network Resource Exclude Item', level=BASIC, + fields={ + 'asso_vpn_network_exclude_item': AssoField2( + verbose_name='VPN Network Exclude Item', + tgt='configuration.vsite_config.vpn.NetGroupExcludedItem.asso', + mul='1', pos='left', optional=True), + }) vpn_netpool = FieldGroup(writable=False, verbose_name='VPN Netpool', level=BASIC, fields={ - 'asso_vpn_netpool' : AssoField2(verbose_name='VPN Netpool', - tgt='configuration.vsite_config.vpn.VPNNetpool.asso', mul='1', pos='left', optional=True), - }) + 'asso_vpn_netpool': AssoField2(verbose_name='VPN Netpool', + tgt='configuration.vsite_config.vpn.VPNNetpool.asso', mul='1', pos='left', + optional=True), + }) vpn_netpool_iprange = FieldGroup(writable=False, verbose_name='VPN Netpool Dynamic Iprange', level=BASIC, fields={ - 'asso_vpn_netpool_iprange' : AssoField2(verbose_name='VPN Netpool Dynamic IPrange', - tgt='configuration.vsite_config.vpn.IPRange.asso', mul='1', pos='left', optional=True), - }) + 'asso_vpn_netpool_iprange': AssoField2(verbose_name='VPN Netpool Dynamic IPrange', + tgt='configuration.vsite_config.vpn.IPRange.asso', mul='1', pos='left', + optional=True), + }) role_group = FieldGroup(writable=False, verbose_name='Role Group', level=BASIC, fields={ - 'asso_role_group' : AssoField2(verbose_name='Role Group', - tgt='configuration.vsite_config.role.RoleGroup.asso', mul='1', pos='left', optional=True), - }) + 'asso_role_group': AssoField2(verbose_name='Role Group', + tgt='configuration.vsite_config.role.RoleGroup.asso', mul='1', pos='left', + optional=True), + }) role_qualification = FieldGroup(writable=False, verbose_name="Role Qualification", level=BASIC, fields={ - 'asso_role_qualification' : AssoField2(verbose_name="Role Qualification", - tgt='configuration.vsite_config.role.RoleQualification.asso', mul='1', pos='left', optional=True), - }) + 'asso_role_qualification': AssoField2(verbose_name="Role Qualification", + tgt='configuration.vsite_config.role.RoleQualification.asso', mul='1', + pos='left', optional=True), + }) role_condition = FieldGroup(writable=False, verbose_name="Role Condition", level=BASIC, fields={ - 'asso_role_condition' : AssoField2(verbose_name="Role Condition", - tgt='configuration.vsite_config.role.RoleCondition.asso', mul='1', pos='left', optional=True), - }) + 'asso_role_condition': AssoField2(verbose_name="Role Condition", + tgt='configuration.vsite_config.role.RoleCondition.asso', mul='1', pos='left', + optional=True), + }) role_netpool_resource = FieldGroup(writable=False, verbose_name='Role Netpool Resource', level=BASIC, fields={ - 'asso_role_netpool_resource' : AssoField2(verbose_name='Role Netpool Resource', - tgt='configuration.vsite_config.role.RoleNetpoolResource.asso', mul='1', pos='left', optional=True), - }) + 'asso_role_netpool_resource': AssoField2(verbose_name='Role Netpool Resource', + tgt='configuration.vsite_config.role.RoleNetpoolResource.asso', + mul='1', pos='left', optional=True), + }) role_vpn_group_resource = FieldGroup(writable=False, verbose_name='Role VPN Group Resource', level=BASIC, fields={ - 'asso_role_vpn_group_resource' : AssoField2(verbose_name='Role Resource', - tgt='configuration.vsite_config.role.RoleVPNGroupResource.asso', mul='1', pos='left', optional=True), - }) + 'asso_role_vpn_group_resource': AssoField2(verbose_name='Role Resource', + tgt='configuration.vsite_config.role.RoleVPNGroupResource.asso', + mul='1', pos='left', optional=True), + }) acl_group = FieldGroup(writable=False, verbose_name='ACL', level=BASIC, fields={ - 'asso_acl_group' : AssoField2(verbose_name='ACL Group', - tgt='configuration.vsite_config.acl.ACLGroup.asso', mul='1', pos='left', optional=True), + 'asso_acl_group': AssoField2(verbose_name='ACL Group', + tgt='configuration.vsite_config.acl.ACLGroup.asso', mul='1', pos='left', + optional=True), # 'asso_acl_resource' : AssoField2(verbose_name='ACL Resource', # tgt='configuration.vsite_config.acl.ACLResource.asso', mul='1', pos='left', optional=True), # 'asso_acl_rule' : AssoField2(verbose_name='ACL Rule', # tgt='configuration.vsite_config.acl.ACLRule.asso', mul='1', pos='left', optional=True), - }) + }) acl_resource = FieldGroup(writable=False, verbose_name='ACL Resource', level=BASIC, fields={ - 'asso_acl_resource' : AssoField2(verbose_name='ACL Resource', - tgt='configuration.vsite_config.acl.ACLResource.asso', mul='1', pos='left', optional=True), - }) + 'asso_acl_resource': AssoField2(verbose_name='ACL Resource', + tgt='configuration.vsite_config.acl.ACLResource.asso', mul='1', pos='left', + optional=True), + }) acl_rule = FieldGroup(writable=False, verbose_name='ACL Rule', level=BASIC, fields={ - 'asso_acl_rule' : AssoField2(verbose_name='ACL Rule', - tgt='configuration.vsite_config.acl.ACLRule.asso', mul='1', pos='left', optional=True), - }) + 'asso_acl_rule': AssoField2(verbose_name='ACL Rule', + tgt='configuration.vsite_config.acl.ACLRule.asso', mul='1', pos='left', + optional=True), + }) class Meta: verbose_name = 'Vsite config' - list_config_options={'columns':[ - {'name':'vs_name'}, - {'name':'device_name'}, - {'name':'site_FQDM'}, - {'name':'ip'}, - {'name':'site_type'}, - {'name':'description'}, - {'name':'parent_site'}, + list_config_options = {'columns': [ + {'name': 'vs_name'}, + {'name': 'device_name'}, + {'name': 'site_FQDM'}, + {'name': 'ip'}, + {'name': 'site_type'}, + {'name': 'description'}, + {'name': 'parent_site'}, ]} + class VPN_Network_Group_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_vpn_resource_group' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) - class VPN_Network_Include_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_vpn_network_item' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_vpn_resource_group' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + + class VPN_Network_Include_Sync(Action): + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_vpn_network_item' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class VPN_Network_Exclude_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_vpn_network_exclude_item' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_vpn_network_exclude_item' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class VPN_Netpool_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_vpn_netpool' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_vpn_netpool' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class VPN_Netpool_IPRange_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_vpn_netpool_iprange' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_vpn_netpool_iprange' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) class ACL_Group_Sync(Action): verbose_name = _('Sync to Device') @@ -310,9 +344,10 @@ ('insert', 'Insert'), ('delete', 'Delete'), )), DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), + values=['__field:cm.device_mgmt.device.Device.name']), CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), ) + class ACL_Resource_Sync(Action): verbose_name = _('Sync to Device') action_name = _('Sync') @@ -328,128 +363,135 @@ ('insert', 'Insert'), ('delete', 'Delete'), )), DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), + values=['__field:cm.device_mgmt.device.Device.name']), CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + ) + class ACL_Rule_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_acl_rule' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_acl_rule' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) class Role_Group_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_role_group' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) - class Role_Qualification_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_role_qualification' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_role_group' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + + class Role_Qualification_Sync(Action): + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_role_qualification' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class Role_Condition_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_role_condition' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_role_condition' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class Role_Netpool_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_role_netpool_resource' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_role_netpool_resource' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class Role_VPN_Sync(Action): - verbose_name = _('Sync to Device') - action_name = _('Sync') - btn_class = 'btn-danger slow-action' - alert_msg = "" - process_title = "" - related = 'asso_role_vpn_group_resource' - forever = False - config_change = True - instance_mul = '1..*' - option_fields = ( - EnumField(verbose_name='Option', default='insert', name='option', values=( - ('insert', 'Insert'), ('delete', 'Delete'), - )), - DynamicMultiEnumField(name='device', verbose_name='devices', - values=['__field:cm.device_mgmt.device.Device.name']), - CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), - ) + verbose_name = _('Sync to Device') + action_name = _('Sync') + btn_class = 'btn-danger slow-action' + alert_msg = "" + process_title = "" + related = 'asso_role_vpn_group_resource' + forever = False + config_change = True + instance_mul = '1..*' + option_fields = ( + EnumField(verbose_name='Option', default='insert', name='option', values=( + ('insert', 'Insert'), ('delete', 'Delete'), + )), + DynamicMultiEnumField(name='device', verbose_name='devices', + values=['__field:cm.device_mgmt.device.Device.name']), + CharField(name='vsite_name', verbose_name=_('Vsite Name'), length='1..32'), + ) + class Refresh(Action): verbose_name = _('Refresh') action_name = _('Refresh') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False config_change = True instance_mul = '1..*' + class Manager(CLIManager): def _get_query_set(self): db = DB.get_connected_db() @@ -458,7 +500,7 @@ db.close() key = ['vs_name', 'device_name', 'site_FQDM', 'ip', 'site_type', 'description', 'parent_site'] result = [dict(zip(key, each)) for each in data] - + for each in result: each['country_code'] = 'CN' each['Province'] = 'default' @@ -467,15 +509,17 @@ each['unit'] = 'default' each['email'] = 'default' self._model._meta.mark_delay_query(each) - + return QuerySet(self._model, result) + def _perform_VPN_Network_Group_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'vpn', 'VPNResourceGroup']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: @@ -496,17 +540,22 @@ specific_line = 'vpn resource group "%s"' % instance.name insert_or_delete = DELETE - #get vpn groupitem network + # get vpn groupitem network reg_str = 'vpn resource groupitem network "%s" "(?P.*?)" (?P[0-9]+)' % instance.name - rtn_item = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) + rtn_item = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, + MATCHALL) for item in rtn_item[0]: - cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.name, item['net_resource'], int(item['net_type'])) + cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.name, item['net_resource'], + int(item['net_type'])) cli_list.append(cli_str) - #get vpn groupexcludeditem network + # get vpn groupexcludeditem network reg_str = 'vpn resource groupexcludeditem network "%s" "(?P.*?)" (?P[0-9]+)' % instance.name - rtn_excluded = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) + rtn_excluded = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, + MATCHALL) for item in rtn_excluded[0]: - cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.name, item['net_resource'], int(item['net_type'])) + cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.name, + item['net_resource'], + int(item['net_type'])) cli_list.append(cli_str) asso_data = {} @@ -518,30 +567,37 @@ result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_VPN_Network_Include_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'vpn', 'NetGroupItem']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: for item in instance_result: - if item.vpn_group_name == asso_pk['vpn_group_name'] and item.net_resource == asso_pk['net_resource']: + if item.vpn_group_name == asso_pk['vpn_group_name'] and item.net_resource == asso_pk[ + 'net_resource']: instance_list.append(item) result = '' for instance in instance_list: if options['option'] == 'insert': - cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, instance.net_type) specific_line = '#VPN configuration\n' insert_or_delete = INSERT else: - cli_str = 'no vpn resource groupitem network "%s" "%s"' % (instance.vpn_group_name, instance.net_resource) - specific_line = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + cli_str = 'no vpn resource groupitem network "%s" "%s"' % (instance.vpn_group_name, + instance.net_resource) + specific_line = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, + instance.net_type) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -549,30 +605,38 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_VPN_Network_Exclude_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'vpn', 'NetGroupExcludedItem']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: for item in instance_result: - if item.vpn_group_name == asso_pk['vpn_group_name'] and item.net_resource == asso_pk['net_resource']: + if item.vpn_group_name == asso_pk['vpn_group_name'] and item.net_resource == asso_pk[ + 'net_resource']: instance_list.append(item) result = '' for instance in instance_list: if options['option'] == 'insert': - cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, + instance.net_type) specific_line = '#VPN configuration\n' insert_or_delete = INSERT else: - cli_str = 'no vpn resource groupexcludeditem network "%s" "%s"' % (instance.vpn_group_name, instance.net_resource) - specific_line = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + cli_str = 'no vpn resource groupexcludeditem network "%s" "%s"' % (instance.vpn_group_name, + instance.net_resource) + specific_line = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, + instance.net_type) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -580,13 +644,15 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_VPN_Netpool_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'vpn', 'VPNNetpool']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: @@ -606,12 +672,14 @@ cli_list.append(cli_str) specific_line = 'vpn netpool name "%s"' % instance.name insert_or_delete = DELETE - - #get netpool iprange + + # get netpool iprange reg_str = 'vpn netpool iprange dynamic "%s" (?P[0-9|\.|:|\w]+) (?P[0-9|\.|:|\w]+) "(?P.*?)"' % instance.name - rtn_iprange = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) + rtn_iprange = parse_specific_config(pk_list, '#VPN configuration\n', '#aaa configuration\n', reg_str, + MATCHALL) for item in rtn_iprange[0]: - cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.name, item['first_ip'], item['last_ip'], item['unit_name']) + cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.name, item['first_ip'], + item['last_ip'], item['unit_name']) cli_list.append(cli_str) asso_data = {} @@ -622,30 +690,42 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_VPN_Netpool_IPRange_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'vpn', 'IPRange']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: for item in instance_result: - if item.netpool_name == asso_pk['netpool_name'] and item.first_ip == asso_pk['first_ip'] and item.last_ip == asso_pk['last_ip']: + if item.netpool_name == asso_pk['netpool_name'] and item.first_ip == asso_pk[ + 'first_ip'] and item.last_ip == asso_pk['last_ip']: instance_list.append(item) result = '' for instance in instance_list: if options['option'] == 'insert': - cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) + cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], + instance.unit_name) specific_line = '#VPN configuration\n' insert_or_delete = INSERT else: - cli_str = 'no vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) - specific_line = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) + cli_str = 'no vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], + instance.unit_name) + specific_line = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], + instance.unit_name) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -661,7 +741,8 @@ sess = get_current_session() acl_group_model = get_model('cm', ['configuration', 'vsite_config', 'acl', 'ACLGroup']) acl_group_manager = acl_group_model.get_manager(sess) - acl_group_instance = acl_group_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + acl_group_instance = acl_group_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) acl_group_list = [] for asso_pk in asso_pk_list: @@ -678,7 +759,7 @@ cli_str = 'no acl resourcegroup %s' % item.name specific_line = 'acl resourcegroup %s "%s" "%s"' % (item.group_type, item.name, item.description) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -686,13 +767,15 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += ret if ret != None else '' return result + def _perform_ACL_Resource_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() acl_resource_model = get_model('cm', ['configuration', 'vsite_config', 'acl', 'ACLResource']) acl_resource_manager = acl_resource_model.get_manager(sess) - acl_resource_instance = acl_resource_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + acl_resource_instance = acl_resource_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) acl_resource_list = [] for asso_pk in asso_pk_list: @@ -709,7 +792,7 @@ cli_str = 'no acl resource "%s" "%s"' % (item.group_name, item.resource) specific_line = 'acl resource "%s" "%s"' % (item.group_name, item.resource) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -717,30 +800,38 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_ACL_Rule_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() acl_resource_model = get_model('cm', ['configuration', 'vsite_config', 'acl', 'ACLRule']) acl_resource_manager = acl_resource_model.get_manager(sess) - acl_resource_instance = acl_resource_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + acl_resource_instance = acl_resource_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) acl_resource_list = [] for asso_pk in asso_pk_list: for item in acl_resource_instance: - if item.target_name == asso_pk['target_name'] and item.group_name == asso_pk['group_name'] and item.target_type == asso_pk['target_type']: + if item.target_name == asso_pk['target_name'] and item.group_name == asso_pk[ + 'group_name'] and item.target_type == asso_pk['target_type']: acl_resource_list.append(item) result = '' for instance in acl_resource_list: if options['option'] == 'insert': - cli_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, instance.action, instance.priority, instance.target_type) + cli_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, + instance.action, instance.priority, + instance.target_type) specific_line = '#acl configuration\n' insert_or_delete = INSERT else: - cli_str = 'no acl rule "%s" "%s" "%s"' % (instance.target_name, instance.group_name, instance.target_type) - specific_line = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, instance.action, instance.priority, instance.target_type) + cli_str = 'no acl rule "%s" "%s" "%s"' % (instance.target_name, instance.group_name, + instance.target_type) + specific_line = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, + instance.action, instance.priority, + instance.target_type) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -755,7 +846,8 @@ sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'RoleGroup']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: @@ -770,9 +862,10 @@ insert_or_delete = INSERT else: cli_str = 'no role name "%s"' % (instance.name) - specific_line = 'role name "%s" "%s" %d' % (instance.name, instance.description, int(instance.priority)) + specific_line = 'role name "%s" "%s" %d' % (instance.name, instance.description, + int(instance.priority)) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -780,13 +873,15 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_Role_Qualification_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'RoleQualification']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: @@ -797,19 +892,22 @@ for instance in instance_list: cli_list = [] if options['option'] == 'insert': - cli_str = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, instance.description) + cli_str = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, + instance.description) cli_list.append(cli_str) specific_line = '#role configuration\n' insert_or_delete = INSERT else: cli_str = 'no role qualification "%s" "%s"' % (instance.role_name, instance.name) cli_list.append(cli_str) - specific_line = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, instance.description) + specific_line = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, + instance.description) insert_or_delete = DELETE - - #get role condition + + # get role condition reg_str = 'role condition "%s" "%s" "(?P.*?)"' % (instance.role_name, instance.name) - rtn_condition = parse_specific_config(pk_list, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) + rtn_condition = parse_specific_config(pk_list, '#role configuration\n', '#acl configuration\n', reg_str, + MATCHALL) for item in rtn_condition[0]: cli_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.name, item['content']) cli_list.append(cli_str) @@ -822,30 +920,36 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_Role_Condition_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'RoleCondition']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: for item in instance_result: - if item.role_name == asso_pk['role_name'] and item.qualification_name == asso_pk['qualification_name'] and item.content == asso_pk['content']: + if item.role_name == asso_pk['role_name'] and item.qualification_name == asso_pk[ + 'qualification_name'] and item.content == asso_pk['content']: instance_list.append(item) result = '' for instance in instance_list: if options['option'] == 'insert': - cli_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) + cli_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) specific_line = '#role configuration\n' insert_or_delete = INSERT else: - cli_str = 'no role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) - specific_line = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) + cli_str = 'no role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) + specific_line = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -853,13 +957,15 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_Role_Netpool_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() instance_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'RoleNetpoolResource']) instance_manager = instance_model.get_manager(sess) - instance_result = instance_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + instance_result = instance_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) instance_list = [] for asso_pk in asso_pk_list: @@ -876,7 +982,7 @@ cli_str = 'no role resource netpool "%s" "%s"' % (instance.role_name, instance.netpool_name) specific_line = 'role resource netpool "%s" "%s"' % (instance.role_name, instance.netpool_name) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -884,13 +990,15 @@ ret = vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete) result += (ret + cli_str + '\n') if ret != None else '' return result + def _perform_Role_VPN_Sync(self, options): pk_list = options['__pk_list'][0] asso_pk_list = options['__asso_pk_list'] sess = get_current_session() acl_resource_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'RoleVPNGroupResource']) acl_resource_manager = acl_resource_model.get_manager(sess) - acl_resource_instance = acl_resource_manager.filter({"asso":[{'device_name': pk_list['device_name'], 'vs_name':pk_list['vs_name']}]}) + acl_resource_instance = acl_resource_manager.filter( + {"asso": [{'device_name': pk_list['device_name'], 'vs_name': pk_list['vs_name']}]}) acl_resource_list = [] for asso_pk in asso_pk_list: @@ -904,10 +1012,12 @@ specific_line = '#role configuration\n' insert_or_delete = INSERT else: - cli_str = 'no role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, instance.vpn_group_name) - specific_line = 'role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, instance.vpn_group_name) + cli_str = 'no role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, + instance.vpn_group_name) + specific_line = 'role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, + instance.vpn_group_name) insert_or_delete = DELETE - + asso_data = {} for device in options['device']: asso_data['device_name'] = device @@ -925,8 +1035,8 @@ device_info = get_rest_info_from_device(device_name)[0] cli_str = "no virtual site name '%s' \n YES" % vs_name - #print "ssl csr", cli_str - data = {'cmd':cli_str} + # print "ssl csr", cli_str + data = {'cmd': cli_str} running_params = json.dumps(data) ret = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) @@ -935,35 +1045,39 @@ db = DB.get_connected_db() db.execute_sql(delete_sql) db.close() - - #remove vsite from config file + + # remove vsite from config file ret = remove_vsite_from_config(instance.vs_name, instance.device_name) return ret + def _insert(self, instance): def insert_failed(vs_name, device_info): cli_str = "no virtual site name '%s' \n YES" % vs_name - data = {'cmd':cli_str} + data = {'cmd': cli_str} running_params = json.dumps(data) ret = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) return ret + device_info = get_rest_info_from_device(instance.device_name)[0] if not device_info: raise ModelQueryException(CLICmdError(__("Device Name is not existed!"))) - cli_str = "virtual site name '%s' '%s' '%s' '%s' \n" % (instance.vs_name, instance.description, instance.site_type, instance.parent_site) - cli_str += "virtual site ip '%s' %s %d \n" % (instance.vs_name, instance.ip.split(':')[0], int(instance.ip.split(':')[1])) + cli_str = "virtual site name '%s' '%s' '%s' '%s' \n" % (instance.vs_name, instance.description, + instance.site_type, instance.parent_site) + cli_str += "virtual site ip '%s' %s %d \n" % (instance.vs_name, instance.ip.split(':')[0], + int(instance.ip.split(':')[1])) cli_str += "virtual site domain '%s' '%s' \n" % (instance.vs_name, instance.site_FQDM) try: - sync_config = json.dumps({'config':cli_str}) + sync_config = json.dumps({'config': cli_str}) except Exception as e: return 'Can not parse the virtual site info to json.' - #Add vsite for device + # Add vsite for device ret = send_command_device(device_info, '/rest/ag/global/batch_cli', sync_config) if ret == 'field': return 'Connect device failed.' - #This ip-port is already configured by a virtual site - #There is already virtual site \"rest_test\". - #This domain name already exists + # This ip-port is already configured by a virtual site + # There is already virtual site \"rest_test\". + # This domain name already exists if ret.find('There is already virtual site') != -1: raise ModelQueryException(CLICmdError(__(ret))) if ret.find('This ip-port is already configured by a virtual site') != -1: @@ -973,33 +1087,36 @@ insert_failed(instance.vs_name, device_info) raise ModelQueryException(CLICmdError(__(ret))) - #create vsite ssl csr + # create vsite ssl csr cli_str = "ssl csr 2048 sha256 \nYES\n" - cli_str += "%s\n%s\n%s\n%s\n%s\nY\n%s\nNO\n" % (instance.country_code, instance.Province, instance.city, instance.organization, instance.unit, instance.email) - data = {'cmd':cli_str, 'vsite_name':instance.vs_name} + cli_str += "%s\n%s\n%s\n%s\n%s\nY\n%s\nNO\n" % (instance.country_code, instance.Province, instance.city, + instance.organization, instance.unit, instance.email) + data = {'cmd': cli_str, 'vsite_name': instance.vs_name} running_params = json.dumps(data) - + ret = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) if ret == 'field': insert_failed(instance.vs_name, device_info) raise ModelQueryException(CLICmdError(__('Connect device failed.'))) - #start ssl + # start ssl cli_str = 'ssl start\n' - data = {'cmd':cli_str, 'vsite_name':instance.vs_name} + data = {'cmd': cli_str, 'vsite_name': instance.vs_name} running_params = json.dumps(data) ret = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) if ret == 'field': insert_failed(instance.vs_name, device_info) raise ModelQueryException(CLICmdError(__('Connect device failed.'))) - #Error while trying to start ssl + # Error while trying to start ssl if ret.find('Error while trying to start ssl') != -1: insert_failed(instance.vs_name, device_info) raise ModelQueryException(CLICmdError(__(ret))) - #insert to database and get site config from device + # insert to database and get site config from device db = DB.get_connected_db() - save_sql = "INSERT INTO vsite_list(vs_name, device_name, site_FQDM, ip, site_type, description, parent_site) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (instance.vs_name, instance.device_name, instance.site_FQDM, instance.ip, instance.site_type, instance.description, instance.parent_site) + save_sql = "INSERT INTO vsite_list(vs_name, device_name, site_FQDM, ip, site_type, description, parent_site) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( + instance.vs_name, instance.device_name, instance.site_FQDM, instance.ip, instance.site_type, + instance.description, instance.parent_site) db.execute_sql(save_sql) db.close() @@ -1011,7 +1128,7 @@ raise ModelQueryException(CLICmdError(__('open vsite file failed!'))) cli_str = 'show running\n' - data = {'cmd':cli_str, 'vsite_name':instance.vs_name} + data = {'cmd': cli_str, 'vsite_name': instance.vs_name} running_params = json.dumps(data) content = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) if content == 'field': @@ -1022,20 +1139,21 @@ vsite_file.close() result = add_vsite_to_config(instance.vs_name, instance.device_name) return ret + # def _update_ip(self, instance): # ip = instance.ip.split(':')[0] # port = instance.ip.split(':')[1] # cli_str = 'virtual site ip ' def _perform_Refresh(self, options): - #configuration refresh {u'__pk_list': [{u'vs_name': u'bug', u'device_name': u'liyi_2017'}]} + # configuration refresh {u'__pk_list': [{u'vs_name': u'bug', u'device_name': u'liyi_2017'}]} pk_list = options['__pk_list'][0] vs_name = pk_list['vs_name'] device_name = pk_list['device_name'] file_path = DEFAULT_VSITE_PATH + device_name + '-' + vs_name device_info = get_rest_info_from_device(device_name)[0] - data = {'cmd' : 'show running', 'vsite_name' : vs_name} + data = {'cmd': 'show running', 'vsite_name': vs_name} running_params = json.dumps(data) ret = send_command_device(device_info, '/rest/ag/global/cli_extend', running_params) @@ -1047,6 +1165,3 @@ vs_file.write(ret) vs_file.close() return - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py (working copy) @@ -1,45 +1,52 @@ -from hive.imports.model import * +import datetime + +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + from cm.lib.parse_configfile import INSERT, DELETE -from cm.models.configuration.vsite_config import vs_insert_delete, CM_check -from cm.lib.parse_configfile import parse_specific_config, DEFAULT_CONFIG_FILE_PATH, delete_specific_info_from_config, sync_vsite_to_config -from hive.model.legacycli import MATCHALL, MATCHONE -import os, datetime, difflib, time +from cm.lib.parse_configfile import parse_specific_config, delete_specific_info_from_config, \ + sync_vsite_to_config from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, update_cm_role_devicelist +from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ + update_cm_role_devicelist +from cm.models.configuration.vsite_config import vs_insert_delete, CM_check from cm.models.tasking import GLOBAL_TASK -from apscheduler.schedulers.background import BackgroundScheduler -from django.db.models.query import QuerySet -import json -__=_ +from hive.imports.model import * +from hive.model.legacycli import MATCHALL +__ = _ + class ACLGroup(ANModel): asso = AssoField2(verbose_name="Vsite Device", tgt='configuration.vsite_config.VsiteConfig.asso_acl_group', - mul='*', pos='right', optional=True) + mul='*', pos='right', optional=True) group_type = CharField(verbose_name='ACL Group Type', length='1..32', primary_key=True) name = CharField(verbose_name='ACL Group Name', length='1..32', primary_key=True) description = TextField(verbose_name='ACL Group Description', length='1..1024', optional=True) - + class Meta: verbose_name = 'ACL Group' - list_config_options={'columns':[ - {'name':'group_type'}, - {'name':'name'}, - {'name':'description'}, + list_config_options = {'columns': [ + {'name': 'group_type'}, + {'name': 'name'}, + {'name': 'description'}, ]} + class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #acl resourcegroup fileshare "aaa_fileshare" "" + # acl resourcegroup fileshare "aaa_fileshare" "" reg_str = 'acl resourcegroup (?P.*?) "(?P.*?)" "(?P.*?)"' - rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, MATCHALL) + rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, + MATCHALL) if type(rtn) == str: raise ModelQueryException(CLICmdError(__(rtn))) for item in rtn[0]: item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): cli_str = 'acl resourcegroup %s "%s" "%s"' % (instance.group_type, instance.name, instance.description) ret = vs_insert_delete(instance.asso[0], cli_str, '#acl configuration\n', INSERT) @@ -54,37 +61,41 @@ if ret: raise ModelQueryException(CLICmdError(__(ret))) - #delete all of the ACL group + # delete all of the ACL group vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + instance.asso[0]['device_name'] + '-' + instance.asso[0]['vs_name'] filter_str = ['acl resource "%s".*?' % instance.name, - 'acl rule .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#acl configuration\n', '#url policy configuration\n', filter_str) + 'acl rule .* "%s".*?' % instance.name] + ret = delete_specific_info_from_config(vs_file_path, '#acl configuration\n', '#url policy configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) - + sync_vsite_to_config(instance.asso[0]['vs_name'], instance.asso[0]['device_name']) mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'VsiteConfig'])) return + class ACLResource(ANModel): asso = AssoField2(verbose_name="Vsite Device", tgt='configuration.vsite_config.VsiteConfig.asso_acl_resource', - mul='*', pos='right', optional=True) + mul='*', pos='right', optional=True) group_name = CharField(verbose_name='ACL Group Name', length='1..32', primary_key=True) resource = CharField(verbose_name='ACL Resource', length='1..128', primary_key=True) class Meta: verbose_name = 'ACL Resource' - list_config_options={'columns':[ - {'name':'group_name'}, - {'name':'resource'}, + list_config_options = {'columns': [ + {'name': 'group_name'}, + {'name': 'resource'}, ]} + class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #acl resource "aaa_fileshare" "\\\\10.10.10.1\\directory" + # acl resource "aaa_fileshare" "\\\\10.10.10.1\\directory" reg_str = 'acl resource "(?P.*?)" "(?P.*?)"' - rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, MATCHALL) + rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, + MATCHALL) if type(rtn) == str: raise ModelQueryException(CLICmdError(__(rtn))) for item in rtn[0]: @@ -92,14 +103,16 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): cli_str = 'acl resource "%s" "%s"' % (instance.group_name, instance.resource) ret = vs_insert_delete(instance.asso[0], cli_str, '#acl configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no acl resource aaa_fileshare "\\10.10.10.1\directory" + # no acl resource aaa_fileshare "\\10.10.10.1\directory" specific_str = 'acl resource "%s" "%s"' % (instance.group_name, instance.resource) cli_str = 'no acl resource "%s" "%s"' % (instance.group_name, instance.resource) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) @@ -107,62 +120,75 @@ raise ModelQueryException(CLICmdError(__(ret))) return + class ACLRule(ANModel): asso = AssoField2(verbose_name="Vsite Device", tgt='configuration.vsite_config.VsiteConfig.asso_acl_rule', - mul='*', pos='right', optional=True) - target_name = CharField(verbose_name='Target Name', length='1..32', primary_key=True, help_text='[User Name | Group Name | Role Name]') + mul='*', pos='right', optional=True) + target_name = CharField(verbose_name='Target Name', length='1..32', primary_key=True, + help_text='[User Name | Group Name | Role Name]') group_name = CharField(verbose_name='ACL Group Name', length='1..32', primary_key=True) action = CharField(verbose_name='Action', length='1..16', help_text='[permit | deny]') priority = Int32Field(verbose_name=_('Acl Priority'), scope='1..1000') - target_type = CharField(verbose_name='Target Type', length='1..8', default='R', primary_key=True, help_text='[R(role) | U(user) | G(group)]') + target_type = CharField(verbose_name='Target Type', length='1..8', default='R', primary_key=True, + help_text='[R(role) | U(user) | G(group)]') class Meta: verbose_name = 'ACL Rule' - list_config_options={'columns':[ - {'name':'target_name'}, - {'name':'group_name'}, - {'name':'action'}, - {'name':'priority'}, - {'name':'target_type'}, + list_config_options = {'columns': [ + {'name': 'target_name'}, + {'name': 'group_name'}, + {'name': 'action'}, + {'name': 'priority'}, + {'name': 'target_type'}, ]} + class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #acl rule "ddd" "bbb_fileshare" "PERMIT" 1000 "R" + # acl rule "ddd" "bbb_fileshare" "PERMIT" 1000 "R" reg_str = 'acl rule "(?P.*?)" "(?P.*?)" "(?P.*?)" (?P[0-9]+) "(?P.*?)"' - rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, MATCHALL) + rtn = parse_specific_config(asso_data, '#acl configuration\n', '#url policy configuration\n', reg_str, + MATCHALL) if type(rtn) == str: raise ModelQueryException(CLICmdError(__(rtn))) for item in rtn[0]: item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) - return QuerySet(self._model, rtn[0]) + return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - cli_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, instance.action, instance.priority, instance.target_type) + cli_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, instance.action, + instance.priority, instance.target_type) ret = vs_insert_delete(instance.asso[0], cli_str, '#acl configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no acl rule bbb bbb_fileshare R - specific_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, instance.action, int(instance.priority), instance.target_type) + # no acl rule bbb bbb_fileshare R + specific_str = 'acl rule "%s" "%s" "%s" %d "%s"' % (instance.target_name, instance.group_name, + instance.action, int(instance.priority), + instance.target_type) cli_str = 'no acl rule "%s" "%s" "%s"' % (instance.target_name, instance.group_name, instance.target_type) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + class CMACLResource(ANModel): id = Int32Field(primary_key=True, configurable=False) group_name = CharField(verbose_name='ACL Group Name', length='1..64') resource = CharField(verbose_name='ACL Resource', length='1..128') description = CharField(verbose_name='description', length='1..128', editable=True) - status = Int32Field(verbose_name='Status', default=0, configurable=False, optional=True) #0-new 1-success 2-failed + status = Int32Field(verbose_name='Status', default=0, configurable=False, + optional=True) # 0-new 1-success 2-failed time = CharField(verbose_name='Modify time', length='1..64', configurable=False, optional=True) device_list = CharField(verbose_name='Device List', length='1..1024', configurable=False, optional=True) class Meta: verbose_name = 'ACL Resource' + class Sync(Action): verbose_name = _('Sync') action_name = _('Sync') @@ -173,15 +199,18 @@ option_fields = ( # DynamicMultiEnumField(name='device', verbose_name='devices', # values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']), - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), EnumField(verbose_name='Action', default='insert', name='action', values=( ('insert', 'Insert'), ('delete', 'Delete'), )), ) + class Check(Action): verbose_name = _('Check') action_name = _('Check') @@ -190,8 +219,10 @@ forever = False instance_mul = '1..*' option_fields = ( - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), ) + class Init_DeviceList(Action): verbose_name = 'Init Device List' action_name = 'Init_DeviceList' @@ -199,9 +230,9 @@ forever = False instance_mul = '1' option_fields = ( - CharField(verbose_name='Option', name='option'), - CharField(verbose_name='Device List', name='device_list') - ) + CharField(verbose_name='Option', name='option'), + CharField(verbose_name='Device List', name='device_list') + ) class Manager(CLIManager): def _get_query_set(self): @@ -211,11 +242,13 @@ db.close() key = ['id', 'group_name', 'description', 'status', 'time', 'device_list', 'resource'] result = [dict(zip(key, each)) for each in data] - return QuerySet(self._model, result) + return QuerySet(self._model, result) def _insert(self, instance): db = DB.get_connected_db() - save_sql = "INSERT INTO cm_acl_resource(group_name, description, time, resource) values('%s', '%s', '%s', '%s')" % (instance.group_name, instance.description, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), instance.resource) + save_sql = "INSERT INTO cm_acl_resource(group_name, description, time, resource) values('%s', '%s', '%s', '%s')" % ( + instance.group_name, instance.description, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + instance.resource) db.execute_sql(save_sql) db.close() return @@ -231,10 +264,11 @@ description = instance.description time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - update_sql = "UPDATE cm_acl_resource set time='%s', description='%s' where id=%d" % (time, description, instance.id) + update_sql = "UPDATE cm_acl_resource set time='%s', description='%s' where id=%d" % (time, description, + instance.id) db.execute_sql(update_sql) db.close() - return + return def _perform_Sync(self, options): sess = get_current_session() @@ -244,7 +278,7 @@ service_list = options['service'] db = DB.get_connected_db() for service in service_list: - #select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service + # select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service select_sql = "with raw as (select service.name, service.service_name, device.name as dname from service left join ds on service.id=ds.s_id left join device on device.id=ds.d_id where service.name='%s') select array_agg(dname), raw.service_name from raw group by raw.name, raw.service_name;" % service data = db.fetchall(select_sql) key = ['device_list', 'service_name'] @@ -253,38 +287,41 @@ for pk in options['__pk_list']: time.sleep(1) - cm_manager = cm_acl_model.get_manager(sess).filter({'id':pk['id']}) + cm_manager = cm_acl_model.get_manager(sess).filter({'id': pk['id']}) group_name = cm_manager[0].group_name resource = cm_manager[0].resource - #pk_list = options['__pk_list'][0] + # pk_list = options['__pk_list'][0] new_url = '/rest/ag/global/batch_cli' running_params = [] - for device in device_list: + for device in device_list: if options['action'] == 'insert': cli_str = 'sw %s\n acl resource "%s" "%s"\n' % (device[0]['service_name'], group_name, resource) elif options['action'] == 'delete': - cli_str = 'sw %s\n no acl resource "%s" "%s"\n' % (device[0]['service_name'], group_name, resource) - #running_params = json.dumps({'config':cli_str}) - running_params.append({'params':json.dumps({'config':cli_str}), 'device_list':device[0]['device_list']}) + cli_str = 'sw %s\n no acl resource "%s" "%s"\n' % (device[0]['service_name'], group_name, + resource) + # running_params = json.dumps({'config':cli_str}) + running_params.append( + {'params': json.dumps({'config': cli_str}), 'device_list': device[0]['device_list']}) task_name = '%s-%s' % (cm_manager[0].description, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) - if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) - #data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + if date < datetime.datetime.now(): + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - pk_dict = {'table':'cm_acl_resource', 'id':pk['id']} + pk_dict = {'table': 'cm_acl_resource', 'id': pk['id']} new_schedule = {} description = " %s ACL[%s] to services[%s]\n" % (options['action'], resource, ','.join(service_list)) description += " Sync create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' @@ -292,8 +329,10 @@ new_schedule['args'] = [new_url, running_params, pk_dict, options['action'], task_name] new_schedule_str = json.dumps(new_schedule) - #print new_schedule_str - add_job_into_database(task_name, 'sync_acl', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=','.join(service_list)) + # print new_schedule_str + add_job_into_database(task_name, 'sync_acl', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=','.join(service_list)) sched = BackgroundScheduler() sched.add_job(sync_to_device, 'date', next_run_time=date, args=new_schedule['args']) GLOBAL_TASK.add(task_name, sched) @@ -306,7 +345,7 @@ service_list = options['service'] db = DB.get_connected_db() for service in service_list: - #select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service + # select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service select_sql = "with raw as (select service.name, service.service_name, device.name as dname from service left join ds on service.id=ds.s_id left join device on device.id=ds.d_id where service.name='%s') select array_agg(dname), raw.service_name from raw group by raw.name, raw.service_name;" % service data = db.fetchall(select_sql) key = ['device_list', 'service_name'] @@ -319,24 +358,26 @@ pk_list = options['__pk_list'] for pk in pk_list: - cm_manager = cm_acl_model.get_manager(sess).filter({'id':pk['id']}) + cm_manager = cm_acl_model.get_manager(sess).filter({'id': pk['id']}) src_str.append('acl resource "%s" "%s"\n' % (cm_manager[0].group_name, cm_manager[0].resource)) - + result_diff = "" running_params = [] - for device in device_list: - params = json.dumps({"cmd":"show acl resource", "vsite_name":"%s" % (device[0]['service_name'])}) - running_params.append({'params':[params,], 'device_list':device[0]['device_list']}) + for device in device_list: + params = json.dumps({"cmd": "show acl resource", "vsite_name": "%s" % (device[0]['service_name'])}) + running_params.append({'params': [params, ], 'device_list': device[0]['device_list']}) new_url = '/rest/ag/global/cli_extend' - result_diff = CM_check(new_url, running_params, ['acl resource "%(res_grp_name)s" "%(resource)s"\n'], src_str) + result_diff = CM_check(new_url, running_params, ['acl resource "%(res_grp_name)s" "%(resource)s"\n'], + src_str) return result_diff + def _perform_Init_DeviceList(self, options): if options['option'] not in ['delete', 'insert']: raise ModelQueryException(CLICmdError(__('Internal error.'))) else: - pk_dict = {'table':'cm_acl_resource', 'id':options['__pk_list'][0]['id']} + pk_dict = {'table': 'cm_acl_resource', 'id': options['__pk_list'][0]['id']} update_cm_role_devicelist(options['device_list'].split(','), pk_dict, options['option'], None) return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py (working copy) @@ -1,39 +1,47 @@ -from hive.imports.model import * +import datetime + +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -from cm.lib.parse_configfile import DEFAULT_CONFIG_FILE_PATH, parse_specific_config, delete_specific_info_from_config, sync_vsite_to_config + from cm.lib.parse_configfile import INSERT, DELETE -from cm.models.configuration.vsite_config import vs_insert_delete, CM_check -from hive.model.legacycli import MATCHALL, MATCHONE -import os, datetime, json, difflib, time -#from cm.lib.sqlite_db import DB +from cm.lib.parse_configfile import parse_specific_config, delete_specific_info_from_config, \ + sync_vsite_to_config from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, update_cm_role_devicelist +from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ + update_cm_role_devicelist +from cm.models.configuration.vsite_config import vs_insert_delete, CM_check from cm.models.tasking import GLOBAL_TASK -from apscheduler.schedulers.background import BackgroundScheduler -from django.db.models.query import QuerySet -__=_ +from hive.imports.model import * +from hive.model.legacycli import MATCHALL +__ = _ + + class RoleGroup(ANModel): - asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_group', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_group', + mul='*', pos='right', optional=True) name = CharField(verbose_name='Role Group Name', primary_key=True, length='1..32') description = TextField(verbose_name='Description', length='1..256', optional=True) priority = Int32Field(verbose_name=_('Role Priority'), scope='1..2000') + # asso_qualification = AssoField2(verbose_name='Role Qualification', hidden=True, # tgt='configuration.vsite_config.role.RoleQualification.role_name', mul='1', pos='right', optional=True) class Meta: verbose_name = 'Role Group' - list_config_options={'columns':[ - {'name':'name'}, - {'name':'description'}, - {'name':'priority'}, + list_config_options = {'columns': [ + {'name': 'name'}, + {'name': 'description'}, + {'name': 'priority'}, ]} + class Manager(CLIManager): def _get(self, pk_dict=None): return self._filter(pk_dict).get(pk_dict) + def _filter(self, filter_dict): - #print 'role group', filter_dict + # print 'role group', filter_dict asso_data = filter_dict['asso'][0] reg_str = 'role name "(?P.*?)" "(?P.*?)" (?P[0-9]+)' rtn = parse_specific_config(asso_data, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) @@ -43,37 +51,41 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #role name role_test role_test 1 + # role name role_test role_test 1 cli_str = 'role name "%s" "%s" %d' % (instance.name, instance.description, instance.priority) ret = vs_insert_delete(instance.asso[0], cli_str, '#role configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no role name role_test + # no role name role_test specific_str = 'role name "%s" "%s" %d' % (instance.name, instance.description, int(instance.priority)) cli_str = 'no role name "%s"' % (instance.name) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) - #delete all of the role group + # delete all of the role group vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + instance.asso[0]['device_name'] + '-' + instance.asso[0]['vs_name'] filter_str = ['role qualification "%s".*?' % instance.name, - 'role condition "%s".*?' % instance.name, - 'role resource .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', filter_str) + 'role condition "%s".*?' % instance.name, + 'role resource .* "%s".*?' % instance.name] + ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) sync_vsite_to_config(instance.asso[0]['vs_name'], instance.asso[0]['device_name']) mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'VsiteConfig'])) - return + return + class RoleQualification(ANModel): - asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_qualification', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_qualification', + mul='*', pos='right', optional=True) # role_name = AssoField2(verbose_name='Role Name', tgt='configuration.vsite_config.role.RoleGroup.asso_qualification', # mul='*', pos='left') role_name = CharField(verbose_name='Role Name', primary_key=True, length='1..32') @@ -82,72 +94,78 @@ class Meta: verbose_name = 'Role Qualification' - list_config_options={'columns':[ - {'name':'role_name'}, - {'name':'name'}, - {'name':'description'}, + list_config_options = {'columns': [ + {'name': 'role_name'}, + {'name': 'name'}, + {'name': 'description'}, ]} class Manager(CLIManager): def _get(self, pk_dict=None): return self._filter(pk_dict).get(pk_dict) + def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #role qualification "bug_role" "time" "time" + # role qualification "bug_role" "time" "time" reg_str = 'role qualification "(?P.*?)" "(?P.*?)" "(?P.*?)"' rtn = parse_specific_config(asso_data, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) if type(rtn) == str: raise ModelQueryException(CLICmdError(__(rtn))) for item in rtn[0]: item['asso'] = filter_dict['asso'] - #item['role_name'] = [{'name': item['role_name'], 'asso':filter_dict['asso']}] - #item['role_name'] = [{'name': item['role_name']}] + # item['role_name'] = [{'name': item['role_name'], 'asso':filter_dict['asso']}] + # item['role_name'] = [{'name': item['role_name']}] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #role qualification "bug_role" "time" "time" + # role qualification "bug_role" "time" "time" cli_str = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, instance.description) ret = vs_insert_delete(instance.asso[0], cli_str, '#role configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no role qualification "bug_role" "time" - specific_str = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, instance.description) + # no role qualification "bug_role" "time" + specific_str = 'role qualification "%s" "%s" "%s"' % (instance.role_name, instance.name, + instance.description) cli_str = 'no role qualification "%s" "%s"' % (instance.role_name, instance.name) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) - - #delete all of the role condition of qualification + + # delete all of the role condition of qualification vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + instance.asso[0]['device_name'] + '-' + instance.asso[0]['vs_name'] filter_str = ['role condition .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', filter_str) + ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) sync_vsite_to_config(instance.asso[0]['vs_name'], instance.asso[0]['device_name']) - return + return + class RoleCondition(ANModel): - asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_condition', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Vsite Device', tgt='configuration.vsite_config.VsiteConfig.asso_role_condition', + mul='*', pos='right', optional=True) role_name = CharField(verbose_name='Role Name', length='1..32', primary_key=True) qualification_name = CharField(verbose_name='Qualification Name', length='1..32', primary_key=True) content = CharField(verbose_name='Content', length='1..64', primary_key=True) class Meta: verbose_name = 'Role Condition' - list_config_options={'columns':[ - {'name':'role_name'}, - {'name':'qualification_name'}, - {'name':'content'}, + list_config_options = {'columns': [ + {'name': 'role_name'}, + {'name': 'qualification_name'}, + {'name': 'content'}, ]} class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #role condition "bug_role" "user" "USERNAME IS 'c'" + # role condition "bug_role" "user" "USERNAME IS 'c'" reg_str = 'role condition "(?P.*?)" "(?P.*?)" "(?P.*?)"' rtn = parse_specific_config(asso_data, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -156,39 +174,46 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #role condition "bug_role" "year" "LOGINYEAR IS 2016,2017" - cli_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) + # role condition "bug_role" "year" "LOGINYEAR IS 2016,2017" + cli_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) ret = vs_insert_delete(instance.asso[0], cli_str, '#role configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no role condition "bug_role" "year" "LOGINYEAR IS 2016,2017" - specific_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) - cli_str = 'no role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, instance.content) + # no role condition "bug_role" "year" "LOGINYEAR IS 2016,2017" + specific_str = 'role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) + cli_str = 'no role condition "%s" "%s" "%s"' % (instance.role_name, instance.qualification_name, + instance.content) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + class RoleNetpoolResource(ANModel): - asso = AssoField2(verbose_name="Vsite Device", tgt='configuration.vsite_config.VsiteConfig.asso_role_netpool_resource', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name="Vsite Device", + tgt='configuration.vsite_config.VsiteConfig.asso_role_netpool_resource', + mul='*', pos='right', optional=True) role_name = CharField(verbose_name='Role Name', length='1..32', primary_key=True) netpool_name = CharField(verbose_name='Netpool Name', length='1..32', primary_key=True) - + class Meta: verbose_name = 'Role Netpool Resource' - list_config_options={'columns':[ - {'name':'role_name'}, - {'name':'netpool_name'}, + list_config_options = {'columns': [ + {'name': 'role_name'}, + {'name': 'netpool_name'}, ]} class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #role resource netpool "bug_role" "bug_netpool_1" + # role resource netpool "bug_role" "bug_netpool_1" reg_str = 'role resource netpool "(?P.*?)" "(?P.*?)"' rtn = parse_specific_config(asso_data, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -196,16 +221,18 @@ for item in rtn[0]: item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) - return QuerySet(self._model, rtn[0]) + return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #role resource netpool "bug_role" "bug_netpool" + # role resource netpool "bug_role" "bug_netpool" cli_str = 'role resource netpool "%s" "%s"' % (instance.role_name, instance.netpool_name) ret = vs_insert_delete(instance.asso[0], cli_str, '#role configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no role resource netpool "bug_role" "bug_netpool" + # no role resource netpool "bug_role" "bug_netpool" specific_str = 'role resource netpool "%s" "%s"' % (instance.role_name, instance.netpool_name) cli_str = 'no role resource netpool "%s" "%s"' % (instance.role_name, instance.netpool_name) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) @@ -213,22 +240,25 @@ raise ModelQueryException(CLICmdError(__(ret))) return + class RoleVPNGroupResource(ANModel): - asso = AssoField2(verbose_name="Vsite Device", tgt='configuration.vsite_config.VsiteConfig.asso_role_vpn_group_resource', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name="Vsite Device", + tgt='configuration.vsite_config.VsiteConfig.asso_role_vpn_group_resource', + mul='*', pos='right', optional=True) role_name = CharField(verbose_name='Role Name', length='1..32', primary_key=True) vpn_group_name = CharField(verbose_name='VPN Group Name', length='1..32', primary_key=True) - + class Meta: verbose_name = 'Role Netpool Resource' - list_config_options={'columns':[ - {'name':'role_name'}, - {'name':'vpn_group_name'}, + list_config_options = {'columns': [ + {'name': 'role_name'}, + {'name': 'vpn_group_name'}, ]} + class Manager(CLIManager): def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #role resource vpnresourcegroup "bug_role" "vpn_group_name" + # role resource vpnresourcegroup "bug_role" "vpn_group_name" reg_str = 'role resource vpnresourcegroup "(?P.*?)" "(?P.*?)"' rtn = parse_specific_config(asso_data, '#role configuration\n', '#acl configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -236,16 +266,18 @@ for item in rtn[0]: item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) - return QuerySet(self._model, rtn[0]) + return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #role resource vpnresourcegroup "bug_role" "bug_vpn_group" + # role resource vpnresourcegroup "bug_role" "bug_vpn_group" cli_str = 'role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, instance.vpn_group_name) ret = vs_insert_delete(instance.asso[0], cli_str, '#role configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no role resource vpnresourcegroup "bug_role" "bug_vpn_group" + # no role resource vpnresourcegroup "bug_role" "bug_vpn_group" specific_str = 'role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, instance.vpn_group_name) cli_str = 'no role resource vpnresourcegroup "%s" "%s"' % (instance.role_name, instance.vpn_group_name) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) @@ -253,12 +285,14 @@ raise ModelQueryException(CLICmdError(__(ret))) return + class CMRoleGroup(ANModel): """ Get data from database """ name = CharField(verbose_name='Name', primary_key=True, length='1..32') - #priority = Int32Field(verbose_name='Priority', default=1) + # priority = Int32Field(verbose_name='Priority', default=1) description = CharField(verbose_name='Description', length='1..128', editable=True) - status = Int32Field(verbose_name='Status', default=0, configurable=False, optional=True) #0-new 1-success 2-failed + status = Int32Field(verbose_name='Status', default=0, configurable=False, + optional=True) # 0-new 1-success 2-failed time = CharField(verbose_name='Modify time', length='1..64', configurable=False, optional=True) device_list = CharField(verbose_name='Device List', length='1..1024', configurable=False, optional=True) @@ -275,15 +309,18 @@ option_fields = ( # DynamicMultiEnumField(name='device', verbose_name='devices', # values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']), - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), EnumField(verbose_name='Action', default='insert', name='action', values=( ('insert', 'Insert'), ('delete', 'Delete'), )), ) + class Check(Action): verbose_name = _('Check') action_name = _('Check') @@ -291,10 +328,12 @@ confirm_msg = 'Are you sure you want to do that?' forever = False instance_mul = '1..*' - option_fields = ( - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), - + option_fields = ( + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), + ) + class Init_DeviceList(Action): verbose_name = 'Init Device List' action_name = 'Init_DeviceList' @@ -302,9 +341,9 @@ forever = False instance_mul = '1' option_fields = ( - CharField(verbose_name='Option', name='option'), - CharField(verbose_name='Device List', name='device_list') - ) + CharField(verbose_name='Option', name='option'), + CharField(verbose_name='Device List', name='device_list') + ) class Manager(CLIManager): def _get_query_set(self): @@ -314,11 +353,14 @@ db.close() key = ['name', 'description', 'status', 'time', 'device_list'] result = [dict(zip(key, each)) for each in data] - return QuerySet(self._model, result) + return QuerySet(self._model, result) def _insert(self, instance): db = DB.get_connected_db() - save_sql = "INSERT INTO cm_role(name, description, time) values('%s', '%s', '%s')" % (instance.name, instance.description, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + save_sql = "INSERT INTO cm_role(name, description, time) values('%s', '%s', '%s')" % (instance.name, + instance.description, + datetime.datetime.now().strftime( + '%Y-%m-%d %H:%M:%S')) db.execute_sql(save_sql) db.close() return @@ -327,16 +369,17 @@ description = instance.description time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - update_sql = "UPDATE cm_role set time='%s', description='%s' where name='%s'" % (time, description, instance.name) + update_sql = "UPDATE cm_role set time='%s', description='%s' where name='%s'" % (time, description, + instance.name) db.execute_sql(update_sql) db.close() - return + return def _delete_instance(self, instance): db = DB.get_connected_db() delete_sql = "DELETE FROM cm_role where name='%s'" % instance.name db.execute_sql(delete_sql) - #delete acl/vpn resource of role group + # delete acl/vpn resource of role group delete_sql = "DELETE FROM cm_acl_resource where group_name='network_%s'" % instance.name db.execute_sql(delete_sql) delete_sql = "DELETE FROM cm_vpn_resource where group_name ='%s'" % instance.name @@ -345,11 +388,11 @@ mark_expire_all(get_model('cm', ['configuration', 'vsite_config', 'vpn', 'CMVPNResource'])) db.close() return - + def _perform_Sync(self, options): sess = get_current_session() cm_role_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'CMRoleGroup']) - #pk_list = options['__pk_list'][0] + # pk_list = options['__pk_list'][0] device_list = [] service_list = options['service'] db = DB.get_connected_db() @@ -359,49 +402,56 @@ key = ['device_list', 'service_name'] device_list.append([dict(zip(key, each)) for each in data]) db.close() - #[[{'service_name': 'intranet', 'device_list': 'Intranet'}], [{'service_name': 'log_test', 'device_list': '10.8.11.35'}]] - + # [[{'service_name': 'intranet', 'device_list': 'Intranet'}], [{'service_name': 'log_test', 'device_list': '10.8.11.35'}]] + for pk in options['__pk_list']: - cm_manager = cm_role_model.get_manager(sess).filter({'name':pk['name']}) + cm_manager = cm_role_model.get_manager(sess).filter({'name': pk['name']}) time.sleep(1) name = pk['name'] new_url = '/rest/ag/global/batch_cli' - + running_params = [] for device in device_list: if options['action'] == 'insert': cli_str = 'sw %s\n role name "%s"\n role qualification "%s" "%s"\n vpn resource group "%s"\n role resource vpnresourcegroup "%s" "%s"\n acl resourcegroup network network_%s\n acl rule "%s" network_%s permit\n role condition "%s" "%s" "GROUPNAME IS %s"' \ - % (device[0]['service_name'], name, name, name, name, name, name, name, name, name, name, name, name) + % (device[0]['service_name'], name, name, name, name, name, name, name, name, name, + name, name, name) elif options['action'] == 'delete': - cli_str = 'sw %s\n no role name "%s"\n no vpn resource group "%s"\n no acl resourcegroup network_%s\n ' % (device[0]['service_name'], name, name, name) - #running_params = json.dumps({'config':cli_str}) - running_params.append({'params':json.dumps({'config':cli_str}), 'device_list':device[0]['device_list']}) + cli_str = 'sw %s\n no role name "%s"\n no vpn resource group "%s"\n no acl resourcegroup network_%s\n ' % ( + device[0]['service_name'], name, name, name) + # running_params = json.dumps({'config':cli_str}) + running_params.append( + {'params': json.dumps({'config': cli_str}), 'device_list': device[0]['device_list']}) task_name = '%s-%s' % (cm_manager[0].description, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) - #data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - + # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') if date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - - pk_dict = {'table':'cm_role', 'name':name} + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + + pk_dict = {'table': 'cm_role', 'name': name} new_schedule = {} description = " %s Role[%s] to service[%s]\n" % (options['action'], name, ','.join(service_list)) description += " Sync create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'sync_to_device' new_schedule['args'] = [new_url, running_params, pk_dict, options['action'], task_name] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'sync_role', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=','.join(service_list)) + add_job_into_database(task_name, 'sync_role', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=','.join(service_list)) sched = BackgroundScheduler() sched.add_job(sync_to_device, 'date', next_run_time=date, args=new_schedule['args']) @@ -415,13 +465,13 @@ service_list = options['service'] db = DB.get_connected_db() for service in service_list: - #select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service + # select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service select_sql = "with raw as (select service.name, service.service_name, device.name as dname from service left join ds on service.id=ds.s_id left join device on device.id=ds.d_id where service.name='%s') select array_agg(dname), raw.service_name from raw group by raw.name, raw.service_name;" % service data = db.fetchall(select_sql) key = ['device_list', 'service_name'] device_list.append([dict(zip(key, each)) for each in data]) db.close() - #device_list [[{'service_name': 'intranet', 'device_list': 'Intranet'}], [{'service_name': 'log_test', 'device_list': '10.8.11.35'}]] + # device_list [[{'service_name': 'intranet', 'device_list': 'Intranet'}], [{'service_name': 'log_test', 'device_list': '10.8.11.35'}]] # {u'__pk_list': [{u'name': u'array'}, {u'name': u'array_network'}], u'devices': {u'device': [u'10.8.2.55']}} pk_list = options['__pk_list'] @@ -432,19 +482,20 @@ running_params = [] for device in device_list: - params = json.dumps({"cmd":"show role name", "vsite_name":"%s" % device[0]['service_name']}) - running_params.append({'params':[params,], 'device_list':device[0]['device_list']}) + params = json.dumps({"cmd": "show role name", "vsite_name": "%s" % device[0]['service_name']}) + running_params.append({'params': [params, ], 'device_list': device[0]['device_list']}) new_url = '/rest/ag/global/cli_extend' result_diff = CM_check(new_url, running_params, ['role name "%(role_name)s" "" 1\n'], src_str) return result_diff + def _perform_Init_DeviceList(self, options): if options['option'] not in ['delete', 'insert']: raise ModelQueryException(CLICmdError(__('Internal error.'))) else: - pk_dict = {'table':'cm_role', 'name':options['__pk_list'][0]['name']} + pk_dict = {'table': 'cm_role', 'name': options['__pk_list'][0]['name']} update_cm_role_devicelist(options['device_list'].split(','), pk_dict, options['option'], None) return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py (working copy) @@ -1,21 +1,25 @@ -from hive.imports.model import * +import datetime + +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from cm.lib.parse_configfile import INSERT, DELETE, delete_specific_info_from_config, sync_vsite_to_config from cm.lib.parse_configfile import parse_specific_config -from cm.models.configuration.vsite_config import vs_insert_delete, CM_check -from cm.lib.parse_configfile import INSERT, DELETE, delete_specific_info_from_config, DEFAULT_CONFIG_FILE_PATH, sync_vsite_to_config -from hive.model.legacycli import MATCHALL, MATCHONE -import os, datetime, difflib, time from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, update_cm_role_devicelist +from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ + update_cm_role_devicelist +from cm.models.configuration.vsite_config import vs_insert_delete, CM_check from cm.models.tasking import GLOBAL_TASK -from apscheduler.schedulers.background import BackgroundScheduler -from django.db.models.query import QuerySet -import json -__=_ +from hive.imports.model import * +from hive.model.legacycli import MATCHALL +__ = _ + + class NetGroupItem(ANModel): - asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_network_item', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_network_item', + mul='*', pos='right', optional=True) vpn_group_name = CharField(verbose_name='VPN Group Name', length='1..32', primary_key=True) net_resource = CharField(verbose_name=_('NetWork Resource'), length='1..64', primary_key=True) net_type = EnumField(verbose_name=_('Type'), values=( @@ -26,18 +30,19 @@ class Meta: verbose_name = 'Network-type VPN Resource Item' - list_config_options={'columns':[ - {'name':'vpn_group_name'}, - {'name':'net_resource'}, - {'name':'net_type'}, - ], 'click_enable':False} + list_config_options = {'columns': [ + {'name': 'vpn_group_name'}, + {'name': 'net_resource'}, + {'name': 'net_type'}, + ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): return self._filter(pk_dict).get(pk_dict) + def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #asso_vs_info = asso_data['asso'][0] + # asso_vs_info = asso_data['asso'][0] reg_str = 'vpn resource groupitem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)' rtn = parse_specific_config(asso_data, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -46,25 +51,32 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #vpn resource group network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" 1 - cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + # vpn resource group network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" 1 + cli_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, + instance.net_type) ret = vs_insert_delete(instance.asso[0], cli_str, '#VPN configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no vpn resource group network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" - specific_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, int(instance.net_type)) + # no vpn resource group network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" + specific_str = 'vpn resource groupitem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, + int(instance.net_type)) cli_str = 'no vpn resource groupitem network "%s" "%s"' % (instance.vpn_group_name, instance.net_resource) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + class NetGroupExcludedItem(ANModel): - asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_network_exclude_item', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Resource Group', + tgt='configuration.vsite_config.VsiteConfig.asso_vpn_network_exclude_item', + mul='*', pos='right', optional=True) vpn_group_name = CharField(verbose_name='VPN Group Name', length='1..32', primary_key=True) net_resource = CharField(verbose_name=_('NetWork Resource'), length='1..64', primary_key=True) net_type = EnumField(verbose_name=_('Type'), values=( @@ -75,18 +87,19 @@ class Meta: verbose_name = 'Network-type VPN Resource Item' - list_config_options={'columns':[ - {'name':'vpn_group_name'}, - {'name':'net_resource'}, - {'name':'net_type'}, - ], 'click_enable':False} + list_config_options = {'columns': [ + {'name': 'vpn_group_name'}, + {'name': 'net_resource'}, + {'name': 'net_type'}, + ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): return self._filter(pk_dict).get(pk_dict) + def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #asso_vs_info = asso_data['asso'][0] + # asso_vs_info = asso_data['asso'][0] reg_str = 'vpn resource groupexcludeditem network "(?P.*?)" "(?P.*?)" (?P[0-9]+)' rtn = parse_specific_config(asso_data, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -95,36 +108,45 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #vpn resource groupexcludeditem network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" 1 - cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, instance.net_type) + # vpn resource groupexcludeditem network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" 1 + cli_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, instance.net_type) ret = vs_insert_delete(instance.asso[0], cli_str, '#VPN configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no vpn resource groupexcludeditem network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" - specific_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, instance.net_resource, int(instance.net_type)) - cli_str = 'no vpn resource groupexcludeditem network "%s" "%s"' % (instance.vpn_group_name, instance.net_resource) + # no vpn resource groupexcludeditem network "bug_vpn_group" "10.8.7.86/255.255.255.255:0-65535" + specific_str = 'vpn resource groupexcludeditem network "%s" "%s" %d' % (instance.vpn_group_name, + instance.net_resource, + int(instance.net_type)) + cli_str = 'no vpn resource groupexcludeditem network "%s" "%s"' % (instance.vpn_group_name, + instance.net_resource) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + class VPNResourceGroup(ANModel): - asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_resource_group', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Resource Group', + tgt='configuration.vsite_config.VsiteConfig.asso_vpn_resource_group', + mul='*', pos='right', optional=True) name = CharField(verbose_name='VPN Resource Group', primary_key=True, length='1..64') - # asso_net_item = AssoField2(verbose_name=_('Network-type VPN Resource Item'), + + # asso_net_item = AssoField2(verbose_name=_('Network-type VPN Resource Item'), # tgt='configuration.vsite_config.vpn.NetGroupItem.asso', mul='1', pos='left', optional=True) # asso_net_excluded_item = AssoField2(verbose_name=_('Network-type VPN Resource Item'), # tgt='configuration.vsite_config.vpn.NetGroupExcludedItem.asso', mul='1', pos='left', optional=True) class Meta: verbose_name = 'VPN Resource Group' - list_config_options={'columns':[ - {'name':'name'}, - ]} + list_config_options = {'columns': [ + {'name': 'name'}, + ]} class Manager(CLIManager): def _filter(self, filter_dict): @@ -137,38 +159,42 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #vpn resource group "bug_vpn_group" + # vpn resource group "bug_vpn_group" cli_str = 'vpn resource group "%s"' % instance.name ret = vs_insert_delete(instance.asso[0], cli_str, '#VPN configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no vpn resource group "bug_vpn_group" + # no vpn resource group "bug_vpn_group" specific_str = 'vpn resource group "%s"' % instance.name cli_str = 'no vpn resource group "%s"' % instance.name ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) - #delete all of the vpn group + # delete all of the vpn group vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + instance.asso[0]['device_name'] + '-' + instance.asso[0]['vs_name'] - #print 'vs_file_path', vs_file_path + # print 'vs_file_path', vs_file_path filter_str = ['vpn resource .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#VPN configuration\n', '#aaa configuration\n', filter_str) + ret = delete_specific_info_from_config(vs_file_path, '#VPN configuration\n', '#aaa configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) - + sync_vsite_to_config(instance.asso[0]['vs_name'], instance.asso[0]['device_name']) return - + class IPRange(ANModel): - asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_netpool_iprange', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Resource Group', + tgt='configuration.vsite_config.VsiteConfig.asso_vpn_netpool_iprange', + mul='*', pos='right', optional=True) netpool_name = CharField(verbose_name=_('VPN Netpool name'), primary_key=True, length='1..31') first_ip = IPAddressField(verbose_name=_('First IP Address'), primary_key=True) last_ip = IPAddressField(verbose_name=_('Last IP Address'), primary_key=True) @@ -176,18 +202,19 @@ class Meta: verbose_name = 'IP Address Range' - list_config_options={'columns':[ - {'name':'netpool_name'}, - {'name':'first_ip'}, - {'name':'last_ip'}, - ], 'click_enable':False} + list_config_options = {'columns': [ + {'name': 'netpool_name'}, + {'name': 'first_ip'}, + {'name': 'last_ip'}, + ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): return self._filter(pk_dict).get(pk_dict) + def _filter(self, filter_dict): asso_data = filter_dict['asso'][0] - #asso_vs_info = asso_data['asso'][0] + # asso_vs_info = asso_data['asso'][0] reg_str = 'vpn netpool iprange dynamic "(?P.*?)" (?P[0-9|\.|:|\w]+) (?P[0-9|\.|:|\w]+) "(?P.*?)"' rtn = parse_specific_config(asso_data, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) if type(rtn) == str: @@ -196,27 +223,39 @@ item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #vpn netpool iprange dynamic "bug_netpool" 10.8.2.60 10.8.2.61 "" - cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) + # vpn netpool iprange dynamic "bug_netpool" 10.8.2.60 10.8.2.61 "" + cli_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], instance.unit_name) ret = vs_insert_delete(instance.asso[0], cli_str, '#VPN configuration\n', INSERT) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no netpool name "bug_netpool" - specific_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) - cli_str = 'no vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, instance.first_ip.values()[0], instance.last_ip.values()[0], instance.unit_name) + # no netpool name "bug_netpool" + specific_str = 'vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], + instance.unit_name) + cli_str = 'no vpn netpool iprange dynamic "%s" %s %s "%s"' % (instance.netpool_name, + instance.first_ip.values()[0], + instance.last_ip.values()[0], + instance.unit_name) ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) return + class VPNNetpool(ANModel): - asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_netpool', - mul='*', pos='right', optional=True) + asso = AssoField2(verbose_name='Resource Group', tgt='configuration.vsite_config.VsiteConfig.asso_vpn_netpool', + mul='*', pos='right', optional=True) name = CharField(verbose_name=_('VPN Netpool name'), primary_key=True, length='1..31') enable_nat = BooleanField(verbose_name='Enable Nat') + # ip_range = FieldGroup(writable=False, verbose_name='IP Range', level=BASIC, editable=True, fields={ # 'asso_ip_address' : AssoField2(verbose_name=_('IP Address'), # tgt='configuration.vsite_config.vpn.IPAddress.asso', mul='1', pos='left', optional=True), @@ -224,9 +263,9 @@ class Meta: verbose_name = 'VPN Netpool' - list_config_options={'columns':[ - {'name':'name'}, - {'name':'enable_nat'} + list_config_options = {'columns': [ + {'name': 'name'}, + {'name': 'enable_nat'} ]} class Manager(CLIManager): @@ -238,16 +277,18 @@ raise ModelQueryException(CLICmdError(__(rtn))) for item in rtn[0]: reg_str = 'vpn netpool nat "%s" "useglobal"' % item['name'] - nat_enable = parse_specific_config(asso_data, '#VPN configuration\n', '#aaa configuration\n', reg_str, MATCHALL) + nat_enable = parse_specific_config(asso_data, '#VPN configuration\n', '#aaa configuration\n', reg_str, + MATCHALL) if len(nat_enable[0]): item['enable_nat'] = True else: item['enable_nat'] = False item['asso'] = filter_dict['asso'] self._model._meta.mark_delay_query(item) - return QuerySet(self._model, rtn[0]) + return QuerySet(self._model, rtn[0]) + def _insert(self, instance): - #vpn netpool name "bug_netpool" + # vpn netpool name "bug_netpool" cli_str = 'vpn netpool name "%s"' % instance.name ret = vs_insert_delete(instance.asso[0], cli_str, '#VPN configuration\n', INSERT) if ret: @@ -258,28 +299,32 @@ if ret: raise ModelQueryException(CLICmdError(__(ret))) return + def _delete_instance(self, instance): - #no netpool name "bug_netpool" + # no netpool name "bug_netpool" specific_str = 'vpn netpool name "%s"' % instance.name cli_str = 'no vpn netpool name "%s"' % instance.name ret = vs_insert_delete(instance.asso[0], cli_str, specific_str, DELETE) if ret: raise ModelQueryException(CLICmdError(__(ret))) - - #delete all of the vpn netpool + + # delete all of the vpn netpool vsite_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'vsite') vs_file_path = vsite_path + '/' + instance.asso[0]['device_name'] + '-' + instance.asso[0]['vs_name'] filter_str = ['vpn netpool .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#VPN configuration\n', '#aaa configuration\n', filter_str) + ret = delete_specific_info_from_config(vs_file_path, '#VPN configuration\n', '#aaa configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) - filter_str = ['role resource netpool .* "%s".*?' % instance.name] - ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', filter_str) + filter_str = ['role resource netpool .* "%s".*?' % instance.name] + ret = delete_specific_info_from_config(vs_file_path, '#role configuration\n', '#acl configuration\n', + filter_str) if ret: raise ModelQueryException(CLICmdError(__(ret))) sync_vsite_to_config(instance.asso[0]['vs_name'], instance.asso[0]['device_name']) return + class CMVPNResource(ANModel): id = Int32Field(primary_key=True, configurable=False) group_name = CharField(verbose_name='VPN Group Name', length='1..32') @@ -289,9 +334,11 @@ (2, 'Application'), (0, 'Both'), )) - group_type = BooleanField(verbose_name='Group Type', lexical=('included', 'excluded'), display=('included','excluded')) + group_type = BooleanField(verbose_name='Group Type', lexical=('included', 'excluded'), + display=('included', 'excluded')) description = CharField(verbose_name='description', length='1..128', editable=True) - status = Int32Field(verbose_name='Status', default=0, configurable=False, optional=True) #0-new 1-success 2-failed + status = Int32Field(verbose_name='Status', default=0, configurable=False, + optional=True) # 0-new 1-success 2-failed time = CharField(verbose_name='Modify time', length='1..64', configurable=False, optional=True) device_list = CharField(verbose_name='Device List', length='1..1024', configurable=False, optional=True) @@ -308,11 +355,13 @@ option_fields = ( # DynamicMultiEnumField(name='device', verbose_name='devices', # values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']), - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), EnumField(verbose_name='Action', default='insert', name='action', values=( ('insert', 'Insert'), ('delete', 'Delete'), )), @@ -325,9 +374,9 @@ forever = False instance_mul = '1' option_fields = ( - CharField(verbose_name='Option', name='option'), - CharField(verbose_name='Device List', name='device_list') - ) + CharField(verbose_name='Option', name='option'), + CharField(verbose_name='Device List', name='device_list') + ) class Check(Action): verbose_name = _('Check') @@ -337,8 +386,10 @@ forever = False instance_mul = '1..*' option_fields = ( - DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', '__field:cm.device_mgmt.service.Services.name']), + DynamicMultiEnumField(verbose_name='service', name='service', values=['__value:---Please Select---', + '__field:cm.device_mgmt.service.Services.name']), ) + class Manager(CLIManager): def _get_query_set(self): db = DB.get_connected_db() @@ -348,12 +399,14 @@ key = ['id', 'group_name', 'description', 'status', 'time', 'device_list', 'resource', 'type', 'group_type'] result = [dict(zip(key, each)) for each in data] - return QuerySet(self._model, result) + return QuerySet(self._model, result) def _insert(self, instance): - #group_type = 'included' if instance.group_type else 'excluded' + # group_type = 'included' if instance.group_type else 'excluded' db = DB.get_connected_db() - save_sql = "INSERT INTO cm_vpn_resource(group_name, description, time, resource, type, group_type) values('%s', '%s', '%s', '%s', %d, '%s')" % (instance.group_name, instance.description, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), instance.resource, instance.type, instance.group_type) + save_sql = "INSERT INTO cm_vpn_resource(group_name, description, time, resource, type, group_type) values('%s', '%s', '%s', '%s', %d, '%s')" % ( + instance.group_name, instance.description, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + instance.resource, instance.type, instance.group_type) db.execute_sql(save_sql) db.close() return @@ -369,10 +422,11 @@ description = instance.description time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - update_sql = "UPDATE cm_vpn_resource set time='%s', description='%s' where id=%d" % (time, description, instance.id) + update_sql = "UPDATE cm_vpn_resource set time='%s', description='%s' where id=%d" % (time, description, + instance.id) db.execute_sql(update_sql) db.close() - return + return def _perform_Sync(self, options): sess = get_current_session() @@ -382,7 +436,7 @@ service_list = options['service'] db = DB.get_connected_db() for service in service_list: - #select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service + # select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service select_sql = "with raw as (select service.name, service.service_name, device.name as dname from service left join ds on service.id=ds.s_id left join device on device.id=ds.d_id where service.name='%s') select array_agg(dname), raw.service_name from raw group by raw.name, raw.service_name;" % service data = db.fetchall(select_sql) key = ['device_list', 'service_name'] @@ -391,40 +445,49 @@ for pk in options['__pk_list']: time.sleep(1) - cm_manager = cm_vpn_model.get_manager(sess).filter({'id':pk['id']}) + cm_manager = cm_vpn_model.get_manager(sess).filter({'id': pk['id']}) group_name = cm_manager[0].group_name resource = cm_manager[0].resource type = cm_manager[0].type group_type = 'included' if cm_manager[0].group_type else 'excluded' - #pk_list = options['__pk_list'][0] + # pk_list = options['__pk_list'][0] new_url = '/rest/ag/global/batch_cli' running_params = [] for device in device_list: if options['action'] == 'insert': - cli_str = 'sw %s\n vpn resource groupitem network "%s" "%s" %d\n' % (device[0]['service_name'], group_name, resource, type) if group_type == 'included' else 'sw %s\n vpn resource groupexcludeditem network "%s" "%s" %d\n' % (device[0]['service_name'], group_name, resource, type) + cli_str = 'sw %s\n vpn resource groupitem network "%s" "%s" %d\n' % (device[0]['service_name'], + group_name, resource, + type) if group_type == 'included' else 'sw %s\n vpn resource groupexcludeditem network "%s" "%s" %d\n' % ( + device[0]['service_name'], group_name, resource, type) elif options['action'] == 'delete': - cli_str = 'sw %s\n no vpn resource groupitem network "%s" "%s" \n' % (device[0]['service_name'], group_name, resource) if group_type == 'included' else 'sw %s\n no vpn resource groupexcludeditem network "%s" "%s" \n' % (device[0]['service_name'], group_name, resource) - #running_params = json.dumps({'config':cli_str}) - running_params.append({'params':json.dumps({'config':cli_str}), 'device_list':device[0]['device_list']}) + cli_str = 'sw %s\n no vpn resource groupitem network "%s" "%s" \n' % (device[0]['service_name'], + group_name, + resource) if group_type == 'included' else 'sw %s\n no vpn resource groupexcludeditem network "%s" "%s" \n' % ( + device[0]['service_name'], group_name, resource) + # running_params = json.dumps({'config':cli_str}) + running_params.append( + {'params': json.dumps({'config': cli_str}), 'device_list': device[0]['device_list']}) task_name = '%s-%s' % (cm_manager[0].description, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) - #data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + if date < datetime.datetime.now(): + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - pk_dict = {'table':'cm_vpn_resource', 'id':pk['id']} + pk_dict = {'table': 'cm_vpn_resource', 'id': pk['id']} new_schedule = {} description = " %s VPN[%s] to services[%s]\n" % (options['action'], resource, ','.join(service_list)) description += " Sync create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' @@ -432,7 +495,9 @@ new_schedule['args'] = [new_url, running_params, pk_dict, options['action'], task_name] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'sync_vpn', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=','.join(service_list)) + add_job_into_database(task_name, 'sync_vpn', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, + device_list=','.join(service_list)) sched = BackgroundScheduler() sched.add_job(sync_to_device, 'date', next_run_time=date, args=new_schedule['args']) GLOBAL_TASK.add(task_name, sched) @@ -445,7 +510,7 @@ service_list = options['service'] db = DB.get_connected_db() for service in service_list: - #select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service + # select_sql = "SELECT device_list, service_name FROM service WHERE name='%s'" % service select_sql = "with raw as (select service.name, service.service_name, device.name as dname from service left join ds on service.id=ds.s_id left join device on device.id=ds.d_id where service.name='%s') select array_agg(dname), raw.service_name from raw group by raw.name, raw.service_name;" % service data = db.fetchall(select_sql) key = ['device_list', 'service_name'] @@ -458,29 +523,37 @@ pk_list = options['__pk_list'] for pk in pk_list: - cm_manager = cm_vpn_model.get_manager(sess).filter({'id':pk['id']}) + cm_manager = cm_vpn_model.get_manager(sess).filter({'id': pk['id']}) if cm_manager[0].group_type: - src_str.append('vpn resource groupitem network "%s" "%s" %d\n' % (cm_manager[0].group_name, cm_manager[0].resource, cm_manager[0].type)) + src_str.append('vpn resource groupitem network "%s" "%s" %d\n' % (cm_manager[0].group_name, + cm_manager[0].resource, + cm_manager[0].type)) else: - src_str.append('vpn resource groupexcludeditem network "%s" "%s" %d\n' % (cm_manager[0].group_name, cm_manager[0].resource, cm_manager[0].type)) + src_str.append('vpn resource groupexcludeditem network "%s" "%s" %d\n' % (cm_manager[0].group_name, + cm_manager[0].resource, + cm_manager[0].type)) result_diff = "" running_params = [] - for device in device_list: - params = [json.dumps({"cmd":"show vpn resource groupitem network", "vsite_name":"%s" % device[0]['service_name']}), json.dumps({"cmd":"show vpn resource groupexcludeditem network", "vsite_name":"%s" % device[0]['service_name']})] - running_params.append({'params':params, 'device_list':device[0]['device_list']}) + for device in device_list: + params = [json.dumps( + {"cmd": "show vpn resource groupitem network", "vsite_name": "%s" % device[0]['service_name']}), + json.dumps({"cmd": "show vpn resource groupexcludeditem network", + "vsite_name": "%s" % device[0]['service_name']})] + running_params.append({'params': params, 'device_list': device[0]['device_list']}) new_url = '/rest/ag/global/cli_extend' - parse_list = ['vpn resource groupitem network "%(resourcegroup)s" "%(netresource)s" %(type)d\n', 'vpn resource groupexcludeditem network "%(resourcegroup)s" "%(netresource)s" %(type)d\n'] + parse_list = ['vpn resource groupitem network "%(resourcegroup)s" "%(netresource)s" %(type)d\n', + 'vpn resource groupexcludeditem network "%(resourcegroup)s" "%(netresource)s" %(type)d\n'] result_diff = CM_check(new_url, running_params, parse_list, src_str) return result_diff def _perform_Init_DeviceList(self, options): - #{u'__pk_list': [{u'id': 1}], u'device_list': u'10.8.2.55', u'option': u'delete'} + # {u'__pk_list': [{u'id': 1}], u'device_list': u'10.8.2.55', u'option': u'delete'} if options['option'] not in ['delete', 'insert']: raise ModelQueryException(CLICmdError(__('Internal error.'))) else: - pk_dict = {'table':'cm_vpn_resource', 'id':options['__pk_list'][0]['id']} + pk_dict = {'table': 'cm_vpn_resource', 'id': options['__pk_list'][0]['id']} update_cm_role_devicelist(options['device_list'].split(','), pk_dict, options['option'], None) return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (working copy) @@ -1,32 +1,33 @@ -# hash restapi passowrd with md5 by Jason Chou -import hashlib -from django.utils.encoding import force_bytes +# hash restapi password with md5 by Jason Chou +import datetime +import re +import time +import uuid -from hive.imports.model import * +import elasticsearch +from apscheduler.schedulers.background import BackgroundScheduler +from django.db.models.query import QuerySet +from django.utils.encoding import force_bytes from django.utils.translation import gettext_lazy as _ -from hive.utils import get_device_type, standard_model_type -from cm.lib.postgres_db import DB + from cm.lib.communication import send_https_rest_request, modify_url, L -from cm.lib.elk import put_template, get_elk -from cm.lib.parse_configfile import parse_vsite_from_configfile, DEFAULT_CONFIG_FILE_PATH, DEFAULT_VSITE_PATH -from hive.model.query import mark_expire_all -from hive.model.loading import get_model +from cm.lib.elk import get_elk +from cm.lib.libbasic_operation import send_command_to_device, send_cli_to_device, insert_default_tmpkey, \ + delete_all_tmpkey +from cm.lib.postgres_db import DB from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_backup -from cm.models.tasking import GLOBAL_TASK -from cm.models.system.license import check_license -from apscheduler.schedulers.background import BackgroundScheduler -import uuid -import time, os, re from cm.models.audit.user.user_body import body_login_detail, body_device_vsite -import json -import elasticsearch +from cm.models.system.license import check_license +from cm.models.tasking import GLOBAL_TASK from djproject.an_settings import * -from cm.lib.libbasic_operation import send_command_to_device, send_cli_to_device, insert_default_tmpkey, delete_all_tmpkey -from django.db.models.query import QuerySet -import datetime +from hive.imports.model import * +from hive.model.loading import get_model +from hive.model.query import mark_expire_all +from hive.utils import get_device_type, standard_model_type -__=_ +__ = _ + def get_rest_info_from_device(device_name): db = DB.get_connected_db() fetchall_sql = '''SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, name, enable_password FROM device where name='%s' ''' % device_name @@ -34,10 +35,12 @@ db.close() if not data: return None - key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', 'enable_password'] + key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', + 'enable_password'] result = [dict(zip(key, each)) for each in data] return result + def get_rest_info_from_device_id(device_id): db = DB.get_connected_db() fetchall_sql = '''SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, name, enable_password FROM device where id='%s' ''' % device_id @@ -45,10 +48,12 @@ db.close() if not data: return None - key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', 'enable_password'] + key = ['ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'name', + 'enable_password'] result = [dict(zip(key, each)) for each in data] return result + def get_rest_info_from_host(device_id): db = DB.get_connected_db() fetchall_sql = '''SELECT name, ip, restapi_port, restapi_username, restapi_password, connection FROM host where id='%s' ''' % device_id @@ -60,16 +65,17 @@ result = [dict(zip(key, each)) for each in data] return result + class IPPools(ANModel): - #index = IntegerField(hidden=True, primary_key=True) - asso = AssoField2(verbose_name=_('Device'), tgt='device_mgmt.device.Device.asso_ip_pool', - mul='*', pos='right', optional=True, hidden=True) + # index = IntegerField(hidden=True, primary_key=True) + asso = AssoField2(verbose_name=_('Device'), tgt='device_mgmt.device.Device.asso_ip_pool', + mul='*', pos='right', optional=True, hidden=True) ip_pool_range = CharField(verbose_name=_('IP Pool Range'), primary_key=True) class Meta: verbose_name = 'IP Pools' - list_config_options={ - 'add_button_hide': True, 'delete_button_hide': True, 'click_enable':False + list_config_options = { + 'add_button_hide': True, 'delete_button_hide': True, 'click_enable': False } # repr_from_instance = lambda instance: '%s' % instance.ip_pool_range # pk_to_id = lambda instance: '%s' % instance.ip_pool_range @@ -90,100 +96,112 @@ # i += 1 return QuerySet(self._model, result) + class Device(ANModel): - #Now you can using the 'default' fieldgroup to defined the id, name and type ..., or defined the field without fieldgroup + # Now you can using the 'default' fieldgroup to defined the id, name and type ..., or defined the field without fieldgroup default = FieldGroup(verbose_name=_('Base Settings'), level=BASIC, fields={ 'zone': CharField(verbose_name=_('Zone'), optional=True, configurable=False, editable=False), # 'zone' : AssoField2(verbose_name=_('Zone'), tgt='device_mgmt.zone.Zone.device', # mul='*', pos='right', configurable=False, editable=False, optional=True), - 'id' : CharField(verbose_name='ID', primary_key=True, configurable=False, optional=True, editable=False, length='1..64'), - 'name' : CharField(verbose_name='Device Name', primary_key=True, editable=False, length='1..32'), - 'type' : CharField(verbose_name='Device Type', primary_key=True, editable=False), - 'ip' : IPAddressField(verbose_name='IP Address', editable=False), - 'protocol' : CharField(verbose_name='Protocol', editable=False), - 'group_name_from' : CharField(verbose_name='Group Name From', editable=False, default='new', optional=True), - 'restapi_port' : PortField(verbose_name='RESTful API Port', default=9997, editable=False), - 'restapi_account' : GroupField(verbose_name='RESTful API Account', editable=True, custom_display=lambda val: 'username: '+val['restapi_username'] + '
password: ********', fields={ - 'restapi_username' : CharField(verbose_name='Username', length='1..16'), - 'restapi_password' : PasswordField(verbose_name='Password', length='1..32') + 'id': CharField(verbose_name='ID', primary_key=True, configurable=False, optional=True, editable=False, + length='1..64'), + 'name': CharField(verbose_name='Device Name', primary_key=True, editable=False, length='1..32'), + 'type': CharField(verbose_name='Device Type', primary_key=True, editable=False), + 'ip': IPAddressField(verbose_name='IP Address', editable=False), + 'protocol': CharField(verbose_name='Protocol', editable=False), + 'group_name_from': CharField(verbose_name='Group Name From', editable=False, default='new', optional=True), + 'restapi_port': PortField(verbose_name='RESTful API Port', default=9997, editable=False), + 'restapi_account': GroupField(verbose_name='RESTful API Account', editable=True, + custom_display=lambda val: 'username: ' + val[ + 'restapi_username'] + '
password: ********', fields={ + 'restapi_username': CharField(verbose_name='Username', length='1..16'), + 'restapi_password': PasswordField(verbose_name='Password', length='1..32') }), - 'console_account' : GroupField(verbose_name='Web Console Account', editable=True, custom_display=lambda val: 'username: '+val['console_username'] + '
password: ********', fields={ - 'console_username' : CharField(verbose_name='Username', length='1..16'), - 'console_password' : PasswordField(verbose_name='Password', length='1..32') + 'console_account': GroupField(verbose_name='Web Console Account', editable=True, + custom_display=lambda val: 'username: ' + val[ + 'console_username'] + '
password: ********', fields={ + 'console_username': CharField(verbose_name='Username', length='1..16'), + 'console_password': PasswordField(verbose_name='Password', length='1..32') }), - 'connection' : BooleanField(verbose_name='Connection', editable=False, configurable=False, display=('connection', 'unconnection'), optional=True), - 'status' : BooleanField(verbose_name='Status', configurable=False, editable=False, display=('new', 'active'), optional=True), - 'log_enable' : BooleanField(verbose_name='Collection Log', display=('enable', 'disable'), default=False), + 'connection': BooleanField(verbose_name='Connection', editable=False, configurable=False, + display=('connection', 'unconnection'), optional=True), + 'status': BooleanField(verbose_name='Status', configurable=False, editable=False, display=('new', 'active'), + optional=True), + 'log_enable': BooleanField(verbose_name='Collection Log', display=('enable', 'disable'), default=False), # 'snmp_enable' : BooleanField(verbose_name='Collection SNMP', display=('enable', 'disable'), default=False), - 'snmp_general' : GroupField(verbose_name='SNMP General', optional=True, editable=True, fields={ - 'snmp_enable' : BooleanField(verbose_name='SNMP Enable'), - 'community' : CharField(verbose_name='Community', optional=True) - }), - 'license_key' : CharField(verbose_name='License Key', editable=False, configurable=False, optional=True, length='1..128'), - 'webui_port' : PortField(verbose_name='Webui Port', default=8888, editable=True), - 'enable_password' : PasswordField(verbose_name='Enable Password', length='1..32', optional=True), - 'backup_enable' : BooleanField(verbose_name='Backup Configuraion', display=('enable', 'disable'), default=False, optional=True), - }) + 'snmp_general': GroupField(verbose_name='SNMP General', optional=True, editable=True, fields={ + 'snmp_enable': BooleanField(verbose_name='SNMP Enable'), + 'community': CharField(verbose_name='Community', optional=True) + }), + 'license_key': CharField(verbose_name='License Key', editable=False, configurable=False, optional=True, + length='1..128'), + 'webui_port': PortField(verbose_name='Webui Port', default=8888, editable=True), + 'enable_password': PasswordField(verbose_name='Enable Password', length='1..32', optional=True), + 'backup_enable': BooleanField(verbose_name='Backup Configuraion', display=('enable', 'disable'), default=False, + optional=True), + }) advance_settings = FieldGroup(verbose_name=_('Advance Settings'), level=BASIC, fields={ - 'gateway_domain' : CharField(verbose_name='Gateway Domain', optional=True, length='1..128'), - 'location' : CharField(verbose_name='Location', optional=True, length='1..128'), - 'firewall_ip' : IPAddressField(verbose_name='Firewall IP', editable=True, optional=True), - 'intranet_ip' : IPAddressField(verbose_name='Intranet IP', editable=True, optional=True), - 'own' : CharField(verbose_name='Belong to', optional=True, length='1..64'), - 'extend_fields' : CharField(verbose_name='Extend Fields', length='1..8192', optional=True, configurable=False) - }) + 'gateway_domain': CharField(verbose_name='Gateway Domain', optional=True, length='1..128'), + 'location': CharField(verbose_name='Location', optional=True, length='1..128'), + 'firewall_ip': IPAddressField(verbose_name='Firewall IP', editable=True, optional=True), + 'intranet_ip': IPAddressField(verbose_name='Intranet IP', editable=True, optional=True), + 'own': CharField(verbose_name='Belong to', optional=True, length='1..64'), + 'extend_fields': CharField(verbose_name='Extend Fields', length='1..8192', optional=True, configurable=False) + }) version = FieldGroup(writable=False, verbose_name=_('Version'), level=STATS, visible_edit=True, fields={ - 'host_name' : CharField(verbose_name="Host name"), - 'system_cpu' : CharField(verbose_name="System CPU"), - 'system_ram' : CharField(verbose_name="System RAM"), - 'system_boot_time' : CharField(verbose_name="System boot time"), - 'system_up_time' : CharField(verbose_name="System up time"), - 'platform_bld_date' : CharField(verbose_name="Platform Bld Date"), - 'ssl_hw' : CharField(verbose_name="SSL HW"), - 'compression_hw' : CharField(verbose_name="Compression HW"), - 'network_interface' : CharField(verbose_name="Network Interface"), - 'model' : CharField(verbose_name="Model"), - 'serial_number' : CharField(verbose_name="Serial Number"), - 'license_feature' : CharField(verbose_name="License Featrues"), - 'license_date' : CharField(verbose_name="License Date"), - 'build_version' : CharField(verbose_name="Build Version"), - 'license_limit' : CharField(verbose_name="License Limit"), + 'host_name': CharField(verbose_name="Host name"), + 'system_cpu': CharField(verbose_name="System CPU"), + 'system_ram': CharField(verbose_name="System RAM"), + 'system_boot_time': CharField(verbose_name="System boot time"), + 'system_up_time': CharField(verbose_name="System up time"), + 'platform_bld_date': CharField(verbose_name="Platform Bld Date"), + 'ssl_hw': CharField(verbose_name="SSL HW"), + 'compression_hw': CharField(verbose_name="Compression HW"), + 'network_interface': CharField(verbose_name="Network Interface"), + 'model': CharField(verbose_name="Model"), + 'serial_number': CharField(verbose_name="Serial Number"), + 'license_feature': CharField(verbose_name="License Featrues"), + 'license_date': CharField(verbose_name="License Date"), + 'build_version': CharField(verbose_name="Build Version"), + 'license_limit': CharField(verbose_name="License Limit"), }) ip_pools = FieldGroup(writable=False, verbose_name=_('IP Pools'), level=BASIC, fields={ - 'asso_ip_pool' : AssoField2(verbose_name='IP Pools', - tgt='device_mgmt.device.IPPools.asso', mul='1', pos='left', optional=True), - }) - device_group = AssoField2(verbose_name=_('Device Group'), tgt='device_mgmt.device_group.DeviceGroup.device_list', mul='*', pos='right') + 'asso_ip_pool': AssoField2(verbose_name='IP Pools', + tgt='device_mgmt.device.IPPools.asso', mul='1', pos='left', optional=True), + }) + device_group = AssoField2(verbose_name=_('Device Group'), tgt='device_mgmt.device_group.DeviceGroup.device_list', + mul='*', pos='right') class Meta: verbose_name = 'Managed Devices' show_im_export_button = False - list_config_options={ - 'columns':[ - {'name':'name'}, - {'name':'type'}, - {'name':'ip'}, - {'name':'restapi_port'}, - {'name':'restapi_account'}, - {'name':'connection'}, - {'name':'status'}, - {'name':'license_key'}, - {'name':'asso_ip_pool'}, - {'name':'log_enable'}, + list_config_options = { + 'columns': [ + {'name': 'name'}, + {'name': 'type'}, + {'name': 'ip'}, + {'name': 'restapi_port'}, + {'name': 'restapi_account'}, + {'name': 'connection'}, + {'name': 'status'}, + {'name': 'license_key'}, + {'name': 'asso_ip_pool'}, + {'name': 'log_enable'}, # {'name':'snmp_enable'}, - {'name':'gateway_domain'}, - {'name':'location'}, - {'name':'firewall_ip'}, - {'name':'intranet_ip'}, - {'name':'webui_port'}, - ], 'select_button_enable' : True, 'export_button_enable' : True} - help_text = _("When registering, please make sure a RESTful API account has already been configured on target device and the RESTful API service has been turned on 'https' as well. If not, please refer to CLI handbook, using command 'restapi on [protocol] [port]' and 'user [password] [level]' to configure them first." ) + {'name': 'gateway_domain'}, + {'name': 'location'}, + {'name': 'firewall_ip'}, + {'name': 'intranet_ip'}, + {'name': 'webui_port'}, + ], 'select_button_enable': True, 'export_button_enable': True} + help_text = _( + "When registering, please make sure a RESTful API account has already been configured on target device and the RESTful API service has been turned on 'https' as well. If not, please refer to CLI handbook, using command 'restapi on [protocol] [port]' and 'user [password] [level]' to configure them first.") class Activate(Action): verbose_name = _('Activate device') action_name = _('Activate') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -215,13 +233,13 @@ process_title = "" forever = False config_change = True - instance_mul = '1' + instance_mul = '1' class Refresh(Action): verbose_name = _('Refresh') action_name = _('Refresh') btn_class = 'btn-danger slow-action' - #confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" + # confirm_msg = "Are you sure you want to reset the configurations of selected bond interfaces?" alert_msg = "" process_title = "" forever = False @@ -293,7 +311,10 @@ fetchall_sql = '''SELECT id, zone, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, console_username, console_password, connection, status, version, license_key, gateway_domain, location, firewall_ip, intranet_ip, type, extend_fields, own, log_enable, snmp_general, webui_port, device_group, enable_password, extend FROM device order by create_time''' data = db.fetchall(fetchall_sql) db.close() - key = ['id', 'zone', 'name', 'ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'console_username', 'console_password', 'connection', 'status', 'version', 'license_key', 'gateway_domain', 'location', 'firewall_ip', 'intranet_ip', 'type', 'extend_fields', 'own', 'log_enable', 'snmp_general', 'webui_port', 'device_group', 'enable_password', 'extend'] + key = ['id', 'zone', 'name', 'ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', + 'console_username', 'console_password', 'connection', 'status', 'version', 'license_key', + 'gateway_domain', 'location', 'firewall_ip', 'intranet_ip', 'type', 'extend_fields', 'own', + 'log_enable', 'snmp_general', 'webui_port', 'device_group', 'enable_password', 'extend'] result = [dict(zip(key, each)) for each in data] for each in result: del each['extend'] @@ -304,7 +325,7 @@ each['restapi_account'] = {'restapi_username': each['restapi_username'], 'restapi_password': None} del each['restapi_username'] del each['restapi_password'] - each['console_account'] = {'console_username':each['console_username'], 'console_password': None} + each['console_account'] = {'console_username': each['console_username'], 'console_password': None} del each['console_username'] del each['console_password'] if each['connection'] == 'connected': @@ -320,22 +341,22 @@ if each['version']: replace_result = each['version'].replace('\\n', '\n') rtn = cli_parse(replace_result, [ - RegexParser('Host name : (.+?)\n', MATCHONE), - RegexParser('System CPU : (.+?)\n', MATCHONE), - RegexParser('System RAM : (.+?)\n', MATCHONE), - RegexParser('System boot time : (.+?)\n', MATCHONE), - RegexParser('System up time : (.+?)\n', MATCHONE), - RegexParser('Platform Bld Date : (.+?)\n', MATCHONE), - RegexParser('SSL HW : (.+?)\n', MATCHONE), - RegexParser('Compression HW : (.+?)\n', MATCHONE), - RegexParser('Network Interface : (.+?)\n', MATCHONE), - RegexParser('Model : (.+?)\n', MATCHONE), - RegexParser('Serial Number : (.+?)\n', MATCHONE), - RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), - RegexParser('Licensed Limits : (.+?)\n', MATCHONE), - RegexParser('Licensed Features : (.*)', MATCHONE, reflags=re.S), - RegexParser('License Date : (.*?)\n', MATCHONE), - ]) + RegexParser('Host name : (.+?)\n', MATCHONE), + RegexParser('System CPU : (.+?)\n', MATCHONE), + RegexParser('System RAM : (.+?)\n', MATCHONE), + RegexParser('System boot time : (.+?)\n', MATCHONE), + RegexParser('System up time : (.+?)\n', MATCHONE), + RegexParser('Platform Bld Date : (.+?)\n', MATCHONE), + RegexParser('SSL HW : (.+?)\n', MATCHONE), + RegexParser('Compression HW : (.+?)\n', MATCHONE), + RegexParser('Network Interface : (.+?)\n', MATCHONE), + RegexParser('Model : (.+?)\n', MATCHONE), + RegexParser('Serial Number : (.+?)\n', MATCHONE), + RegexParser('([ArrayOS|InfosecOS].+?)\n', MATCHONE), + RegexParser('Licensed Limits : (.+?)\n', MATCHONE), + RegexParser('Licensed Features : (.*)', MATCHONE, reflags=re.S), + RegexParser('License Date : (.*?)\n', MATCHONE), + ]) each['host_name'] = rtn[0][0] if rtn[0] else '' each['system_cpu'] = rtn[1][0] if rtn[1] else '' each['system_ram'] = rtn[2][0] if rtn[2] else '' @@ -350,7 +371,8 @@ each['build_version'] = rtn[11][0] if rtn[11] else '' each['license_limit'] = rtn[12][0] if rtn[12] else '' each['license_date'] = rtn[-1][0] if rtn[-1] else '' - each['license_feature'] = re.sub("\s+", ", ", rtn[-2][0].split('License Key')[0]).strip(', ') if rtn[-2] else '' + each['license_feature'] = re.sub("\s+", ", ", rtn[-2][0].split('License Key')[0]).strip(', ') if \ + rtn[-2] else '' del each['version'] if each['snmp_general']: @@ -365,7 +387,8 @@ try: if len(license_date) > 8: license_date = license_date[2:] - each['license_date'] = "Expires on " + time.strftime("%b %d %G", time.strptime(license_date, "%Y%m%d")) + each['license_date'] = "Expires on " + time.strftime("%b %d %G", + time.strptime(license_date, "%Y%m%d")) except Exception as e: each['license_date'] = "Permanent" each['group_name_from'] = '' @@ -375,7 +398,7 @@ def _filter(self, filter_dict): if 'device_group' in filter_dict: return self.all().filter(filter_dict) - return self.all() + return self.all() def _get_stats(self): return self._get_query_set() @@ -390,7 +413,7 @@ # data = {"cmd" : "show running"} # new_url = '/rest/%s/cli_extend' % get_device_type(device_info[0]['type'].lower()) # running_params = json.dumps(data) - + # try: # rest_response_data = send_https_rest_request('POST', new_url, running_params, ip, port, username, password) # except: @@ -399,7 +422,7 @@ # config_result = json.loads(rest_response_data['body'])['contents'] # else: # raise ModelQueryException(CLICmdError(__("The request is not found!"))) - + # device_path = os.path.join(DEFAULT_CONFIG_FILE_PATH, 'device') # if not os.path.isdir(device_path): # os.makedirs(device_path) @@ -439,16 +462,19 @@ def _insert(self, instance): data = instance.get_field_dict() check_license(data["type"]) + def insert_ip_pool(device, device_id): response_data = send_cli_to_device(device, 'show running all "iprange dynamic"', mode='enable') if response_data['message'] == 'success': rtn = cli_parse(response_data['data'], [ - RegexParser('vpn netpool iprange dynamic "(?P.*?)" (?P.*?) (?P.*?) ""', MATCHALL) + RegexParser('vpn netpool iprange dynamic "(?P.*?)" (?P.*?) (?P.*?) ""', + MATCHALL) ]) db = DB.get_connected_db() for value in rtn[0]: ip_pool_range = value['first_ip'] + '-' + value['last_ip'] - save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES " + "('%s', '%s')" % (ip_pool_range, device_id) + save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES " + "('%s', '%s')" % ( + ip_pool_range, device_id) db.execute_sql(save_sql) db.close() elif response_data['message'] == 'requestfailed': @@ -456,6 +482,7 @@ else: raise ModelQueryException(CLICmdError(__('Activation Failed'))) return + data['id'] = uuid.uuid1() data['zone'] = "default" data['ip'] = data['ip'].values()[0] @@ -466,7 +493,7 @@ res_data = db.fetchall(fetchall_sql) if res_data and res_data[0]: raise ModelQueryException(CLICmdError(__('Device Group "%s" is exist.' % data['device_group']))) - + insert_sql = "INSERT INTO device_group(name) values ('%s')" % data['device_group'] db.execute_sql(insert_sql) db.close() @@ -485,10 +512,12 @@ rest_response_data = send_cli_to_device(data, "show version", mode='enable') else: new_url = modify_url('/rest/device_type/system/SystemInfo/version', data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', data['ip'], data['restapi_port'], data['restapi_username'], data['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', data['ip'], data['restapi_port'], + data['restapi_username'], data['restapi_password']) except Exception as e: if 'timed out' in str(e): - raise ModelQueryException(CLICmdError(__('Register timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) + raise ModelQueryException(CLICmdError( + __('Register timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) raise ModelQueryException(CLICmdError(__('Register Failed.'))) else: if data['protocol'] == 'xmlrpc': @@ -497,19 +526,21 @@ elif rest_response_data['message'] == 'requestfailed': raise ModelQueryException(CLICmdError(__('Register Failed.'))) else: - raise ModelQueryException(CLICmdError(__('Register Failed! Please make sure the XML RPC configuration is correct!'))) + raise ModelQueryException( + CLICmdError(__('Register Failed! Please make sure the XML RPC configuration is correct!'))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) data['version'] = rest_body['SystemInfo']['version'] elif rest_response_data['status'] == 401: - raise ModelQueryException(CLICmdError(__('Register Failed! Please make sure the RESTful API username/password is correct!'))) + raise ModelQueryException(CLICmdError( + __('Register Failed! Please make sure the RESTful API username/password is correct!'))) else: raise ModelQueryException(CLICmdError(__('Refresh Version Failed!'))) rtn = cli_parse(data['version'], [ - RegexParser('License Key : (.+?)\n', MATCHONE), - RegexParser('Model : (.+?)\n', MATCHONE), - ]) + RegexParser('License Key : (.+?)\n', MATCHONE), + RegexParser('Model : (.+?)\n', MATCHONE), + ]) if rtn[0]: data['license_key'] = rtn[0][0] data['status'] = 'active' @@ -517,14 +548,15 @@ data['license_key'] = None data['status'] = 'new' model_str = rtn[1][0] if rtn[1] else None + def get_device_model(model_string): if model_string: - #Array AVX vAPV - #Array AVX vxAG - #Array APV xxx - #Array vAPV - #Array AG xxx - #Array vxAG + # Array AVX vAPV + # Array AVX vxAG + # Array APV xxx + # Array vAPV + # Array AG xxx + # Array vxAG if model_string.find('Array APV') != -1: return 'APV' if model_string.find('Array AG') != -1: @@ -533,15 +565,15 @@ return 'vAPV' if model_string.find('vxAG') != -1: return 'vxAG' - #INFOSEC Ccypher vNSAE - #INFOSEC Ccypher vNetOpti - #INFOSEC Ccypher vNetGate - #INFOSEC vNSAE - #INFOSEC vNetOpti - #INFOSEC vNetGate - #INFOSEC NSAE xxx - #INFOSEC NetOpti xxx - #INFOSEC NetGate xxx + # INFOSEC Ccypher vNSAE + # INFOSEC Ccypher vNetOpti + # INFOSEC Ccypher vNetGate + # INFOSEC vNSAE + # INFOSEC vNetOpti + # INFOSEC vNetGate + # INFOSEC NSAE xxx + # INFOSEC NetOpti xxx + # INFOSEC NetGate xxx if model_string.find('vNetOpti') != -1: return 'vNetOpti' if model_string.find('vNSAE') != -1: @@ -554,8 +586,8 @@ return 'NSAE' if model_string.find('INFOSEC NetGate') != -1: return 'NetGate' - #Array vASF - #Array ASF + # Array vASF + # Array ASF if model_string.find('Array ASF') != -1: return 'ASF' if model_string.find('vASF') != -1: @@ -565,23 +597,27 @@ if model_string.find('vNetWAF') != -1: return 'vNetWAF' return None + model = get_device_model(model_str) if not model: - raise ModelQueryException(CLICmdError(__('Register Failed! Invalid model! Only %s are supported.')%', '.join(DEVICE_STD_LIST))) + raise ModelQueryException(CLICmdError( + __('Register Failed! Invalid model! Only %s are supported.') % ', '.join(DEVICE_STD_LIST))) if model != data['type']: - raise ModelQueryException(CLICmdError(__('Register Failed! Current device model is %s but you choose %s.')%(model, data['type']))) + raise ModelQueryException(CLICmdError( + __('Register Failed! Current device model is %s but you choose %s.') % (model, data['type']))) # else: # raise ModelQueryException(CLICmdError(__('Register Failed! Please make sure device model is right.'))) db = DB.get_connected_db() - save_sql = "INSERT INTO device(id, zone, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, console_username, console_password, connection, status, license_key, gateway_domain, location, firewall_ip, intranet_ip, version, type, own, log_enable, webui_port, device_group, enable_password) Values " + "('%(id)s', '%(zone)s', '%(name)s', '%(ip)s', '%(protocol)s', '%(restapi_port)s', '%(restapi_username)s', '%(restapi_password)s', '%(console_username)s', '%(console_password)s', 'connected', '%(status)s', '%(license_key)s', '%(gateway_domain)s', '%(location)s', '%(firewall_ip)s', '%(intranet_ip)s', '%(version)s', '%(type)s', '%(own)s', '%(log_enable)d', '%(webui_port)d', '%(device_group)s', '%(enable_password)s')"%data + save_sql = "INSERT INTO device(id, zone, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, console_username, console_password, connection, status, license_key, gateway_domain, location, firewall_ip, intranet_ip, version, type, own, log_enable, webui_port, device_group, enable_password) Values " + "('%(id)s', '%(zone)s', '%(name)s', '%(ip)s', '%(protocol)s', '%(restapi_port)s', '%(restapi_username)s', '%(restapi_password)s', '%(console_username)s', '%(console_password)s', 'connected', '%(status)s', '%(license_key)s', '%(gateway_domain)s', '%(location)s', '%(firewall_ip)s', '%(intranet_ip)s', '%(version)s', '%(type)s', '%(own)s', '%(log_enable)d', '%(webui_port)d', '%(device_group)s', '%(enable_password)s')" % data db.execute_sql(save_sql) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - insert_config_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type, device_id) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (data['name'], create_time, create_time, 'device', '', data['type'], data['id']) + insert_config_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type, device_id) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( + data['name'], create_time, create_time, 'device', '', data['type'], data['id']) db.execute_sql(insert_config_sql) db.close() - #file_path = self.get_config_from_device(data['ip'], data['restapi_port'], data['restapi_username'], data['restapi_password'], data['type'], data['name']) + # file_path = self.get_config_from_device(data['ip'], data['restapi_port'], data['restapi_username'], data['restapi_password'], data['type'], data['name']) ### XXX : Need to check if ip pool is used in AMP 3.0 if data['type'].lower() in VPN_TYPE_LIST: try: @@ -592,12 +628,12 @@ db.execute_sql(delete_sql) db.close() raise ModelQueryException(CLICmdError(__('Get %s ip pool failed!' % data['type']))) - #parse vsite name, ip... from global config - #parse_vsite_config_from_configfile(file_path, data['name']) - #parse all vsite config from 'show running all', and write to vsite file - #parse_vsite_from_configfile(file_path, data['name']) + # parse vsite name, ip... from global config + # parse_vsite_config_from_configfile(file_path, data['name']) + # parse all vsite config from 'show running all', and write to vsite file + # parse_vsite_from_configfile(file_path, data['name']) - #backup the config file + # backup the config file if data["backup_enable"]: date = datetime.datetime.now() + datetime.timedelta(seconds=1) device_info = get_rest_info_from_device(data['name'])[0] @@ -606,11 +642,13 @@ GLOBAL_TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} - description = " Backup config create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' + description = " Backup config create time: " + datetime.datetime.now().strftime( + '%Y-%m-%d %H:%M:%S') + '\n' new_schedule['func'] = 'config_backup' new_schedule['args'] = [task_name, device_info, "Backup when adding device."] new_schedule_str = json.dumps(new_schedule) - add_job_into_database(task_name, 'config_backup', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=data['name']) + add_job_into_database(task_name, 'config_backup', description, 'date', 'waiting', + date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=data['name']) sched = BackgroundScheduler() sched.add_job(config_backup, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) @@ -621,10 +659,11 @@ # insert device default template key-value insert_default_tmpkey(data['ip']) - #create index template + # create index template # put_template(data['ip']) # oper_log('info', 'device', 'Added a new device %s' % (data['name'])) return + # #set log host # if data['log_enable']: # cm = __import__("cm") @@ -654,7 +693,7 @@ def _update_log_enable(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET log_enable='%d' WHERE id='%s'" %(instance.log_enable, device_id) + update_sql = "UPDATE device SET log_enable='%d' WHERE id='%s'" % (instance.log_enable, device_id) db.execute_sql(update_sql) db.close() return @@ -662,7 +701,8 @@ def _update_device_group(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET device_group='%s' WHERE id='%s'" %(instance.device_group[0]['name'], device_id) + update_sql = "UPDATE device SET device_group='%s' WHERE id='%s'" % (instance.device_group[0]['name'], + device_id) db.execute_sql(update_sql) db.close() mark_expire_all(get_model('cm', ['device_mgmt', 'device_group', 'DeviceGroup'])) @@ -673,12 +713,14 @@ try: send_cli_to_device(each, cmd) except Exception as e: - raise ModelQueryException(CLICmdError(__('Set device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException( + CLICmdError(__('Set device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) else: if each['protocol'] == 'xmlrpc' or "nsae" in each['type'].lower() or "netopti" in each['type'].lower(): response_data = send_cli_to_device(each, "show snmp", mode='enable') if response_data['message'] != 'success': - raise ModelQueryException(CLICmdError(__('Get device<%s> snmp failed. %s' % (each['ip_address'], response_data['data'])))) + raise ModelQueryException(CLICmdError( + __('Get device<%s> snmp failed. %s' % (each['ip_address'], response_data['data'])))) else: try: result = cli_parse(response_data['data'], [ @@ -701,18 +743,23 @@ if result[6]: snmp_general["v3user"] = result[6]['v3user'].replace('\"', '') db = DB.get_connected_db() - update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" %(json.dumps(snmp_general), device_id) + update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" % ( + json.dumps(snmp_general), device_id) db.execute_sql(update_sql) db.close() except Exception as e: - raise ModelQueryException(CLICmdError(__('Update device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException( + CLICmdError(__('Update device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) else: if each['type'].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST: try: new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/general', each['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], each['restapi_port'], each['restapi_username'], each['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], + each['restapi_port'], each['restapi_username'], + each['restapi_password']) except Exception as e: - raise ModelQueryException(CLICmdError(__('Get device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException( + CLICmdError(__('Get device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -730,25 +777,32 @@ if 'v3_user' in rest_body['SNMPGeneral']: for user in rest_body['SNMPGeneral']['v3_user']: if 'username' in user and 'auth_password' in user and 'level' in user: - v3user = user['username']+' '+user['auth_password']+' '+user['level'] + v3user = user['username'] + ' ' + user['auth_password'] + ' ' + \ + user['level'] if 'priv_password' in user: - v3user += ' '+user['priv_password'] + v3user += ' ' + user['priv_password'] snmp_general['v3user'] = v3user break db = DB.get_connected_db() - update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" %(json.dumps(snmp_general), device_id) + update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" % ( + json.dumps(snmp_general), device_id) db.execute_sql(update_sql) db.close() except Exception as e: - raise ModelQueryException(CLICmdError(__('Update device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Update device<%s> snmp failed. %s' % (each['ip_address'], str(e))))) elif each['type'].lower() in VPN_TYPE_LIST: snmp_general = {} try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_version', each['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], each['restapi_port'], each['restapi_username'], each['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_version', + each['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], + each['restapi_port'], each['restapi_username'], + each['restapi_password']) except Exception as e: - raise ModelQueryException(CLICmdError(__('Get device<%s> SNMP version failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Get device<%s> SNMP version failed. %s' % (each['ip_address'], str(e))))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -757,13 +811,18 @@ if 'snmp_version' in rest_body['SNMPGeneral']: snmp_general['snmp_version'] = rest_body['SNMPGeneral']['snmp_version'] except Exception as e: - raise ModelQueryException(CLICmdError(__('Set device<%s> snmp version failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Set device<%s> snmp version failed. %s' % (each['ip_address'], str(e))))) try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_enable', each['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], each['restapi_port'], each['restapi_username'], each['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/snmp_enable', + each['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], + each['restapi_port'], each['restapi_username'], + each['restapi_password']) except Exception as e: - raise ModelQueryException(CLICmdError(__('Get device<%s> SNMP enable failed. %s' % (each['iip_addressp'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Get device<%s> SNMP enable failed. %s' % (each['iip_addressp'], str(e))))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -772,13 +831,18 @@ if 'snmp_enable' in rest_body['SNMPGeneral']: snmp_general['snmp_enable'] = rest_body['SNMPGeneral']['snmp_enable'] except Exception as e: - raise ModelQueryException(CLICmdError(__('Set device<%s> snmp enable failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Set device<%s> snmp enable failed. %s' % (each['ip_address'], str(e))))) try: - new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/community', each['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], each['restapi_port'], each['restapi_username'], each['restapi_password']) + new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/community', + each['type']) + rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], + each['restapi_port'], each['restapi_username'], + each['restapi_password']) except Exception as e: - raise ModelQueryException(CLICmdError(__('Get device<%s> SNMP community failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Get device<%s> SNMP community failed. %s' % (each['ip_address'], str(e))))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -787,12 +851,16 @@ if 'community' in rest_body['SNMPGeneral']: snmp_general['community'] = rest_body['SNMPGeneral']['community'] except Exception as e: - raise ModelQueryException(CLICmdError(__('Set device<%s> snmp community failed. %s' % (each['ip_address'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Set device<%s> snmp community failed. %s' % (each['ip_address'], str(e))))) try: new_url = modify_url('/rest/device_type/admintools/snmp/SNMPGeneral/v3_user', each['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], each['restapi_port'], each['restapi_username'], each['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', each['ip_address'], + each['restapi_port'], each['restapi_username'], + each['restapi_password']) except Exception as e: - raise ModelQueryException(CLICmdError(__('Get device<%s> SNMP v3_user failed. %s' % (each['name'], str(e))))) + raise ModelQueryException( + CLICmdError(__('Get device<%s> SNMP v3_user failed. %s' % (each['name'], str(e))))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -801,16 +869,19 @@ if 'v3_user' in rest_body['SNMPGeneral']: for user in rest_body['SNMPGeneral']['v3_user']: if 'username' in user and 'auth_password' in user and 'level' in user: - v3user = user['username']+' '+user['auth_password']+' '+user['level'] + v3user = user['username'] + ' ' + user['auth_password'] + ' ' + \ + user['level'] if 'priv_password' in user: - v3user += ' '+user['priv_password'] + v3user += ' ' + user['priv_password'] snmp_general['v3user'] = v3user break except Exception as e: - raise ModelQueryException(CLICmdError(__('Set device<%s> snmp v3_user failed. %s' % (each['name'], str(e))))) + raise ModelQueryException(CLICmdError( + __('Set device<%s> snmp v3_user failed. %s' % (each['name'], str(e))))) db = DB.get_connected_db() - update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" %(json.dumps(snmp_general), device_id) + update_sql = "UPDATE device SET snmp_general='%s' WHERE id='%s'" % (json.dumps(snmp_general), + device_id) db.execute_sql(update_sql) db.close() return @@ -853,7 +924,7 @@ update_sql = "UPDATE device SET own='%s' WHERE id='%s'" % (instance.own, device_id) db.execute_sql(update_sql) db.close() - return + return def _update_location(self, instance): device_id = instance.id @@ -866,7 +937,8 @@ def _update_firewall_ip(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET firewall_ip='%s' WHERE id='%s'" % (instance.firewall_ip.values()[0], device_id) + update_sql = "UPDATE device SET firewall_ip='%s' WHERE id='%s'" % (instance.firewall_ip.values()[0], + device_id) db.execute_sql(update_sql) db.close() return @@ -874,7 +946,8 @@ def _update_intranet_ip(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET intranet_ip='%s' WHERE id='%s'" % (instance.intranet_ip.values()[0], device_id) + update_sql = "UPDATE device SET intranet_ip='%s' WHERE id='%s'" % (instance.intranet_ip.values()[0], + device_id) db.execute_sql(update_sql) db.close() return @@ -890,22 +963,25 @@ def _update_restapi_account(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET restapi_username='%s', restapi_password='%s' WHERE id='%s'" % (instance.restapi_account['restapi_username'], instance.restapi_account['restapi_password'], device_id) + update_sql = "UPDATE device SET restapi_username='%s', restapi_password='%s' WHERE id='%s'" % ( + instance.restapi_account['restapi_username'], instance.restapi_account['restapi_password'], device_id) db.execute_sql(update_sql) db.close() - return + return def _update_console_account(self, instance): device_id = instance.id db = DB.get_connected_db() - update_sql = "UPDATE device SET console_username='%s', console_password='%s' WHERE id='%s'" % (instance.console_account['console_username'], instance.console_account['console_password'], device_id) + update_sql = "UPDATE device SET console_username='%s', console_password='%s' WHERE id='%s'" % ( + instance.console_account['console_username'], instance.console_account['console_password'], device_id) db.execute_sql(update_sql) db.close() - return + return def _update_enable_password(self, instance): db = DB.get_connected_db() - update_sql = "UPDATE device SET enable_password='%s' WHERE id='%s'" % (instance.enable_password, instance.id) + update_sql = "UPDATE device SET enable_password='%s' WHERE id='%s'" % (instance.enable_password, + instance.id) db.execute_sql(update_sql) db.close() return @@ -920,9 +996,9 @@ def _delete_instance(self, instance): data = instance.get_field_dict() - #check the task + # check the task db = DB.get_connected_db() - select_sql = 'SELECT name, state, device_list from task;' + select_sql = 'SELECT name, state, device_list from task;' select_data = db.fetchall(select_sql) key = ['name', 'state', 'device_list'] result = [dict(zip(key, each)) for each in select_data] @@ -930,22 +1006,23 @@ if data['name'] in item['device_list'] and item['state'] != 'failed' and item['state'] != 'done': raise ModelQueryException(CLICmdError(__('Sorry, please delete the task:%s first.' % item['name']))) - #get backup file list - select_sql = "SELECT name, device_type FROM file_list WHERE type='backup' and modify_time='%s'" % data['name'] + # get backup file list + select_sql = "SELECT name, device_type FROM file_list WHERE type='backup' and modify_time='%s'" % data[ + 'name'] select_data = db.fetchall(select_sql) key = ['name', 'device_type'] backup_result = [dict(zip(key, each)) for each in select_data] - delete_sql = "DELETE FROM device WHERE id = " + "'%s'"%data['id'] + delete_sql = "DELETE FROM device WHERE id = " + "'%s'" % data['id'] db.execute_sql(delete_sql) L.remove_lock(data['ip'].values()[0]) - - #delete from file_list, delete device item and backup item + + # delete from file_list, delete device item and backup item delete_sql = "DELETE FROM file_list WHERE modify_time= '%s'" % data['name'] db.execute_sql(delete_sql) - #remove the backup config file + # remove the backup config file for item in backup_result: backup_path = DEFAULT_CONFIG_FILE_PATH + 'backup/' + item['name'] if os.path.exists(backup_path): @@ -954,7 +1031,7 @@ # need to remove both .all_cfg_tar file and dir path os.popen("rm -rf %s" % backup_path[0:-12]).read() - #delete vsite from vsite_list + # delete vsite from vsite_list select_sql = "SELECT vs_name FROM vsite_list WHERE device_name='%s'" % data['name'] select_data = db.fetchall(select_sql) key = ['vs_name'] @@ -963,13 +1040,13 @@ delete_sql = "DELETE FROM vsite_list WHERE device_name='%s'" % data['name'] db.execute_sql(delete_sql) db.close() - - #remove the vsite config file + + # remove the vsite config file for item in result: vsite_path = DEFAULT_VSITE_PATH + data['name'] + '-' + item['vs_name'] if os.path.exists(vsite_path): os.remove(vsite_path) - #remove the device config file + # remove the device config file file_path = DEFAULT_CONFIG_FILE_PATH + 'device/' + data['name'] if os.path.exists(file_path): os.remove(file_path) @@ -990,18 +1067,29 @@ return def refresh_version(self, item): + + # ToDo: duplicated function 'update_device_connected' to avoid syntax error, provide clean fix. + def update_device_connected(device_id): + db = DB.get_connected_db() + update_sql = "UPDATE device SET connection='%s' WHERE id='%s'" % ('unconnected', device_id) + db.execute_sql(update_sql) + db.close() + data = {} try: if item[0]['protocol'] == 'xmlrpc': rest_response_data = send_cli_to_device(item[0], "show version", mode='enable') else: new_url = modify_url('/rest/device_type/system/SystemInfo/version', item[0]['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', item[0]['ip'], item[0]['restapi_port'], item[0]['restapi_username'], item[0]['restapi_password']) + rest_response_data = send_https_rest_request('GET', new_url, '', item[0]['ip'], + item[0]['restapi_port'], item[0]['restapi_username'], + item[0]['restapi_password']) except Exception as e: if 'timed out' in str(e): update_device_connected(item[0]['id']) mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) - raise ModelQueryException(CLICmdError(__('Timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) + raise ModelQueryException(CLICmdError( + __('Timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) raise ModelQueryException(CLICmdError(__('Refresh Version Failed!'))) else: if item[0]['protocol'] == 'xmlrpc': @@ -1010,19 +1098,21 @@ elif rest_response_data['message'] == 'requestfailed': raise ModelQueryException(CLICmdError(__('Refresh Version Failed!'))) else: - raise ModelQueryException(CLICmdError(__('Register Failed! Please make sure the XML RPC configuration is correct!'))) + raise ModelQueryException( + CLICmdError(__('Register Failed! Please make sure the XML RPC configuration is correct!'))) else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) data['version'] = rest_body['SystemInfo']['version'] elif rest_response_data['status'] == 401: - raise ModelQueryException(CLICmdError(__('Refresh Failed! Please make sure the RESTful API username/password is correct!'))) + raise ModelQueryException(CLICmdError( + __('Refresh Failed! Please make sure the RESTful API username/password is correct!'))) else: raise ModelQueryException(CLICmdError(__('Refresh Version Failed!'))) - + rtn = cli_parse(data['version'], [ - RegexParser('License Key : (.+?)\n', MATCHONE), - ]) + RegexParser('License Key : (.+?)\n', MATCHONE), + ]) if rtn[0]: data['license_key'] = rtn[0][0] data['status'] = 'active' @@ -1030,7 +1120,8 @@ data['license_key'] = None data['status'] = 'new' db = DB.get_connected_db() - update_sql = "UPDATE device SET license_key='%s', status='%s', connection='%s', version='%s' WHERE id='%s'" % (data['license_key'], data['status'], 'connected', data['version'], item[0]['id']) + update_sql = "UPDATE device SET license_key='%s', status='%s', connection='%s', version='%s' WHERE id='%s'" % ( + data['license_key'], data['status'], 'connected', data['version'], item[0]['id']) db.execute_sql(update_sql) db.close() return @@ -1040,9 +1131,9 @@ db = DB.get_connected_db() update_sql = "UPDATE device SET connection='%s' WHERE id='%s'" % ('unconnected', device_id) db.execute_sql(update_sql) - db.close() + db.close() - def refresh_ip_pool(item): + def refresh_ip_pool(item): response_data = send_cli_to_device(item[0], 'show running all "iprange dynamic"', mode='enable') if response_data['message'] == 'requestfailed': update_device_connected(item[0]['id']) @@ -1050,19 +1141,22 @@ raise ModelQueryException(CLICmdError(__('Can not connect'))) elif response_data['message'] == 'success': rtn = cli_parse(response_data['data'], [ - RegexParser('vpn netpool iprange dynamic "(?P.*?)" (?P.*?) (?P.*?) ""', MATCHALL) + RegexParser('vpn netpool iprange dynamic "(?P.*?)" (?P.*?) (?P.*?) ""', + MATCHALL) ]) db = DB.get_connected_db() delete_sql = "DELETE FROM ip_pool WHERE device_id = " + "'%s'" % item[0]['id'] db.execute_sql(delete_sql) for value in rtn[0]: ip_pool_range = value['first_ip'] + '-' + value['last_ip'] - save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES " + "('%s', '%s')" % (ip_pool_range, item[0]['id']) + save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES " + "('%s', '%s')" % ( + ip_pool_range, item[0]['id']) db.execute_sql(save_sql) db.close() else: - raise ModelQueryException(CLICmdError(__('Activation Failed'))) + raise ModelQueryException(CLICmdError(__('Activation Failed'))) return + pk_list = options['__pk_list'] device_id_list = [each['id'] for each in pk_list] device_ip = [] @@ -1070,27 +1164,31 @@ for item in device_id_list: fetchall_sql = "SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, enable_password FROM device WHERE id = " + "'%s'" % item data = db.fetchall(fetchall_sql) - key = ['ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password'] + key = ['ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', + 'enable_password'] result = [dict(zip(key, each)) for each in data] result[0]['id'] = item device_ip.append(result) db.close() - #todo refresh config file + # todo refresh config file for item in device_ip: self.refresh_version(item) if get_device_type(item[0]['type'].lower()) in VPN_TYPE_LIST: refresh_ip_pool(item) mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) - + def _perform_Activate(self, options): def update_device_status(device_id, status, license_name): db = DB.get_connected_db() if license_name: - update_sql = "UPDATE device SET status='%s', connection='%s', license_key='%s' WHERE id='%s'" % (status, 'connected', license_name, device_id) + update_sql = "UPDATE device SET status='%s', connection='%s', license_key='%s' WHERE id='%s'" % ( + status, 'connected', license_name, device_id) else: - update_sql = "UPDATE device SET status='%s', connection='%s' WHERE id='%s'" % (status, 'connected', device_id) + update_sql = "UPDATE device SET status='%s', connection='%s' WHERE id='%s'" % (status, 'connected', + device_id) db.execute_sql(update_sql) db.close() + pk_list = options['__pk_list'] license_name = options['license_name'].strip('"') @@ -1100,10 +1198,12 @@ device_id_list = [each['id'] for each in pk_list] db = DB.get_connected_db() - fetchall_sql = "SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, enable_password, id FROM device WHERE id = " + "'%s'" % device_id_list[0] + fetchall_sql = "SELECT ip_address, protocol, restapi_port, restapi_username, restapi_password, type, enable_password, id FROM device WHERE id = " + "'%s'" % \ + device_id_list[0] data = db.fetchall(fetchall_sql) db.close() - key = ['ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password', 'id'] + key = ['ip', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password', + 'id'] result = [dict(zip(key, each)) for each in data] response_data = send_cli_to_device(result[0], str_cmd) @@ -1120,7 +1220,8 @@ else: update_device_status(device_id_list[0], 'new', '') mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) - raise ModelQueryException(CLICmdError(__('This operation is not allowed when the system is running on an AVX appliance.'))) + raise ModelQueryException(CLICmdError( + __('This operation is not allowed when the system is running on an AVX appliance.'))) else: update_device_status(device_id_list[0], 'new', '') mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) @@ -1129,6 +1230,7 @@ update_device_status(device_id_list[0], 'new', '') mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) raise ModelQueryException(CLICmdError(__('Internal error.'))) + # def send_command_to_device(self, device_info, url, running_params): # try: # rest_response_data = send_https_rest_request('POST', url, running_params, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) @@ -1147,10 +1249,11 @@ url = '/rest/%s/global/cli_extend' % _type data = {'cmd': 'show statistics virtual'} parser = RegexParser('\s+Virtual site (?P.*?)' + - '\s+Statistics .*?' + - '\s+Active sessions\s+(?P[0-9]+)' + - '\s+Maximum Active sessions\s+(?P[0-9]+)' + - '\s+Today\'s Maximum Active sessions\s+(?P[0-9]+)', MATCHALL, reflags=re.S) + '\s+Statistics .*?' + + '\s+Active sessions\s+(?P[0-9]+)' + + '\s+Maximum Active sessions\s+(?P[0-9]+)' + + '\s+Today\'s Maximum Active sessions\s+(?P[0-9]+)', MATCHALL, + reflags=re.S) running_params = json.dumps(data) content = send_command_to_device(device_info, url, running_params) @@ -1158,7 +1261,7 @@ raise ModelQueryException(CLICmdError(__('Get info from device failed.'))) result = cli_parse(content, parser) - return {'result':result} + return {'result': result} def _perform_CheckSession(self, options): pk_list = options['__pk_list'][0] @@ -1169,23 +1272,26 @@ url = '/rest/%s/global/cli_extend' % _type session_result = [] - data = {'cmd':'show session active', 'vsite_name':vsite_name} + data = {'cmd': 'show session active', 'vsite_name': vsite_name} running_params = json.dumps(data) content = send_command_to_device(device_info, url, running_params) if not content or content == 'failed' or type(content) == str: raise ModelQueryException(CLICmdError(__('Get session from virtual site(%s) failed.' % vsite_name))) user_list = content[1]['list'] for user in user_list: - session_result.append({"vsite_name":vsite_name, "user_name":user["User Name"].strip(), "session_id":int(user["Session ID"].strip(), 16), "host_name":host_name}) + session_result.append({"vsite_name": vsite_name, "user_name": user["User Name"].strip(), + "session_id": int(user["Session ID"].strip(), 16), "host_name": host_name}) - #get data from elasticsearch + # get data from elasticsearch log_result = [] for session in session_result: - body_text = body_login_detail % (session['user_name'], session['host_name'], session['vsite_name'], str(session['session_id'])) + body_text = body_login_detail % (session['user_name'], session['host_name'], session['vsite_name'], + str(session['session_id'])) try: es = get_elk() - res = es._search(device_type=_type.upper(), doc_type="syslog", body=json.loads(body_text), _source=["time", "useragent", "aaa_method"]) + res = es._search(device_type=_type.upper(), doc_type="syslog", body=json.loads(body_text), + _source=["time", "useragent", "aaa_method"]) except Exception as e: raise ModelQueryException(CLICmdError(__('Connect elasticsearch failed. %s' % str(e)))) if not es: @@ -1201,14 +1307,14 @@ try: es = get_elk() res = es._search_index(device_ip, doc_type="syslog", body=json.loads(body_device_vsite)) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError as e: raise ModelQueryException(CLICmdError(__(e.info["error"]["reason"]))) except Exception as e: raise ModelQueryException(CLICmdError(__(str(e)))) if not res: return None result = res["aggregations"]["vsite_list"]["buckets"] - return {"result":[item["key"] for item in result]} + return {"result": [item["key"] for item in result]} def get_service_info(self, device_type): db = DB.get_connected_db() @@ -1219,7 +1325,7 @@ data = [dict(zip(key, each)) for each in result] service_list = {} for item in data: - service_list[item["device_ip"]] = item + service_list[item["device_ip"]] = item return service_list def _perform_GetDeviceIpAndVsite(self, options): @@ -1232,11 +1338,12 @@ result = [] for device in device_list: - data = self._perform_GetVsite({"device_ip":device.ip}) + data = self._perform_GetVsite({"device_ip": device.ip}) if not data or device.ip not in service_list: continue vsite_list = list(set(service_list[device.ip]["vsite"]).intersection(set(data["result"]))) for vsite in vsite_list: index = service_list[device.ip]["vsite"].index(vsite) - result.append({"device_name":device.name, "device_ip":device.ip, "vsite":vsite, "service_name":service_list[device.ip]["service_name"][index]}) - return {"result":result} + result.append({"device_name": device.name, "device_ip": device.ip, "vsite": vsite, + "service_name": service_list[device.ip]["service_name"][index]}) + return {"result": result} Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py (working copy) @@ -1,15 +1,11 @@ import datetime -import json -import os -import time + +from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ from cm.lib.communication import L -from cm.lib.parse_configfile import DEFAULT_CONFIG_FILE_PATH, DEFAULT_VSITE_PATH from cm.lib.postgres_db import DB -from django.utils.translation import gettext_lazy as _ -from djproject.an_settings import * from hive.imports.model import * -from django.db.models.query import QuerySet __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py (working copy) @@ -1,32 +1,39 @@ -from hive.imports.model import * -from cm.lib.postgres_db import DB +import json + +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + from cm.lib.communication import send_https_rest_request, modify_url +from cm.lib.postgres_db import DB from cm.models.device_mgmt.device import get_rest_info_from_device from djproject.an_settings import * -from django.db.models.query import QuerySet -import json -from hive.utils import andebug, get_device_type, standard_model_type -__=_ +from hive.imports.model import * +__ = _ + + class HACluster(ANModel): - id = IntegerField(verbose_name = 'HA Cluster ID', primary_key=True, configurable=False) + id = IntegerField(verbose_name='HA Cluster ID', primary_key=True, configurable=False) name = CharField(verbose_name='HA Cluster Name', primary_key=True, length='1..64') device_list = CharField(verbose_name='Device List', editable=True, optional=True) - device_type = CharField(verbose_name='Device Type', optional = True) - enable_ha = BooleanField(verbose_name='Enable Ha', editable=True, lexical=('enable', 'disable'), display=('enable','disable')) - ha_status = FieldGroup(writable=False, verbose_name='HA Status', optional=True, level=STATS, visible_edit=True, fields={ - 'ha_status' : CLITextField(verbose_name='HA Status'), - }) - ha_config = FieldGroup(writable=False, verbose_name='HA Config', optional=True, level=STATS, visible_edit=True, fields={ - 'ha_config' : CLITextField(verbose_name='HA Config'), - }) + device_type = CharField(verbose_name='Device Type', optional=True) + enable_ha = BooleanField(verbose_name='Enable Ha', editable=True, lexical=('enable', 'disable'), + display=('enable', 'disable')) + ha_status = FieldGroup(writable=False, verbose_name='HA Status', optional=True, level=STATS, visible_edit=True, + fields={ + 'ha_status': CLITextField(verbose_name='HA Status'), + }) + ha_config = FieldGroup(writable=False, verbose_name='HA Config', optional=True, level=STATS, visible_edit=True, + fields={ + 'ha_config': CLITextField(verbose_name='HA Config'), + }) ha_log = FieldGroup(writable=False, verbose_name='HA Log', optional=True, level=STATS, visible_edit=True, fields={ - 'ha_log' : CLITextField(verbose_name='HA Log'), + 'ha_log': CLITextField(verbose_name='HA Log'), + }) + ha_condition = FieldGroup(writable=False, verbose_name='HA Condition', optional=True, level=STATS, + visible_edit=True, fields={ + 'ha_condition': CLITextField(verbose_name='HA Condition'), }) - ha_condition = FieldGroup(writable=False, verbose_name='HA Condition', optional=True, level=STATS, visible_edit=True, fields={ - 'ha_condition' : CLITextField(verbose_name='HA Condition'), - }) class Meta: verbose_name = 'HA Cluster' @@ -37,7 +44,7 @@ select_sql = "with raw as (select ha_cluster.*, device.name as dname from ha_cluster left join hc_device on hc_device.g_id=ha_cluster.id left join device on device.id=hc_device.d_id) select raw.id, raw.name, raw.device_type, raw.extend_fields, array_agg(dname) from raw group by raw.id, raw.name, raw.device_type, raw.extend_fields;" data = db.fetchall(select_sql) db.close() - key = ['id', 'name', 'device_type','extend_fields', 'device_list'] + key = ['id', 'name', 'device_type', 'extend_fields', 'device_list'] result = [dict(zip(key, each)) for each in data] for each in result: @@ -51,7 +58,7 @@ else: each['enable_ha'] = False del each['extend_fields'] - + return QuerySet(self._model, result) def update_hc_device(self, device_list, g_id): @@ -75,8 +82,9 @@ db.close() raise ModelQueryException(CLICmdError(__('The group name has existed!'))) - extend_fields = json.dumps({'enable_ha':instance.enable_ha}) - insert_sql = ''' INSERT into ha_cluster(name, device_type, extend_fields) values('%s', '%s', '%s') ''' % (instance.name, instance.device_type, extend_fields) + extend_fields = json.dumps({'enable_ha': instance.enable_ha}) + insert_sql = ''' INSERT into ha_cluster(name, device_type, extend_fields) values('%s', '%s', '%s') ''' % ( + instance.name, instance.device_type, extend_fields) db.execute_sql(insert_sql) select_sql = "select id from ha_cluster where name='%s'" % instance.name result = db.fetchone(select_sql) @@ -96,9 +104,9 @@ data = db.fetchone(select_sql) if not data[0]: extend_data = {} - else: + else: extend_data = json.loads(data[0]) - extend_data.update({'enable_ha':instance.enable_ha}) + extend_data.update({'enable_ha': instance.enable_ha}) update_sql = "update ha_cluster set extend_fields='%s' where name='%s'" % (json.dumps(extend_data), name) db.execute_sql(update_sql) db.close() @@ -107,11 +115,11 @@ def _delete_instance(self, instance): db = DB.get_connected_db() delete_sql = ''' DELETE FROM ha_cluster where name='%s' ''' % instance.name - #update_sql = ''' UPDATE device set ha_cluster='' where ha_cluster='%s' ''' % instance.name + # update_sql = ''' UPDATE device set ha_cluster='' where ha_cluster='%s' ''' % instance.name db.execute_sql(delete_sql) - #db.execute_sql(update_sql) + # db.execute_sql(update_sql) db.close() - #mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) + # mark_expire_all(get_model('cm', ['device_mgmt', 'device', 'Device'])) return def get_cmd_result(self, cmd, device_name): @@ -119,17 +127,19 @@ data = {'cmd': cmd} config_params = json.dumps(data) new_url = modify_url('/rest/device_type/cli_extend', device_info['type']) - rest_response_data = send_https_rest_request('POST', new_url, config_params, device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + rest_response_data = send_https_rest_request('POST', new_url, config_params, device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) try: config_info = json.loads(rest_response_data['body'])['contents'] except Exception as e: raise ModelQueryException(CLICmdError(__("Can't connect the device('%s')." % device_name))) - return (config_info, device_info['type']) + return (config_info, device_info['type']) def _get_ha_status(self, pk_dict): db = DB.get_connected_db() - #select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] + # select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] select_sql = "select name from hc_device left join device on d_id=id where g_id=%d" % pk_dict['id'] device_list = db.fetchall(select_sql) db.close() @@ -139,11 +149,11 @@ return None result, device_type = self.get_cmd_result('show ha status', device_name) - return json.dumps({'device':device_name, 'data':result}) + return json.dumps({'device': device_name, 'data': result}) def _get_ha_config(self, pk_dict): db = DB.get_connected_db() - #select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] + # select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] select_sql = "select name from hc_device left join device on d_id=id where g_id=%d" % pk_dict['id'] device_list = db.fetchall(select_sql) db.close() @@ -154,7 +164,7 @@ result, device_type = self.get_cmd_result('show ha config', device_name) if device_type.lower() in ADC_TYPE_LIST: - return json.dumps({'device':device_name, 'data':result}) + return json.dumps({'device': device_name, 'data': result}) else: config_list = [] for each in result: @@ -163,11 +173,11 @@ for key, value in each.items(): config = "%s %s" % (config, value) config_list.append(config.strip()) - return json.dumps({'device':device_name, 'data':'\n'.join(config_list)}) + return json.dumps({'device': device_name, 'data': '\n'.join(config_list)}) def _get_ha_log(self, pk_dict): db = DB.get_connected_db() - #select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] + # select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] select_sql = "select name from hc_device left join device on d_id=id where g_id=%d" % pk_dict['id'] device_list = db.fetchall(select_sql) db.close() @@ -177,12 +187,11 @@ return None result, device_type = self.get_cmd_result('show ha log 100', device_name) - return json.dumps({'device':device_name, 'data':result}) - + return json.dumps({'device': device_name, 'data': result}) def _get_ha_condition(self, pk_dict): db = DB.get_connected_db() - #select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] + # select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] select_sql = "select name from hc_device left join device on d_id=id where g_id=%d" % pk_dict['id'] device_list = db.fetchall(select_sql) db.close() @@ -195,7 +204,7 @@ device_type = device_info['type'] if device_type.lower() in ADC_TYPE_LIST: result = self.get_cmd_result('show monitor condition', device_name)[0] - return json.dumps({'device':device_name, 'data':result}) + return json.dumps({'device': device_name, 'data': result}) else: result = self.get_cmd_result('show ha condition', device_name)[0] - return json.dumps({'device':device_name, 'data':result}) + return json.dumps({'device': device_name, 'data': result}) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py (working copy) @@ -1,19 +1,24 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ +import datetime + from apscheduler.schedulers.background import BackgroundScheduler -from cm.models.tasking import GLOBAL_TASK +from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ + from cm.lib.postgres_db import DB from cm.lib.task_scheduler import monitor_alert, get_global_settings -import json, datetime, logging -from django.db.models.query import QuerySet -__=_ +from cm.models.tasking import GLOBAL_TASK +from hive.imports.model import * +__ = _ + logger = logging.getLogger('hive.debug') + class MonitorSettings(ANModel): interval = IntegerField(verbose_name='Interval Time') inhibit_time = IntegerField(verbose_name='Inhibit Time') - sslcert_check_enable = BooleanField(verbose_name='SSLCert Check Enable', display=('on', 'off'), default=True, editable=True) + sslcert_check_enable = BooleanField(verbose_name='SSLCert Check Enable', display=('on', 'off'), default=True, + editable=True) assign = CharField(verbose_name='Assign', condition=ValueCondition('sslcert_check_enable', (True)), default="") message = CharField(verbose_name='Message', condition=ValueCondition('sslcert_check_enable', (True)), default="") @@ -23,15 +28,15 @@ class Manager(CLIManager): def _get_query_set(self): - return_data = {'interval':5, 'inhibit_time':0, 'sslcert_check_enable':0, 'assign':"", "message":""} + return_data = {'interval': 5, 'inhibit_time': 0, 'sslcert_check_enable': 0, 'assign': "", "message": ""} select_interval_sql = "select name, settings from global_settings where name='alert_interval';"; select_inhibit_sql = "select name, settings from global_settings where name='alert_inhibit';"; - #select_sslcertcheck_sql = "select name, settings from global_settings where name='sslcert_check';" + # select_sslcertcheck_sql = "select name, settings from global_settings where name='sslcert_check';" select_alertinfo_sql = "select extend, enable from monitor where name='ssl_cert_alert';" db = DB.get_connected_db() data = db.fetchone(select_interval_sql) if data: - return_data['interval'] = int(data[1]) + return_data['interval'] = int(data[1]) data = db.fetchone(select_inhibit_sql) if data: @@ -60,20 +65,25 @@ if data: return True return False + def _update_interval(self, instance): if self.check_field('alert_interval'): - execute_sql = "update global_settings set settings='%s' where name='alert_interval'" % str(instance.interval) + execute_sql = "update global_settings set settings='%s' where name='alert_interval'" % str( + instance.interval) else: - execute_sql = "insert into global_settings(name, settings) values('%s', '%s')" % ('alert_interval', str(instance.interval)) + execute_sql = "insert into global_settings(name, settings) values('%s', '%s')" % ('alert_interval', + str(instance.interval)) db = DB.get_connected_db() db.execute_sql(execute_sql) db.close() - + def _update_inhibit_time(self, instance): if self.check_field('alert_inhibit'): - execute_sql = "update global_settings set settings='%s' where name='alert_inhibit'" % str(instance.inhibit_time) + execute_sql = "update global_settings set settings='%s' where name='alert_inhibit'" % str( + instance.inhibit_time) else: - execute_sql = "insert into global_settings(name, settings) values('%s', '%s')" % ('alert_inhibit', str(instance.inhibit_time)) + execute_sql = "insert into global_settings(name, settings) values('%s', '%s')" % ('alert_inhibit', + str(instance.inhibit_time)) db = DB.get_connected_db() db.execute_sql(execute_sql) db.close() @@ -87,6 +97,7 @@ return data[0] else: return False + """ def check_global_setting(self): select_sql = "select name from global_settings where name='ssl_cert_alert';" @@ -97,16 +108,17 @@ return False return True """ - + def update_insert_ssl_cert(self, enable, assign, message): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') data = { - "name" : "ssl_cert_alert", + "name": "ssl_cert_alert", "level": 0, "target_type": 2, "enable": enable, - "modify_time" : now, - "extend": json.dumps({"assign":assign.split(','), "message":message, "type_id":["4000"], "condition":{}}) + "modify_time": now, + "extend": json.dumps( + {"assign": assign.split(','), "message": message, "type_id": ["4000"], "condition": {}}) } insert_flag = self.check_ssl_cert() @@ -125,7 +137,7 @@ db = DB.get_connected_db() db.execute_sql(execute_sql) - #db.execute_sql(execute_global_sql) + # db.execute_sql(execute_global_sql) db.close() return self.check_ssl_cert() @@ -141,11 +153,11 @@ sched.add_job(monitor_alert, 'interval', minutes=60 * 24, args=[10, "", monitor_id]) if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) - GLOBAL_TASK.add(task_name, sched) + GLOBAL_TASK.add(task_name, sched) else: GLOBAL_TASK.remove(task_name) - + class MonitorMsg(ANModel): id = IntegerField(primary_key=True) alert_time = CharField(verbose_name='Alert Time') @@ -160,15 +172,15 @@ class Meta: verbose_name = 'Monitor Message' list_config_options = { - 'columns' : [ - {'name' : 'alert_time'}, - {'name' : 'deal_time'}, - {'name' : 'alert_name'}, - {'name' : 'extend'}, - {'name' : 'target_name'}, - {'name' : 'target_type'}, - {'name' : 'alert_status'} - ], 'add_button_hide' : True + 'columns': [ + {'name': 'alert_time'}, + {'name': 'deal_time'}, + {'name': 'alert_name'}, + {'name': 'extend'}, + {'name': 'target_name'}, + {'name': 'target_type'}, + {'name': 'alert_status'} + ], 'add_button_hide': True } class SearchAlert(Action): @@ -181,14 +193,14 @@ config_change = True instance_mul = '0' option_fields = ( - #CharField(name='target_type', verbose_name='Target Type', length='1..64', default='all'), - IntegerField(name='target_type', verbose_name='Target Type', default=15), # all:15 + # CharField(name='target_type', verbose_name='Target Type', length='1..64', default='all'), + IntegerField(name='target_type', verbose_name='Target Type', default=15), # all:15 CharField(name='target_name', verbose_name='Target Name', length='1..64', default='all'), CharField(name='alert_type', verbose_name='Alert Type', length='1..64', default='all'), - IntegerField(name='level', verbose_name='Level', default=15), # all:15 + IntegerField(name='level', verbose_name='Level', default=15), # all:15 CharField(name='begin_time', verbose_name='Begin Time'), CharField(name='end_time', verbose_name='End Time'), - IntegerField(name='deal_status', verbose_name='Deal Status', default=0) # all:15 + IntegerField(name='deal_status', verbose_name='Deal Status', default=0) # all:15 ) class Manager(CLIManager): @@ -196,7 +208,8 @@ db = DB.get_connected_db() condition = "where create_time between '%s' and '%s'" % (options['begin_time'], options['end_time']) if options['deal_status'] != 15: - select_sql = "select monitor_id from monitor_msg %s and deal_status=%d group by monitor_id;" % (condition, options['deal_status']) + select_sql = "select monitor_id from monitor_msg %s and deal_status=%d group by monitor_id;" % ( + condition, options['deal_status']) else: select_sql = "select monitor_id from monitor_msg %s group by monitor_id ;" % condition data = db.fetchall(select_sql) @@ -206,6 +219,7 @@ monitor_id.append(item[0]) return monitor_id + def get_monitor_info(self, monitor_id, level): condition = '' if level != 15: @@ -238,7 +252,8 @@ db = DB.get_connected_db() data = db.fetchall(select_sql) db.close() - key = ['id', 'monitor_id', 'create_time', 'deal_time', 'deal_status', 'target_type', 'target_name', 'extend'] + key = ['id', 'monitor_id', 'create_time', 'deal_time', 'deal_status', 'target_type', 'target_name', + 'extend'] result = [dict(zip(key, each)) for each in data] for each in result: each['comment'] = each['extend']['comment'] if 'comment' in each['extend'] else "" @@ -281,24 +296,26 @@ if type_id in alert_type: result.append(item) break - return result + return result def _perform_SearchAlert(self, options): - #{u'alert_type': u'all', u'deal_status': u'0', u'__pk_list': [], u'level': u'15', u'target_name': u'all', u'target_type': u'all', u'end_time': u'2018-10-09 01:41:17', u'begin_time': u'2018-05-07 01:41:17'} - monitor_condition = {'deal_status':int(options['deal_status']), 'begin_time':options['begin_time'], 'end_time':options['end_time']} + # {u'alert_type': u'all', u'deal_status': u'0', u'__pk_list': [], u'level': u'15', u'target_name': u'all', u'target_type': u'all', u'end_time': u'2018-10-09 01:41:17', u'begin_time': u'2018-05-07 01:41:17'} + monitor_condition = {'deal_status': int(options['deal_status']), 'begin_time': options['begin_time'], + 'end_time': options['end_time']} - #get monitor_id + # get monitor_id monitor_id = self.get_monitor_id(monitor_condition) - #get monitor_info - #{'id': 3, 'type_id': [u'1', u'2'], 'condition': u'cpu_usage>1 or mem_usage>1', 'name': 'alert', 'level': 0} + # get monitor_info + # {'id': 3, 'type_id': [u'1', u'2'], 'condition': u'cpu_usage>1 or mem_usage>1', 'name': 'alert', 'level': 0} monitor_info = self.get_monitor_info(monitor_id, int(options['level'])) monitor_info = self.filter_monitor_info(monitor_info, options['alert_type']) - #get monitor_msg_info - monitor_condition.update({'target_type':int(options['target_type']), 'target_name':options['target_name']}) + # get monitor_msg_info + monitor_condition.update( + {'target_type': int(options['target_type']), 'target_name': options['target_name']}) monitor_msg_info = self.get_monitor_msg_info(monitor_condition) - #{'monitor_id': 4, 'comment': u'...', 'deal_status': 0, 'target_name': '10.8.11.35', 'target_type': 0, 'note': '', 'create_time': datetime.datetime(2018, 7, 9, 15, 19, 25), 'deal_time': None} + # {'monitor_id': 4, 'comment': u'...', 'deal_status': 0, 'target_name': '10.8.11.35', 'target_type': 0, 'note': '', 'create_time': datetime.datetime(2018, 7, 9, 15, 19, 25), 'deal_time': None} result = [] for monitor_item in monitor_info: @@ -310,12 +327,13 @@ if not monitor_item['monitor_msg']: continue result.append(monitor_item) - return {'result':result} + return {'result': result} def _get_query_set(self): db = DB.get_connected_db() select_sql = "SELECT id, monitor_id, create_time, deal_time, deal_status, target_type, target_name, extend from monitor_msg;" - key = ['id', 'monitor_id', 'alert_time', 'deal_time', 'alert_status', 'target_type', 'target_name', 'extend'] + key = ['id', 'monitor_id', 'alert_time', 'deal_time', 'alert_status', 'target_type', 'target_name', + 'extend'] data = db.fetchall(select_sql) result = [dict(zip(key, each)) for each in data] for item in result: @@ -323,7 +341,7 @@ key = ['name'] data = db.fetchall(select_detail_sql) result_detail = [dict(zip(key, each)) for each in data] - item.update({'alert_name':result_detail[0]['name']}) + item.update({'alert_name': result_detail[0]['name']}) del item['monitor_id'] db.close() return QuerySet(self._model, result) @@ -332,7 +350,8 @@ def _update_alert_status(self, instance): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - update_sql = "UPDATE monitor_msg set deal_status=%d, extend=extend||'{\"note\":\"%s\"}'::jsonb, deal_time='%s' where id=%d" % (int(instance.alert_status), instance.note, now, instance.id) + update_sql = "UPDATE monitor_msg set deal_status=%d, extend=extend||'{\"note\":\"%s\"}'::jsonb, deal_time='%s' where id=%d" % ( + int(instance.alert_status), instance.note, now, instance.id) db.execute_sql(update_sql) db.close() return @@ -343,6 +362,7 @@ db.execute_sql(delete_sql) db.close() + class Monitor(ANModel): id = IntegerField(primary_key=True, configurable=False) name = CharField(verbose_name='Monitor Name', length='1..64') @@ -356,26 +376,26 @@ modify_time = CharField(verbose_name='Modify Time', configurable=False, optional=True) assign = CharField(verbose_name='Assign List', optional=True, length='1..2048', editable=True) level = EnumField(verbose_name='Monitor Level', editable=True, values=( - (0, 'Serious'), - (1, 'Major'), - (2, 'Minor') - )) + (0, 'Serious'), + (1, 'Major'), + (2, 'Minor') + )) class Meta: verbose_name = 'Monitor' list_config_options = { - 'columns' : [ - {'name' : 'name'}, - {'name' : 'target_type'}, - {'name' : 'target_list'}, - {'name' : 'type_id'}, - {'name' : 'interval'}, - {'name' : 'condition'}, - {'name' : 'message'}, - {'name' : 'enable'}, - {'name' : 'modify_time'}, - {'name' : 'assign'}, - {'name' : 'level'} + 'columns': [ + {'name': 'name'}, + {'name': 'target_type'}, + {'name': 'target_list'}, + {'name': 'type_id'}, + {'name': 'interval'}, + {'name': 'condition'}, + {'name': 'message'}, + {'name': 'enable'}, + {'name': 'modify_time'}, + {'name': 'assign'}, + {'name': 'level'} ] } @@ -397,17 +417,19 @@ def _perform_DealAlert(self, options): msg_id = options['monitor_msg_id'].split(',') if not len(msg_id): - raise ModelQueryException(CLICmdError(__("name:%s has existed!" % (data['name'])))) + raise ModelQueryException(CLICmdError(__("name:%s has existed!" % (options['name'])))) condition = "where " for item in msg_id: condition += "id=%d or " % int(item) condition = condition.strip(' ').strip('or') now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') db = DB.get_connected_db() - update_sql = "UPDATE monitor_msg set deal_status=1, extend=extend||'{\"note\":\"%s\"}'::jsonb, deal_time='%s' %s;" % (options['note'], now, condition) + update_sql = "UPDATE monitor_msg set deal_status=1, extend=extend||'{\"note\":\"%s\"}'::jsonb, deal_time='%s' %s;" % ( + options['note'], now, condition) db.execute_sql(update_sql) db.close() return + def _get_query_set(self): db = DB.get_connected_db() select_device_sql = "select id, name, array_agg, interval, level, target_type, modify_time, enable, extend from dm_view;" @@ -430,7 +452,7 @@ each['target_list'] = '' each['condition'] = json.dumps(each['condition']) return_data.append(each) - + return QuerySet(self._model, return_data) def update_dm(self, device_list, monitor_id): @@ -456,7 +478,6 @@ db.execute_sql(insert_sql) db.close() return - def check_repeater(self, name): db = DB.get_connected_db() @@ -472,11 +493,13 @@ sched.add_job(monitor_alert, 'interval', minutes=interval, args=args) if GLOBAL_TASK.get_schedule(task_name): GLOBAL_TASK.remove(task_name) - GLOBAL_TASK.add(task_name, sched) + GLOBAL_TASK.add(task_name, sched) + def remove_monitor(self, task_name): if task_name: GLOBAL_TASK.remove(task_name) return + def get_device_ip(self, device_list): db = DB.get_connected_db() device_ip = [] @@ -496,6 +519,7 @@ if item in data: data['extend'][item] = data[item] if item not in list_flag else data[item].split(',') del data[item] + def split_extend_field(self, data, field_list, list_flag): for item in field_list: if item in data['extend']: @@ -504,11 +528,11 @@ del data['extend'] def _insert(self, instance): - #{'enable': True, 'name': u'alert', 'device_list': u'device', 'interval': 2, 'email': '', - #'send_email': False, 'modify_time': '', 'message': u'heiheihei', 'id': 0, 'condition': u'cpu>=100'} + # {'enable': True, 'name': u'alert', 'device_list': u'device', 'interval': 2, 'email': '', + # 'send_email': False, 'modify_time': '', 'message': u'heiheihei', 'id': 0, 'condition': u'cpu>=100'} now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') data = instance.get_field_dict() - + if self.check_repeater(data['name']): raise ModelQueryException(CLICmdError(__("name:%s has existed!" % (data['name'])))) @@ -543,7 +567,7 @@ if data['enable']: task_name = '%d-%s' % (id, data['name']) args = [data['target_type'], data['target_list'].split(','), id] - #args = [data['target_type'], data['target_list'].split(','), id, user_info] + # args = [data['target_type'], data['target_list'].split(','), id, user_info] global_settings = get_global_settings() self.start_monitor(task_name, args, interval_time) return @@ -562,8 +586,7 @@ continue return user_info_list ''' - - + def _update(self, instance): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') data = instance.get_field_dict() @@ -580,8 +603,7 @@ else: global_settings = get_global_settings() interval_time = int(global_settings['interval']) - - + data['extend'] = json.dumps(data['extend']) update_sql = "UPDATE monitor SET interval='%(interval)d', level='%(level)d', enable='%(enable)d', target_type='%(target_type)d', modify_time='%(now)s', extend='%(extend)s' where id='%(id)d'" % data @@ -597,7 +619,7 @@ if data['enable']: task_name = '%d-%s' % (data['id'], data['name']) args = [data['target_type'], data['target_list'].split(','), data['id']] - #args = [data['target_type'], data['target_list'].split(','), data['id'], user_info] + # args = [data['target_type'], data['target_list'].split(','), data['id'], user_info] self.start_monitor(task_name, args, interval_time) else: task_name = '%d-%s' % (data['id'], data['name']) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py (working copy) @@ -1,37 +1,34 @@ +from django.utils.translation import gettext_lazy as _ + from hive.imports.model import * -from hive.model.manager import ANManager -from django.utils.translation import ugettext_lazy as _ -from hive.session import ANSession, current_app -from hive.router import get_current_session -from django.shortcuts import redirect -from hive.node.model import * -from hive.utils import get_current_session, andebug + __ = _ + class DNSServer(ANModel): server_ip = IPAddressField(verbose_name=_('IP'), primary_key=True) class Meta: verbose_name = _('DNS Server') show_im_export_button = False - list_config_options={'click_enable': False} + list_config_options = {'click_enable': False} class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable() - result = self.cli.cmd('show ip nameserver', - RegexParser('name server (?P[0-9|\.|:|\w]+)', MATCHALL, reflags=re.M)) + result = self.cli.cmd('show ip nameserver', + RegexParser('name server (?P[0-9|\.|:|\w]+)', MATCHALL, reflags=re.M)) return QuerySet(self._model, result) def _insert(self, instance): self.cli.set_config() result = self.cli.cmd('ip nameserver %s' % instance.server_ip.values()[0], - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return def _delete(self, pk_list): self.cli.set_config() for each_pk in pk_list: result = self.cli.cmd('no ip nameserver %s' % each_pk["server_ip"].values()[0], - BlankParser(nonblank_exception=CLICmdError, supplement=True)) - return \ No newline at end of file + BlankParser(nonblank_exception=CLICmdError, supplement=True)) + return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py (working copy) @@ -1,15 +1,19 @@ -from hive.imports.model import * +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from hive.imports.model import * from hive.node.model import * -from hive.utils import get_current_session, andebug -from django.db.models.query import QuerySet + __ = _ + class InterfaceIP(ANModel): - interface = AssoField2(verbose_name=_('Interface'), tgt='network.interface.SystemInterface.addresses', monitorable=True, mul='*', pos='right', primary_key=True) + interface = AssoField2(verbose_name=_('Interface'), tgt='network.interface.SystemInterface.addresses', + monitorable=True, mul='*', pos='right', primary_key=True) ip = IPAddressField(verbose_name=_('IP'), primary_key=True, editable=True) - mask_prefix = UnionField(verbose_name=_('Mask/Prefix'), editable=True, primary_key=True, fields={'mask':NetmaskField(verbose_name=_('Mask')), - 'prefix':Uint32Field(verbose_name=_('Prefix'))}) + mask_prefix = UnionField(verbose_name=_('Mask/Prefix'), editable=True, primary_key=True, + fields={'mask': NetmaskField(verbose_name=_('Mask')), + 'prefix': Uint32Field(verbose_name=_('Prefix'))}) class Meta: verbose_name = _('Interface Address') @@ -17,57 +21,68 @@ class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable() - parser = RegexParser('ip address "(?P\S+)" (?P[0-9|\.|:|\w]+) (?P[0-9|\.]+)', MATCHALL, reflags=re.M) + parser = RegexParser( + 'ip address "(?P\S+)" (?P[0-9|\.|:|\w]+) (?P[0-9|\.]+)', MATCHALL, + reflags=re.M) result = self.cli.cmd('show ip address', parser) for each_result in result: each_result['interface'] = [{'interface_name': each_result['interface']}] if each_result['mask_prefix_str'].find('.') == -1: - each_result['mask_prefix'] = {'prefix':each_result['mask_prefix_str']} + each_result['mask_prefix'] = {'prefix': each_result['mask_prefix_str']} else: - each_result['mask_prefix'] = {'mask':each_result['mask_prefix_str']} + each_result['mask_prefix'] = {'mask': each_result['mask_prefix_str']} del each_result['mask_prefix_str'] return QuerySet(self._model, result) def _insert(self, instance): self.cli.set_config() - result = self.cli.cmd('ip address "%s" %s %s' % (instance.interface[0]["interface_name"], instance.ip.values()[0], instance.mask_prefix.values()[0]), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + result = self.cli.cmd( + 'ip address "%s" %s %s' % (instance.interface[0]["interface_name"], instance.ip.values()[0], + instance.mask_prefix.values()[0]), + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return def _update(self, instance): self.cli.set_config() - result = self.cli.cmd('ip address "%s" %s %s' % (instance.interface[0]["interface_name"], instance.ip.values()[0], instance.mask_prefix.values()[0]), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + result = self.cli.cmd( + 'ip address "%s" %s %s' % (instance.interface[0]["interface_name"], instance.ip.values()[0], + instance.mask_prefix.values()[0]), + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return def _delete(self, pk_list): self.cli.set_config() for each_pk in pk_list: - result = self.cli.cmd('no ip address "%s" %s %s' % (each_pk["interface"][0]["interface_name"], each_pk["ip"].values()[0], each_pk["mask_prefix"].values()[0]), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + result = self.cli.cmd( + 'no ip address "%s" %s %s' % (each_pk["interface"][0]["interface_name"], each_pk["ip"].values()[0], + each_pk["mask_prefix"].values()[0]), + BlankParser(nonblank_exception=CLICmdError, supplement=True)) mark_expire_all(InterfaceIP) return + class SystemInterface(ANModel): interface_name = CharField(verbose_name=_('Interface Name'), editable=False, primary_key=True) inf_type = EnumField(verbose_name=_('Interface Type'), values=( - ('system', 'System'), - ('mgmt', 'Management') - ), editable=False) + ('system', 'System'), + ('mgmt', 'Management') + ), editable=False) # ip = IPAddressField(verbose_name=_('IP'), editable=True, optional=True) - addresses = AssoField2(verbose_name=_('Interface IP'), tgt='network.interface.InterfaceIP.interface', monitorable=True, mul='1', pos='left', optional=True) + addresses = AssoField2(verbose_name=_('Interface IP'), tgt='network.interface.InterfaceIP.interface', + monitorable=True, mul='1', pos='left', optional=True) mac = MACAddressField(verbose_name=_('MAC'), editable=False, optional=True) mtu = Uint32Field(verbose_name=_('MTU'), editable=False) class Meta: verbose_name = _('System Interface') - list_config_options={ - 'columns':[ - {'name':'interface_name'}, - {'name':'ip'}, - {'name':'mask_prefix'}, - ]} + list_config_options = { + 'columns': [ + {'name': 'interface_name'}, + {'name': 'ip'}, + {'name': 'mask_prefix'}, + ]} + class Modify_IP(Action): verbose_name = _('Modify IP') action_name = _('Modify') @@ -78,11 +93,13 @@ config_change = True instance_mul = '0' option_fields = ( - IPAddressField(verbose_name=_('IP'), name='ip'), - CharField(verbose_name=_('Interface Name'), name='interface_name'), - UnionField(verbose_name=_('Mask/Prefix'), name='mask', fields={'mask':NetmaskField(verbose_name=_('Mask')), - 'prefix':Uint32Field(verbose_name=_('Prefix'))}) + IPAddressField(verbose_name=_('IP'), name='ip'), + CharField(verbose_name=_('Interface Name'), name='interface_name'), + UnionField(verbose_name=_('Mask/Prefix'), name='mask', fields={'mask': NetmaskField(verbose_name=_('Mask')), + 'prefix': Uint32Field( + verbose_name=_('Prefix'))}) ) + class Delete_IP(Action): verbose_name = _('Delete IP') action_name = _('Delete') @@ -93,11 +110,13 @@ config_change = True instance_mul = '0' option_fields = ( - IPAddressField(verbose_name=_('IP'), name='ip'), - CharField(verbose_name=_('Interface Name'), name='interface_name'), - UnionField(verbose_name=_('Mask/Prefix'), name='mask', fields={'mask':NetmaskField(verbose_name=_('Mask')), - 'prefix':Uint32Field(verbose_name=_('Prefix'))}) + IPAddressField(verbose_name=_('IP'), name='ip'), + CharField(verbose_name=_('Interface Name'), name='interface_name'), + UnionField(verbose_name=_('Mask/Prefix'), name='mask', fields={'mask': NetmaskField(verbose_name=_('Mask')), + 'prefix': Uint32Field( + verbose_name=_('Prefix'))}) ) + class Set_Mgmt(Action): verbose_name = _('Set Mgmt') action_name = _('Set') @@ -114,32 +133,42 @@ mask = options['mask'].values()[0] self.cli.set_config() result = self.cli.cmd('ip address %s %s %s' % (options['interface_name'], ip, mask), - RegexParser('Invalid', MATCHONE, match_exception=CLICmdWarning, match_msg='Set IP error.')) + RegexParser('Invalid', MATCHONE, match_exception=CLICmdWarning, + match_msg='Set IP error.')) mark_expire_all(get_model('cm', ['network', 'interface', 'SystemInterface'])) return result + def _perform_Delete_IP(self, options): ip = options['ip'].values()[0] mask = options['mask'].values()[0] self.cli.set_config() result = self.cli.cmd('no ip address %s %s %s' % (options['interface_name'], ip, mask), - RegexParser('Invalid', MATCHONE, match_exception=CLICmdWarning, match_msg='Delete IP error.')) + RegexParser('Invalid', MATCHONE, match_exception=CLICmdWarning, + match_msg='Delete IP error.')) mark_expire_all(get_model('cm', ['network', 'interface', 'SystemInterface'])) return result + def _perform_Set_Mgmt(self, options): pk_list = options['__pk_list'][0] interface_name = pk_list['interface_name'] self.cli.set_config() result = self.cli.cmd('system management interface %s' % interface_name, - RegexParser('Failed', MATCHONE, match_exception=CLICmdWarning, match_msg='Set managerment interface failed.')) + RegexParser('Failed', MATCHONE, match_exception=CLICmdWarning, + match_msg='Set managerment interface failed.')) mark_expire_all(get_model('cm', ['network', 'interface', 'SystemInterface'])) return result + def _get_query_set(self): self.cli.set_enable() - parser_ether = RegexParser('(?P[^\n]+?):\s+flags=[^>]+>\s+mtu\s+(?P[0-9]+)((?!ether).)*?' + - '\s+ether\s+(?P[\w|0-9|:]+)((?!RX packets).)*?', MATCHALL, reflags=re.S) - parser_name = RegexParser('(?P[^\n]+?):\s+flags=[^>]+>\s+mtu\s+(?P[0-9]+)((?!RX packets).)*?', MATCHALL, reflags=re.S) - - mgmt = self.cli.cmd('show system management interface', RegexParser('system management interface (?P.*?)\n', MATCHONE, reflags=re.S)) + parser_ether = RegexParser( + '(?P[^\n]+?):\s+flags=[^>]+>\s+mtu\s+(?P[0-9]+)((?!ether).)*?' + + '\s+ether\s+(?P[\w|0-9|:]+)((?!RX packets).)*?', MATCHALL, reflags=re.S) + parser_name = RegexParser( + '(?P[^\n]+?):\s+flags=[^>]+>\s+mtu\s+(?P[0-9]+)((?!RX packets).)*?', MATCHALL, + reflags=re.S) + + mgmt = self.cli.cmd('show system management interface', + RegexParser('system management interface (?P.*?)\n', MATCHONE, reflags=re.S)) result = self.cli.cmd('show interface', parser_name) res = [] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py (working copy) @@ -1,12 +1,15 @@ -from hive.imports.model import * from django.utils.translation import gettext_lazy as _ -from hive.node.model import * + +from hive.imports.model import * + __ = _ + class DefaultRoutingSetting(ANModel): - gateway_ip = GroupField(verbose_name=_('Gateway IP'), fields={'ipv4':IPv4AddressField(verbose_name=_('IPv4'),optional=True), - 'ipv6':IPv6AddressField(verbose_name=_('IPv6'), optional=True)}, optional=True) - + gateway_ip = GroupField(verbose_name=_('Gateway IP'), + fields={'ipv4': IPv4AddressField(verbose_name=_('IPv4'), optional=True), + 'ipv6': IPv6AddressField(verbose_name=_('IPv6'), optional=True)}, optional=True) + class Clear(Action): verbose_name = _('Clear') action_name = _('Clear') @@ -20,13 +23,14 @@ profile = True verbose_name = _('Default Route') show_im_export_button = False - + class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable(force=True) - parser = RegexParser('IPv[4|6]\s+route:((?!default).)*?\s+default\s+(?P[0-9|:|\.|\w]+)', MATCHALL, reflags=re.S) + parser = RegexParser('IPv[4|6]\s+route:((?!default).)*?\s+default\s+(?P[0-9|:|\.|\w]+)', + MATCHALL, reflags=re.S) result = self.cli.cmd('show ip route', parser) - ret = {'gateway_ip':{'ipv4':'', 'ipv6':''}} + ret = {'gateway_ip': {'ipv4': '', 'ipv6': ''}} if result: for each_res in result: if is_ipv4(each_res['gateway_ip_str']): @@ -35,19 +39,18 @@ ret['gateway_ip']['ipv6'] = each_res['gateway_ip_str'] data = QuerySet(self._model, [ret]) return data - - + def _update_gateway_ip(self, instance, old_values): self.cli.set_config() if instance.gateway_ip: if instance.gateway_ip['ipv4']: if old_values['gateway_ip']['ipv4'] != instance.gateway_ip['ipv4']: result = self.cli.cmd('ip route default %s' % (instance.gateway_ip['ipv4']), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) if instance.gateway_ip['ipv6']: if old_values['gateway_ip']['ipv6'] != instance.gateway_ip['ipv6']: result = self.cli.cmd('ip route default %s' % (instance.gateway_ip['ipv6']), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return def _perform_Clear(self, options): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py (working copy) @@ -1,17 +1,21 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ -from hive.utils import andebug from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ + +from hive.imports.model import * + __ = _ + + class AccessControl(ANModel): webui_settings = FieldGroup(verbose_name=_('WebUI Settings'), level=BASIC, editable=True, fields={ - 'enable_webui' : BooleanField(verbose_name=_('Enable WebUI')), - 'webui_port' : PortField(verbose_name=_('WebUI Port'),default=8888,scope='1025..65000'), + 'enable_webui': BooleanField(verbose_name=_('Enable WebUI')), + 'webui_port': PortField(verbose_name=_('WebUI Port'), default=8888, scope='1025..65000'), }) - webui_ssl_cert = FieldGroup(writable=False, verbose_name=_('WebUI Certificate'), level=BASIC, editable=True, fields={ - 'cert_file': CLITextField(verbose_name=_('Certificate'), optional=True, default="") - }) + webui_ssl_cert = FieldGroup(writable=False, verbose_name=_('WebUI Certificate'), level=BASIC, editable=True, + fields={ + 'cert_file': CLITextField(verbose_name=_('Certificate'), optional=True, default="") + }) class Import_Cert(Action): verbose_name = _('Import Certificate') @@ -24,18 +28,21 @@ condition_rule = EmptyValueCondition('cert_file') option_fields = ( - UnionField(name='using',verbose_name=_('Using'), optional=True, fields={ - 'local': GroupField(verbose_name=_('Local File'),fields={ - 'cert': ImportLocalFileField(verbose_name=_('PEM File'), button_text=__('Choose Local File'), maxFileSize=500000000), - }), - 'online': GroupField(verbose_name=_('TFTP/FTP/HTTP'),fields={ - 'url': URLField(verbose_name=_('URL'), pattern='((tftp|ftp|http)://)', pattern_msg='Please specify tftp:// ftp:// or http://',optional=True), - }), - 'manual_input': GroupField(verbose_name=_('Manual Input'),fields={ - 'cert_content': TextField(verbose_name=_('Cert Content')), - }), + UnionField(name='using', verbose_name=_('Using'), optional=True, fields={ + 'local': GroupField(verbose_name=_('Local File'), fields={ + 'cert': ImportLocalFileField(verbose_name=_('PEM File'), button_text=__('Choose Local File'), + maxFileSize=500000000), + }), + 'online': GroupField(verbose_name=_('TFTP/FTP/HTTP'), fields={ + 'url': URLField(verbose_name=_('URL'), pattern='((tftp|ftp|http)://)', + pattern_msg='Please specify tftp:// ftp:// or http://', optional=True), + }), + 'manual_input': GroupField(verbose_name=_('Manual Input'), fields={ + 'cert_content': TextField(verbose_name=_('Cert Content')), + }), }), ) + class Clear_Cert(Action): verbose_name = _('Clear Certificate') action_name = _('Clear') @@ -45,11 +52,11 @@ process_title = "Clear..." config_change = True condition_rule = NonemptyValueCondition('cert_file') - + class Meta: profile = True verbose_name = _('Access Control') - + class Manager(CLIManager): def _get_query_set(self): ret = {} @@ -61,14 +68,16 @@ def _get_webui_settings(self): self.cli.set_enable() result = self.cli.cmd('show webui settings', - RegexParser('The WebUI is (?PON|OFF) using port (?P[0-9]+)\n', MATCHONE, reflags=re.S)) + RegexParser( + 'The WebUI is (?PON|OFF) using port (?P[0-9]+)\n', + MATCHONE, reflags=re.S)) data = { - 'enable_webui': True if result['enable_webui'] == 'ON' else False, - 'webui_port': result['webui_port'], - } + 'enable_webui': True if result['enable_webui'] == 'ON' else False, + 'webui_port': result['webui_port'], + } return data - + def _update_enable_webui(self, instance): self.cli.set_config() on_or_off = 'on' if instance.enable_webui else 'off' @@ -91,25 +100,33 @@ if 'local' in options['using']: self.cli.set_config() result = self.cli.cmd('webui ssl import pem "file://%s"' % (options['using']['local']['cert']), - RegexParser('import failed', match_exception=CLICmdError, match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), - RegexParser('already exists', match_exception=CLICmdError, match_msg='The WebUI SSL certificate already exists.'), + RegexParser('import failed', match_exception=CLICmdError, + match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), + RegexParser('already exists', match_exception=CLICmdError, + match_msg='The WebUI SSL certificate already exists.'), RegexParser('import successful', match_exception=CLICmdNormal, exclusive=True), BlankParser(nonblank_exception=CLICmdError, supplement=True)) elif 'online' in options['using']: self.cli.set_config() result = self.cli.cmd('webui ssl import pem "%s"' % (options['using']['online']['url']), - RegexParser('import failed', match_exception=CLICmdError, match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), - RegexParser('already exists', match_exception=CLICmdError, match_msg='The WebUI SSL certificate already exists.'), + RegexParser('import failed', match_exception=CLICmdError, + match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), + RegexParser('already exists', match_exception=CLICmdError, + match_msg='The WebUI SSL certificate already exists.'), RegexParser('import successful', match_exception=CLICmdNormal, exclusive=True), BlankParser(nonblank_exception=CLICmdError, supplement=True)) elif 'manual_input' in options['using']: self.cli.set_config() - result = self.cli.cmd('webui ssl import pem\n%s\n...\n' % (options['using']['manual_input']['cert_content']), - RegexParser('import failed', match_exception=CLICmdError, match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), - RegexParser('already exists', match_exception=CLICmdError, match_msg='The WebUI SSL certificate already exists.'), - RegexParser('import successful', match_exception=CLICmdNormal, exclusive=True), - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + result = self.cli.cmd( + 'webui ssl import pem\n%s\n...\n' % (options['using']['manual_input']['cert_content']), + RegexParser('import failed', match_exception=CLICmdError, + match_msg='WebUI SSL certificate import failed. Invalid certificate format!'), + RegexParser('already exists', match_exception=CLICmdError, + match_msg='The WebUI SSL certificate already exists.'), + RegexParser('import successful', match_exception=CLICmdNormal, exclusive=True), + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return result + def _perform_Clear_Cert(self, options): self.cli.set_config() result = self.cli.cmd("clear webui ssl cert") @@ -117,17 +134,19 @@ class AdminAAASettings(ANModel): - enable_aaa_authorize = BooleanField(verbose_name=_('External Authorization'),default=False) - enable_aaa = BooleanField(verbose_name=_('Enable Admin AAA'),default=False) - priority = EnumField(condition=ValueCondition('enable_aaa',True),verbose_name = _('Priority'),default=0,optional=True,values=( - ('0','0'), - ('1','1'), - )) - method = EnumField(verbose_name = _('Method'),default='radius',optional=True,values=( - ('radius','RADIUS'), - ('tac_x','TAC_X'), - )) - admin_aaa_server = AssoField2(verbose_name='Admin AAA Server', tgt='system.AdminAAAServer.asso', mul='1', pos='left', optional=True) + enable_aaa_authorize = BooleanField(verbose_name=_('External Authorization'), default=False) + enable_aaa = BooleanField(verbose_name=_('Enable Admin AAA'), default=False) + priority = EnumField(condition=ValueCondition('enable_aaa', True), verbose_name=_('Priority'), default=0, + optional=True, values=( + ('0', '0'), + ('1', '1'), + )) + method = EnumField(verbose_name=_('Method'), default='radius', optional=True, values=( + ('radius', 'RADIUS'), + ('tac_x', 'TAC_X'), + )) + admin_aaa_server = AssoField2(verbose_name='Admin AAA Server', tgt='system.AdminAAAServer.asso', mul='1', + pos='left', optional=True) class Clear(Action): verbose_name = _('Clear') @@ -141,21 +160,22 @@ class Meta: profile = True verbose_name = _('Admin AAA Settings') - + class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable() result = self.cli.cmd('show admin aaa all', - RegexParser('admin aaa (?Pon|off)', MATCHONE, reflags=re.S), - RegexParser('admin aaa on (?P0|1)', MATCHONE, reflags=re.S), - RegexParser('admin aaa method (?PRADIUS|TAC_X)', MATCHONE, reflags=re.S), - RegexParser('admin aaa authorize (?Pon|off)', MATCHONE, reflags=re.S)) + RegexParser('admin aaa (?Pon|off)', MATCHONE, reflags=re.S), + RegexParser('admin aaa on (?P0|1)', MATCHONE, reflags=re.S), + RegexParser('admin aaa method (?PRADIUS|TAC_X)', MATCHONE, reflags=re.S), + RegexParser('admin aaa authorize (?Pon|off)', MATCHONE, + reflags=re.S)) rtn_dict = { - 'enable_aaa':True if result[0]['enable_aaa'] == 'on' else False, - 'priority':result[1]['priority'] if result[1] else '0', - 'method':result[2]['method'] if result[2] else '', - 'enable_aaa_authorize':True if result[3]['enable_aaa_authorize'] == 'on' else False, - } + 'enable_aaa': True if result[0]['enable_aaa'] == 'on' else False, + 'priority': result[1]['priority'] if result[1] else '0', + 'method': result[2]['method'] if result[2] else '', + 'enable_aaa_authorize': True if result[3]['enable_aaa_authorize'] == 'on' else False, + } self._model._meta.mark_delay_query(rtn_dict) return QuerySet(self._model, [rtn_dict]) @@ -164,10 +184,10 @@ self.cli.set_config() if instance.enable_aaa_authorize: result = self.cli.cmd('admin aaa authorize on', - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) else: result = self.cli.cmd('admin aaa authorize off', - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return result @UpdatingFields(['enable_aaa', 'priority']) @@ -175,32 +195,32 @@ self.cli.set_config() if instance.enable_aaa: result = self.cli.cmd('admin aaa on %s' % instance.priority, - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) else: result = self.cli.cmd('admin aaa off', - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return result def _update_method(self, instance): self.cli.set_config() result = self.cli.cmd('admin aaa method %s' % instance.method, - BlankParser(nonblank_exception=CLICmdError, supplement=True)) - return result + BlankParser(nonblank_exception=CLICmdError, supplement=True)) + return result def _perform_Clear(self, options): self.cli.set_config() result = self.cli.cmd('clear admin aaa all') - return result + return result class AdminAAAServer(ANModel): asso = AssoField2(hidden=True, tgt='system.AdminAAASettings.admin_aaa_server', mul='*', pos='right', optional=True) - server_id = EnumField(verbose_name = _('Server ID'), default='es01', values=( - ('es01','es01'), - ('es02','es02'), - ), primary_key=True) + server_id = EnumField(verbose_name=_('Server ID'), default='es01', values=( + ('es01', 'es01'), + ('es02', 'es02'), + ), primary_key=True) host_name = CharField(verbose_name=_('Host Name')) - aaa_port = PortField(verbose_name=_('Port')) + aaa_port = PortField(verbose_name=_('Port')) secret = CharField(verbose_name=_('Secret'), optional=True) class Meta: @@ -211,9 +231,11 @@ def _get_query_set(self): self.cli.set_enable() result = self.cli.cmd('show admin aaa all', - RegexParser('admin aaa server (?Pes01|es02) "(?P.*?)" (?P[0-9]+) "(?P.*?)"', MATCHALL, reflags=re.S)) + RegexParser( + 'admin aaa server (?Pes01|es02) "(?P.*?)" (?P[0-9]+) "(?P.*?)"', + MATCHALL, reflags=re.S)) for data in result: - self._model._meta.mark_delay_query(data) + self._model._meta.mark_delay_query(data) return QuerySet(self._model, result) def _filter(self, filter_dict): @@ -226,9 +248,9 @@ data = instance.get_field_dict() self.cli.set_config() result = self.cli.cmd('admin aaa server %(server_id)s "%(host_name)s" %(aaa_port)u "%(secret)s"' % data, - BlankParser(nonblank_exception=CLICmdError, supplement=True)) + BlankParser(nonblank_exception=CLICmdError, supplement=True)) return result - + def _delete(self, pk_list): self.cli.set_config() for each in pk_list: @@ -241,5 +263,5 @@ data = instance.get_field_dict() self.cli.set_config() result = self.cli.cmd('admin aaa server %(server_id)s "%(host_name)s" %(aaa_port)u "%(secret)s"' % data, - BlankParser(nonblank_exception=CLICmdError, supplement=True)) - return result \ No newline at end of file + BlankParser(nonblank_exception=CLICmdError, supplement=True)) + return result Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py (working copy) @@ -1,7 +1,9 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ +from hive.imports.model import * + + class BackupConfig(ANModel): class BackupStartupConfig(Action): verbose_name = _('Backup Startup Configuration') @@ -23,9 +25,9 @@ class Manager(CLIManager): def _get_query_set(self): return QuerySet(self._model, [{}]) - + def _perform_BackupStartupConfig(self): self.cli.set_config() result = self.cli.cmd('write memory', - BlankParser(nonblank_exception=CLICmdError, supplement=True)) - return result \ No newline at end of file + BlankParser(nonblank_exception=CLICmdError, supplement=True)) + return result Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py (working copy) @@ -1,9 +1,12 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ -from djproject.an_settings import CM_ConfigFile_PATH -import os, cm.conf from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ +import cm.conf +from hive.imports.model import * + +__ = _ + + class HostSettings(ANModel): hostname = CharField(verbose_name=_('Host Name'), length='1..64') vsite_name = CharField(verbose_name=_('Vsite Name'), length='1..64') @@ -17,21 +20,21 @@ process_title = "" forever = False config_change = True - + class Meta: profile = True verbose_name = _('Host Settings') - + class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable() output = self.cli.cmd('show host') data = { 'hostname': output.strip(), - 'vsite_name':cm.conf.VSITE_NAME, + 'vsite_name': cm.conf.VSITE_NAME, } return QuerySet(self._model, [data]) - + def _update_hostname(self, instance): self.cli.set_config() result = self.cli.cmd('host "%s"' % instance.hostname, @@ -39,15 +42,15 @@ return def _update_vsite_name(self, instance): - #sed -i -e 's|NETWORK_CARD = \'ens32\'|NETWORK_CARD = \'abcdefg\'|' __init__.py + # sed -i -e 's|NETWORK_CARD = \'ens32\'|NETWORK_CARD = \'abcdefg\'|' __init__.py file_path = CM_ConfigFile_PATH + '/__init__.py' - str_cli = "sed -i -e \"s|VSITE_NAME = '%s'|VSITE_NAME = '%s'|\" %s" % (cm.conf.VSITE_NAME, instance.vsite_name, file_path) + str_cli = "sed -i -e \"s|VSITE_NAME = '%s'|VSITE_NAME = '%s'|\" %s" % (cm.conf.VSITE_NAME, + instance.vsite_name, file_path) if os.system(str_cli) != 0: raise ModelQueryException(CLICmdError(__('Update vsite name failed!'))) cm.conf.VSITE_NAME = instance.vsite_name - return + return - def _perform_Reset(self, options): self.cli.set_config() result = self.cli.cmd('no hostname') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py (working copy) @@ -1,27 +1,30 @@ -from hive.imports.model import * +import datetime + +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -from hive.session import check_license_expire + from cm.lib.postgres_db import DB from djproject.an_settings import * -import datetime -from django.db.models.query import QuerySet -__=_ +from hive.imports.model import * +from hive.session import check_license_expire +__ = _ + def get_license(): session = get_current_session() session.cli.set_enable() license_status = { "device_limit": 0, - "adc_limit": 0, - "vpn_limit": 0, - "waf_limit": 0 + "adc_limit": 0, + "vpn_limit": 0, + "waf_limit": 0 } result = session.cli.cmd('show version', [ - RegexParser('Licensed Devices : (?P[0-9]*)', MATCHONE), - RegexParser('Licensed ADCs : (?P[0-9]*)', MATCHONE), - RegexParser('Licensed VPNs : (?P[0-9]*)', MATCHONE), - RegexParser('Licensed WAFs : (?P[0-9]*)', MATCHONE)]) + RegexParser('Licensed Devices : (?P[0-9]*)', MATCHONE), + RegexParser('Licensed ADCs : (?P[0-9]*)', MATCHONE), + RegexParser('Licensed VPNs : (?P[0-9]*)', MATCHONE), + RegexParser('Licensed WAFs : (?P[0-9]*)', MATCHONE)]) if result: for each in result: if "device_limit" in each and each["device_limit"]: @@ -34,12 +37,14 @@ license_status["waf_limit"] = int(each["waf_limit"]) return license_status + def _check_license_expire(): session = get_current_session() res = check_license_expire(session) return res + def get_device_num(device_type): condition_list = [] if device_type.lower() in ADC_TYPE_LIST: @@ -65,8 +70,9 @@ db.close() return result[0] else: - return 0 + return 0 + def check_license(device_type): expire = _check_license_expire() if expire[0]: @@ -90,6 +96,7 @@ else: raise ModelQueryException(CLICmdError(__('Invalid device type.'))) + class License(ANModel): license_key = CharField(verbose_name='License Key', length='1..256', optional=True) serial_num = CharField(verbose_name='Serial Number') @@ -109,30 +116,31 @@ def _get_query_set(self): self.cli.set_enable() output = self.cli.cmd('show version', - [RegexParser('License Key : (?P.*)', MATCHONE), - RegexParser('Serial Number : (?P.*)', MATCHONE), - RegexParser('Licensed Devices : (?P[0-9]+)', MATCHONE), - RegexParser('Expiration Date : (?P.*)', MATCHONE), - RegexParser('Licensed ADCs : (?P.*)', MATCHONE), - RegexParser('Licensed VPNs : (?P.*)', MATCHONE), - RegexParser('Licensed WAFs : (?P.*)', MATCHONE), - RegexParser('Licensed VPN License Session : (?P.*)', MATCHONE)]) + [RegexParser('License Key : (?P.*)', MATCHONE), + RegexParser('Serial Number : (?P.*)', MATCHONE), + RegexParser('Licensed Devices : (?P[0-9]+)', MATCHONE), + RegexParser('Expiration Date : (?P.*)', MATCHONE), + RegexParser('Licensed ADCs : (?P.*)', MATCHONE), + RegexParser('Licensed VPNs : (?P.*)', MATCHONE), + RegexParser('Licensed WAFs : (?P.*)', MATCHONE), + RegexParser('Licensed VPN License Session : (?P.*)', MATCHONE)]) expire_time = "" if output[3]: exp_str = output[3]['expiration_data'].split('on')[-1].strip() - expire_time = datetime.datetime.strptime(exp_str, '%b %d %Y').strftime("%Y-%m-%d") if exp_str != "Permanent" else "Permanent" + expire_time = datetime.datetime.strptime(exp_str, '%b %d %Y').strftime( + "%Y-%m-%d") if exp_str != "Permanent" else "Permanent" data = { - 'license_key':output[0]['license_key'] if output[0] else '', - 'serial_num':output[1]['serial_num'], - 'device_num':output[2]['device_num'] if output[2] else 0, - 'adc_num':output[4]['adc_num'] if output[4] else 0, - 'vpn_num':output[5]['vpn_num'] if output[5] else 0, - 'waf_num':output[6]['waf_num'] if output[6] else 0, + 'license_key': output[0]['license_key'] if output[0] else '', + 'serial_num': output[1]['serial_num'], + 'device_num': output[2]['device_num'] if output[2] else 0, + 'adc_num': output[4]['adc_num'] if output[4] else 0, + 'vpn_num': output[5]['vpn_num'] if output[5] else 0, + 'waf_num': output[6]['waf_num'] if output[6] else 0, 'vpn_lic_sess': output[7]['vpn_lic_sess'] if output[7]['vpn_lic_sess'] != "No Limitation" else 99999999, 'expiration_data': expire_time, - 'status':'' - } + 'status': '' + } if data['expiration_data']: data['status'] = 'Normal' if data['expiration_data'] != 'Permanent': @@ -143,18 +151,12 @@ data['status'] = 'Expiration' elif result.days < 30: data['status'] = 'Warning' - + return QuerySet(self._model, [data]) + def _update_license_key(self, instance): self.cli.set_config() output = self.cli.cmd('system license "%s"' % instance.license_key.strip('"')) if output.lower().find('failed') != -1: result = cli_parse(output, BlankParser(nonblank_exception=CLICmdError, supplement=True)) return - - - - - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py (working copy) @@ -1,28 +1,33 @@ -from hive.imports.model import * +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from hive.imports.model import * from hive.model.query import mark_expire_all -from django.db.models.query import QuerySet + class NTPSettings(ANModel): enable_ntp = BooleanField(verbose_name=_('Enable NTP')) - ntp_server = AssoField2(verbose_name=_('NTP Servers'), tgt='system.NTPServer.asso', mul=1, pos='left', optional=True) + ntp_server = AssoField2(verbose_name=_('NTP Servers'), tgt='system.NTPServer.asso', mul=1, pos='left', + optional=True) ntp_stats = FieldGroup(writable=False, verbose_name=_('NTP Statistics'), level=STATS, - fields={ - 'ntp_stats': CLITextField(verbose_name=_('NTP Statistics')), - }) + fields={ + 'ntp_stats': CLITextField(verbose_name=_('NTP Statistics')), + }) stats = FieldGroup(writable=False, verbose_name=_('Ntp Statistics'), level=STATS, stats_source="userland", - fields={ - 'sysstats_reset':StorableField(verbose_name=_('system reset statistics'), fields=Uint64Field()), - 'packets_received':StorableField(verbose_name=_('Packets received'), fields=Uint64Field()), - 'bad':StorableField(verbose_name=_('Bad length or format'), fields=Uint64Field()), - 'auth_failed':StorableField(verbose_name=_('authentication failed'), fields=Uint64Field()), - 'declined':StorableField(verbose_name=_('Declined'), fields=Uint64Field()), - 'restricted':StorableField(verbose_name=_('Restricted'), fields=Uint64Field()), - 'rate_limited':StorableField(verbose_name=_('Rate limited'), fields=Uint64Field()), - 'KoD_responses':StorableField(verbose_name=_('KoD responses'), fields=Uint64Field()), - 'processed_for_time':StorableField(verbose_name=_('Processed for time'), fields=Uint64Field()), - }) + fields={ + 'sysstats_reset': StorableField(verbose_name=_('system reset statistics'), + fields=Uint64Field()), + 'packets_received': StorableField(verbose_name=_('Packets received'), fields=Uint64Field()), + 'bad': StorableField(verbose_name=_('Bad length or format'), fields=Uint64Field()), + 'auth_failed': StorableField(verbose_name=_('authentication failed'), fields=Uint64Field()), + 'declined': StorableField(verbose_name=_('Declined'), fields=Uint64Field()), + 'restricted': StorableField(verbose_name=_('Restricted'), fields=Uint64Field()), + 'rate_limited': StorableField(verbose_name=_('Rate limited'), fields=Uint64Field()), + 'KoD_responses': StorableField(verbose_name=_('KoD responses'), fields=Uint64Field()), + 'processed_for_time': StorableField(verbose_name=_('Processed for time'), + fields=Uint64Field()), + }) class Clear_ntp(Action): verbose_name = _('Clear') @@ -39,19 +44,19 @@ class Meta: profile = True verbose_name = _('NTP Settings') - + class Manager(CLIManager): def _get_query_set(self): self.cli.set_enable() self.cli.set_enable() result = self.cli.cmd('show ntp', - RegexParser('ntp (?Pon|off)', MATCHONE, reflags=re.S)) + RegexParser('ntp (?Pon|off)', MATCHONE, reflags=re.S)) if result: rtn_dict = { - 'enable_ntp':True if result['enable_ntp'] == 'on' else False - } + 'enable_ntp': True if result['enable_ntp'] == 'on' else False + } else: - rtn_dict = { 'enable_ntp':False } + rtn_dict = {'enable_ntp': False} self._model._meta.mark_delay_query(rtn_dict) return QuerySet(self._model, [rtn_dict]) @@ -78,13 +83,14 @@ result = self.cli.cmd('show ntp') return result + class NTPServer(ANModel): ip = IPAddressField(verbose_name=_('NTP Server IP'), primary_key=True) version = EnumField(verbose_name=_('NTP Server Version'), default=4, primary_key=True, optional=True, values=( - (1, '1'), - (2, '2'), - (3, '3'), - (4, '4'))) + (1, '1'), + (2, '2'), + (3, '3'), + (4, '4'))) asso = AssoField2(tgt='system.NTPSettings.ntp_server', mul='*', pos='right', hidden=True, default=[{}]) class Meta: @@ -94,13 +100,13 @@ def _get_query_set(self): self.cli.set_enable() result = self.cli.cmd('show ntp\n', - EasyParser('ntp server ', ['?ip', '?version'])) + EasyParser('ntp server ', ['?ip', '?version'])) return QuerySet(self._model, result) def _insert(self, instance): data = { - 'ip':instance.ip.values()[0], - 'version':instance.version, + 'ip': instance.ip.values()[0], + 'version': instance.version, } self.cli.set_config() result = self.cli.cmd('ntp server %(ip)s %(version)u' % data, Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py (working copy) @@ -1,8 +1,9 @@ -from cm.router import _extension_log +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from cm.router import _extension_log from hive.imports.model import * from hive.utils import andebug, update_frontend_index_html -from django.db.models.query import QuerySet __ = _ @@ -142,4 +143,3 @@ self.cli.set_config() self.cli.cmd('system reboot noninteractive') return - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py (working copy) @@ -1,9 +1,11 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ from datetime import datetime + from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ +from hive.imports.model import * + class TimeSettings(ANModel): system_time = DateTimeField(verbose_name=_('System Time'), cache_expire_time=10) system_timezone = TimezoneField(verbose_name=_('System Timezone'), cache_expire_time=10) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py (working copy) @@ -1,11 +1,12 @@ import datetime -import json -from cm.lib.postgres_db import DB +from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ + +from cm.lib.postgres_db import DB from hive.imports.model import * from hive.model.query import clear_cache_all -from django.db.models.query import QuerySet +from hive.session import ANSession __ = _ @@ -14,10 +15,10 @@ default = FieldGroup(editable=False, fields={ 'username': CharField(primary_key=True, length='1..16'), 'passwords': PasswordField(length='1..128', verbose_name='Password', editable=True, - pattern='^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9]).{8,}$', - error_messages={'invalid': 'The password must be at least 8 characters in length ' - 'and consist of a combination of the following: ' - 'uppercase, lowercase and numeric.'}), + pattern='^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9]).{8,}$', + error_messages={'invalid': 'The password must be at least 8 characters in length ' + 'and consist of a combination of the following: ' + 'uppercase, lowercase and numeric.'}), 'email': CharField(length='1..128', optional=True, editable=True), 'phone_num': CharField(length='1..128', optional=True, editable=True), 'level': EnumField(verbose_name=_('User Level'), default='config', values=( Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py (working copy) @@ -1,17 +1,19 @@ -from hive.imports.model import * -from django.utils.translation import gettext_lazy as _ -#from cm.lib.sqlite_db import DB -from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import TaskScheduler, update_job_runtime, upload_job, append_job_description, Scheduler_func_list, update_job_state -import datetime, json -from apscheduler.schedulers.background import BackgroundScheduler -from cm.lib.parse_configfile import DEFAULT_CONFIG_FILE_PATH +import datetime import shutil + +from apscheduler.schedulers.background import BackgroundScheduler from django.db.models.query import QuerySet -import os -__=_ +from django.utils.translation import gettext_lazy as _ + +from cm.lib.postgres_db import DB +from cm.lib.task_scheduler import TaskScheduler, update_job_runtime, append_job_description, \ + Scheduler_func_list, update_job_state +from hive.imports.model import * + +__ = _ GLOBAL_TASK = TaskScheduler() + class Tasks(ANModel): id = IntegerField(primary_key=True) name = CharField(verbose_name='Task Name', length='1..64', primary_key=True) @@ -21,22 +23,22 @@ ('config_backup', 'Config Backup'), ('config_custom', 'Config Custom'), ('config_clone', 'Config Clone'), - ('config_recover', 'Config Recover'), - ('config_init', 'Config Init'), - ('config_check', 'Config Check'), - ('config', 'Config'), - ('sync_role', 'Sync Role'), - ('sync_acl', 'Sync Acl'), - ('sync_vpn', 'Sync VPN'), - ('sync_any', 'Sync Any') + ('config_recover', 'Config Recover'), + ('config_init', 'Config Init'), + ('config_check', 'Config Check'), + ('config', 'Config'), + ('sync_role', 'Sync Role'), + ('sync_acl', 'Sync Acl'), + ('sync_vpn', 'Sync VPN'), + ('sync_any', 'Sync Any') ), primary_key=True) - #trigger = EnumField(verbose_name='Trigger', ) + # trigger = EnumField(verbose_name='Trigger', ) state = EnumField(verbose_name='Status', default='waiting', values=( ('done', 'Done'), - ('ongoing', 'Ongoing'), + ('ongoing', 'Ongoing'), ('waiting', 'Waiting'), ('failed', 'Failed'), - ), primary_key=True) + ), primary_key=True) next_run_time = CharField(verbose_name='Execution time', length='1..64') description = TextField(verbose_name='Description', length='1..10240', optional=True) result_field = TextField(verbose_name='Result Field', length='1..1024', optional=True) @@ -46,16 +48,16 @@ class Meta: verbose_name = 'Tasking' show_im_export_button = False - list_config_options={ - 'columns':[ - {'name':'name'}, - {'name':'type'}, - {'name':'state'}, - {'name':'next_run_time'}, - {'name':'description'}, - {'name':'result_field'}, - {'name':'device_list'} - ],'add_button_hide':True, 'click_enable':False} + list_config_options = { + 'columns': [ + {'name': 'name'}, + {'name': 'type'}, + {'name': 'state'}, + {'name': 'next_run_time'}, + {'name': 'description'}, + {'name': 'result_field'}, + {'name': 'device_list'} + ], 'add_button_hide': True, 'click_enable': False} class Modify(Action): verbose_name = _('Modify Job') @@ -67,6 +69,7 @@ option_fields = ( DateTimeField(verbose_name=_('Expire Time'), name='expire_time'), ) + class Restart(Action): verbose_name = _('Restart Job') action_name = _('Restart') @@ -78,8 +81,10 @@ EnumField(verbose_name='Option', default='immediate', name='option', values=( ('immediate', 'Immediate'), ('deferred', 'Deferred'), )), - DateTimeField(verbose_name='Expire Time', name='expire_time', condition=ValueCondition('option', ['deferred'])), + DateTimeField(verbose_name='Expire Time', name='expire_time', + condition=ValueCondition('option', ['deferred'])), ) + class Clear(Action): verbose_name = _('Clear') action_name = _('Clear') @@ -94,7 +99,8 @@ fetchall_sql = '''SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task order by next_run_time;''' data = db.fetchall(fetchall_sql) db.close() - key = ['id', 'name', 'type', 'state', 'next_run_time', 'description', 'result_field', 'device_list', 'scheduler'] + key = ['id', 'name', 'type', 'state', 'next_run_time', 'description', 'result_field', 'device_list', + 'scheduler'] result = [dict(zip(key, each)) for each in data] date_res = [] interval_res = [] @@ -105,6 +111,7 @@ date_res.append(each) res = sorted(date_res, key=lambda x: x["next_run_time"]) + interval_res return QuerySet(self._model, res) + def _delete_instance(self, instance): data = instance.get_field_dict() db = DB.get_connected_db() @@ -152,15 +159,17 @@ def _perform_Modify(self, options): pk_list = options['__pk_list'][0] if pk_list['state'] == 'waiting': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < datetime.datetime.now(): #maybe need delay 3 seconds - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - + if date < datetime.datetime.now(): # maybe need delay 3 seconds + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + task_name = pk_list['name'] sched = GLOBAL_TASK.get_schedule(task_name) if sched: - #todo if not find the sched, restart it immediate + # todo if not find the sched, restart it immediate jobs = sched.get_jobs()[-1] jobs.modify(next_run_time=date) update_job_runtime(task_name, date.strftime('%Y-%m-%d %H:%M:%S')) @@ -180,6 +189,7 @@ return data[0][0] else: return None + pk_list = options['__pk_list'][0] if pk_list['state'] != 'failed': @@ -190,20 +200,23 @@ if not scheduler_str: raise ModelQueryException(CLICmdError(__('Error can\'t find this scheduler!'))) new_scheduler = json.loads(scheduler_str) - + if options['option'] == 'immediate': date = datetime.datetime.now() + datetime.timedelta(seconds=3) elif options['option'] == 'deferred': - time_str = options['expire_time'][:19] #'2014-03-28T06:59:52' + time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' date = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S') - if date < datetime.datetime.now(): - raise ModelQueryException(CLICmdError(__('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - + if date < datetime.datetime.now(): + raise ModelQueryException(CLICmdError( + __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) + task_name = pk_list['name'] update_job_runtime(task_name, date.strftime('%Y-%m-%d %H:%M:%S')) sched = BackgroundScheduler() - sched.add_job(Scheduler_func_list[new_scheduler['func']], 'date', next_run_time=date, args=new_scheduler['args'], id=task_name) + sched.add_job(Scheduler_func_list[new_scheduler['func']], 'date', next_run_time=date, + args=new_scheduler['args'], id=task_name) description = "Restart time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' append_job_description(task_name, description) @@ -213,12 +226,3 @@ GLOBAL_TASK.remove(task_name) GLOBAL_TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) - -# class Task_update(ANModel): -# pass - -# class Task_config(ANModel): -# pass - -# class Task_upload(ANModel): -# pass Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py (working copy) @@ -1,11 +1,10 @@ -from hive.imports.model import * -from django.utils.translation import ugettext_lazy as _ -from hive.utils import andebug -from cm.lib.postgres_db import DB -from cm.lib.communication import call_restapi -import uuid, re, json, os, time from django.db.models.query import QuerySet +from django.utils.translation import gettext_lazy as _ +from cm.lib.communication import call_restapi +from cm.lib.postgres_db import DB +from hive.imports.model import * + __ = _ @@ -186,7 +185,7 @@ device_id = instance.id db = DB.get_connected_db() update_sql = "UPDATE host SET restapi_username='%s', restapi_password='%s' WHERE id='%s'" % ( - instance.restapi_account['restapi_username'], instance.restapi_account['restapi_password'], device_id) + instance.restapi_account['restapi_username'], instance.restapi_account['restapi_password'], device_id) db.execute_sql(update_sql) db.close() return @@ -238,4 +237,4 @@ save_sql = "UPDATE host SET version = '%s' WHERE id = '%s'" % (version, device_id) db.execute_sql(save_sql) db.close() - return \ No newline at end of file + return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (working copy) @@ -1,29 +1,31 @@ -import os -from django.http import HttpResponse -from hive.utils import andebug, HiveEnvironment, get_current_session, update_frontend_index_html, get_device_type -from django.shortcuts import redirect -import json import base64 -from djproject.an_settings import * -from cm.lib.communication import send_https_rest_request, modify_url -import time, datetime +import datetime import re -from jinja2 import Environment, PackageLoader, ChoiceLoader +import subprocess +import time +from importlib import reload + +from django.http import HttpResponse from django.template import RequestContext -from cm.lib.postgres_db import DB -from hive.model.loading import get_model -from hive.model.query import mark_expire_all from django.views.decorators.csrf import csrf_exempt -from hive.session import temp_session, current_app, check_license_expire -import subprocess -from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE, EasyParser -from cm.lib.libbasic_operation import get_rest_info_from_device, send_command_to_device, oper_log, send_cli_to_device -from cm.models.tasking import GLOBAL_TASK +from jinja2 import PackageLoader + +from cm.lib.communication import send_https_rest_request, modify_url +from cm.lib.libbasic_operation import get_rest_info_from_device, send_command_to_device, oper_log, send_cli_to_device, \ + get_ip_address +from cm.lib.parse_configfile import TarFiles from cm.lib.task_scheduler import remove_job_from_database +from cm.models.tasking import GLOBAL_TASK +from djproject.an_settings import * from hive.log_utils import * -from cm.lib.parse_configfile import TarFiles +from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE +from hive.model.loading import get_model +from hive.model.query import mark_expire_all +from hive.session import temp_session, check_license_expire +from hive.utils import HiveEnvironment, get_current_session, update_frontend_index_html, get_device_type -#/cm/configuration/configuration_file/[save_config | get_config]/file_type/file_name + +# /cm/configuration/configuration_file/[save_config | get_config]/file_type/file_name def get_configuration(request, params): default_dir = DEFAULT_CONFIG_FILE_PATH if not os.path.isdir(default_dir): @@ -42,8 +44,9 @@ f.write(save_config_content) f.close() os.remove(file_path + '/' + file_name + '.bak') - #update file modify time - update_sql = "UPDATE file_list SET modify_time='%s' where name='%s'" % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), file_name) + # update file modify time + update_sql = "UPDATE file_list SET modify_time='%s' where name='%s'" % ( + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), file_name) db = DB.get_connected_db() db.execute_sql(update_sql) db.close() @@ -77,17 +80,26 @@ if response_data['message'] == 'success': return HttpResponse(json.dumps([True, response_data['data']]), content_type='application/json') elif response_data['message'] == 'requestfailed': - return HttpResponse(json.dumps([False, 'Failed to send request to device. Please check the connection of device.']), content_type='application/json') + return HttpResponse( + json.dumps([False, 'Failed to send request to device. Please check the connection of device.']), + content_type='application/json') elif response_data['message'] == 'parsefailed': - return HttpResponse(json.dumps([False, 'Failed to parse the response from device. Please check device restful API service.']), content_type='application/json') + return HttpResponse(json.dumps( + [False, 'Failed to parse the response from device. Please check device restful API service.']), + content_type='application/json') elif response_data['message'] == 'parsefailed': - return HttpResponse(json.dumps([False, 'Failed to send request to device. Please check restapi username and password of device.']), content_type='application/json') - return HttpResponse(json.dumps([True, 'Failed to get configuration from device because interval error.']), content_type='application/json') + return HttpResponse(json.dumps([False, + 'Failed to send request to device. Please check restapi username and password of device.']), + content_type='application/json') + return HttpResponse( + json.dumps([True, 'Failed to get configuration from device because interval error.']), + content_type='application/json') else: file_type = "customize" if file_type == "template" else file_type file_path = default_dir + file_type + '/' + file_name if not os.path.exists(file_path): - return HttpResponse(json.dumps([False, 'The Config File Can not Find!']), content_type='application/json') + return HttpResponse(json.dumps([False, 'The Config File Can not Find!']), + content_type='application/json') try: f = open(file_path, 'r') content = f.read() @@ -98,12 +110,14 @@ except Exception as e: if f: f.close() - return HttpResponse(json.dumps([False, 'Read Config File Failed!']), content_type='application/json') + return HttpResponse(json.dumps([False, 'Read Config File Failed!']), + content_type='application/json') elif params.split('/')[0] == 'get_output': task_name = params.split('/')[2] file_path = "%soutput/%s" % (default_dir, task_name) if not os.path.exists(file_path): - return HttpResponse(json.dumps([False, 'The Output File Can not Find!']), content_type='application/json') + return HttpResponse(json.dumps([False, 'The Output File Can not Find!']), + content_type='application/json') try: res_type = "text" f = open(file_path, 'r') @@ -128,14 +142,18 @@ device_name = '/'.join(params.split('/')[1:]) device_info = get_rest_info_from_device(device_name) # device_info [{'restapi_port': 9997, 'restapi_password': u'admin', 'ip_address': u'10.8.2.54', 'restapi_username': u'array', 'type': u'AG'}] - + response_data = send_cli_to_device(device_info[0], 'show virtual site name', mode='enable') if response_data['message'] == 'requestfailed': - return HttpResponse(json.dumps([False, 'Send request to device failed!']), content_type='application/json') + return HttpResponse(json.dumps([False, 'Send request to device failed!']), + content_type='application/json') elif response_data['message'] == 'failed': - return HttpResponse(json.dumps([False, 'Get Config from device failed!']), content_type='application/json') + return HttpResponse(json.dumps([False, 'Get Config from device failed!']), + content_type='application/json') - vsite_parser = RegexParser('virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', MATCHALL) + vsite_parser = RegexParser( + 'virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', + MATCHALL) ret = [] if device_info[0]['protocol'] == 'xmlrpc': output = cli_parse(response_data['data'], vsite_parser) @@ -148,6 +166,7 @@ return HttpResponse(json.dumps([False, 'Get config file failed!']), content_type='application/json') + def encode(key, clear): enc = [] for i in range(len(clear)): @@ -156,6 +175,7 @@ enc.append(enc_c) return base64.urlsafe_b64encode("".join(enc)) + def large_screen(request): django_ctx = RequestContext(request) env = HiveEnvironment(loader=PackageLoader('vclient', 'dist')) @@ -163,6 +183,7 @@ ctx = {'csrf_token': django_ctx['csrf_token']} return HttpResponse(template.render(ctx)) + def vnc_host_info(request): host_id = json.loads(request.POST["data"])["host_id"] expires = json.loads(request.POST["data"])["expires"] @@ -185,8 +206,9 @@ "expires": expires })) } - return HttpResponse(json.dumps(res), content_type='application/json') + return HttpResponse(json.dumps(res), content_type='application/json') + def device_version(request): device_id = request.POST["device_id"] @@ -201,8 +223,9 @@ res["state"] = True res["version"] = version[0][0] - return HttpResponse(json.dumps(res), content_type='application/json') + return HttpResponse(json.dumps(res), content_type='application/json') + def get_enable_mode(request): auto_mode = False @@ -216,6 +239,7 @@ return HttpResponse(json.dumps({"auto_mode": auto_mode}), content_type='application/json') + def set_enable_mode(request): auto_mode = json.loads(request.POST["auto_mode"]) state = False @@ -234,26 +258,36 @@ return HttpResponse(json.dumps({"state": state, "msg": msg}), content_type='application/json') + def get_available_extension(request): available_ext_list = [] subprocess.getstatusoutput("yum --disablerepo=* --enablerepo=array clean metadata") check_status, check_output = subprocess.getstatusoutput("yum check-update --disablerepo=* --enablerepo=array") if check_status == 256: - if check_output.find("Timeout")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository because connect to the repository timeout."}), content_type='application/json') - elif check_output.find("Cannot find a valid baseurl for repo")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Please set yum repository by CLI 'system repo '."}), content_type='application/json') - elif check_output.find("Could not resolve host")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository. Please check IP nameserver and yum repository URL."}), content_type='application/json') - elif check_output.find("Connection refused")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository. Please check the port of yum repository URL."}), content_type='application/json') + if check_output.find("Timeout") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository because connect to the repository timeout."}), + content_type='application/json') + elif check_output.find("Cannot find a valid baseurl for repo") != -1: + return HttpResponse( + json.dumps({"state": False, "data": "Please set yum repository by CLI 'system repo '."}), + content_type='application/json') + elif check_output.find("Could not resolve host") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository. Please check IP nameserver and yum repository URL."}), + content_type='application/json') + elif check_output.find("Connection refused") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository. Please check the port of yum repository URL."}), + content_type='application/json') else: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository."}), + content_type='application/json') a_status, a_output = subprocess.getstatusoutput("yum --disablerepo=* --enablerepo=array list available") a_parser = RegexParser('(?P\S+)\s+(?P\S+)\s+(?P\S+)', MATCHALL, reflags=re.S) - if a_output.find("Available Packages")!= -1: + if a_output.find("Available Packages") != -1: available_ext = a_output.split("Available Packages")[1] else: available_ext = '' @@ -275,21 +309,23 @@ if available_ext_list: res = [] for each_pack in available_ext_list: - status, output = subprocess.getstatusoutput('yum info --disablerepo="*" --enablerepo="array" %s' % each_pack) + status, output = subprocess.getstatusoutput( + 'yum info --disablerepo="*" --enablerepo="array" %s' % each_pack) if status: - return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), + content_type='application/json') else: if output: b_parser = RegexParser('Available Packages\nName\s*:\s*(?P[^\n]*)\n' + - 'Arch\s*:\s*(?P[^\n]*)\n' + - 'Version\s*:\s*(?P[^\n]*)\n' + - 'Release\s*:\s*(?P[^\n]*)\n' + - 'Size\s*:\s*(?P[^\n]*)\n' + - 'Repo\s*:\s*(?P[^\n]*)\n' + - 'Summary\s*:\s*(?P[^\n]*)\n' + - 'URL\s*:\s*(?P[^\n]*)\n' + - 'License\s*:\s*(?P[^\n]*)\n' + - 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) + 'Arch\s*:\s*(?P[^\n]*)\n' + + 'Version\s*:\s*(?P[^\n]*)\n' + + 'Release\s*:\s*(?P[^\n]*)\n' + + 'Size\s*:\s*(?P[^\n]*)\n' + + 'Repo\s*:\s*(?P[^\n]*)\n' + + 'Summary\s*:\s*(?P[^\n]*)\n' + + 'URL\s*:\s*(?P[^\n]*)\n' + + 'License\s*:\s*(?P[^\n]*)\n' + + 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) result = cli_parse(output, b_parser) if result and result[0]: @@ -298,6 +334,7 @@ else: return HttpResponse(json.dumps({"state": True, "data": []}), content_type='application/json') + def check_extension_obsoleted(ext_name): obsoleted_ext_name = "array_" + ext_name s, o = subprocess.getstatusoutput("rpm -aq | grep %s" % obsoleted_ext_name) @@ -305,10 +342,14 @@ return 1 return 0 + def install_extension(request, name): if check_extension_obsoleted(name): _extension_log(name, "install", "failed") - return HttpResponse(json.dumps({"state": False, "data": "Error: Failed to install extension '%s' beacause there is a obsoleted extension '%s'.Current system version do not support extension '%s' any more. Please remove it first and try to install '%s'." % (name, "array_" + name, "array_" + name, name)}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, + "data": "Error: Failed to install extension '%s' beacause there is a obsoleted extension '%s'.Current system version do not support extension '%s' any more. Please remove it first and try to install '%s'." % ( + name, "array_" + name, "array_" + name, name)}), + content_type='application/json') # check the dependency dep = check_extension_dep(name) if not dep[0]: @@ -320,28 +361,33 @@ if status: _extension_log(name, "install", "failed") oper_log('error', 'extensions', 'Failed to install the extension %s.' % name) - return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), + content_type='application/json') else: update_frontend_index_html() return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') + def remove_extension(request, name): status, output = subprocess.getstatusoutput("/bin/sh %sext_opr.sh remove %s" % (webui_path, name)) if status: _extension_log(name, "remove", "failed") oper_log('error', 'extensions', 'Failed to remove the extension %s.' % name) - return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), + content_type='application/json') else: oper_log('info', 'extensions', 'Remove the extension %s successfully.' % name) update_frontend_index_html() return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') + def check_extension_dep(ext_name): status, output = subprocess.getstatusoutput("yum deplist %s" % ext_name) if status: return [False, "Failed to check the extension dependency."] else: - ext_dep_parser = RegexParser('dependency:\s+(?P[^\n]+)\n\s+Unsatisfied dependency', MATCHALL, reflags=re.S) + ext_dep_parser = RegexParser('dependency:\s+(?P[^\n]+)\n\s+Unsatisfied dependency', MATCHALL, + reflags=re.S) result = cli_parse(output, ext_dep_parser) if len(result) == 0: @@ -349,6 +395,7 @@ else: return [False, "Unsatisfied dependency: %s" % (", ").join([each["dependency"] for each in result])] + def update_extension(request, name): # check the dependency dep = check_extension_dep(name) @@ -361,12 +408,14 @@ if status: _extension_log(name, "update", "failed") oper_log('error', 'extensions', 'Failed to update the extension %s.' % name) - return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), + content_type='application/json') else: oper_log('info', 'extensions', 'Update the extension %s successfully.' % name) update_frontend_index_html() return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') + def get_extensions(request): get_sidebar = False check_status = 0 @@ -379,51 +428,53 @@ check_status, check_output = subprocess.getstatusoutput("yum check-update --disablerepo=* --enablerepo=array") session = get_current_session() session.cli.set_enable() - system_version = session.cli.cmd("show version", RegexParser('AMP\s+(?P[^\.]+\.AMP(\.[0-9]+)+)'))["version"] - + system_version = session.cli.cmd("show version", RegexParser('AMP\s+(?P[^\.]+\.AMP(\.[0-9]+)+)'))[ + "version"] + extensions_res = [] extension_list = [] root = extension_path available_ext_parser = RegexParser('Name\s*:\s*(?P[^\n]*)\n' + - 'Arch\s*:\s*(?P[^\n]*)\n' + - 'Version\s*:\s*(?P[^\n]*)\n' + - 'Release\s*:\s*(?P[^\n]*)\n' + - 'Size\s*:\s*(?P[^\n]*)\n' + - 'Repo\s*:\s*(?P[^\n]*)\n' + - 'Summary\s*:\s*(?P[^\n]*)\n' + - 'URL\s*:\s*(?P[^\n]*)\n' + - 'License\s*:\s*(?P[^\n]*)\n' + - 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) + 'Arch\s*:\s*(?P[^\n]*)\n' + + 'Version\s*:\s*(?P[^\n]*)\n' + + 'Release\s*:\s*(?P[^\n]*)\n' + + 'Size\s*:\s*(?P[^\n]*)\n' + + 'Repo\s*:\s*(?P[^\n]*)\n' + + 'Summary\s*:\s*(?P[^\n]*)\n' + + 'URL\s*:\s*(?P[^\n]*)\n' + + 'License\s*:\s*(?P[^\n]*)\n' + + 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) ext_info_parse = RegexParser('Name\s*:\s*(?P[^\n]*)\n' + - 'Version\s*:\s*(?P[^\n]*)\n' + - 'Release\s*:\s*(?P[^\n]*)\n' + - 'Architecture\s*:\s*(?P[^\n]*)\n' + - 'Install Date\s*:\s*(?P[^\n]*)\n' + - 'Group\s*:\s*(?P[^\n]*)\n' + - 'Size\s*:\s*(?P[^\n]*)\n' + - 'License\s*:\s*(?P[^\n]*)\n' + - 'Signature\s*:\s*(?P[^\n]*)\n' + - 'Source RPM\s*:\s*(?P[^\n]*)\n' + - 'Build Date\s*:\s*(?P[^\n]*)\n' + - 'Build Host\s*:\s*(?P[^\n]*)\n' + - 'Relocations\s*:\s*(?P[^\n]*)\n' + - 'URL\s*:\s*(?P[^\n]*)\n' + - 'Summary\s*:\s*(?P[^\n]*)\n' + - 'Description\s*:\s*(?P[^\n]+)', MATCHALL, reflags=re.S) + 'Version\s*:\s*(?P[^\n]*)\n' + + 'Release\s*:\s*(?P[^\n]*)\n' + + 'Architecture\s*:\s*(?P[^\n]*)\n' + + 'Install Date\s*:\s*(?P[^\n]*)\n' + + 'Group\s*:\s*(?P[^\n]*)\n' + + 'Size\s*:\s*(?P[^\n]*)\n' + + 'License\s*:\s*(?P[^\n]*)\n' + + 'Signature\s*:\s*(?P[^\n]*)\n' + + 'Source RPM\s*:\s*(?P[^\n]*)\n' + + 'Build Date\s*:\s*(?P[^\n]*)\n' + + 'Build Host\s*:\s*(?P[^\n]*)\n' + + 'Relocations\s*:\s*(?P[^\n]*)\n' + + 'URL\s*:\s*(?P[^\n]*)\n' + + 'Summary\s*:\s*(?P[^\n]*)\n' + + 'Description\s*:\s*(?P[^\n]+)', MATCHALL, reflags=re.S) if not os.path.exists(root): - os.system('mkdir %s'%root) + os.system('mkdir %s' % root) else: items = os.listdir(root) for item in items: path = os.path.join(root, item) if os.path.isdir(path) and os.listdir(path): if get_sidebar: - with open(path+'/metadata.json', 'r') as f: + with open(path + '/metadata.json', 'r') as f: json_obj = json.loads(f.read()) # if not get_sidebar: json_obj['status'] = 'disabled' - ext_name = json_obj['name'] if json_obj['name'][:6] != "array_" else json_obj['name'].split("_", 1)[1] + ext_name = json_obj['name'] if json_obj['name'][:6] != "array_" else \ + json_obj['name'].split("_", 1)[1] if os.path.exists(frontend_extensions_path + ext_name): json_obj['status'] = 'enabled' extensions_res.append(json_obj) @@ -440,14 +491,18 @@ else: result = cli_parse(output, ext_info_parse)[0] # check the main version number of extensions with system main version number. - result["ver_diff"] = True if result["version"].split(".")[:2] != system_version.split(".")[2:4] else False + result["ver_diff"] = True if result["version"].split(".")[:2] != system_version.split(".")[ + 2:4] else False try: - result["install_ts"] = time.mktime(datetime.datetime.strptime(result["install_date"], '%a %b %d %H:%M:%S %Y').timetuple()) + result["install_ts"] = time.mktime( + datetime.datetime.strptime(result["install_date"], '%a %b %d %H:%M:%S %Y').timetuple()) except ValueError: try: - result["install_ts"] = time.mktime(datetime.datetime.strptime(result["install_date"], '%a %d %b %Y %I:%M:%S %p %Z').timetuple()) + result["install_ts"] = time.mktime(datetime.datetime.strptime(result["install_date"], + '%a %d %b %Y %I:%M:%S %p %Z').timetuple()) except ValueError: - result["install_ts"] = time.mktime(datetime.datetime.strptime(result["install_date"], '%a %b %d %I:%M:%S %Y').timetuple()) + result["install_ts"] = time.mktime( + datetime.datetime.strptime(result["install_date"], '%a %b %d %I:%M:%S %Y').timetuple()) if os.path.exists(frontend_extensions_path + item): result["status"] = "enabled" else: @@ -457,7 +512,7 @@ extensions_res.append(result) # query all extension available packages for checking updatable at the same time. - if not get_sidebar and extension_list and check_status!= 256 and "Timeout" not in check_output: + if not get_sidebar and extension_list and check_status != 256 and "Timeout" not in check_output: extension_s = (" ").join(extension_list) status, output = subprocess.getstatusoutput('yum info --disablerepo="*" --enablerepo="array" %s' % extension_s) @@ -472,6 +527,7 @@ return HttpResponse(json.dumps(extensions_res), content_type='application/json') + def check_vpn_license_session(): session = get_current_session() session.cli.set_enable() @@ -488,43 +544,51 @@ return True return False + def enable_extension(request, name): name = name.split(',') for each in name: # check vpn_license_server because this ext should be licensed by AMP license and it's session should > 0 if each == "vpn_license_server": if check_vpn_license_session() == False: - return HttpResponse(json.dumps({"state": False, "msg": "Failed to enable extension vpn_license_server because licensed VPN license session is 0."}), content_type='application/json') - os.system('/bin/sh %s/%s/start.sh'%(extension_path, each if each[:6] != "array_" else each.split("_", 1)[1])) + return HttpResponse(json.dumps({"state": False, + "msg": "Failed to enable extension vpn_license_server because licensed VPN license session is 0."}), + content_type='application/json') + os.system('/bin/sh %s/%s/start.sh' % (extension_path, each if each[:6] != "array_" else each.split("_", 1)[1])) oper_log('info', 'extensions', 'Enable the extension %s successfully.' % each) update_frontend_index_html() return HttpResponse(json.dumps({"state": True, "msg": ""}), content_type='application/json') + def disable_extension(request, name): name = name.split(',') for each in name: - os.system('/bin/sh %s/%s/stop.sh'%(extension_path, each if each[:6] != "array_" else each.split("_", 1)[1])) + os.system('/bin/sh %s/%s/stop.sh' % (extension_path, each if each[:6] != "array_" else each.split("_", 1)[1])) oper_log('info', 'extensions', 'Disable the extension %s successfully.' % each) update_frontend_index_html() return HttpResponse(json.dumps(True), content_type='application/json') + def get_product_device_type(request): response = { - "VPN_TYPE_LIST": VPN_TYPE_LIST, - "ADC_TYPE_LIST": ADC_TYPE_LIST, - "WAF_TYPE_LIST": WAF_TYPE_LIST, + "VPN_TYPE_LIST": VPN_TYPE_LIST, + "ADC_TYPE_LIST": ADC_TYPE_LIST, + "WAF_TYPE_LIST": WAF_TYPE_LIST, "DEVICE_TYPE_LIST": DEVICE_TYPE_LIST, - "DEVICE_STD_LIST": DEVICE_STD_LIST + "DEVICE_STD_LIST": DEVICE_STD_LIST } return HttpResponse(json.dumps(response), content_type='application/json') + def _extension_log(name, action, result): s_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) - insert_sql = "insert into ext_log(name, action, time, result) values('%s', '%s', '%s', '%s')" % (name, action, s_time, result) + insert_sql = "insert into ext_log(name, action, time, result) values('%s', '%s', '%s', '%s')" % (name, action, + s_time, result) db = DB.get_connected_db() db.execute_sql(insert_sql) db.close() + def get_extension_log(request): select_sql = "select name, action, time, result from ext_log;" try: @@ -547,8 +611,10 @@ except: if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to query the extensions log."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to query the extensions log."}), + content_type='application/json') + def _get_operation_log(data): offset = int(data["start"]) limit = int(data["number"]) @@ -578,7 +644,8 @@ else: order_str += "desc" - select_sql = "select username, client_ip, time, level, module, operation, id from operation_log %s %s offset %d limit %d" % (search_str, order_str, offset, limit) + select_sql = "select username, client_ip, time, level, module, operation, id from operation_log %s %s offset %d limit %d" % ( + search_str, order_str, offset, limit) count_sql = "select count(id) from operation_log %s" % (search_str) try: db = DB.get_connected_db() @@ -590,17 +657,19 @@ for each in data: result.append(dict(zip(['username', 'client_ip', 'time', 'level', 'module', 'operation', 'id'], each))) total_num = total[0][0] - return {"state": True, "data": result, "total":total_num} + return {"state": True, "data": result, "total": total_num} except: if db: db.close() return {"state": False, "data": "Failed to query the operation log."} + def get_operation_log(request): data = json.loads(request.POST.get('post_data', None)) res = _get_operation_log(data) return HttpResponse(json.dumps(res), content_type='application/json') + def clear_operation_log(request): select_sql = "delete from operation_log" try: @@ -612,8 +681,10 @@ raise if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to clear the operation log."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to clear the operation log."}), + content_type='application/json') + def del_operation_log(request): id = request.POST.get('id', None) if id is not None: @@ -627,8 +698,10 @@ raise if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to delete the operation log."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to delete the operation log."}), + content_type='application/json') + def get_user_info(request): res = {} # if 'django_language' in request.COOKIES: @@ -650,6 +723,7 @@ res['root'] = manager.get({'username': session.username}).get_field_dict()['root'] return HttpResponse(json.dumps(res), content_type='application/json') + def get_user_auth(request): session = get_current_session() try: @@ -658,18 +732,24 @@ data = db.fetchone(fetchone_sql) if data: if data[1]: - return HttpResponse(json.dumps({"state": True, "data": dict(zip(['username', 'auth'], data))}), content_type='application/json') + return HttpResponse(json.dumps({"state": True, "data": dict(zip(['username', 'auth'], data))}), + content_type='application/json') else: - #no auth info, default allow all auth - return HttpResponse(json.dumps({"state": True, "data": {'username':session.username, 'auth':'\"all\"'}}), content_type='application/json') + # no auth info, default allow all auth + return HttpResponse( + json.dumps({"state": True, "data": {'username': session.username, 'auth': '\"all\"'}}), + content_type='application/json') else: - #here means no auth info, like default account(username: array) - return HttpResponse(json.dumps({"state": True, "data": {'username':session.username, 'auth':'\"all\"'}}), content_type='application/json') + # here means no auth info, like default account(username: array) + return HttpResponse(json.dumps({"state": True, "data": {'username': session.username, 'auth': '\"all\"'}}), + content_type='application/json') except: if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to query the user auth."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to query the user auth."}), + content_type='application/json') + def get_system_version(request): session = get_current_session() session.cli.set_enable() @@ -678,48 +758,62 @@ if output["version"]: return HttpResponse(json.dumps({"state": True, "data": output}), content_type='application/json') else: - return HttpResponse(json.dumps({"state": False, "data": "Failed to match the AMP version."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to match the AMP version."}), + content_type='application/json') else: - return HttpResponse(json.dumps({"state": False, "data": "Failed to query the AMP version."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to query the AMP version."}), + content_type='application/json') + def get_host_name(request): session = get_current_session() session.cli.set_enable() ret = session.cli.cmd('show hostname').strip() return HttpResponse(json.dumps({"hostname": ret}), content_type='application/json') + def get_update_version(request): available_ext_list = [] subprocess.getstatusoutput("yum --disablerepo=* --enablerepo=array clean metadata") check_status, check_output = subprocess.getstatusoutput("yum check-update --disablerepo=* --enablerepo=array") if check_status == 256: - if check_output.find("Timeout")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository because connect to the repository timeout."}), content_type='application/json') - elif check_output.find("Cannot find a valid baseurl for repo")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Please set yum repository by CLI 'system repo '."}), content_type='application/json') - elif check_output.find("Could not resolve host")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository. Please check IP nameserver and yum repository URL."}), content_type='application/json') - elif check_output.find("Connection refused")!= -1: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository. Please check the port of yum repository URL."}), content_type='application/json') + if check_output.find("Timeout") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository because connect to the repository timeout."}), + content_type='application/json') + elif check_output.find("Cannot find a valid baseurl for repo") != -1: + return HttpResponse( + json.dumps({"state": False, "data": "Please set yum repository by CLI 'system repo '."}), + content_type='application/json') + elif check_output.find("Could not resolve host") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository. Please check IP nameserver and yum repository URL."}), + content_type='application/json') + elif check_output.find("Connection refused") != -1: + return HttpResponse(json.dumps({"state": False, + "data": "Failed to connect the yum repository. Please check the port of yum repository URL."}), + content_type='application/json') else: - return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to connect the yum repository."}), + content_type='application/json') status, output = subprocess.getstatusoutput("yum --disablerepo=* --enablerepo=array info amp") if status: - return HttpResponse(json.dumps({"state": False, "data": "Failed to check the amp version."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to check the amp version."}), + content_type='application/json') parser = RegexParser('Available Packages\nName\s*:\s*(?P[^\n]*)\n' + - 'Arch\s*:\s*(?P[^\n]*)\n' + - 'Version\s*:\s*(?P[^\n]*)\n' + - 'Release\s*:\s*(?P[^\n]*)\n' + - 'Size\s*:\s*(?P[^\n]*)\n' + - 'Repo\s*:\s*(?P[^\n]*)\n' + - 'Summary\s*:\s*(?P[^\n]*)\n' + - 'URL\s*:\s*(?P[^\n]*)\n' + - 'License\s*:\s*(?P[^\n]*)\n' + - 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) + 'Arch\s*:\s*(?P[^\n]*)\n' + + 'Version\s*:\s*(?P[^\n]*)\n' + + 'Release\s*:\s*(?P[^\n]*)\n' + + 'Size\s*:\s*(?P[^\n]*)\n' + + 'Repo\s*:\s*(?P[^\n]*)\n' + + 'Summary\s*:\s*(?P[^\n]*)\n' + + 'URL\s*:\s*(?P[^\n]*)\n' + + 'License\s*:\s*(?P[^\n]*)\n' + + 'Description\s*:\s*(?P[^\n]*)', MATCHALL, reflags=re.S) res = cli_parse(output, parser) @@ -728,11 +822,13 @@ else: return HttpResponse(json.dumps({"state": True, "data": []}), content_type='application/json') + def update_version_by_repo(requset): status, output = subprocess.getstatusoutput("yum update amp -y") if status: oper_log('error', 'system', 'Faile to update version from repository.') - return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to execute the command."}), + content_type='application/json') else: # update_frontend_index_html() session = get_current_session() @@ -759,7 +855,8 @@ return HttpResponse(json.dumps({"state": True, "data": result}), content_type='application/json') db.close() - key = ['device_name', 'ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password'] + key = ['device_name', 'ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', + 'enable_password'] device_list = [dict(zip(key, each)) for each in data] slb_parser = RegexParser('slb virtual (?P.*?) "(?P.*?)" (?P.*?)', MATCHALL) @@ -776,7 +873,7 @@ response_data = send_cli_to_device(each, 'show slb virtual all', mode='enable') if response_data['message'] != 'success': - continue + continue # if not content: # return HttpResponse(json.dumps({"state": False, "data": "Cannot query the device <%s> slb virtual data." % each[1]})) # if content == 'failed' or content == 'time out': @@ -825,17 +922,17 @@ result.append({ "service_name": item['vs_name'], "service_type": each["service_type"], - "device_name": each["device_name"], - "cert_status": cert_status, - "exp_time": time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(timestamp)) + "device_name": each["device_name"], + "cert_status": cert_status, + "exp_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) }) else: result.append({ "service_name": item['vs_name'], "service_type": each["service_type"], - "device_name": each["device_name"], - "cert_status": "NoCertificate", - "exp_time": "" + "device_name": each["device_name"], + "cert_status": "NoCertificate", + "exp_time": "" }) return HttpResponse(json.dumps({"state": True, "data": result}), content_type='application/json') @@ -858,10 +955,13 @@ return HttpResponse(json.dumps({"state": True, "data": result}), content_type='application/json') db.close() - key = ['device_name', 'ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', 'enable_password'] + key = ['device_name', 'ip_address', 'protocol', 'restapi_port', 'restapi_username', 'restapi_password', 'type', + 'enable_password'] device_list = [dict(zip(key, each)) for each in data] - vsite_parser = RegexParser('virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', MATCHALL) + vsite_parser = RegexParser( + 'virtual site name "(?P.*?)" "(?P.*?)" (?P.*?) (?P.*?)', + MATCHALL) ssl_cert_parser = [ RegexParser('Not Before: (?P.*?) GMT', MATCHONE), RegexParser('Not After : (?P.*?) GMT', MATCHONE) @@ -891,8 +991,9 @@ if cert_device_list: for each in cert_device_list: ssl_url = '/rest/%s/global/cli_extend' % get_device_type(each["type"]) - data_cert = {'cmd':'show ssl certificate', 'vsite_name': each["vsite_name"]} - device_info = {"ip_address": each["ip_address"], "restapi_port": each["restapi_port"], "restapi_username": each["restapi_username"], "restapi_password": each["restapi_password"]} + data_cert = {'cmd': 'show ssl certificate', 'vsite_name': each["vsite_name"]} + device_info = {"ip_address": each["ip_address"], "restapi_port": each["restapi_port"], + "restapi_username": each["restapi_username"], "restapi_password": each["restapi_password"]} running_params = json.dumps(data_cert) cert_content = send_command_to_device(device_info, ssl_url, running_params) cert_result = cli_parse(cert_content, ssl_cert_parser) @@ -908,28 +1009,31 @@ cert_status = "Expired" result.append({ "service_name": each['vsite_name'], - "device_name": each["device_name"], - "cert_status": cert_status, - "exp_time": time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(timestamp)) + "device_name": each["device_name"], + "cert_status": cert_status, + "exp_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) }) else: result.append({ "service_name": each['vsite_name'], - "device_name": each["device_name"], - "cert_status": "NoCertificate", - "exp_time": "" + "device_name": each["device_name"], + "cert_status": "NoCertificate", + "exp_time": "" }) return HttpResponse(json.dumps({"state": True, "data": result}), content_type='application/json') + def check_adc_ssl_cert(requset): ret = os.popen("/usr/bin/python /ca/bin/check_adc_ssl.py").read() return HttpResponse(ret) + def check_build_info(request): ret = os.popen("/usr/bin/python /ca/bin/check_build.py").read() return HttpResponse(ret, content_type='application/json') + def fetch_schedule_backup_all(request): result = { "status": False, @@ -955,6 +1059,7 @@ return HttpResponse(json.dumps(result), content_type='application/json') + def clear_schedule_backup_all(request): task_name_list = json.loads(request.POST.get('task_name_list', None)) @@ -973,11 +1078,14 @@ return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') else: oper_log('error', 'configuration', 'Faile to clear schedule backup all task.') - return HttpResponse(json.dumps({"state": False, "data": "Task name list is empty."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Task name list is empty."}), + content_type='application/json') + def get_debug_logs(request): ret = os.popen("/usr/bin/python /ca/bin/check_adc_ssl.py").read() + def get_log_basic(request): result = { "status": True, @@ -991,6 +1099,7 @@ return HttpResponse(json.dumps(result), content_type='application/json') + def update_log_basic(request): val = {} data = json.loads(request.POST.get('post_data', None)) @@ -998,21 +1107,22 @@ if "enable_log" in data: val["enable"] = 1 if data["enable_log"] else 0 else: - result = {"status":False, "data":"Invalid Post Data"} + result = {"status": False, "data": "Invalid Post Data"} return HttpResponse(json.dumps(result), content_type='application/json') if "log_level" in data: val["level"] = data["log_level"] else: - result = {"status":False, "data":"Invalid Post Data"} + result = {"status": False, "data": "Invalid Post Data"} return HttpResponse(json.dumps(result), content_type='application/json') ret = set_log_settings(val) oper_log('info', 'system', 'Update log basic successfully.') return HttpResponse(json.dumps(ret), content_type='application/json') else: oper_log('error', 'system', 'Failed to update log basic.') - result = {"status":False, "data":"Invalid Post Data"} + result = {"status": False, "data": "Invalid Post Data"} return HttpResponse(json.dumps(result), content_type='application/json') + def get_log_host(request): result = { "status": True, @@ -1021,12 +1131,13 @@ data = get_log_settings() for key, val in data["host"].items(): result["data"].append({ - "host_ip":val[0], - "port":val[1], - "protocol":val[2] + "host_ip": val[0], + "port": val[1], + "protocol": val[2] }) return HttpResponse(json.dumps(result), content_type='application/json') + def add_log_host(request): data = json.loads(request.POST.get('post_data', None)) if data: @@ -1035,9 +1146,10 @@ return HttpResponse(json.dumps(ret), content_type='application/json') else: oper_log('error', 'system', 'Failed to add log basic.') - result = {"status":False, "data":"Invalid Post Data"} + result = {"status": False, "data": "Invalid Post Data"} return HttpResponse(json.dumps(result), content_type='application/json') + def delete_log_host(request): data = json.loads(request.POST.get('post_data', None)) if data: @@ -1045,16 +1157,18 @@ oper_log('info', 'system', 'Delete log basic successfully.') return HttpResponse(json.dumps(ret), content_type='application/json') else: - result = {"status":False, "data":"Invalid Post Data"} + result = {"status": False, "data": "Invalid Post Data"} oper_log('error', 'system', 'Failed to delete log basic.') return HttpResponse(json.dumps(result), content_type='application/json') + def get_license_expire(request): session = get_current_session() res = check_license_expire(session) return HttpResponse(json.dumps(res), content_type='application/json') + def _get_task_pagination(data): offset = int(data["start"]) limit = int(data["number"]) @@ -1069,7 +1183,8 @@ search_str = "where %s" % (" and ").join(search_str_list) if search_str_list else "" - select_sql = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task %s ORDER BY next_run_time desc offset %d LIMIT %d" % (search_str, offset, limit) + select_sql = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task %s ORDER BY next_run_time desc offset %d LIMIT %d" % ( + search_str, offset, limit) count_sql = "SELECT count(id) FROM task %s" % search_str try: @@ -1080,7 +1195,9 @@ result = [] if data: for each in data: - result.append(dict(zip(['id', 'name', 'type', 'state', 'next_run_time', 'description', 'result_field', 'device_list', 'scheduler'], each))) + result.append(dict( + zip(['id', 'name', 'type', 'state', 'next_run_time', 'description', 'result_field', 'device_list', + 'scheduler'], each))) total_num = total[0][0] return {"state": True, "data": result, "total": total_num} except: @@ -1088,31 +1205,37 @@ db.close() return {"state": False, "data": "Failed to query the task list."} + def get_task_pagination(request): data = json.loads(request.POST.get('post_data', None)) res = _get_task_pagination(data) return HttpResponse(json.dumps(res), content_type='application/json') + def get_task_and_log(request): data = json.loads(request.POST.get('post_data', None)) offset = int(data["start"]) limit = int(data["number"]) - log = _get_operation_log({"start": offset, "number": limit, "search": {}, "sort_predicate": "time", "sort_reverse": ""}) + log = _get_operation_log( + {"start": offset, "number": limit, "search": {}, "sort_predicate": "time", "sort_reverse": ""}) task = _get_task_pagination({"start": offset, "number": limit, "search": {}}) return HttpResponse(json.dumps({"log": log, "task": task}), content_type='application/json') + def get_task_by_name(request): post_data = json.loads(request.POST.get('post_data', None)) name = post_data['name'] config_type = post_data['type'] sql_res2 = [] db = DB.get_connected_db() - fetchall_sql = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task WHERE name like '%%%s%%' order by next_run_time DESC;" % (name + "-" if config_type != 'system' else name) + fetchall_sql = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task WHERE name like '%%%s%%' order by next_run_time DESC;" % ( + name + "-" if config_type != 'system' else name) if name.endswith(".cfg") or name.endswith(".all_cfg_tar"): name = name[:-4] if name.endswith(".cfg") else name[:-12] - fetchall_sql2 = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task WHERE name like '%%%s%%' and type = 'config_check' order by next_run_time DESC;" % (name + "-") + fetchall_sql2 = "SELECT id, name, type, state, next_run_time, description, result_field, device_list, scheduler FROM task WHERE name like '%%%s%%' and type = 'config_check' order by next_run_time DESC;" % ( + name + "-") sql_res2 = db.fetchall(fetchall_sql2) sql_res = db.fetchall(fetchall_sql) db.close() @@ -1130,6 +1253,7 @@ return HttpResponse(json.dumps(res), content_type='application/json') + def get_config_template_key(request): sql_res = None try: @@ -1146,13 +1270,15 @@ return HttpResponse(json.dumps(result), content_type='application/json') + def add_config_template_key(request): data = json.loads(request.POST.get('post_data', None)) res = {"state": False, "msg": ""} try: db = DB.get_connected_db() - insert_sql = "insert into config_template(key, default_value, description) values('%s', '%s', '%s')" % (data["key"], data["default_value"], data["description"]) + insert_sql = "insert into config_template(key, default_value, description) values('%s', '%s', '%s')" % ( + data["key"], data["default_value"], data["description"]) db.execute_sql(insert_sql) db.close() update_new_default_tmpkey(data["key"], data["default_value"]) @@ -1164,6 +1290,7 @@ return HttpResponse(json.dumps(res), content_type='application/json') + def delete_config_template_key(request): data = json.loads(request.POST.get('post_data', None)) res = {"state": False, "msg": ""} @@ -1182,13 +1309,15 @@ return HttpResponse(json.dumps(res), content_type='application/json') + def edit_config_template_key(request): data = json.loads(request.POST.get('post_data', None)) res = {"state": False, "msg": ""} try: db = DB.get_connected_db() select_sql = "SELECT default_value FROM config_template WHERE key = '%s'" % data["key"] - update_sql = "UPDATE config_template SET default_value='%s', description='%s' WHERE key = '%s'" % (data["default_value"], data["description"], data["key"]) + update_sql = "UPDATE config_template SET default_value='%s', description='%s' WHERE key = '%s'" % ( + data["default_value"], data["description"], data["key"]) select_res = db.fetchall(select_sql) old_value = select_res[0][0] db.execute_sql(update_sql) @@ -1203,6 +1332,7 @@ return HttpResponse(json.dumps(res), content_type='application/json') + def edit_default_tmpvalue(key, value, old_value): db = DB.get_connected_db() tmp_value_res = db.fetchall("SELECT id, key, value FROM device_config_template WHERE key = '%s'" % key) @@ -1215,6 +1345,7 @@ db.close() return + def update_new_default_tmpkey(key, default_value=None, add_flag=True): db = DB.get_connected_db() @@ -1223,7 +1354,9 @@ ip_list = [each[0] for each in ip_list_res] for each in ip_list: - update_sql = "insert into device_config_template(device_id, key, value) values('%s', '%s', '%s')" % (each, key, default_value) + update_sql = "insert into device_config_template(device_id, key, value) values('%s', '%s', '%s')" % (each, + key, + default_value) db.execute_sql(update_sql) else: delete_sql = "delete from device_config_template where key = '%s'" % key @@ -1231,6 +1364,7 @@ db.close() return + def query_all_tmpkeyval(request): try: db = DB.get_connected_db() @@ -1272,6 +1406,7 @@ } return HttpResponse(json.dumps(resp), content_type='application/json') + def edit_config_template_default_value(request): data = json.loads(request.POST.get('post_data', None)) device_ip = None @@ -1283,16 +1418,19 @@ db = DB.get_connected_db() for each in data: if each["key"] != "device_ip": - update_sql = "UPDATE device_config_template SET value = '%s' WHERE device_id = '%s' AND key = '%s'" % (each["value"], device_ip, each["key"]) + update_sql = "UPDATE device_config_template SET value = '%s' WHERE device_id = '%s' AND key = '%s'" % ( + each["value"], device_ip, each["key"]) db.execute_sql(update_sql) db.close() except: if db: db.close() - return HttpResponse(json.dumps({"state": False, "msg": "Failed to edit config template deafult value."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "msg": "Failed to edit config template deafult value."}), + content_type='application/json') return HttpResponse(json.dumps({"state": True, "msg": ""}), content_type='application/json') + def get_device_ha_list(request): resp = { "error_msg": "", @@ -1313,7 +1451,8 @@ } try: new_url = modify_url('/rest/device_type/ha/units/HAGenetalSettings', each['type']) - response_data = send_https_rest_request('GET', new_url, '', each['ip'], each['port'], each['username'], each['password']) + response_data = send_https_rest_request('GET', new_url, '', each['ip'], each['port'], + each['username'], each['password']) except Exception as e: errMsg = 'Get device<%s> ha genetal settings failed. %s\n' % (each['ip'], str(e)) resp["error_msg"] = resp["error_msg"] + errMsg @@ -1326,7 +1465,8 @@ ha_settings["device_name"] = each["name"] resp["device_ha"].append(ha_settings) except: - resp["error_msg"] += 'Get device<%s> ha genetal settings failed. Device Interval error.\n' % each['ip'] + resp["error_msg"] += 'Get device<%s> ha genetal settings failed. Device Interval error.\n' % \ + each['ip'] else: resp["error_msg"] += 'Get device<%s> ha genetal settings failed. Interval error.\n' % each['ip'] elif request.method == 'POST': @@ -1335,7 +1475,9 @@ try: new_url = modify_url('/rest/device_type/ha/units/HAGenetalSettings', device_info['type']) - response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) if response_data['status'] == 200: rest_body = json.loads(response_data['body']) ha_settings = rest_body['HAGenetalSettings'][0] @@ -1345,9 +1487,9 @@ except Exception as e: resp["error_msg"] = 'Get device ha genetal settings failed. %s\n' % str(e) - return HttpResponse(json.dumps(resp), content_type='application/json') + def get_device_ha_units(request): device_name = json.loads(request.POST.get('device_name', None)) @@ -1358,7 +1500,9 @@ try: new_url = modify_url('/rest/device_type/ha/units/HAUnit', device_info['type']) - response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) if response_data['status'] == 200: rest_body = json.loads(response_data['body']) res = rest_body['HAUnit'] @@ -1376,6 +1520,7 @@ return HttpResponse(json.dumps({"state": state, "data": res, "msg": err_msg}), content_type='application/json') + def get_device_ha_groups(request): device_name = json.loads(request.POST.get('device_name', None)) @@ -1386,7 +1531,9 @@ try: new_url = modify_url('/rest/device_type/ha/groups/HAGroup', device_info['type']) - response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + response_data = send_https_rest_request('GET', new_url, '', device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) if response_data['status'] == 200: rest_body = json.loads(response_data['body']) res = rest_body['HAGroup'] @@ -1404,6 +1551,7 @@ return HttpResponse(json.dumps({"state": state, "data": res, "msg": err_msg}), content_type='application/json') + def swith_ha_group(request): post_data = json.loads(request.POST.get('post_data', None)) device_name = post_data["device_name"] @@ -1417,7 +1565,9 @@ try: new_url = modify_url('/rest/device_type/cli_extend', device_info['type']) data = {'cmd': 'ha group switch %s %s' % (unit_name, group_id)} - response_data = send_https_rest_request('POST', new_url, json.dumps(data), device_info['ip_address'], device_info['restapi_port'], device_info['restapi_username'], device_info['restapi_password']) + response_data = send_https_rest_request('POST', new_url, json.dumps(data), device_info['ip_address'], + device_info['restapi_port'], device_info['restapi_username'], + device_info['restapi_password']) if response_data['status'] == 200: rest_body = json.loads(response_data['body']) contents = rest_body["contents"].strip() @@ -1430,6 +1580,7 @@ return HttpResponse(json.dumps({"state": state, "msg": err_msg}), content_type='application/json') + def import_license_key(request): data = json.loads(request.POST.get('post_data', None)) license_key = data["license_key"] @@ -1444,21 +1595,25 @@ else: return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') + @csrf_exempt def device_change_result(request): data = json.loads(request.raw_post_data) - change_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()) + change_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) db = DB.get_connected_db() if data: for each in data["details"]: - insert_sql = "INSERT INTO vpn_device_change_result(device_id, result, msg, field, old_value, new_value, change_time) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % (data["device_id"], 1 if data["result"] else 0, data["msg"], each["name"], each["old_value"], each["new_value"], change_time) + insert_sql = "INSERT INTO vpn_device_change_result(device_id, result, msg, field, old_value, new_value, change_time) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( + data["device_id"], 1 if data["result"] else 0, data["msg"], each["name"], each["old_value"], + each["new_value"], change_time) db.execute_sql(insert_sql) db.close() return HttpResponse(json.dumps({"status": True}), content_type='application/json') + def get_device_change_log(request): device_id = request.POST["device_id"] @@ -1470,13 +1625,15 @@ except: if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to fetch the device change log."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to fetch the device change log."}), + content_type='application/json') key = ['device_id', 'result', 'msg', 'field', 'old_value', 'new_value', 'change_time'] result = [dict(zip(key, each)) for each in data] return HttpResponse(json.dumps({"state": True, "data": result}), content_type='application/json') + def clear_device_change_log(request): device_id = request.POST["device_id"] @@ -1488,10 +1645,12 @@ except: if db: db.close() - return HttpResponse(json.dumps({"state": False, "data": "Failed to clear the device change log."}), content_type='application/json') + return HttpResponse(json.dumps({"state": False, "data": "Failed to clear the device change log."}), + content_type='application/json') return HttpResponse(json.dumps({"state": True, "data": ""}), content_type='application/json') + def export_adc_ssl_cert_file(request): device_name = request.POST["device_name"] host_name = request.POST['host_name'] @@ -1501,7 +1660,6 @@ cm = __import__("cm") reload(cm.conf) network_card = cm.conf.NETWORK_CARD - #get config file from device try: local_ip = get_ip_address(network_card) except Exception as e: @@ -1512,7 +1670,8 @@ file_path = host_name if domain_name: file_path += '_' + domain_name - cmd = 'wget --header="Authorization: Basic %s" "https://%s:%d/pest/localfile/certificatefile/%s/%s" --no-check-certificate -O "/ca/config/backup/%s"' % (auth_str, device_info['ip_address'], device_info['restapi_port'], file_path, file_name, file_name) + cmd = 'wget --header="Authorization: Basic %s" "https://%s:%d/pest/localfile/certificatefile/%s/%s" --no-check-certificate -O "/ca/config/backup/%s"' % ( + auth_str, device_info['ip_address'], device_info['restapi_port'], file_path, file_name, file_name) try: os.popen(cmd).read() except: @@ -1522,6 +1681,7 @@ else: return HttpResponse(json.dumps({"state": False, "data": "Failed to export the backup file"})) + def remove_adc_ssl_cert_file(request): file_name = request.POST['file_name'] try: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/tree.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/tree.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/tree.py (working copy) @@ -1,18 +1,10 @@ -#!/usr/bin/env python -# encoding: utf-8 -""" -tree.py +from collections import OrderedDict as D -Created by Li Qian on 2013-06-04. -Copyright (c) 2013 Array Networks. All rights reserved. -""" - from hive.imports.node_tree import * -from collections import OrderedDict as D from views import * tree = D([ - ('ajax', {'cls':MenuNode, 'hidden':True, 'sub_nodes':D([ - ('webconsole', {'cls':WebConsoleNode}), + ('ajax', {'cls': MenuNode, 'hidden': True, 'sub_nodes': D([ + ('webconsole', {'cls': WebConsoleNode}), ])}), ]) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/urls.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/urls.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/urls.py (working copy) @@ -1,58 +1,60 @@ from django.urls import re_path + from cm.router import * + app_urlpatterns = ['', - re_path(r'^cm/configuration/device/get_schedule_backup_all$', fetch_schedule_backup_all), - re_path(r'^cm/configuration/device/clear_schedule_backup_all$', clear_schedule_backup_all), - re_path(r'^cm/configuration/configuration_file/(?P.*)$', get_configuration), - re_path(r'^cm/va_mgmt/vnc_host_info$', vnc_host_info), - re_path(r'^cm/get_device_type', get_product_device_type), - re_path(r'^cm/volume_license/device_version$', device_version), - re_path(r'^cm/volume_license/get_enable_mode$', get_enable_mode), - re_path(r'^cm/volume_license/set_enable_mode$', set_enable_mode), - re_path(r'^cm/volume_license/get_device_change_log$', get_device_change_log), - re_path(r'^cm/volume_license/clear_device_change_log$', clear_device_change_log), - re_path(r'^cm/device_change_result$', device_change_result), - re_path(r'^get_available_extension$', get_available_extension), - re_path(r'^install_extension/(?P.*)$', install_extension), - re_path(r'^remove_extension/(?P.*)$', remove_extension), - re_path(r'^update_extension/(?P.*)$', update_extension), - re_path(r'^get_extensions$', get_extensions), - re_path(r'^enable_extension/(?P.*)$', enable_extension), - re_path(r'^disable_extension/(?P.*)$', disable_extension), - re_path(r'^cm/get_extension_log$', get_extension_log), - re_path(r'^cm/get_operation_log$', get_operation_log), - re_path(r'^cm/get_task_pagination$', get_task_pagination), - re_path(r'^cm/get_task_by_name$', get_task_by_name), - re_path(r'^cm/get_task_and_log$', get_task_and_log), - re_path(r'^cm/clear_operation_log$', clear_operation_log), - re_path(r'^cm/del_operation_log$', del_operation_log), - re_path(r'^cm/get_user_info$', get_user_info), - re_path(r'^cm/get_user_auth$', get_user_auth), - re_path(r'^cm/get_system_version$', get_system_version), - re_path(r'^cm/get_host_name$', get_host_name), - re_path(r'^cm/get_adc_ssl_cert$', get_adc_ssl_cert), - re_path(r'^cm/get_vpn_ssl_cert$', get_vpn_ssl_cert), - re_path(r'^cm/check_adc_ssl_cert$', check_adc_ssl_cert), - re_path(r'^cm/check_build_info$', check_build_info), - re_path(r'^cm/system/get_update_version$', get_update_version), - re_path(r'^cm/system/update_version_by_repo$', update_version_by_repo), - re_path(r'^cm/get_debug_logs$', get_debug_logs), - re_path(r'^cm/get_log_basic$', get_log_basic), - re_path(r'^cm/update_log_basic$', update_log_basic), - re_path(r'^cm/get_log_host$', get_log_host), - re_path(r'^cm/add_log_host$', add_log_host), - re_path(r'^cm/delete_log_host$', delete_log_host), - re_path(r'^cm/get_license_expire$', get_license_expire), - re_path(r'^mainshow$', large_screen), - re_path(r'^cm/get_config_template_key$', get_config_template_key), - re_path(r'^cm/add_config_template_key$', add_config_template_key), - re_path(r'^cm/delete_config_template_key$', delete_config_template_key), - re_path(r'^cm/edit_config_template_key$', edit_config_template_key), - re_path(r'^cm/query_all_tmpkeyval$', query_all_tmpkeyval), - re_path(r'^cm/edit_config_template_default_value$', edit_config_template_default_value), - re_path(r'^cm/get_device_ha_list$', get_device_ha_list), - re_path(r'^cm/get_device_ha_units$', get_device_ha_units), - re_path(r'^cm/get_device_ha_groups$', get_device_ha_groups), - re_path(r'^cm/swith_ha_group$', swith_ha_group), - re_path(r'^cm/import_license_key$', import_license_key), -] + re_path(r'^cm/configuration/device/get_schedule_backup_all$', fetch_schedule_backup_all), + re_path(r'^cm/configuration/device/clear_schedule_backup_all$', clear_schedule_backup_all), + re_path(r'^cm/configuration/configuration_file/(?P.*)$', get_configuration), + re_path(r'^cm/va_mgmt/vnc_host_info$', vnc_host_info), + re_path(r'^cm/get_device_type', get_product_device_type), + re_path(r'^cm/volume_license/device_version$', device_version), + re_path(r'^cm/volume_license/get_enable_mode$', get_enable_mode), + re_path(r'^cm/volume_license/set_enable_mode$', set_enable_mode), + re_path(r'^cm/volume_license/get_device_change_log$', get_device_change_log), + re_path(r'^cm/volume_license/clear_device_change_log$', clear_device_change_log), + re_path(r'^cm/device_change_result$', device_change_result), + re_path(r'^get_available_extension$', get_available_extension), + re_path(r'^install_extension/(?P.*)$', install_extension), + re_path(r'^remove_extension/(?P.*)$', remove_extension), + re_path(r'^update_extension/(?P.*)$', update_extension), + re_path(r'^get_extensions$', get_extensions), + re_path(r'^enable_extension/(?P.*)$', enable_extension), + re_path(r'^disable_extension/(?P.*)$', disable_extension), + re_path(r'^cm/get_extension_log$', get_extension_log), + re_path(r'^cm/get_operation_log$', get_operation_log), + re_path(r'^cm/get_task_pagination$', get_task_pagination), + re_path(r'^cm/get_task_by_name$', get_task_by_name), + re_path(r'^cm/get_task_and_log$', get_task_and_log), + re_path(r'^cm/clear_operation_log$', clear_operation_log), + re_path(r'^cm/del_operation_log$', del_operation_log), + re_path(r'^cm/get_user_info$', get_user_info), + re_path(r'^cm/get_user_auth$', get_user_auth), + re_path(r'^cm/get_system_version$', get_system_version), + re_path(r'^cm/get_host_name$', get_host_name), + re_path(r'^cm/get_adc_ssl_cert$', get_adc_ssl_cert), + re_path(r'^cm/get_vpn_ssl_cert$', get_vpn_ssl_cert), + re_path(r'^cm/check_adc_ssl_cert$', check_adc_ssl_cert), + re_path(r'^cm/check_build_info$', check_build_info), + re_path(r'^cm/system/get_update_version$', get_update_version), + re_path(r'^cm/system/update_version_by_repo$', update_version_by_repo), + re_path(r'^cm/get_debug_logs$', get_debug_logs), + re_path(r'^cm/get_log_basic$', get_log_basic), + re_path(r'^cm/update_log_basic$', update_log_basic), + re_path(r'^cm/get_log_host$', get_log_host), + re_path(r'^cm/add_log_host$', add_log_host), + re_path(r'^cm/delete_log_host$', delete_log_host), + re_path(r'^cm/get_license_expire$', get_license_expire), + re_path(r'^mainshow$', large_screen), + re_path(r'^cm/get_config_template_key$', get_config_template_key), + re_path(r'^cm/add_config_template_key$', add_config_template_key), + re_path(r'^cm/delete_config_template_key$', delete_config_template_key), + re_path(r'^cm/edit_config_template_key$', edit_config_template_key), + re_path(r'^cm/query_all_tmpkeyval$', query_all_tmpkeyval), + re_path(r'^cm/edit_config_template_default_value$', edit_config_template_default_value), + re_path(r'^cm/get_device_ha_list$', get_device_ha_list), + re_path(r'^cm/get_device_ha_units$', get_device_ha_units), + re_path(r'^cm/get_device_ha_groups$', get_device_ha_groups), + re_path(r'^cm/swith_ha_group$', swith_ha_group), + re_path(r'^cm/import_license_key$', import_license_key), + ] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/views.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/views.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/views.py (working copy) @@ -1,14 +1,14 @@ # Create your views here. -from hive.node import BaseNode +from urllib.parse import unquote + from cm.lib.console import WebConsoleRequestHandler -from hive.utils import get_current_session, andebug -from hive.model.loading import get_model -import json, os -from urllib import unquote from cm.lib.postgres_db import DB from djproject.an_settings import * -#from hive.forms.models import modelform_factory +from hive.model.loading import get_model +from hive.node import BaseNode +from hive.utils import get_current_session + class DashboardNode(BaseNode): _template_name = 'dashboard.html' @@ -16,8 +16,10 @@ class EmptyNode(BaseNode): _template_name = 'todo.html' + class SearchNode(BaseNode): _template_name = 'search.html' + def _get_context(self, request, session): device = get_model('cm', ['device_mgmt', 'device', 'Device']) sess = get_current_session() @@ -26,20 +28,24 @@ for each in device_ip.all(): ip_list.append(each.ip.values()[0]) super_ctx = super(SearchNode, self)._get_context(request, session) - return dict(super_ctx.items() + {'ip_list':ip_list}.items()) + return dict(super_ctx.items() + {'ip_list': ip_list}.items()) class GraphicsNode(BaseNode): _template_name = 'graphics.html' + class GraphSelectorNode(BaseNode): _template_name = 'box_graph_selector.html' + def _get_context(self, request, session): super_ctx = super(GraphSelectorNode, self)._get_context(request, session) - return dict(super_ctx.items() + {'MODE':'main'}.items()) + return dict(super_ctx.items() + {'MODE': 'main'}.items()) + class GraphSelectorSubNode(BaseNode): _template_name = 'box_graph_selector.html' + def _get_context(self, request, session): model_path = request.GET['path'] super_ctx = super(GraphSelectorSubNode, self)._get_context(request, session) @@ -47,7 +53,7 @@ model = get_model('cm', model_path.split('.')) sess = get_current_session() manager = model.get_manager(sess) - return dict(super_ctx.items() + {'MODE':'sub', 'MODEL':model, 'MANAGER':manager}.items()) + return dict(super_ctx.items() + {'MODE': 'sub', 'MODEL': model, 'MANAGER': manager}.items()) else: data = self._load_graphs_config_from_json() graph_owner = request.GET['type'] @@ -56,15 +62,21 @@ else: device_ip = '' if model_path == 'graph_model': - return dict(super_ctx.items() + {'MODE': model_path, 'DATA': data[graph_owner]['models'], 'DATA_TYPE': graph_owner, 'IP': device_ip}.items()) + return dict(super_ctx.items() + {'MODE': model_path, 'DATA': data[graph_owner]['models'], + 'DATA_TYPE': graph_owner, 'IP': device_ip}.items()) elif model_path == 'graph': graph_model_name = request.GET['name'] - return dict(super_ctx.items() + {'MODE': model_path, 'GRAPH_MODEL_NAME':graph_model_name, 'DATA':data[graph_owner]['models'], 'DATA_TYPE': graph_owner, 'IP': device_ip}.items()) + return dict(super_ctx.items() + {'MODE': model_path, 'GRAPH_MODEL_NAME': graph_model_name, + 'DATA': data[graph_owner]['models'], 'DATA_TYPE': graph_owner, + 'IP': device_ip}.items()) else: - #sub_graph + # sub_graph parent_name = request.GET['parent'] graph_model_name = request.GET['name'] - return dict(super_ctx.items() + {'MODE': model_path, 'PARENT_MODEL_NAME': parent_name, 'GRAPH_MODEL_NAME':graph_model_name, 'DATA':data[graph_owner]['models'][parent_name]['sub_models'], 'DATA_TYPE': graph_owner, 'IP': device_ip}.items()) + return dict(super_ctx.items() + {'MODE': model_path, 'PARENT_MODEL_NAME': parent_name, + 'GRAPH_MODEL_NAME': graph_model_name, + 'DATA': data[graph_owner]['models'][parent_name]['sub_models'], + 'DATA_TYPE': graph_owner, 'IP': device_ip}.items()) def _load_graphs_config_from_json(self): os.chdir(CM_ConfigFile_PATH) @@ -77,9 +89,10 @@ data = json.loads(file_data) return data + class WebConsoleNode(BaseNode): _template_name = 'box_webconsole.html' - + def init(self, request, session): os.system('rm -rf /root/.ssh/known_hosts') width = request.GET.get('w', 160) @@ -93,7 +106,7 @@ data = db.fetchone(select_sql) if data: if data[1].lower() in ADC_TYPE_LIST + WAF_TYPE_LIST or data[6] == 'xmlrpc': - result = {'ip_address': data[0], 'username':data[2], 'password':data[3]} + result = {'ip_address': data[0], 'username': data[2], 'password': data[3]} elif data[1].lower() in VPN_TYPE_LIST: """ cm=# select ip_address, type, console_username, console_password, restapi_username, restapi_password from device where id='6986be62-3113-11ea-898c-fce1fb0f7013'; @@ -103,14 +116,15 @@ (1 row) AG's restapi account is actually ssh account """ - result = {'ip_address': data[0], 'username':data[4], 'password':data[5]} + result = {'ip_address': data[0], 'username': data[4], 'password': data[5]} else: select_sql = "select ip, restapi_username, restapi_password from host where id='%s'" % device_id data = db.fetchone(select_sql) if data: - result = {'ip_address': data[0], 'username':data[1], 'password':data[2]} + result = {'ip_address': data[0], 'username': data[1], 'password': data[2]} db.close() return result + if not device_id: return json.dumps([False, 'Failed to connect console. Device id is not specified.']) if device_id == 'localhost': @@ -121,7 +135,8 @@ if not data: return json.dumps([False, 'Failed to connect console. Device info can not be found.']) sessid = data['ip_address'] - pid = WebConsoleRequestHandler().create_console(sessid, data['username'], data['password'], width, height, data['ip_address']) + pid = WebConsoleRequestHandler().create_console(sessid, data['username'], data['password'], width, height, + data['ip_address']) if pid: return json.dumps([True, pid, sessid]) else: @@ -133,7 +148,7 @@ int(request.GET.get('h')), unquote(request.GET.get('k')), request.GET.get('pid')) - + def _get_context(self, request, session): device = get_model('cm', ['device_mgmt', 'device', 'Device']) sess = get_current_session() @@ -144,11 +159,13 @@ width = request.GET.get('w', 160) height = request.GET.get('h', 24) super_ctx = super(WebConsoleNode, self)._get_context(request, session) - return dict(super_ctx.items() + {'SID': session.sessid, 'width':width, 'height':height, 'ip_list':ip_list}.items()) + return dict( + super_ctx.items() + {'SID': session.sessid, 'width': width, 'height': height, 'ip_list': ip_list}.items()) class SystemStatusNode(BaseNode): _template_name = 'box_system_status.html' + def _get_context(self, request, session): sess = get_current_session() system_status_model = get_model('cm', ['system_status', 'SystemStatus']) @@ -157,7 +174,7 @@ curr_time = system_status_model._meta.get_field('curr_time') connect_devices = system_status_model._meta.get_field('connect_devices') cm_version = system_status_model._meta.get_field('cm_version') - + super_ctx = super(SystemStatusNode, self)._get_context(request, session) return dict(super_ctx.items() + { 'system_status_model': system_status_model, @@ -168,6 +185,7 @@ 'cm_version': cm_version, }.items()) + class ConfigurationNode(BaseNode): _template_name = 'configuration.html' @@ -179,5 +197,4 @@ for each in device_id.all(): ip_list.append(each.ip.values()[0]) super_ctx = super(ConfigurationNode, self)._get_context(request, session) - return dict(super_ctx.items() + {'SID': session.sessid, 'ip_list':ip_list}.items()) - + return dict(super_ctx.items() + {'SID': session.sessid, 'ip_list': ip_list}.items()) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py (working copy) @@ -1,4 +1,6 @@ -import json, os, configparser +import json +import os + import lib.toml as toml AN_APPS = ( @@ -7,64 +9,63 @@ LEGACY_CLI_AGENT = True -LEGACY_CLI_AGENT_IP = '127.0.0.1' #PRODUCTMODE -#LEGACY_CLI_AGENT_IP = '10.23.1.98' #DEVMODE - update to your test system's IP +LEGACY_CLI_AGENT_IP = '127.0.0.1' # PRODUCTMODE +# LEGACY_CLI_AGENT_IP = '' #DEVMODE - update to your test system's IP # use "backend -c" to exec enable level cmds -FAST_CLI_CMD = True #PRODUCTMODE -#FAST_CLI_CMD = False #DEVMODE +FAST_CLI_CMD = True # PRODUCTMODE +# FAST_CLI_CMD = False #DEVMODE # Available Themes to choose from -THEMES = ['Default','Rose','Clark'] +THEMES = ['Default', 'Rose', 'Clark'] # Available Languages to choose from, see hive/lang.py for detail LANGS = ('en', 'zh-cn', 'zh-tw', 'ja') -# System-level default language, can be modified by CLI +# CLI can modify the system-level default language SYS_DEFAULT_LANG = 'en' # mongodb settings -DBHOST='127.0.0.1' -DBPORT=27017 +DBHOST = '127.0.0.1' +DBPORT = 27017 -PUBLIC_PATHS=['/login', '/logout', '/login_app', '/CacheClean', '/jsi18n/', '/webui_on', '/webui_off', '/restapi_on/', '/restapi_off', '/ReloadAppNode/', '/UpdateFrontend', '/cm/import_license_key'] - -# Test use only -PSEUDO_MANAGER_SHELF_FILE = '/home/tiankai/src/test.shelf' #update it to your local absolute path +PUBLIC_PATHS = ['/login', '/logout', '/login_app', '/CacheClean', '/jsi18n/', '/webui_on', '/webui_off', '/restapi_on/', + '/restapi_off', '/ReloadAppNode/', '/UpdateFrontend', '/cm/import_license_key'] +TEST_IP_POOL = [] PSEUDO_MANAGER_TEST_MODE = False -TEST_IP_POOL = ['192.168.2.111','192.168.2.112','192.168.2.113','192.168.2.114'] +PSEUDO_MANAGER_SHELF_FILE = '' -#Cache manager +# Cache manager CACHE_SWITCH = True -#config file +# config file CONFIG_FILE_DIR = "/var/crash/statmon/" CONFIG_FILE = CONFIG_FILE_DIR + "webui_settings.conf" -#For RRDtool +# For RRDtool DIR_STR = "/var/crash/statmon/" MAX_FILES_COUNT = 1000 RRDCACHED_SOCKET = "unix:/var/crash/rrdcached.sock" -#For SQLite +# For SQLite DB_FILE_PATH = "/var/crash/log_server.db" -#For whoosh +# For whoosh WHOOSH_INDEX_PATH = "/var/crash/whoosh" # For CLI "webui language" LANGUAGE_CONFIG_FILE = "/var/tmp/new_webui_lang" ###PRODUCTMODE### -WEBUI_ON_OFF = False +WEBUI_ON_OFF = False RESTAPI_ON_OFF = False RESTAPI_PORT = "9997" ###DEVMODE### -# WEBUI_ON_OFF = True +# WEBUI_ON_OFF = True # RESTAPI_ON_OFF = True # RESTAPI_PORT = "8888" -#*************************************** for CM *************************************** +# *************************************** for CM *************************************** ENCODE_SECRET_KEY = "qL1.xC67*" ###PRODUCTMODE### @@ -72,8 +73,8 @@ extension_path = '/ca/extensions' ###DEVMODE### -# webui_path = '/Users/zhangjiawei/svn_codes/amp/trunk/src/webui/webui/htdocs/new/src/' -# extension_path = '/Users/zhangjiawei/svn_codes/amp/trunk/extensions' +# webui_path = '/opt/src/webui/webui/htdocs/new/src/' +# extension_path = '/opt/extensions' frontend_path = webui_path + 'client/' frontend_modules_path = frontend_path + 'app/modules/' @@ -88,7 +89,7 @@ CMDATA = {} with open(CM_ConfigFile_PATH + '/cm.json', 'r') as f: - CMDATA = json.load(f) + CMDATA = json.load(f) if os.path.exists('/var/opt/composer/ui/conf/composer_ui.toml'): config = toml.load("/var/opt/composer/ui/conf/composer_ui.toml") @@ -100,24 +101,24 @@ CMDATA['COMPOSER_LOGO_PATH'] = config['app']['logoPath'] ###DEVMODE### -# CMDATA['COMPOSER_KEY'] = "wjb59724bf8534d3cb" -# CMDATA['COMPOSER_SECRET'] = "59e0340f0df438d329cd308210091cd8" -# CMDATA['INFLUXDB_IP'] = "10.23.1.98" -# CMDATA['ELASTICSEARCH_IP'] = "10.23.1.98" -# CMDATA['COMPOSER_IP'] = "10.23.1.98" +# CMDATA['COMPOSER_KEY'] = "" +# CMDATA['COMPOSER_SECRET'] = "" +# CMDATA['INFLUXDB_IP'] = "" +# CMDATA['ELASTICSEARCH_IP'] = "" +# CMDATA['COMPOSER_IP'] = "" if not os.path.exists(CM_ConfigFile_PATH + '/rs_block.json'): with open(CM_ConfigFile_PATH + '/rs_block.json', 'w') as f: f.write(json.dumps({})) AN_VPN_LIST = CMDATA['PRODUCT_CATEGORY']['VPN'] -OTHER_VPN_LIST = [] #just for oem, should be empty +OTHER_VPN_LIST = [] # just for oem, should be empty AN_ADC_LIST = CMDATA['PRODUCT_CATEGORY']['ADC'] -OTHER_ADC_LIST = [] #just for oem, should be empty +OTHER_ADC_LIST = [] # just for oem, should be empty AN_WAF_LIST = CMDATA['PRODUCT_CATEGORY']['WAF'] -OTHER_WAF_LIST = [] #just for oem, should be empty +OTHER_WAF_LIST = [] # just for oem, should be empty VPN_TYPE_LIST = AN_VPN_LIST + OTHER_VPN_LIST ADC_TYPE_LIST = AN_ADC_LIST + OTHER_ADC_LIST Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py (working copy) @@ -1,9 +1,9 @@ # Django settings for djproject project. DEBUG = False -#DEBUG = True #DEVMODE +# DEBUG = True #DEVMODE TEMPLATE_DEBUG = DEBUG -ALLOWED_HOSTS = ['*'] # for django 1.5+ +ALLOWED_HOSTS = ['*'] # for django 1.5+ ADMINS = ( # ('Your Name', 'your_email@example.com'), @@ -13,12 +13,12 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - 'NAME': '', # Or path to database file if using sqlite3. - 'USER': '', # Not used with sqlite3. - 'PASSWORD': '', # Not used with sqlite3. - 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. - 'PORT': '', # Set to empty string for default. Not used with sqlite3. + 'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': '', # Or path to a database file if using sqlite3. + 'USER': '', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string by default. Not used with sqlite3. } } @@ -32,7 +32,7 @@ # http://www.i18nguy.com/unicode/language-identifiers.html LOCALE_PATHS = ( '/ca/webui/htdocs/new/src/locale', - #'/path/to/your/locale',#DEVMODE + # '/path/to/your/locale',#DEVMODE ) SITE_ID = 1 @@ -45,7 +45,7 @@ # calendars according to the current locale. USE_L10N = True -# If you set this to False, Django will not use timezone-aware datetimes. +# If you set this to False, Django will not use timezone-aware datetime. USE_TZ = False # Absolute filesystem path to the directory that will hold user-uploaded files. @@ -79,17 +79,17 @@ STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', + # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) -# Make this unique, and don't share it with anybody. +# Make this unique and don't share it with anybody. SECRET_KEY = '^av)^9uv!y447(&l@%vydqipos_v^+f4@8n3-d_i2wt^xht02t' # List of callables that know how to import templates from various sources. TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', + # 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( @@ -107,7 +107,7 @@ CSRF_COOKIE_SECURE = True ROOT_URLCONF = 'djproject.urls' -# Python dotted path to the WSGI application used by Django's runserver. +# Python dotted the path to the WSGI application used by Django's runserver. WSGI_APPLICATION = 'djproject.wsgi.application' TEMPLATE_DIRS = ( @@ -117,20 +117,20 @@ ) INSTALLED_APPS = ( - #'django.contrib.auth', - #'django.contrib.contenttypes', - #'django.contrib.sessions', - #'django.contrib.sites', - #'django.contrib.messages', - #'django.contrib.staticfiles', + # 'django.contrib.auth', + # 'django.contrib.contenttypes', + # 'django.contrib.sessions', + # 'django.contrib.sites', + # 'django.contrib.messages', + # 'django.contrib.staticfiles', # Uncomment the next line to enable the admin: # 'django.contrib.admin', # Uncomment the next line to enable admin documentation: # 'django.contrib.admindocs', - #'statici18n',#DEVMODE - for generating javascript i18n files + # 'statici18n',#DEVMODE - for generating JavaScript i18n files ) -#UPLOAD SETTINGS +# UPLOAD SETTINGS FILE_UPLOAD_DIR = '/tmp/webui_upload/' CONFIG_UPLOAD_STORE_DIR = '/ca/package/' @@ -151,11 +151,11 @@ }, 'simple': { 'format': '%(levelname)s - %(message)s', - 'datefmt' : "%d/%b/%Y %H:%M:%S" + 'datefmt': "%d/%b/%Y %H:%M:%S" }, 'standard': { - 'format' : "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", - 'datefmt' : "%d/%b/%Y %H:%M:%S" + 'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + 'datefmt': "%d/%b/%Y %H:%M:%S" }, }, 'filters': { @@ -169,32 +169,32 @@ 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, - 'console':{ + 'console': { 'level': 'INFO', 'class': 'logging.StreamHandler', 'formatter': 'simple' }, - 'request': { - 'level':'DEBUG', - 'class':'logging.handlers.RotatingFileHandler', - 'filename': '/var/log/request.log', - 'maxBytes': 1024*1024*5, # 5 MB - 'backupCount': 5, - 'formatter':'simple', - }, - 'avx': { - 'level':'DEBUG', - 'class':'logging.handlers.RotatingFileHandler', - 'filename': '/var/log/avxwebui.log', - 'maxBytes': 1024*1024*5, # 5 MB - 'backupCount': 5, - 'formatter':'simple', - }, + 'request': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': '/var/log/request.log', + 'maxBytes': 1024 * 1024 * 5, # 5 MB + 'backupCount': 5, + 'formatter': 'simple', + }, + 'avx': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': '/var/log/avxwebui.log', + 'maxBytes': 1024 * 1024 * 5, # 5 MB + 'backupCount': 5, + 'formatter': 'simple', + }, 'logfile': { - 'level':'DEBUG', - 'class':'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', 'filename': '/var/log/hive.log', - 'maxBytes': 1024*1024*50, # 50 MB + 'maxBytes': 1024 * 1024 * 50, # 50 MB 'backupCount': 2, 'formatter': 'standard', }, @@ -228,7 +228,7 @@ 'hive.debug': { 'handlers': ['logfile'], 'level': 'INFO', - #'level': 'DEBUG', #DEVMODE + # 'level': 'DEBUG', #DEVMODE }, } } Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/composer.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/composer.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/composer.py (working copy) @@ -7,19 +7,20 @@ import subprocess import time -import lib.toml as toml -import lib.yaml as yaml -from cm.lib.libbasic_operation import oper_log +import httpx from django.http import HttpResponse +from django.views import View from django.views.decorators.csrf import csrf_exempt from django.views.decorators.gzip import gzip_page -from djproject.an_settings import extension_path, CMDATA from elasticsearch import Elasticsearch + +import lib.toml as toml +import lib.yaml as yaml +from cm.lib.libbasic_operation import oper_log +from djproject.an_settings import extension_path, CMDATA from hive.custom_exceptions import generic_exception as ge from hive.utils import andebug, upload_receive from lib.response import FileResponse -from django.views import View -import httpx ELASTIC_SEARCH_CONFIG_FILE = '/etc/elasticsearch/elasticsearch.yml' @@ -304,6 +305,7 @@ if os.path.exists("logo.png"): os.remove("logo.png") return HttpResponse(json.dumps({'is_success': True})) + return None def acquisition_config(request): @@ -328,6 +330,7 @@ os.system('service acm_syslogd restart') time.sleep(3) return HttpResponse(json.dumps({'is_success': True})) + return None def elastic_config(request): @@ -335,6 +338,7 @@ return HttpResponse(json.dumps({'is_success': True})) elif request.method == 'POST': return HttpResponse(json.dumps({'is_success': True})) + return None def kibana_config(request): @@ -365,6 +369,7 @@ os.system('systemctl restart kibana') time.sleep(3) return HttpResponse(json.dumps({'is_success': True})) + return None def composer_ui_config(request): @@ -392,6 +397,7 @@ toml.dump(config, f) fcntl.flock(f.fileno(), fcntl.LOCK_UN) return HttpResponse(json.dumps({'is_success': True})) + return None def composer_config(request, app): @@ -407,6 +413,7 @@ return is_external_drive_available() elif app == 'switch_log_location': return change_log_location(request) + return None def is_external_drive_available(): @@ -464,7 +471,7 @@ 'message': "Invalid location specified: {}. Valid values are 'internal' and 'external'".format(log_location) }), content_type='application/json') - # Read existing configuration + # Read the existing configuration with open(ELASTIC_SEARCH_CONFIG_FILE, 'r') as configfile: lines = configfile.readlines() @@ -522,6 +529,7 @@ try: oper_log('info', 'system', 'Executing command :{}'.format(command)) subprocess.check_call(command, shell=True) + return None except subprocess.CalledProcessError as e: oper_log('info', 'system', 'Executing command :{} failed with error:{}'.format(command, e.output)) return HttpResponse( @@ -536,6 +544,8 @@ 'message': "An unexpected error occurred", 'data': str(e) }), content_type='application/json') + return None + return None # if request.method == 'GET': @@ -602,21 +612,24 @@ if request.method == 'GET': return HttpResponse(json.dumps({'running': procIsRunning('acm_syslogd')})) elif request.method == 'POST': - pass + return None + return None def elastic_status(request): if request.method == 'GET': return HttpResponse(json.dumps({'running': procIsRunning('elasticsearch')})) elif request.method == 'POST': - pass + return None + return None def kibana_status(request): if request.method == 'GET': return HttpResponse(json.dumps({'running': procIsRunning('kibana')})) elif request.method == 'POST': - pass + return None + return None def composer_status(request, app): @@ -626,3 +639,4 @@ return elastic_status(request) elif app == 'kibana': return kibana_status(request) + return None Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (working copy) @@ -1,15 +1,16 @@ -import os import json +import os import subprocess from django.http import HttpResponse, StreamingHttpResponse -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log -from hive.utils import andebug, get_current_session -from hive.services.service_utils import ServiceUtils -from hive.services.schedule_backup import schedule_backup_job, fetch_schedule_backup_data, remove_backup_cron_job -from hive.services.backup_service_utils import persist_remote_storage_details, fetch_remote_storage_details +from hive.custom_exceptions import generic_exception as ge from hive.services.backup_service import BackupConfig +from hive.services.backup_service_utils import persist_remote_storage_details, fetch_remote_storage_details +from hive.services.schedule_backup import schedule_backup_job, fetch_schedule_backup_data, remove_backup_cron_job +from hive.services.service_utils import ServiceUtils +from hive.utils import get_current_session BACKUP_SCRIPT_FILE = "/ca/webui/htdocs/new/src/cm/backup/backup.sh" BACKUP_LOG_FILE = "/var/log/backup.log" @@ -43,7 +44,7 @@ "BackupConfig; BackupConfig().perform_backup(destination='{}')".format(backup_target) ) # Creating another process so that it runs in background - p1 = subprocess.Popen(["python", "-c", cmd], + p1 = subprocess.Popen(["python3", "-c", cmd], stdin=subprocess.PIPE, close_fds=True) p1.stdin.write(output.encode()) @@ -157,7 +158,8 @@ 'message': "Day of the month and month need to be specified as the frequency is yearly." }), content_type='application/json', status=400) - return schedule_backup_job(backup_target, frequency, hours, minutes, day_of_the_week, day_of_the_month, month, request) + return schedule_backup_job(backup_target, frequency, hours, minutes, day_of_the_week, day_of_the_month, month, + request) def get_schedule_backup_info(request): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (working copy) @@ -1,14 +1,13 @@ -import os import json -import subprocess from django.http import HttpResponse -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log -from hive.utils import andebug, get_current_session +from hive.custom_exceptions import generic_exception as ge from hive.services.restore_service import RestoreConfig -from hive.session import ANSession from hive.services.service_utils import ServiceUtils +from hive.session import ANSession +from hive.utils import get_current_session def handle_restore_req(request, path=None): @@ -127,7 +126,8 @@ raise HttpResponse(json.dumps({ 'error': 500, "status": "Failed", - "message": "Error while restoring backup : {}".format(e.message)}), content_type='application/json') + "message": "Error while restoring backup : {}".format(e.message)}), + content_type='application/json') return HttpResponse(json.dumps({ "status": "In-Progress", Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py (working copy) @@ -1,7 +1,8 @@ -from django.http import HttpResponse import json +from django.http import HttpResponse + class GenericError(Exception): """Exception raised for errors related to creating partition for secondary drive.""" @@ -41,4 +42,5 @@ self.description = description def __str__(self): - return "Parameter: {}, Error Code: {}, Description: {}".format(self.parm_name, self.error_code, self.description) + return "Parameter: {}, Error Code: {}, Description: {}".format(self.parm_name, self.error_code, + self.description) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/db/llb_stats_queries.py (working copy) @@ -1,11 +1,11 @@ -import time import hashlib import http.client import json +import time + +from cm.lib.libbasic_operation import oper_log from djproject.an_settings import CMDATA from hive.custom_exceptions import generic_exception as ge -from hive.utils import andebug -from cm.lib.libbasic_operation import oper_log class LLBStatsDB: @@ -34,7 +34,8 @@ else: query_string += "&time=" + timestamp + "&sign=" + m.hexdigest() - http_client = http.client.HTTPConnection(CMDATA['COMPOSER_IP'], str(CMDATA['COMPOSER_PORT']), True, timeout=15) + http_client = http.client.HTTPConnection(CMDATA['COMPOSER_IP'], str(CMDATA['COMPOSER_PORT']), True, + timeout=15) http_client.request(req_dict['method'], req_dict["url"] + "?" + query_string, req_payload, req_dict['config']) @@ -357,4 +358,4 @@ def format_query_string(query): """Formats query string to ensure proper URL encoding""" return query.strip().replace(' ', "%20").replace('<', "%3C").replace('>', "%3E").replace(',', "%2C") \ - .replace(';', "%3B").replace('+', "%2B").replace('/', "%2F").replace('|', "%7C").replace('=', "%3D") \ No newline at end of file + .replace(';', "%3B").replace('+', "%2B").replace('/', "%2F").replace('|', "%7C").replace('=', "%3D") Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/document.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/document.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/document.py (working copy) @@ -8,10 +8,11 @@ from importlib import import_module from xml.dom import minidom +from jinja2 import Environment, FileSystemLoader + import tools.xmltodict as xmltodict from hive.lang import HIVE_LANGUAGES, set_current_lang, get_current_lang from hive.utils import get_current_session -from jinja2 import Environment, FileSystemLoader # Here we extends standard ElementTree classes to implement the source_line feature @@ -21,7 +22,8 @@ source_line = kwargs.pop('source_line', 0) super(MyElement, self).__init__(*args, **kwargs) self.source_line = source_line - + + class MyTreeBuilder(ET.TreeBuilder): def start(self, tag, attrs, source_line=0): self._flush() @@ -32,6 +34,7 @@ self._tail = 0 return elem + class MyXMLParser(ET.XMLParser): def _start(self, tag, attrib_in): fixname = self._fixname @@ -49,14 +52,16 @@ attrib = {} if attrib_in: for i in range(0, len(attrib_in), 2): - attrib[fixname(attrib_in[i])] = fixtext(attrib_in[i+1]) + attrib[fixname(attrib_in[i])] = fixtext(attrib_in[i + 1]) return self.target.start(tag, attrib, source_line=self.parser.CurrentLineNumber) + def xml_parse(source): builder = MyTreeBuilder(element_factory=MyElement) parser = MyXMLParser(target=builder) return ET.parse(source, parser=parser) + class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' @@ -73,43 +78,51 @@ self.FAIL = '' self.ENDC = '' + class DOCUMENT_OK: text = 'OK' + class DOCUMENT_PENDING: text = 'PENDING' - + + class DOCUMENT_NEW: text = 'NEW' + def status_str2obj(text): for each in (DOCUMENT_OK, DOCUMENT_PENDING, DOCUMENT_NEW): if each.text == text: return each raise ValueError(str(text) + ' is not a valid document status') - + + def prettify(elem): """Return a pretty-printed XML string for the Element. """ rough_string = ET.tostring(elem, encoding='utf-8') rough_string = '\n' + rough_string reparsed = minidom.parseString(rough_string) - return '\n'.join([line for line in reparsed.toprettyxml(indent=' '*4).split('\n') if line.strip()]) - #return reparsed.toprettyxml(indent=" ") + return '\n'.join([line for line in reparsed.toprettyxml(indent=' ' * 4).split('\n') if line.strip()]) + # return reparsed.toprettyxml(indent=" ") + def get_doc_top_pkgs(app): get_app = import_module('.loading', 'hive.model').get_app app_pkg = get_app(app) return [each._meta.name for each in app_pkg.sub_packages.values()] + def get_doc_xml_path(app, lang, top_pkg, enguser=None): - if app in ['nsae','netopti','fortinet','ax','apv']: + if app in ['nsae', 'netopti', 'fortinet', 'ax', 'apv']: app = 'apv' if not enguser: - return os.path.dirname(os.path.realpath(__file__))+'/../'+app+'/doc/'+lang+'/'+top_pkg+'.xml' + return os.path.dirname(os.path.realpath(__file__)) + '/../' + app + '/doc/' + lang + '/' + top_pkg + '.xml' else: - return '/var/tmp/'+app+'/doc/'+enguser+'/'+lang+'/'+top_pkg+'.xml' + return '/var/tmp/' + app + '/doc/' + enguser + '/' + lang + '/' + top_pkg + '.xml' + # For Viewing Current Loaded XML file at eng mode. def get_current_loaded_xml_path(app_name, top_pkg): sess = get_current_session() @@ -121,24 +134,26 @@ else: xml_path = get_doc_xml_path(app_name, language, top_pkg) return xml_path - -_app_xml_trees = {} # cache for xml trees with structure {:{:, ...}, ...} + +_app_xml_trees = {} # cache for xml trees with structure {:{:, ...}, ...} + + def get_app_element_tree(app_name, top_pkg): global _app_xml_trees sess = get_current_session() - + if sess and sess.engmode: app_xml_trees = sess.eng_app_xml_trees else: app_xml_trees = _app_xml_trees - + # now check the cache language = get_current_lang() ele_tree_cached = app_xml_trees.get(app_name, {}).get(language, {}).get(top_pkg, None) if ele_tree_cached: return ele_tree_cached - + # parse from the xml path if sess and sess.engmode: xml_path = get_doc_xml_path(app_name, language, top_pkg, enguser=sess.username) @@ -151,10 +166,10 @@ try: ele_tree = xml_parse(xml_path) except IOError: - return None # the file should be found, because the original XML doc file is always there + return None # the file should be found, because the original XML doc file is always there except ET.ParseError: return None - + # fill in the cache if app_name not in app_xml_trees: app_xml_trees[app_name] = {} @@ -162,9 +177,10 @@ app_xml_trees[app_name][language] = {} if top_pkg not in app_xml_trees[app_name][language]: app_xml_trees[app_name][language][top_pkg] = ele_tree - + return ele_tree + def get_package_element(app_name, path): if len(path) == 0: return None @@ -176,12 +192,13 @@ if len(left_path) == 0: # this is the top package (the root node) return top_pkg_root - + # normal packages - xpath = './' + '/'.join(["package[@name='"+pkg_name+"']" for pkg_name in left_path]) + xpath = './' + '/'.join(["package[@name='" + pkg_name + "']" for pkg_name in left_path]) pkg = top_pkg_root.find(xpath) return pkg - + + def get_model_element(app_name, path): (top_pkg, left_path) = (path[0], path[1:]) top_pkg_ele_tree = get_app_element_tree(app_name, top_pkg) @@ -190,35 +207,38 @@ top_pkg_root = top_pkg_ele_tree.getroot() model = left_path[-1] packages = left_path[:-1] - xpath = './' + '/'.join(["package[@name='"+pkg_name+"']" for pkg_name in packages]) + '/' + "model[@name='"+model+"']" + xpath = './' + '/'.join( + ["package[@name='" + pkg_name + "']" for pkg_name in packages]) + '/' + "model[@name='" + model + "']" model_ele = top_pkg_root.find(xpath) return model_ele + def my_unicode(src): return str(src) if src is not None else '' + def check_verbose_name(verbose_name, target_obj): if (verbose_name and verbose_name[0].islower()): print('Warning: verbose_name should be capitalized - %s - %s' % (verbose_name, target_obj)) if ('\r' in verbose_name or '\n' in verbose_name): print('Warning: verbose_name should not include line break - %s - %s' % (verbose_name, target_obj)) - - + + # this function should only be called from command line def generate_xml(app, lang, target_path): if lang == 'all': for each_lang in HIVE_LANGUAGES.keys(): generate_xml(app, each_lang, target_path) return - + # set current lang so that the model layer will read from correct language xml file. print(bcolors.FAIL + 'Generating doc XML for language %s' % (lang) + bcolors.ENDC) set_current_lang(lang) # The target dir path if not target_path: - target_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__))+'/../'+app+'/doc/'+lang) - + target_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/../' + app + '/doc/' + lang) + # Create the dir if it doesn't exists if not os.path.exists(target_path): os.makedirs(target_path) @@ -229,7 +249,7 @@ app_pkg.reload_document() # Now start generating per-top-package xml files inside target_path directory - #root = ET.Element('application', attrib={'name':app_pkg._meta.app_label}) + # root = ET.Element('application', attrib={'name':app_pkg._meta.app_label}) def gen_pkg_ele(pkg, pkg_ele): verbose_name_ele = ET.SubElement(pkg_ele, 'verbose_name') verbose_name_ele.text = my_unicode(pkg.verbose_name) @@ -238,12 +258,12 @@ help_text_ele.text = my_unicode(pkg.help_text) help_link_ele = ET.SubElement(pkg_ele, 'help_link') help_link_ele.text = my_unicode(pkg.help_link) - + for sub_model in pkg.models.values(): if not sub_model._meta.generate_xml: continue model_document_status = sub_model._meta.document_status - sub_model_ele = ET.SubElement(pkg_ele, 'model', attrib={'name':sub_model._meta.object_name}) + sub_model_ele = ET.SubElement(pkg_ele, 'model', attrib={'name': sub_model._meta.object_name}) verbose_name_ele = ET.SubElement(sub_model_ele, 'verbose_name') verbose_name_ele.text = my_unicode(sub_model._meta.verbose_name) check_verbose_name(verbose_name_ele.text, sub_model) @@ -251,13 +271,13 @@ help_text_ele.text = my_unicode(sub_model._meta.help_text) help_link_ele = ET.SubElement(sub_model_ele, 'help_link') help_link_ele.text = my_unicode(sub_model._meta.help_link) - #for restapi doc usage - #load data between first - #if not exist, generate a new element + # for restapi doc usage + # load data between first + # if not exist, generate a new element if sub_model._meta.document is not None: current_restapi_examples_ele = sub_model._meta.document.find('./restapi_examples') if current_restapi_examples_ele is not None: - #using old value to create a element + # using old value to create a element sub_model_ele.append(current_restapi_examples_ele) # else: # #create default element for per model @@ -281,17 +301,19 @@ if model_document_status is DOCUMENT_OK and field_grp_document_status is DOCUMENT_NEW: print('field grp %s is NEW' % field_grp.name) model_document_status = DOCUMENT_PENDING - field_grp_ele = ET.SubElement(sub_model_ele, 'fieldgrp', attrib={'name':field_grp.name}) + field_grp_ele = ET.SubElement(sub_model_ele, 'fieldgrp', attrib={'name': field_grp.name}) verbose_name_ele = ET.SubElement(field_grp_ele, 'verbose_name') verbose_name_ele.text = my_unicode(field_grp.verbose_name) check_verbose_name(verbose_name_ele.text, field_grp) help_text_ele = ET.SubElement(field_grp_ele, 'help_text') help_text_ele.text = my_unicode(field_grp.help_text) - + for field in field_grp.fields: if field.is_inherited() or field.hidden: continue - field_ele = ET.SubElement(field_grp_ele, 'field', attrib={'name':field.name, 'type':field.type_name}) + field_ele = ET.SubElement(field_grp_ele, 'field', + attrib={'name': field.name, 'type': field.type_name}) + # returns the status of the field def gen_field_ele(f, f_ele): status = f.document_status @@ -304,16 +326,20 @@ choices_ele = ET.SubElement(f_ele, 'choices') if f.values: for each_choice in f.values: - choice_ele = ET.SubElement(choices_ele, 'choice', attrib={'value':my_unicode(each_choice[0])}) + choice_ele = ET.SubElement(choices_ele, 'choice', + attrib={'value': my_unicode(each_choice[0])}) choice_verbose_name_ele = ET.SubElement(choice_ele, 'verbose_name') - choice_verbose_name_ele.text = my_unicode(each_choice[1] if len(each_choice) >=2 else '') + choice_verbose_name_ele.text = my_unicode( + each_choice[1] if len(each_choice) >= 2 else '') choice_help_text_ele = ET.SubElement(choice_ele, 'help_text') - choice_help_text_ele.text = my_unicode(each_choice[2] if len(each_choice) >= 3 else '') + choice_help_text_ele.text = my_unicode( + each_choice[2] if len(each_choice) >= 3 else '') sub_fields = f.get_sub_fields() if sub_fields is not None: sub_fields_ele = ET.SubElement(f_ele, 'subfields') for sub_field in sub_fields: - sub_field_ele = ET.SubElement(sub_fields_ele, 'field', attrib={'name':sub_field.name, 'type':sub_field.type_name}) + sub_field_ele = ET.SubElement(sub_fields_ele, 'field', attrib={'name': sub_field.name, + 'type': sub_field.type_name}) sub_status = gen_field_ele(sub_field, sub_field_ele) if sub_status == DOCUMENT_NEW and status == DOCUMENT_OK: status = DOCUMENT_PENDING @@ -331,7 +357,7 @@ if model_document_status is DOCUMENT_OK and action_document_status is DOCUMENT_NEW: print('action %s is NEW' % action.name) model_document_status = DOCUMENT_PENDING - action_ele = ET.SubElement(sub_model_ele, 'action', attrib={'name':action.name}) + action_ele = ET.SubElement(sub_model_ele, 'action', attrib={'name': action.name}) action_name_ele = ET.SubElement(action_ele, 'action_name') action_name_ele.text = my_unicode(action.action_name) verbose_name_ele = ET.SubElement(action_ele, 'verbose_name') @@ -349,13 +375,14 @@ process_msg_ele.text = my_unicode(action.process_msg) finish_msg_ele = ET.SubElement(action_ele, 'finish_msg') finish_msg_ele.text = my_unicode(action.finish_msg) - #for restapi doc usage - #load data between first - #if not exist, generate a new element + # for restapi doc usage + # load data between first + # if not exist, generate a new element if sub_model._meta.document is not None: - current_restapi_examples_ele = sub_model._meta.document.find("./action[@name='"+action.name+"']/restapi_examples") + current_restapi_examples_ele = sub_model._meta.document.find( + "./action[@name='" + action.name + "']/restapi_examples") if current_restapi_examples_ele is not None: - #using old value to create a element + # using old value to create a element action_ele.append(current_restapi_examples_ele) # else: # #create default element for per action @@ -375,7 +402,8 @@ for field in action.option_fields: if field.is_inherited() or field.hidden: continue - field_ele = ET.SubElement(action_ele, 'field', attrib={'name':field.name, 'type':field.type_name}) + field_ele = ET.SubElement(action_ele, 'field', attrib={'name': field.name, 'type': field.type_name}) + # returns the status of the field def gen_field_ele(f, f_ele): status = f.document_status @@ -388,16 +416,20 @@ choices_ele = ET.SubElement(f_ele, 'choices') if f.values: for each_choice in f.values: - choice_ele = ET.SubElement(choices_ele, 'choice', attrib={'value':my_unicode(each_choice[0])}) + choice_ele = ET.SubElement(choices_ele, 'choice', + attrib={'value': my_unicode(each_choice[0])}) choice_verbose_name_ele = ET.SubElement(choice_ele, 'verbose_name') - choice_verbose_name_ele.text = my_unicode(each_choice[1] if len(each_choice) >=2 else '') + choice_verbose_name_ele.text = my_unicode( + each_choice[1] if len(each_choice) >= 2 else '') choice_help_text_ele = ET.SubElement(choice_ele, 'help_text') - choice_help_text_ele.text = my_unicode(each_choice[2] if len(each_choice) >= 3 else '') + choice_help_text_ele.text = my_unicode( + each_choice[2] if len(each_choice) >= 3 else '') sub_fields = f.get_sub_fields() if sub_fields is not None: sub_fields_ele = ET.SubElement(f_ele, 'subfields') for sub_field in sub_fields: - sub_field_ele = ET.SubElement(sub_fields_ele, 'field', attrib={'name':sub_field.name, 'type':sub_field.type_name}) + sub_field_ele = ET.SubElement(sub_fields_ele, 'field', attrib={'name': sub_field.name, + 'type': sub_field.type_name}) sub_status = gen_field_ele(sub_field, sub_field_ele) if sub_status == DOCUMENT_NEW and status == DOCUMENT_OK: status = DOCUMENT_PENDING @@ -411,72 +443,79 @@ action_document_status = DOCUMENT_PENDING # finally, set the model status - #if sub_model._meta.path[0] in ('ha', 'system', 'admintools') or 'ssl' in sub_model._meta.path or 'llb' in sub_model._meta.path: + # if sub_model._meta.path[0] in ('ha', 'system', 'admintools') or 'ssl' in sub_model._meta.path or 'llb' in sub_model._meta.path: # sub_model_ele.set('status', 'OK') - #else: + # else: sub_model_ele.set('status', model_document_status.text) - + for sub_pkg in pkg.sub_packages.values(): - sub_pkg_ele = ET.SubElement(pkg_ele, 'package', attrib={'name':sub_pkg._meta.name}) + sub_pkg_ele = ET.SubElement(pkg_ele, 'package', attrib={'name': sub_pkg._meta.name}) gen_pkg_ele(sub_pkg, sub_pkg_ele) - + # for each top-packages of the app, generate a XML file for top_pkg in app_pkg.sub_packages.values(): - top_pkg_ele = ET.Element('package', attrib={'name':top_pkg._meta.name, 'lang':lang}) + top_pkg_ele = ET.Element('package', attrib={'name': top_pkg._meta.name, 'lang': lang}) gen_pkg_ele(top_pkg, top_pkg_ele) top_pkg_path = target_path + '/' + top_pkg._meta.name + '.xml' - + exists = False if os.path.isfile(top_pkg_path): exists = True - - target_file = codecs.open(top_pkg_path+'.new', 'w', 'utf-8') + + target_file = codecs.open(top_pkg_path + '.new', 'w', 'utf-8') target_file.write(prettify(top_pkg_ele)) target_file.close() if exists: - if filecmp.cmp(top_pkg_path, top_pkg_path+'.new'): + if filecmp.cmp(top_pkg_path, top_pkg_path + '.new'): # new file is equal with old one - os.remove(top_pkg_path+'.new') + os.remove(top_pkg_path + '.new') print("File unchanged: %s" % (bcolors.OKBLUE + top_pkg_path + bcolors.ENDC)) else: - print("Successfully updated XML file into %s" % (bcolors.OKGREEN + top_pkg_path + '.new' + bcolors.ENDC)) - os.system("diff -up %s %s > %s" % (top_pkg_path, top_pkg_path+'.new', top_pkg_path+'.diff')) - print("And the diff is made at %s" % (bcolors.OKBLUE + top_pkg_path+'.diff' + bcolors.ENDC)) + print( + "Successfully updated XML file into %s" % (bcolors.OKGREEN + top_pkg_path + '.new' + bcolors.ENDC)) + os.system("diff -up %s %s > %s" % (top_pkg_path, top_pkg_path + '.new', top_pkg_path + '.diff')) + print("And the diff is made at %s" % (bcolors.OKBLUE + top_pkg_path + '.diff' + bcolors.ENDC)) else: print("Successfully generated XML file %s" % (bcolors.OKGREEN + top_pkg_path + '.new' + bcolors.ENDC)) + def get_xml_reference(pkg, ele, default=""): if ele is None: return default - + rtn = ele.text sess = get_current_session() if sess and sess.engmode and sess.doc_engmode: - return (rtn if rtn else "") + ' ' % (pkg, str(ele.source_line)) + return ( + rtn if rtn else "") + ' ' % ( + pkg, str(ele.source_line)) else: return rtn if rtn else default - + + def standarlize_xml(src_path, target_path): ele_tree = xml_parse(src_path) root = ele_tree.getroot() target_file = codecs.open(target_path, 'w', 'utf-8') target_file.write(prettify(root)) target_file.close() - + def my_unicode(src): return str(src) if src is not None else '' + # Generate RESTful API Spec HTML def rest_gen_model_spec(app, target_dir): get_app = import_module('.loading', 'hive.model').get_app app_pkg = get_app(app) app_path = target_dir + '/' + app - + # jinja2 env - env = Environment(loader = FileSystemLoader(os.path.abspath(os.path.dirname(os.path.realpath(__file__))+'/templates/docs'))) - + env = Environment( + loader=FileSystemLoader(os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/templates/docs'))) + # create the dir if not exist if not os.path.exists(app_path): os.makedirs(app_path) @@ -485,26 +524,28 @@ app_index_path = app_path + '/index.html' template = env.get_template('model_app_index.html') with codecs.open(app_index_path, "w", 'utf-8') as fh: - fh.write(template.render({'APP':app_pkg, 'app':app, 'DOCROOT':'./', 'STATIC':'../static/'})) + fh.write(template.render({'APP': app_pkg, 'app': app, 'DOCROOT': './', 'STATIC': '../static/'})) fh.close() - + # generate a package's files def gen_pkg(pkg, pkg_dir, is_root=False): # create the dir if not exist if not os.path.exists(pkg_dir): os.makedirs(pkg_dir) - + # compute the depths of current file - depth = len(pkg.path)+1 + depth = len(pkg.path) + 1 if not is_root: # generate package's index rst pkg_index_path = pkg_dir + '/index.html' template = env.get_template('model_pkg_index.html') with codecs.open(pkg_index_path, "w", 'utf-8') as fh: - fh.write(template.render({'PACKAGE':pkg, 'APP':app_pkg, 'app':app, 'DOCROOT':'/media/docs/rest/'+app+'/', 'STATIC':'../'*depth+'static/'})) + fh.write(template.render( + {'PACKAGE': pkg, 'APP': app_pkg, 'app': app, 'DOCROOT': '/media/docs/rest/' + app + '/', + 'STATIC': '../' * depth + 'static/'})) fh.close() - + # generate sub models' rsts for sub_model in pkg.models.values(): model_path = pkg_dir + '/' + sub_model._meta.module_name + '.html' @@ -514,17 +555,23 @@ template = env.get_template('model_model.html') # load examples from xml file model_examples = {} - #following example can be one or more - model_examples['delete_instance'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two - model_examples['create_instance'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost - model_examples['update_instance'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/host_status - model_examples['filter_instance_list'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost?name=test_two - model_examples['instance_fieldgroup'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/advanced_options - model_examples['instance_field'] = [] #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/host_status - model_examples['instance_all_fields'] = [] #just for non profile model, /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two - model_examples['instance_asso_fields'] = {} #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni - model_examples['instance_asso_delete_instance'] = {} #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni/test_two-www.aaa.com - model_examples['instance_asso_create_instance'] = {} #/rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni + # following example can be one or more + model_examples['delete_instance'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two + model_examples['create_instance'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost + model_examples[ + 'update_instance'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/host_status + model_examples['filter_instance_list'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost?name=test_two + model_examples[ + 'instance_fieldgroup'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/advanced_options + model_examples['instance_field'] = [] # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/host_status + model_examples[ + 'instance_all_fields'] = [] # just for non profile model, /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two + model_examples[ + 'instance_asso_fields'] = {} # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni + model_examples[ + 'instance_asso_delete_instance'] = {} # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni/test_two-www.aaa.com + model_examples[ + 'instance_asso_create_instance'] = {} # /rest/apv/loadbalancing/slb/ssl/SSLVirtualHost/test_two/ssl_sni action_examples = {} if sub_model._meta.document is not None: restapi_examples_ele = sub_model._meta.document.find('./restapi_examples') @@ -533,28 +580,30 @@ for each_example_ele in model_examples_ele: each_example_ele_str = ET.tostring(each_example_ele, encoding="utf-8", method="xml") each_example_dict = xmltodict.parse(each_example_ele_str, encoding="utf-8")["example"] - if not each_example_dict['method'] or not each_example_dict['url'] or not each_example_dict['response_data']: - #all fields mustn't be empty execept request_data + if not each_example_dict['method'] or not each_example_dict['url'] or not each_example_dict[ + 'response_data']: + # all fields mustn't be empty execept request_data continue if each_example_dict['url'].find('/meta') != -1: model_examples['meta'] = each_example_dict else: - if '/'.join(each_example_dict['url'].split('/rest/')[1].split('/')[:-1]) == '%s/'%app + '/'.join(sub_model._meta.path): + if '/'.join(each_example_dict['url'].split('/rest/')[1].split('/')[ + :-1]) == '%s/' % app + '/'.join(sub_model._meta.path): if each_example_dict['method'] == 'GET': model_examples['instance_all_fields'].append(each_example_dict) elif each_example_dict['method'] == 'DELETE': model_examples['delete_instance'].append(each_example_dict) - if each_example_dict['url'].split('/rest/')[1] == '%s/'%app + '/'.join(sub_model._meta.path): + if each_example_dict['url'].split('/rest/')[1] == '%s/' % app + '/'.join(sub_model._meta.path): if each_example_dict['method'] == 'GET': - #for profile model, there is only one instance - #for non profile model, there can be more than one instances. + # for profile model, there is only one instance + # for non profile model, there can be more than one instances. model_examples['instance_list'] = each_example_dict elif each_example_dict['method'] == 'POST': model_examples['create_instance'].append(each_example_dict) elif each_example_dict['method'] == 'PUT': - #profile model instance update + # profile model instance update model_examples['update_instance'].append(each_example_dict) - if '%s/'%app + '/'.join(sub_model._meta.path) + '?' in each_example_dict['url']: + if '%s/' % app + '/'.join(sub_model._meta.path) + '?' in each_example_dict['url']: model_examples['filter_instance_list'].append(each_example_dict) if each_example_dict['url'].split('/')[-1] in sub_model._meta.field_groups.keys(): if each_example_dict['method'] == 'GET': @@ -564,31 +613,35 @@ field = sub_model._meta.get_field(each_example_dict['url'].split('/')[-1]) if each_example_dict['method'] == 'PUT': model_examples['update_instance'].append(each_example_dict) - elif each_example_dict['method'] == 'GET': + elif each_example_dict['method'] == 'GET': if field.type_name == 'asso': model_examples['instance_asso_fields'][field_name] = each_example_dict else: model_examples['instance_field'].append(each_example_dict) - elif each_example_dict['method'] == 'POST': + elif each_example_dict['method'] == 'POST': if field.is_composition_asso: model_examples['instance_asso_create_instance'][field_name] = each_example_dict if each_example_dict['url'].split('/')[-2] in [field.name for field in sub_model._meta.fields]: field_name = each_example_dict['url'].split('/')[-2] field = sub_model._meta.get_field(each_example_dict['url'].split('/')[-2]) - if each_example_dict['method'] == 'DELETE': + if each_example_dict['method'] == 'DELETE': if field.is_composition_asso: model_examples['instance_asso_delete_instance'][field_name] = each_example_dict for action in sub_model._meta.actions: action_examples[action.name] = [] - action_examples_ele = sub_model._meta.document.find("./action[@name='"+action.name+"']/restapi_examples") + action_examples_ele = sub_model._meta.document.find( + "./action[@name='" + action.name + "']/restapi_examples") if action_examples_ele is not None: action_examples_ele = action_examples_ele.getchildren() for each_action_example_ele in action_examples_ele: - each_action_example_ele_str = ET.tostring(each_action_example_ele, encoding="utf-8", method="xml") - each_action_example_dict = xmltodict.parse(each_action_example_ele_str, encoding="utf-8")["example"] - if not each_action_example_dict['method'] or not each_action_example_dict['url'] or not each_action_example_dict['response_data']: - #all fields mustn't be empty execept request_data + each_action_example_ele_str = ET.tostring(each_action_example_ele, encoding="utf-8", + method="xml") + each_action_example_dict = xmltodict.parse(each_action_example_ele_str, encoding="utf-8")[ + "example"] + if not each_action_example_dict['method'] or not each_action_example_dict['url'] or not \ + each_action_example_dict['response_data']: + # all fields mustn't be empty execept request_data continue action_examples[action.name].append(each_action_example_dict) @@ -599,12 +652,15 @@ except: clis = [] with codecs.open(model_path, "w", 'utf-8') as fh: - fh.write(template.render({'MODEL':sub_model, 'MODEL_EXAMPLES': model_examples, 'ACTION_EXAMPLES': action_examples, 'PACKAGE':pkg, 'APP':app_pkg, 'app':app, 'CLIS':clis, 'DOCROOT':'/media/docs/rest/'+app+'/', 'STATIC':'../'*depth+'static/'})) + fh.write(template.render( + {'MODEL': sub_model, 'MODEL_EXAMPLES': model_examples, 'ACTION_EXAMPLES': action_examples, + 'PACKAGE': pkg, 'APP': app_pkg, 'app': app, 'CLIS': clis, + 'DOCROOT': '/media/docs/rest/' + app + '/', 'STATIC': '../' * depth + 'static/'})) fh.close() - + # generate sub packages' rsts for sub_pkg in pkg.sub_packages.values(): sub_pkg_dir = pkg_dir + '/' + sub_pkg._meta.name gen_pkg(sub_pkg, sub_pkg_dir) - - gen_pkg(app_pkg, app_path, is_root=True) \ No newline at end of file + + gen_pkg(app_pkg, app_path, is_root=True) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/engineer.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/engineer.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/engineer.py (working copy) @@ -1,16 +1,26 @@ -from jinja2 import Environment, PackageLoader, ChoiceLoader -from hive.utils import andebug, HiveEnvironment, get_current_session -from hive.document import get_doc_xml_path, get_current_loaded_xml_path, xml_parse, get_doc_top_pkgs, standarlize_xml -import os, os.path, time, shutil, sys, difflib, subprocess as commands, traceback -import json import codecs -from django.template import RequestContext +import difflib import html as cgi -from hive.model.query import manager_cache, DelayedQuery -from django.http import HttpResponse +import json +import os +import os.path +import shutil +import subprocess as commands +import sys +import time +import traceback from collections import OrderedDict + +from django.http import HttpResponse +from django.template import RequestContext +from jinja2 import PackageLoader, ChoiceLoader + from cm.lib.postgres_db import DB +from hive.document import get_doc_xml_path, get_current_loaded_xml_path, xml_parse, get_doc_top_pkgs, standarlize_xml +from hive.model.query import manager_cache +from hive.utils import HiveEnvironment, get_current_session + def document(request): django_ctx = RequestContext(request) lang = request.GET.get('lang', 'en') @@ -30,7 +40,7 @@ 'detail_url': '/eng/doc_detail/?lang=%s&app=%s&pkg=%s' % (lang, app, each_pkg), 'mtime': time.ctime(os.path.getmtime(orig_path)), }) - + # The updated XML file info update_path = get_doc_xml_path(app, lang, each_pkg, enguser=sess.username) update_info = { @@ -38,21 +48,24 @@ 'exists': os.path.isfile(update_path), } if update_info['exists']: - update_info['url'] = '/eng/download_doc/%s_%s.xml?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, each_pkg, lang, app, each_pkg, sess.username) - update_info['detail_url'] = '/eng/doc_detail/?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, app, each_pkg, sess.username) + update_info['url'] = '/eng/download_doc/%s_%s.xml?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, each_pkg, lang, + app, each_pkg, + sess.username) + update_info['detail_url'] = '/eng/doc_detail/?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, app, each_pkg, + sess.username) update_info['mtime'] = time.ctime(os.path.getmtime(update_path)) - update_info['diff_url'] = '/eng/diff_doc/?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, app, each_pkg, sess.username) + update_info['diff_url'] = '/eng/diff_doc/?lang=%s&app=%s&pkg=%s&enguser=%s' % (lang, app, each_pkg, + sess.username) update_info['stat'] = os.stat(update_path) UPDATE_FILES.append(update_info) - - + env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) template = env.get_template('/eng_document.html') - ctx = {'SESSION':get_current_session(), 'MEDIA':'/media/', 'LANG':lang, 'APP':app, - 'ENGUSER':sess.username if sess.username else '', - 'TOP_PKGS':top_pkgs, - 'ORIG_FILES':ORIG_FILES, 'UPDATE_FILES':UPDATE_FILES, 'csrf_token':django_ctx['csrf_token']} - + ctx = {'SESSION': get_current_session(), 'MEDIA': '/media/', 'LANG': lang, 'APP': app, + 'ENGUSER': sess.username if sess.username else '', + 'TOP_PKGS': top_pkgs, + 'ORIG_FILES': ORIG_FILES, 'UPDATE_FILES': UPDATE_FILES, 'csrf_token': django_ctx['csrf_token']} + # Get the info of NEW/PENDING models """ with codecs.open(current_path, 'r', encoding='utf-8') as xmlfile: @@ -81,9 +94,10 @@ ctx['CURRENT_PATH'] = current_path xmlfile.close() """ - + return template.render(ctx) + def download_doc(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -93,6 +107,7 @@ return xmlfile.read() return 'Package File Not Found' + def doc_detail(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -104,9 +119,9 @@ env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) template = env.get_template('/eng_document_detail.html') - ctx = {'SESSION':sess, 'MEDIA':'/media/', 'LANG':lang, 'APP':app, - 'ENGUSER':enguser, - 'PACKAGE':pkg} + ctx = {'SESSION': sess, 'MEDIA': '/media/', 'LANG': lang, 'APP': app, + 'ENGUSER': enguser, + 'PACKAGE': pkg} with codecs.open(current_path, 'r', encoding='utf-8') as xmlfile: all_lines = xmlfile.readlines() @@ -115,18 +130,19 @@ new_model_eles_extended = [] pending_model_eles = ele_tree.findall(".//model[@status='PENDING']") pending_model_eles_extended = [] - for (eles, eles_extended) in ((new_model_eles, new_model_eles_extended), (pending_model_eles, pending_model_eles_extended)): + for (eles, eles_extended) in ((new_model_eles, new_model_eles_extended), + (pending_model_eles, pending_model_eles_extended)): for each_ele in eles: line = each_ele.source_line - start = line-10 if line >= 10 else 0 - end = line+10 + start = line - 10 if line >= 10 else 0 + end = line + 10 bias = line - start text_lines = cgi.escape(''.join(all_lines[start:end])) ele_dict = { - 'ele':each_ele, - 'text_lines':text_lines, - 'bias':bias, - 'line':line, + 'ele': each_ele, + 'text_lines': text_lines, + 'bias': bias, + 'line': line, } eles_extended.append(ele_dict) ctx['NEW_MODEL_ELES'] = new_model_eles_extended @@ -136,6 +152,7 @@ return template.render(ctx) + def view_loaded_doc(request): app = request.GET.get('app', 'apv') line = request.GET.get('line', None) @@ -143,23 +160,25 @@ bias = None if line: line = int(line) - start = line-10 if line >= 10 else 0 - end = line+10 + start = line - 10 if line >= 10 else 0 + end = line + 10 bias = line - start with codecs.open(get_current_loaded_xml_path(app, pkg), 'r', encoding='utf-8') as xmlfile: if line: text_lines = xmlfile.readlines()[start:end] else: text_lines = xmlfile.readlines() - + env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) template = env.get_template('/eng_view_loaded_doc.html') - ctx = {'MEDIA':'/media/', 'PATH':get_current_loaded_xml_path(app, pkg), 'CONTENT':cgi.escape(''.join(text_lines))} + ctx = {'MEDIA': '/media/', 'PATH': get_current_loaded_xml_path(app, pkg), + 'CONTENT': cgi.escape(''.join(text_lines))} if line: ctx['BIAS'] = bias ctx['LINE'] = line return template.render(ctx) - + + def diff_doc(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -171,9 +190,11 @@ from_lines = from_file_p.readlines() with codecs.open(to_file, 'r', encoding='utf-8') as to_file_p: to_lines = to_file_p.readlines() - diff = difflib.HtmlDiff().make_file(from_lines, to_lines, from_file, to_file).replace('charset=ISO-8859-1', 'charset=UTF-8') + diff = difflib.HtmlDiff().make_file(from_lines, to_lines, from_file, to_file).replace('charset=ISO-8859-1', + 'charset=UTF-8') return diff + def upload_doc(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -182,7 +203,7 @@ orig_path = get_doc_xml_path(app, lang, pkg) update_path = get_doc_xml_path(app, lang, pkg, enguser=sess.username) update_dir = os.path.dirname(update_path) - + if not os.path.exists(update_dir): os.makedirs(update_dir) new_file = request.FILES['file'] @@ -190,9 +211,10 @@ for chunk in new_file.chunks(): destination.write(chunk) standarlize_xml(update_path, update_path) - + sess.reload_document() - + + def clone_doc(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -210,7 +232,8 @@ except IOError: sess.reload_document() return json.dumps([False, orig_path, update_path]) - + + def remove_doc(request): lang = request.GET.get('lang', 'english') app = request.GET.get('app', 'apv') @@ -226,23 +249,27 @@ sess.reload_document() return json.dumps([False]) + def ml_download(request): l = request.GET.get('lang', 'en') t = request.GET.get('type', 'py') file_name = 'django.po' if t == 'py' else 'djangojs.po' - with codecs.open(os.path.dirname(os.path.realpath(__file__))+'/../locale/'+l+'/LC_MESSAGES/'+file_name, 'r', encoding='utf-8') as pofile: + with codecs.open(os.path.dirname(os.path.realpath(__file__)) + '/../locale/' + l + '/LC_MESSAGES/' + file_name, 'r', + encoding='utf-8') as pofile: return pofile.read() + def engineering_utils(request): - django_ctx = RequestContext(request) - sess = get_current_session() - - env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) - template = env.get_template('/eng_utils.html') - ctx = {'SESSION':get_current_session(), 'MEDIA':'/media/', 'CACHE':manager_cache, - 'ENGUSER':sess.username if sess.username else '', 'csrf_token':django_ctx['csrf_token']} - return template.render(ctx) + django_ctx = RequestContext(request) + sess = get_current_session() + env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) + template = env.get_template('/eng_utils.html') + ctx = {'SESSION': get_current_session(), 'MEDIA': '/media/', 'CACHE': manager_cache, + 'ENGUSER': sess.username if sess.username else '', 'csrf_token': django_ctx['csrf_token']} + return template.render(ctx) + + def get_cache_list(request): result = [] for key, instance in manager_cache._cache_dict.iteritems(): @@ -256,13 +283,15 @@ 'value': str(instance.get_attr_raw(each_field.name)) } data['value'].append(d) - result.append(data) + result.append(data) return json.dumps(result) + def clear_cache(request): manager_cache.clear_all() return json.dumps([True]) + def toggle_engmode(request): action = request.GET.get('action', 'on') sess = get_current_session() @@ -273,6 +302,7 @@ sess.reload_document() return json.dumps([True]) + def view_log_info(request): data = OrderedDict() data['ps auxwww | grep backend'] = commands.getoutput('ps auxwww | grep backend') @@ -281,24 +311,26 @@ data['tail -n 200 /var/log/request.log'] = commands.getoutput('tail -n 200 /var/log/request.log') return HttpResponse(json.dumps(data)) + def view_debug_info(request): rst_data = [] for threadId, stack in sys._current_frames().items(): rst = { - 'thread_id':threadId, - 'trace':[], - 'local_vars':[] + 'thread_id': threadId, + 'trace': [], + 'local_vars': [] } new_stack = stack while new_stack: rst['local_vars'].insert(0, str(new_stack.f_locals)) new_stack = new_stack.f_back for filename, lineno, name, line in traceback.extract_stack(stack): - tmp_tb = {'file':filename, "line":lineno, "function":name, "code": line.strip() if line else ""} + tmp_tb = {'file': filename, "line": lineno, "function": name, "code": line.strip() if line else ""} rst['trace'].append(tmp_tb) rst_data.append(rst) return HttpResponse(json.dumps(rst_data)) + def execute_command(request): password = request.POST.get('password') command = request.POST.get('command') @@ -313,6 +345,7 @@ 'result': 'Wrong Password!' })) + def fetch_retry_time(request): try: fetch_sql = "select retry_time from vpn_retry_time" @@ -326,6 +359,7 @@ return HttpResponse(json.dumps({"state": True, "data": retry_time})) + def update_retry_time(request): retry_time = int(request.POST.get('retry_time')) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/exceptions.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/exceptions.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/exceptions.py (working copy) @@ -1,15 +1,21 @@ import copy + class FieldError(Exception): """Some kind of problem with a model field.""" pass + class ModelDefinitionError(Exception): pass + NON_FIELD_ERRORS = '__all__' + + class ValidationError(Exception): """An error while validating data.""" + def __init__(self, message, code=None, params=None): import operator from django.utils.encoding import force_str @@ -20,7 +26,7 @@ """ if isinstance(message, dict): self.message_dict = copy.copy(message) - for key,value in self.message_dict.iteritems(): + for key, value in self.message_dict.iteritems(): self.message_dict[key] = ', '.join(value) # Reduce each list of messages into a single list. message = reduce(operator.add, message.values()) @@ -39,7 +45,7 @@ # AttributeError: ValidationError instance has no attribute 'args' # See http://www.python.org/doc/current/tut/node10.html#handling if hasattr(self, 'message_dict'): - return '
'.join(['%s - %s' % (key, value) for key,value in self.message_dict.iteritems()]) + return '
'.join(['%s - %s' % (key, value) for key, value in self.message_dict.iteritems()]) return '
'.join(self.messages) def __repr__(self): @@ -64,9 +70,9 @@ import operator from django.utils.encoding import force_str from functools import reduce - + if not message: - message = [] # Don't use funtion's default argument value, because it is static! + message = [] # Don't use funtion's default argument value, because it is static! """ ModelQueryException can be passed any object that can be printed (usually a string), a list of objects or a dictionary. @@ -92,14 +98,14 @@ # AttributeError: ValidationError instance has no attribute 'args' # See http://www.python.org/doc/current/tut/node10.html#handling if hasattr(self, 'message_dict'): - return '
'.join(['%s - %s' % (key, value) for key,value in self.message_dict.iteritems()]) + return '
'.join(['%s - %s' % (key, value) for key, value in self.message_dict.iteritems()]) return '
'.join(self.messages) def __repr__(self): if hasattr(self, 'message_dict'): return 'ModelQueryException(%s)' % repr(self.message_dict) return 'ModelQueryException(%s)' % repr(self.messages) - + def has_error(self): for each_message in self.orig_messages: if hasattr(each_message, 'is_error'): @@ -115,11 +121,11 @@ detail = [each_message.type, str(each_message)] total_detail.append(detail) return total_detail - + def merge(self, another_exception): # merge the orig_messages self.orig_messages.extend(another_exception.orig_messages) - + # merge the message_dict and messages self.messages.extend(another_exception.messages) if hasattr(another_exception, 'message_dict'): @@ -127,24 +133,28 @@ self.message_dict.update(another_exception.message_dict) else: self.message_dict = copy.copy(another_exception.message_dict) - + def is_empty(self): return len(self.messages) == 0 - + class ActionPerformException(Exception): pass - + + class DBException(Exception): pass + class FieldDoesNotExist(Exception): pass + class ManagerImplError(Exception): pass + class RestfulException(Exception): def __init__(self, message, code=None): self.code = code - self.message = message \ No newline at end of file + self.message = message Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/gen_doc.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/gen_doc.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/gen_doc.py (working copy) @@ -36,7 +36,7 @@ target_path = arg elif opt == '-i': src_path = arg - + if mode == 'generate': if lang == 'all' and target_path != '': print("Error: you have to specify the language in order to generate at custom file path") @@ -54,4 +54,3 @@ standarlize_xml(src_path, target_path) except IOError: raise - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/lang.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/lang.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/lang.py (working copy) @@ -1,20 +1,22 @@ # encoding: utf-8 -from django.http import HttpResponse from django.http import Http404 -from hive.utils import get_current_session -from djproject.an_settings import LANGS +from django.http import HttpResponse from django.utils.translation import get_language, activate + +from djproject.an_settings import LANGS from hive.model.query import clear_cache_all +from hive.utils import get_current_session HIVE_LANGUAGES = { - # language code: (language name, language verbose name) - 'en': ('en', u'English'), - 'zh-cn': ('zh_CN', u'简体中文'), + # language code: (language name, language verbose name) + 'en': ('en', u'English'), + 'zh-cn': ('zh_CN', u'简体中文'), 'zh-tw': ('zh_TW', u'繁体中文'), - 'ja': ('ja', u'日本語'), + 'ja': ('ja', u'日本語'), } LANGS_LOCALE_DIR = [HIVE_LANGUAGES[x][0] for x in HIVE_LANGUAGES.keys()] + def switch_lang(request): if 'new_lang' not in request.POST: raise Http404 @@ -27,22 +29,27 @@ clear_cache_all() return response + langs_info = None + + def get_langs_info(): - global langs_info - if not langs_info: - langs_info = [{'code':code, 'name':HIVE_LANGUAGES[code][0], 'local_name':HIVE_LANGUAGES[code][1]} for code in LANGS] - return langs_info + global langs_info + if not langs_info: + langs_info = [{'code': code, 'name': HIVE_LANGUAGES[code][0], 'local_name': HIVE_LANGUAGES[code][1]} for code in + LANGS] + return langs_info + # get the current language code def get_current_lang(): - lang = get_language() - if lang == 'en-us': - return 'en' - else: - return lang + lang = get_language() + if lang == 'en-us': + return 'en' + else: + return lang + # manually set the current language code def set_current_lang(lang): - return activate(lang) - + return activate(lang) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/llb_stats.py (working copy) @@ -1,10 +1,10 @@ -import os +import json from django.http import HttpResponse -import json -from hive.db.llb_stats_queries import LLBStatsDB -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log +from hive.custom_exceptions import generic_exception as ge +from hive.db.llb_stats_queries import LLBStatsDB from hive.utils import andebug @@ -15,6 +15,7 @@ return get_llb_stats(request) elif path == 'stats/historical': return get_llb_historical_data(request) + return None elif path == 'monitoring_data' and request.method == 'POST': return get_monitoring_data(request) else: @@ -130,6 +131,7 @@ # Process each data point units = [] + unit = 0 for value in values: if data_type == 'historical': unit = get_llb_historical_col_values(col_indices, value) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_location.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_location.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_location.py (working copy) @@ -1,11 +1,11 @@ -import subprocess import json from django.http import HttpResponse + from cm.lib.libbasic_operation import oper_log -from .services import log_location_service as log_service from hive.custom_exceptions import generic_exception as ge from hive.util import constants as const +from .services import log_location_service as log_service def handle_log_location_app(request, app): @@ -238,4 +238,4 @@ # Handle unexpected exceptions oper_log('error', 'system', 'An unexpected error occurred: {}'.format(e)) e.message = 'An unexpected error occurred: {}'.format(e) - return ge.handle_exception(e) \ No newline at end of file + return ge.handle_exception(e) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/log_utils.py (working copy) @@ -1,20 +1,21 @@ +import logging import queue -import threading import socket +import threading + from cm.lib.postgres_db import DB -import logging -import queue _log_host_queue_dict = {} _log_host_thread_dict = {} _log_settings = { - 'enable':1, - 'level':1, - 'host':{} + 'enable': 1, + 'level': 1, + 'host': {} } logger = logging.getLogger('hive.debug') + # main func for log host thread def log_host_process_main(config): global _log_host_queue_dict @@ -22,13 +23,13 @@ ip = config[0] port = config[1] protocol = config[2] - key = "%s--%d--%d" %(ip, port, protocol) + key = "%s--%d--%d" % (ip, port, protocol) queue = _log_host_queue_dict[key] if protocol == 0: - #TCP - client = socket.socket(socket.AF_INET,socket.SOCK_STREAM) - client.connect((ip,port)) - + # TCP + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect((ip, port)) + while _log_settings["enable"] and key in _log_host_queue_dict: try: item = queue.get(True, 1) @@ -37,16 +38,17 @@ continue client.close() elif protocol == 1: - #UDP + # UDP client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) while _log_settings["enable"] and key in _log_host_queue_dict: try: item = queue.get(True, 1) - client.sendto(item, (ip,port)) + client.sendto(item, (ip, port)) except queue.Empty: continue client.close() + def log_host_init(key, config): qu = queue.Queue() _log_host_queue_dict[key] = qu @@ -54,11 +56,12 @@ _log_host_thread_dict[key] = th th.start() + # add new host def log_host_add(config): global _log_settings if len(_log_settings['host'].keys()) >= 5: - return {"status":False, "data":"Can not add more host."} + return {"status": False, "data": "Can not add more host."} db = DB.get_connected_db() save_sql = "INSERT INTO log_host(host, port, protocol) values('%s', %d, %d)" % (config[0], config[1], config[2]) try: @@ -67,33 +70,36 @@ db.conn.rollback() db.close() if str(e).find("duplicate key") != -1: - return {"status":False, "data":"Host already exist."} + return {"status": False, "data": "Host already exist."} else: - logger.error('Add log host failed, %s, %s.' %(str(config), str(e))) - return {"status":False, "data":"Internal Error."} + logger.error('Add log host failed, %s, %s.' % (str(config), str(e))) + return {"status": False, "data": "Internal Error."} db.close() - key = "%s--%d--%d" %(config[0], config[1], config[2]) + key = "%s--%d--%d" % (config[0], config[1], config[2]) _log_settings['host'][key] = [config[0], config[1], config[2]] if _log_settings['enable']: log_host_init(key, config) - return {"status":True, "data":{}} + return {"status": True, "data": {}} + # del host def log_host_del(config): global _log_settings global _log_host_queue_dict global _log_host_thread_dict db = DB.get_connected_db() - save_sql = "delete from log_host where host='%s' and port = %d and protocol = %d" % (config[0], config[1], config[2]) + save_sql = "delete from log_host where host='%s' and port = %d and protocol = %d" % (config[0], config[1], + config[2]) db.execute_sql(save_sql) db.close() - key = "%s--%d--%d" %(config[0], config[1], config[2]) + key = "%s--%d--%d" % (config[0], config[1], config[2]) del _log_settings['host'][key] del _log_host_queue_dict[key] del _log_host_thread_dict[key] - return {"status":True, "data":{}} + return {"status": True, "data": {}} + # load and init log host when log on def log_host_load(): global _log_settings @@ -101,6 +107,7 @@ for key, host in _log_settings['host'].items(): log_host_init(key, host) + # load log config when startup def log_settings_load(): global _log_settings @@ -117,17 +124,20 @@ data = db.fetchall(select_sql) db.close() for host in data: - key = "%s--%d--%d" %(host[0], host[1], host[2]) + key = "%s--%d--%d" % (host[0], host[1], host[2]) _log_settings['host'][key] = [host[0], host[1], host[2]] if _log_settings['enable']: log_host_load() + log_settings_load() + def get_log_settings(): global _log_settings return _log_settings + def set_log_settings(data): global _log_settings global _log_host_queue_dict @@ -140,14 +150,14 @@ _log_settings['level'] = int(data["level"]) if _log_settings['enable'] != int(data["enable"]): if _log_settings['enable']: - #disable log function, stop all thread + # disable log function, stop all thread _log_settings['enable'] = int(data["enable"]) _log_host_queue_dict = {} _log_host_thread_dict = {} else: - #enable log functin, start thread + # enable log functin, start thread log_settings_load() - return {"status":True, "data":{}} + return {"status": True, "data": {}} def send_log_to_hosts(msg): @@ -156,18 +166,19 @@ try: queue.put(msg, False) except queue.Full: - logger.error('Send log <%s> to host <%s> failed.' %(msg, key)) + logger.error('Send log <%s> to host <%s> failed.' % (msg, key)) + def log_check(level): global _log_settings log_level_int = { - 'debug':0, - 'info':1, - 'notice':2, - 'warning':3, - 'error':4, - 'critical':5 + 'debug': 0, + 'info': 1, + 'notice': 2, + 'warning': 3, + 'error': 4, + 'critical': 5 } if _log_settings['enable'] and _log_settings['level'] <= log_level_int[level]: return True - return False \ No newline at end of file + return False Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/action.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/action.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/action.py (working copy) @@ -1,15 +1,10 @@ -#!/usr/bin/env python -""" -action.py - -Created by Li Qian on 2013-08-12. -Copyright (c) 2013 Array Networks. All rights reserved. -""" import json import types -from hive.utils import andebug -from hive.document import get_xml_reference, DOCUMENT_OK, DOCUMENT_NEW, DOCUMENT_PENDING +from filecmp import cmp +from hive.document import get_xml_reference, DOCUMENT_OK, DOCUMENT_NEW + + class Action(object): """ This class only get instanciated while performing an action @@ -17,41 +12,41 @@ """ name = '' verbose_name = '' - action_name = '' # displayed in the button + action_name = '' # displayed in the button help_text = '' - confirm_msg = '' # confirm dialog displayed before processing - alert_msg = '' # alert dialog displayed before processing - process_title = '' # title of the progress bar - process_msg = '' # message displayed besides the progress bar - process_type = 'info' # for icon in the progress bar + confirm_msg = '' # confirm dialog displayed before processing + alert_msg = '' # alert dialog displayed before processing + process_title = '' # title of the progress bar + process_msg = '' # message displayed besides the progress bar + process_type = 'info' # for icon in the progress bar finish_msg = '' # message alerted when successfully finished option_fields = () btn_class = 'btn-primary' - full_progress = True # whether to always display full progress bar rather than progressing as time goes - total_time = 0 # total time of progress bar (useful only when full_progress is False) - forever = False # Forever display the progress bar even if the action returns from backend (for reboot/shutdown) - config_change = True # this action will change config or stats of the model or field. False: this action only change the display message of the model or field. - refresh_page = False # if set to true, will try to refresh current page when finish action. This may used when one field action will change another field value in the same page. + full_progress = True # whether to always display full progress bar rather than progressing as time goes + total_time = 0 # total time of progress bar (useful only when full_progress is False) + forever = False # Forever display the progress bar even if the action returns from backend (for reboot/shutdown) + config_change = True # this action will change config or stats of the model or field. False: this action only change the display message of the model or field. + refresh_page = False # if set to true, will try to refresh current page when finish action. This may used when one field action will change another field value in the same page. option_page_url = '' perform_url = '' - condition_rule=None #An instance of :py:class:`Condition` class for determining whether this action is available at current status of the model. - instance_mul='*' # IntRange: '1', '*', '0..1', '1..*' &etc. used for list or asso list. + condition_rule = None # An instance of :py:class:`Condition` class for determining whether this action is available at current status of the model. + instance_mul = '*' # IntRange: '1', '*', '0..1', '1..*' &etc. used for list or asso list. # The follow parameters are used to set the location to display the action. One action can display in many place. - related = '' # set the related field. If this is set, the action will display in the field widget. - related_group = '' # set the related field group. If this is set, the action will display in the header of the group tab in the instance config page. - model_action = None # True/False/None. If True, this action will display in model list page for none profile model and will display in a standalone tab for profile model. - # If None and 'related' 'related_group' not set, will same as set to True. + related = '' # set the related field. If this is set, the action will display in the field widget. + related_group = '' # set the related field group. If this is set, the action will display in the header of the group tab in the instance config page. + model_action = None # True/False/None. If True, this action will display in model list page for none profile model and will display in a standalone tab for profile model. + # If None and 'related' 'related_group' not set, will same as set to True. creation_counter = 0 - load_config_flag = False #True/False. If True, webui will load new config and logout. + load_config_flag = False # True/False. If True, webui will load new config and logout. cmd_pipe = False def __init__(self, model, session): - #self._instance = instance - #self._model = instance._meta._model + # self._instance = instance + # self._model = instance._meta._model self._model = model self._session = session self._manager = self._model.get_manager(session) - + def __call__(self, options=None, **kwargs): if options: result = self.perform(options=options, **kwargs) @@ -59,44 +54,46 @@ result = self.perform(**kwargs) if type(result) is types.DictType: res = result.copy() - res.update({'config_change':self.config_change, 'refresh_page':self.refresh_page,'msg':''}) + res.update({'config_change': self.config_change, 'refresh_page': self.refresh_page, 'msg': ''}) return res elif type(result) is types.StringType: - return {'config_change':self.config_change, 'refresh_page':self.refresh_page,'msg':result} + return {'config_change': self.config_change, 'refresh_page': self.refresh_page, 'msg': result} elif type(result) is types.ListType: - #some output of cli parser is [None] or [[],[]] - #but we don't need it. + # some output of cli parser is [None] or [[],[]] + # but we don't need it. if len(result) > 0: result = '' - return {'config_change':self.config_change, 'refresh_page':self.refresh_page, 'msg':result if result else ''} + return {'config_change': self.config_change, 'refresh_page': self.refresh_page, 'msg': result if result else ''} class __metaclass__(type): def __new__(cls, name, bases, attrs): for action in bases: Dict = action.__dict__ for name in attrs.keys(): - if name in ['verbose_name','action_name','help_text','confirm_msg','alert_msg','process_title','process_msg','finish_msg']: - attrs['_%s'%name] = attrs[name] + if name in ['verbose_name', 'action_name', 'help_text', 'confirm_msg', 'alert_msg', 'process_title', + 'process_msg', 'finish_msg']: + attrs['_%s' % name] = attrs[name] del attrs[name] - for name in Dict.keys() : + for name in Dict.keys(): if name not in attrs.keys(): - if name in ['verbose_name','action_name','help_text','confirm_msg','alert_msg','process_title','process_msg','finish_msg']: - attrs['_%s'%name] = Dict[name] + if name in ['verbose_name', 'action_name', 'help_text', 'confirm_msg', 'alert_msg', 'process_title', + 'process_msg', 'finish_msg']: + attrs['_%s' % name] = Dict[name] elif name == 'creation_counter': attrs[name] = Dict[name] - setattr(Action, name, Dict[name]+1) + setattr(Action, name, Dict[name] + 1) else: attrs[name] = Dict[name] return type.__new__(cls, name, bases, attrs) - def __cmp__(cls, other): + def __cmp__(cls, other): # This is needed because bisect does not take a comparison function. return cmp(cls.creation_counter, other.creation_counter) @property def document(cls): if cls.model._meta.document is not None: - xpath = "./action[@name='"+cls.name+"']" + xpath = "./action[@name='" + cls.name + "']" return cls.model._meta.document.find(xpath) return None @@ -104,7 +101,7 @@ def verbose_name(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/verbose_name" + xpath = "./action[@name='" + cls.name + "']/verbose_name" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._verbose_name) return cls._verbose_name @@ -116,7 +113,7 @@ def action_name(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/action_name" + xpath = "./action[@name='" + cls.name + "']/action_name" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._action_name) return cls._action_name @@ -127,8 +124,8 @@ @property def help_text(cls): if cls.document is not None: - pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/help_text" + pkg = cls.model._meta.path[0] + xpath = "./action[@name='" + cls.name + "']/help_text" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._help_text) return cls._help_text @@ -140,7 +137,7 @@ def confirm_msg(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/confirm_msg" + xpath = "./action[@name='" + cls.name + "']/confirm_msg" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._confirm_msg) return cls._confirm_msg @@ -152,7 +149,7 @@ def alert_msg(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/alert_msg" + xpath = "./action[@name='" + cls.name + "']/alert_msg" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._alert_msg) return cls._alert_msg @@ -164,7 +161,7 @@ def process_title(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/process_title" + xpath = "./action[@name='" + cls.name + "']/process_title" title = get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._process_title) if title == '': return cls.verbose_name @@ -184,7 +181,7 @@ def process_msg(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/process_msg" + xpath = "./action[@name='" + cls.name + "']/process_msg" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._process_msg) return cls._process_msg @@ -196,7 +193,7 @@ def finish_msg(cls): if cls.document is not None: pkg = cls.model._meta.path[0] - xpath = "./action[@name='"+cls.name+"']/finish_msg" + xpath = "./action[@name='" + cls.name + "']/finish_msg" return get_xml_reference(pkg, cls.model._meta.document.find(xpath), default=cls._finish_msg) return cls._finish_msg @@ -207,12 +204,12 @@ @property def document_status(cls): if cls.model._meta.document is not None: - xpath = "./action[@name='"+cls.name+"']" + xpath = "./action[@name='" + cls.name + "']" action_ele = cls.model._meta.document.find(xpath) if action_ele is None: return DOCUMENT_NEW else: - return DOCUMENT_OK + return DOCUMENT_OK else: return DOCUMENT_NEW @@ -221,13 +218,14 @@ if cls.condition_rule and instance: return cls.condition_rule.check(instance) return True - + def perform(self, **kwargs): if 'options' in kwargs and '__asso_pk_list' in kwargs['options']: - kwargs['options']['__asso_pk_list'] = [json.loads(pk_str) if pk_str else '' for pk_str in kwargs['options']['__asso_pk_list']] - if hasattr(self._manager, '_perform_'+self.name): - return getattr(self._manager, '_perform_'+self.name)(**kwargs) - + kwargs['options']['__asso_pk_list'] = [json.loads(pk_str) if pk_str else '' for pk_str in + kwargs['options']['__asso_pk_list']] + if hasattr(self._manager, '_perform_' + self.name): + return getattr(self._manager, '_perform_' + self.name)(**kwargs) + @classmethod def contribute_to_class(cls, model, name): cls.name = name @@ -254,8 +252,8 @@ 'alert_msg': cls.alert_msg, 'title': str(cls.process_title), 'full_progress': cls.full_progress, - 'total_time':cls.total_time, - 'process_type': cls.process_type, + 'total_time': cls.total_time, + 'process_type': cls.process_type, 'process_msg': cls.process_msg, 'forever': cls.forever, 'finish_msg': cls.finish_msg, Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py (working copy) @@ -1,36 +1,29 @@ -#!/usr/bin/env python -# encoding: utf-8 -""" -ajax.py - -Created by Li Qian on 2013-07-24. -Copyright (c) 2013 Array Networks. All rights reserved. -""" - -import sys -import os -from hive.model.loading import get_app, get_model -from django.http import Http404 -import json -from hive.exceptions import ValidationError, ModelQueryException, ActionPerformException, FieldDoesNotExist, ManagerImplError import copy +import json +import os import time -from jinja2 import Environment, PackageLoader, ChoiceLoader -from hive.utils import HiveEnvironment, andebug, url2obj, IntRange -from hive.model.manager import ListManager -from hive.model.utils import reduce_asso_value_item, get_complex_field, get_config_list_data, save_config_list_data -from jinja2.exceptions import TemplateNotFound -from django.template import RequestContext import urllib -from hive.notification import send_notification + +from django.http import Http404 +from django.template import RequestContext from django.utils.encoding import smart_str -from djproject.an_settings import TEST_IP_POOL, DIR_STR, MAX_FILES_COUNT, RRDCACHED_SOCKET -#import rrdtool from django.utils.translation import gettext_lazy as _ +from jinja2 import PackageLoader, ChoiceLoader +from jinja2.exceptions import TemplateNotFound +from djproject.an_settings import DIR_STR +from hive.exceptions import ValidationError, ModelQueryException, ActionPerformException, ManagerImplError +from hive.model.loading import get_app, get_model +from hive.model.manager import ListManager +from hive.model.utils import get_complex_field, get_config_list_data +from hive.notification import send_notification +from hive.utils import HiveEnvironment, andebug, url2obj, IntRange + + def main(): pass + class ModelAjaxHandler(object): def __init__(self, request, session, app, model, action): self.request = request @@ -41,7 +34,7 @@ self.model = model self.manager = self.model.get_manager(session) - + def render(self): if hasattr(self, self.action): request_dict = dict(self.request.REQUEST) @@ -53,22 +46,22 @@ return self.box(self.action[4:], **request_dict) else: raise Http404 - + def default(self): raise Http404('default action is not yet implemented') def box(self, sub_action, **kwargs): django_ctx = RequestContext(self.request) env = HiveEnvironment(loader=ChoiceLoader([ - PackageLoader(self.app._meta.app_label, 'templates'), - PackageLoader('hive', 'templates')])) + PackageLoader(self.app._meta.app_label, 'templates'), + PackageLoader('hive', 'templates')])) try: template = env.get_template('/'.join(self.model._meta.path) + '/box_' + sub_action + '.html') except TemplateNotFound: template = env.get_template('box_' + sub_action + '.html') - ctx = {'REQUEST':self.request, 'SESSION':self.session, 'MEDIA':'/media/', - 'MODEL':self.model, 'MANAGER':self.manager, 'csrf_token':django_ctx['csrf_token'], 'PARAMS':kwargs} - + ctx = {'REQUEST': self.request, 'SESSION': self.session, 'MEDIA': '/media/', + 'MODEL': self.model, 'MANAGER': self.manager, 'csrf_token': django_ctx['csrf_token'], 'PARAMS': kwargs} + if self.model._meta.profile: try: ctx['OBJECT'] = self.manager.all()[0] @@ -82,28 +75,29 @@ except KeyError: PARAMSTR = self.request.GET.urlencode() ctx['PARAMSTR'] = PARAMSTR - + # get the dict for determining concrete class - get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key + get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key concrete_dict = get_dict.get('concrete', None) if concrete_dict: concrete_dict = url2obj(concrete_dict) andebug('hive.debug', 'Got concrete dict %s' % str(concrete_dict)) - + # If this is a virtual super class, add a context variable if self.model._meta.abstract and concrete_dict: try: concrete_model = self.model._meta.get_child_by_field_cond(concrete_dict) except KeyError: - raise Http404('not such child model based on param dict %s - %s' % (str(concrete_dict), str(self.request.GET))) + raise Http404( + 'not such child model based on param dict %s - %s' % (str(concrete_dict), str(self.request.GET))) concrete_manager = self.session.get_model_manager(self.app._meta.app, concrete_model._meta.path) ctx['CONCRETE_MODEL'] = concrete_model ctx['CONCRETE_MANAGER'] = concrete_manager - + return template.render(ctx) - + def perform(self, **kwargs): - args = dict(kwargs) # make a copy + args = dict(kwargs) # make a copy try: action_name = args['action'] del args['action'] @@ -125,13 +119,13 @@ elif e.get_detail(): return json.dumps([True, e.get_detail()]) return json.dumps([False, str(e)]) - + return json.dumps([True, msg]) - + def list(self, **kwargs): # make a copy because we might change the cached queryset # qs = copy.copy(self.manager.all()) - + # Check and process the filter # format of the filter should be {:, ...} def determine(instance, filters): @@ -139,6 +133,7 @@ if getattr(instance, field_name) != value: return False return True + """ if 'filter' in kwargs: filters = json.loads(urllib.unquote(kwargs['filter'])) @@ -149,11 +144,10 @@ qs = self.manager.filter(filters) else: qs = copy.copy(self.manager.all()) - - + rtn = [] for each in qs: - each_rtn = {'pk':each.pk_dict(), 'repr':each.repr()} + each_rtn = {'pk': each.pk_dict(), 'repr': each.repr()} if 'with_fields' in kwargs: each_rtn['fields'] = each.get_field_dict() rtn.append(each_rtn) @@ -162,7 +156,7 @@ def recent_objs(self, **kwargs): objs = self.session.pref.get_recent_objs(self.model._meta.path) return json.dumps(objs) - + def add(self, **kwargs): try: json_str = kwargs['post_data'] @@ -183,8 +177,8 @@ return json.dumps([True, new_obj.url_params(), e.get_detail()]) elif str(e): return json.dumps([True, new_obj.url_params(), [['warning', str(e)]]]) - - send_notification('info', 'A new %s is added.' % self.model._meta.verbose_name) + + send_notification('info', 'A new %s is added.' % self.model._meta.verbose_name) return json.dumps([True, new_obj.url_params(), []]) def delete(self, **kwargs): @@ -194,12 +188,13 @@ if len(pk_list) == 1: # first try to get the instance, so we can get the real instance model. instance = self.manager.get(pk_list[0]) - send_notification('info', '%s (%s) is deleted.' % (instance.repr_by_pk(pk_list[0]), instance._meta.verbose_name)) + send_notification('info', '%s (%s) is deleted.' % (instance.repr_by_pk(pk_list[0]), + instance._meta.verbose_name)) else: send_notification('info', '%d %s are deleted.' % (len(pk_list), self.model._meta.verbose_name)) return self.manager.delete(pk_list) except KeyError as e: - return json.dumps([True, [['warning','Instance not found in system: ' + e.message]]]) + return json.dumps([True, [['warning', 'Instance not found in system: ' + e.message]]]) except ModelQueryException as e: if e.has_error(): return json.dumps([False, str(e)]) @@ -207,7 +202,7 @@ return json.dumps([True, e.get_detail()]) else: return json.dumps([True, []]) - + def delete2(self, **kwargs): try: pk_list = json.loads(urllib.unquote(kwargs['pk'])) @@ -215,32 +210,33 @@ self.manager.delete(pk_list) return json.dumps([True, []]) except KeyError as e: - return json.dumps([True, [['warning','Instance not found in system: ' + e.message]]]) + return json.dumps([True, [['warning', 'Instance not found in system: ' + e.message]]]) except ModelQueryException as e: if e.has_error(): return json.dumps([False, str(e)]) else: if len(pk_list) == 1: - send_notification('info', '%s (%s) is deleted.' % (self.model.repr_by_pk(pk_list[0]), self.model._meta.verbose_name)) + send_notification('info', '%s (%s) is deleted.' % (self.model.repr_by_pk(pk_list[0]), + self.model._meta.verbose_name)) else: send_notification('info', '%d %s are deleted.' % (len(pk_list), self.model._meta.verbose_name)) if e.get_detail(): return json.dumps([True, e.get_detail()]) else: return json.dumps([True, []]) - + def statistics(self, **kwargs): ret = [] try: qs = self.manager.all() - ret = json.dumps([{'pk':each.pk_dict(), 'repr':each.repr(), 'fields':each.get_statistics()} for each in qs]) + ret = json.dumps( + [{'pk': each.pk_dict(), 'repr': each.repr(), 'fields': each.get_statistics()} for each in qs]) except KeyError: raise Http404 return ret - def stats_table(self, **kwargs): - response_data = {} # response obj + response_data = {} # response obj # --- Get input agrs --- data_start = 0 @@ -258,17 +254,17 @@ response_data["draw"] = kwargs["draw"] if "order[0][column]" in kwargs: sortByIdx = str(kwargs["order[0][column]"]) - sort_dict['field'] = kwargs["columns["+ sortByIdx +"][name]"] + sort_dict['field'] = kwargs["columns[" + sortByIdx + "][name]"] sort_dict['order'] = kwargs["order[0][dir]"] - field_list = [] # fields list in base class + field_list = [] # fields list in base class i = 0 - while "columns["+str(i)+"][name]" in kwargs: - if kwargs["columns["+str(i)+"][name]"] == 'external_data_column': + while "columns[" + str(i) + "][name]" in kwargs: + if kwargs["columns[" + str(i) + "][name]"] == 'external_data_column': get_external_data = True i += 1 continue - field_list.append(kwargs["columns["+str(i)+"][name]"]) + field_list.append(kwargs["columns[" + str(i) + "][name]"]) i += 1 if "initial_filter" in kwargs: @@ -277,22 +273,25 @@ search_dict["search_fields"] = field_list if "search[value]" in kwargs: search_dict["search_str"] = kwargs["search[value]"] - + # --- Start get data --- try: - ret_data = self.manager.get_stats_list(start=data_start, max_num=data_length, search=search_dict, filter_list=filter_dict, sort_dict=sort_dict) + ret_data = self.manager.get_stats_list(start=data_start, max_num=data_length, search=search_dict, + filter_list=filter_dict, sort_dict=sort_dict) response_data["data"] = ret_data['data'] response_data["recordsTotal"] = ret_data['total'] response_data["recordsFiltered"] = ret_data['total'] - except ManagerImplError: - ret = [] # data in response obj - andebug('hive.debug', 'filter_search_sort: %s, %s, %s' % (str(filter_dict), str(search_dict), str(sort_dict))) - ret_data = self.manager.filter_search_sort(start=data_start, length=data_length, filter_dict=filter_dict, search_dict=search_dict, sort_dict=sort_dict) + except ManagerImplError: + ret = [] # data in response obj + andebug('hive.debug', + 'filter_search_sort: %s, %s, %s' % (str(filter_dict), str(search_dict), str(sort_dict))) + ret_data = self.manager.filter_search_sort(start=data_start, length=data_length, filter_dict=filter_dict, + search_dict=search_dict, sort_dict=sort_dict) instance_list = ret_data["data"] response_data["recordsTotal"] = ret_data["total"] response_data["recordsFiltered"] = ret_data["total"] for each in instance_list: - sub_ret = {} # save all data for one instance + sub_ret = {} # save all data for one instance ins_manager = self.session.get_model_manager(self.app._meta.app, each._meta.path) # add normal data # first get all fields value in base class fields list @@ -303,49 +302,49 @@ value = ', '.join(value_list) sub_ret[fn] = value # second get left fields value for each children instance - if get_external_data: + if get_external_data: tmp_ret = ins_manager.get_external_data_for_list(each, field_list, 'stats') sub_ret['external'] = tmp_ret ret.append(sub_ret) - + response_data["data"] = ret - return json.dumps(response_data) + return json.dumps(response_data) def config_table_columns(self, **kwargs): - columns = [] # the format of columns should support DataTables aoColumns require + columns = [] # the format of columns should support DataTables aoColumns require parent_model = self.model - + # get the PARAMSTR to pass when switching between different concrete classes try: copied_get_dict = self.request.GET.copy() copied_get_dict.pop('concrete') - get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key + get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key concrete_dict = get_dict.get('concrete', None) if concrete_dict: concrete_dict = url2obj(concrete_dict) - + # If this is a virtual super class, add a context variable if self.model._meta.abstract and concrete_dict: parent_model = self.model._meta.get_child_by_field_cond(concrete_dict) except KeyError: pass - + columns = parent_model._meta.get_list_columns() return json.dumps(columns) def config_table(self, **kwargs): - response_data = {} # response obj - + response_data = {} # response obj + # get the PARAMSTR to pass when switching between different concrete classes try: copied_get_dict = self.request.GET.copy() copied_get_dict.pop('concrete') - get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key + get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key concrete_dict = get_dict.get('concrete', None) if concrete_dict: concrete_dict = url2obj(concrete_dict) - + # If this is a virtual super class, add a context variable if self.model._meta.abstract and concrete_dict: parent_model = self.model._meta.get_child_by_field_cond(concrete_dict) @@ -370,35 +369,35 @@ response_data["draw"] = kwargs["draw"] if "order[0][column]" in kwargs: sortByIdx = str(kwargs["order[0][column]"]) - sort_dict['field'] = kwargs["columns["+ sortByIdx +"][name]"] + sort_dict['field'] = kwargs["columns[" + sortByIdx + "][name]"] sort_dict['order'] = kwargs["order[0][dir]"] - field_list = [] # fields list in base class + field_list = [] # fields list in base class i = 0 - while "columns["+str(i)+"][name]" in kwargs: - if kwargs["columns["+str(i)+"][name]"] == 'external_data_column': + while "columns[" + str(i) + "][name]" in kwargs: + if kwargs["columns[" + str(i) + "][name]"] == 'external_data_column': get_external_data = True i += 1 continue - if kwargs["columns["+str(i)+"][name]"] == 'check_box_column': + if kwargs["columns[" + str(i) + "][name]"] == 'check_box_column': i += 1 continue - field_list.append(kwargs["columns["+str(i)+"][name]"]) - if "columns["+str(i)+"][search][value]" in kwargs and kwargs["columns["+str(i)+"][search][value]"]: - filter_dict[kwargs["columns["+str(i)+"][name]"]] = kwargs["columns["+str(i)+"][search][value]"] + field_list.append(kwargs["columns[" + str(i) + "][name]"]) + if "columns[" + str(i) + "][search][value]" in kwargs and kwargs["columns[" + str(i) + "][search][value]"]: + filter_dict[kwargs["columns[" + str(i) + "][name]"]] = kwargs["columns[" + str(i) + "][search][value]"] i += 1 has_init_filter = False if "initial_filter" in kwargs: initial_filter = url2obj(kwargs["initial_filter"]) has_init_filter = True - + for key, value in initial_filter.iteritems(): filter_dict[key] = value search_dict["search_fields"] = field_list if "search[value]" in kwargs: search_dict["search_str"] = kwargs["search[value]"] - + # ---- A special case for concrete_dict ---- # When change the concrete_dict, the firset request is used to recreate the table, a filter request will be send # immediately after the first request, so for the first, we didn't need to return the really data. @@ -406,36 +405,41 @@ response_data["data"] = [] response_data["recordsTotal"] = 0 response_data["recordsFiltered"] = 0 - return json.dumps(response_data) - - # --- Start get data --- + return json.dumps(response_data) + + # --- Start get data --- try: - ret_data = self.manager.get_config_list(start=data_start, max_num=data_length, search=search_dict, filter_list=filter_dict, sort_dict=sort_dict) + ret_data = self.manager.get_config_list(start=data_start, max_num=data_length, search=search_dict, + filter_list=filter_dict, sort_dict=sort_dict) response_data["data"] = ret_data['data'] response_data["recordsTotal"] = ret_data['total'] response_data["recordsFiltered"] = ret_data['total'] - except ManagerImplError: - #--------First try to get data from level two cache -------# + except ManagerImplError: + # --------First try to get data from level two cache -------# try: # level two cache still has some problem, so skip it for now, we will fix the level two cache later raise KeyError if has_init_filter: raise KeyError - ret_data = get_config_list_data(model=self.manager._model, start=data_start, max_num=data_length, search_dict=search_dict, filter_dict=filter_dict, sort_dict=sort_dict) + ret_data = get_config_list_data(model=self.manager._model, start=data_start, max_num=data_length, + search_dict=search_dict, filter_dict=filter_dict, sort_dict=sort_dict) response_data["data"] = ret_data['data'] response_data["recordsTotal"] = ret_data['total'] response_data["recordsFiltered"] = ret_data['total'] except KeyError: - #--------Get data from level two cache failed, try to get data in normal way ------# - ret = [] # data in response obj - andebug('hive.debug', 'filter_search_sort: %s, %s, %s' % (str(filter_dict), str(search_dict), str(sort_dict))) - ret_data = self.manager.filter_search_sort(start=data_start, length=data_length, filter_dict=filter_dict, search_dict=search_dict, sort_dict=sort_dict) + # --------Get data from level two cache failed, try to get data in normal way ------# + ret = [] # data in response obj + andebug('hive.debug', + 'filter_search_sort: %s, %s, %s' % (str(filter_dict), str(search_dict), str(sort_dict))) + ret_data = self.manager.filter_search_sort(start=data_start, length=data_length, + filter_dict=filter_dict, search_dict=search_dict, + sort_dict=sort_dict) instance_list = ret_data["data"] response_data["recordsTotal"] = ret_data["total"] response_data["recordsFiltered"] = ret_data["total"] for each in instance_list: - sub_ret = {} # save all data for one instance + sub_ret = {} # save all data for one instance ins_manager = self.session.get_model_manager(self.app._meta.app, each._meta.path) # add normal data # first get all fields value in base class fields list @@ -446,27 +450,27 @@ value = ', '.join(value_list) sub_ret[fn] = value # second get left fields value for each children instance - if get_external_data: + if get_external_data: tmp_ret = ins_manager.get_external_data_for_list(each, field_list) sub_ret['external'] = tmp_ret - #add hide data in response, hide data include "pk_str", "action", "functions" + # add hide data in response, hide data include "pk_str", "action", "functions" hide_data = ins_manager.get_internal_data_for_list(each) sub_ret['internal'] = hide_data - ret.append(sub_ret) + ret.append(sub_ret) response_data["data"] = ret # level two cache still has some problem, so skip it for now, we will fix the level two cache later - #if not filter_dict: + # if not filter_dict: # save_config_list_data(manager=self.manager, field_list=field_list, external_data=get_external_data) - return json.dumps(response_data) + return json.dumps(response_data) def popup_add(self, **kwargs): try: parent_field_name = kwargs['field'] except KeyError: raise Http404('no field name provided') - + django_ctx = RequestContext(self.request) if self.model._meta.profile: raise Http404 @@ -476,14 +480,15 @@ template = env.get_template('/'.join(self.model._meta.path) + '/popup_add.html') except TemplateNotFound: template = env.get_template('popup_add.html') - ctx = {'MODEL':self.model, 'MANAGER':self.manager, 'csrf_token':django_ctx['csrf_token'], 'parent_field_name':parent_field_name} + ctx = {'MODEL': self.model, 'MANAGER': self.manager, 'csrf_token': django_ctx['csrf_token'], + 'parent_field_name': parent_field_name} # This should not be a virtual super class if self.model._meta.abstract and len(self.request.GET) > 0: raise Http404 return template.render(ctx) - + def tree(self, **kwargs): group = '' search = '' @@ -493,16 +498,16 @@ if "OrderBy" in kwargs: order = kwargs["OrderBy"] if "Search" in kwargs: - search = kwargs["Search"] - + search = kwargs["Search"] + ret = self.manager.get_tree_for_module(group=group, order=order, search=search) - + return json.dumps(ret) - + def histogram(self, **kwargs): - ret = {"data":[], "ticks":[], "series":[]} + ret = {"data": [], "ticks": [], "series": []} field = [] - sendbc='' + sendbc = '' max_num = 0 order = False if "field" in kwargs: @@ -515,7 +520,7 @@ sendbc = kwargs["sendbc"] if "max" in kwargs: max_num = int(kwargs["max"]) - + tmp_ret = self.manager.get_fields_for_module(field=field, order=order) length = len(field) for i in range(length): @@ -527,23 +532,23 @@ count += 1 if count > max_num: break - + ret['ticks'].append(i[0]) j = 0 while j < length: ret["data"][j].append(i[1][j]) j = j + 1 - + ret["sendbc"] = sendbc - return json.dumps(ret) - + return json.dumps(ret) + def stat_fields(self, **kwargs): rtn = [] for group_name, field_group in self.model._meta.field_groups.iteritems(): if field_group.isstats: for field in field_group.fields: if field.type_name == 'StorableField': - rtn.append({'name':field.name, 'verbose_name': str(field.verbose_name)}) + rtn.append({'name': field.name, 'verbose_name': str(field.verbose_name)}) return json.dumps(rtn) def action_check(self, **kwargs): @@ -562,7 +567,7 @@ ret = num_check.validate(instance_len) if not ret: rtn[action.name] = False - continue + continue for each_ins in instance_list: ret = action.condition(each_ins) if not ret: @@ -572,18 +577,19 @@ def model_quantity_check(self, **kwargs): # get the dict for determining concrete class - get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key + get_dict = self.request.GET.dict() # don't use dict(...) because Django QueryDict have a list for one key concrete_dict = get_dict.get('concrete', None) if concrete_dict: concrete_dict = url2obj(concrete_dict) andebug('hive.debug', 'Got concrete dict %s' % str(concrete_dict)) - + # If this is a virtual super class, add a context variable if self.model._meta.abstract and concrete_dict: try: concrete_model = self.model._meta.get_child_by_field_cond(concrete_dict) except KeyError: - raise Http404('not such child model based on param dict %s - %s' % (str(concrete_dict), str(self.request.GET))) + raise Http404( + 'not such child model based on param dict %s - %s' % (str(concrete_dict), str(self.request.GET))) concrete_manager = self.session.get_model_manager(self.app._meta.app, concrete_model._meta.path) instance_count = concrete_manager.all().count() if instance_count < concrete_model._meta.quantity.get_max(): @@ -601,8 +607,8 @@ # {'field':[xxx, ...]} keys = json.loads(kwargs['keys']) ret = { - 'result':False, - 'error_msg':[] + 'result': False, + 'error_msg': [] } webui_graph_mod = get_model('apv', ['system', 'AccessControl']) webui_graph_manager = webui_graph_mod.get_manager(self.session) @@ -628,14 +634,15 @@ mod = self.model file_name = DIR_STR + instance._meta._model.__name__ + instance.pk_str() + field + ".rrd" if not os.path.isfile(file_name): - ret['error_msg'].append(_('%s - %s: Fails to save statistics data.') % (instance.repr(), str(field_obj.verbose_name))) + ret['error_msg'].append( + _('%s - %s: Fails to save statistics data.') % (instance.repr(), str(field_obj.verbose_name))) continue ret['result'] = True return json.dumps(ret) - #get model list data - #if specify keys, it will get field values based on keys - #otherwise, it will get all field values + # get model list data + # if specify keys, it will get field values based on keys + # otherwise, it will get all field values def get_list_data(self, **kwargs): if isinstance(self.manager, ListManager): rtn = self.manager.get_config_list()['data'] @@ -665,15 +672,15 @@ else: each_rtn = each.get_field_dict() each_rtn['pk'] = each.pk_str() - each_rtn['pk_id']= each.pk_id() + each_rtn['pk_id'] = each.pk_id() rtn.append(each_rtn) return json.dumps(rtn) - #get asso model list data - #This function can also be used to get data from a ANModel by filter_dict + # get asso model list data + # This function can also be used to get data from a ANModel by filter_dict def get_asso_list_data(self, **kwargs): rtn = [] - filter_dict={} + filter_dict = {} rest = True if 'rest' in kwargs and not kwargs['rest']: rest = False @@ -681,7 +688,7 @@ initial_filter = kwargs['initial_filter'] except KeyError: raise Http404('no initial_filter provided') - + initial_filter = url2obj(initial_filter) for key, value in initial_filter.iteritems(): filter_dict[key] = value @@ -707,18 +714,18 @@ else: each_rtn = each.get_field_dict() each_rtn['pk'] = each.pk_str() - each_rtn['pk_id']= each.pk_id() + each_rtn['pk_id'] = each.pk_id() rtn.append(each_rtn) return json.dumps(rtn) - #get filter model + # get filter model def get_filter_list_data(self, **kwargs): - filter_dict={} + filter_dict = {} try: initial_filter = kwargs['initial_filter'] except KeyError: raise Http404('no initial_filter provided') - + qs_list = [] flag_list = False initial_filter = url2obj(initial_filter) @@ -756,7 +763,7 @@ def get_statistics_data(self, **kwargs): # {'instance':[pk_dict1, pk_dict2, ...], 'field':[xxx, ...]} keys = json.loads(kwargs['keys']) - TIMEFORMAT='%Y-%m-%d %X' + TIMEFORMAT = '%Y-%m-%d %X' ret = {} if 'key' in kwargs: ret['key'] = kwargs['key'] @@ -772,7 +779,7 @@ ret['end_time'] = time.strftime(TIMEFORMAT, time.localtime(end_time)) ret['data'] = [] ret['data_name'] = [] - + file_err = False for pk_dict_str in keys['instance']: pk_dict = eval(pk_dict_str) @@ -784,18 +791,19 @@ try: file_name = DIR_STR + instance._meta._model.__name__ + instance.pk_str() + field + ".rrd" field_obj = instance._meta.get_field(field) - #result = rrdtool.fetch(str(file_name), 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) + # result = rrdtool.fetch(str(file_name), 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) result = [] start = result[0][0] end = result[0][1] freq = result[0][2] - num = (end-start)/freq + num = (end - start) / freq data = [] for i in range(num): value = result[2][i][0] if value == None: continue - data.append([time.strftime(TIMEFORMAT, time.localtime(start+i*freq)), field_obj.to_python(value)]) + data.append( + [time.strftime(TIMEFORMAT, time.localtime(start + i * freq)), field_obj.to_python(value)]) ret['data'].append(data) field_obj = instance._meta.get_field(field) ret['data_name'].append(instance.repr() + '-' + str(field_obj.verbose_name)) @@ -805,6 +813,7 @@ ret['error'] = str(_('Statistics information is not saved.')) return json.dumps(ret) + class ObjectAjaxHandler(object): def __init__(self, request, session, app, model, action, pk): self.request = request @@ -815,19 +824,19 @@ self.model = model self.manager = self.model.get_manager(session) - + # if pk is not None if pk: # decode pk's each value from url encoding for key, value in pk.iteritems(): pk[key] = url2obj(value) - + # pk is None when it's a profile Model's object try: self.object = self.manager.get(pk) except KeyError: raise - #raise Http404('No such object') + # raise Http404('No such object') self.pk_id = self.object.pk_id() def render(self): @@ -848,21 +857,21 @@ def box(self, sub_action, **kwargs): django_ctx = RequestContext(self.request) env = HiveEnvironment(loader=ChoiceLoader([ - PackageLoader(self.app._meta.app_label, 'templates'), - PackageLoader('hive', 'templates')])) + PackageLoader(self.app._meta.app_label, 'templates'), + PackageLoader('hive', 'templates')])) try: template = env.get_template('/'.join(self.model._meta.path) + '/box_' + sub_action + '.html') except TemplateNotFound: template = env.get_template('/box_' + sub_action + '.html') - ctx = {'REQUEST':self.request, 'SESSION':self.session, 'MEDIA':'/media/', - 'MODEL':self.model, 'MANAGER':self.manager, 'OBJECT':self.object, - 'csrf_token':django_ctx['csrf_token'], 'PARAMS':kwargs} - + ctx = {'REQUEST': self.request, 'SESSION': self.session, 'MEDIA': '/media/', + 'MODEL': self.model, 'MANAGER': self.manager, 'OBJECT': self.object, + 'csrf_token': django_ctx['csrf_token'], 'PARAMS': kwargs} + return template.render(ctx) def statistics(self, **kwargs): try: - TIMEFORMAT='%Y-%m-%d %X' + TIMEFORMAT = '%Y-%m-%d %X' start = time.time() + int(kwargs['start_time']) end = time.time() + int(kwargs['end_time']) @@ -872,9 +881,10 @@ return json.dumps(ret) except KeyError: raise Http404 - # only for profile model + + # only for profile model def perform(self, **kwargs): - args = dict(kwargs) # make a copy + args = dict(kwargs) # make a copy try: action_name = args['action'] del args['action'] @@ -896,7 +906,7 @@ elif e.get_detail(): return json.dumps([True, e.get_detail()]) return json.dumps([False, str(e)]) - + return json.dumps([True, msg]) def field(self, **kwargs): @@ -906,7 +916,7 @@ raise Http404('no field name provided') fetch_type = kwargs.get('type', 'value') - + try: field_value = self.manager.get_field(self.object.pk_dict(), field_name) except AttributeError: @@ -918,11 +928,11 @@ # XXX default type is 'value' return json.dumps(field_value) - #get profile model fields' data - #if specify keys, it will get field values based on keys - #otherwise, it will get all fields' values + # get profile model fields' data + # if specify keys, it will get field values based on keys + # otherwise, it will get all fields' values def fields(self, **kwargs): - #qs = self.manager.all()[0] + # qs = self.manager.all()[0] data = {} rest = True if 'rest' in kwargs and not kwargs['rest']: @@ -933,7 +943,7 @@ if rest: data[field_name] = self.object.get_rest_field(field_name) else: - data[field_name]= getattr(self.object, field_name, None) + data[field_name] = getattr(self.object, field_name, None) except Exception as e: if rest: data = self.object.get_rest_field_dict() @@ -960,43 +970,44 @@ except AttributeError: raise Http404('no such field_group %s' % field_group) - return json.dumps(result) - + return json.dumps(result) + def export(self, **kwargs): try: field_name = kwargs['field'] except KeyError: raise Http404('no field name provided') - + try: field_value = self.manager.get_field(self.object.pk_dict(), field_name) except AttributeError: raise Http404('no such field %s' % field_name) - + return self.model._meta.get_field(field_name).value_to_display(val=field_value) - + def popup_chart(self, **kwargs): try: field_name = kwargs['field'] except KeyError: raise Http404('no field name provided') - + field = self.model._meta.get_field(field_name) if not field or not field.group.isstats: raise Http404('field not found or not a stats field') - + style = kwargs.get('style', 'chart_small') env = HiveEnvironment(loader=PackageLoader('hive', 'templates')) - + template = env.get_template('popup_chart.html') - return template.render({'OBJECT':self.object, 'MODEL':self.model, 'MANAGER':self.manager, 'STYLE':style, 'FIELD':field}) - + return template.render( + {'OBJECT': self.object, 'MODEL': self.model, 'MANAGER': self.manager, 'STYLE': style, 'FIELD': field}) + def update(self, **kwargs): try: json_str = kwargs['post_data'] json_dict = json.loads(json_str) old_value_dict = dict(self.object.get_field_dict(withstats=False)) - for key,value in json_dict.iteritems(): + for key, value in json_dict.iteritems(): setattr(self.object, key, value) except KeyError: raise Http404 @@ -1004,7 +1015,7 @@ self.object.clean_fields() except ValidationError as e: # roll back - for key,value in old_value_dict.iteritems(): + for key, value in old_value_dict.iteritems(): setattr(self.object, key, value) return json.dumps([False, str(e)]) try: @@ -1017,7 +1028,7 @@ return json.dumps([True, self.object.url_params(), e.get_detail()]) else: return json.dumps([True, self.object.url_params(), []]) - + if self.model._meta.profile: send_notification('info', '%s is updated.' % (self.model._meta.verbose_name)) else: @@ -1034,16 +1045,17 @@ template = env.get_template('/'.join(self.model._meta.path) + '/monitor_sub_page.html') except TemplateNotFound: template = env.get_template('monitor_sub_page.html') - ctx = {'SESSION':self.session, 'MODEL':self.model, 'MANAGER':self.manager, 'OBJECT':self.object, 'csrf_token':django_ctx['csrf_token']} - + ctx = {'SESSION': self.session, 'MODEL': self.model, 'MANAGER': self.manager, 'OBJECT': self.object, + 'csrf_token': django_ctx['csrf_token']} + return template.render(ctx) - + pos = 2 if "pos" in kwargs: pos = int(kwargs["pos"]) - + ret = self.manager.get_tree_for_instance(pos=pos, instance=self.object) - + return json.dumps(ret) def stat_fields(self, **kwargs): @@ -1052,7 +1064,7 @@ if field_group.isstats: for field in field_group.fields: if field.type_name == 'StorableField': - rtn.append({'name':field.name, 'verbose_name': str(field.verbose_name)}) + rtn.append({'name': field.name, 'verbose_name': str(field.verbose_name)}) return json.dumps(rtn) def asso_action_check(self, **kwargs): @@ -1072,20 +1084,21 @@ ret = num_check.validate(instance_len) if not ret: rtn[action.name] = False - continue - + continue + ret = action.condition(self.object) if not ret: rtn[action.name] = False break rtn[action.name] = True return json.dumps(rtn) + def check_statistics_data(self, **kwargs): # {'field':[xxx, ...]} keys = json.loads(kwargs['keys']) ret = { - 'result':False, - 'error_msg':[] + 'result': False, + 'error_msg': [] } webui_graph_mod = get_model('apv', ['system', 'AccessControl']) webui_graph_manager = webui_graph_mod.get_manager(self.session) @@ -1105,14 +1118,16 @@ mod = self.model file_name = DIR_STR + instance._meta._model.__name__ + instance.pk_str() + field + ".rrd" if not os.path.isfile(file_name): - ret['error_msg'].append(_('%s - %s: Fails to save statistics data.') % (instance.repr(), str(field_obj.verbose_name))) - continue + ret['error_msg'].append( + _('%s - %s: Fails to save statistics data.') % (instance.repr(), str(field_obj.verbose_name))) + continue ret['result'] = True return json.dumps(ret) + def get_statistics_data(self, **kwargs): # {'field':[xxx, ...]} keys = json.loads(kwargs['keys']) - TIMEFORMAT='%Y-%m-%d %X' + TIMEFORMAT = '%Y-%m-%d %X' ret = {} if 'key' in kwargs: ret['key'] = kwargs['key'] @@ -1128,7 +1143,7 @@ ret['end_time'] = time.strftime(TIMEFORMAT, time.localtime(end_time)) ret['data'] = [] ret['data_name'] = [] - + file_err = False instance = self.object for field in keys['field']: @@ -1136,26 +1151,26 @@ file_name = DIR_STR + instance._meta._model.__name__ + instance.pk_str() + field + ".rrd" field_obj = instance._meta.get_field(field) ret['data_name'].append(instance.repr() + '-' + str(field_obj.verbose_name)) - #result = rrdtool.fetch(str(file_name), 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) + # result = rrdtool.fetch(str(file_name), 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) result = [] start = result[0][0] end = result[0][1] freq = result[0][2] - num = (end-start)/freq + num = (end - start) / freq data = [] for i in range(num): value = result[2][i][0] if value == None: continue - data.append([time.strftime(TIMEFORMAT, time.localtime(start+i*freq)), field_obj.value_to_num(val=field_obj.to_python(value))]) + data.append([time.strftime(TIMEFORMAT, time.localtime(start + i * freq)), + field_obj.value_to_num(val=field_obj.to_python(value))]) ret['data'].append(data) except: file_err = True if file_err and not ret['data']: ret['error'] = str(_('Fails to save statistics data.')) return json.dumps(ret) - + if __name__ == '__main__': main() - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/association.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/association.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/association.py (working copy) @@ -1,17 +1,19 @@ from hive.model.base import ANModel from hive.utils import IntRange -import copy + class AssoEnd(object): """ Just a small class for modeling the two ends of the association """ + def __init__(self, model=None, ref=None, mul='*'): self.model = model assert issubclass(self.model, ANModel), 'Association must be connecting two model classes' self.ref = ref if ref else self.model._meta.module_name self.mul = IntRange(mul, late_parse=False) - + + class Association(object): """Association objects @@ -27,7 +29,8 @@ :param assoc_model: The association class to attach additional fields :type assoc_model: ANModel - """ + """ + def __init__(self, left_params=None, right_params=None, ref=None, assoc_model=None): self._left = AssoEnd(**left_params) self._right = AssoEnd(**right_params) @@ -55,25 +58,26 @@ def has_ref(self, ref_name): return ref_name in (self.ref, self._left.ref, self._right.ref) - + def has_ends_ref(self, ref_name): return ref_name in (self._left.ref, self._right.ref) - + def check_multiplicity(self, ref, count): return self._ref_dict[ref].mul.validate(count) - + def is_single_target(self, ref): return str(self._ref_dict[ref].mul) == '1' - - + @property def ref_dict(self): return self._ref_dict - + + class AssoEnd2(object): """ Just a small class for modeling the two ends of the association """ + def __init__(self, field=None, mul='*', comp={}, pos=None): self.field = field self.model = self.field.model @@ -83,7 +87,8 @@ self.mul = mul self.comp = comp self.pos = pos - + + class Association2(object): def __init__(self, field1, field2, index1, index2): if field1.pos == 'left' and field2.pos == 'right': @@ -94,40 +99,40 @@ self._right = AssoEnd2(field1, field1.mul[index1], field1.comp, field1.pos) else: assert 0, 'These two fields has invalid position definitions: %s, %s' % (str(field1), str(field2)) - field1.register_association(self, field2, index1, index2) # will later call model's add_association + field1.register_association(self, field2, index1, index2) # will later call model's add_association field2.register_association(self, field1, index2, index1) - + self._register_inherited_models(field1.model, field1.name, index1, index2) self._register_inherited_models(field2.model, field2.name, index2, index1) - + def _register_inherited_models(self, parent_model, field_name, asso_index, target_asso_index): for sub_class in parent_model._meta.children: inherited_field = sub_class._meta.get_field(field_name) - self.register_inherited_field(inherited_field, asso_index, target_asso_index) # don't override if it exists + self.register_inherited_field(inherited_field, asso_index, target_asso_index) # don't override if it exists self._register_inherited_models(sub_class, field_name, asso_index, target_asso_index) - + def register_inherited_field(self, inherited_field, asso_index, target_asso_index, override=False): assert inherited_field.is_inherited() # inheritance always happen before association establishment # so we should register the association to the inherited fields tgt_field = self._right.field if inherited_field.name == self._left.field.name else self._left.field inherited_field.register_association(self, tgt_field, asso_index, target_asso_index) - + def _end_by_pos(self, pos): if pos == 'left': return self._left else: return self._right - + def _tgt_end(self, pos): if pos == 'left': return self._right else: return self._left - + def check_multiplicity(self, pos, count): return self._tgt_end(pos).mul.validate(count) - + def get_ends(self, field): this_end = self._end_by_pos(field.pos) if this_end is self._left: @@ -135,13 +140,10 @@ else: tgt_end = self._left return (this_end, tgt_end) - + def is_one_on_one(self): return str(self._left.mul) == '1' and str(self._right.mul) == '1' def is_composition(self, pos): # this end's multiplicity should be 1 return repr(self._end_by_pos(pos).mul) in ['1'] - - - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py (working copy) @@ -3,37 +3,36 @@ """ import copy +import html +import json import sys +from collections import OrderedDict +from functools import partial from functools import update_wrapper -from hive.utils import ClassProperty, Singleton, ObjCache, dump_dict, get_current_session, andebug, anerror, obj2url -import hive.model.manager # Imported to register signal handler. from django.conf import settings -from djproject.an_settings import PSEUDO_MANAGER_TEST_MODE from django.core.exceptions import (ObjectDoesNotExist, - MultipleObjectsReturned, FieldError, NON_FIELD_ERRORS) + MultipleObjectsReturned) +from django.db.models.fields.related import ManyToOneRel +from django.utils.encoding import smart_str, force_str + +from djproject.an_settings import PSEUDO_MANAGER_TEST_MODE from hive.exceptions import ValidationError -from django.core import validators -# ANModel imports -from hive.model.fields import FieldDoesNotExist -from hive.model.options import Options, ListOptions from hive.model import signals +from hive.model.fields import FieldDoesNotExist from hive.model.loading import register_models, get_model from hive.model.manager import PseudoManager +from hive.model.options import Options from hive.model.query import DelayedQuery, check_field_expire_in_cache from hive.model.utils import normalize_asso_value, CONDITIONAL_NONE -# ANModel imports end -from functools import partial -from django.utils.encoding import smart_str, force_str -from django.db.models.fields.related import ManyToOneRel -import json -from collections import OrderedDict -import html +from hive.utils import dump_dict, get_current_session, andebug, anerror, obj2url + class ANModelBase(type): """ Metaclass for all models. As a meta class, all model classes are its instances. That is, the __new__ function would be triggered to create every model class based on its definition. """ + def __new__(cls, name, bases, attrs): super_new = super(ANModelBase, cls).__new__ parents = [b for b in bases if isinstance(b, ANModelBase)] @@ -57,32 +56,33 @@ base_meta = getattr(new_class, '_meta', None) model_module = sys.modules[new_class.__module__] - app_label, package_path = parse_module_path(model_module) + app_label, package_path = parse_module_path(model_module) package_path.append(name) kwargs = {"app_label": app_label, "path": package_path} - # Create an Option class and add as the _meta new_class.add_to_class('_meta', meta_type(meta, **kwargs)) # Append the manager class attr_manager = attrs.pop('Manager', None) super_manager = attrs.pop('SuperManager', None) - + # Pseudo manager if not attr_manager or PSEUDO_MANAGER_TEST_MODE: attr_manager = PseudoManager - #new_class.add_to_class('_manager', attr_manager) + # new_class.add_to_class('_manager', attr_manager) if not abstract: new_class.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', - tuple(x.DoesNotExist - for x in parents if hasattr(x, '_meta') and not x._meta.abstract) - or (ObjectDoesNotExist,), module)) + tuple(x.DoesNotExist + for x in parents if hasattr(x, + '_meta') and not x._meta.abstract) + or (ObjectDoesNotExist,), module)) new_class.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', - tuple(x.MultipleObjectsReturned - for x in parents if hasattr(x, '_meta') and not x._meta.abstract) - or (MultipleObjectsReturned,), module)) + tuple(x.MultipleObjectsReturned + for x in parents if hasattr(x, + '_meta') and not x._meta.abstract) + or (MultipleObjectsReturned,), module)) if base_meta and not base_meta.abstract: # Non-abstract child classes inherit some attributes from their # non-abstract parent (unless an ABC comes before it in the @@ -152,9 +152,9 @@ # little differently from normal models. # # XXX Comment these out to get the abstract class from view layer - #attr_meta.abstract = False - #new_class.Meta = attr_meta - #return new_class + # attr_meta.abstract = False + # new_class.Meta = attr_meta + # return new_class pass new_class.add_to_class('_manager', attr_manager) if super_manager: @@ -209,6 +209,7 @@ signals.class_prepared.send(sender=cls) + class ANModel(object): """ Parent class of all models @@ -260,7 +261,7 @@ else: val = field.get_default(None) setattr(self, field.attname, val) - + for field in left_fields: # Here we pass the instance "self" to "get_default" with risk !!! # some other fields may not yet be ready and this is not a complete instance @@ -278,11 +279,11 @@ if kwargs: raise TypeError("'%s' is an invalid keyword argument for this function" % kwargs.keys()[0]) super(ANModel, self).__init__() - + # A model instance should not have other attributes except the field values. Or the Python pickle interface used by # Psuedo managers won't work well. In this case, we might implement a method "asso_links" if necessary. # - #self.asso_links = SortedDict() + # self.asso_links = SortedDict() signals.post_init.send(sender=self.__class__, instance=self) def __unicode__(self): @@ -311,16 +312,16 @@ def _get_pk_val(self): return self.pk_str() - + def get_attr_raw(self, attr): return object.__getattribute__(self, attr) - + def is_delayed_query(self, attr): if self.get_attr_raw(attr) == DelayedQuery: return True else: return False - + # this is quite similary with __getattribute__, but will not return default value if the value is CONDITIONAL_NONE def get_attr_real(self, attr): if self.get_attr_raw(attr) == DelayedQuery: @@ -342,6 +343,7 @@ return False except KeyError: return False + # a little bit ugly here raw_val = object.__getattribute__(self, 'get_attr_raw')(attr) if raw_val is DelayedQuery: @@ -349,7 +351,7 @@ field = self._meta.get_field(attr) if field.condition_check(self): mgr = self.__class__.get_manager(get_current_session()) - + # this call might have a side effect of setting other fields of # the instance andebug('hive.debug', 'calling get_field from a DelayedQuery value of field %s' % attr) @@ -358,16 +360,16 @@ setattr(self, attr, CONDITIONAL_NONE) return field.get_default(self) elif check_value_expire(self, attr, raw_val): - #if raw_val is DelayedQuery: + # if raw_val is DelayedQuery: # make query and get the value mgr = self.__class__.get_manager(get_current_session()) - + # this call might have a side effect of setting other fields of # the instance andebug('hive.debug', 'calling get_field from a DelayedQuery value of field %s' % attr) return mgr.get_field_by_instance(self, attr) elif raw_val is CONDITIONAL_NONE: - return self._meta.get_field(attr).get_default(self) # keep the raw value as CONDITIONAL_NONE + return self._meta.get_field(attr).get_default(self) # keep the raw value as CONDITIONAL_NONE else: return raw_val @@ -382,7 +384,7 @@ :param session: ``ANSession`` """ return cls._manager(cls, session) - try: + try: if cls._manager_cache is cls._meta.parents.keys()[0]._manager_cache: cls._manager_cache = {} except IndexError: @@ -392,7 +394,7 @@ if session not in cls._manager_cache: cls._manager_cache[session] = cls._manager(cls, session) return cls._manager_cache[session] - + def serializable_value(self, field_name): """ Returns the value of the field name for this instance. If there's @@ -428,27 +430,32 @@ exclude = [] errors = {} - left_fields=[] + left_fields = [] for f in self._meta.fields: if f.name in exclude: continue - + raw_value = self.get_attr_raw(f.attname) if raw_value in (DelayedQuery, CONDITIONAL_NONE): continue - + # Complement or validate the external component field if f.is_external_component(): if f.is_empty_value(raw_value): raw_value = f.get_default(self) else: - if normalize_asso_value(raw_value, in_place=False) != normalize_asso_value(f.get_default(self), in_place=False): - raise ValidationError('Field %s is an external component field with invalid value: %s - %s' % (f.attname, raw_value, f.get_default(self))) - + if normalize_asso_value(raw_value, in_place=False) != normalize_asso_value(f.get_default(self), + in_place=False): + raise ValidationError( + 'Field %s is an external component field with invalid value: %s - %s' % (f.attname, + raw_value, + f.get_default( + self))) + # This was the original django way: # Skip validation for empty fields with optional=True. The developer # is responsible for making sure they have a valid value. - #if f.optional and raw_value in validators.EMPTY_VALUES: + # if f.optional and raw_value in validators.EMPTY_VALUES: # continue # if f.is_empty_value(raw_value): @@ -460,14 +467,14 @@ # !!! need to check condition. left_fields.append(f) continue - #raise ValidationError('Field %s must not be empty values' % (f.attname)) + # raise ValidationError('Field %s must not be empty values' % (f.attname)) try: setattr(self, f.attname, f.clean(raw_value, self)) except ValidationError as e: errors[str(f.verbose_name)] = e.messages if errors: raise ValidationError(errors) - + # Filter conditional fields self.condition_filtering() # After condition check, some empty value field may set to CONDITIONAL_NONE, so check empty and optional again. @@ -478,14 +485,13 @@ else: raise ValidationError('Field %s must not be empty values' % (f.attname)) - def condition_filtering(self): for grp in self._meta.field_groups.values(): grp_condition_pass = True # 1. Check per-group condition if grp.condition and not grp.condition.check(self): grp_condition_pass = False - + # 2. Check per-field conditions and make necessary marking for f in grp.fields: if self.get_attr_raw(f.attname) is DelayedQuery: @@ -500,7 +506,7 @@ if self.get_attr_raw(f.attname) == CONDITIONAL_NONE: setattr(self, f.attname, f.get_default(self)) else: - setattr(self, f.attname, CONDITIONAL_NONE) + setattr(self, f.attname, CONDITIONAL_NONE) def validate_unique(self, exclude=None): """ @@ -513,18 +519,19 @@ raise ValidationError(errors) # if visible_edit_only is true, will ignore withstats and withhidden - def get_field_dict(self, exclude=None, serialize=False, group=None, withstats=True, withhidden=False, visible_edit_only=False, need_escape=False, ignore_condition=False): + def get_field_dict(self, exclude=None, serialize=False, group=None, withstats=True, withhidden=False, + visible_edit_only=False, need_escape=False, ignore_condition=False): """ Get a dictionary of fields in format of name->field """ ret = {} if exclude is None: exclude = [] - + if group is None: fields = self._meta.fields else: fields = self._meta.field_groups[group].fields - + for f in fields: if f.name in exclude: continue @@ -557,7 +564,7 @@ else: each_dict = html.escape(each_dict) ret[f.attname] = val - + if serialize: try: return json.dumps(ret) @@ -565,6 +572,7 @@ anerror('hive.debug', 'Fail to serialize JSON data %s' % str(ret)) raise return ret + def escape_dict(self, each_dict): for x in each_dict: if type(each_dict[x]) == str: @@ -608,38 +616,40 @@ assert field_name, "field_name must be specified" f = self._meta.get_field(field_name) return f.value_to_rest(self) - + def repr(self): # if this is a profile, return model's verbose name if self._meta.profile: return str(self._meta.verbose_name) - + # if Meta.repr_from_instance is not None, call it if self._meta.repr_from_instance: return self._meta.repr_from_instance(self) elif hasattr(self, 'name'): return getattr(self, 'name') else: - return ' - '.join([smart_str(pk_field.value_to_display(obj=self, style=False)) for pk_field in self._meta.pk_set]) + return ' - '.join( + [smart_str(pk_field.value_to_display(obj=self, style=False)) for pk_field in self._meta.pk_set]) @classmethod def repr_by_pk(cls, pk): if cls._meta.repr_by_pk: return cls._meta.repr_by_pk(pk) else: - return ' - '.join([smart_str(cls._meta.get_field(pk_field).value_to_display(val=pk_value, style=False)) for pk_field, pk_value in pk.iteritems() if pk_value]) - + return ' - '.join([smart_str(cls._meta.get_field(pk_field).value_to_display(val=pk_value, style=False)) for + pk_field, pk_value in pk.iteritems() if pk_value]) + def pk_str(self): if self._meta.profile: return '' - + return dump_dict(self.pk_dict()) - + def pk_dict(self): if self._meta.profile: return {} - - #pk_values = [(pk_field.name, pk_field.value_to_string(self)) for pk_field in self._meta.pk_set] + + # pk_values = [(pk_field.name, pk_field.value_to_string(self)) for pk_field in self._meta.pk_set] # Don't use value_to_string here in case of nesting asso field pk_values = [(pk_field.name, pk_field._get_val_from_obj(self)) for pk_field in self._meta.pk_set] return dict(pk_values) @@ -650,7 +660,6 @@ return True return False - def pk_id(self): if self._meta.profile: return None @@ -668,15 +677,16 @@ return cls._meta.asso_ref_to_instance_id(asso_instance_id, asso_ref) else: return asso_ref - + def url_params(self): - return '&'.join(['%s=%s' % (name, obj2url(value)) for name,value in self.pk_dict().iteritems()]) - + return '&'.join(['%s=%s' % (name, obj2url(value)) for name, value in self.pk_dict().iteritems()]) + def get_statistics(self, start_time=None, end_time=None, field=None, cal_method=None): # Get statistics for this module or one field in this module. if field != None: field = self._meta.get_field(field) - return field._get_data(start_time=start_time, end_time=end_time, cal_method=cal_method, instance_pk=self.pk_str()) + return field._get_data(start_time=start_time, end_time=end_time, cal_method=cal_method, + instance_pk=self.pk_str()) else: ret = {} for group_name, g in self._meta.field_groups.iteritems(): @@ -686,10 +696,10 @@ return ret def get_ajax_url(self, action, param_dict={}): - param_str = '&'.join([str(key)+'='+str(value) for key,value in param_dict.iteritems()]) + param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.iteritems()]) if param_str != '': param_str = '?' + param_str - + if self._meta.profile: return '/api/' + self._meta.app_label + '/' + '/'.join(self._meta.path) + '/_' + action + param_str else: @@ -699,12 +709,14 @@ # the values should be encoded into url string # they are later decoded at the ObjectAjaxHandler's __init__ pk_item_list.append(obj2url(val)) - return '/api/' + self._meta.app_label + '/' + '/'.join(self._meta.path) + '/_' + action + '/' + '/'.join(pk_item_list) + param_str - + return '/api/' + self._meta.app_label + '/' + '/'.join(self._meta.path) + '/_' + action + '/' + '/'.join( + pk_item_list) + param_str + # def tree_node_str(self, status=0): - colour = ['#00FA00', '#19E100', '#32C800', '#4B9600', '#647D00', '#7D6400', '#964B00', '#C83200', '#E11900', '#FA0000'] - return ''+ self.repr() + '' + colour = ['#00FA00', '#19E100', '#32C800', '#4B9600', '#647D00', '#7D6400', '#964B00', '#C83200', '#E11900', + '#FA0000'] + return '' + self.repr() + '' @property def related_actions(self): @@ -720,8 +732,10 @@ ############################################## def get_absolute_url(opts, func, self, *args, **kwargs): - return settings.ABSOLUTE_URL_OVERRIDES.get('%s.%s' % (opts.app_label, opts.module_name), func)(self, *args, **kwargs) + return settings.ABSOLUTE_URL_OVERRIDES.get('%s.%s' % (opts.app_label, opts.module_name), func)(self, *args, + **kwargs) + def parse_module_path(module): """ Parse a Python module like ``apv.models.loadbalancing.slb`` into (app_label, package_path) like ('apv', ['loadbalancing','slb']) @@ -731,7 +745,7 @@ module_name = module.__name__ path_list = module_name.split('.') if path_list[0] == 'hive': - #need to skip model under hive + # need to skip model under hive return ('hive', ['base']) ispkg = module.__file__.endswith('__init__.py') or module.__file__.endswith('__init__.pyc') try: @@ -739,11 +753,10 @@ except ValueError: raise TypeError("module's __name__ must contain 'models': %s" % (module_name)) - app_label = path_list[models_idx-1] - package_path = path_list[models_idx+1:] if ispkg else path_list[models_idx+1:-1] + app_label = path_list[models_idx - 1] + package_path = path_list[models_idx + 1:] if ispkg else path_list[models_idx + 1:-1] return (app_label, package_path) - ######## # MISC # @@ -752,7 +765,8 @@ def subclass_exception(name, parents, module): return type(name, parents, {'__module__': module}) -# + +# class ANList(ANModel): """ Parent class of all custom list Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py (working copy) @@ -1,17 +1,18 @@ -import copy from bisect import bisect +from filecmp import cmp +from django.utils.encoding import smart_str +from django.utils.translation import gettext_lazy as _ + from hive import exceptions +from hive.document import get_xml_reference, DOCUMENT_OK, DOCUMENT_NEW +from hive.exceptions import FieldDoesNotExist from hive.model import validators -from django.utils.translation import gettext_lazy as _ -from django.utils.encoding import smart_str +from hive.model.utils import CONDITIONAL_NONE +from hive.model.utils import get_value_asso_idx +from hive.utils import * from hive.widgets.form import TextInput, StaticFormWidget from hive.widgets.status import Stat -from hive.utils import * -from hive.model.utils import get_value_asso_idx -from hive.model.utils import CONDITIONAL_NONE -from hive.exceptions import FieldDoesNotExist -from hive.document import get_xml_reference, DOCUMENT_OK, DOCUMENT_NEW, DOCUMENT_PENDING class NOT_PROVIDED: @@ -19,6 +20,7 @@ """ pass + # A guide to Field parameters: # # * name: The name of the field specifed in the model. @@ -87,15 +89,15 @@ type_name = 'base' type_default = NOT_PROVIDED type_arguments = {} - + # These track each time a Field instance is created. Used to retain order. creation_counter = 0 - default_validators = [] # Default set of validators + default_validators = [] # Default set of validators default_error_messages = { 'invalid_choice': u'Value %r is not a valid choice.', 'null': u'This field cannot be null.', 'unique': u'%(model_name)s with this %(field_label)s ' - u'already exists.', + u'already exists.', } def _description(self): @@ -105,12 +107,15 @@ return _(u'Type of the field value should be: %(field_type)s') % { 'field_type': self.__class__.__name__ } + description = property(_description) def __init__(self, verbose_name=None, name=None, primary_key=False, - indexed=None, unique=None, optional=False, - default=NOT_PROVIDED, editable=NOT_PROVIDED, configurable=True, hidden=False, hide_edit=False, hide_list=False, help_text='', - validators=[], error_messages=None, condition=None, basic_info=False, object_switch=False, cache_expire_time=0, visible_edit=None, custom_display=NOT_PROVIDED, statistics_source="kernel"): + indexed=None, unique=None, optional=False, + default=NOT_PROVIDED, editable=NOT_PROVIDED, configurable=True, hidden=False, hide_edit=False, + hide_list=False, help_text='', + validators=[], error_messages=None, condition=None, basic_info=False, object_switch=False, + cache_expire_time=0, visible_edit=None, custom_display=NOT_PROVIDED, statistics_source="kernel"): self.name = name self._verbose_name = verbose_name self.primary_key = primary_key @@ -125,17 +130,17 @@ self.hide_list = hide_list self._help_text = help_text self.group = None - self.model = None # keeps None for nested fields - self.root_model = None # The original model this field is inherited from - self.nesting_field = None # parent field (UnionField/GroupField/ListField) for nested fields + self.model = None # keeps None for nested fields + self.root_model = None # The original model this field is inherited from + self.nesting_field = None # parent field (UnionField/GroupField/ListField) for nested fields self._actions = [] - self.basic_info = basic_info # for displaying in the Basic Info block of per-instance status page - self.object_switch = object_switch # for displaying at the first place of the per-instance status page + self.basic_info = basic_info # for displaying in the Basic Info block of per-instance status page + self.object_switch = object_switch # for displaying at the first place of the per-instance status page self.condition = condition self.cache_expire_time = cache_expire_time self.visible_edit = visible_edit - self.custom_display=custom_display - self.derived = False # this is copied from parent class as a DerivedField + self.custom_display = custom_display + self.derived = False # this is copied from parent class as a DerivedField # Adjust the appropriate creation counter, and save our local copy. self.creation_counter = Field.creation_counter @@ -223,7 +228,7 @@ sub classes. """ pass - + def clean(self, value, model_instance): """ Convert the value's type and run validation. Validation errors @@ -243,7 +248,7 @@ @property def unique(self): return self._unique or self.primary_key - + def _set_attributes_from_name(self, name): if not self.name: self.name = name @@ -253,14 +258,14 @@ # This is also called when sub classes inherit the field from parent class def contribute_to_class(self, cls, name): - if self.model: # this field is inherited + if self.model: # this field is inherited if not self.root_model: self.root_model = self.model self.model = cls self.default_override() else: self.model = cls - + self._set_attributes_from_name(name) cls._meta.add_field(self, self.group) if self.editable is NOT_PROVIDED: @@ -269,7 +274,7 @@ else: self.editable = True # if specially provided at definition, should keep the value - + """ if self.choices: setattr(cls, 'get_%s_display' % self.name, @@ -278,12 +283,12 @@ self.init_sub_fields() # only used by nested fields (UnionField/ListField/GroupField/EnumField) - #@per_sess_cached_property + # @per_sess_cached_property @property def document(self): if self.model: if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']" result = self.model._meta.document.find(xpath) if result is None: for parent in self.model._meta.parents: @@ -294,36 +299,36 @@ return result elif self.nesting_field: if self.nesting_field.document is not None: - xpath = "./subfields/field[@name='"+self.name+"']" + xpath = "./subfields/field[@name='" + self.name + "']" return self.nesting_field.document.find(xpath) elif self._actions: if self._actions[0].document is not None: - xpath = "./field[@name='"+self.name+"']" + xpath = "./field[@name='" + self.name + "']" return self._actions[0].document.find(xpath) - + return None @property - def verbose_name(self): # XXX consider using self.document here? pros and cons? + def verbose_name(self): # XXX consider using self.document here? pros and cons? if self.is_inherited(): return self.get_original_field().verbose_name if self.model: if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']/verbose_name" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']/verbose_name" pkg = self.path.split('.')[0] return get_xml_reference(pkg, self.model._meta.document.find(xpath), default=self._verbose_name) elif self.nesting_field: # this is a nested field, get doc from its parent field if self.nesting_field.document is not None: - xpath = "./subfields/field[@name='"+self.name+"']/verbose_name" + xpath = "./subfields/field[@name='" + self.name + "']/verbose_name" pkg = self.path.split('.')[0] return get_xml_reference(pkg, self.nesting_field.document.find(xpath), default=self._verbose_name) elif self._actions: if self._actions[0].document is not None: - xpath = "./field[@name='"+self.name+"']/verbose_name" + xpath = "./field[@name='" + self.name + "']/verbose_name" pkg = self._actions[0].model._meta.path[0] return get_xml_reference(pkg, self._actions[0].document.find(xpath), default=self._verbose_name) @@ -332,27 +337,27 @@ @verbose_name.setter def verbose_name(self, new_value): self._verbose_name = new_value - + @property - def help_text(self): # XXX consider using self.document here? pros and cons? + def help_text(self): # XXX consider using self.document here? pros and cons? if self.is_inherited(): return self.get_original_field().help_text if self.model: if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']/help_text" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']/help_text" pkg = self.path.split('.')[0] return get_xml_reference(pkg, self.model._meta.document.find(xpath), default=self._help_text) elif self.nesting_field: # this is a nested field, get doc from its parent field if self.nesting_field.document is not None: - xpath = "./subfields/field[@name='"+self.name+"']/help_text" + xpath = "./subfields/field[@name='" + self.name + "']/help_text" pkg = self.path.split('.')[0] return get_xml_reference(pkg, self.nesting_field.document.find(xpath), default=self._help_text) elif self._actions: if self._actions[0].document is not None: - xpath = "./field[@name='"+self.name+"']/help_text" + xpath = "./field[@name='" + self.name + "']/help_text" pkg = self._actions[0].model._meta.path[0] return get_xml_reference(pkg, self._actions[0].document.find(xpath), default=self._help_text) @@ -367,7 +372,7 @@ if self.document is None: return DOCUMENT_NEW else: - return DOCUMENT_OK # this field already exist in current document + return DOCUMENT_OK # this field already exist in current document @property def document_text(self): @@ -378,7 +383,7 @@ def get_sub_fields(self): return None - + def get_attname(self): return self.name @@ -440,7 +445,6 @@ rtn = val return rtn - def value_to_string(self, obj=None, val=None): """ Returns a string value of this field from the passed obj. @@ -491,12 +495,12 @@ just a shortcut """ return self.value_to_display(obj=obj, style=True, text=False) - + def value_split(self, value): # used for GroupField, return [[sub_field1,val1],[sub_field2,val2]] # for other field, return [[field_name,value]] return [[str(self.verbose_name), self.value_to_lexical(val=value)]] - + def value_to_num(self, obj=None, val=None): value = self._get_val_from_obj_or_val(obj, val) if type(value) == float: @@ -524,7 +528,7 @@ self._optional = value optional = property(get_optional, set_optional) - + @property def required(self): # if this field is not optional and there's no default value provided, it is required; @@ -532,32 +536,34 @@ return True if not self.optional and self.default is NOT_PROVIDED else False def static_widget(self, model_instance=None, manager=None, context_value=None, context_repr=None): - return StaticFormWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, context_value=context_value, context_repr=context_repr) - + return StaticFormWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, + context_value=context_value, context_repr=context_repr) + def default_widget(self, model_instance=None, manager=None, **kwargs): # The default default widget is a simple TextInput widget that ignores all other keyword parameters - return TextInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, required=self.required) + return TextInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, + required=self.required) def status_widget(self, model_instance=None, manager=None, **kwargs): # The default stat widget is a simple Stat widget that ignores all other keyword parameters return Stat(name=self.name, label=self.verbose_name, help_text=self.help_text, **kwargs) - + def statistics_widget(self, model_instance=None, manager=None, **kwargs): # The default stat widget is a simple Stat widget that ignores all other keyword parameters return Stat(name=self.name, label=self.verbose_name, help_text=self.help_text, **kwargs) - + def value_from_object(self, obj): """ Returns the value of this field in the given model instance. """ return getattr(obj, self.attname) - + def register_related_action(self, action_cls): self._actions.append(action_cls) - + def has_related_action(self): return self.related_actions() != [] - + def related_actions(self, instance=None): tmp_acts = [] for each_act in self._actions: @@ -583,11 +589,11 @@ def set_group(self, group): self.group = group - + def get_pseudo_value(self): # should be implemented by sub classes raise ValueError('The function is not implemented by this field type') - + @property def is_scalar_stats(self): return self.group.isstats and not self.basic_info and not self.object_switch @@ -600,43 +606,43 @@ return self.root_model._meta.get_field(self.name) else: return None - + # Some field type (e.g. AssoField) have some default # overriding behavior even if there's no OverrideField defined def default_override(self): # to be implemented by various field types pass - + # This function is called by OverrideField def override(self, default=None, optional=None, editable=None, configurable=None, hidden=None, **kwargs): if default is not None: - self.default=default + self.default = default if optional is not None: - self.optional=optional + self.optional = optional if editable is not None: self.editable = editable if configurable is not None: self.configurable = configurable if hidden is not None: self.hidden = hidden - + def is_external_component(self): if self.type_name != 'asso': return False - + if len(self.asso) != 1: return False - + if self.asso[0] and hasattr(self.asso[0], 'is_one_on_one') and self.asso[0].is_one_on_one(): target_pk = self.target_end[0].model._meta.pk_set if len(target_pk) == 1 and self.target_end[0].field in target_pk: return True return False - + def is_external_component_target(self): if self.type_name != 'asso': return False - + # As an external component target, the asso count # doesn't have to be 1. e.g. HTTPSettings # We simply check the first asso's target field as @@ -644,38 +650,37 @@ if hasattr(self, 'tgt_field'): return self.tgt_field[0].is_external_component() return False - + def get_external_component_obj(self, instance): assert self.is_external_component() target_manager = self.target_end[0].model.get_manager(get_current_session()) target_pk = self.value_from_object(instance)[0] return target_manager.get(target_pk) - - + def is_composition_asso(self): if self.type_name != 'asso': return False # all assos must be composition asso_idx = 0 - for each_asso in self.asso: + for each_asso in self.asso: if each_asso and hasattr(each_asso, 'is_composition') and each_asso.is_composition(self.pos): - pass # check next + pass # check next else: return False asso_idx += 1 return True - + def is_composition_asso_target(self): if self.type_name != 'asso': return False - + if hasattr(self, 'tgt_field'): for each_tgt_field in self.tgt_field: if not each_tgt_field.is_composition_asso(): return False - return True # all tgt fields are composition asso + return True # all tgt fields are composition asso return False - + # the condition / asso_condition in add form is implemented at the front-end def visible_in_add(self): if self.group.isstats: @@ -683,7 +688,7 @@ if self.hidden or not self.configurable or self.is_composition_asso(): return False return True - + # the condition / asso_condition in edit form is implemented at the Group's visible_in_edit def visible_in_edit(self): if self.visible_edit is not None: @@ -693,7 +698,7 @@ if self.hidden or self.hide_edit or self.is_external_component_target(): return False return True - + @property def path(self): if self.model: @@ -716,7 +721,7 @@ # 1. Check group condition if self.group and self.group.condition and not self.group.condition.check(instance): return False - + # 2. Check field conditions field_condition_pass = True if self.condition and not self.condition.check(instance): @@ -733,22 +738,28 @@ return asso_idx + class BASIC: """ Macro for the ``level`` of :py:class:`FieldGroup` """ pass + + class ADVANCED: """ Macro for the ``level`` of :py:class:`FieldGroup` """ pass + + class STATS: """ Macro for the ``level`` of :py:class:`FieldGroup` """ pass + class FieldGroup(object): """ Groups to organize fields of a model class. @@ -778,23 +789,26 @@ """ # These track each time a Field Group instance is created. Used to retain order. creation_counter = 1 + # 0 is left for the auto-created default group - - def __init__(self, name=None, verbose_name=None, help_text=None, writable=True, level=BASIC, fields=None, condition=None, editable=NOT_PROVIDED, optional=False, override=False, subjoin=False, visible_edit=None, stats_source='kernel', custom_tab=None): + + def __init__(self, name=None, verbose_name=None, help_text=None, writable=True, level=BASIC, fields=None, + condition=None, editable=NOT_PROVIDED, optional=False, override=False, subjoin=False, + visible_edit=None, stats_source='kernel', custom_tab=None): self.name = name self._verbose_name = verbose_name self._help_text = help_text self.writable = writable self.level = level self._field_dict = fields if fields else {} - self._fields = [] # initialize at contribute_to_class + self._fields = [] # initialize at contribute_to_class self.condition = condition self.editable = editable self._optional = optional self.override = override self.subjoin = subjoin - self.stats_source = stats_source - self.custom_tab = custom_tab + self.stats_source = stats_source + self.custom_tab = custom_tab # Adjust the appropriate creation counter, and save our local copy. if self.name == 'default': self.creation_counter = 0 @@ -806,21 +820,21 @@ def __cmp__(self, other): return (self.creation_counter > other.creation_counter) - (self.creation_counter < other.creation_counter) - + @property def fields(self): return self._fields - + def get_optional(self): if self.level is STATS: return True return self._optional - + def set_optional(self, value): self._optional = value - + optional = property(get_optional, set_optional) - + def _set_attributes_from_name(self, name): if not self.name: self.name = name @@ -838,8 +852,8 @@ if old_group: for each in old_group._fields: cls._meta.del_field(each) - - self._fields = [] # reset the list here in case its copied from parent cls + + self._fields = [] # reset the list here in case its copied from parent cls if self.subjoin: # add all fields belong to this group in parent class into _fields old_group = cls._meta.get_field_group(self.name) @@ -852,11 +866,11 @@ self._fields.insert(bisect(self._fields, field), field) cls._meta.add_field_group(self) - #@cached_property + # @cached_property @property def verbose_name(self): if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.name+"']/verbose_name" + xpath = "./fieldgrp[@name='" + self.name + "']/verbose_name" pkg = self.model._meta.path[0] return get_xml_reference(pkg, self.model._meta.document.find(xpath), default=self._verbose_name) return self._verbose_name @@ -864,12 +878,12 @@ @verbose_name.setter def verbose_name(self, new_value): self._verbose_name = new_value - - #@cached_property + + # @cached_property @property def help_text(self): if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.name+"']/help_text" + xpath = "./fieldgrp[@name='" + self.name + "']/help_text" pkg = self.model._meta.path[0] return get_xml_reference(pkg, self.model._meta.document.find(xpath), default=self._help_text) return self._help_text @@ -881,15 +895,15 @@ @property def document_status(self): if self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.name+"']" + xpath = "./fieldgrp[@name='" + self.name + "']" field_grp_ele = self.model._meta.document.find(xpath) if field_grp_ele is None: return DOCUMENT_NEW else: - return DOCUMENT_OK # this field already exist in current document + return DOCUMENT_OK # this field already exist in current document else: return DOCUMENT_NEW - + def append_field(self, field): # append a field to the group after bound to class # e.g. add field to the default group @@ -900,7 +914,7 @@ def del_field(self, field): del self._field_dict[field.name] self._fields.remove(field) - + def __deepcopy__(self, memodict): new = copy.copy(self) new._field_dict = copy.deepcopy(self._field_dict, memodict) @@ -909,9 +923,11 @@ @property def isbasic(self): return self.level == BASIC + @property def isstats(self): return self.level == STATS + @property def is_cli_statistics(self): if self.isstats and self.fields: @@ -919,6 +935,7 @@ if field.get_internal_type() == "CLITextField": return True return False + @property def is_asso_statistics(self): if self.isstats and self.fields: @@ -926,16 +943,17 @@ if field.get_internal_type() == "AssoField2": return True return False + @property def has_scalar_stats(self): for each_field in self.fields: if each_field.is_scalar_stats: return True return False - + def visible_in_edit(self, instance=None): if self.visible_edit is not None: - if self.visible_edit: + if self.visible_edit: # check condition if instance and self.condition: if self.condition.check(instance) is False: @@ -945,28 +963,31 @@ return False if self.isstats: return False - + # first check the value condition and asso value conditions if instance and self.condition: - andebug('hive.debug', 'instance %s, self.condition %s, self.condition.check(instance) %s' % (str(instance), str(self.condition), str(self.condition.check(instance)))) + andebug('hive.debug', 'instance %s, self.condition %s, self.condition.check(instance) %s' % (str(instance), + str(self.condition), + str(self.condition.check( + instance)))) if self.condition.check(instance) is False: return False - + for field in self.fields: if field.visible_in_edit(): return True - + return False # whether this group only contains hidden fields def all_hidden(self): - if not self.fields: #for FieldGroup without fields but bound to an action, e.g. admintools-troubleshooting-Tools + if not self.fields: # for FieldGroup without fields but bound to an action, e.g. admintools-troubleshooting-Tools return False for field in self.fields: if not field.hidden: return False return True - + def is_special_fieldgrp(self): if len(self.fields) == 1 and self.fields[0].type_name == 'clitext': return True @@ -975,7 +996,7 @@ def register_related_action(self, action_cls): self._actions.append(action_cls) - + def related_actions(self, instance=None): tmp_acts = [] for each_act in self._actions: @@ -983,21 +1004,25 @@ tmp_acts.append(each_act) return tmp_acts + class DefaultValueCondition(object): """ It specific field default value should be decided by some target value. """ + def __init__(self, fieldname, data): self._fieldname = fieldname self._data = data - + def to_dict(self): - return {'field':self._fieldname, 'value':self._data} + return {'field': self._fieldname, 'value': self._data} + class Condition(object): """ A class for describing specific conditions to determine whether a field group or field is available at current status of its belonging model class. """ + def __init__(self): self._model = None pass @@ -1013,21 +1038,23 @@ Check whether the condition matches. Should be overwritten by subclasses. """ return True - + def __and__(self, other): return AndCondition(self, other) - + def __or__(self, other): return OrCondition(self, other) + class ValueCondition(Condition): """ The condition that specific field value should be equal to some target value. """ + def __init__(self, fieldname, fieldvalue): super(ValueCondition, self).__init__() self._fieldname = fieldname - if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here + if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here self._fieldvalue = [fieldvalue] else: self._fieldvalue = fieldvalue @@ -1037,10 +1064,11 @@ if realvalue in self._fieldvalue: return True return False - + def to_dict(self): - return {'field':self._fieldname, 'value':self._fieldvalue} + return {'field': self._fieldname, 'value': self._fieldvalue} + class GroupFieldValueCondition(Condition): def __init__(self, groupname, fieldname, fieldvalue): super(GroupFieldValueCondition, self).__init__() @@ -1062,18 +1090,18 @@ return False def to_dict(self): - return {'group_name':self._groupname, 'field':self._fieldname, 'value':self._fieldvalue} + return {'group_name': self._groupname, 'field': self._fieldname, 'value': self._fieldvalue} - class ExceptionValueCondition(Condition): """ The condition that specific field value should NOT equal to some target value. """ + def __init__(self, fieldname, fieldvalue): super(ExceptionValueCondition, self).__init__() self._fieldname = fieldname - if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here + if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here self._fieldvalue = [fieldvalue] else: self._fieldvalue = fieldvalue @@ -1083,14 +1111,16 @@ if realvalue in self._fieldvalue: return False return True - + def to_dict(self): - return {'field':self._fieldname, 'value':self._fieldvalue, 'except':True} + return {'field': self._fieldname, 'value': self._fieldvalue, 'except': True} + class NonemptyValueCondition(Condition): """ The condition that specific field value should NOT equal to some target value. """ + def __init__(self, fieldname): super(NonemptyValueCondition, self).__init__() self._fieldname = fieldname @@ -1100,14 +1130,16 @@ if realvalue: return True return False - + def to_dict(self): - return {'field':self._fieldname, 'nonempty':True} + return {'field': self._fieldname, 'nonempty': True} + class EmptyValueCondition(Condition): """ The condition that specific field value should NOT equal to some target value. """ + def __init__(self, fieldname): super(EmptyValueCondition, self).__init__() self._fieldname = fieldname @@ -1117,19 +1149,21 @@ if realvalue: return False return True - + def to_dict(self): - return {'field':self._fieldname, 'nonempty':False} + return {'field': self._fieldname, 'nonempty': False} + class AssoValueCondition(Condition): """ The condition that specific field of some associated model object should be equal to some target value. """ + def __init__(self, asso_fieldname, fieldname, fieldvalue): super(AssoValueCondition, self).__init__() self._asso_fieldname = asso_fieldname self._fieldname = fieldname - if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here + if type(fieldvalue) is not list: # XXX if the value itself is a list type, we are wrong here self._fieldvalue = [fieldvalue] else: self._fieldvalue = fieldvalue @@ -1144,50 +1178,53 @@ except IndexError: return False return getattr(target_obj, self._fieldname, None) in self._fieldvalue - + def to_dict(self): - return {'asso_field':self._asso_fieldname, 'field':self._fieldname, 'value':self._fieldvalue} + return {'asso_field': self._asso_fieldname, 'field': self._fieldname, 'value': self._fieldvalue} + class AndCondition(Condition): def __init__(self, *args): super(AndCondition, self).__init__() self._sub_conditions = list(args) - + def check(self, obj): for each in self._sub_conditions: andebug('hive.debug', 'checking sub condition: %s' % str(each.to_dict())) if not each.check(obj): return False return True - + def to_dict(self): # XXX simply return the first sub condition's dict here # we will consider an enhancement together with the frontend rst = [] for each in self._sub_conditions: rst.append(each.to_dict()) - return {'multi':True,'conditions':rst,'rel':'and'} - + return {'multi': True, 'conditions': rst, 'rel': 'and'} + + class OrCondition(Condition): def __init__(self, *args): super(OrCondition, self).__init__() self._sub_conditions = list(args) - + def check(self, obj): for each in self._sub_conditions: andebug('hive.debug', 'checking sub condition: %s' % str(each.to_dict())) if each.check(obj): return True return False - + def to_dict(self): # XXX simply return the first sub condition's dict here # we will consider an enhancement together with the frontend rst = [] for each in self._sub_conditions: rst.append(each.to_dict()) - return {'multi':True,'conditions':rst,'rel':'or'} - + return {'multi': True, 'conditions': rst, 'rel': 'or'} + + def get_parent_field_types(field_class): """ returns a list of field classes from which the class @@ -1196,12 +1233,14 @@ mro_list = [i for i in field_class.__mro__ if i is not field_class and i is not object and issubclass(i, Field)] return mro_list + # DerivedField is designed for adding value constraints to sub classes' fields to determine # the concrete class type class DerivedField(object): """ A Fake Field class for delegate a SingleValueValidator to the copied field from a virtual parent model class. """ + def __init__(self, value=None): self._value = value @@ -1222,7 +1261,7 @@ else: copied_parent_field.default = self._value copied_parent_field.validators.append(validators.SingleValueValidator(self._value)) - + copied_parent_field.derived = True # Establish "children" data structure of ancestor model classes @@ -1233,34 +1272,36 @@ del self + class OverrideField(object): def __init__(self, **kwargs): self.kwargs = dict(kwargs) - + def contribute_to_class(self, cls, name): # find the field from the fields # that have already been loaded # from parent model copied_parent_field = None - + try: copied_parent_field = cls._meta.get_field(name) except FieldDoesNotExist: raise ValueError('OverrideField %s must has the same name as the parent\'s field' % name) - + # call the field's override() function copied_parent_field.override(**self.kwargs) + class ReplaceField(object): def __init__(self, field, **kwargs): self.field = field(**kwargs) - + def contribute_to_class(self, cls, name): # find the field from the fields # that have already been loaded # from parent model copied_parent_field = None - + try: copied_parent_field = cls._meta.get_field(name) except FieldDoesNotExist: @@ -1269,7 +1310,3 @@ cls._meta.real_del_field(copied_parent_field) self.field.creation_counter = copied_parent_field.creation_counter self.field.contribute_to_class(cls, name) - - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py (working copy) @@ -1,31 +1,29 @@ -import copy -import json -from bisect import bisect -import sys, tempfile, os -import time +import html +import os import re -import random -from hive.utils import IntRange,andebug, get_current_session, is_ipv4, is_ipv6, text_add_style +from bisect import bisect + +from django.utils.translation import gettext_lazy as _ + +from djproject.an_settings import TEST_IP_POOL, DIR_STR from hive import exceptions from hive.model import validators -from django.utils.translation import gettext_lazy as _ -from django.utils.encoding import smart_str +from hive.model.association import Association2 from hive.model.fields import Field, get_parent_field_types +from hive.model.loading import get_model +from hive.model.utils import * +from hive.utils import IntRange, get_current_session, is_ipv4, is_ipv6, text_add_style +from hive.utils import nd_iter_space, dict_combine from hive.widgets.form import * -from hive.widgets.status import Stat,ActionStatus,Gauge,SquareStatus,PlainStatus,PlainInfo,BarStatus from hive.widgets.statistics import LineChart -from hive.model.association import Association, Association2 -from hive.model.utils import * -from hive.utils import andebug, nd_iter_space, cached_property, dict_combine -from djproject.an_settings import TEST_IP_POOL, DIR_STR, MAX_FILES_COUNT, RRDCACHED_SOCKET -from hive.model.loading import get_app, get_model -import html +from hive.widgets.status import ActionStatus, Gauge, SquareStatus, PlainStatus, PlainInfo, BarStatus # The values to use for "blank" in ChoiceFields. Will be appended to the start # of most "choices" lists. BLANK_CHOICE_DASH = [("", "---------")] BLANK_CHOICE_NONE = [("", "None")] + class BooleanField(Field): # Attributes of field types type_name = 'boolean' @@ -36,7 +34,8 @@ } description = _("Boolean (Either True or False)") - def __init__(self, verbose_name=None, name=None, lexical=('True','False'), display=(_('Enabled'), _('Disabled')), style=('text-true fa-check-circle','text-false fa-times-circle'), **kwargs): + def __init__(self, verbose_name=None, name=None, lexical=('True', 'False'), display=(_('Enabled'), _('Disabled')), + style=('text-true fa-check-circle', 'text-false fa-times-circle'), **kwargs): self.lexical = lexical self.display = display self.style = style @@ -47,7 +46,7 @@ # this is for the _get_xxx interfaces of manager implementation if self.is_empty_value(value): value = self.get_default(None) - + assert value is not None if value in (True, False): # if value is 1 or 0 than it's equal to True or False, but we want @@ -78,7 +77,7 @@ val = self._get_val_from_obj_or_val(obj, val) if val is None: val = self.default - return str(self.lexical[1-int(val)]) + return str(self.lexical[1 - int(val)]) def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) @@ -86,31 +85,34 @@ if value is None: return rtn if text: - rtn = self.lexical[1-int(value)] + rtn = self.lexical[1 - int(value)] if style: - rtn = text_add_style(rtn, self.style[1-int(value)]) + rtn = text_add_style(rtn, self.style[1 - int(value)]) return rtn - #if value == True: + # if value == True: # return '' - #else: + # else: # return '' def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name) if model_instance else None default = self.get_default(model_instance) - if self.display != ('',''): - return Switch(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, default=default, + if self.display != ('', ''): + return Switch(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, + value=value, default=default, lexical=self.lexical, display=self.display, **kwargs) else: - return CheckboxInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, default=default, **kwargs) - + return CheckboxInput(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, default=default, **kwargs) + def status_widget(self, model_instance=None, manager=None, **kwargs): value = self.value_to_lexical(model_instance) if model_instance else "" lexical = value - widget_type = kwargs.get('type','table') + widget_type = kwargs.get('type', 'table') if widget_type == 'table': - return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, lexical=lexical, - instance=model_instance, **kwargs) + return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + lexical=lexical, + instance=model_instance, **kwargs) def get_iterable_value(self, bias): if self.has_default(): @@ -118,11 +120,12 @@ return self.get_default(None) else: bias -= 1 - if bias%2 == 1: + if bias % 2 == 1: return True else: return False - + + class CharField(Field): # Attributes of field types type_name = 'string' @@ -136,8 +139,9 @@ super(CharField, self).__init__(verbose_name, name, **kwargs) # setup length validator - length_str_list = [length, self.__class__.type_arguments.get('length', None)] +\ - [parent_class.type_arguments.get('length', None) for parent_class in get_parent_field_types(self.__class__)] + length_str_list = [length, self.__class__.type_arguments.get('length', None)] + \ + [parent_class.type_arguments.get('length', None) for parent_class in + get_parent_field_types(self.__class__)] length_str_list.reverse() self.length = None for length_str in length_str_list: @@ -145,18 +149,19 @@ continue self.length = IntRange.merge(self.length, IntRange(length_str).set_parent(self.length)) self.validators.append(validators.LengthValidator(self.length)) - + # setup pattern validator self.pattern = pattern type_pattern = self.type_arguments.get('pattern', None) dbquote_pattern = '^[^"]+$' dbquote_pattern_msg = 'Invalid value! Double quote character is not allowed.' - if not allow_dbquote: #allow_dbquote=True means Double quote character is allowed. + if not allow_dbquote: # allow_dbquote=True means Double quote character is allowed. if self.pattern: self.validators.append(validators.RegexValidator(pattern, pattern_msg)) self.validators.append(validators.RegexValidator(dbquote_pattern, dbquote_pattern_msg)) elif type_pattern: - self.validators.append(validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) + self.validators.append( + validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) self.validators.append(validators.RegexValidator(dbquote_pattern, dbquote_pattern_msg)) else: self.validators.append(validators.RegexValidator(dbquote_pattern, dbquote_pattern_msg)) @@ -164,16 +169,17 @@ if self.pattern: self.validators.append(validators.RegexValidator(pattern, pattern_msg)) elif type_pattern: - self.validators.append(validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) - - self.examples=examples - self.style=style + self.validators.append( + validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) + + self.examples = examples + self.style = style self.style_class = style_class - + def to_python(self, value): if value is None: value = self.get_default(None) - if isinstance(value, str): # or value is None: (value is never None here) + if isinstance(value, str): # or value is None: (value is never None here) return value return smart_str(value) @@ -197,10 +203,11 @@ def status_widget(self, model_instance=None, manager=None, **kwargs): value = self._get_val_from_obj(model_instance) or '' lexical = value - widget_type = kwargs.get('type','table') + widget_type = kwargs.get('type', 'table') if widget_type == 'table': - return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, lexical=lexical, - instance=model_instance, **kwargs) + return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + lexical=lexical, + instance=model_instance, **kwargs) def get_iterable_value(self, bias): if self.has_default(): @@ -209,12 +216,13 @@ else: bias -= 1 if self.examples: - return self.examples[bias%len(self.examples)] + return self.examples[bias % len(self.examples)] elif 'name' in self.name and self.primary_key: return 'test_' + self.model._meta.module_name + str(bias) else: return 'test_' + self.name + '_' + str(bias) - + + class TextField(Field): # Attributes of field types type_name = 'text' @@ -228,8 +236,9 @@ super(TextField, self).__init__(verbose_name, name, **kwargs) # setup length validator - length_str_list = [length, self.__class__.type_arguments.get('length', None)] +\ - [parent_class.type_arguments.get('length', None) for parent_class in get_parent_field_types(self.__class__)] + length_str_list = [length, self.__class__.type_arguments.get('length', None)] + \ + [parent_class.type_arguments.get('length', None) for parent_class in + get_parent_field_types(self.__class__)] length_str_list.reverse() self.length = None for length_str in length_str_list: @@ -244,15 +253,16 @@ if self.pattern: self.validators.append(validators.RegexValidator(pattern, pattern_msg)) elif type_pattern: - self.validators.append(validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) + self.validators.append( + validators.RegexValidator(type_pattern, self.type_arguments.get('pattern_msg', None))) - self.examples=examples - self.style=style + self.examples = examples + self.style = style def to_python(self, value): if value is None: value = self.get_default(None) - if isinstance(value, str): # or value is None: (value is never None here) + if isinstance(value, str): # or value is None: (value is never None here) return value return smart_str(value) @@ -266,16 +276,18 @@ value = getattr(model_instance, self.name) if model_instance else None default = self.get_default(model_instance) return TextArea(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, - value=value, required=self.required, default=default, style=self.style, **kwargs) + value=value, required=self.required, default=default, style=self.style, **kwargs) def status_widget(self, model_instance=None, manager=None, **kwargs): value = self._get_val_from_obj(model_instance) or '' lexical = value - widget_type = kwargs.get('type','table') + widget_type = kwargs.get('type', 'table') if widget_type == 'table': - return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, lexical=lexical, - instance=model_instance, **kwargs) - + return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + lexical=lexical, + instance=model_instance, **kwargs) + + class JSONField(Field): # Attributes of field types type_name = 'json' @@ -290,7 +302,7 @@ def to_python(self, value): if value is None: value = self.get_default(None) - if isinstance(value, str): # or value is None: (value is never None here) + if isinstance(value, str): # or value is None: (value is never None here) return value return value @@ -304,16 +316,18 @@ value = getattr(model_instance, self.name) if model_instance else None default = self.get_default(model_instance) return TextArea(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, - value=value, required=self.required, default=default, **kwargs) + value=value, required=self.required, default=default, **kwargs) def status_widget(self, model_instance=None, manager=None, **kwargs): value = self._get_val_from_obj(model_instance) or '' lexical = value - widget_type = kwargs.get('type','table') + widget_type = kwargs.get('type', 'table') if widget_type == 'table': - return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, lexical=lexical, - instance=model_instance, **kwargs) + return PlainInfo(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + lexical=lexical, + instance=model_instance, **kwargs) + class EnumField(Field): # Attributes of field types type_name = 'enum' @@ -337,22 +351,22 @@ each_rtn_value = list(each_value) # extend the list with two empty strings so that its length # is always bigger than 3 - each_rtn_value.extend(['','']) + each_rtn_value.extend(['', '']) field_doc_ele = self.document if field_doc_ele is not None: - xpath = "./choices/choice[@value='"+str(each_rtn_value[0])+"']" + xpath = "./choices/choice[@value='" + str(each_rtn_value[0]) + "']" choice_ele = field_doc_ele.find(xpath) if choice_ele is not None: each_rtn_value[1] = choice_ele.findtext("./verbose_name", default=each_rtn_value[1]) each_rtn_value[2] = choice_ele.findtext("./help_text", default=each_rtn_value[2]) rtn_values.append(each_rtn_value) - + return rtn_values @values.setter def values(self, new_value): - self._values = values - + self._values = new_value + def to_python(self, value): if value is None: value = self.get_default(None) @@ -389,7 +403,7 @@ value = initial_value return Select(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, choices=self.values, default=self.default, **kwargs) - + def status_widget(self, model_instance=None, manager=None, **kwargs): assert model_instance is not None; assert manager is not None; @@ -398,16 +412,18 @@ if item[0] == value: break display = self.value_to_display(val=value) - + if self.has_related_action() and self.group.isstats: - return ActionStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, lexical=display, - status_values=self.values, actions=manager.get_available_actions(model_instance, self.name), + return ActionStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + lexical=display, + status_values=self.values, + actions=manager.get_available_actions(model_instance, self.name), instance=model_instance, **kwargs) else: help_text = item[2] if len(item) >= 3 else '' return PlainInfo(name=self.name, label=self.verbose_name, help_text=help_text, value=value, lexical=display, - instance=model_instance, **kwargs) - + instance=model_instance, **kwargs) + def get_iterable_value(self, bias): if self.has_default(): if bias == 0: @@ -415,7 +431,7 @@ else: bias -= 1 length = len(self.values) - return self.values[bias%length][0] + return self.values[bias % length][0] @property def document_text(self): @@ -427,6 +443,7 @@ tmp += '
  • %s (%s)
    • ' % (str(each[0]), each[1]) return self.help_text + 'Available values:
      ' + tmp + '
    ' + class MultiEnumField(Field): # Attributes of field types type_name = 'multienum' @@ -446,10 +463,11 @@ each_rtn_value = list(each_value) # extend the list with two empty strings so that its length # is always bigger than 3 - each_rtn_value.extend(['','']) + each_rtn_value.extend(['', '']) # XXX self.model may be None when this is a sub field inside a GroupField/UnionField if self.model and self.model._meta.document is not None: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']/choices/choice[@value='"+str(each_rtn_value[0])+"']" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']/choices/choice[@value='" + str( + each_rtn_value[0]) + "']" choice_ele = self.model._meta.document.find(xpath) if choice_ele is not None: each_rtn_value[1] = choice_ele.findtext("./verbose_name", default=each_rtn_value[1]) @@ -460,7 +478,7 @@ @values.setter def values(self, new_value): - self._values = values + self._values = new_value def to_python(self, value): if value is None: @@ -473,7 +491,7 @@ if item[0] == each or str(item[0]) == each: found = True new_value.append(item[0]) - break # to next value + break # to next value if not found: raise exceptions.ValidationError(_(u'The value is not one of the valid value for the field.')) return new_value @@ -485,7 +503,7 @@ for item in self.values: if item[0] == each: new_value.append(item[1]) - break # to next value + break # to next value return ', '.join(new_value) def _get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH): @@ -500,9 +518,11 @@ # the initial value is always from the concrete-class selector. if initial_value: value = initial_value - return MultiSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, + return MultiSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, choices=self.values, default=self.default, **kwargs) - + + class FloatField(Field): # Attributes of field types type_name = 'float' @@ -526,7 +546,7 @@ assert value is not None try: if self.accuracy is not None: - return round(float(value), self.accuracy) + return round(float(value), self.accuracy) else: return float(value) except (TypeError, ValueError): @@ -539,14 +559,15 @@ return None else: return float(value) - + def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) if self.unit_name: return str(value) + ' ' + str(self.unit_name) else: return str(value) - + + class IntegerField(Field): # Attributes of field types type_name = 'int' @@ -562,8 +583,9 @@ super(IntegerField, self).__init__(verbose_name, name, **kwargs) # setup scope validator - scope_str_list = [scope, self.__class__.type_arguments.get('scope', None)] +\ - [parent_class.type_arguments.get('scope', None) for parent_class in get_parent_field_types(self.__class__)] + scope_str_list = [scope, self.__class__.type_arguments.get('scope', None)] + \ + [parent_class.type_arguments.get('scope', None) for parent_class in + get_parent_field_types(self.__class__)] scope_str_list.reverse() self.scope = None for scope_str in scope_str_list: @@ -593,7 +615,7 @@ return None else: return int(value) - + def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) if self.unit_name: @@ -610,17 +632,21 @@ value = getattr(model_instance, self.name) if model_instance else None default = self.get_default(model_instance) return TextInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, - value=value, required=self.required, default=default, unit_name=self.unit_name, note=self.note, **kwargs) - + value=value, required=self.required, default=default, unit_name=self.unit_name, note=self.note, + **kwargs) + def status_widget(self, model_instance=None, manager=None, **kwargs): widget_type = kwargs.get('type', 'square') if widget_type == 'square': return SquareStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name}), **kwargs) + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name}), **kwargs) else: return PlainStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name, 'type':'lexical'}), **kwargs) - + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name, 'type': 'lexical'}), + **kwargs) + def get_pseudo_value(self): ran = random() name_list = ['in', 'out', 'Network Traffic'] @@ -631,7 +657,7 @@ return ran.randint(self.scope.get_min(), self.scope.get_max(), inc) else: return ran.randint(1, 100, inc) - + def get_iterable_value(self, bias): if self.has_default(): if bias == 0: @@ -642,61 +668,71 @@ return self.scope.get_iterable_value(bias) else: return bias - + def override(self, scope=None, **kwargs): super(IntegerField, self).override(**kwargs) if scope: self.scope = IntRange(scope) # XXX remove original validator - self.validators.append(validators.ScopeValidator(self.scope)) + self.validators.append(validators.ScopeValidator(self.scope)) def value_to_num(self, obj=None, val=None): value = self._get_val_from_obj_or_val(obj, val) return int(value) + class Int8Field(IntegerField): - type_arguments = {'scope':'-128..127'} + type_arguments = {'scope': '-128..127'} + class Int16Field(IntegerField): - type_arguments = {'scope':'-32768..32767'} + type_arguments = {'scope': '-32768..32767'} + class Int32Field(IntegerField): type_name = 'int32' - type_arguments = {'scope':'-2147483648..2147483647'} + type_arguments = {'scope': '-2147483648..2147483647'} + class Int64Field(IntegerField): type_name = 'int64' - type_arguments = {'scope':'-9223372036854775808..9223372036854775807'} + type_arguments = {'scope': '-9223372036854775808..9223372036854775807'} + class Uint8Field(IntegerField): type_name = 'uint8' - type_arguments = {'scope':'0..255'} + type_arguments = {'scope': '0..255'} + class Uint16Field(IntegerField): type_name = 'uint16' - type_arguments = {'scope':'0..65535'} + type_arguments = {'scope': '0..65535'} + class Uint32Field(IntegerField): type_name = 'uint32' - type_arguments = {'scope':'0..4294967295'} + type_arguments = {'scope': '0..4294967295'} + class Uint64Field(IntegerField): type_name = 'uint64' - type_arguments = {'scope':'0..18446744073709551615L'} + type_arguments = {'scope': '0..18446744073709551615L'} + class PortField(Uint16Field): type_name = 'port' + class UnionField(Field): type_name = 'UnionField' - + def __init__(self, verbose_name=None, name=None, fields={}, **kwargs): super(UnionField, self).__init__(verbose_name, name, **kwargs) self.fields = fields self.field_list = [] assert self.fields != {}, 'The UnionField must not contain empty field dict' - - @property + + @property def unit_name(self): # XXX should specially implement for UnionField # all fields should have the same unit_names @@ -709,8 +745,8 @@ except StopIteration: # never happens because we already checked the dict non-empty return '' - - @property + + @property def value_limit(self): # XXX should specially implement for UnionField # all fields should have the same value_limit @@ -730,7 +766,7 @@ # first it must be a dictionary if type(value) is not dict: raise exceptions.ValidationError('UnionField\'s value must be a dictionary') - + # copy it into a new dict new_value = {} for field_name, field in self.fields.iteritems(): @@ -738,14 +774,13 @@ if field_name in value: new_value[field_name] = field.to_python(value[field_name]) return new_value - + raise exceptions.ValidationError('UnionField does not have one of the sub field key') - def validate(self, value, model_instance): for field_name, field in self.fields.iteritems(): if field_name in value: - field.validate(value[field_name], model_instance) + field.validate(value[field_name], model_instance) return raise exceptions.ValidationError('UnionField does not have one of the sub field key') @@ -760,7 +795,7 @@ def get_sub_fields(self): return self.field_list - + def get_pseudo_value(self): result = {} for key, field in self.fields.iteritems(): @@ -768,7 +803,7 @@ # only one option is enough break return result - + def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) rtn_str = '' @@ -788,17 +823,22 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): return UnionWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, - field_list=self.field_list, required=self.required, model_instance=model_instance, manager=manager, **kwargs) - + field_list=self.field_list, required=self.required, model_instance=model_instance, + manager=manager, **kwargs) + def status_widget(self, model_instance=None, manager=None, **kwargs): # XXX should re-implement for UnionField widget_type = kwargs.get('type', 'square') if widget_type == 'square': return SquareStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name}), fields=self.fields, **kwargs) + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name}), fields=self.fields, + **kwargs) else: return PlainStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name, 'type':'lexical'}), **kwargs) + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name, 'type': 'lexical'}), + **kwargs) def get_iterable_value(self, bias): if self.has_default(): @@ -814,9 +854,9 @@ nd_value = it.next() i += 1 rtn_dict = {} - - #print nd_value - + + # print nd_value + chosen_key = self.fields.keys()[nd_value[0] % len(self.fields.keys())] rtn_dict[chosen_key] = self.fields[chosen_key].get_iterable_value(nd_value[1]) @@ -865,9 +905,12 @@ def value_to_string(self, obj=None, val=None): return str(self.value_to_rest(obj, val)) + class IPv4AddressField(CharField): type_name = 'ip4addr' - type_arguments = {'pattern':'^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$', 'pattern_msg':_('Invalid IP address!')} + type_arguments = { + 'pattern': '^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$', + 'pattern_msg': _('Invalid IP address!')} description = _("IPv4 address") def get_internal_type(self): @@ -894,7 +937,8 @@ def value_to_string(self, obj=None, val=None): return self.value_to_rest(obj, val) - + + class IPAddressField(UnionField): type_name = 'ipaddr' type_arguments = {} @@ -903,11 +947,14 @@ def __init__(self, verbose_name=None, name=None, optional=False, **kwargs): if 'default' in kwargs and type(kwargs['default']) in (list, tuple): - super(IPAddressField, self).__init__(verbose_name, name, fields={'ipv4':IPv4AddressField(verbose_name='IPv4', optional=optional, default=kwargs['default'][0]), - 'ipv6':IPv6AddressField(verbose_name='IPv6', optional=optional, default=kwargs['default'][1])}, optional=optional, **kwargs) + super(IPAddressField, self).__init__(verbose_name, name, fields={ + 'ipv4': IPv4AddressField(verbose_name='IPv4', optional=optional, default=kwargs['default'][0]), + 'ipv6': IPv6AddressField(verbose_name='IPv6', optional=optional, default=kwargs['default'][1])}, + optional=optional, **kwargs) else: - super(IPAddressField, self).__init__(verbose_name, name, fields={'ipv4':IPv4AddressField(verbose_name='IPv4', optional=optional), - 'ipv6':IPv6AddressField(verbose_name='IPv6', optional=optional)}, optional=optional, **kwargs) + super(IPAddressField, self).__init__(verbose_name, name, fields={ + 'ipv4': IPv4AddressField(verbose_name='IPv4', optional=optional), + 'ipv6': IPv6AddressField(verbose_name='IPv6', optional=optional)}, optional=optional, **kwargs) def get_internal_type(self): return "IPAddressField" @@ -915,13 +962,13 @@ def to_python(self, value): if self.is_empty_value(value): value = self.get_default(None) - + assert value is not None if type(value) in (str, str): if is_ipv4(value): - return {'ipv4':value} + return {'ipv4': value} elif is_ipv6(value): - return {'ipv6':value} + return {'ipv6': value} elif type(value) is dict: return value msg = self.error_messages['invalid'] % str(value) @@ -956,9 +1003,12 @@ def value_to_string(self, obj=None, val=None): return self.value_to_rest(obj, val) + class IPv6AddressField(CharField): type_name = 'ip6addr' - type_arguments = {'pattern':'^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$', 'pattern_msg':_('Invalid IP address!')} + type_arguments = { + 'pattern': '^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$', + 'pattern_msg': _('Invalid IP address!')} description = _("IPv6 address") def get_internal_type(self): @@ -974,17 +1024,19 @@ return data[0] else: return '' - + def value_to_string(self, obj=None, val=None): return self.value_to_rest(obj, val) + class AssoField2(Field): type_name = 'asso' description = _("New Association Field") type_arguments = {} # comp: compatibility - def __init__(self, verbose_name=None, name=None, tgt=None, mul='*', comp={}, pos=None, monitorable=False, mutex=True, scroll=False, **kwargs): + def __init__(self, verbose_name=None, name=None, tgt=None, mul='*', comp={}, pos=None, monitorable=False, + mutex=True, scroll=False, **kwargs): super(AssoField2, self).__init__(verbose_name, name, **kwargs) # setup asso validator # XXX @@ -997,19 +1049,19 @@ # if this field can monitorable, but the tgt_field is not monitorable, the tgt model will hide and the remote model of the tgt model will show. # like: slb_group-->member-->rs, when open slb_group, will not display member info, but display rs info. self.monitorable = monitorable - self.mutex = mutex # If there are multiple assos, values can only belong to one same asso. - + self.mutex = mutex # If there are multiple assos, values can only belong to one same asso. + self.asso_count = len(self.tgt) - self.asso = [None] * self.asso_count # a list of one or multiple associations - self.tgt_field = [None] * self.asso_count # a list of target fields in one or multiple associations - self.this_end = [None] * self.asso_count # a list of local ends in one or multiple associations - self.target_end = [None] * self.asso_count # a list of target ends in one or multiple associations - self.target_asso_idx = [0] * self.asso_count # asso_idx in target field + self.asso = [None] * self.asso_count # a list of one or multiple associations + self.tgt_field = [None] * self.asso_count # a list of target fields in one or multiple associations + self.this_end = [None] * self.asso_count # a list of local ends in one or multiple associations + self.target_end = [None] * self.asso_count # a list of target ends in one or multiple associations + self.target_asso_idx = [0] * self.asso_count # asso_idx in target field # for asso field, if the field is override, we need to re find the idx and tgt_idx when register_association self.is_override = False - + def type_default(self, instance): - if self.is_external_component(): # asso_count must be 1 + if self.is_external_component(): # asso_count must be 1 if instance: tgt_pk = {} # from target model, the asso_count might be more than 1 @@ -1019,14 +1071,14 @@ if tgt_field.target_end[idx].model == self.model: tgt_to_self_idx = idx break - tgt_pk[tgt_field.name] = [dict_combine(instance.pk_dict(), {'_asso_idx':tgt_to_self_idx})] + tgt_pk[tgt_field.name] = [dict_combine(instance.pk_dict(), {'_asso_idx': tgt_to_self_idx})] normalize_asso_value_item(tgt_pk) return [tgt_pk] else: - return None # still returns an empty value if instance is None + return None # still returns an empty value if instance is None else: return [] - + def default_override(self): # compatibility is defautly empty for sub-classes inheritting the field self.mul = '' @@ -1048,15 +1100,15 @@ self.comp = comp self.pos = pos self.monitorable = monitorable - self.mutex = mutex # If there are multiple assos, values can only belong to one same asso. + self.mutex = mutex # If there are multiple assos, values can only belong to one same asso. self.asso_count = len(self.tgt) - self.asso = [None] * self.asso_count # a list of one or multiple associations - self.tgt_field = [None] * self.asso_count # a list of target fields in one or multiple associations - self.this_end = [None] * self.asso_count # a list of local ends in one or multiple associations - self.target_end = [None] * self.asso_count # a list of target ends in one or multiple associations - self.target_asso_idx = [0] * self.asso_count # asso_idx in target field + self.asso = [None] * self.asso_count # a list of one or multiple associations + self.tgt_field = [None] * self.asso_count # a list of target fields in one or multiple associations + self.this_end = [None] * self.asso_count # a list of local ends in one or multiple associations + self.target_end = [None] * self.asso_count # a list of target ends in one or multiple associations + self.target_asso_idx = [0] * self.asso_count # asso_idx in target field self.is_override = True - + def register_target_asso_while_loading(self, tgt_field, asso_index): # asso_index is the index of asso in local field # now find the index of asso in target field @@ -1073,9 +1125,10 @@ break tgt_asso_index += 1 if tgt_asso_index >= tgt_field.asso_count: - raise exceptions.ModelDefinitionError('Could not find target matching association, %s, %s' % (self.path, tgt_field.tgt)) + raise exceptions.ModelDefinitionError( + 'Could not find target matching association, %s, %s' % (self.path, tgt_field.tgt)) new_asso = Association2(self, tgt_field, asso_index, tgt_asso_index) - + def register_association(self, association, tgt_field, asso_index, target_asso_index): if self.is_override: # this field is override field, the tgt list may different with parent field, so re find the idx. @@ -1099,13 +1152,13 @@ self.target_asso_idx[asso_index] = target_asso_index # register to model self.model._meta.add_association(association, self) - + def get_value_length(self, val, asso_idx): if self.asso_count == 1: return len(val) else: return len([a for a in val if get_value_asso_idx(a) == asso_idx]) - + def value_to_string(self, obj=None, val=None): val = self._get_val_from_obj_or_val(obj, val) val = copy.deepcopy(val) @@ -1135,7 +1188,8 @@ if self.is_external_component(): rtn_dict = target_instance.get_rest_field_dict(withstats=False, exclude=[ignore_field]) elif self.is_composition_asso(): - rtn_dict = target_instance.get_rest_field_dict(withstats=False, group='default', exclude=[ignore_field]) + rtn_dict = target_instance.get_rest_field_dict(withstats=False, group='default', + exclude=[ignore_field]) rtn_dict['pk'] = target_instance.pk_str() rtn.append(rtn_dict) elif self.is_composition_asso_target(): @@ -1143,7 +1197,7 @@ rtn = self.value_to_string(obj, val) else: rtn = '' - else:#For non-composition AssoField, return simply a list of instance_id strings. + else: # For non-composition AssoField, return simply a list of instance_id strings. rtn = [] for each in val: if '_asso_idx' in each.keys(): @@ -1160,7 +1214,7 @@ if value: for each in value: copied_value = dict(each) - asso_idx = get_value_asso_idx(copied_value, pop=True) # don't change the original value + asso_idx = get_value_asso_idx(copied_value, pop=True) # don't change the original value if style: display_val = self.target_end[asso_idx].model.repr_by_pk(copied_value) + '     ' url = get_ajax_url_by_pk(self.target_end[asso_idx].model, copied_value, 'box_instance_config') @@ -1176,7 +1230,7 @@ if self.scroll: return '
    ' + ''.join(str_list) + '
    ' return ''.join(str_list) - + def to_python(self, value): if value is None: value = self.get_default(None) @@ -1191,7 +1245,8 @@ else: each['_asso_idx'] = int(each['_asso_idx']) return value - elif type(value) in (str, str):#just for restful api, the full format is :, and format is -- + elif type(value) in (str, + str): # just for restful api, the full format is :, and format is -- def instance_id_to_pk(instance_id, target_model): l1, l2 = [], [] l1 = [pk_field.name for pk_field in target_model._meta.pk_set] @@ -1203,13 +1258,14 @@ else: if len(l1) == 1: l2 = [instance_id] - else:#more than one pk model - l2 = instance_id.split('-') + else: # more than one pk model + l2 = instance_id.split('-') if len(l1) == len(l2): - pk_dict = dict(zip(l1,l2)) + pk_dict = dict(zip(l1, l2)) return pk_dict else: return None + if ":" in value: l = value.split(":") li = l[1].split(".") @@ -1228,7 +1284,7 @@ return [tgt_pk] else: raise exceptions.ValidationError('AssoField\'s value must be dict list') - + def validate(self, value, model_instance): """ Check that the instance's value obeys the @@ -1236,10 +1292,11 @@ """ for idx in range(self.asso_count): rtn = self.asso[idx].check_multiplicity(self.pos, self.get_value_length(value, idx)) - if rtn: # pass validation only if one asso passes + if rtn: # pass validation only if one asso passes return - - raise exceptions.ValidationError('Multiplicity validation error for field %s, association %s' % (self, self.asso)) + + raise exceptions.ValidationError( + 'Multiplicity validation error for field %s, association %s' % (self, self.asso)) # Check whether this field is a primary key def post_validation(self, value, model_instance): @@ -1249,9 +1306,9 @@ """ # A model instance should not have other attributes except the field values. Or the Python pickle interface used by # Psuedo managers won't work well. In this case, we might implement a method "asso_links" if necessary. - #model_instance.asso_links[self.asso] = value + # model_instance.asso_links[self.asso] = value pass - + # notice that self.ref is the ref name for target model cls or asso cls # # pk_dict is to pass into the target end in case he needs it to reference back @@ -1278,8 +1335,9 @@ target_asso_field = target_meta.associations[self.asso] fixed_assos[target_asso_field.name] = pk_dict - #print "===--- %s, %s, %s" % (self.tgt_field.model, self.tgt_field.model._meta.pk_set, fixed_assos) - return self.tgt_field.model._meta.gen_test_dataset_for_fieldset(self.target_end.model._meta.pk_set, count=1, fixed_assos=fixed_assos) + # print "===--- %s, %s, %s" % (self.tgt_field.model, self.tgt_field.model._meta.pk_set, fixed_assos) + return self.tgt_field.model._meta.gen_test_dataset_for_fieldset(self.target_end.model._meta.pk_set, count=1, + fixed_assos=fixed_assos) def static_widget(self, model_instance=None, manager=None, context_value=None, context_repr=None): context_obj_dict = None @@ -1289,15 +1347,19 @@ if context_value and manager: assert type(context_value[0]) is dict # context_value[0] should contain "_asso_idx" or it is 0 - copied_value = dict(context_value[0]) # the original context_value should be directly transported to frontend + copied_value = dict( + context_value[0]) # the original context_value should be directly transported to frontend asso_idx = get_value_asso_idx(copied_value, pop=True) # this obj_dict is designed for "get_asso_value" interface at frontend (to implement asso-value conditions) - context_obj_dict = self.target_end[asso_idx].model.get_manager(manager._session).get(copied_value).get_field_dict(withstats=False) + context_obj_dict = self.target_end[asso_idx].model.get_manager(manager._session).get( + copied_value).get_field_dict(withstats=False) except KeyError: raise - - return StaticFormWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, asso_idx=asso_idx, context_value=context_value, context_repr=context_repr, context_obj_dict=context_obj_dict) + return StaticFormWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, asso_idx=asso_idx, + context_value=context_value, context_repr=context_repr, + context_obj_dict=context_obj_dict) + def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name, None) if model_instance else None target_models = [self.target_end[idx].model for idx in range(self.asso_count)] @@ -1305,7 +1367,7 @@ # at this case, must provide model_instance # this means, this widget should never be at an "add" form if not model_instance: - return ReferenceWidget() # just for doc system XXX + return ReferenceWidget() # just for doc system XXX external_obj = self.get_external_component_obj(model_instance) return ReferenceWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, link=external_obj.get_ajax_url('box_instance_config'), **kwargs) @@ -1320,68 +1382,82 @@ # at the case of composition, must provide model_instance # this means, this widget should never be at an "add" form if not model_instance: - return AssoListBox() # just for doc system XXX - initial_filters = [{self.tgt_field[idx].name: [dict_combine(model_instance.pk_dict(), {'_asso_idx':self.target_asso_idx[idx]})]} for idx in range(self.asso_count)] + return AssoListBox() # just for doc system XXX + initial_filters = [{self.tgt_field[idx].name: [ + dict_combine(model_instance.pk_dict(), {'_asso_idx': self.target_asso_idx[idx]})]} for idx in + range(self.asso_count)] self_pk = model_instance.pk_dict() self_repr = model_instance.repr() - + target_muls = [repr(self.target_end[idx].mul) for idx in range(self.asso_count)] choices_urls = [target_models[idx]._meta.get_ajax_url('list') for idx in range(self.asso_count)] - popup_add_urls = [(target_models[idx]._meta.get_ajax_url('box_model_add_wizard') if target_models[idx]._meta.managable else '') for idx in range(self.asso_count)] - popup_del_urls = [(target_models[idx]._meta.get_ajax_url('delete2') if target_models[idx]._meta.managable else '') for idx in range(self.asso_count)] - popup_edit_urls = [(target_models[idx]._meta.get_ajax_url('box_instance_config') if target_models[idx]._meta.managable else '') for idx in range(self.asso_count)] - + popup_add_urls = [(target_models[idx]._meta.get_ajax_url('box_model_add_wizard') if target_models[ + idx]._meta.managable else '') for idx in range(self.asso_count)] + popup_del_urls = [ + (target_models[idx]._meta.get_ajax_url('delete2') if target_models[idx]._meta.managable else '') for idx + in range(self.asso_count)] + popup_edit_urls = [(target_models[idx]._meta.get_ajax_url('box_instance_config') if target_models[ + idx]._meta.managable else '') for idx in range(self.asso_count)] + if not composition: - return AssoBox(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, required=self.required, asso_count=self.asso_count, + return AssoBox(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, + required=self.required, asso_count=self.asso_count, choices_urls=choices_urls, initial_filters=initial_filters, mutex=self.mutex, - popup_add_urls=popup_add_urls, popup_del_urls=popup_del_urls, popup_edit_urls=popup_edit_urls, target_models=target_models, - composition=composition, target_muls=target_muls, target_fields=self.tgt_field, target_asso_idxs=self.target_asso_idx, + popup_add_urls=popup_add_urls, popup_del_urls=popup_del_urls, + popup_edit_urls=popup_edit_urls, target_models=target_models, + composition=composition, target_muls=target_muls, target_fields=self.tgt_field, + target_asso_idxs=self.target_asso_idx, self_pk=self_pk, self_repr=self_repr, **kwargs) else: if model_instance: action_options = { - 'pk_str': model_instance.pk_str(), - 'action_check_url':model_instance.get_ajax_url('asso_action_check', {'field_name':self.name}) - } + 'pk_str': model_instance.pk_str(), + 'action_check_url': model_instance.get_ajax_url('asso_action_check', {'field_name': self.name}) + } else: action_options = { - 'pk_str': None, - 'action_check_url':'' - } - return AssoListBox(name=self.name, label=self.verbose_name, help_text=self.help_text, required=self.required, asso_count=self.asso_count, + 'pk_str': None, + 'action_check_url': '' + } + return AssoListBox(name=self.name, label=self.verbose_name, help_text=self.help_text, + required=self.required, asso_count=self.asso_count, popup_add_urls=popup_add_urls, composition=composition, - target_models=target_models, target_muls=target_muls, target_fields=self.tgt_field, target_asso_idxs=self.target_asso_idx, - initial_filters=initial_filters, self_pk=self_pk, self_repr=self_repr, actions=self.related_actions(model_instance), action_options=action_options, **kwargs) + target_models=target_models, target_muls=target_muls, target_fields=self.tgt_field, + target_asso_idxs=self.target_asso_idx, + initial_filters=initial_filters, self_pk=self_pk, self_repr=self_repr, + actions=self.related_actions(model_instance), action_options=action_options, + **kwargs) def status_widget(self, model_instance=None, manager=None, **kwargs): - widget_type = kwargs.get('type','list') + widget_type = kwargs.get('type', 'list') if widget_type == 'list': return self.default_widget(model_instance, manager, **kwargs) + class ListField(Field): type_name = 'ListField' type_default = [] - + def __init__(self, verbose_name=None, name=None, fields=None, **kwargs): super(ListField, self).__init__(verbose_name, name, **kwargs) self.fields = fields - + def to_python(self, value): # first it must be a list if type(value) is not list: raise exceptions.ValidationError('ListField\'s value must be a list') - + # copy it into a new list new_value = [] for each_value in value: new_value.append(self.fields.to_python(each_value)) return new_value - + def validate(self, value, model_instance): for each_value in value: self.fields.validate(each_value, model_instance) - + def init_sub_fields(self): # will be called by "contribute_to_class" self.fields._set_attributes_from_name(self.name) @@ -1397,7 +1473,7 @@ return [] else: return value - + def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) if style: @@ -1413,18 +1489,20 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): return ListWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, - sub_field=self.fields, required=self.required, model_instance=model_instance, manager=manager, **kwargs) + sub_field=self.fields, required=self.required, model_instance=model_instance, manager=manager, + **kwargs) + class GroupField(Field): type_name = 'GroupField' - + def __init__(self, verbose_name=None, name=None, fields={}, **kwargs): super(GroupField, self).__init__(verbose_name, name, **kwargs) self.fields = fields assert self.fields != {}, 'The GroupField must not contain empty field dict' self.field_list = [] - - @property + + @property def unit_name(self): # all fields should have the same unit_names try: @@ -1436,8 +1514,8 @@ except StopIteration: # never happens because we already checked the dict non-empty return '' - - @property + + @property def value_limit(self): # all fields should have the same value_limit try: @@ -1459,7 +1537,7 @@ if not value and not not self.optional: return {} - + # copy it into a new dict new_value = {} for field_name, field in self.fields.iteritems(): @@ -1477,14 +1555,14 @@ if field.condition: self.optional = True if field_name in value: - field.validate(value[field_name], model_instance) - if field_name not in value and not self.optional: + field.validate(value[field_name], model_instance) + if field_name not in value and not self.optional: raise exceptions.ValidationError('GroupField has incomplete value - %s is missing' % field_name) - field.validate(value[field_name], model_instance) + field.validate(value[field_name], model_instance) def init_sub_fields(self): # will be called by "contribute_to_class" - self.field_list=[] + self.field_list = [] for sub_field_name, sub_field in self.fields.iteritems(): sub_field._set_attributes_from_name(sub_field_name) sub_field.nesting_field = self @@ -1493,7 +1571,7 @@ def get_sub_fields(self): return self.field_list - + def get_pseudo_value(self): result = {} for key, field in self.fields.iteritems(): @@ -1504,7 +1582,7 @@ value = self._get_val_from_obj_or_val(obj, val) rtn_str = '' for key, field in self.fields.iteritems(): - rtn_str += key+': '+field.value_to_display(val=value[key])+' ' + rtn_str += key + ': ' + field.value_to_display(val=value[key]) + ' ' rtn_str = html.escape(rtn_str) return rtn_str @@ -1519,20 +1597,25 @@ for each in sub_val: result.append([each[0], each[1]]) return result - + def status_widget(self, model_instance=None, manager=None, **kwargs): widget_type = kwargs.get('type', 'square') if widget_type == 'square': return SquareStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name}), fields=self.fields, **kwargs) + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name}), fields=self.fields, + **kwargs) else: return PlainStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, ajax_url=model_instance.get_ajax_url('field', {'field':self.name, 'type':'lexical'}), **kwargs) + unit_name=self.unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name, 'type': 'lexical'}), + **kwargs) def default_widget(self, model_instance=None, manager=None, **kwargs): return GroupWidget(name=self.name, label=self.verbose_name, help_text=self.help_text, - field_list=self.field_list, required=self.required, model_instance=model_instance, manager=manager, **kwargs) - + field_list=self.field_list, required=self.required, model_instance=model_instance, + manager=manager, **kwargs) + def get_iterable_value(self, bias): if self.has_default(): if bias == 0: @@ -1546,20 +1629,23 @@ nd_value = it.next() i += 1 rtn_dict = {} - + i = 0 for sub_field_name, sub_field in self.fields.iteritems(): rtn_dict[sub_field_name] = sub_field.get_iterable_value(nd_value[i]) i += 1 - + return rtn_dict - + + class PortRangeField(GroupField): def __init__(self, verbose_name=None, name=None, **kwargs): - super(PortRangeField, self).__init__(verbose_name, name, fields={'low':PortField(verbose_name=_('Start Port')), 'high':PortField(verbose_name=_('End Port'))}, **kwargs) - + super(PortRangeField, self).__init__(verbose_name, name, fields={'low': PortField(verbose_name=_('Start Port')), + 'high': PortField(verbose_name=_('End Port'))}, + **kwargs) + def custom_display(self, val): - rtn_str = str(val['low'])+'-'+str(val['high']) + rtn_str = str(val['low']) + '-' + str(val['high']) return rtn_str def get_iterable_value(self, bias): @@ -1571,44 +1657,59 @@ from_value = 80 to_value = 90 while to_value < 65535: - rtn = {'low':from_value, 'high':to_value} + rtn = {'low': from_value, 'high': to_value} from_value = to_value to_value += 10 return rtn + class IPv4RangeField(GroupField): def __init__(self, verbose_name=None, name=None, **kwargs): - super(IPv4RangeField, self).__init__(verbose_name, name, fields={'low':IPv4AddressField(verbose_name=_('Start IP')), 'high':IPv4AddressField(verbose_name=_('End IP'))}, **kwargs) + super(IPv4RangeField, self).__init__(verbose_name, name, + fields={'low': IPv4AddressField(verbose_name=_('Start IP')), + 'high': IPv4AddressField(verbose_name=_('End IP'))}, **kwargs) def custom_display(self, val): - rtn_str = self.fields['low'].value_display(val=val['low']) + ' - ' + self.fields['high'].value_to_display(val=val['high']) + rtn_str = self.fields['low'].value_display(val=val['low']) + ' - ' + self.fields['high'].value_to_display( + val=val['high']) return rtn_str + class IPv6RangeField(GroupField): def __init__(self, verbose_name=None, name=None, **kwargs): - super(IPv6RangeField, self).__init__(verbose_name, name, fields={'low':IPv6AddressField(verbose_name=_('Start IP')), 'high':IPv6AddressField(verbose_name=_('End IP'))}, **kwargs) - + super(IPv6RangeField, self).__init__(verbose_name, name, + fields={'low': IPv6AddressField(verbose_name=_('Start IP')), + 'high': IPv6AddressField(verbose_name=_('End IP'))}, **kwargs) + def custom_display(self, val): - rtn_str = self.fields['low'].value_display(val=val['low']) + ' - ' + self.fields['high'].value_display(val=val['high']) + rtn_str = self.fields['low'].value_display(val=val['low']) + ' - ' + self.fields['high'].value_display( + val=val['high']) return rtn_str + class URLField(CharField): type_name = 'url' - type_arguments = {'pattern': '^(http://|https://|ftp://|ldap://)', 'pattern_msg':'Invalid URL!'} + type_arguments = {'pattern': '^(http://|https://|ftp://|ldap://)', 'pattern_msg': 'Invalid URL!'} + def __init__(self, verbose_name=None, name=None, length=None, **kwargs): super(URLField, self).__init__(verbose_name, name, **kwargs) + class ImportURLField(CharField): type_name = 'importurl' - type_arguments = {'pattern': '^(http://|https://|ftp://)', 'pattern_msg':'Invalid URL!'} + type_arguments = {'pattern': '^(http://|https://|ftp://)', 'pattern_msg': 'Invalid URL!'} + def __init__(self, verbose_name=None, name=None, style='long', **kwargs): super(ImportURLField, self).__init__(verbose_name, name, **kwargs) self.style = style + class ImportLocalFileField(Field): type_name = 'importlocalfile' - type_arguments = {'pattern': '^(http://|https://|ftp://)', 'pattern_msg':'Invalid URL!'} - def __init__(self, verbose_name=None, name=None, button_text=None, maxFileSize=None, acceptFileSuffix=None, **kwargs): + type_arguments = {'pattern': '^(http://|https://|ftp://)', 'pattern_msg': 'Invalid URL!'} + + def __init__(self, verbose_name=None, name=None, button_text=None, maxFileSize=None, acceptFileSuffix=None, + **kwargs): super(ImportLocalFileField, self).__init__(verbose_name, name, **kwargs) self.button_text = button_text self.maxFileSize = maxFileSize @@ -1617,53 +1718,75 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name) if model_instance else None return ImportLocalFile(name=self.name, label=self.verbose_name, help_text=self.help_text, value=value, - validators=self.validators, button_text=self.button_text, maxFileSize=self.maxFileSize, acceptFileSuffix=self.acceptFileSuffix, **kwargs) + validators=self.validators, button_text=self.button_text, maxFileSize=self.maxFileSize, + acceptFileSuffix=self.acceptFileSuffix, **kwargs) + class RegexField(CharField): def __init__(self, verbose_name=None, name=None, **kwargs): super(RegexField, self).__init__(verbose_name, name, **kwargs) + class EmailField(CharField): type_name = 'email' - type_arguments = {'pattern':'[^@]+@[^@]+\.[^@]+', 'pattern_msg':'Enter a valid e-mail address.'} + type_arguments = {'pattern': '[^@]+@[^@]+\.[^@]+', 'pattern_msg': 'Enter a valid e-mail address.'} + def __init__(self, verbose_name=None, name=None, **kwargs): super(EmailField, self).__init__(verbose_name, name, **kwargs) + class DurationField(IntegerField): def __init__(self, verbose_name=None, name=None, unit_name=_('Seconds'), **kwargs): super(DurationField, self).__init__(verbose_name, name, **kwargs) self.unit_name = unit_name + class HostnameField(CharField): - type_arguments = {'pattern':'^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$|^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$|^', 'pattern_msg':_('Invalid IP address or hostname')} + type_arguments = { + 'pattern': '^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$|^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$|^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$|^', + 'pattern_msg': _('Invalid IP address or hostname')} + def __init__(self, verbose_name=None, name=None, **kwargs): super(HostnameField, self).__init__(verbose_name, name, **kwargs) + class NetmaskField(CharField): - type_arguments = {'pattern':'^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$|^([0-9]|[1-2][0-9]|3[0-2])$', 'pattern_msg':_('Invalid Netmask')} + type_arguments = { + 'pattern': '^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9]{1,2})){3}$|^([0-9]|[1-2][0-9]|3[0-2])$', + 'pattern_msg': _('Invalid Netmask')} + def __init__(self, verbose_name=None, name=None, **kwargs): super(NetmaskField, self).__init__(verbose_name, name, **kwargs) + class NetprefixField(IntegerField): - type_arguments = {'scope':'0..128'} + type_arguments = {'scope': '0..128'} + def __init__(self, verbose_name=None, name=None, **kwargs): super(NetprefixField, self).__init__(verbose_name, name, **kwargs) - + + class IntRangeField(Field): def __init__(self, verbose_name=None, name=None, **kwargs): super(IntRangeField, self).__init__(verbose_name, name, **kwargs) + class MACAddressField(CharField): - type_arguments = {'pattern':'^([0-9a-fA-F]{1,2})(([/\s:][0-9a-fA-F]{1,2}){5})$', 'pattern_msg':'Invalid MAC address'} + type_arguments = {'pattern': '^([0-9a-fA-F]{1,2})(([/\s:][0-9a-fA-F]{1,2}){5})$', + 'pattern_msg': 'Invalid MAC address'} + def __init__(self, verbose_name=None, name=None, **kwargs): super(MACAddressField, self).__init__(verbose_name, name, **kwargs) + class AutoIDField(Int32Field): - type_default = -1 # -1 means not yet initialized value + type_default = -1 # -1 means not yet initialized value + def __init__(self, verbose_name=None, name=None, **kwargs): super(AutoIDField, self).__init__(verbose_name, name, **kwargs) self.configurable = False + class PasswordField(CharField): def __init__(self, verbose_name=None, name=None, confirm=True, **kwargs): super(PasswordField, self).__init__(verbose_name, name, **kwargs) @@ -1679,15 +1802,19 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name) if model_instance else None default = self.get_default(model_instance) - return PasswordInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, + return PasswordInput(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, required=self.required, default=default, confirm=self.confirm, **kwargs) + class LDAPDNField(Field): def __init__(self, verbose_name=None, name=None, **kwargs): super(LDAPDNField, self).__init__(verbose_name, name, **kwargs) + class EnumOrderField(Field): type_name = 'enumorder' + def __init__(self, verbose_name=None, name=None, values=[], **kwargs): super(EnumOrderField, self).__init__(verbose_name, name, **kwargs) self.values = list(values) @@ -1695,7 +1822,7 @@ # the default order is what defined at the schema def type_default(self, instance): return range(len(self.values)) - + def to_python(self, value): if value is None: value = self.get_default(None) @@ -1713,11 +1840,13 @@ return EnumOrder(name=self.name, label=self.verbose_name, help_text=self.help_text, values=self.values, **kwargs) + class StorableField(Field): # Attributes of field types type_name = 'StorableField' - def __init__(self, verbose_name=None, name=None, samp_freq=10, samp_life=7, retrieve_type=KEEPING, fields=None, **kwargs): + def __init__(self, verbose_name=None, name=None, samp_freq=10, samp_life=7, retrieve_type=KEEPING, fields=None, + **kwargs): super(StorableField, self).__init__(verbose_name, name, **kwargs) self.editable = False # Sampling Frequency. default 10 seconds @@ -1730,7 +1859,7 @@ else: self.fields = fields self.cache_expire_time = self.samp_freq - + @property def unit_name(self): if hasattr(self.fields, 'unit_name'): @@ -1743,16 +1872,18 @@ value = self.get_default(None) # delegate to the fields return self.fields.to_python(value) - + def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) return self.fields.value_to_display(obj=None, val=value, style=style, text=text) + """ INPUT: start_time: timestamp, default is one day(86400 seconds) before end_time end_time: timestamp, default is now cal_method: Calculation Method {differencing|keeping|...} """ + def _get_data(self, start_time=None, end_time=None, cal_method=None, instance_pk=''): # get data from MongoDB global MethodList @@ -1762,23 +1893,23 @@ start_time = end_time - 86400.0 if cal_method == None: cal_method = self.retrieve_type.__name__ - + mod = self.model file_name = DIR_STR + mod.__name__ + instance_pk + self.attname + ".rrd" try: - #result = rrdtool.fetch(file_name, 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) + # result = rrdtool.fetch(file_name, 'AVERAGE', '-s', str(int(start_time)), '-e', str(int(end_time)), "--daemon", RRDCACHED_SOCKET) result = [] start = result[0][0] end = result[0][1] freq = result[0][2] - num = (end-start)/freq + num = (end - start) / freq data = [] - TIMEFORMAT='%Y-%m-%d %X' + TIMEFORMAT = '%Y-%m-%d %X' for i in range(num): value = result[2][i][0] if value == None: continue - data.append([time.strftime(TIMEFORMAT, time.localtime(start+i*freq)), self.fields.to_python(value)]) + data.append([time.strftime(TIMEFORMAT, time.localtime(start + i * freq)), self.fields.to_python(value)]) file_err = False except: data = None @@ -1795,9 +1926,9 @@ if value_limit != None: ret['value_limit'] = value_limit if file_err: - ret['error'] = 'Failed to save data for field '+ str(self.verbose_name) + ret['error'] = 'Failed to save data for field ' + str(self.verbose_name) return ret - + def _get_data_old(self, start_time=None, end_time=None, cal_method=None, instance_pk=''): # get data from MongoDB global MethodList @@ -1807,19 +1938,20 @@ start_time = end_time - 86400.0 if cal_method == None: cal_method = self.retrieve_type.__name__ - + db = DBProcess(db_name=self.model._meta.app_label) - #col_name = self.model._meta.path[-1] + # col_name = self.model._meta.path[-1] col_name = '.'.join(self.model._meta.path) - rets = db.select(col_name, {'pk':instance_pk, 'field':self.attname, 'time':{"$gte":start_time, "$lte":end_time}}) - #try: + rets = db.select(col_name, + {'pk': instance_pk, 'field': self.attname, 'time': {"$gte": start_time, "$lte": end_time}}) + # try: return get_method(cal_method).process(self, rets) - #except: + # except: # return None def validate(self, value, model_instance): # delegate to the fields - return self.fields.validate(value, model_instance) + return self.fields.validate(value, model_instance) def init_sub_fields(self): # will be called by "contribute_to_class" @@ -1832,10 +1964,10 @@ def get_sub_fields(self): return self.fields.get_sub_fields() - + def default_widget(self, model_instance=None, manager=None, **kwargs): return self.statistics_widget(model_instance, manager, **kwargs) - + def status_widget(self, model_instance=None, manager=None, **kwargs): widget_type = kwargs.get('type', 'progress') if widget_type in ('progress'): @@ -1848,22 +1980,26 @@ # we need to do a check to see the data is available for this field if not model_instance: # Translators: This should be internal error - return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, error_msg=_('Instance is not available.'), **kwargs) + return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, + error_msg=_('Instance is not available.'), **kwargs) # check manager interface is available try: val = getattr(model_instance, self.attname) except Exception as e: - return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, error_msg=str(e), **kwargs) + return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, + error_msg=str(e), **kwargs) # check webui graph is enabled webui_graph_mod = get_model('apv', ['system', 'AccessControl']) webui_graph_manager = webui_graph_mod.get_manager(get_current_session()) if not webui_graph_manager.all()[0].enable_webui_graph: - return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, error_msg=_('WebUI graph is disabled.'), **kwargs) + return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, + error_msg=_('WebUI graph is disabled.'), **kwargs) # check rrd file is available mod = self.model file_name = DIR_STR + mod.__name__ + model_instance.pk_str() + self.attname + ".rrd" if not os.path.isfile(file_name): - return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, error_msg=_('Statistics Information is not saved.'), **kwargs) + return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, error=True, + error_msg=_('Statistics Information is not saved.'), **kwargs) style = kwargs.pop('style') # get the parent mod @@ -1871,53 +2007,59 @@ while par_mode._meta.parents: par_mode = par_mode._meta.parents.keys()[-1] # to keep the url same with graph selector page, we have to create the url in client side. - book_mark_url = {'base_url':par_mode._meta.get_ajax_url("box_instance_graph"), - 'option':{"field": [self.name], "instance": [str(model_instance.pk_dict())], "instance_repr":model_instance.repr()} + book_mark_url = {'base_url': par_mode._meta.get_ajax_url("box_instance_graph"), + 'option': {"field": [self.name], "instance": [str(model_instance.pk_dict())], + "instance_repr": model_instance.repr()} } - return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, ajax_url=model_instance.get_ajax_url('get_statistics_data', {'keys':json.dumps({'field':[self.attname]})}), - samp_freq=self.samp_freq, unit_name=self.unit_name, style=style, book_mark_url=book_mark_url, **kwargs) + return LineChart(name=self.name, label=self.verbose_name, help_text=self.help_text, + ajax_url=model_instance.get_ajax_url('get_statistics_data', + {'keys': json.dumps({'field': [self.attname]})}), + samp_freq=self.samp_freq, unit_name=self.unit_name, style=style, book_mark_url=book_mark_url, + **kwargs) def get_pseudo_value(self): return self.fields.get_pseudo_value() - + @property def value_limit(self): if hasattr(self.fields, 'value_limit'): return self.fields.value_limit else: return None - + def value_to_num(self, obj=None, val=None): value = self._get_val_from_obj_or_val(obj, val) return self.fields.value_to_num(val=value) - + def get_iterable_value(self, bias): return self.fields.get_iterable_value(bias) - + + class ByteField(IntegerField): type_name = "Byte" - + def __init__(self, verbose_name=None, name=None, unit_name=_('Bytes'), **kwargs): super(ByteField, self).__init__(verbose_name, name, **kwargs) self.unit_name = unit_name + class PercentField(Field): # value of this type of field is a dict: # {'percent':25, 'num':5, 'total':20} type_name = "Percent" type_arguments = {} - + def __init__(self, verbose_name=None, name=None, unit_name=_('%'), num_unit_name='', **kwargs): super(PercentField, self).__init__(verbose_name, name, **kwargs) self.unit_name = unit_name self.num_unit_name = num_unit_name - + def to_python(self, value): if value is None: value = self.get_default(None) if type(value) in (int, float): - new_value = {'percent':value, 'num':value, 'total':100} + new_value = {'percent': value, 'num': value, 'total': 100} elif type(value) is dict: new_value = value @@ -1928,12 +2070,12 @@ raise exceptions.ValidationError('PercentField does not have complete value') if 'percent' not in new_value: - new_value['percent'] = new_value['num']/new_value['total'] + new_value['percent'] = new_value['num'] / new_value['total'] return new_value def custom_display(self, val): return str(val.get('percent', '0')) + str(self.unit_name) - + def validate(self, value, model_instance): pass @@ -1942,20 +2084,21 @@ if widget_type == 'square': return Gauge(name=self.name, label=self.verbose_name, help_text=self.help_text, unit_name=self.unit_name, num_unit_name=self.num_unit_name, - ajax_url=model_instance.get_ajax_url('field', {'field':self.name}), **kwargs) + ajax_url=model_instance.get_ajax_url('field', {'field': self.name}), **kwargs) elif widget_type == 'progress': return BarStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, unit_name=self.unit_name, num_unit_name=self.num_unit_name, - ajax_url=model_instance.get_ajax_url('field', {'field':self.name}), **kwargs) + ajax_url=model_instance.get_ajax_url('field', {'field': self.name}), **kwargs) else: return PlainStatus(name=self.name, label=self.verbose_name, help_text=self.help_text, - unit_name=self.unit_name, num_unit_name=self.num_unit_name, - ajax_url=model_instance.get_ajax_url('field', {'field':self.name, 'type':'lexical'}), **kwargs) - + unit_name=self.unit_name, num_unit_name=self.num_unit_name, + ajax_url=model_instance.get_ajax_url('field', {'field': self.name, 'type': 'lexical'}), + **kwargs) + def get_pseudo_value(self): ran = random() num = ran.randint(0, 100) - return {'percent':num, 'num':num, 'total':100} + return {'percent': num, 'num': num, 'total': 100} def value_to_num(self, obj=None, val=None): value = self._get_val_from_obj_or_val(obj, val) @@ -1963,13 +2106,13 @@ return value.get("percent") except: return None - - + @property def value_limit(self): # temporarily return 0-100 - return {'min':0, 'max':100} + return {'min': 0, 'max': 100} + class DateTimeField(Field): # Attributes of field types type_name = 'datetime' @@ -1983,8 +2126,10 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name) if model_instance else None - return DateTimeInput(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, **kwargs) - + return DateTimeInput(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, **kwargs) + + class TimezoneField(Field): type_name = 'timezone' @@ -1996,8 +2141,10 @@ def default_widget(self, model_instance=None, manager=None, **kwargs): value = getattr(model_instance, self.name) if model_instance else None - return TimezoneSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, **kwargs) + return TimezoneSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, **kwargs) + class CLITextField(Field): # Attributes of field types type_name = 'clitext' @@ -2012,11 +2159,12 @@ def to_python(self, value): if value is None: value = self.get_default(None) - if isinstance(value, str): # or value is None: (value is never None here) + if isinstance(value, str): # or value is None: (value is never None here) return value return smart_str(value) -#values: ['__field:xxxx', '__value:xxxx'] + +# values: ['__field:xxxx', '__value:xxxx'] class DynamicEnumField(Field): # Attributes of field types type_name = 'dynamic_enum' @@ -2035,22 +2183,23 @@ each_rtn_value = list(each_value) # extend the list with two empty strings so that its length # is always bigger than 3 - each_rtn_value.extend(['','']) + each_rtn_value.extend(['', '']) # XXX self.model may be None when this is a sub field inside a GroupField/UnionField if self.model and self.model._meta.document: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']/choices/choice[@value='"+str(each_rtn_value[0])+"']" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']/choices/choice[@value='" + str( + each_rtn_value[0]) + "']" choice_ele = self.model._meta.document.find(xpath) if choice_ele: each_rtn_value[1] = choice_ele.findtext("./verbose_name", default=each_rtn_value[1]) each_rtn_value[2] = choice_ele.findtext("./help_text", default=each_rtn_value[2]) rtn_values.append(each_rtn_value) - + return rtn_values @values.setter def values(self, new_value): - self._values = values - + self._values = new_value + def get_values(self, base_manager=None, base_pk_list=None): ret = [] session = get_current_session() @@ -2080,9 +2229,9 @@ if type(sub_value) in (str, str): sub_value_list = sub_value.split(', ') for each in sub_value_list: - ret.append([each, field.value_to_display(val=each)]) - else: - ret.append([sub_value, field.value_to_display(val=sub_value)]) + ret.append([each, field.value_to_display(val=each)]) + else: + ret.append([sub_value, field.value_to_display(val=sub_value)]) if key_type == '__value': ret.append([key_value, key_value]) return ret @@ -2119,7 +2268,6 @@ return Select(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, value=value, choices=values, default=self.default, **kwargs) - def get_iterable_value(self, bias): if self.has_default(): if bias == 0: @@ -2127,7 +2275,9 @@ else: bias -= 1 length = len(self.values) - return self.values[bias%length][0] + return self.values[bias % length][0] + + class DynamicMultiEnumField(Field): # Attributes of field types type_name = 'multienum' @@ -2147,10 +2297,11 @@ each_rtn_value = list(each_value) # extend the list with two empty strings so that its length # is always bigger than 3 - each_rtn_value.extend(['','']) + each_rtn_value.extend(['', '']) # XXX self.model may be None when this is a sub field inside a GroupField/UnionField if self.model and self.model._meta.document: - xpath = "./fieldgrp[@name='"+self.group.name+"']/field[@name='"+self.name+"']/choices/choice[@value='"+str(each_rtn_value[0])+"']" + xpath = "./fieldgrp[@name='" + self.group.name + "']/field[@name='" + self.name + "']/choices/choice[@value='" + str( + each_rtn_value[0]) + "']" choice_ele = self.model._meta.document.find(xpath) if choice_ele: each_rtn_value[1] = choice_ele.findtext("./verbose_name", default=each_rtn_value[1]) @@ -2161,7 +2312,7 @@ @values.setter def values(self, new_value): - self._values = values + self._values = new_value def get_values(self, base_manager=None, base_pk_list=None): ret = [] @@ -2192,9 +2343,9 @@ if type(sub_value) in (str, str): sub_value_list = sub_value.split(', ') for each in sub_value_list: - ret.append([each, field.value_to_display(val=each)]) - else: - ret.append([sub_value, field.value_to_display(val=sub_value)]) + ret.append([each, field.value_to_display(val=each)]) + else: + ret.append([sub_value, field.value_to_display(val=sub_value)]) if key_type == '__value': ret.append([key_value, key_value]) return ret @@ -2210,7 +2361,7 @@ if item[0] == each or str(item[0]) == each: found = True new_value.append(item[0]) - break # to next value + break # to next value if not found: raise exceptions.ValidationError(_(u'The value is not one of the valid value for the field.')) return new_value @@ -2222,7 +2373,7 @@ for item in self.values: if item[0] == each: new_value.append(item[1]) - break # to next value + break # to next value return ', '.join(new_value) def _get_choices(self, include_blank=True, blank_choice=BLANK_CHOICE_DASH): @@ -2241,5 +2392,6 @@ if 'base_pk_list' in kwargs: base_pk_list = [eval(each) if each else "" for each in kwargs['base_pk_list'] if each] values = self.get_values(manager, base_pk_list) - return MultiSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, validators=self.validators, + return MultiSelect(name=self.name, label=self.verbose_name, help_text=self.help_text, + validators=self.validators, value=value, choices=values, default=[], **kwargs) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py (working copy) @@ -1,26 +1,31 @@ +import errno +import re +import socket +import subprocess +import threading +import time from collections import OrderedDict +from filecmp import cmp + from django.conf import settings -import socket, sys, time, re -from hive.utils import anlog, aninfo, anerror, andebug, escapeshellarg, HiveWebPipe, _thread_locals, get_current_session -from hive.exceptions import ModelQueryException + from djproject.an_settings import FAST_CLI_CMD -import subprocess -import threading -import json -import errno +from hive.exceptions import ModelQueryException +from hive.utils import anerror, andebug, escapeshellarg, HiveWebPipe, _thread_locals, get_current_session WEBUI_AGENT_PORT = 4545 CLI_TIMEOUT = 60.0 + class CliFeeder(threading.Thread): def __init__(self, cli, pipe, cmd, session_id, timeout): threading.Thread.__init__(self) self.cli = cli self.pipe = pipe self.cmd = cmd - self.session_id = session_id # this is not python session id, but is webui_agent session id. + self.session_id = session_id # this is not python session id, but is webui_agent session id. self.timeout = timeout - self.buffer = '' # this is used to store str when it got from socket doesn't end with '\r\n' + self.buffer = '' # this is used to store str when it got from socket doesn't end with '\r\n' def run(self): s = self.cli @@ -28,12 +33,13 @@ if not cmd[-1] == '\n': cmd = cmd + '\n' s._connect(60 * 5) - + session_id = self.session_id cmd_len = 1024 - len(session_id.encode('utf-8')) - len(s._username.encode('utf-8')) - 3 cmd_str = cmd.encode('utf-8') while len(cmd_str) > 0: - each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode('utf-8') + F_EOS + each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode( + 'utf-8') + F_EOS s._send(each_cmd) if len(cmd_str) > cmd_len: cmd_str = cmd_str[cmd_len:] @@ -41,13 +47,14 @@ cmd_str = '' _thread_locals._hive_socket.setblocking(0) while not self.pipe.is_finished(): - if len(self.pipe.right_read())>0: + if len(self.pipe.right_read()) > 0: cmd = self.pipe.right_read() self.pipe.left_write("") if not cmd[-1] == '\n': cmd = cmd + '\n' cmd_str = cmd.encode('utf-8') - each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode('utf-8') + F_EOS + each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode( + 'utf-8') + F_EOS _thread_locals._hive_socket.settimeout(self.timeout) s._send(each_cmd) _thread_locals._hive_socket.setblocking(0) @@ -61,7 +68,7 @@ s._disconnect() raise CLIConnBroken("socket connection broken %s" % (e)) if data.find(F_EOP) != -1: - data = data.replace(F_EOP,'') + data = data.replace(F_EOP, '') cmd = "quit\n" cmd_str = cmd.encode('utf-8') each_cmd = cmd_str + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode('utf-8') + F_EOS @@ -89,6 +96,7 @@ class CLIMode(object): """ USER | ENABLE | CONFIG | ENGINEER modes of a CLI connection """ + def __init__(self, lexical, level): self._lexical = lexical self._level = level @@ -101,6 +109,7 @@ return cmp(self._level, another._level) return True + MODE_USER = CLIMode('USER', 0) MODE_ENABLE = CLIMode('ENABLE', 1) MODE_CONFIG = CLIMode('CONFIG', 2) @@ -114,43 +123,60 @@ # This line is just for loading the settings xxx = settings.INSTALLED_APPS + class CLIConnFail(Exception): """ Connection Failure """ pass + + class CLITimeout(Exception): """ CLI Execution timeout """ pass + + class CLIConnBroken(Exception): """ CLI Connection is Broken """ pass + + class CLIEnablePassError(Exception): """ Enable Password is Wrong """ pass + + class CLIConfigModeHeld(Exception): """ Config Mode is Held By Another CLI/WebUI Session """ pass + + class CLICmdException(Exception): """ Base Exception Class for all CLI Command Execution Exceptions """ type = '' pass + + class CLICmdError(CLICmdException): """ Error Level Exception of CLI Command Execution """ - is_error=True - + is_error = True + + class CLICmdWarning(CLICmdException): """ Warning Level Exception of CLI Command Execution - This is a Successful Execution """ - is_error=False + is_error = False type = 'warning' - + + class CLICmdInfo(CLICmdException): """ Info Level Exception of CLI Command Execution - This is a Successful Execution """ - is_error=False + is_error = False type = 'info' + class CLICmdNormal(CLICmdException): """ Info Level Exception of CLI Command Execution - This is a Successful Execution """ - is_error=False + is_error = False + class ANCLIConn(object): """ Legacy CLI Connection to webui_agent @@ -174,14 +200,14 @@ self._address = address self._mode = MODE_USER self._enable_pass = '' - #self._is_set_enable = False + # self._is_set_enable = False if sessid: self._sessid = sessid else: self._sessid = str(self.creation_counter) - #self._connect() + # self._connect() def _set_creation_counter(self): self.creation_counter = ANCLIConn.creation_counter @@ -204,7 +230,7 @@ raise CLIConnFail s.settimeout(timeout) - if getattr(_thread_locals, '_hive_socket', None) !=None: + if getattr(_thread_locals, '_hive_socket', None) != None: _thread_locals._hive_socket.close() _thread_locals._hive_socket = s @@ -248,7 +274,7 @@ raise CLITimeout except socket.error as e: # why connect again ??? - #self._connect() + # self._connect() self._disconnect() raise CLIConnBroken("socket connection broken %s" % (e)) @@ -268,8 +294,8 @@ f_eop = chr(252) username = self._username cmd = escapeshellarg(cmd) - - cmd = '/ca/bin/backend -u ' + username + ' -c ' + cmd + + cmd = '/ca/bin/backend -u ' + username + ' -c ' + cmd cmd = cmd.encode('utf-8') + f_eop (status, output) = subprocess.getstatusoutput(cmd) output = output.strip(f_eop) @@ -279,7 +305,6 @@ return cli_parse(output, args[0]) else: return cli_parse(output, list(args)) - def cmd(self, cmd, *args, **kwargs): """ Execute the command and parse the command output from CLI. @@ -296,29 +321,30 @@ return self.cmd_direct(cmd, *args) if not cmd[-1] == '\n': cmd = cmd + '\n' - #if self._socket is None: + # if self._socket is None: self._connect() timeout = kwargs.get('timeout', None) if timeout: _thread_locals._hive_socket.settimeout(timeout) - #self._send(self._username + F_EOU + cmd + F_EOC + self._sessid + F_EOS) + # self._send(self._username + F_EOU + cmd + F_EOC + self._sessid + F_EOS) session_id = 'webui' + self._sessid + '-' + str(_thread_locals._hive_thread_id) - + # the packet has to be 1k to work with read_write_until_token1 # in webui_agent cmd_len = 1024 - len(session_id.encode('utf-8')) - len(self._username.encode('utf-8')) - 3 - + # XXX check first \n less than cmd_len - + cmd_str = cmd.encode('utf-8') while len(cmd_str) > 0: - each_cmd = cmd_str[:cmd_len] + F_EOC + self._username.encode('utf-8') + F_EOU + session_id.encode('utf-8') + F_EOS + each_cmd = cmd_str[:cmd_len] + F_EOC + self._username.encode('utf-8') + F_EOU + session_id.encode( + 'utf-8') + F_EOS self._send(each_cmd) if len(cmd_str) > cmd_len: cmd_str = cmd_str[cmd_len:] else: cmd_str = '' - + output = self._receive(F_EOP) andebug('an.model.cli', 'cli command execution: "%s", output: "%s"' % (cmd, output)) # For now, as a limitation of webui_agent, we always establish new TCP connection @@ -343,7 +369,6 @@ 'type': 'pipe_action', } return result - @staticmethod def get_conn(username, sessid=None, address="127.0.0.1"): @@ -360,7 +385,7 @@ # create new connection return ANCLIConn(username, sessid, address) - + def get_mode(self): """ Get the mode of current CLI connection by executing command 'mode?' @@ -381,7 +406,7 @@ self._enable_pass = enable_pass def _user2enable(self): - result = self.cmd('en\n'+self._enable_pass+'\n') + result = self.cmd('en\n' + self._enable_pass + '\n') if not result.strip() == 'Enable password:': raise CLIEnablePassError @@ -397,7 +422,7 @@ result = self.cmd('quit\n') if not result.strip() == '': raise CLICmdError - + def _quit2enable(self): result = self.cmd('exit\n') if not result.strip() == '': @@ -422,7 +447,7 @@ elif curr == MODE_USER: self._user2enable() elif curr > MODE_ENABLE and not permit_higher: - self._quit2enable() + self._quit2enable() def set_config(self, permit_higher=False): if not get_current_session().allow_config: @@ -449,8 +474,10 @@ _thread_locals._is_set_enable = False return self._quit2user() + class ANCLIParser(object): """ Base class for legacy CLI parsers """ + def __init__(self, exclusive=False, supplement=False): self.exclusive = exclusive self.supplement = supplement @@ -466,6 +493,7 @@ """ raise TypeError('This function should be overwritten.') + class EasyParser(ANCLIParser): """ A easy parser splitting command output lines with spaces, and match these data with a given ``match_list``. @@ -477,6 +505,7 @@ means specific data must equals the target data or else the line is ignored. :type match_list: list of str """ + def __init__(self, prefix=None, match_list=None, strict_prefix=False, **kwargs): super(EasyParser, self).__init__(**kwargs) self._prefix = prefix or "" @@ -504,7 +533,7 @@ if self._strict_prefix and index > 0: # in strict_prefix mode, match prefix from the beginning return None - handle_list = cmd_output[index+len(self._prefix):].split(' ') + handle_list = cmd_output[index + len(self._prefix):].split(' ') in_quote = False quoted_string = "" for each in handle_list: @@ -517,7 +546,7 @@ in_quote = True quoted_string += each[1:] elif each[-1] == '"': - assert 0,"Incoupled quotes at CLI response %s" % cmd_output + assert 0, "Incoupled quotes at CLI response %s" % cmd_output else: final_list.append(each) else: @@ -527,12 +556,12 @@ final_list.append(quoted_string) quoted_string = "" elif each[0] == '"' and len(each) > 1: - assert 0, "Incoupled quotes at CLI response %s" % cmd_output + assert 0, "Incoupled quotes at CLI response %s" % cmd_output else: quoted_string += (' ' + each) for index in range(0, len(final_list)): - if not final_list[index] or index > len(self._match_list)-1 or not self._match_list[index]: + if not final_list[index] or index > len(self._match_list) - 1 or not self._match_list[index]: continue mark = self._match_list[index][0] if mark == '?': @@ -542,7 +571,7 @@ return None else: assert 0, "invalid format in match_list" - + # if there are left items in the _match_list, just set them to None # the later process of Model layer will set them to default values index += 1 @@ -551,8 +580,8 @@ index += 1 return result - + class BlankParser(ANCLIParser): """ Parser that checks whether the output of command is empty and raise exceptions as necessary. @@ -565,6 +594,7 @@ :param nonblank_msg: Similar as blank_msg but at non-blank case. :type nonblank_msg: str """ + def __init__(self, blank_exception=None, nonblank_exception=None, blank_msg=None, nonblank_msg=None, **kwargs): super(BlankParser, self).__init__(**kwargs) self._blank_exception = blank_exception @@ -582,21 +612,36 @@ raise self._blank_exception(self._blank_msg if self._blank_msg else cmd_output) return cmd_output + class WHOLE: pass + + class DICT: pass + + class LIST: pass + + class RAW: pass + + class AUTO: pass + + class MATCHONE: pass + + class MATCHALL: pass + +# ToDo: Fix unicode import / what is it. class RegexParser(ANCLIParser): """ A Parser matching the output with a regex pattern and return results. @@ -628,7 +673,9 @@ :param unmatch_msg: Similar as match_msg but at unmatch case. :type unmatch_msg: str """ - def __init__(self, regex, mode=MATCHONE, result=AUTO, match_exception=None, unmatch_exception=None, reflags=0, match_msg=None, unmatch_msg=None, **kwargs): + + def __init__(self, regex, mode=MATCHONE, result=AUTO, match_exception=None, unmatch_exception=None, reflags=0, + match_msg=None, unmatch_msg=None, **kwargs): super(RegexParser, self).__init__(**kwargs) self._regex = re.compile(regex, reflags) if type(regex) in (str, unicode) else regex self._match_exception = match_exception @@ -676,7 +723,7 @@ return match_obj else: raise ValueError - + elif self._mode == MATCHALL: match_obj = self._regex.finditer(cmd_output) empty = True @@ -706,7 +753,7 @@ else: raise ValueError - # raise necessary exception + # raise necessary exception if empty: if self._unmatch_exception: raise self._unmatch_exception(self._unmatch_msg if self._unmatch_msg else cmd_output) @@ -717,8 +764,8 @@ return matches else: raise ValueError - + def cli_parse(cmd_output, parser_obj): """ Implementation of the CLI output parser **workflow** @@ -767,7 +814,3 @@ return res else: raise TypeError('Third argument of cmd should be an ANCLIParser or list of it') - - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py (working copy) @@ -2,17 +2,16 @@ Utilities for loading model classes and the packages that contain them. """ -from djproject import an_settings +import pkgutil +import threading from collections import OrderedDict from importlib import import_module + from django.utils.module_loading import module_has_submodule -from hive.model.package import ANPackage -import sys -import os -import threading -import pkgutil -from hive.utils import anlog, anerror, andebug +from djproject import an_settings +from hive.model.package import ANPackage +from hive.utils import anerror, andebug __all__ = ('get_apps', 'get_app', 'get_package', 'get_model', 'register_models', 'get_app_errors', 'app_cache_ready') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading_bac.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading_bac.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading_bac.py (working copy) @@ -1,18 +1,18 @@ "Utilities for loading models and the modules that contain them." +import os +import sys +import threading +from importlib import import_module + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.utils.datastructures import SortedDict -from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule -import sys -import os -import threading - __all__ = ('get_apps', 'get_app', 'get_models', 'get_model', 'register_models', - 'load_app', 'app_cache_ready') + 'load_app', 'app_cache_ready') + class AppCache(object): """ A cache that stores installed applications and their models. Used to @@ -22,25 +22,25 @@ # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66531. __shared_state = dict( # Keys of app_store are the model modules for each application. - app_store = SortedDict(), + app_store={}, # Mapping of installed app_labels to model modules for that app. - app_labels = {}, + app_labels={}, # Mapping of app_labels to a dictionary of model names to model code. # May contain apps that are not installed. - app_models = SortedDict(), + app_models={}, # Mapping of app_labels to errors raised when trying to import the app. - app_errors = {}, + app_errors={}, # -- Everything below here is only used when populating the cache -- - loaded = False, - handled = {}, - postponed = [], - nesting_level = 0, - write_lock = threading.RLock(), - _get_models_cache = {}, + loaded=False, + handled={}, + postponed=[], + nesting_level=0, + write_lock=threading.RLock(), + _get_models_cache={}, ) def __init__(self): @@ -181,13 +181,12 @@ self._populate() if app_mod: if app_mod in self.app_store: - app_list = [self.app_models.get(self._label_for(app_mod), - SortedDict())] + app_list = [self.app_models.get(self._label_for(app_mod), {})] else: app_list = [] else: if only_installed: - app_list = [self.app_models.get(app_label, SortedDict()) + app_list = [self.app_models.get(app_label, {}) for app_label in self.app_labels.iterkeys()] else: app_list = self.app_models.itervalues() @@ -213,7 +212,7 @@ self._populate() if only_installed and app_label not in self.app_labels: return None - return self.app_models.get(app_label, SortedDict()).get(model_name.lower()) + return self.app_models.get(app_label, {}).get(model_name.lower()) def register_models(self, app_label, *models): """ @@ -223,7 +222,7 @@ # Store as 'name: model' pair in a dictionary # in the app_models dictionary model_name = model._meta.object_name.lower() - model_dict = self.app_models.setdefault(app_label, SortedDict()) + model_dict = self.app_models.setdefault(app_label, {}) if model_name in model_dict: # The same model may be imported via different paths (e.g. # appname.models and project.appname.models). We use the source @@ -238,6 +237,7 @@ model_dict[model_name] = model self._get_models_cache.clear() + cache = AppCache() # These methods were always module level, so are kept that way for backwards Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py (working copy) @@ -1,55 +1,61 @@ # encoding: utf-8 -from hive.model.query import * -from djproject.an_settings import PSEUDO_MANAGER_SHELF_FILE, CACHE_SWITCH -import shelve -from hive.exceptions import ValidationError, ModelQueryException, ManagerImplError -import random -from hive.utils import andebug, dump_dict, get_current_session, getcallargs, dict_combine -from hive.model.utils import reduce_asso_value_item, get_complex_field, CONDITIONAL_NONE -import itertools as it import inspect +import itertools as it +import shelve + +from django.db.models.query import QuerySet from django.utils.encoding import smart_str -from ctypes import * -from hive.session import current_app + +from cm import company from cm.lib.libbasic_operation import oper_log -import json -from django.db.models.query import QuerySet -import copy +from djproject.an_settings import PSEUDO_MANAGER_SHELF_FILE +from hive.exceptions import ManagerImplError +from hive.model.loading import get_model +from hive.model.query import * +from hive.model.utils import reduce_asso_value_item, get_complex_field, CONDITIONAL_NONE +from hive.session import current_app +from hive.utils import andebug, dump_dict, getcallargs, dict_combine -from hive.model.stats import STAILQ_HEAD - exec("import %s.company as company" % current_app()) try: from hive.model.stats import * except: pass + class UndefinedValue: pass + # decorator for _get_ functions of manager implementation def QueryingFields(field_name_list): def decorator(F): # register this function to the manager class's _custom_gets F._custom_gets = field_name_list return F + return decorator + # decorator for _get_ functions of manager implementation def QueryingGroups(group_name_list): def decorator(F): # register this function to the manager class's _custom_gets F._custom_group_gets = group_name_list return F + return decorator + def UpdatingFields(field_name_list): def decorator(F): # register this function to the manager class's _custom_updates F._custom_updates = field_name_list return F + return decorator + # used for manger interface, the first arg for F is 'self': Manager for xxx. # if field_name_list is set, only mark these field expire, do not mark whole instance and model expire. # if field_name_list not set, make whole instance expire. @@ -57,14 +63,14 @@ # if pk_list not set, mark expire on all instances in this model. def ImpactModule(model_path, pk_list=None, field_name_list=None): def decorator(F): - def new_function(*callvarargs,**callkeywords): + def new_function(*callvarargs, **callkeywords): args, varargname, kwname = inspect.getargspec(F)[:3] dctCallArgs = getcallargs(args, varargname, kwname, - callvarargs, callkeywords) + callvarargs, callkeywords) model = get_model(dctCallArgs['self']._model._meta.app_label, model_path.split('.')) if not model: raise ManagerImplError('decorator for %s has wrong arguments', F) - #mark cache expire + # mark cache expire if pk_list: if field_name_list: for pk_dict in pk_list: @@ -83,27 +89,30 @@ mark_expire_field(ins, field) else: mark_expire_all(model) - return F(*callvarargs,**callkeywords) + return F(*callvarargs, **callkeywords) + return new_function + return decorator + class ANManager(object): _custom_gets = {} _custom_gets_reverse = {} - _custom_updates = {} # {:} - _custom_updates_reverse = {} # {:[, , ...]} - + _custom_updates = {} # {:} + _custom_updates_reverse = {} # {:[, , ...]} + def __init__(self, model, session): self._model = model self._session = session - + @classmethod def contribute_to_class(manager_cls, model_cls, name): manager_cls._custom_gets = {} manager_cls._custom_gets_reverse = {} manager_cls._custom_updates = {} manager_cls._custom_updates_reverse = {} - + # register the _custom_gets for method_name, method in manager_cls.__dict__.iteritems(): if hasattr(method, "_custom_group_gets"): @@ -119,21 +128,20 @@ manager_cls._custom_gets_reverse[method] = method._custom_gets for each_field_name in method._custom_gets: manager_cls._custom_gets[each_field_name] = method - + if hasattr(method, "_custom_updates"): manager_cls._custom_updates_reverse[method] = method._custom_updates for each_field_name in method._custom_updates: manager_cls._custom_updates[each_field_name] = method setattr(model_cls, name, manager_cls) - + def __repr__(self): - return 'Manager for '+ '.'.join(self._model._meta.path) + ' for session %s' % self._session + return 'Manager for ' + '.'.join(self._model._meta.path) + ' for session %s' % self._session def __str__(self): - return 'Manager for '+ '.'.join(self._model._meta.path) + ' for session %s' % self._session + return 'Manager for ' + '.'.join(self._model._meta.path) + ' for session %s' % self._session - ##################################### # Public methods # To be called by frontend view layer @@ -195,10 +203,10 @@ if path_list[0] == "loadbalancing": path_list[0] = "lb" # TODO: Fix Library issue and then uncomment - #rawdata = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all") - #rawdata.restype = POINTER(STAILQ_HEAD) + # rawdata = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all") + # rawdata.restype = POINTER(STAILQ_HEAD) errcode_ptr = pointer(errcode) - #rtn_head = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all")(errcode_ptr) + # rtn_head = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all")(errcode_ptr) return None except: return None @@ -237,21 +245,26 @@ self.reset_after_change() # XXX check insert success or failed? mark_expire_all(model_instance._meta._model, clear_all_if_has_asso=True) - oper_log('error', model_instance._meta.path[-2], 'Failed to do create operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do create operation <%s>.' % model_instance.repr_by_pk( + model_instance.pk_dict())) raise e else: - oper_log('error', model_instance._meta.path[-2], 'Failed to do create operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) - raise ModelQueryException('This instance is already existed.') + oper_log('error', model_instance._meta.path[-2], + 'Failed to do create operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + raise ModelQueryException('This instance is already existed.') except ModelQueryException as e: self.reset_after_change() # XXX check insert success or failed? mark_expire_all(model_instance._meta._model, clear_all_if_has_asso=True) - oper_log('error', model_instance._meta.path[-2], 'Failed to do create operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do create operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) raise e self.reset_after_change() # XXX check insert success or failed? mark_expire_all(model_instance._meta._model, clear_all_if_has_asso=True) - oper_log('info', model_instance._meta.path[-2], '<%s> has been created.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('info', model_instance._meta.path[-2], + '<%s> has been created.' % model_instance.repr_by_pk(model_instance.pk_dict())) return result def update(self, model_instance, field_list=None, old_values=None): @@ -277,15 +290,15 @@ if field_list == None: field_list = [each_field.name for each_field in self._model._meta.fields] update_all = True - + done_list = [] error_list = [] - exception = ModelQueryException() # an empty exception + exception = ModelQueryException() # an empty exception for field_name in field_list: # if the field is already handled, pass it if field_name in done_list: continue - + if model_instance.is_delayed_query(field_name): if update_all: # just pass the fields which are DelayedQuery @@ -346,36 +359,43 @@ except ManagerImplError: try: if 'old_values' in inspect.getargspec(self._update).args: - result = self._update(model_instance, old_values) + result = self._update(model_instance, old_values) else: - result = self._update(model_instance) + result = self._update(model_instance) mark_expire_one(model_instance._meta._model, model_instance.pk_dict()) except ManagerImplError as e: - oper_log('error', model_instance._meta.path[-2], 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do update operation <%s>.' % model_instance.repr_by_pk( + model_instance.pk_dict())) exception.merge(e) except ModelQueryException as e: - oper_log('error', model_instance._meta.path[-2], 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do update operation <%s>.' % model_instance.repr_by_pk( + model_instance.pk_dict())) exception.merge(e) except ModelQueryException as e: - oper_log('error', model_instance._meta.path[-2], 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) exception.merge(e) - - #if len(error_list) > 0: + + # if len(error_list) > 0: # raise ManagerImplError('Fields ' + ','.join(error_list) + ' of model %s are not updatable' % self._model) - + self.reset_after_change() if old_instance: mark_expire_one(old_instance._meta._model, old_instance.pk_dict()) if not exception.is_empty(): andebug('hive.debug', 'raising exception at manager\'s update: %s' % exception) - oper_log('error', model_instance._meta.path[-2], 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('error', model_instance._meta.path[-2], + 'Failed to do update operation <%s>.' % model_instance.repr_by_pk(model_instance.pk_dict())) raise exception if model_instance.pk_dict(): - oper_log('info', model_instance._meta.path[-2], '<%s> has been updated.' % model_instance.repr_by_pk(model_instance.pk_dict())) + oper_log('info', model_instance._meta.path[-2], + '<%s> has been updated.' % model_instance.repr_by_pk(model_instance.pk_dict())) else: oper_log('info', model_instance._meta.path[-2], 'Field of %s has been updated.' % field_name) - + def delete(self, pk_list, clear_cache=True): # try to call _delete_instance try: @@ -397,7 +417,8 @@ try: self._delete(pk_list) except ModelQueryException as e: - oper_log('error', self._model._meta.path[-2], 'Failed to do delete operation <%s>.' % self._model.repr_by_pk(instance.pk_dict())) + oper_log('error', self._model._meta.path[-2], + 'Failed to do delete operation <%s>.' % self._model.repr_by_pk(instance.pk_dict())) self.reset_after_change() raise e # if module do not have _get_query_set or _get_stats, there are not "all" cache for this module, so mark_expire_all can not expire the instance cache. @@ -405,11 +426,13 @@ mark_expire_one(self._model, each_pk) except ModelQueryException as e: self.reset_after_change() - oper_log('error', self._model._meta.path[-2], 'Failed to do delete operation <%s>.' % self._model.repr_by_pk(instance.pk_dict())) + oper_log('error', self._model._meta.path[-2], + 'Failed to do delete operation <%s>.' % self._model.repr_by_pk(instance.pk_dict())) raise e - oper_log('info', self._model._meta.path[-2], '<%s> has been deleted.' % self._model.repr_by_pk(instance.pk_dict())) + oper_log('info', self._model._meta.path[-2], + '<%s> has been deleted.' % self._model.repr_by_pk(instance.pk_dict())) self.reset_after_change() - + def get(self, pk_dict=None): new_pk = None if pk_dict: @@ -459,7 +482,6 @@ # handle KeyError return instance - # This is called with pk_dict (no instance) to get value of a field # # first check cache, if cache for the instance doesnexist or expire, call the get() to get the instance and update cache. @@ -492,7 +514,6 @@ instance = get_cache_one(self._model, pk_dict) return getattr(instance, field_name) - # we have an instance, the instance must not be expired. so we doncare about cache here. # # called with an instance, this is always called as a Delayed Request, @@ -507,8 +528,8 @@ the_pk_dict = instance.pk_dict() or None # for profile classes, the input pk_dict could be None or {} - if hasattr(self, '_get_all_'+field_name): - value = getattr(self, '_get_all_'+field_name)() #[{'pk_dict':xxx, '':xxx}, ...] + if hasattr(self, '_get_all_' + field_name): + value = getattr(self, '_get_all_' + field_name)() # [{'pk_dict':xxx, '':xxx}, ...] # cannot call validate here because we don't have a instance to reference # so just call to_python @@ -524,18 +545,19 @@ update_cache_field(tmp_instance, field_name, each[field_name]) result = instance.get_attr_raw(field_name) if result is DelayedQuery: - raise ManagerImplError('function _get_all_%s did not return value for instance:%s ' % (field_name, instance.pk_str())) + raise ManagerImplError( + 'function _get_all_%s did not return value for instance:%s ' % (field_name, instance.pk_str())) if result is CONDITIONAL_NONE: return getattr(instance, field_name) return result except ValidationError as e: raise ModelQueryException(e) - elif hasattr(self, '_get_'+field_name): - if the_pk_dict is None: # for profile only - value = getattr(self, '_get_'+field_name)() + elif hasattr(self, '_get_' + field_name): + if the_pk_dict is None: # for profile only + value = getattr(self, '_get_' + field_name)() else: - value = getattr(self, '_get_'+field_name)(the_pk_dict) - + value = getattr(self, '_get_' + field_name)(the_pk_dict) + # cannot call validate here because we don't have a instance to reference # so just call to_python if not CACHE_SWITCH: @@ -545,7 +567,8 @@ update_cache_field(instance, field_name, value) result = instance.get_attr_raw(field_name) if result is DelayedQuery: - raise ManagerImplError('function _get_%s did not return value for instance:%s ' % (field_name, instance.pk_str())) + raise ManagerImplError( + 'function _get_%s did not return value for instance:%s ' % (field_name, instance.pk_str())) if result is CONDITIONAL_NONE: return getattr(instance, field_name) return result @@ -554,8 +577,8 @@ else: custom_get_func = self._custom_gets.get(field_name, None) if custom_get_func: - if the_pk_dict is None: # for profile only - rtn_dict = custom_get_func(self) # this is an unbound method, call with self + if the_pk_dict is None: # for profile only + rtn_dict = custom_get_func(self) # this is an unbound method, call with self else: rtn_dict = custom_get_func(self, the_pk_dict) if type(rtn_dict) == dict: @@ -583,10 +606,12 @@ if not rst and ins_pk == the_pk_dict: result = self._model._meta.get_field(field_name).to_python(each_rtn[field_name]) return result - + result = instance.get_attr_raw(field_name) if result is DelayedQuery: - raise ManagerImplError('function _custom_gets for %s did not return value for instance:%s ' % (field_name, instance.pk_str())) + raise ManagerImplError( + 'function _custom_gets for %s did not return value for instance:%s ' % (field_name, + instance.pk_str())) if result is CONDITIONAL_NONE: return getattr(instance, field_name) return result @@ -596,7 +621,9 @@ # of target model field = self._model._meta.get_field(field_name) if field.is_composition_asso(): - andebug('hive.debug', 'no custom getting for field %s, as this is a composition asso, deligate to target model\'s filter' % (field.name)) + andebug('hive.debug', + 'no custom getting for field %s, as this is a composition asso, deligate to target model\'s filter' % ( + field.name)) rtn = [] for idx in range(field.asso_count): target_manager = field.target_end[idx].model.get_manager(self._session) @@ -608,21 +635,22 @@ break filter_dict = {} if the_pk_dict: - filter_dict[field.tgt_field[idx].name] = [dict_combine(the_pk_dict, {'_asso_idx':target_end_idx})] + filter_dict[field.tgt_field[idx].name] = [ + dict_combine(the_pk_dict, {'_asso_idx': target_end_idx})] else: - filter_dict[field.tgt_field[idx].name] = [{'_asso_idx':target_end_idx}] + filter_dict[field.tgt_field[idx].name] = [{'_asso_idx': target_end_idx}] target_objs = target_manager.filter(filter_dict) andebug('hive.debug', 'got target objs: %s' % (target_objs)) for each in target_objs: andebug('hive.debug', 'obj pk_dict: %s' % (each.pk_dict())) - rtn.extend([dict_combine(x.pk_dict(), {'_asso_idx':idx}) for x in target_objs]) + rtn.extend([dict_combine(x.pk_dict(), {'_asso_idx': idx}) for x in target_objs]) update_cache_field(instance, field_name, rtn) return rtn # come to here means there's not any custom implementation of "_get" # deligate to get(pk_dict) field = self._model._meta.get_field(field_name) - #try: + # try: if field.group.isstats: fast_data = False try: @@ -681,7 +709,9 @@ if raw_value is DelayedQuery: field = self._model._meta.get_field(field_name) if field.type_name == 'asso': - andebug('hive.debug', 'no custom getting for field %s, as this is a asso field, deligate to target model\'s filter' % (field.name)) + andebug('hive.debug', + 'no custom getting for field %s, as this is a asso field, deligate to target model\'s filter' % ( + field.name)) rtn = [] for idx in range(field.asso_count): target_manager = field.target_end[idx].model.get_manager(self._session) @@ -691,33 +721,34 @@ if field.tgt_field[idx].target_end[tmp_idx].model == self._model: target_end_idx = tmp_idx break - filter_dict = dict_combine(the_pk_dict, {'_asso_idx':target_end_idx}) + filter_dict = dict_combine(the_pk_dict, {'_asso_idx': target_end_idx}) target_obj_list = target_manager.all() for target_ins in target_obj_list: tgt_raw_value = target_ins.get_attr_raw(field.tgt_field[idx].name) if tgt_raw_value is DelayedQuery: continue if filter_dict in tgt_raw_value: - rtn.extend(dict_combine(target_ins.pk_dict(), {'_asso_idx':idx})) + rtn.extend(dict_combine(target_ins.pk_dict(), {'_asso_idx': idx})) update_cache_field(the_instance, field_name, rtn) return rtn else: - raise ManagerImplError('model %s has not implemented _get functions for field %s' % (self._model, field_name)) + raise ManagerImplError( + 'model %s has not implemented _get functions for field %s' % (self._model, field_name)) if raw_value is CONDITIONAL_NONE: return getattr(the_instance, field_name) return raw_value - #except AttributeError as e: + # except AttributeError as e: # raise ManagerImplError('model %s has not implemented _get functions for field %s, case 2 with msg %s' % (self._model, field_name, e.message)) - + def get_pks(self): - return [e.pk_dict() for e in self.all()] - + return [e.pk_dict() for e in self.all()] + def get_stats_pks(self): try: return [e.pk_dict() for e in self.stats_all()] except ManagerImplError: return [] - + def clear(self): pass @@ -763,8 +794,6 @@ def _get(self, pk_dict=None): raise ManagerImplError('model %s has not implemented _get', self._model) - - ##################################### # Special public methods ##################################### @@ -778,7 +807,8 @@ if serializable: ret = [] for action in available_list: - t = {'name':action.name, 'verbose_name':str(action.verbose_name), 'btn_class':action.btn_class, 'option':[]} + t = {'name': action.name, 'verbose_name': str(action.verbose_name), 'btn_class': action.btn_class, + 'option': []} t['url'] = model_instance.get_ajax_url('perform') if action.options: for option in action.options: @@ -787,19 +817,24 @@ return ret else: return available_list - + def get_available_functions(self, model_instance, serializable=False): if model_instance._meta.app_label == 'apv': model_instance._meta.app_label = company.PRODUCT_KEYWORD available_list = {} - available_list['popup_config'] = {'name':'Edit', 'type':'popup', 'data':model_instance.get_ajax_url('box_instance_config')} + available_list['popup_config'] = {'name': 'Edit', 'type': 'popup', + 'data': model_instance.get_ajax_url('box_instance_config')} default_node = model_instance._meta.get_default_node() if default_node: - available_list['detail']={'name':'Detail', 'type':'href', 'data':'/'+model_instance._meta.app_label+'/'+'/'.join(default_node.path) + '/_inst_status?'+ model_instance.url_params()} - available_list['delete']={'name':'Delete', 'type':'ajax', 'data':{'data':{'pk':model_instance.pk_str()}, 'url':self._model._meta.get_ajax_url('delete')}} + available_list['detail'] = {'name': 'Detail', 'type': 'href', + 'data': '/' + model_instance._meta.app_label + '/' + '/'.join( + default_node.path) + '/_inst_status?' + model_instance.url_params()} + available_list['delete'] = {'name': 'Delete', 'type': 'ajax', 'data': {'data': {'pk': model_instance.pk_str()}, + 'url': self._model._meta.get_ajax_url( + 'delete')}} return available_list - + def perform_action(self, name, options=None, **kwargs): # XXX exception handling pk_list = [] @@ -820,21 +855,24 @@ if action_cls.config_change: if not pk_list: mark_expire_all(self._model) - oper_log('info', self._model._meta.path[-2], 'The %s action is successfully.' % str(action_cls.verbose_name)) + oper_log('info', self._model._meta.path[-2], + 'The %s action is successfully.' % str(action_cls.verbose_name)) else: for each_pk in pk_list: - oper_log('info', self._model._meta.path[-2], 'The %s action is successfully <%s>.' % (str(action_cls.verbose_name), self._model.repr_by_pk(each_pk))) + oper_log('info', self._model._meta.path[-2], + 'The %s action is successfully <%s>.' % (str(action_cls.verbose_name), + self._model.repr_by_pk(each_pk))) mark_expire_one(self._model, each_pk) return ret # return a dict, {'stat': <'stat-ok'|'stat-info'|'stat-warning'|'stat-error'>, 'msg':'xxx'}. def get_quality_status(self, instance=None, include_asso=False): # to be overwritten by sub classes - return {'stat':'stat-none', 'msg':''} - + return {'stat': 'stat-none', 'msg': ''} + def get_display_name(self, instance=None): return instance.repr() - + def get_asso_list(self, instance=None, point=0): ret = [] for f in self._model._meta.fields: @@ -845,10 +883,10 @@ 'type': 'Menu', 'children': [] } - ret.append(tmp) - + ret.append(tmp) + return ret - + # if 'pos' is 0, means this obj is on the left of a asso, need to find the models one the right; # if 'pos' is 1, means this obj is on the right of a asso, need to find the models one the left; # if 'pos' is 2, means to find all assos on both side; @@ -865,23 +903,23 @@ continue else: asso_field_done_list.append(field.attname) - + # find correct fields, check pos if not pos % 2 and field.pos == 'left': ins_pos = 0 elif pos > 0 and field.pos == 'right': - #mul = field.asso._right.mul.__repr__() + # mul = field.asso._right.mul.__repr__() ins_pos = 1 else: continue - + pks = field._get_val_from_obj(instance) if pks: tmp = { - 'data': str(field.verbose_name), - 'state': 'open', - 'children': [] - } + 'data': str(field.verbose_name), + 'state': 'open', + 'children': [] + } for each_pk in pks: if '_asso_idx' in each_pk: asso_idx = each_pk['_asso_idx'] @@ -891,25 +929,25 @@ remote_manager = remote_model.get_manager(self._session) remote_ins = remote_manager.get(each_pk) remote_hc = remote_manager.get_quality_status(instance=remote_ins) - if field.tgt_field[asso_idx].monitorable: + if field.tgt_field[asso_idx].monitorable: tmp['children'].append( - { - 'data': { - 'title': remote_manager.get_display_name(remote_ins), - 'attr': { - 'class': "stat " + remote_hc['stat'], - 'data-content': remote_hc['msg'] - } - }, - 'state': 'closed', - 'metadata': { - 'pos': ins_pos, - 'url': remote_ins.get_ajax_url('tree'), - 'page_url': remote_ins.get_ajax_url('box_instance_monitor') - }, - 'multi': False, - } - ) + { + 'data': { + 'title': remote_manager.get_display_name(remote_ins), + 'attr': { + 'class': "stat " + remote_hc['stat'], + 'data-content': remote_hc['msg'] + } + }, + 'state': 'closed', + 'metadata': { + 'pos': ins_pos, + 'url': remote_ins.get_ajax_url('tree'), + 'page_url': remote_ins.get_ajax_url('box_instance_monitor') + }, + 'multi': False, + } + ) else: # we will not display remote_model info, we will get the tgt_model for remote and diaplay it. sub_asso_field_done_list = [] @@ -938,55 +976,55 @@ rr_ins = rr_manager.get(each_rpk) rr_hc = rr_manager.get_quality_status(instance=rr_ins) tmp['children'].append( - { - 'data': { - 'primary':{ - 'title': rr_manager.get_display_name(rr_ins), - 'attr': { - 'class': "stat " + rr_hc['stat'], - 'data-content': rr_hc['msg'] - } - }, - 'secondary':{ - 'title': remote_manager.get_display_name(remote_ins), - 'attr': { - 'class': "stat " + remote_hc['stat'], - 'data-content': remote_hc['msg'] - }, - }, - }, - 'state': 'closed', - 'metadata': { - 'pos': ins_pos, - 'url': rr_ins.get_ajax_url('tree'), - 'primary': { - 'page_url': rr_ins.get_ajax_url('box_instance_monitor') - }, - 'secondary':{ - 'page_url': remote_ins.get_ajax_url('box_instance_monitor') - } - }, - 'multi': True, + { + 'data': { + 'primary': { + 'title': rr_manager.get_display_name(rr_ins), + 'attr': { + 'class': "stat " + rr_hc['stat'], + 'data-content': rr_hc['msg'] } - ) + }, + 'secondary': { + 'title': remote_manager.get_display_name(remote_ins), + 'attr': { + 'class': "stat " + remote_hc['stat'], + 'data-content': remote_hc['msg'] + }, + }, + }, + 'state': 'closed', + 'metadata': { + 'pos': ins_pos, + 'url': rr_ins.get_ajax_url('tree'), + 'primary': { + 'page_url': rr_ins.get_ajax_url('box_instance_monitor') + }, + 'secondary': { + 'page_url': remote_ins.get_ajax_url('box_instance_monitor') + } + }, + 'multi': True, + } + ) ret.append(tmp) return ret - + def get_tree_for_module(self, group=None, order=None, search=None): ret = { - 'data': str(self._model._meta.verbose_name), - 'metadata': { - 'url': self._model._meta.get_ajax_url('tree'), - 'page_url': self._model._meta.get_ajax_url('box_module_monitor') - }, - 'pk':self._model.__name__, - 'type': 'Menu', - 'groupby': group, - 'orderby': order, - 'children': [], - } - + 'data': str(self._model._meta.verbose_name), + 'metadata': { + 'url': self._model._meta.get_ajax_url('tree'), + 'page_url': self._model._meta.get_ajax_url('box_module_monitor') + }, + 'pk': self._model.__name__, + 'type': 'Menu', + 'groupby': group, + 'orderby': order, + 'children': [], + } + index_list = self._model._meta.get_indexed_list() ret["group"] = index_list["group"] ret["order"] = index_list["order"] @@ -1004,24 +1042,24 @@ if search: no_found = True for k, v in sh.iteritems(): - if k=='-' and ins.repr().find(v) > -1: + if k == '-' and ins.repr().find(v) > -1: no_found = False elif str(getattr(ins, k, '')).find(v) > -1: no_found = False - + if no_found: continue tmp = { - 'data':self.get_display_name(ins), - 'hc':self.get_quality_status(instance=ins), - 'group':'Not Set', - 'order':getattr(ins, order, ''), - 'pk': ins.repr(), - 'url': ins.get_ajax_url('tree'), - 'page_url': ins.get_ajax_url('box_instance_monitor'), - 'state': '-', - 'pos': 2 - } + 'data': self.get_display_name(ins), + 'hc': self.get_quality_status(instance=ins), + 'group': 'Not Set', + 'order': getattr(ins, order, ''), + 'pk': ins.repr(), + 'url': ins.get_ajax_url('tree'), + 'page_url': ins.get_ajax_url('box_instance_monitor'), + 'state': '-', + 'pos': 2 + } if group_field: tmp['group'] = group_field.value_to_display(obj=ins) for ins_field in ins._meta.fields: @@ -1031,18 +1069,21 @@ tmp_ret.append(tmp) if order: - tmp_ret.sort(key=lambda x:x["order"], reverse=True) + tmp_ret.sort(key=lambda x: x["order"], reverse=True) if group: - tmp_ret.sort(key=lambda x:x["group"]) - for i,j in it.groupby(tmp_ret, lambda x:x["group"]): + tmp_ret.sort(key=lambda x: x["group"]) + for i, j in it.groupby(tmp_ret, lambda x: x["group"]): ret['children'].append( { 'data': i, 'state': 'open', - 'children': [{'data':{'title': x['data'], 'attr': {'class':"stat " + x['hc']['stat'], 'data-content':x['hc']['msg']}}, 'state':x['state'], 'metadata':{'url':x['url'], 'page_url':x['page_url'], 'pos': 2}} for x in j] + 'children': [{'data': {'title': x['data'], 'attr': {'class': "stat " + x['hc']['stat'], + 'data-content': x['hc']['msg']}}, + 'state': x['state'], + 'metadata': {'url': x['url'], 'page_url': x['page_url'], 'pos': 2}} for x in j] } ) - + else: for i in tmp_ret: ret['children'].append( @@ -1051,7 +1092,7 @@ 'title': i["data"], 'attr': { 'class': "stat " + i['hc']['stat'], - 'data-content':i['hc']['msg'] + 'data-content': i['hc']['msg'] } }, 'state': i["state"], @@ -1065,7 +1106,7 @@ if (len(ret['children']) > 0): ret['state'] = 'open' return ret - + def get_fields_for_module(self, field, order): ret = [] ins_list = self.stats_all() @@ -1075,14 +1116,16 @@ val = getattr(ins, fn, '0') tmp.append(val) ret.append([ins.repr(), tmp]) + def val(s): - return s[1][0] + return s[1][0] + ret.sort(key=val, reverse=order) return ret - + def get_config_list(self, start=0, max_num=0, search=None, filter_list=None, sort_dict=None): raise ManagerImplError('model %s has not implemented get_config_list', self._model) - + def get_stats_list(self, start=0, max_num=0, search=None, filter_list=None, sort_dict=None): raise ManagerImplError('model %s has not implemented get_stats_list', self._model) @@ -1116,6 +1159,7 @@ ret = self.filter(filter_dict)[0:] else: ret = self.all()[0:] + # search_dict: {search_str:xxx, search_fields:[field_name1, field_name2, ...]} def ins_search(instance, search_dict): for field_name in search_dict["search_fields"]: @@ -1126,8 +1170,10 @@ if value.upper().find(search_dict["search_str"].upper()) != -1: return True return False + if search_dict and "search_str" in search_dict and search_dict["search_str"]: - ret = [x for x in ret if ins_search(x, search_dict)] + ret = [x for x in ret if ins_search(x, search_dict)] + def ins_sort(instance1, instance2): field_name = sort_dict["field"] field = None @@ -1145,7 +1191,7 @@ if field: length = len(value1) idx = 0 - for idx in range(length): + for idx in range(length): if not idx < len(value2): return 1 try: @@ -1159,6 +1205,7 @@ return 0 else: return 0 + if sort_dict: if sort_dict["order"] == 'desc': order = True @@ -1169,7 +1216,7 @@ if length <= 0: response["data"] = ret[start:] else: - response["data"] = ret[start:start+length] + response["data"] = ret[start:start + length] response["total"] = len(ret) return response @@ -1177,6 +1224,7 @@ # to be overwritten by sub classes, if not defined, will use default check in ajax.py raise ManagerImplError('model %s has not implemented action_check', self._model) + class CLIManager(ANManager): @property def cli(self): @@ -1187,7 +1235,8 @@ return andebug('hive.debug', 'Reset manager after change actions') return self.cli.set_enable(force=True) - + + class VirtualClassManager(CLIManager): def _shared_parsing(self, cmd_output): # the cmd_output should be passed from a super virtual manager @@ -1203,16 +1252,16 @@ if sub_data: data.merge(sub_data) return data - - def _get_query_set(self): + + def _get_query_set(self): # using the _shared_parsing interface for all the deriving fields if hasattr(self, '_shared_cmd'): data = QuerySet(self._model) - + # first run the command self.cli.set_enable() cmd_output = self.cli.cmd(self._shared_cmd) - + # then loop all children and gather the results if not self._model._meta.children: raise ValueError("this is not a virtual class") @@ -1233,7 +1282,7 @@ if sub_data: data.merge(sub_data) return data - + # using the ugly slow way data = QuerySet(self._model) if self._model._meta.children: @@ -1251,7 +1300,7 @@ else: raise ValueError("this is not a virtual class") return data - + def delete(self, pk_list, clear_cache=True): try: self._delete(pk_list) @@ -1272,12 +1321,13 @@ mark_expire_all(self._model) raise e return - + + class PseudoManager(ANManager): def __init__(self, model, session): super(PseudoManager, self).__init__(model, session) self.shelf_file = PSEUDO_MANAGER_SHELF_FILE - + def _get_query_set(self): andebug('hive.debug', 'entering pseudo manager\'s get_query_set') data = QuerySet(self._model) @@ -1306,11 +1356,11 @@ shelf_dict = shelve.open(self.shelf_file) key = '.'.join(self._model._meta.path) if not shelf_dict.has_key(key): - data = QuerySet(self._model) # Empty queryset + data = QuerySet(self._model) # Empty queryset else: data = shelf_dict[key] shelf_dict.close() - + # if this is a profile, and the data storage is empty # _insert an initial object with all fields set as default value if self._model._meta.profile and data.count() == 0: @@ -1321,19 +1371,18 @@ shelf_dict = shelve.open(self.shelf_file) key = '.'.join(self._model._meta.path) if not shelf_dict.has_key(key): - return QuerySet(self._model) # Empty queryset - + return QuerySet(self._model) # Empty queryset + data = shelf_dict[key] shelf_dict.close() - + return data - def _insert(self, model_instance): for field in model_instance._meta.fields: if getattr(model_instance, field.name, None) is None: setattr(model_instance, field.name, DelayedQuery) - + shelf_dict = shelve.open(self.shelf_file) key = '.'.join(self._model._meta.path) if shelf_dict.has_key(key): @@ -1344,9 +1393,9 @@ new_qs = QuerySet(self._model) new_qs.append_obj(model_instance) shelf_dict[key] = new_qs - + shelf_dict.close() - + # check external component fields # and insert to the target model for each_field in self._model._meta.fields: @@ -1356,10 +1405,10 @@ target_args = {} target_args[each_field.tgt_field[0].name] = [model_instance.pk_dict()] target_obj = target_model(**target_args) - target_obj.clean_fields() # every external component models should be optional or have non-Empty default values + target_obj.clean_fields() # every external component models should be optional or have non-Empty default values target_model.get_manager(self._session)._insert(target_obj) return True - + def _update(self, model_instance, field_list=[]): shelf_dict = shelve.open(self.shelf_file) key = '.'.join(self._model._meta.path) @@ -1371,7 +1420,7 @@ raise ValueError('No this class stored in database') shelf_dict.close() return True - + def _delete(self, pk_list): for pk in pk_list: assert type(pk) is dict @@ -1394,7 +1443,7 @@ key = '.'.join(self._model._meta.path) if shelf_dict.has_key(key): data = shelf_dict[key] - pk = dump_dict(pk) + pk = dump_dict(pk) old_obj = data._cache_dict[pk] idx = data._cache.index(old_obj) del data._cache_dict[pk] @@ -1414,16 +1463,15 @@ target_pk[each_field.tgt_field[0].name] = [pk] target_model.get_manager(self._session)._delete([target_pk]) return True - - + def _clear(self): shelf_dict = shelve.open(self.shelf_file) key = '.'.join(self._model._meta.path) if shelf_dict.has_key(key): - shelf_dict[key] = QuerySet(self._model) # Empty queryset + shelf_dict[key] = QuerySet(self._model) # Empty queryset shelf_dict.close() return True - + def get_field_by_instance(self, instance, field_name): # the default implementation is slow ! field = self._model._meta.get_field(field_name) @@ -1434,7 +1482,7 @@ return field.get_pseudo_value() else: return field.get_pseudo_value() - + def get_field(self, pk_dict, field_name): field = self._model._meta.get_field(field_name) if not field.group.isstats or field.object_switch: @@ -1444,13 +1492,14 @@ return field.get_pseudo_value() else: return field.get_pseudo_value() - + def __repr__(self): - return 'Pseudo Manager for '+ '.'.join(self._model._meta.path) + ' for session %s' % self._session + return 'Pseudo Manager for ' + '.'.join(self._model._meta.path) + ' for session %s' % self._session def __str__(self): - return 'Pseudo Manager for '+ '.'.join(self._model._meta.path) + ' for session %s' % self._session + return 'Pseudo Manager for ' + '.'.join(self._model._meta.path) + ' for session %s' % self._session + class ListManager(CLIManager): def _get_config_list(self, filter_list=None): raise ManagerImplError('model %s has not implemented _get_config_list', self._model) @@ -1460,6 +1509,7 @@ # [{'field_name1':[display_value, search_sort_value], 'field_name2':[display_value, search_sort_value], ...}, ...] ret = self._get_config_list(filter_list) + # --- Following code is used to do search and sort ---- def ins_search(instance, search_dict): for field_name in search_dict["search_fields"]: @@ -1469,9 +1519,11 @@ if smart_str(ret_val).upper().find(search_dict["search_str"].upper()) != -1: return True return False + # search: {search_str:xxx, search_fields:[field_name1, field_name2, ...]} if search and "search_str" in search: - ret = [x for x in ret if ins_search(x, search)] + ret = [x for x in ret if ins_search(x, search)] + def ins_sort(instance1, instance2): field_name = sort_dict["field"] value1 = '' @@ -1481,6 +1533,7 @@ if field_name in instance2: value2 = instance2[field_name][1] return (value1 > value2) - (value1 < value2) + if sort_dict: if sort_dict["order"] == 'desc': order = True @@ -1503,7 +1556,6 @@ if max_num <= 0: response["data"] = rstdata[start:] else: - response["data"] = rstdata[start:start+max_num] + response["data"] = rstdata[start:start + max_num] response["total"] = len(rstdata) return response - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/node.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/node.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/node.py (working copy) @@ -1,58 +1,57 @@ - class ANNode(object): - creation_counter = 1 - def __init__(self, name=None, verbose_name=None): - self.name = name - self.verbose_name = verbose_name - self.creation_counter = ANNode.creation_counter - ANNode.creation_counter += 1 - self.primary_key = False + creation_counter = 1 - # may be called after deep copying from base model class - def contribute_to_class(self, cls, name): - self.name = name - self.model = cls - cls._meta.add_node(self) + def __init__(self, name=None, verbose_name=None): + self.name = name + self.verbose_name = verbose_name + self.creation_counter = ANNode.creation_counter + ANNode.creation_counter += 1 + self.primary_key = False - def __cmp__(self, other): - # This is needed because bisect does not take a comparison function. - return (self.creation_counter > other.creation_counter) - (self.creation_counter < other.creation_counter) + # may be called after deep copying from base model class + def contribute_to_class(self, cls, name): + self.name = name + self.model = cls + cls._meta.add_node(self) - # descriptor as used in models - class descriptor(object): - def __init__(self, name): - self._name = name - - def __get__(self, instance, owner): - return getattr(instance, "_"+self._name) + def __cmp__(self, other): + # This is needed because bisect does not take a comparison function. + return (self.creation_counter > other.creation_counter) - (self.creation_counter < other.creation_counter) - def __set__(self, instance, value): - setattr(instance, "_"+self._name, value) + # descriptor as used in models + class descriptor(object): + def __init__(self, name): + self._name = name - def __delete__(self, instance): - delattr(instance, "_"+self._name) + def __get__(self, instance, owner): + return getattr(instance, "_" + self._name) + def __set__(self, instance, value): + setattr(instance, "_" + self._name, value) + def __delete__(self, instance): + delattr(instance, "_" + self._name) + class ContainerNode(ANNode): - def __init__(self, sub_model): - self.sub_model = sub_model + def __init__(self, sub_model): + self.sub_model = sub_model - class descriptor(object): - def __init__(self, name): - self._name = name - - def __get__(self, instance, owner): - return instance.get_container(self._name) + class descriptor(object): + def __init__(self, name): + self._name = name + def __get__(self, instance, owner): + return instance.get_container(self._name) + + class ListNode(ANNode): - def __init__(self, sub_model): - self.sub_model = sub_model + def __init__(self, sub_model): + self.sub_model = sub_model - class descriptor(object): - def __init__(self, name): - self._name = name - - def __get__(self, instance, owner): - return instance.get_list(self._name) + class descriptor(object): + def __init__(self, name): + self._name = name + def __get__(self, instance, owner): + return instance.get_list(self._name) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py (working copy) @@ -2,42 +2,45 @@ Implementation of the ``Options`` class whose instance is for containing meta attributes of a model class. """ +import json import re from bisect import bisect -import sys -import copy -import json -from hive.utils import andebug +from collections import OrderedDict + from django.conf import settings -import hive.model.fields as fields -from hive.model.loading import app_cache_ready, get_model -from hive.utils import IntRange, nd_iter_coordinate, nd_iter_space, andebug, per_sess_cached_property, clear_per_sess_property_cache, dump_dict, get_current_lang, is_ipv4, is_ipv6 -from django.utils.translation import activate, deactivate_all, get_language from django.utils.encoding import force_str, smart_str -from collections import OrderedDict +from django.utils.translation import activate, deactivate_all, get_language from django.utils.translation import gettext_lazy as _ + import hive.exceptions as exceptions -from hive.model.query import DelayedQuery -from hive.document import get_model_element, get_xml_reference, status_str2obj, DOCUMENT_OK, DOCUMENT_NEW, DOCUMENT_PENDING +import hive.model.fields as fields +from hive.document import get_model_element, get_xml_reference, status_str2obj, DOCUMENT_NEW from hive.model.fields import DefaultValueCondition +from hive.model.loading import app_cache_ready, get_model +from hive.model.query import DelayedQuery +from hive.utils import IntRange, nd_iter_coordinate, nd_iter_space, andebug, clear_per_sess_property_cache, dump_dict, \ + get_current_lang, is_ipv4, is_ipv6 + try: from hive.model.stats import * except: pass # Calculate the verbose_name by converting from InitialCaps to "uppercase with spaces". -get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).title().strip() +get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', + class_name).title().strip() # original function of Django -#get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).lower().strip() +# get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).lower().strip() DEFAULT_NAMES = ('verbose_name', 'verbose_name_plural', 'ordering', 'unique_together', 'permissions', 'get_latest_by', 'order_with_respect_to', 'app_label', 'help_text', 'field_group_sequence', - 'abstract', 'managable', 'proxy', 'auto_created', 'show_im_export_button','hide_import_button', + 'abstract', 'managable', 'proxy', 'auto_created', 'show_im_export_button', 'hide_import_button', 'quantity', 'profile', 'path', 'repr_from_instance', 'repr_by_pk', 'button_submit_text', 'addable', 'list_config_options', 'list_stats_options', '_verbose_name', '_help_text', 'generate_xml', 'cache_expire_time', 'meta_type', 'licenses', 'pk_to_id', 'id_to_pk', 'asso_ref_to_instance_id') + class Options(object): """ All attributes except for fields are contained in the ``Meta`` class (actually just a container), which are collected at model class loading phase to establish an :py:class:`Options ` object, this object is accessible using ``._meta`` or ``._meta``. All these magic are cont rolled with the **meta class** of ``ANModel`` - the :py:class:`ANModelBase `. @@ -63,6 +66,7 @@ :param cache_expire_time: expire time for the model value in Cache. default is 0 (not expire) :type cache_expire_time: int """ + def __init__(self, meta, app_label=None, path=None): self.hide_import_button = False self.field_group_sequence = [] @@ -77,8 +81,8 @@ self._help_text = None self.ordering = [] self.licenses = [] - self.unique_together = [] - self.permissions = [] + self.unique_together = [] + self.permissions = [] self.object_name, self.app_label = None, app_label self.path = path self.get_latest_by = None @@ -89,7 +93,7 @@ self.managable = True self.addable = True self.cache_expire_time = 0 - self.field_cache_expire = {} # {:True/False, ...} + self.field_cache_expire = {} # {:True/False, ...} # this is for list widget # list_config_options # { @@ -146,12 +150,12 @@ self.abstract_parents = OrderedDict() self.children = OrderedDict() self.conditional_deriving_fields = [] - + self.duplicate_targets = {} self.auto_created = False self.quantity = '0..65535' self.profile = False - self._associations = OrderedDict() # Reference from association objs to AssoFields + self._associations = OrderedDict() # Reference from association objs to AssoFields # To handle various inheritance situations, we need to track where # managers came from (concrete or abstract base classes). @@ -165,8 +169,8 @@ self.repr_by_pk = None self._actions = [] self.object_switch_field = None - - #self.document = get_model_element(self.app_label, self.path) + + # self.document = get_model_element(self.app_label, self.path) self._package = None self._default_node = None self.button_submit_text = lambda options: _('Create the %s') % str(options.verbose_name) @@ -177,7 +181,7 @@ self.generate_xml = True self.stats_mapper = None - def get_addable_list(self,current_model): + def get_addable_list(self, current_model): addable_list = [] child_class = current_model._meta.children.keys() for cls in child_class: @@ -194,8 +198,6 @@ return addable_list def contribute_to_class(self, cls, name): - from django.db import connection - from django.db.backends.util import truncate_name cls._meta = self self._model = cls @@ -215,7 +217,7 @@ if name.startswith('_'): del meta_attrs[name] if name == 'verbose_name' or name == 'help_text': - meta_attrs['_'+name] = meta_attrs[name] + meta_attrs['_' + name] = meta_attrs[name] del meta_attrs[name] for attr_name in DEFAULT_NAMES: if attr_name in meta_attrs: @@ -238,7 +240,6 @@ self.quantity = IntRange(self.quantity) - # Any leftover attributes must be invalid. if meta_attrs != {}: raise TypeError("'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys())) @@ -277,15 +278,18 @@ def add_field_group(self, group): """ Add a ``FieldGroup`` object to the model class """ if self.object_name == 'HTTPVirtualService': - andebug('hive.debug', 'field group added into the sorted dict: %s %d bisec %d' % (group.name, group.creation_counter, bisect(self.field_groups.values(), group))) + andebug('hive.debug', + 'field group added into the sorted dict: %s %d bisec %d' % (group.name, group.creation_counter, + bisect(self.field_groups.values(), + group))) self.field_groups.insert(bisect(self.field_groups.values(), group), group.name, group) if self.object_name == 'HTTPVirtualService': for each_name, each_value in self.field_groups.iteritems(): andebug('hive.debug', 'field group now includes: %s %d' % (each_name, each_value.creation_counter)) - + def get_field_group(self, group_name): # get FieldGroup from the model class, if not found, return None - if group_name in self.field_groups: + if group_name in self.field_groups: return self.field_groups[group_name] else: return None @@ -297,38 +301,39 @@ """ if not group: if 'default' not in self.field_groups: - default_grp = fields.FieldGroup(name='default', verbose_name=_('Basic Settings'), editable=(True if self.profile else False)) + default_grp = fields.FieldGroup(name='default', verbose_name=_('Basic Settings'), + editable=(True if self.profile else False)) self.field_groups.insert(0, 'default', default_grp) # the creation counter of 'default' field group is set to 0 # the default group won't call the contribute_to_class method, so we should set the class reference manually here default_grp.model = self._model - + self.field_groups['default'].append_field(field) - + # Insert the given field in the order in which it was created, using # the "creation_counter" attribute of the field. self.local_fields.insert(bisect(self.local_fields, field), field) self._setup_index(field) if field.object_switch: self.object_switch_field = field - + if hasattr(self, '_field_cache'): del self._field_cache del self._field_name_cache if hasattr(self, '_name_map'): del self._name_map - + def del_field(self, field): """ Delete a ``Field`` object to the model class. !!! Only delete in local_fields, do NOT delete in field_group """ self._remove_index(field) self.local_fields.remove(field) - + if field.object_switch: self.object_switch_field = None - + if hasattr(self, '_field_cache'): del self._field_cache del self._field_name_cache @@ -339,14 +344,14 @@ def real_del_field(self, field): """ Delete a ``Field`` object to the model class and in the field_group. """ - + self._remove_index(field) self.local_fields.remove(field) field.group.del_field(field) if field.object_switch: self.object_switch_field = None - + if hasattr(self, '_field_cache'): del self._field_cache del self._field_name_cache @@ -397,9 +402,10 @@ raw = force_str(self.verbose_name) activate(lang) return raw + verbose_name_raw = property(verbose_name_raw) - #@per_sess_cached_property + # @per_sess_cached_property @property def document(self): return get_model_element(self.app_label, self.path) @@ -411,11 +417,11 @@ top_pkg = self.path[0] return get_xml_reference(top_pkg, self.document.find('./verbose_name'), default=self._verbose_name) return self._verbose_name - + @verbose_name.setter def verbose_name(self, new_value): self._verbose_name = new_value - + @property def help_text(self): " The help text of the model " @@ -423,7 +429,7 @@ top_pkg = self.path[0] return get_xml_reference(top_pkg, self.document.find('./help_text'), default=self._help_text) return self._help_text - + @help_text.setter def help_text(self, new_value): self._help_text = new_value @@ -434,7 +440,7 @@ if self.document is not None: return self.document.findtext('./help_link', default='') return '' - + def get_help_link(self): if self.help_link: return get_current_lang() + '/' + self.help_link @@ -442,23 +448,24 @@ return self._package.get_help_link() else: return '' - + def set_package(self, package): self._package = package - + @property def document_status(self): if self.document is not None: # read the status from document status_str = self.document.get('status', default='OK') - return status_str2obj(status_str) # this model already exist in current document + return status_str2obj(status_str) # this model already exist in current document else: return DOCUMENT_NEW - + def reload_document(self): clear_per_sess_property_cache(self) + # clear all field grps' and fields' property cache - #for each_field_grp in self.field_groups.values(): + # for each_field_grp in self.field_groups.values(): # clear_property_cache(each_field_grp) def reload_per_field(f): clear_per_sess_property_cache(f) @@ -468,7 +475,7 @@ for each_field in self.fields: reload_per_field(each_field) - + def _fields(self): """ Returns the list of field objects @@ -482,8 +489,9 @@ except AttributeError: self._fill_fields_cache() return self._field_name_cache + fields = property(_fields) - + def get_fields_with_model(self): """ Returns a sequence of (field, model) pairs for all fields. The "model" @@ -536,7 +544,7 @@ return cache[name] except KeyError: raise fields.FieldDoesNotExist('%s has no field named %r' - % (self.object_name, name)) + % (self.object_name, name)) def get_all_field_names(self): """ @@ -564,7 +572,7 @@ continue rtn_list.append(each_field) return rtn_list - + def _init_name_map(self): """ Initialises the field name -> field object mapping. @@ -592,7 +600,7 @@ res.insert(0, parent) return res raise TypeError('%r is not an ancestor of this model' - % model._meta.module_name) + % model._meta.module_name) def get_parent_list(self): """ @@ -630,7 +638,7 @@ if not hasattr(self, '_ordered_objects'): objects = [] # TODO - #for klass in get_models(get_app(self.app_label)): + # for klass in get_models(get_app(self.app_label)): # opts = klass._meta # if opts.order_with_respect_to and opts.order_with_respect_to.rel \ # and self == opts.order_with_respect_to.rel.to._meta: @@ -644,20 +652,21 @@ assert association assert field self._associations[association] = field - + # obsoleted !!! for old Associations only - def add_association_obsolete(self, association, end, target_end): + def add_association_obsolete(self, association, end, target_end): """ Add an association to the model class, This should be called after the model is already loaded """ self._associations[association] = None - + # Register the association to the corresponding asso field for each in self.fields: # Importing the builtin.py will cause round importing. # So check the interface function here rather than # using 'isinstance' - if getattr(each, 'obsoleted', False) and hasattr(each, 'register_association') and association.has_ref(each.ref): + if getattr(each, 'obsoleted', False) and hasattr(each, 'register_association') and association.has_ref( + each.ref): each.register_association(association, end, target_end) self._associations[association] = each break @@ -669,7 +678,7 @@ @property def associations(self): return self._associations - + @property def parents(self): """ Returns a ``SortedDict`` of parents of the field class @@ -677,25 +686,26 @@ Here 'parents' means class inheritence tree rather than model package tree """ return self.abstract_parents - + def register_deriving_field(self, field_name, condition_value, sub_class): if field_name not in self.conditional_deriving_fields: self.conditional_deriving_fields.append(field_name) - + if sub_class in self.children: self.children[sub_class][field_name] = condition_value else: - self.children[sub_class] = SortedDict({field_name:condition_value}) - + self.children[sub_class] = {field_name: condition_value} + def get_all_concrete_children(self): final_rtn_list = [] + def _all_direct_children(model_meta, rtn_list): rtn_list.extend([each for each in model_meta.children.keys() if not each._meta.abstract]) for each in model_meta.children.keys(): _all_direct_children(each._meta, rtn_list) + _all_direct_children(self, final_rtn_list) return final_rtn_list - def get_child_by_field_cond(self, field_cond_dict): """ Get child class by a dict of field conditions @@ -722,7 +732,7 @@ if current_matching_field_count > max_matching_field_count: max_matching_field_count = current_matching_field_count matching_sub_class = sub_class - + # Search is over for current level, enter next level if matching_sub_class is None: raise KeyError('No sub class matching these field conditions') @@ -730,7 +740,7 @@ return matching_sub_class._meta.get_child_by_field_cond(field_cond_dict) else: return matching_sub_class - + def _find_deriving_fields(self, children_data, field_dict): for sub_class in children_data: for field_name in children_data[sub_class]: @@ -749,7 +759,7 @@ @property def all_deriving_data(self): - deriving_data={} + deriving_data = {} self._all_deriving_data(self.children, deriving_data) return deriving_data @@ -758,36 +768,36 @@ field_dict = {} self._find_deriving_fields(self.children, field_dict) return field_dict.keys() - #return str(field_dict.keys()) + # return str(field_dict.keys()) def get_ajax_url(self, action, param_dict=None): if param_dict: - param_str = '&'.join([str(key)+'='+str(value) for key,value in param_dict.iteritems()]) + param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.iteritems()]) if param_str != '': param_str = '?' + param_str else: param_str = '' return '/api/' + self.app_label + '/' + '/'.join(self.path) + '/_' + action + param_str - + def append_action_cls(self, name, action_cls): # register actions to related fields - #if action_cls.related: - #self.get_field(action_cls.related).register_related_action(action_cls) + # if action_cls.related: + # self.get_field(action_cls.related).register_related_action(action_cls) for field in action_cls.option_fields: field.register_related_action(action_cls) field.init_sub_fields() self._actions.insert(bisect(self._actions, action_cls), action_cls) - + @property def actions(self): return self._actions - + def get_action_by_name(self, name): for each in self.actions: if each.name == name: return each return None - + @property def explicit_actions(self): tmp_acts = [] @@ -795,7 +805,7 @@ if each_act.model_action and each_act.condition(): tmp_acts.append(each_act) return tmp_acts - + def gen_dataset_for_update(self, existing_queryset, count=100): # for now, ignore existing_queryset it = nd_iter_space(len(self.fields)) @@ -819,7 +829,7 @@ dataset_list.append(dataset) i += 1 return dataset_list - + def gen_test_dataset_for_fieldset(self, fieldset, count=100, pk_dict_list=None, fixed_assos={}): it = nd_iter_coordinate(len(fieldset)) dataset_list = [] @@ -829,30 +839,31 @@ while i < count: bias_list = it.next() dataset = {} - + field_validation_error = 0 for j in range(len(fieldset)): - #print self.verbose_name + "in gen_basic_dataset j - " + str(j) + "-" + self.fields[j].name + # print self.verbose_name + "in gen_basic_dataset j - " + str(j) + "-" + self.fields[j].name # Filter out non-critical association fields if isinstance(fieldset[j], fields.builtin.AssoField2): fixed_asso_value = fixed_assos.get(fieldset[j].name, None) pk_dict = pk_dict_list[i] if pk_dict_list else None # Should ASSERT when both fixed_asso_value and pk_dict are None - #print "=== pk_dict %s, fixed_value %s" % (str(pk_dict), str(fixed_asso_value)) - dataset[fieldset[j].name] = fieldset[j].get_basic_case_value(pk_dict=pk_dict, fixed_value=fixed_asso_value) - #print "==== get result from get_basic_case_value: " + str(dataset[fieldset[j].name]) + # print "=== pk_dict %s, fixed_value %s" % (str(pk_dict), str(fixed_asso_value)) + dataset[fieldset[j].name] = fieldset[j].get_basic_case_value(pk_dict=pk_dict, + fixed_value=fixed_asso_value) + # print "==== get result from get_basic_case_value: " + str(dataset[fieldset[j].name]) else: dataset[fieldset[j].name] = fieldset[j].get_iterable_value(bias_list[j]) - #print "got value - " + str(dataset[fieldset[j].name]) + # print "got value - " + str(dataset[fieldset[j].name]) try: dataset[fieldset[j].name] = fieldset[j].clean(dataset[fieldset[j].name], None) except exceptions.ValidationError: field_validation_error += 1 - + # if there is validation error, skip this case if field_validation_error > 0: continue - + # Check whether dataset is repeated if dataset in dataset_list: repeated += 1 @@ -864,34 +875,35 @@ repeated = 0 dataset_list.append(dataset) i += 1 - - #print dataset_list + + # print dataset_list return dataset_list - + def gen_test_dataset(self, existing_data=None, fixed_pk=None, count=100): per_fieldset_results = [] left_fields = self.get_config_fields() pk_dict_list = None - + # primary key set if len(self.pk_set) > 0: pk_dict_list = self.gen_test_dataset_for_fieldset(self.pk_set, count) per_fieldset_results.append(pk_dict_list) for each_field in self.pk_set: left_fields.remove(each_field) - + # unique sets for name, unique_set in self.unique_sets.iteritems(): if name is True: # this is a primary key set continue - per_fieldset_results.append(self.gen_test_dataset_for_fieldset(unique_set, count, pk_dict_list=pk_dict_list)) + per_fieldset_results.append( + self.gen_test_dataset_for_fieldset(unique_set, count, pk_dict_list=pk_dict_list)) for each_field in unique_set: left_fields.remove(each_field) - + # all left fields into a fieldset per_fieldset_results.append(self.gen_test_dataset_for_fieldset(left_fields, count, pk_dict_list=pk_dict_list)) - + # the first list, must copy it out !!! or else the "update" below will cause bug merged_dataset_list = copy.deepcopy(per_fieldset_results[0]) @@ -910,7 +922,7 @@ return True for m in self.children: return m._meta.has_statistics - + return False @property @@ -918,29 +930,29 @@ for name in self.field_groups: group = self.field_groups[name] if group.isstats and group.is_cli_statistics == False: - return True + return True for m in self.children: return m._meta.has_not_only_cli_stats - return False + return False @property def has_not_only_asso_stats(self): for name in self.field_groups: group = self.field_groups[name] if group.isstats and group.is_asso_statistics == False: - return True + return True for m in self.children: return m._meta.has_not_only_asso_stats - return False + return False @property def stats_from_userland(self): for name in self.field_groups: group = self.field_groups[name] if group.isstats and group.is_cli_statistics == False and group.stats_source == "userland": - return True + return True for m in self.children: return m._meta.stats_from_userland return False @@ -949,20 +961,20 @@ def stats_mapper_cls(self): if self.stats_mapper == None: self.stats_mapper = CreateMapper(self._model) - return self.stats_mapper - + return self.stats_mapper + @property def has_fast_stats_interface(self): path_list = copy.copy(self.path) if path_list[0] == "loadbalancing": path_list[0] = "lb" - result = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all", False) + result = getattr(libustat, "ustat_" + '_'.join(path_list) + "_all", False) if result != False: - result = True + result = True return result - + def get_indexed_list(self): - ret = {"group":{}, "order":{}, "search":{}} + ret = {"group": {}, "order": {}, "search": {}} for field in self.fields: if field.name in self.all_deriving_fields: ret["group"][field.name] = str(field.verbose_name) @@ -983,9 +995,9 @@ else: if len(l1) == 1: l2 = [instance_id] - else:#more than one pk model - #if object name itself include "-": - #for example: '"vs-1"', 'g1-"-xxx-"', '"-xxx-"-g1', 'g1-"rs-2"', '"rs-2"-g1', ... + else: # more than one pk model + # if object name itself include "-": + # for example: '"vs-1"', 'g1-"-xxx-"', '"-xxx-"-g1', 'g1-"rs-2"', '"rs-2"-g1', ... if '"' in instance_id: items = re.findall(r'"\S.*?"', instance_id) new_items = [item.strip('"') for item in items] @@ -998,11 +1010,11 @@ l2.append(each) else: l2.append(new_items[i]) - i+=1 + i += 1 else: l2 = instance_id.split('-') if len(l1) == len(l2): - pk_dict = dict(zip(l1,l2)) + pk_dict = dict(zip(l1, l2)) else: pk_dict = None if pk_dict: @@ -1011,12 +1023,11 @@ pk_dict[pk_field_name] = pk_field.to_python(instance_id) return pk_dict - def targets_id_to_pk_list(self, targets_id, is_pk_str): pk_dict = self.instance_id_to_pk(targets_id) pk_list = [] if pk_dict: - if is_pk_str:#true means ["{}"], false means [{}] + if is_pk_str: # true means ["{}"], false means [{}] pk_list.append(dump_dict(pk_dict)) else: pk_list.append(pk_dict) @@ -1032,12 +1043,12 @@ fields.append(each.name) ret[field_group.name] = fields return ret - + def get_group_condition_dict(self): ret = {} for field_group in self.field_groups.values(): if field_group.condition: - ret[field_group.name] = field_group.condition.to_dict() + ret[field_group.name] = field_group.condition.to_dict() return ret def get_field_default_value_confition_dict(self): @@ -1046,7 +1057,7 @@ if isinstance(field.default, DefaultValueCondition): ret[field.name] = field.default.to_dict() return ret - + def get_field_condition_dict(self): ret = {} for field in self.fields: @@ -1062,13 +1073,13 @@ get_sub_field_dict['owngroupfield'] = field.name ret[sub_field.name] = get_sub_field_dict return ret - + def mark_delay_query(self, data_dict): for field in self.fields: if field.name not in data_dict: data_dict[field.name] = DelayedQuery return data_dict - + # type: 'config'|'stats' def get_list_columns(self, type='config', format=False, custom_options=None): columns = [] @@ -1077,9 +1088,11 @@ custom_options = self.list_config_options if "check_box_hide" not in custom_options or not custom_options["check_box_hide"]: # add check box as the first column - columns.append({'name':'check_box_column', 'title':'', 'width':'20px', 'orderable':False, 'defaultContent':''}) + columns.append( + {'name': 'check_box_column', 'title': '', + 'width': '20px', 'orderable': False, 'defaultContent': ''}) if "columns" in custom_options and custom_options["columns"]: - columns_list = copy.deepcopy(custom_options["columns"]) + columns_list = copy.deepcopy(custom_options["columns"]) for column in columns_list: column['data'] = column['name'] if 'title' not in column: @@ -1089,13 +1102,13 @@ field = self.get_field(field_name) column['title'] = str(field.verbose_name) except: - column['title'] = 'Error:no title for '+ field_name + column['title'] = 'Error:no title for ' + field_name else: - column['title'] = str(column['title']) + column['title'] = str(column['title']) if 'defaultContent' not in column: column['defaultContent'] = '/' columns.append(column) - + else: # default: get all config fields name in base class for gn, g in self.field_groups.iteritems(): @@ -1103,16 +1116,18 @@ for f in g.fields: if f.hidden or f.hide_list: continue - columns.append({'name':f.attname, 'title': str(f.verbose_name), 'defaultContent':'/', 'data':f.attname}) + columns.append({'name': f.attname, 'title': str(f.verbose_name), 'defaultContent': '/', + 'data': f.attname}) # For config list, the -1 column is for 'external data' if "external_data_enable" not in custom_options or custom_options["external_data_enable"]: - columns.append({'title': '', 'name':'external_data_column', 'orderable':False, 'defaultContent':'
    '}) + columns.append({'title': '', 'name': 'external_data_column', 'orderable': False, + 'defaultContent': '
    '}) elif type == 'stats': # for stats, the first column is for pk_str, the second one is for name and the last column is for 'More' - columns.append({'name': 'repr','title':'Name', 'data':'repr'}) + columns.append({'name': 'repr', 'title': 'Name', 'data': 'repr'}) custom_options = self.list_stats_options if "columns" in custom_options and custom_options["columns"]: - columns_list = copy.deepcopy(custom_options["columns"]) + columns_list = copy.deepcopy(custom_options["columns"]) for column in columns_list: column['data'] = column['name'] if 'title' not in column: @@ -1122,13 +1137,13 @@ field = self.get_field(field_name) column['title'] = str(field.verbose_name) except: - column['title'] = 'Error:no title for '+ field_name + column['title'] = 'Error:no title for ' + field_name else: - column['title'] = str(column['title']) + column['title'] = str(column['title']) if 'defaultContent' not in column: column['defaultContent'] = '/' columns.append(column) - + else: # default: get all config fields name in stats class for gn, g in self.field_groups.iteritems(): @@ -1136,13 +1151,15 @@ for f in g.fields: if f.hidden: continue - columns.append({'name':f.attname, 'title': str(f.verbose_name), 'defaultContent':'/', 'data':f.attname}) - columns.append({'title': '', 'name':'external_data_column', 'orderable':False, 'defaultContent':'
    '}) + columns.append({'name': f.attname, 'title': str(f.verbose_name), 'defaultContent': '/', + 'data': f.attname}) + columns.append({'title': '', 'name': 'external_data_column', 'orderable': False, + 'defaultContent': '
    '}) if format: return json.dumps(columns) else: return columns - + def get_list_config(self, custom_options=None, type='config'): if type == 'config': if not custom_options: @@ -1150,36 +1167,36 @@ else: ret = dict(custom_options) ret['columns'] = self.get_list_columns(type, False, custom_options) - # list_config_options - # { - # columns: [ - # { - # 'name': field_name, must set; for asso field, support use '-' to get value for a - # field in the target model. Example: for RealService, use 'group_member.group' - # to show the group name. If use '-', 'title' must be set. - # 'title': display name, optional, default will use the verbose name of the field; - # if the 'name' is not a field_name for current class, 'title' must set - # 'width': any CSS value (3em, 20px etc) - # 'className': adds a class to each cell in a column - # 'defaultContent': if no data is set for this column, the default content will display - # }, - # ... - # ], default get all fields in basic fieldGroup. - # check_box_hide: True/False, default False; show(False) or hide(True) the check box - # add_button_hide: True/False, default False; show(False) or hide(True) the add button. - # If 'model._meta.addable' is False, 'add_button_hide' default value is True. - # add_button_url: String, if 'add_button_hide' is False, this must set - # delete_button_hide: True/False, default False; show(False) or hide(True) the delete button - # delete_button_url: String, if 'delete_button_hide' is False, this must set - # delete_ingore_warning: True/False, default False; Ingore waring result message if set to True, only Error and Info will be shown. - # click_enable: True/False, default True; if True, click a row will jump to a new page - # external_data_enable: True/False, default True; if Ture, a column will add in the last, and click it - # will show more data - # filter_config:[{'name':xx, 'display_name':xx, 'values':[[value1, display_value1],[value2, display_value2]], 'deriving':True/False}] - # used to create filter control. default: all indexed field in columns. - # quantity: quantity limitation of the model class. default meta.quantity.max - # } - # + # list_config_options + # { + # columns: [ + # { + # 'name': field_name, must set; for asso field, support use '-' to get value for a + # field in the target model. Example: for RealService, use 'group_member.group' + # to show the group name. If use '-', 'title' must be set. + # 'title': display name, optional, default will use the verbose name of the field; + # if the 'name' is not a field_name for current class, 'title' must set + # 'width': any CSS value (3em, 20px etc) + # 'className': adds a class to each cell in a column + # 'defaultContent': if no data is set for this column, the default content will display + # }, + # ... + # ], default get all fields in basic fieldGroup. + # check_box_hide: True/False, default False; show(False) or hide(True) the check box + # add_button_hide: True/False, default False; show(False) or hide(True) the add button. + # If 'model._meta.addable' is False, 'add_button_hide' default value is True. + # add_button_url: String, if 'add_button_hide' is False, this must set + # delete_button_hide: True/False, default False; show(False) or hide(True) the delete button + # delete_button_url: String, if 'delete_button_hide' is False, this must set + # delete_ingore_warning: True/False, default False; Ingore waring result message if set to True, only Error and Info will be shown. + # click_enable: True/False, default True; if True, click a row will jump to a new page + # external_data_enable: True/False, default True; if Ture, a column will add in the last, and click it + # will show more data + # filter_config:[{'name':xx, 'display_name':xx, 'values':[[value1, display_value1],[value2, display_value2]], 'deriving':True/False}] + # used to create filter control. default: all indexed field in columns. + # quantity: quantity limitation of the model class. default meta.quantity.max + # } + # if 'check_box_hide' not in ret: ret['custom_options'] = False if 'add_button_hide' not in ret: @@ -1212,7 +1229,8 @@ field = self.get_field(name) if field.indexed or field.name in self.all_deriving_fields: if field.type_name == 'enum': - tmp_ret = {'name':field.attname, 'display_name':str(field.verbose_name), 'values':[], 'deriving':False} + tmp_ret = {'name': field.attname, 'display_name': str(field.verbose_name), 'values': [], + 'deriving': False} if field.name in self.all_deriving_fields: tmp_ret['deriving'] = True for each in field.values: @@ -1232,24 +1250,26 @@ ret['auto_refresh_time'] = 10 return json.dumps(ret) - + def set_default_node(self, node): self._default_node = node - + def get_default_node(self): if self._default_node: return self._default_node - + for parent in self.parents.keys(): if parent._meta._default_node: return parent._meta._default_node - + return None + class ListOptions(Options): """ To use this type of meta, set "meta_type" to ListOptions in schema. """ + # compare with get_list_columns() in Options, support set columns as field in other model # use 'model' to set the model path def get_list_columns(self, type='config', format=False, custom_options=None): @@ -1259,9 +1279,11 @@ custom_options = self.list_config_options if "check_box_hide" not in custom_options or not custom_options["check_box_hide"]: # add check box as the first column - columns.append({'name':'check_box_column', 'title':'', 'width':'20px', 'orderable':False, 'defaultContent':''}) + columns.append( + {'name': 'check_box_column', 'title': '', + 'width': '20px', 'orderable': False, 'defaultContent': ''}) if "columns" in custom_options and custom_options["columns"]: - columns_list = copy.deepcopy(custom_options["columns"]) + columns_list = copy.deepcopy(custom_options["columns"]) for column in columns_list: column['data'] = column['name'] if 'title' not in column: @@ -1278,13 +1300,13 @@ field = self.get_field(field_name) column['title'] = str(field.verbose_name) except: - column['title'] = 'Error:no title for '+ field_name + column['title'] = 'Error:no title for ' + field_name else: - column['title'] = str(column['title']) + column['title'] = str(column['title']) if 'defaultContent' not in column: column['defaultContent'] = '/' columns.append(column) - + else: # default: get all config fields name in base class for gn, g in self.field_groups.iteritems(): @@ -1292,16 +1314,18 @@ for f in g.fields: if f.hidden or f.hide_list: continue - columns.append({'name':f.attname, 'title': str(f.verbose_name), 'defaultContent':'/', 'data':f.attname}) + columns.append({'name': f.attname, 'title': str(f.verbose_name), 'defaultContent': '/', + 'data': f.attname}) # For config list, the -1 column is for 'external data' if "external_data_enable" not in custom_options or custom_options["external_data_enable"]: - columns.append({'title': '', 'name':'external_data_column', 'orderable':False, 'defaultContent':'
    '}) + columns.append({'title': '', 'name': 'external_data_column', 'orderable': False, + 'defaultContent': '
    '}) elif type == 'stats': # for stats, the first column is for pk_str, the second one is for name and the last column is for 'More' - columns.append({'name': 'repr','title':'Name', 'data':'repr'}) + columns.append({'name': 'repr', 'title': 'Name', 'data': 'repr'}) custom_options = self.list_stats_options if "columns" in custom_options and custom_options["columns"]: - columns_list = copy.deepcopy(custom_options["columns"]) + columns_list = copy.deepcopy(custom_options["columns"]) for column in columns_list: column['data'] = column['name'] if 'title' not in column: @@ -1318,13 +1342,13 @@ field = self.get_field(field_name) column['title'] = str(field.verbose_name) except: - column['title'] = 'Error:no title for '+ field_name + column['title'] = 'Error:no title for ' + field_name else: - column['title'] = str(column['title']) + column['title'] = str(column['title']) if 'defaultContent' not in column: column['defaultContent'] = '/' columns.append(column) - + else: # default: get all config fields name in stats class for gn, g in self.field_groups.iteritems(): @@ -1332,8 +1356,10 @@ for f in g.fields: if f.hidden: continue - columns.append({'name':f.attname, 'title': str(f.verbose_name), 'defaultContent':'/', 'data':f.attname}) - columns.append({'title': '', 'name':'external_data_column', 'orderable':False, 'defaultContent':'
    '}) + columns.append({'name': f.attname, 'title': str(f.verbose_name), 'defaultContent': '/', + 'data': f.attname}) + columns.append({'title': '', 'name': 'external_data_column', 'orderable': False, + 'defaultContent': '
    '}) if format: return json.dumps(columns) else: @@ -1348,12 +1374,13 @@ pk_field = meta.get_field(pk_fields[i]) if pk_field.type_name == 'UnionField': if is_ipv4(l[i]): - l[i] = {'ipv4':l[i]} + l[i] = {'ipv4': l[i]} elif is_ipv6(l[1]): - l[i] = {'ipv6':l[i]} + l[i] = {'ipv6': l[i]} else: l[i] = str(l[i]) else: l[i] = pk_field.to_python(l[i]) return l + return f Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/package.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/package.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/package.py (working copy) @@ -3,10 +3,14 @@ """ import json from collections import OrderedDict + from django.utils.encoding import smart_str, force_str -from hive.utils import anlog, anerror, andebug, ObjCache, per_sess_cached_property, clear_per_sess_property_cache, get_current_lang + from hive.document import get_package_element, get_xml_reference +from hive.utils import anerror, ObjCache, clear_per_sess_property_cache, \ + get_current_lang + def create_tree(package, tree): ret = {} if tree['type'] == 'Menu': @@ -24,15 +28,18 @@ model = tmp.models[model_name] ret['data'] = str(model._meta.verbose_name) if model._meta.profile: - ret['metadata'] = {"url":"/hide_tree", "page_url":model._meta.get_ajax_url('box_instance_monitor')} + ret['metadata'] = {"url": "/hide_tree", "page_url": model._meta.get_ajax_url('box_instance_monitor')} else: - ret['metadata'] = {"url":model._meta.get_ajax_url('tree'), "page_url":model._meta.get_ajax_url('box_module_monitor')} - + ret['metadata'] = {"url": model._meta.get_ajax_url('tree'), + "page_url": model._meta.get_ajax_url('box_module_monitor')} + return ret + class ANPackageMeta(object): """ Stores the package meta info. Every :class:`ANPackage` object has a field ``_meta`` pointing to it's meta object """ + def __init__(self): self.app = '' self.app_label = '' @@ -43,9 +50,10 @@ self.parent_package = None self.models = OrderedDict() self.verbose_name = '' - self.help_text = '' - self.child_seq = [] # for RESTful API documentation + self.help_text = '' + self.child_seq = [] # for RESTful API documentation + class ANPackage(object): """ Create a model package from existing Python module. @@ -54,6 +62,7 @@ :param module: The Python module to create the package from. The module might contains ``verbose_name`` and ``help_text`` attributes :type module: module """ + def __init__(self, app_name, module): path_list = module.__name__.split('.') try: @@ -80,7 +89,7 @@ self._meta.help_text = getattr(module, 'help_text', '') self._meta.monitor_tree = getattr(module, 'monitor_tree', {}) self._meta.child_seq = getattr(module, 'child_seq', []) - #self._meta.document = get_package_element(self._meta.app_label, self._meta.path) + # self._meta.document = get_package_element(self._meta.app_label, self._meta.path) def append_package(self, name, ele): """ Append a sub package @@ -134,7 +143,7 @@ rtn.extend(self.models.keys()) return rtn - #@per_sess_cached_property + # @per_sess_cached_property @property def document(self): return get_package_element(self._meta.app_label, self._meta.path) @@ -154,14 +163,14 @@ top_pkg = '' if len(self.path) == 0 else self.path[0] return get_xml_reference(top_pkg, self.document.find('./help_text'), default=self._meta.help_text) return self._meta.help_text - + @property def help_link(self): " A link to the user guide for the package " if self.document is not None: return self.document.findtext('./help_link', default='') return '' - + def get_help_link(self): if self.help_link: return get_current_lang() + '/' + self.help_link @@ -169,7 +178,7 @@ return self.parent_package.get_help_link() else: return '' - + def __getattr__(self, name): try: if name in self._meta.sub_packages: @@ -184,7 +193,6 @@ def __iter__(self): return iter(self._meta.models) - def __unicode__(self): return self._meta.verbose_name @@ -199,40 +207,43 @@ if hasattr(self._meta, 'verbose_name'): return force_str(self._meta.verbose_name) return '%s object' % self.__class__.__name__ - - def get_ajax_url(self, param_str=''): + + def get_ajax_url(self, param_str=''): return '/api/' + self._meta.app_label + '/' + '/'.join(self._meta.path) + '/' + param_str - + def get_tree(self, tojson=True): if self._meta.monitor_tree: ret = create_tree(self, self._meta.monitor_tree) else: - ret = {'data': str(self._meta.verbose_name), 'children':[]} + ret = {'data': str(self._meta.verbose_name), 'children': []} for k in self.models: m = self.models[k] if not m._meta.parents and m._meta.has_statistics: - sub = {'data': str(m._meta.verbose_name), 'metadata': {'url': m._meta.get_ajax_url('tree'), "page_url":m._meta.get_ajax_url('box_module_monitor')}} + sub = {'data': str(m._meta.verbose_name), 'metadata': {'url': m._meta.get_ajax_url('tree'), + "page_url": m._meta.get_ajax_url( + 'box_module_monitor')}} ret['children'].append(sub) for k in self.sub_packages: p = self.sub_packages[k] sub = p.get_tree(False) if sub['children']: ret['children'].append(sub) - + if tojson: return json.dumps(ret) else: return ret - + def reload_document(self): clear_per_sess_property_cache(self) - #self._meta.document = get_package_element(self._meta.app_label, self._meta.path) + # self._meta.document = get_package_element(self._meta.app_label, self._meta.path) for sub_model in self.models.values(): sub_model._meta.reload_document() for sub_pkg in self.sub_packages.values(): sub_pkg.reload_document() -@ObjCache('session','package') + +@ObjCache('session', 'package') class ANPackageAgent(object): def __init__(self, package, session): self._package = package @@ -276,8 +287,7 @@ return child.get_manager(self._session) def __repr__(self): - return 'ANPackageAgent for '+self._package.__repr__() + return 'ANPackageAgent for ' + self._package.__repr__() def __str__(self): - return 'ANPackageAgent for '+self._package.__str__() - + return 'ANPackageAgent for ' + self._package.__str__() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/query.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/query.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/query.py (working copy) @@ -1,19 +1,21 @@ import copy -import itertools -import sys -from hive.exceptions import ValidationError, ModelQueryException, FieldDoesNotExist -import json -from hive.utils import dump_dict, andebug, anerror -from hive.model.utils import normalize_asso_value, get_value_asso_idx, reduce_asso_value_item, clear_config_list_data, clear_config_list_all import time + from djproject.an_settings import CACHE_SWITCH +from hive.exceptions import ValidationError, ModelQueryException +from hive.model.utils import normalize_asso_value, get_value_asso_idx, reduce_asso_value_item, clear_config_list_data, \ + clear_config_list_all +from hive.utils import dump_dict, andebug, anerror + class DelayedQuery: pass + class CacheCleared: pass - + + class QuerySet(object): def __init__(self, model_cls, initial_value=None): self._cache = [] @@ -27,10 +29,10 @@ new_model.clean_fields() except TypeError as e: raise - #raise ModelQueryException(e.message) + # raise ModelQueryException(e.message) except ValidationError as e: raise - #raise ModelQueryException(e) + # raise ModelQueryException(e) # pk field should be cleaned here if CACHE_SWITCH: for pk_field in model_cls._meta.pk_set: @@ -61,6 +63,7 @@ if compare_left != compare_right: return False return True + return [x for x in self._cache if determine(x, filter_dict)] def get(self, pk_dict): @@ -70,7 +73,7 @@ else: pk_str = dump_dict(pk_dict) return self.__getitem__(pk_str) - + def get_field(self, pk_dict, field_name): obj = self.get(pk_dict) return getattr(obj, field_name, None) @@ -78,11 +81,11 @@ def append_obj(self, model_instance): if model_instance.pk_str() in self._cache_dict: err_str = model_instance.repr() - #err_str = ', '.join(['%s - %s' % (key, value) for key,value in model_instance.pk_dict().iteritems()]) + # err_str = ', '.join(['%s - %s' % (key, value) for key,value in model_instance.pk_dict().iteritems()]) raise ModelQueryException('Already has this object: ' + err_str) self._cache_dict[model_instance.pk_str()] = model_instance self._cache.append(model_instance) - + def update_obj(self, model_instance): old_obj = self._cache_dict[model_instance.pk_str()] idx = self._cache.index(old_obj) @@ -94,18 +97,18 @@ def merge(self, qs): for instance in qs._cache: self.append_obj(instance) - + def remove(self, instance): try: self._cache.remove(instance) except ValueError: raise - + try: del self._cache_dict[instance.pk_str()] except KeyError: raise - + def __repr__(self): return 'QuerySet:' + repr(self._cache) @@ -126,20 +129,21 @@ return self._cache_dict[k] else: raise TypeError - + def __copy__(self): - newone = type(self)(None) # doesn't need to specify the model here + newone = type(self)(None) # doesn't need to specify the model here newone._cache = copy.copy(self._cache) newone._cache_dict = copy.copy(self._cache_dict) return newone + class QueryCache(object): def __init__(self, model_cls, initial_value=None): - self._cache_dict = {} # all instances for all modules - self._cache_time = {} # time for each insrance, key is same with _cache_dict - self._cache_module = {} # index list in _cache_dict for each module - self._cache_module_time = {} # time for each index list in _cache_module, key is same with _cache_module - self._cache_module_stat = {} # stat for each module; 1-config ready; 2-stat ready; 3-both ready, key is same with _cache_module + self._cache_dict = {} # all instances for all modules + self._cache_time = {} # time for each insrance, key is same with _cache_dict + self._cache_module = {} # index list in _cache_dict for each module + self._cache_module_time = {} # time for each index list in _cache_module, key is same with _cache_module + self._cache_module_stat = {} # stat for each module; 1-config ready; 2-stat ready; 3-both ready, key is same with _cache_module if initial_value: for each_dict in initial_value: try: @@ -153,7 +157,7 @@ key = '.'.join(model_cls._meta.path) + new_model.pk_str() assert not key in self._cache_dict, 'Primary key not unique in a queryset' self._cache_dict[key] = new_model - self._cache_time[key] = {'_instance':time.time()} + self._cache_time[key] = {'_instance': time.time()} def set_stat(self, module, stat): key = module._meta.app_label + '.' + '.'.join(module._meta.path) @@ -162,7 +166,7 @@ self._cache_module_stat[key] = old | stat except KeyError: self._cache_module_stat[key] = stat - + def get_stat(self, module): key = module._meta.app_label + '.' + '.'.join(module._meta.path) return self._cache_module_stat[key] @@ -170,7 +174,7 @@ def get_module_time(self, module): key = module._meta.app_label + '.' + '.'.join(module._meta.path) return self._cache_module_time[key] - + def set_module(self, module, index_list): key = module._meta.app_label + '.' + '.'.join(module._meta.path) self._cache_module[key] = index_list @@ -181,7 +185,7 @@ if key in self._cache_module: return self._cache_module[key] return None - + def remove_module(self, module): clear_config_list_data(module) key = module._meta.app_label + '.' + '.'.join(module._meta.path) @@ -207,14 +211,14 @@ del self._cache_module_stat[key] except KeyError: pass - + def get(self, module, pk_dict): key = '.'.join(module._meta.path) + dump_dict(pk_dict) return self.__getitem__(key) - + def get_by_index(self, key): return self.__getitem__(key) - + def get_field(self, module, pk_dict, field_name): obj = self.get(module, pk_dict) return getattr(obj, field_name, None) @@ -223,12 +227,12 @@ key = '.'.join(model_instance._meta._model._meta.path) + model_instance.pk_str() if key in self._cache_dict: err_str = model_instance.repr() - #err_str = ', '.join(['%s - %s' % (key, value) for key,value in model_instance.pk_dict().iteritems()]) + # err_str = ', '.join(['%s - %s' % (key, value) for key,value in model_instance.pk_dict().iteritems()]) raise ModelQueryException('Already has this object: ' + err_str) self._cache_dict[key] = model_instance - self._cache_time[key] = {'_instance':time.time()} + self._cache_time[key] = {'_instance': time.time()} return key - + def update_obj(self, model_instance): key = '.'.join(model_instance._meta._model._meta.path) + model_instance.pk_str() if key in self._cache_dict: @@ -240,7 +244,7 @@ self._cache_time[key][field.attname] = time.time() else: self._cache_dict[key] = model_instance - self._cache_time[key] = {'_instance':time.time()} + self._cache_time[key] = {'_instance': time.time()} return key def updata_time(self, model_instance, fields=None): @@ -250,7 +254,7 @@ for f in fields: self._cache_time[key][f] = time.time() else: - self._cache_time[key] = {'_instance':time.time()} + self._cache_time[key] = {'_instance': time.time()} except KeyError: anerror('hive.debug', 'Fail updating time with invalid key: %s' % key) @@ -272,7 +276,7 @@ del self._cache_time[key][field] except KeyError: anerror('hive.debug', 'Fail removing time with invalid key: %s' % key) - + def merge(self, qs): ret = [] for instance in qs._cache: @@ -293,14 +297,14 @@ del self._cache_time[key] except KeyError: pass - + def clear_all(self): self._cache_dict.clear() self._cache_time.clear() - self._cache_module.clear() + self._cache_module.clear() self._cache_module_stat.clear() clear_config_list_all() - + def __repr__(self): return 'QueryCache' @@ -316,6 +320,7 @@ else: return None + manager_cache = QueryCache(None) """ @@ -327,26 +332,29 @@ """ + # used to clean fields after cache change # if "model" is set, will clean fields for all instance in the model, and "instance" value will be ingore # if "model" is NOT set, and "instance" is set, only clean fields for this instance. def cache_clean_fields(model=None, instance=None): if model: - #andebug('hive.pro', 'start clean fields for model:%u' % time.time()) + # andebug('hive.pro', 'start clean fields for model:%u' % time.time()) index_list = manager_cache.get_module(model) if index_list: for index in index_list: ins = manager_cache.get_by_index(index) if ins: cache_clean_fields(instance=ins) - #andebug('hive.pro', 'end clean fields for model:%u' % time.time()) + # andebug('hive.pro', 'end clean fields for model:%u' % time.time()) elif instance: instance.clean_fields() - + + def get_ins_key(model_instance): key = '.'.join(model_instance._meta._model._meta.path) + model_instance.pk_str() return key + def set_cache_all(module, queryset, config=False, stats=False, fast_data=False): if not CACHE_SWITCH: return True @@ -362,6 +370,7 @@ if not fast_data: cache_clean_fields(model=module) + def set_cache_one(instance): if not CACHE_SWITCH: return True @@ -370,11 +379,12 @@ manager_cache.updata_time(instance) cache_clean_fields(instance=instance) + def update_cache_all(module, queryset, config=False, stats=False, fast_data=False): if not CACHE_SWITCH: return True - #index_list = manager_cache.get_module(module) - #if index_list == None: + # index_list = manager_cache.get_module(module) + # if index_list == None: clear_config_list_data(module) index_list = [] pk_list = [] @@ -388,6 +398,7 @@ if not fast_data: cache_clean_fields(model=module) + def update_cache_one(module, pk, data, updata_time=False): if not CACHE_SWITCH: return True @@ -403,6 +414,7 @@ else: return False + def update_cache_field(instance, field_name, field_value): if not CACHE_SWITCH: return True @@ -412,6 +424,7 @@ manager_cache.updata_time(instance, [field_name]) cache_clean_fields(instance=instance) + def get_cache_all(module): if not CACHE_SWITCH: return None @@ -429,6 +442,7 @@ return None return ret + # return None if not found def get_cache_one(module, pk, check_expire=True): if not CACHE_SWITCH: @@ -453,18 +467,20 @@ return instance return instance + # to avoid cycle call, make sure the instance or the field is marked as expire before call this function def mark_expire_asso(instance, field_name=None): if not CACHE_SWITCH: return True andebug('hive.debug', 'mark_expire_asso: instance is %s, field_name is %s' % (instance, field_name)) - + def mark_expire_asso_field(field, instance): andebug('hive.debug', 'mark_expire_asso_field: instance is %s, field is %s' % (instance, field.name)) if (instance.is_delayed_query(field.name)): # if the field value is delayed query # we clear the whole cache - andebug('hive.debug', 'mark_expire_asso_field met with DelayedQuery field %s, clearing the whole cache...' % field.name) + andebug('hive.debug', + 'mark_expire_asso_field met with DelayedQuery field %s, clearing the whole cache...' % field.name) manager_cache.clear_all() return CacheCleared pks = field._get_val_from_obj(instance) @@ -482,13 +498,13 @@ rtn = mark_expire_field(ins, field.tgt_field[idx].attname, asso=False) if rtn is CacheCleared: return CacheCleared - + if instance: if field_name: field = instance._meta.get_field(field_name) if field.type_name == 'asso': return mark_expire_asso_field(field, instance) - + else: # all asso field done_fields = {} @@ -500,8 +516,9 @@ rtn = mark_expire_asso_field(field, instance) if rtn is CacheCleared: return CacheCleared - -def mark_expire_all(module, each = True, clear_all_if_has_asso=False): + + +def mark_expire_all(module, each=True, clear_all_if_has_asso=False): if not CACHE_SWITCH: return True andebug('hive.debug', 'mark_expire_all: module is %s, each - %s' % (module, each)) @@ -518,9 +535,10 @@ rtn = mark_expire_asso(ins) if rtn is CacheCleared: return CacheCleared - + manager_cache.remove_module(module) + def mark_expire_one(module, pk): if not CACHE_SWITCH: return True @@ -530,8 +548,8 @@ manager_cache.remove_module(ins._meta._model) manager_cache.remove(module, pk) return mark_expire_asso(ins) - + def mark_expire_field(instance, field_name, asso=True): if not CACHE_SWITCH: return True @@ -548,17 +566,17 @@ return CacheCleared setattr(instance, field_name, DelayedQuery) manager_cache.remove_time(instance, field_name) - + def check_expire_all(module, config=False, stats=False): if not CACHE_SWITCH: return True index_list = manager_cache.get_module(module) if index_list != None: stat = manager_cache.get_stat(module) - if config and not stat&1: + if config and not stat & 1: return True - if stats and not stat&2: + if stats and not stat & 2: return True expire_time = module._meta.cache_expire_time if not expire_time: @@ -568,8 +586,9 @@ if age + expire_time < now: return True return False - return True + return True + def check_expire(module, pk, field_name=None): if not CACHE_SWITCH: return True @@ -592,10 +611,12 @@ return False return True + def clear_cache_all(): manager_cache.clear_all() return True + # Chech the field is expired in cache. If the instance in cache is not same with the instance in parameter, also return False. def check_field_expire_in_cache(model, pk_dict, instance, field_name): if not CACHE_SWITCH: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/rest.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/rest.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/rest.py (working copy) @@ -1,21 +1,20 @@ -from hive.model.loading import get_app -from hive.model.query import mark_expire_one -from django.http import Http404, HttpResponse import json -from hive.exceptions import ValidationError, ModelQueryException, ActionPerformException, FieldDoesNotExist, \ - ManagerImplError, RestfulException -import copy import time -from hive.utils import HiveEnvironment, andebug, url2obj, IntRange +from collections import OrderedDict +from xml.dom.minidom import parseString + from django.utils.translation import gettext_lazy as _ + import tools.dicttoxml as dicttoxml import tools.xmltodict as xmltodict -from xml.dom.minidom import parseString -from collections import OrderedDict -from hive.session import current_app -from cm.lib.postgres_db import DB from cm.lib.activation_server import ActivationServer -from ctypes import POINTER, cast +from cm.lib.postgres_db import DB +from hive.exceptions import ValidationError, ModelQueryException, ActionPerformException, ManagerImplError, \ + RestfulException +from hive.model.loading import get_app +from hive.model.query import mark_expire_one +from hive.session import current_app +from hive.utils import andebug try: from hive.model.stats import * @@ -940,7 +939,7 @@ db.execute_sql(save_sql) except Exception as e: if "unique constraint" in str(e): - pass # The device already exists. + pass # The device already exists. # The following approach was deprecated. # save_sql = "REPLACE INTO discover_device(ip, restapi_port, restapi_username, restapi_password, status, version, timestamp, restapi_protocol) Values " + "('%(device_ip)s', '%(restapi_port)d', '%(restapi_username)s', '%(restapi_password)s', '%(status)s', '%(version)s', '%(time)s', '%(restapi_protocol)s')" % data # db.execute_sql(save_sql) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/signals.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/signals.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/signals.py (working copy) @@ -13,4 +13,4 @@ post_syncdb = Signal() -m2m_changed = Signal() \ No newline at end of file +m2m_changed = Signal() Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/stats.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/stats.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/stats.py (working copy) @@ -1,52 +1,55 @@ -from ctypes import * -import os import copy -import pdb +from ctypes import * -#libustat = None +# libustat = None libustat = cdll.LoadLibrary('/lib/' + 'libustat.so') libustat.ustat_interface_initial() -exclude_model = ['admintools.config_mgmt.ViewConfig', 'admintools.sys_mgmt.SystemLicense', "loadbalancing.slb.vs.http.URLFilter"] +exclude_model = ['admintools.config_mgmt.ViewConfig', 'admintools.sys_mgmt.SystemLicense', + "loadbalancing.slb.vs.http.URLFilter"] class STAILQ_HEAD(Structure): _fields_ = [("stqh_first", c_void_p), - ("stqh_last", POINTER(c_void_p)) - ] + ("stqh_last", POINTER(c_void_p)) + ] + class STAILQ_ENTRY(Structure): _fields_ = [("stqe_next", c_void_p) - ] + ] + class ustat_model_q_item(Structure): _fields_ = [("data", c_void_p), - ("entry", STAILQ_ENTRY) - ] + ("entry", STAILQ_ENTRY) + ] -class StatsModel(Structure): + +class StatsModel(Structure): _fields_ = [] - + + def need_exclude(model): if '.'.join(model._meta.path) in exclude_model: return True return False - def Process_AssoField(model, stats_field, path_list, prefix): for pk_field in model._meta.pk_set: if pk_field.get_internal_type() == "AssoField2": - continue + continue if pk_field.length != None: - if path_list[2] == "healthcheck": - stats_field.append((prefix + "_" + pk_field.name, c_char * (pk_field.length.get_max() + 23))) - else: - stats_field.append((prefix + "_" + pk_field.name, c_char * pk_field.length.get_max())) + if path_list[2] == "healthcheck": + stats_field.append((prefix + "_" + pk_field.name, c_char * (pk_field.length.get_max() + 23))) + else: + stats_field.append((prefix + "_" + pk_field.name, c_char * pk_field.length.get_max())) else: stats_field.append((prefix + "_" + pk_field.name, c_char * 25)) + def Process_UnionField(field, stats_field): max_length = 0 for each_field in field.fields: @@ -61,7 +64,7 @@ if max_length > 40: continue else: - max_length = 40 + max_length = 40 continue if field.fields[each_field].get_internal_type() == "NetmaskField": @@ -82,15 +85,15 @@ if max_length > 10: continue else: - max_length = 10 + max_length = 10 continue - if field.fields[each_field].length.get_max() > max_length: max_length = field.fields[each_field].length.get_max() stats_field.append((field.name, c_char * max_length)) return + def Process_EnumField(field, stats_field): max_length = 0 index = 0 @@ -100,6 +103,7 @@ index = index + 1 stats_field.append((field.name, c_char * max_length)) + def CreateMapper(model): stats_model_fields = {} stats_model_fields["_fields_"] = [] @@ -119,8 +123,9 @@ if pk_field.target_end[index].model in exist_model: index = index + 1 else: - exist_model[pk_field.target_end[index].model] = pk_field.target_end[index].model - gen_str = Process_AssoField(pk_field.target_end[index].model, stats_model_fields["_fields_"], path_list, pk_field.get_attname()) + exist_model[pk_field.target_end[index].model] = pk_field.target_end[index].model + gen_str = Process_AssoField(pk_field.target_end[index].model, stats_model_fields["_fields_"], + path_list, pk_field.get_attname()) continue if pk_field.get_internal_type() == "IPAddressField" or pk_field.get_internal_type() == "IPv4AddressField": @@ -155,7 +160,6 @@ Process_EnumField(pk_field, stats_model_fields["_fields_"]) continue - if pk_field.length != None: if path_list[2] == "healthcheck": stats_model_fields["_fields_"].append((pk_field.attname, c_char * (pk_field.length.get_max() + 23))) @@ -164,8 +168,6 @@ else: stats_model_fields["_fields_"].append((pk_field.attname, c_char * 64)) - - for name in model._meta.field_groups: group = model._meta.field_groups[name] if group.isstats and group.is_cli_statistics == False: @@ -173,28 +175,28 @@ if (each_field.get_internal_type() == "StorableField"): if (each_field.fields.get_internal_type() == "IntegerField"): stats_model_fields["_fields_"].append((each_field.name, c_ulonglong)) - + if (each_field.fields.get_internal_type() == "Int8Field"): stats_model_fields["_fields_"].append((each_field.name, c_byte)) if (each_field.fields.get_internal_type() == "Int16Field"): stats_model_fields["_fields_"].append((each_field.name, c_short)) - + if (each_field.fields.get_internal_type() == "Int32Field"): stats_model_fields["_fields_"].append((each_field.name, c_int)) if (each_field.fields.get_internal_type() == "Int64Field"): stats_model_fields["_fields_"].append((each_field.name, c_long)) - + if (each_field.fields.get_internal_type() == "Uint8Field"): stats_model_fields["_fields_"].append((each_field.name, c_ubyte)) - + if (each_field.fields.get_internal_type() == "Uint16Field"): stats_model_fields["_fields_"].append((each_field.name, c_ushort)) - + if (each_field.fields.get_internal_type() == "Uint32Field"): stats_model_fields["_fields_"].append((each_field.name, c_uint)) - + if (each_field.fields.get_internal_type() == "Uint64Field"): stats_model_fields["_fields_"].append((each_field.name, c_ulong)) @@ -202,7 +204,7 @@ stats_model_fields["_fields_"].append((each_field.name, c_float)) if (each_field.fields.get_internal_type() == "PercentField"): - stats_model_fields["_fields_"].append((each_field.name, c_ulonglong)) + stats_model_fields["_fields_"].append((each_field.name, c_ulonglong)) if (each_field.fields.get_internal_type() == "BooleanField"): stats_model_fields["_fields_"].append((each_field.name, c_ubyte)) @@ -212,5 +214,5 @@ if (each_field.get_internal_type() == "BooleanField"): stats_model_fields["_fields_"].append((each_field.name, c_ubyte)) - - return type("ustat_" + '_'.join(path_list) + "_data_t", (Structure,), stats_model_fields) + + return type("ustat_" + '_'.join(path_list) + "_data_t", (Structure,), stats_model_fields) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/storage.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/storage.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/storage.py (working copy) @@ -1,86 +1,85 @@ -#!/usr/bin/env python -# encoding: utf-8 -""" -storage.py -Created by Wang Zhe on 2013-07-02. -Copyright (c) 2013 Array Networks. All rights reserved. -""" - -from hive.model.loading import get_app, get_model -import logging import json +import logging + from hive.model.legacycli import CLICmdError, BlankParser +from hive.model.loading import get_model from hive.notification import send_notification logger = logging.getLogger('avx') + class JSONFormat: - class Meta: - name = 'json' - # dict to json file - def serialize(self, data, fp): - return json.dump(data, fp, indent=4) + class Meta: + name = 'json' - # json file to dict - def deserialize(self, fp): - return json.load(fp) + # dict to json file + def serialize(self, data, fp): + return json.dump(data, fp, indent=4) + # json file to dict + def deserialize(self, fp): + return json.load(fp) + + class ModelStorageManager(object): - format = JSONFormat() - load_orders = {} - storage_directories = {} - # "write memory" handler, called by WebUI and CLI backend - def save_app_models(self, session, appname): - self.init_app_settings(appname) - data = {} - for key,value in self.load_orders[appname].items(): - if value['persist']: - path_list = key.split('.') - #logger.info(path_list) - try: - model = get_model(appname, path_list) - manager = session.get_model_manager(appname, path_list) - qs = manager.all() - data[key] = [] - for each in qs: - logger.info(each.get_field_dict()) - data[key].append(each.get_field_dict()) - except: - logger.error('Exception while saving %s' % key) - # /var/array/avx_models.json - with open(self.storage_directories[appname] + appname + "_models." + self.format.Meta.name, "w") as out: - self.format.serialize(data, out) - - # Load model config from file storage to dict. e.g. load_model('avx', 'vm.VirtualMachine') - def load_model(self, appname, model_path): - try: - # /var/array/avx_models.json - with open(self.storage_directories[appname] + appname + "_models." + self.format.Meta.name, "r") as fp: - data = self.format.deserialize(fp) - return data[model_path] - except: - logger.error('Exception while loading %s %s' % (appname, model_path) ) - - def init_app_settings(self, appname): - if appname not in self.load_orders: - # Load dict from app.settings.py - m = __import__(appname + '.settings', globals(), locals(),['model_load_orders', 'model_storage_directory']) - # logger.info(m.model_load_orders) - self.load_orders[appname] = m.model_load_orders - self.storage_directories[appname] = '/var/array/' - if 'm.model_storage_directory' in locals(): - # logger.info(m.model_storage_directory) - self.storage_directories[appname] = m.model_storage_directory - + format = JSONFormat() + load_orders = {} + storage_directories = {} + + # "write memory" handler, called by WebUI and CLI backend + def save_app_models(self, session, appname): + self.init_app_settings(appname) + data = {} + for key, value in self.load_orders[appname].items(): + if value['persist']: + path_list = key.split('.') + # logger.info(path_list) + try: + model = get_model(appname, path_list) + manager = session.get_model_manager(appname, path_list) + qs = manager.all() + data[key] = [] + for each in qs: + logger.info(each.get_field_dict()) + data[key].append(each.get_field_dict()) + except: + logger.error('Exception while saving %s' % key) + # /var/array/avx_models.json + with open(self.storage_directories[appname] + appname + "_models." + self.format.Meta.name, "w") as out: + self.format.serialize(data, out) + + # Load model config from file storage to dict. e.g. load_model('avx', 'vm.VirtualMachine') + def load_model(self, appname, model_path): + try: + # /var/array/avx_models.json + with open(self.storage_directories[appname] + appname + "_models." + self.format.Meta.name, "r") as fp: + data = self.format.deserialize(fp) + return data[model_path] + except: + logger.error('Exception while loading %s %s' % (appname, model_path)) + + def init_app_settings(self, appname): + if appname not in self.load_orders: + # Load dict from app.settings.py + m = __import__(appname + '.settings', globals(), locals(), ['model_load_orders', 'model_storage_directory']) + # logger.info(m.model_load_orders) + self.load_orders[appname] = m.model_load_orders + self.storage_directories[appname] = '/var/array/' + if 'm.model_storage_directory' in locals(): + # logger.info(m.model_storage_directory) + self.storage_directories[appname] = m.model_storage_directory + + modelStorageManager = ModelStorageManager() + # /save/exxx url handler class ModelStorageHandler(object): def __init__(self, request, session, appname): self.request = request self.session = session self.appname = appname - + def render(self): if self.appname == 'apv': self.session.cli.set_config() @@ -89,5 +88,4 @@ send_notification('info', 'System configuration is saved (write memory).') else: modelStorageManager.save_app_models(self.session, self.appname) - return json.dumps([{'result':'ok'}]) - + return json.dumps([{'result': 'ok'}]) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/utils.py (working copy) @@ -1,69 +1,75 @@ -#import pymongo -import time -import random -from hive.exceptions import DBException, FieldDoesNotExist -from djproject.an_settings import DBHOST,DBPORT import copy -from hive.utils import dump_dict, andebug, anerror, obj2url -from django.utils.encoding import smart_str +import random import threading +import time -TIMEFORMAT='%Y-%m-%d %X' +import pymongo +from django.utils.encoding import smart_str +from djproject.an_settings import DBHOST, DBPORT +from hive.exceptions import DBException, FieldDoesNotExist +from hive.utils import obj2url + +TIMEFORMAT = '%Y-%m-%d %X' + + class CONDITIONAL_NONE: pass + class DBProcess(object): MG_db = None con_list = {} - - def __init__(self, host=DBHOST, port=DBPORT, db_name='test'): + + def __init__(self, host=DBHOST, port=DBPORT, db_name='test'): try: MG_conn = pymongo.MongoClient(host, port) self.MG_db = MG_conn[db_name] for c in self.MG_db.collection_names(False): self.con_list[c] = 1 - + except: raise DBException - + def getcol(self, col_name, index=None, capped=False, size=200, max=1000000): if col_name in self.con_list: return self.MG_db[col_name] else: - if index == None: - index = [('pk',pymongo.ASCENDING), ('time',pymongo.DESCENDING), ('field',pymongo.ASCENDING)] - + if index is None: + index = [('pk', pymongo.ASCENDING), ('time', pymongo.DESCENDING), ('field', pymongo.ASCENDING)] + if capped: - self.MG_db.create_collection(col_name, capped=True, size=size*1048576, max=max) + self.MG_db.create_collection(col_name, capped=True, size=size * 1048576, max=max) else: pymongo.collection.Collection(self.MG_db, col_name, create=True) self.MG_db[col_name].ensure_index(index) self.con_list[col_name] = 1 return self.MG_db[col_name] - + def insert(self, col_name, data): col = self.getcol(col_name) return col.insert(data, continue_on_error=True) - + def col_insert(self, col, data): return col.insert(data, continue_on_error=True) - + def select(self, col_name, filt={}): col = self.getcol(col_name) return col.find(filt) - + def delete(self, col_name, query): col = self.getcol(col_name) return col.remove(query) - + def insert_time(self, col_name, data): d = {'time': time.time()} d.update(data) return self.insert(col_name, d) + class KEEPING: __name__ = 'keeping' + def process(self, field, input_data): ret = {} data = [] @@ -84,7 +90,8 @@ else: data.append([]) for r in input_data: - data[0].append([time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), field.fields.value_to_num(val=r.get('value'))]) + data[0].append([time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), + field.fields.value_to_num(val=r.get('value'))]) ret['data'] = data ret['data_name'] = data_name ret['unit_name'] = unit_name @@ -92,8 +99,10 @@ ret['value_limit'] = value_limit return ret + class DIFFERENCING: __name__ = 'differencing' + def process(self, field, input_data): ret = {} data = [] @@ -113,9 +122,10 @@ vd = field.fields.value_to_num(val=r.get('value')) i = 0 for n, v in vd.iteritems(): - if preval[i] == 'NoPre': + if preval[i] == 'NoPre': preval[i] = v - data[i].append([time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), (v-preval[i])/samp_freq]) + data[i].append( + [time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), (v - preval[i]) / samp_freq]) preval[i] = v i += 1 else: @@ -124,7 +134,8 @@ for r in input_data: if preval == 'NoPre': preval = field.fields.value_to_num(val=r.get('value')) - data[0].append([time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), (field.fields.value_to_num(val=r.get('value'))-preval)/samp_freq]) + data[0].append([time.strftime(TIMEFORMAT, time.localtime(r.get('time'))), + (field.fields.value_to_num(val=r.get('value')) - preval) / samp_freq]) preval = r.get('value') ret['data'] = data ret['data_name'] = data_name @@ -134,30 +145,33 @@ ret['value_limit'] = value_limit return ret + MethodList = { - KEEPING.__name__:KEEPING(), - DIFFERENCING.__name__:DIFFERENCING(), - } + KEEPING.__name__: KEEPING(), + DIFFERENCING.__name__: DIFFERENCING(), +} + + def get_method(method_name): return MethodList[method_name] - class irandom: int_bas = 0 int_times = 0 int_add = 0 incerase_bas = 0 incerase_times = 0 - def __init__(self): + + def __init__(self): pass - + def set_inc(self, val, vmin, vmax): irandom.incerase_bas += val if irandom.incerase_bas > 9000000: irandom.incerase_bas = vmin return irandom.incerase_bas - + def randint(self, vmin, vmax, increase=False): num = 100 vmin = 0 @@ -169,15 +183,15 @@ return irandom.incerase_bas else: irandom.int_times += 1 - - time = irandom.int_times/(num) - + + time = irandom.int_times / (num) + if time == irandom.int_add: - #return irandom.int_bas + # return irandom.int_bas pass else: irandom.int_add = time - + if time < 5: val = random.randint(1, 5) elif time < 10: @@ -220,11 +234,12 @@ val = random.randint(10, 30) else: val = random.randint(3, 7) - + if time == 400: irandom.int_times = 0 return val - + + # Utility Functions for associations def get_value_asso_idx(val_item, pop=False): if '_asso_idx' in val_item: @@ -232,11 +247,12 @@ else: return 0 + def normalize_asso_value_item(val_item, in_place=True): # For profile models, pk dict is None if val_item is None: return val_item - + assert type(val_item) is dict if not in_place: val_item = copy.copy(val_item) @@ -244,6 +260,7 @@ val_item['_asso_idx'] = 0 return val_item + def normalize_asso_value(val, in_place=True): assert type(val) is list if not in_place: @@ -252,27 +269,30 @@ normalize_asso_value_item(each, in_place=True) return val + def reduce_asso_value_item(val_item, recursive=False): if type(val_item) is not dict: return val_item if '_asso_idx' in val_item: del val_item['_asso_idx'] - + if recursive: for each_key, each_value in val_item.iteritems(): - if type(each_value) is list: # this might be value of an AssoField + if type(each_value) is list: # this might be value of an AssoField reduce_asso_value(each_value, recursive) return val_item + def reduce_asso_value(val, recursive=False): for each in val: reduce_asso_value_item(each, recursive) return val + def get_ajax_url_by_pk(model, pk_dict, action, param_dict=None): param_str = '' if param_dict: - param_str = '&'.join([str(key)+'='+str(value) for key,value in param_dict.iteritems()]) + param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.iteritems()]) if param_str != '': param_str = '?' + param_str @@ -285,8 +305,10 @@ # the values should be encoded into url string # they are later decoded at the ObjectAjaxHandler's __init__ pk_item_list.append(obj2url(val)) - return '/api/' + model._meta.app_label + '/' + '/'.join(model._meta.path) + '/_' + action + '/' + '/'.join(pk_item_list) + param_str + return '/api/' + model._meta.app_label + '/' + '/'.join(model._meta.path) + '/_' + action + '/' + '/'.join( + pk_item_list) + param_str + # support get value for asso target model field; field_name can be 'aaa-bbb', 'aaa-bbb-ccc' # if field_name contaions '-', 'manager' must set # value_type: 'default', 'display', 'display_for_list', @@ -302,7 +324,7 @@ if current_field_value: for each in current_field_value: copied_value = dict(each) - asso_idx = get_value_asso_idx(copied_value, pop=True) # don't change the original value + asso_idx = get_value_asso_idx(copied_value, pop=True) # don't change the original value target_model = current_field.target_end[asso_idx].model target_manager = target_model.get_manager(manager._session) target_instance = target_manager.get(copied_value) @@ -334,32 +356,35 @@ value = field.value_to_display(instance, style=False) else: value = field._get_val_from_obj(instance) - return {field:value} + return {field: value} except KeyError: return None except AttributeError: if field_name == 'repr': - return {field_name:instance.repr()} + return {field_name: instance.repr()} return None except FieldDoesNotExist: if field_name == 'repr': - return {field_name:instance.repr()} + return {field_name: instance.repr()} return None -#-----------------------level two cache---------------------------# + +# -----------------------level two cache---------------------------# external_cache_data = {} + def get_config_list_data(model, start=0, max_num=0, search_dict=None, filter_dict=None, sort_dict=None): key_str = '.'.join(model._meta.path) + '_config_list' data = external_cache_data[key_str] if data == None: # a thread is getting the data, and is not finished raise KeyError + def fun_search(value, search_dict, filter_dict): if filter_dict: for field_name, field_value in filter_dict.iteritems(): if field_name in value: - compare_left = value[field_name+'__ori__'] + compare_left = value[field_name + '__ori__'] else: compare_left = None if compare_left != field_value: @@ -368,19 +393,22 @@ if value['search_value'].upper().find(search_dict["search_str"].upper()) == -1: return False return True + def do_search(search_dict, filter_dict): if search_dict and "search_str" in search_dict and search_dict["search_str"]: return True if filter_dict: return True return False + if do_search(search_dict, filter_dict): - tmp_data = [x for x in data if fun_search(x, search_dict, filter_dict)] + tmp_data = [x for x in data if fun_search(x, search_dict, filter_dict)] data = tmp_data def fun_sort(value1, value2): - field_name = sort_dict["field"] - return (value1[field_name] > value2[field_name]) - (value1[field_name] < value2[field_name]) + field_name = sort_dict["field"] + return (value1[field_name] > value2[field_name]) - (value1[field_name] < value2[field_name]) + if sort_dict: if sort_dict["order"] == 'desc': order = True @@ -391,36 +419,37 @@ if max_num <= 0: response["data"] = data[start:] else: - response["data"] = data[start:start+max_num] + response["data"] = data[start:start + max_num] response["total"] = len(data) return response + class Config_List_Level2_Cache(threading.Thread): def __init__(self, manager, field_list, external_data): threading.Thread.__init__(self) self.manager = manager self.field_list = field_list self.external_data = external_data - + def run(self): ret = self.manager.all()[0:] rst_data = [] for ins in ret: - sub_ret = {"search_value":""} + sub_ret = {"search_value": ""} for field_name in self.field_list: ret_val = get_complex_field(ins, field_name, self.manager, 'display_for_list') if ret_val: value_list = [str(x) for x in ret_val.values()] value = ', '.join(value_list) sub_ret[field_name] = value - sub_ret[field_name+'__ori__'] = getattr(ins, field_name, ""); + sub_ret[field_name + '__ori__'] = getattr(ins, field_name, ""); sub_ret['search_value'] += value # second get left fields value for each children instance - if self.external_data: + if self.external_data: tmp_ret = self.manager.get_external_data_for_list(ins, self.field_list) sub_ret['external'] = tmp_ret - #add hide data in response, hide data include "pk_str", "action", "functions" + # add hide data in response, hide data include "pk_str", "action", "functions" hide_data = self.manager.get_internal_data_for_list(ins) sub_ret['internal'] = hide_data @@ -431,6 +460,7 @@ # if key_str not in external_cache_data, means config changed during get list data, so should not save current data in cache external_cache_data[key_str] = rst_data + def save_config_list_data(manager, field_list, external_data=True): key_str = '.'.join(manager._model._meta.path) + '_config_list' if key_str in external_cache_data: @@ -440,6 +470,7 @@ thread.start() return + def clear_config_list_data(module): key_str = '.'.join(module._meta.path) + '_config_list' if key_str in external_cache_data: @@ -455,6 +486,7 @@ if key_str in external_cache_data: del external_cache_data[key_str] + def clear_config_list_all(): global external_cache_data external_cache_data = {} Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/validators.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/validators.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/validators.py (working copy) @@ -1,18 +1,20 @@ import platform import re import urllib -import urllib.request as urlopen import urllib.parse as urlparse +import urllib.request as urlopen -from hive.exceptions import ValidationError -from django.utils.translation import gettext_lazy as _ from django.utils.encoding import smart_str from django.utils.ipv6 import is_valid_ipv6_address +from django.utils.translation import gettext_lazy as _ + +from hive.exceptions import ValidationError from hive.utils import IntRange # These values, if given to validate(), will trigger the self.required check. EMPTY_VALUES = (None, '', [], (), {}) + class LengthValidator(object): length = IntRange('0..max') @@ -26,27 +28,32 @@ else: value_len = len(value) if not self.length.validate(value_len): - raise ValidationError(_('The length %(length_value)d is out of the valid length range: %(length_limit)s.') % {'length_value':len(value),'length_limit':self.length}) + raise ValidationError( + _('The length %(length_value)d is out of the valid length range: %(length_limit)s.') % { + 'length_value': len(value), 'length_limit': self.length}) @property def js_func(self): return "$HIVE.LengthValidator(%d, %d)" % (self.length.get_min(), self.length.get_max()) - + + class ScopeValidator(object): scope = IntRange('min..max') def __init__(self, scope=None): if scope is not None: - self.scope = scope + self.scope = scope def __call__(self, value): if not self.scope.validate(value): - raise ValidationError(_('The value %(scope_value)d is out of the valid value range: %(scope_limit)s.') % {'scope_value':value,'scope_limit':self.scope}) + raise ValidationError(_('The value %(scope_value)d is out of the valid value range: %(scope_limit)s.') % { + 'scope_value': value, 'scope_limit': self.scope}) @property def js_func(self): return "$HIVE.ScopeValidator(%d, %d)" % (self.scope.get_min(), self.scope.get_max()) + class RegexValidator(object): regex = '' message = _(u'Enter a valid value.') @@ -70,11 +77,12 @@ """ if not self.regex.search(smart_str(value)): raise ValidationError(self.message, code=self.code) - + @property def js_func(self): return "$HIVE.RegexValidator(new RegExp('%s'), \"%s\")" % (self.regex.pattern, str(self.message)) + class MixedCharValidator(object): scope = IntRange('min..max') regex = '' @@ -93,7 +101,7 @@ # Compile the regex if it was not passed pre-compiled. if isinstance(self.regex, str): - self.regex = re.compile(self.regex) + self.regex = re.compile(self.regex) def __call__(self, value): """ @@ -102,15 +110,20 @@ try: value = int(value) if not self.scope.validate(value): - raise ValidationError(_('The value %(scope_value)d is out of the valid value range: %(scope_limit)s.') % {'scope_value':value,'scope_limit':self.scope}) + raise ValidationError( + _('The value %(scope_value)d is out of the valid value range: %(scope_limit)s.') % { + 'scope_value': value, 'scope_limit': self.scope}) except: if not self.regex.search(smart_str(value)): raise ValidationError(self.message, code=self.code) - + @property def js_func(self): - return "$HIVE.MixedCharValidator(%d, %d, new RegExp('%s'), \"%s\")" % (self.scope.get_min(), self.scope.get_max(), self.regex.pattern, str(self.message)) + return "$HIVE.MixedCharValidator(%d, %d, new RegExp('%s'), \"%s\")" % (self.scope.get_min(), + self.scope.get_max(), self.regex.pattern, + str(self.message)) + class SingleValueValidator(object): message = u'The value must be %s' code = 'invalid' @@ -122,24 +135,26 @@ if not value == self.target_value: raise ValidationError(self.message % self.target_value, code=self.code) + class ValueTupleValidator(object): message = u'The value must be one of %s' code = 'invalid' - + def __init__(self, target_values=()): self.target_values = target_values - + def __call__(self, value): if not value in self.target_values: raise ValidationError(self.message % str(self.target_values), code=self.code) + class URLValidator(RegexValidator): regex = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... - r'localhost|' #localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip - r'(?::\d+)?' # optional port + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port r'(?:/?|[/?]\S+)$', re.IGNORECASE) def __init__(self, verify_exists=False, @@ -157,8 +172,8 @@ value = smart_str(value) scheme, netloc, path, query, fragment = urlparse.urlsplit(value) try: - netloc = netloc.encode('idna') # IDN -> ACE - except UnicodeError: # invalid domain part + netloc = netloc.encode('idna') # IDN -> ACE + except UnicodeError: # invalid domain part raise e url = urlparse.urlunsplit((scheme, netloc, path, query, fragment)) super(URLValidator, self).__call__(url) @@ -173,7 +188,7 @@ "The URLField verify_exists argument has intractable security " "and performance issues. Accordingly, it has been deprecated.", DeprecationWarning - ) + ) headers = { "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5", @@ -190,10 +205,10 @@ try: req = urlopen.Request(url, None, headers) req.get_method = lambda: 'HEAD' - #Create an opener that does not support local file access + # Create an opener that does not support local file access opener = urlopen.OpenerDirector() - #Don't follow redirects, but don't treat them as errors either + # Don't follow redirects, but don't treat them as errors either error_nop = lambda *args, **kwargs: True http_error_processor = urlopen.HTTPErrorProcessor() http_error_processor.http_error_301 = error_nop @@ -219,7 +234,7 @@ opener.open(req) except ValueError: raise ValidationError(_(u'Enter a valid URL.'), code='invalid') - except: # urllib2.URLError, httplib.InvalidURL, etc. + except: # urllib2.URLError, httplib.InvalidURL, etc. raise broken_error @@ -229,6 +244,7 @@ except (ValueError, TypeError): raise ValidationError('') + class EmailValidator(RegexValidator): def __call__(self, value): @@ -246,24 +262,30 @@ else: raise + email_re = re.compile( r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom # quoted-string, see also http://tools.ietf.org/html/rfc2822#section-3.2.5 r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-\011\013\014\016-\177])*"' r')@((?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$)' # domain - r'|\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$', re.IGNORECASE) # literal form, ipv4 address (SMTP 4.1.3) + r'|\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$', + re.IGNORECASE) # literal form, ipv4 address (SMTP 4.1.3) validate_email = EmailValidator(email_re, _(u'Enter a valid email address.'), 'invalid') slug_re = re.compile(r'^[-\w]+$') -validate_slug = RegexValidator(slug_re, _(u"Enter a valid string consisting of only letters, numbers, underscores or hyphens."), 'invalid') +validate_slug = RegexValidator(slug_re, + _(u"Enter a valid string consisting of only letters, numbers, underscores or hyphens."), + 'invalid') ipv4_re = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}$') validate_ipv4_address = RegexValidator(ipv4_re, _(u'Enter a valid IPv4 address.'), 'invalid') + def validate_ipv6_address(value): if not is_valid_ipv6_address(value): raise ValidationError(_(u'Enter a valid IPv6 address.'), code='invalid') + def validate_ipv46_address(value): try: validate_ipv4_address(value) @@ -273,12 +295,14 @@ except ValidationError: raise ValidationError(_(u'Enter a valid IPv4 or IPv6 address.'), code='invalid') + ip_address_validator_map = { 'both': ([validate_ipv46_address], _('Enter a valid IPv4 or IPv6 address.')), 'ipv4': ([validate_ipv4_address], _('Enter a valid IPv4 address.')), 'ipv6': ([validate_ipv6_address], _('Enter a valid IPv6 address.')), } + def ip_address_validators(protocol, unpack_ipv4): """ Depending on the given parameters returns the appropriate validators for @@ -295,13 +319,15 @@ raise ValueError("The protocol '%s' is unknown. Supported: %s" % (protocol, ip_address_validator_map.keys())) + comma_separated_int_list_re = re.compile('^[\d,]+$') -validate_comma_separated_integer_list = RegexValidator(comma_separated_int_list_re, _(u'Enter only digits separated by commas.'), 'invalid') +validate_comma_separated_integer_list = RegexValidator(comma_separated_int_list_re, + _(u'Enter only digits separated by commas.'), 'invalid') class BaseValidator(object): compare = lambda self, a, b: a is not b - clean = lambda self, x: x + clean = lambda self, x: x message = _(u'The value %(show_value)s is not the valid value %(limit_value)s.') code = 'limit_value' @@ -318,25 +344,28 @@ params=params, ) + class MaxValueValidator(BaseValidator): compare = lambda self, a, b: a > b message = _(u'Ensure this value is less than or equal to %(limit_value)s.') code = 'max_value' + class MinValueValidator(BaseValidator): compare = lambda self, a, b: a < b message = _(u'Ensure this value is greater than or equal to %(limit_value)s.') code = 'min_value' + class MinLengthValidator(BaseValidator): compare = lambda self, a, b: a < b - clean = lambda self, x: len(x) + clean = lambda self, x: len(x) message = _(u'Ensure this value has at least %(limit_value)d characters (it has %(show_value)d).') code = 'min_length' + class MaxLengthValidator(BaseValidator): compare = lambda self, a, b: a > b - clean = lambda self, x: len(x) + clean = lambda self, x: len(x) message = _(u'Ensure this value has at most %(limit_value)d characters (it has %(show_value)d).') code = 'max_length' - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/__init__.py (working copy) @@ -1,31 +1,31 @@ -import sys -import os -import unittest -import json import copy -from django.http import HttpResponse from collections import OrderedDict + from django.http import Http404 -from jinja2 import Environment, PackageLoader, ChoiceLoader -from hive.utils import andebug, HiveEnvironment, get_current_session, get_current_lang -from hive.model.loading import get_model, get_app -from hive.session import ANSession, current_app +from django.http import HttpResponse from django.template import RequestContext -from djproject.an_settings import THEMES -from hive.lang import get_langs_info from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt -exec("import %s.company as company" % current_app()) +from jinja2 import PackageLoader, ChoiceLoader +from cm import company +from djproject.an_settings import THEMES +from hive.lang import get_langs_info +from hive.utils import HiveEnvironment, get_current_session, get_current_lang + # Available user manual to choose from -MANUAL = [(_('Application Guide'),'app'), (_('CLI Handbook'), 'cli')] +MANUAL = [(_('Application Guide'), 'app'), (_('CLI Handbook'), 'cli')] + class NodeRedirect(Exception): pass + class BaseNode(object): _template_name = None - def __init__(self, parent=None, name=None, oem_app=company.PRODUCT_KEYWORD, verbose_name=None, icon=None, default_action=None, hidden=False, license=''): + + def __init__(self, parent=None, name=None, oem_app=company.PRODUCT_KEYWORD, verbose_name=None, icon=None, + default_action=None, hidden=False, license=''): self.name = name self.app = name if parent is None else parent.app self.oem_app = oem_app if parent is None else parent.oem_app @@ -47,28 +47,30 @@ if parent.hidden == True: self.hidden = True parent.append_child(self) - + @property def verbose_name(self): return self._verbose_name - + def append_child(self, child_node): self.children[child_node.name] = child_node - + def get_child(self, name): return self.children[name] - + def get_first_child(self): return self.children.value_for_index(0) - + def _get_context(self, request, session): django_ctx = RequestContext(request) params = dict(request.REQUEST) params.pop('csrfmiddlewaretoken', None) - return {'NODE':self, 'REQUEST':request, 'SESSION':session, - 'LANGUAGE_CODE':django_ctx['LANGUAGE_CODE'], 'LANGS':get_langs_info(), 'TRADEMASK':company.COMPANY_TRADEMASK, - 'MEDIA':'/media/', 'APPNODE':AppNode.get_app(self.app),'COPYRIGHT':company.COMPANY_COPYRIGHT[get_current_lang()], - 'csrf_token':django_ctx['csrf_token'], 'THEMES':THEMES, 'PARAMS':params, 'MANUAL':MANUAL} + return {'NODE': self, 'REQUEST': request, 'SESSION': session, + 'LANGUAGE_CODE': django_ctx['LANGUAGE_CODE'], 'LANGS': get_langs_info(), + 'TRADEMASK': company.COMPANY_TRADEMASK, + 'MEDIA': '/media/', 'APPNODE': AppNode.get_app(self.app), + 'COPYRIGHT': company.COMPANY_COPYRIGHT[get_current_lang()], + 'csrf_token': django_ctx['csrf_token'], 'THEMES': THEMES, 'PARAMS': params, 'MANUAL': MANUAL} def render(self, request, session, action=None): if not self.has_page(action): @@ -82,7 +84,7 @@ 3. _template_name is set """ if self.default_action: - raise NodeRedirect('/'+self.oem_app+'/'+'/'.join(self.path)+'/_'+self.default_action) + raise NodeRedirect('/' + self.oem_app + '/' + '/'.join(self.path) + '/_' + self.default_action) if hasattr(self, 'index'): return getattr(self, 'index')(request, session) template_name = self._template_name @@ -90,14 +92,14 @@ if hasattr(self, action): return getattr(self, action)(request, session) else: - template_name = getattr(self, '_template_name_'+action) + template_name = getattr(self, '_template_name_' + action) template = self.env.get_template(template_name) return template.render(self._get_context(request, session)) @property def is_licensed(self): return self._licensed - + def is_parent_of(self, node): try: for i, name in enumerate(self.path): @@ -106,47 +108,49 @@ except IndexError: return False return True - + def has_children(self): - return len(self.children)>0 - + return len(self.children) > 0 + def has_menu_children(self): for each in self.children.values(): if each.__class__.__name__ == 'MenuNode': return True return False - + def has_non_menu_children(self): for each in self.children.values(): if each.__class__.__name__ != 'MenuNode': return True return False - + def has_page(self, action=None): if not action: - return (self._template_name is not None) or (self.default_action is not None) or callable(getattr(self, 'index', None)) + return (self._template_name is not None) or (self.default_action is not None) or callable( + getattr(self, 'index', None)) else: - return hasattr(self, '_template_name_'+action) or callable(getattr(self, action, None)) - + return hasattr(self, '_template_name_' + action) or callable(getattr(self, action, None)) + @property def path_string(self): return '/'.join(self.path) - + @property def url(self): if self.has_page(): - return '/'+self.oem_app+'/'+self.path_string + return '/' + self.oem_app + '/' + self.path_string else: return '#' + class AppNode(BaseNode): _cache = {} - + def __init__(self, parent=None, name=None, **kwargs): super(AppNode, self).__init__(parent, name, **kwargs) self.env = HiveEnvironment(loader=ChoiceLoader([ - PackageLoader(name, 'templates'), - PackageLoader('hive', 'templates')])) + PackageLoader(name, 'templates'), + PackageLoader('hive', 'templates')])) # The env will be passed through the hierachy when loading the node tree self.load_tree() @@ -155,8 +159,9 @@ if self.app is 'apv': manager = get_current_session().get_model_manager(self.app, ['admintools', 'sys_mgmt', 'SystemLicense']) features = manager._get_query_set()[0].features.split(' ') - tree_module = __import__(self.name+'.tree') + tree_module = __import__(self.name + '.tree') tree_dict = tree_module.tree.tree + # traverse the tree dict and build the node tree def load_one_node(parent_node, name, params): new_params = copy.deepcopy(params) @@ -193,10 +198,10 @@ new_node = BaseNode(parent=parent_node, name=name, **new_params) for key, value in sub_nodes.iteritems(): load_one_node(new_node, key, value) - + for key, value in tree_dict.iteritems(): load_one_node(self, key, value) - + def get_node_by_path(self, path): action = None if path[-1] and path[-1][0] == '_': @@ -204,28 +209,30 @@ path = path[:-1] tmp = self for each in path: - if each: # except None or '' + if each: # except None or '' tmp = tmp.get_child(each) return (tmp, action) - + @classmethod def get_app(cls, app_name): if app_name not in cls._cache: cls._cache[app_name] = AppNode(name=app_name) return cls._cache[app_name] - + def index(self, request, session): - raise NodeRedirect('/'+self.oem_app+'/index') + raise NodeRedirect('/' + self.oem_app + '/index') + class MenuNode(BaseNode): _template_name_list_sub_nodes = 'list_sub_nodes.html' - #def list_sub_nodes(self, request, session): + + # def list_sub_nodes(self, request, session): # template = self.env.get_template("list_sub_nodes.html") # return template.render(self._get_context(request, session)) - + def has_page(self, action=None): return self.has_non_menu_children() - + @property def url(self): url = '#' @@ -235,10 +242,11 @@ url = child.url break return url - + def index(self, request, session): raise NodeRedirect(self.url) + @csrf_exempt def reload_app_node(request, app_name): if not app_name: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/model.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/model.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/model.py (working copy) @@ -1,14 +1,16 @@ -from hive.node import BaseNode -from hive.model.loading import get_app, get_model from django.http import Http404 from jinja2.exceptions import TemplateNotFound -import copy + +from hive.model.loading import get_app, get_model +from hive.node import BaseNode from hive.utils import url2obj + # turn url params (REQUEST) into pk dict using url2obj def param_dict_to_pk_dict(param_dict): return {k: url2obj(v) for k, v in param_dict.items()} + class ModelNode(BaseNode): def __init__(self, parent=None, name=None, model=None, **kwargs): super(ModelNode, self).__init__(parent, name, **kwargs) @@ -21,18 +23,19 @@ self.default_action = 'edit' else: self.default_action = 'list' - + @property def verbose_name(self): # the definition at the node tree may override the verbose name of model return self._verbose_name if self._verbose_name else self.model._meta.verbose_name - + def _get_context(self, request, session): """ Add two context variables: MODEL and MANAGER """ super_ctx = super(ModelNode, self)._get_context(request, session) - return dict(super_ctx.items() + {'MODEL':self.model, 'MANAGER':session.get_model_manager(self.app, self.model_path), 'SESSION':session}.items()) + return dict( + super_ctx.items() + {'MODEL': self.model, 'MANAGER': session.get_model_manager(self.app, self.model_path), + 'SESSION': session}.items()) - def list(self, request, session): if self.model._meta.profile: raise Http404 @@ -48,7 +51,7 @@ ctx = self._get_context(request, session) ctx['license'] = lic return self.env.get_template('page_license.html').render(ctx) - + try: template = self.env.get_template('/'.join(self.path) + '/list.html') except TemplateNotFound: @@ -71,7 +74,7 @@ ctx = self._get_context(request, session) ctx['OBJECT'] = obj # Comment out for now as we don't support this feature for now. - #session.pref.append_recent_obj(self.model._meta.path, obj) + # session.pref.append_recent_obj(self.model._meta.path, obj) try: template = self.env.get_template('/'.join(self.path) + '/instance_status.html') @@ -81,7 +84,7 @@ else: template = self.env.get_template('instance_status.html') return template.render(ctx) - + def inst_config(self, request, session): if self.model._meta.profile: raise Http404 @@ -101,11 +104,11 @@ except TemplateNotFound: template = self.env.get_template('instance_config.html') return template.render(ctx) - + def add(self, request, session): if self.model._meta.profile: raise Http404 - + try: template = self.env.get_template('/'.join(self.path) + '/add.html') except TemplateNotFound: @@ -121,9 +124,9 @@ concrete_manager = session.get_model_manager(self.app, concrete_model._meta.path) ctx['CONCRETE_MODEL'] = concrete_model ctx['CONCRETE_MANAGER'] = concrete_manager - + return template.render(ctx) - + def class_statistics(self, request, session): if self.model._meta.profile: raise Http404 @@ -132,11 +135,11 @@ except TemplateNotFound: template = self.env.get_template('model_class_statistics.html') return template.render(self._get_context(request, session)) - + def statistics(self, request, session): if self.model._meta.profile: raise Http404 - + # Get the object manager = session.get_model_manager(self.app, self.model_path) try: @@ -145,13 +148,13 @@ raise Http404('No such object') ctx = self._get_context(request, session) ctx['OBJECT'] = obj - + try: template = self.env.get_template('/'.join(self.path) + '/statistics.html') except TemplateNotFound: template = self.env.get_template('model_statistics.html') return template.render(ctx) - + def index(self, request, session): try: template = self.env.get_template('/'.join(self.path) + '/index.html') @@ -174,7 +177,7 @@ ctx = self._get_context(request, session) ctx['license'] = lic return self.env.get_template('page_license.html').render(ctx) - + # Get the profile object manager = session.get_model_manager(self.app, self.model_path) try: @@ -184,7 +187,7 @@ raise Http404('No such object') ctx = self._get_context(request, session) ctx['OBJECT'] = obj - + try: template = self.env.get_template('/'.join(self.path) + '/edit.html') except TemplateNotFound: @@ -193,6 +196,3 @@ else: template = self.env.get_template('model_profile_edit.html') return template.render(ctx) - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/module.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/module.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/node/module.py (working copy) @@ -1,10 +1,8 @@ -from hive.node import BaseNode -from hive.model.loading import get_app, get_package -from django.http import Http404 from jinja2.exceptions import TemplateNotFound -from hive.utils import get_current_session, andebug + +from hive.model.loading import get_app, get_package from hive.model.loading import get_model -import copy +from hive.node import BaseNode class ModuleNode(BaseNode): @@ -20,7 +18,7 @@ def verbose_name(self): # the definition at the node tree may override the verbose name of package return self._verbose_name if self._verbose_name else self.package.verbose_name - + def _get_context(self, request, session): """ Add two context variables: MODEL and MANAGER """ super_ctx = super(ModuleNode, self)._get_context(request, session) @@ -28,24 +26,20 @@ if self.app == 'cm': model = get_model('cm', ['device_mgmt', 'device', 'Device']) manager = model.get_manager(session) - return dict(super_ctx.items() + {'PACKAGE':self.package, 'MANAGER':manager}.items()) + return dict(super_ctx.items() + {'PACKAGE': self.package, 'MANAGER': manager}.items()) else: - return dict(super_ctx.items() + {'PACKAGE':self.package}.items()) + return dict(super_ctx.items() + {'PACKAGE': self.package}.items()) - def monitor(self, request, session): try: template = self.env.get_template('/'.join(self.path) + '/monitor.html') except TemplateNotFound: template = self.env.get_template('page_module_monitor.html') return template.render(self._get_context(request, session)) - + def index(self, request, session): try: template = self.env.get_template('/'.join(self.path) + '/index.html') except TemplateNotFound: template = self.env.get_template('page_module_index.html') return template.render(self._get_context(request, session)) - - - Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/notification.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/notification.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/notification.py (working copy) @@ -1,55 +1,66 @@ -from django.http import HttpResponse -from django.http import Http404 -from hive.utils import get_current_session, andebug, anerror -import json, copy, threading +import copy +import json +import threading from datetime import datetime -from hive.model.loading import get_model + +from django.http import HttpResponse + from cm.lib.postgres_db import DB +from hive.model.loading import get_model +from hive.utils import get_current_session, anerror + # Ring buffer implementation http://blog.csdn.net/xiongaijing/article/details/12912647 class RingBuffer(object): """ class that implements a not-yet-full buffer """ + def __init__(self, size_max): self.max = size_max self.data = [] + class __Full(object): """ class that implements a full buffer """ + def append(self, x): """ Append an element overwriting the oldest one. """ self.data[self.cur] = x - self.cur = (self.cur+1) % self.max + self.cur = (self.cur + 1) % self.max + def tolist(self): """ return list of elements in correct order. """ return self.data[self.cur:] + self.data[:self.cur] + def append(self, x): """ append an element at the end of the buffer. """ self.data.append(x) if len(self.data) == self.max: self.cur = 0 self.__class__ = self.__Full + def tolist(self): """ Return a list of elements from the oldest to the newest. """ return self.data - class Notification(object): notification_buffer = RingBuffer(5) _lock = threading.Lock() - def __init__(self, time, level, initiator=None, session=None, msg_id=None, message=None, url=None, detail=None, model=None): + + def __init__(self, time, level, initiator=None, session=None, msg_id=None, message=None, url=None, detail=None, + model=None): self.time = time self.level = level self.initiator = initiator self.session = session - self.msg_id = msg_id # id of message type rather than the id of message itself + self.msg_id = msg_id # id of message type rather than the id of message itself self.message = message self.url = url self.detail = detail self.model = model - + # status - self.read = {} # a dict of read session IDs - + self.read = {} # a dict of read session IDs + def check_set_read(self, sess): if sess.sessid in self.read: return True @@ -57,17 +68,19 @@ # if it's not read, set it as read self.read[sess.sessid] = True return False - + @classmethod def get_unread_list(cls, sess): with cls._lock: - return [copy.copy(each) for each in cls.notification_buffer.tolist() if (not each.check_set_read(sess)) and (sess is not each.session)] - + return [copy.copy(each) for each in cls.notification_buffer.tolist() if + (not each.check_set_read(sess)) and (sess is not each.session)] + @classmethod def append_to_buffer(cls, obj): with cls._lock: cls.notification_buffer.append(obj) - + + def send_notification(level, message=None, msg_id=None, url=None, detail=None, session=None, model=None): time = datetime.now() # if session is not specified through argument, get it from the thread @@ -77,7 +90,8 @@ anerror('hive.debug', 'New notification: %s %s %s %s %s' % (level, message, msg_id, url, detail)) new_notification = Notification(time, level, initiator, session, msg_id, message, url, detail, model) Notification.append_to_buffer(new_notification) - + + def poll_notification(request): sess = get_current_session() unread_list = Notification.get_unread_list(sess) @@ -115,11 +129,11 @@ if auth_data[1]: auth_res = dict(zip(['username', 'auth'], auth_data)) else: - #no auth info, default allow all auth - auth_res = {'username':sess.username, 'auth':'\"all\"'} + # no auth info, default allow all auth + auth_res = {'username': sess.username, 'auth': '\"all\"'} else: - #here means no auth info, like default account(username: array) - auth_res = {'username':sess.username, 'auth':'\"all\"'} + # here means no auth info, like default account(username: array) + auth_res = {'username': sess.username, 'auth': '\"all\"'} result = { "notification": result_list, Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/preference.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/preference.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/preference.py (working copy) @@ -1,19 +1,20 @@ -from hive.utils import andebug -from djproject.an_settings import AN_APPS, THEMES, CONFIG_FILE_DIR, LANGS, SYS_DEFAULT_LANG, LANGUAGE_CONFIG_FILE, CM_ConfigFile_PATH -import os, time import json +import os -CONFIG_FILE_DIR = '%s/'%CM_ConfigFile_PATH +from djproject.an_settings import THEMES, CONFIG_FILE_DIR, SYS_DEFAULT_LANG, LANGUAGE_CONFIG_FILE, \ + CM_ConfigFile_PATH +CONFIG_FILE_DIR = '%s/' % CM_ConfigFile_PATH + class Preference(object): def __init__(self, session): - #self._session = session + # self._session = session self._username = session._username self._data = {} self._default_values = { - 'graphs':[], - 'recent_obj_list':{}, + 'graphs': [], + 'recent_obj_list': {}, } def read_config(self): @@ -24,33 +25,33 @@ config.close() except ValueError: self._data = self._default_values - + def write_config(self): config = open(CONFIG_FILE_DIR + self._username, 'w') config.write(json.dumps(self._data)) config.close() - + def append_recent_obj(self, model_name, obj): name = '.'.join(model_name) recent_obj_list = self.get(self, 'recent_obj_list') recent_obj_for_model = recent_obj_list.get(name, []) - - new_dict = {'pk':obj.pk_dict(), 'repr':obj.repr()} + + new_dict = {'pk': obj.pk_dict(), 'repr': obj.repr()} recent_obj_for_model.append(new_dict) recent_obj_list[name] = recent_obj_for_model self._data['recent_obj_list'] = recent_obj_list self.write_config() - + def get_recent_objs(self, model_name): name = '.'.join(model_name) recent_obj_list = self.get(self, 'recent_obj_list') return recent_obj_list.get(name, []) - + def set_default_theme(self, theme_name): self.read_config() self._data['user_theme'] = theme_name self.write_config() - + def get_default_theme(self): self.read_config() theme = self._data.get('user_theme', THEMES[0]) @@ -81,18 +82,18 @@ self.read_config() return self._data.get('default_lang', SYS_DEFAULT_LANG) """ - + def get(self, key): self.read_config() default_value = self._default_values.get(key, None) return self._data.get(key, default_value) - + def set(self, key, value): self.read_config() self._data[key] = value self.write_config() return [True] - + # restore to default value def restore(self, key): self.read_config() @@ -102,4 +103,3 @@ pass self.write_config() return [True] - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_pdf_report.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_pdf_report.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_pdf_report.py (working copy) @@ -1,13 +1,13 @@ +from reportlab.lib import colors from reportlab.lib.pagesizes import letter +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle from reportlab.lib.units import inch -from reportlab.lib import colors from reportlab.platypus import SimpleDocTemplate, Table, TableStyle, Paragraph, Spacer, Image -from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle -from hive.utils import andebug + from cm.lib.libbasic_operation import oper_log -report_file_path = "/ca/webui/htdocs/new/src/hive/media/docs/" -logo_path = "/ca/webui/htdocs/new/src/hive/media/static/Array-Logo-en-2024.jpg" +REPORT_FILE_PATH = "/ca/webui/htdocs/new/src/hive/media/docs/" +LOGO_PATH = "/ca/webui/htdocs/new/src/hive/media/static/Array-Logo-en-2024.jpg" class Report: @@ -36,7 +36,7 @@ """ Generate a PDF report with a table and an optional chart. """ - pdf = SimpleDocTemplate(report_file_path + filename, pagesize=letter) + pdf = SimpleDocTemplate(REPORT_FILE_PATH + filename, pagesize=letter) elements = [] styles = getSampleStyleSheet() styles.add(ParagraphStyle( @@ -45,7 +45,7 @@ leftPadding=0 # Add spacing before the paragraph )) - im = Image(logo_path, 2 * inch, 0.8 * inch) + im = Image(LOGO_PATH, 2 * inch, 0.8 * inch) im.hAlign = 'CENTER' elements.append(im) elements.append(Spacer(1, 20)) @@ -121,4 +121,4 @@ pdf.build(elements) except Exception as e: oper_log('error', 'system', e.message) - raise e # Raise to crash and see error \ No newline at end of file + raise e # Raise to crash and see error Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/report/generate_report.py (working copy) @@ -1,16 +1,16 @@ +import json import os +import time from django.http import HttpResponse -import json -import time -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log +from hive.custom_exceptions import generic_exception as ge from hive.db.llb_stats_queries import LLBStatsDB -from hive.report.generate_pdf_report import Report from hive.llb_stats import reformat_data_from_db -from hive.utils import andebug +from hive.report.generate_pdf_report import Report -report_file_path = "/ca/webui/htdocs/new/src/hive/media/docs/" +REPORT_FILE_PATH = "/ca/webui/htdocs/new/src/hive/media/docs/" def handle_report_generation(request, path=None): @@ -59,12 +59,13 @@ intervalStart, intervalEnd = LLBStatsDB.get_time_range(from_time, to_time) start_time = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime(intervalEnd / 1000)) end_time = time.strftime("%Y_%m_%d_%H_%M_%S", - time.localtime(intervalStart / 1000)) + time.localtime(intervalStart / 1000)) report_name = str(category).upper() + "_" + "Statistics_Report_from_%s_to_%s.pdf" % ( - start_time, end_time) + start_time, end_time) - report_generator.generate_pdf_report(report_name, category, data_dict, data_summary_dict, intervalStart, intervalEnd) - if os.path.exists(report_file_path + report_name): + report_generator.generate_pdf_report(report_name, category, data_dict, data_summary_dict, intervalStart, + intervalEnd) + if os.path.exists(REPORT_FILE_PATH + report_name): return HttpResponse(json.dumps({"result": True, "filename": report_name}), content_type='application/json') else: return HttpResponse(json.dumps({"result": False}), content_type='application/json') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/rest_utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/rest_utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/rest_utils.py (working copy) @@ -1,5 +1,4 @@ from django.urls import resolve -from django.urls import re_path _rest_sub_urlpatterns = [] _global_rest_resolver = None Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py (working copy) @@ -1,44 +1,47 @@ # -*- coding: UTF-8 -*- -import os -from django.http import HttpResponse, HttpResponseRedirect -from django.http import Http404 -from hive.utils import andebug, get_current_session, url2obj, upload_receive, UploadResponse, dict_combine, get_device_type, update_frontend_index_html -from hive.model.loading import get_model, get_app, get_model_from_full_path, get_apps -from hive.session import ANSession, current_app, temp_session -from hive.node import AppNode, NodeRedirect -from hive.model.ajax import ModelAjaxHandler, ObjectAjaxHandler -from hive.model.rest import * -from hive.model.storage import ModelStorageHandler -from django.shortcuts import redirect -import json +import collections +import csv +import fcntl import logging -import hive.engineer as eng -from hive.search import * -from hive.model.query import clear_cache_all, mark_expire_all, QuerySet, mark_expire_one -from djproject.settings import FILE_UPLOAD_DIR -import base64 +import re +import subprocess +from urllib import parse as urlparse + +from django.http import HttpResponseRedirect +from django.shortcuts import redirect from django.views.decorators.csrf import csrf_exempt -import csv -from hive.exceptions import ModelQueryException, ManagerImplError, ValidationError, RestfulException -import collections -from hive.model.manager import ANManager -from hive.model.utils import CONDITIONAL_NONE + import djproject.an_settings +import hive.engineer as eng import lib.toml as toml -import fcntl -from lib.response import StreamingHttpResponse, FileResponse -from hive.model.legacycli import CLIEnablePassError, RegexParser -from hive.notification import send_notification from cm.lib.communication import call_restapi, modify_url, send_https_rest_request, L -from cm.models.device_mgmt.device import get_rest_info_from_host, get_rest_info_from_device, get_rest_info_from_device_id -from hive.rest_utils import get_rest_url_rule_num, register_rest_url_rules, get_rest_process_func from cm.lib.libbasic_operation import oper_log, rest_log -from urllib import parse as urlparse +from cm.models.device_mgmt.device import get_rest_info_from_host, get_rest_info_from_device, \ + get_rest_info_from_device_id +from djproject.settings import FILE_UPLOAD_DIR +from hive.model.ajax import ModelAjaxHandler, ObjectAjaxHandler +from hive.model.legacycli import CLIEnablePassError, RegexParser +from hive.model.loading import get_model_from_full_path +from hive.model.manager import ANManager +from hive.model.query import clear_cache_all, mark_expire_all, QuerySet +from hive.model.rest import * +from hive.model.storage import ModelStorageHandler +from hive.model.utils import CONDITIONAL_NONE +from hive.node import AppNode, NodeRedirect +from hive.notification import send_notification +from hive.rest_utils import get_rest_url_rule_num, get_rest_process_func +from hive.search import * +from hive.session import temp_session from hive.utils import _thread_locals +from hive.utils import upload_receive, UploadResponse, dict_combine, \ + get_device_type, update_frontend_index_html +from lib.response import FileResponse try: - exec("from %s.localfile import localfile_path_map" % current_app()) -except BaseException: + # ToDo: fix this later - as localfile is an empty file. + import_statement = f"from {current_app()}.localfile import localfile_path_map" + exec(import_statement) +except Exception: pass logger = logging.getLogger('hive.debug') @@ -240,12 +243,14 @@ response = HttpResponse(DeviceRESTResource(request, 'POST', 'AutoRegistration').render()) else: try: - response = HttpResponse('Please use http basic authentication, and make sure the username/password is correct.') + response = HttpResponse( + 'Please use http basic authentication, and make sure the username/password is correct.') response['WWW-Authenticate'] = 'Basic realm="Cloud API HTTP Auth"' response.status_code = 401 return response except BaseException: - response = HttpResponse('Please use http basic authentication, and make sure the username/password is correct.') + response = HttpResponse( + 'Please use http basic authentication, and make sure the username/password is correct.') response.status_code = 401 return response else: @@ -258,12 +263,14 @@ with temp_session(request, 'cm') as sess: if not sess: try: - response = HttpResponse('Please use http basic authentication, and make sure the username/password is correct.') + response = HttpResponse( + 'Please use http basic authentication, and make sure the username/password is correct.') response['WWW-Authenticate'] = 'Basic realm="RESTful API HTTP Auth"' response.status_code = 401 return response except BaseException: - response = HttpResponse('Please use http basic authentication, and make sure the username/password is correct.') + response = HttpResponse( + 'Please use http basic authentication, and make sure the username/password is correct.') response.status_code = 401 return response try: @@ -481,7 +488,8 @@ temp = filename.split('/') filename = temp[-1] file_path += '/' + '/'.join(temp[0:-1]) - content_type = localfile_path_map[app]['Content-Type'] if 'Content-Type' in localfile_path_map[app] else 'application/octet-stream' + content_type = localfile_path_map[app]['Content-Type'] if 'Content-Type' in localfile_path_map[ + app] else 'application/octet-stream' os.chdir(file_path) file_exist = os.path.exists(filename) if file_exist: @@ -515,7 +523,9 @@ cookie = 'ComposerUICookie=f1f713c9e000f5d3f280adbd124df4f5' else: new_url = '/pest' + request.META['REQUEST_URI'][10:] - rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host.restapi_port, host.restapi_account['restapi_username'], host.restapi_account['restapi_password'], 'https', cookie) + rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host.restapi_port, + host.restapi_account['restapi_username'], + host.restapi_account['restapi_password'], 'https', cookie) return HttpResponse(rest_response_data["body"]) else: if request.META['REQUEST_URI'][0:17] == '/proxy_req/image/': @@ -530,7 +540,9 @@ host_ip = proxy_ip.values()[0] else: host_ip = proxy_ip - rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host.restapi_port, host.restapi_account['restapi_username'], host.restapi_account['restapi_password'], 'https', cookie) + rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host.restapi_port, + host.restapi_account['restapi_username'], + host.restapi_account['restapi_password'], 'https', cookie) return HttpResponse(rest_response_data["body"], content_type="image/png") raise Http404('Host Not Found') @@ -565,7 +577,8 @@ new_url = ('/rest/%s' + request.META['REQUEST_URI'][19:]) % get_device_type(host["type"]) else: new_url = '/pest' + request.META['REQUEST_URI'][14:] - rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host["restapi_port"], host["restapi_username"], host["restapi_password"], 'https', cookie) + rest_response_data = call_restapi(request.method, new_url, request.body, host_ip, host["restapi_port"], + host["restapi_username"], host["restapi_password"], 'https', cookie) return HttpResponse(rest_response_data["body"]) raise Http404('Host Not Found') @@ -630,6 +643,7 @@ rsBlocks = json.loads(res) return rsBlocks + # example: edit block list # post_data={'blocks':['1', '2', '3']} # edit service block list @@ -708,7 +722,8 @@ file_name = handle_uploaded_file(file) if file_name.split('.')[-1] != 'csv': - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), + content_type='application/json') global FILE_CONTENT FILE_CONTENT = [] @@ -742,6 +757,7 @@ json.dump(rsBlocks, dump_f, ensure_ascii=False) return HttpResponse(json.dumps({'is_success': True}), content_type='application/json') + # example: enable service # post_data=[{'service_name':'a', 'device_name':'test', 'enable':True}] # disable service and set group1 weight 2 @@ -772,20 +788,25 @@ try: data = {'cmd': 'show slb group method'} new_url = modify_url('/rest/device_type/cli_extend', each['type']) - group_method_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], each['port'], each['username'], each['password']) + group_method_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], + each['port'], each['username'], + each['password']) # new_url = modify_url('/rest/apv/loadbalancing/slb/group/Group', each['type']) # group_response_data = send_https_rest_request('GET', new_url, '', each['ip'], each['port'], each['username'], each['password']) data = {'cmd': 'show slb group member'} new_url = modify_url('/rest/device_type/cli_extend', each['type']) - group_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], each['port'], each['username'], each['password']) + group_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], + each['port'], each['username'], each['password']) new_url = modify_url('/rest/apv/loadbalancing/slb/rs/RealService', each['type']) - rs_response_data = send_https_rest_request('GET', new_url, '', each['ip'], each['port'], each['username'], each['password']) + rs_response_data = send_https_rest_request('GET', new_url, '', each['ip'], each['port'], + each['username'], each['password']) data = {'cmd': 'show statistics slb real all'} new_url = modify_url('/rest/device_type/cli_extend', each['type']) - statistic_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], each['port'], each['username'], each['password']) + statistic_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], + each['port'], each['username'], each['password']) except Exception as e: errMsg = 'Get device<%s> RealService failed. %s' % (each['ip'], str(e)) resp[each['name']]['error'] = errMsg @@ -887,7 +908,8 @@ try: data = {'targets': deviceMap[device_name]['enable']} new_url = modify_url('/rest/apv/loadbalancing/slb/rs/RealService/_Enable', each['type']) - rs_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], each['port'], each['username'], each['password']) + rs_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], + each['port'], each['username'], each['password']) except Exception as e: resp[each['name']]['error'].append(str(e)) logger.error('Enable device<%s> RealService failed. %s' % (each['ip'], str(e))) @@ -900,7 +922,8 @@ try: data = {'targets': deviceMap[device_name]['disable'], 'force': 0} new_url = modify_url('/rest/apv/loadbalancing/slb/rs/RealService/_Disable', each['type']) - rs_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], each['port'], each['username'], each['password']) + rs_response_data = send_https_rest_request('POST', new_url, json.dumps(data), each['ip'], + each['port'], each['username'], each['password']) except Exception as e: resp[each['name']]['error'].append(str(e)) logger.error('Disable device<%s> RealService failed. %s' % (each['ip'], str(e))) @@ -912,12 +935,18 @@ if len(deviceMap[device_name]['groups']) > 0: cli_list = [] for group in deviceMap[device_name]['groups']: - if group['group_method'] in ['rr', 'lc', 'hi', 'chi', 'pi', 'ph', 'hh', 'hq', 'hc', 'ic', 'rc', 'ec', 'sslsid', 'radchs', 'radchu', 'radpsun', - 'l2rr', 'l2hi', 'l2chi', 'persistence', 'sipcid', 'sipuid', 'sipcidps', 'sipuidps', 'tuxedo', 'prox', 'rdprt', 'pto', 'hip', 'lb', 'diametersid']: - cli_list.append("slb group member \"" + group['group_name'] + "\" \"" + group['service_name'] + "\" " + str(group['weight'])) + if group['group_method'] in ['rr', 'lc', 'hi', 'chi', 'pi', 'ph', 'hh', 'hq', 'hc', 'ic', + 'rc', 'ec', 'sslsid', 'radchs', 'radchu', 'radpsun', + 'l2rr', 'l2hi', 'l2chi', 'persistence', 'sipcid', 'sipuid', + 'sipcidps', 'sipuidps', 'tuxedo', 'prox', 'rdprt', 'pto', + 'hip', 'lb', 'diametersid']: + cli_list.append("slb group member \"" + group['group_name'] + "\" \"" + group[ + 'service_name'] + "\" " + str(group['weight'])) try: new_url = modify_url('/rest/device_type/batch_cli', each['type']) - rs_response_data = send_https_rest_request('POST', new_url, str('\n'.join(cli_list)), each['ip'], each['port'], each['username'], each['password']) + rs_response_data = send_https_rest_request('POST', new_url, str('\n'.join(cli_list)), + each['ip'], each['port'], each['username'], + each['password']) except Exception as e: resp[each['name']]['error'].append(str(e)) logger.error('Set device<%s> RealService weight failed. %s' % (each['ip'], str(e))) @@ -1050,7 +1079,8 @@ break if the_pk_dict: - filter_dict[tgt_field.name] = [dict_combine(the_pk_dict, {'_asso_idx': tgt_to_self_idx})] + filter_dict[tgt_field.name] = [ + dict_combine(the_pk_dict, {'_asso_idx': tgt_to_self_idx})] else: filter_dict[tgt_field.name] = [{'_asso_idx': tgt_to_self_idx}] @@ -1066,7 +1096,8 @@ the_pk_dict = instance.pk_dict() filter_dict = {} if the_pk_dict: - filter_dict[field.tgt_field[0].name] = [dict_combine(the_pk_dict, {'_asso_idx': 0})] + filter_dict[field.tgt_field[0].name] = [ + dict_combine(the_pk_dict, {'_asso_idx': 0})] else: filter_dict[field.tgt_field[0].name] = [{'_asso_idx': 0}] @@ -1114,7 +1145,8 @@ asso_model = b.keys()[0] asso_model_cls_name = asso_model._meta.path[-1] asso_filter_dict = b.values()[0] - export_handler(asso_model, sess, response, asso_filter_dict, asso_info=(asso_model_cls_name, instance.pk_id())) + export_handler(asso_model, sess, response, asso_filter_dict, + asso_info=(asso_model_cls_name, instance.pk_id())) def write_csv_without_composition_asso(response, keys, data_dict, model_info): @@ -1148,7 +1180,8 @@ file_name = handle_uploaded_file(file) if file_name.split('.')[-1] != 'csv': - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), + content_type='application/json') global FILE_CONTENT FILE_CONTENT = [] @@ -1187,7 +1220,8 @@ except ValidationError as e: raise ModelQueryException(e) except ModelQueryException as e: - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': str(e)}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'errorInfo': str(e)}), + content_type='application/json') return HttpResponse(json.dumps({'is_success': True}), content_type='application/json') @@ -1247,7 +1281,8 @@ continue # Because csv data is type str, a dict or a list after csv reading will turn to be type str. # For IPAdressFiled, GroupField, UnionFiled, etc. We should try to change it back to type dict or list. - if field.type_name in ['ipaddr', 'GroupField', 'UnionField', 'multienum', 'ListField', 'enumorder', 'Percent']: + if field.type_name in ['ipaddr', 'GroupField', 'UnionField', 'multienum', 'ListField', 'enumorder', + 'Percent']: value = eval(value) if field.type_name != 'asso': new_value = field.to_python(value) @@ -1392,7 +1427,8 @@ os.system("cp -r /ca/config /var/tmp/backup") os.system("cp -r /ca/webui/htdocs/new/src/cm/conf/ /var/tmp/backup") os.system("/usr/pgsql-10/bin/pg_dump -U postgres -h 127.0.0.1 cm > /var/tmp/backup/cm.sql") - os.system("cd /var/tmp && tar -zcvf - backup | openssl des3 -salt -k " + output["version"] + " -out " + fileName) + os.system( + "cd /var/tmp && tar -zcvf - backup | openssl des3 -salt -k " + output["version"] + " -out " + fileName) os.system("rm -rf /var/tmp/backup") os.chdir("/var/tmp") @@ -1409,10 +1445,10 @@ def upload_setting(request): - file = upload_receive(request) if file.name.split('.')[-1] != 'setting': - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Invalid File Format !'}), + content_type='application/json') try: path = FILE_UPLOAD_DIR if not os.path.exists(path): @@ -1436,8 +1472,10 @@ os.system("dd if=" + file_name + " |openssl des3 -d -k " + output["version"] + " | tar zxf -") os.remove(file_name) if not os.path.exists("/var/tmp/backup"): - oper_log('error', 'system', 'Setting File upload fail. It may result from incorrect Setting File or error transmission. Please check the validation of Setting File and try again.') - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Setting File upload fail. It may result from incorrect Setting File or error transmission. Please check the validation of Setting File and try again.'})) + oper_log('error', 'system', + 'Setting File upload fail. It may result from incorrect Setting File or error transmission. Please check the validation of Setting File and try again.') + return HttpResponse(json.dumps({'is_success': False, + 'errorInfo': 'Setting File upload fail. It may result from incorrect Setting File or error transmission. Please check the validation of Setting File and try again.'})) if os.path.exists("/var/tmp/backup/version"): with open("/var/tmp/backup/version", "r") as f: @@ -1451,12 +1489,14 @@ os.system("rm -rf /ca/package/upload; cp -r /var/tmp/backup/upload /ca/package/") if os.path.exists("/var/tmp/backup/conf"): - os.system("rm -rf /ca/webui/htdocs/new/src/cm/conf/; cp -r /var/tmp/backup/conf /ca/webui/htdocs/new/src/cm/") + os.system( + "rm -rf /ca/webui/htdocs/new/src/cm/conf/; cp -r /var/tmp/backup/conf /ca/webui/htdocs/new/src/cm/") if os.path.exists("/var/tmp/backup/cm.sql"): os.system('/usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -c "drop database cm;"') os.system('/usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -c "create database cm;"') - os.system('/usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -d cm < /var/tmp/backup/cm.sql') + os.system( + '/usr/pgsql-10/bin/psql -h 127.0.0.1 -p 5432 -U postgres -d cm < /var/tmp/backup/cm.sql') db = DB.get_connected_db() data = db.fetchall("SELECT ip_address FROM device") @@ -1466,12 +1506,13 @@ oper_log('info', 'system', 'Upload setting successful.') return HttpResponse(json.dumps({'is_success': clear_cache_all()}), content_type='application/json') oper_log('error', 'system', 'Setting File upload fail. Version mismatch.') - return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Setting File and device version do not match!'}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'errorInfo': 'Setting File and device version do not match!'}), + content_type='application/json') def get_ha_status(request): try: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") obj = { "enabled": False, } @@ -1495,7 +1536,7 @@ def get_ha_setting(request): try: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") config = toml.load(djproject.an_settings.HA_CONF_PATH) config["enabled"] = False if len(res) == 2: @@ -1503,7 +1544,8 @@ if len(tmp) >= 4: config["enabled"] = (tmp[1] == "on") except Exception as e: - return HttpResponse(json.dumps({'is_success': False, 'message': 'Internal Error.'}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'message': 'Internal Error.'}), + content_type='application/json') else: return HttpResponse(json.dumps({'is_success': True, 'data': config}), content_type='application/json') @@ -1524,10 +1566,10 @@ def get_ha_log(request): rst = [] try: - result = commands.getoutput("tail -n 500 /var/log/pg_ha.log").split("\n") + result = subprocess.getoutput("tail -n 500 /var/log/pg_ha.log").split("\n") for line in result: line_list = line.split() - if LOGLEVEL[line_list[2]]: + if LOGLEVEL[line_list[2]]: level = LOGLEVEL[line_list[2]] else: level = line_list[2].strip('[').strip(']') @@ -1546,7 +1588,7 @@ def update_ha_setting(request): data = json.loads(request.POST.get('post_data', None)) try: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' status').split("\n") enabled = False if len(res) == 2: tmp = [i for i in res[1].split(" ") if i != ''] @@ -1565,17 +1607,19 @@ toml.dump(config, f) fcntl.flock(f.fileno(), fcntl.LOCK_UN) if data['enabled']: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' on') + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' on') else: if enabled: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' off') + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' off') else: res = "HA dsiable success" time.sleep(3) else: - return HttpResponse(json.dumps({'is_success': False, 'message': "please close HA first"}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'message': "please close HA first"}), + content_type='application/json') except Exception as e: - return HttpResponse(json.dumps({'is_success': False, 'message': "Internal Error."}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'message': "Internal Error."}), + content_type='application/json') else: if "success" in res: return HttpResponse(json.dumps({'is_success': True, 'message': res}), content_type='application/json') @@ -1585,9 +1629,10 @@ def delete_ha_setting(request): try: - res = commands.getoutput(djproject.an_settings.HA_BIN_PATH + ' clear') + res = subprocess.getoutput(djproject.an_settings.HA_BIN_PATH + ' clear') except Exception as e: - return HttpResponse(json.dumps({'is_success': False, 'message': "Internal Error."}), content_type='application/json') + return HttpResponse(json.dumps({'is_success': False, 'message': "Internal Error."}), + content_type='application/json') else: if "close HA" in res: return HttpResponse(json.dumps({'is_success': False, 'message': res}), content_type='application/json') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py (working copy) @@ -1,18 +1,22 @@ import os -from jinja2 import Environment, PackageLoader, ChoiceLoader -from hive.utils import andebug, HiveEnvironment, Singleton, get_current_session -import djproject.an_settings as an_settings -from whoosh.index import create_in + +from jinja2 import PackageLoader from whoosh.fields import * +from whoosh.index import create_in from whoosh.qparser import QueryParser from whoosh.query import * + +import djproject.an_settings as an_settings from hive.model.loading import get_app, get_model +from hive.utils import HiveEnvironment, Singleton, get_current_session have_no_default_page = {'System Status': '/apv/index/dashboard'} + def my_unicode(src): return str(src) if src is not None else '' - + + @Singleton class ANSearch(object): def __init__(self, app): @@ -21,19 +25,19 @@ self.env = HiveEnvironment(loader=PackageLoader('hive', 'templates')) self.schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)) self.whoosh_setup() - + def whoosh_setup(self): if not os.path.exists(an_settings.WHOOSH_INDEX_PATH): os.mkdir(an_settings.WHOOSH_INDEX_PATH) self.ix = create_in(an_settings.WHOOSH_INDEX_PATH, self.schema) self.writer = self.ix.writer() - + def my_help_text(help_text): if my_unicode(help_text): return ' - ' + my_unicode(help_text) else: return '' - + # iterate all models def gen_pkg(pkg): for sub_model in pkg.models.values(): @@ -43,14 +47,16 @@ for field_grp in sub_model._meta.field_groups.values(): model_content += my_unicode(field_grp.verbose_name) + ' ' model_content += my_unicode(field_grp.help_text) + ' ' - model_content += ', '.join([my_unicode(field.verbose_name) + my_help_text(field.help_text) for field in field_grp.fields]) + model_content += ', '.join( + [my_unicode(field.verbose_name) + my_help_text(field.help_text) for field in field_grp.fields]) model_content = my_unicode(model_content) self.writer.add_document(title=model_title, path=model_path, content=model_content) for sub_pkg in pkg.sub_packages.values(): gen_pkg(sub_pkg) + gen_pkg(self.app_pkg) self.writer.commit() - + def query(self, q, **kwargs): with self.ix.searcher() as searcher: query1 = QueryParser('title', self.ix.schema).parse(q) @@ -78,17 +84,17 @@ 'path': hit['path'], 'highlight': hit.highlights('content'), }) - + template = self.env.get_template('search.html') - return template.render({'SESSION':get_current_session(), 'MEDIA':'/media/', 'RESULTS':new_results, 'PARAMS':kwargs}) + return template.render( + {'SESSION': get_current_session(), 'MEDIA': '/media/', 'RESULTS': new_results, 'PARAMS': kwargs}) + def hive_search_init(app): return ANSearch(app) - + + def hive_search(q, **kwargs): # get the search singleton search = ANSearch(an_settings.AN_APPS[0]) return search.query(q, **kwargs) - - - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (working copy) @@ -1,15 +1,14 @@ -import os +import argparse import datetime -import shutil +import json import logging +import os +import shutil import sys -import json -import argparse sys.path.append('/ca/webui/htdocs/new/src') from cm.lib.libbasic_operation import oper_log from hive.services.service_utils import ServiceUtils -from hive.utils import andebug from hive.custom_exceptions import generic_exception as ge from hive.services.backup_service_utils import fetch_remote_storage_details from cm.lib.postgres_db import DB @@ -81,7 +80,7 @@ # Backup system logs backup_logger.info("Backing up system logs...") exclude_dirs = ["elasticsearch", "composer*.log", "vmware-network*.log", "yum*.log", "backup.log" - , "restore.log"] + , "restore.log"] exclude_flags = " ".join(["--exclude=" + d for d in exclude_dirs]) ServiceUtils.run_command("rsync -av {} /var/log/ {}/logs/".format(exclude_flags, TMP_BACKUP_DIR), backup_logger) @@ -118,7 +117,6 @@ # Update success status in DB self.update_backups_db(BACKUP_FILE, "Completed", 'update') - # Backup PostgreSQL """ Taking backup of postgres DB after updating DB with the Success message. So that during restore the DB contains correct data """ @@ -244,7 +242,7 @@ return HttpResponse(json.dumps({"status": "Completed", "details": "No Backup in progress!"}), - content_type="application/json") + content_type="application/json") except Exception as e: oper_log('error', 'system', "Exception while fetching backup status.") message = str(e.message).replace("'", "") Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service_utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service_utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/backup_service_utils.py (working copy) @@ -1,12 +1,8 @@ -import os import json -import logging from django.http import HttpResponse -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log -from hive.utils import andebug, get_current_session -from hive.services.service_utils import ServiceUtils from cm.lib.postgres_db import DB Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/log_location_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/log_location_service.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/log_location_service.py (working copy) @@ -1,12 +1,11 @@ -import subprocess -import re -import os import datetime -import shutil +import os +import re +import subprocess import zipfile -from hive.custom_exceptions import generic_exception as ge from cm.lib.libbasic_operation import oper_log +from hive.custom_exceptions import generic_exception as ge from hive.util import constants as const @@ -308,4 +307,4 @@ # Handle unexpected exceptions oper_log('info', 'system', 'An error occurred while archiving logs: {}'.format(e)) e.message = 'An error occurred while archiving logs: {}'.format(e) - raise e \ No newline at end of file + raise e Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/restore_service.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/restore_service.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/restore_service.py (working copy) @@ -1,17 +1,17 @@ +import datetime +import json +import logging import os -import subprocess -import tarfile import shutil -import datetime import sys -import logging -import json +import tarfile import time + from django.http import HttpResponse -from hive.utils import andebug, get_current_session -from hive.services.service_utils import ServiceUtils + from hive.custom_exceptions import generic_exception as ge from hive.services.backup_service_utils import fetch_remote_storage_details +from hive.services.service_utils import ServiceUtils sys.path.append('/ca/webui/htdocs/new/src') from cm.lib.libbasic_operation import oper_log Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py (working copy) @@ -1,15 +1,15 @@ -import os +import datetime import json -import subprocess import logging -import datetime +import os +import subprocess from django.http import HttpResponse -from hive.custom_exceptions import generic_exception as ge + from cm.lib.libbasic_operation import oper_log -from hive.utils import andebug, get_current_session -from hive.services.service_utils import ServiceUtils from cm.lib.postgres_db import DB +from hive.custom_exceptions import generic_exception as ge +from hive.services.service_utils import ServiceUtils # Configure Logging backup_cron_logger = logging.getLogger("backup_cron") Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/service_utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/service_utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/services/service_utils.py (working copy) @@ -1,8 +1,6 @@ import subprocess -import datetime -import os - +# ToDo: Read variables from config file. class ServiceUtils: # Set Variables BACKUP_LOG_FILE = "/var/backup.log" Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py (working copy) @@ -1,28 +1,26 @@ -import os -from django.conf import settings -from djproject import an_settings -import socket, sys, time +import base64 +import time +import uuid +from contextlib import contextmanager from datetime import datetime -from hive.utils import anlog, anerror, andebug -from hive.model.legacycli import ANCLIConn, CLIEnablePassError, RegexParser, MATCHONE -from hive.model.loading import get_apps, get_model -from hive.model.package import ANPackageAgent -from jinja2 import Environment, PackageLoader, ChoiceLoader +from datetime import timedelta + from django.http import HttpResponse -from django.template import RequestContext -from . import auth from django.shortcuts import redirect +from django.template import RequestContext +from django.views.decorators.csrf import csrf_exempt +from jinja2 import PackageLoader + +from cm.lib.libbasic_operation import oper_log +from djproject import an_settings +from hive.model.legacycli import ANCLIConn, CLIEnablePassError, RegexParser, MATCHONE +from hive.model.loading import get_apps, get_model +from hive.notification import send_notification from hive.preference import * from hive.utils import HiveEnvironment, andebug, _thread_locals -import base64, uuid, hashlib -from hive.auth import * -from hive.notification import send_notification -from contextlib import contextmanager -from django.views.decorators.csrf import csrf_exempt -from datetime import timedelta -from cm.lib.libbasic_operation import oper_log -import json +from . import auth + class ANSession(object): _session_pool = {} @@ -267,6 +265,7 @@ elif request.method == 'POST': session = None + username = '' if 'hive_sess' in request.COOKIES: session = ANSession.find_session_by_id(request.COOKIES['hive_sess']) if 'username' in request.POST: @@ -346,6 +345,7 @@ oper_log('error', 'login', 'User <%s> failed to login with an invalid password.' % username) return HttpResponse(json.dumps({'error_code': 2, 'msg': 'Invalid username or password.'}), content_type='application/json') + return None def app_logout_handler(request): @@ -505,4 +505,4 @@ except: pass _thread_locals.session = sess - return None # found session continue the process to the router \ No newline at end of file + return None # found session continue the process to the router Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/shared.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/shared.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/shared.py (working copy) @@ -1,41 +1,43 @@ -from jinja2 import Environment, PackageLoader, ChoiceLoader -from hive.utils import andebug, HiveEnvironment, get_current_session -from django.template import RequestContext from django.http import HttpResponse +from django.template import RequestContext +from jinja2 import PackageLoader, ChoiceLoader +from hive.utils import HiveEnvironment, get_current_session + PROTO_PORT_MAP = { - 'http':80, - 'https':443, - 'dns':53, - 'ftp':21, - 'siptcp':5060, - 'sipudp':5060, - 'rtsp':554, - 'tcp':0, - 'tcps':443, - 'udp':0, - 'rdp':3389, - 'radauth':1812, - 'radacct':1813, - 'ftp':21, - 'ftps':990 + 'http': 80, + 'https': 443, + 'dns': 53, + 'ftp': 21, + 'siptcp': 5060, + 'sipudp': 5060, + 'rtsp': 554, + 'tcp': 0, + 'tcps': 443, + 'udp': 0, + 'rdp': 3389, + 'radauth': 1812, + 'radacct': 1813, + 'ftp': 21, + 'ftps': 990 } + def custom_404_view(request, exception): sess = get_current_session() django_ctx = RequestContext(request) env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) template = env.get_template('/404.html') - ctx = {'SESSION':get_current_session(), 'MEDIA':'/media/'} - + ctx = {'SESSION': get_current_session(), 'MEDIA': '/media/'} + return HttpResponse(template.render(ctx)) + def custom_500_view(request, exception): sess = get_current_session() django_ctx = RequestContext(request) env = HiveEnvironment(loader=ChoiceLoader([PackageLoader('hive', 'templates')])) template = env.get_template('/500.html') - ctx = {'SESSION':get_current_session(), 'MEDIA':'/media/'} + ctx = {'SESSION': get_current_session(), 'MEDIA': '/media/'} return HttpResponse(template.render(ctx)) - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py (working copy) @@ -1,11 +1,11 @@ -from hive.utils import andebug -from django.http import HttpResponse, Http404 -from djproject.an_settings import CMDATA -import json import http.client -import subprocess +import json import os -# import time +import subprocess + +from django.http import HttpResponse, Http404 + +from djproject.an_settings import CMDATA from lib.crontab import CronTab MonitorTree = [{ @@ -115,8 +115,9 @@ path['size'] = getdirsize(path['path']) elif path['type'] == "es_query": try: - httpClient = http.client.HTTPConnection(CMDATA['ELASTICSEARCH_IP'], str(CMDATA['ELASTICSEARCH_PORT']), - True, timeout=300) + httpClient = http.client.HTTPConnection(CMDATA['ELASTICSEARCH_IP'], + str(CMDATA['ELASTICSEARCH_PORT']), + True, timeout=300) httpClient.request("GET", path['path']) response = httpClient.getresponse() body = response.read() @@ -209,4 +210,4 @@ "/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s" % (obj['duration'], obj['percent'])) return HttpResponse(content) - raise Http404('Has no such page for action %s' % key) \ No newline at end of file + raise Http404('Has no such page for action %s' % key) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/theme.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/theme.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/theme.py (working copy) @@ -2,6 +2,7 @@ from django.http import Http404 from hive.utils import get_current_session + def switch_theme(request): if 'new_theme' not in request.POST: raise Http404 @@ -10,4 +11,3 @@ sess.theme = new_theme sess.pref.set_default_theme(new_theme) return HttpResponse(new_theme) - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/util/constants.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/util/constants.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/util/constants.py (working copy) @@ -6,4 +6,4 @@ SECONDARY_LOG_LOCATION = '/secondary/elasticsearch' -ELASTIC_SEARCH_CONFIG_FILE = '/etc/elasticsearch/elasticsearch.yml' \ No newline at end of file +ELASTIC_SEARCH_CONFIG_FILE = '/etc/elasticsearch/elasticsearch.yml' Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py (working copy) @@ -1,36 +1,44 @@ -import logging, sys +import base64 +import collections import copy -from jinja2 import Environment -import uuid, base64 -import urllib -import threading import ctypes -import collections +import logging +import sys +import threading +import urllib +import uuid from collections import OrderedDict + +from jinja2 import Environment + _thread_locals = threading.local() -import json, os from django.http import HttpResponse from django.utils.translation import gettext_lazy, ngettext_lazy, get_language from django.utils.text import format_lazy -import djproject.an_settings +import djproject.an_settings import queue import time import hashlib from djproject.an_settings import * - + + def get_current_request(): return getattr(_thread_locals, 'request', None) + def get_current_session(): return getattr(_thread_locals, 'session', None) - + + class ClassProperty(property): def __get__(self, cls, owner): return self.fget.__get__(None, owner)() + class IntRangeParseErr(Exception): pass + def is_number(s): try: float(s) @@ -38,14 +46,16 @@ except ValueError: return False + def toint(symbol, parent): if symbol == '*' or symbol == 'max': return parent.get_max() if parent else sys.maxsize elif symbol == 'min': - return parent.get_min() if parent else (-sys.maxsize-1) + return parent.get_min() if parent else (-sys.maxsize - 1) else: return int(symbol) + class IntRange(object): """ IntRange object @@ -58,6 +68,7 @@ * '2..*' * '1..5|10|20..max' """ + def __init__(self, string, late_parse=True): # parse the input string self._string = str(string) @@ -65,9 +76,9 @@ self.parsed_list = [] if not late_parse: self._parse() - + def _parse(self): - last_parsed = (-sys.maxsize-1) + last_parsed = (-sys.maxsize - 1) for each_scope in self._string.split('|'): scope_list = each_scope.strip().split('..') if len(scope_list) == 1: @@ -84,7 +95,8 @@ if scope_int_list[0] > scope_int_list[1]: raise IntRangeParseErr("Scope right value smaller than left value %s" % each_scope) if scope_int_list[0] < last_parsed: - raise IntRangeParseErr("Left value of scope %s smaller than last parsed value %d" % (each_scope,last_parsed)) + raise IntRangeParseErr( + "Left value of scope %s smaller than last parsed value %d" % (each_scope, last_parsed)) last_parsed = scope_int_list[1] self.parsed_list.append(scope_int_list) @@ -97,14 +109,13 @@ def get_note(self): if not self._is_parsed(): self._parse() - orig_note_list=[] + orig_note_list = [] for each in self.parsed_list: if each[0] == each[1]: orig_note_list.append(str(each[0])) else: - orig_note_list.append(str(each[0])+'-'+str(each[1])) + orig_note_list.append(str(each[0]) + '-' + str(each[1])) return '(%s)' % ', '.join(orig_note_list) - def set_parent(self, parent): self._parent = parent @@ -124,10 +135,10 @@ if not self._is_parsed(): self._parse() - if not type(integer) in [int, long]: + if not type(integer) in [int]: raise ValueError("Input value should be an integer") for (left, right) in self.parsed_list: - if integer >= left and integer <= right: + if left <= integer <= right: return True return False @@ -139,15 +150,15 @@ if left == right: txt_list.append(str(left)) else: - txt_list.append(str(left)+'..'+str(right)) + txt_list.append(str(left) + '..' + str(right)) return '|'.join(txt_list) - + def __str__(self): return self.__repr__() def can_be_zero(self): return self.validate(0) - + @staticmethod def get_range_from_parsed_list(parsed_list): new_range = IntRange('') @@ -187,10 +198,10 @@ for (left, right) in self.parsed_list: borders.append(left) borders.append(right) - + if bias < len(borders): return borders[bias] - + # bias from the non-border list bias_non_border = bias - len(borders) @@ -198,36 +209,39 @@ while 1: for (left, right) in self.parsed_list: cur_value = left - while cur_value < right-1: + while cur_value < right - 1: cur_value += 1 if bias_non_border == 0: return cur_value bias_non_border -= 1 - + class Singleton(object): def __init__(self, aClass): self.aClass = aClass self.instance = None if (hasattr(aClass, '_meta')): self._meta = aClass._meta + def __call__(self, *args, **kargs): if self.instance == None: self.instance = self.aClass(*args, **kargs) return self.instance + class ObjCache(object): """ Decorator class for implementing object cache, every object of the class is a callable that returns a function to get the cached object. """ - def __init__(self, *args): # at @decorator(arg1,arg2) call + + def __init__(self, *args): # at @decorator(arg1,arg2) call self._argnames = args self._cache = {} - def __call__(self, aClass): # at @decorator(arg1,arg2)(Class) call - def onCall(*args, **kwargs): # at Class(args) call + def __call__(self, aClass): # at @decorator(arg1,arg2)(Class) call + def onCall(*args, **kwargs): # at Class(args) call cur_layer = self._cache for argname in self._argnames: try: @@ -241,10 +255,13 @@ else: cur_layer[arg] = cur_layer.get(arg, aClass(*args, **kwargs)) return cur_layer[arg] + return onCall - + + def cached_property(f): """returns a cached property that is calculated by function f""" + def get(self): try: return self._property_cache[f] @@ -258,6 +275,7 @@ return property(get) + # for now we only support clearing all properties # of an instance def clear_property_cache(instance): @@ -266,6 +284,7 @@ except AttributeError: pass + def per_sess_cached_property(f): def get(self): try: @@ -289,6 +308,7 @@ return property(get) + def clear_per_sess_property_cache(instance): sess = get_current_session() try: @@ -298,21 +318,27 @@ except KeyError: pass + def anlog(logger_path, level, msg, *args, **kwargs): return logging.getLogger(logger_path).log(level, msg, *args, **kwargs) + def andebug(logger_path, msg, *args, **kwargs): return anlog(logger_path, logging.DEBUG, msg, *args, **kwargs) + def aninfo(logger_path, msg, *args, **kwargs): return anlog(logger_path, logging.INFO, msg, *args, **kwargs) + def anwarning(logger_path, msg, *args, **kwargs): return anlog(logger_path, logging.WARNING, msg, *args, **kwargs) + def anerror(logger_path, msg, *args, **kwargs): return anlog(logger_path, logging.ERROR, msg, *args, **kwargs) - + + def ancritical(logger_path, msg, *args, **kwargs): return anlog(logger_path, logging.CRITICAL, msg, *args, **kwargs) @@ -324,31 +350,30 @@ self.limit = limit self.current_coord = 0 self.current_step = 0 - self.value = [0]*self.dimension - + self.value = [0] * self.dimension + def __iter__(self): return self - + def next(self): if self.current_step == 0: self.current_step += 1 return self.value # set the value based on current coord/step - self.value = [0]*self.dimension + self.value = [0] * self.dimension self.value[self.current_coord] = self.current_step - + # update the current coord/step (step forward) - if self.current_coord == self.dimension-1: + if self.current_coord == self.dimension - 1: self.current_coord = 0 self.current_step += 1 else: self.current_coord += 1 - + # return the value return self.value - - - + + # Iterate over the dicrete points of a n-dimensional infinite space or within a limitted area class nd_iter_space(object): def __init__(self, dimension, limit=0): @@ -357,7 +382,7 @@ self.limit = limit self.current_cube = 0 self.current_flat = 0 - self.value = [0]*self.dimension + self.value = [0] * self.dimension self.sub_iter = None def __iter__(self): @@ -392,7 +417,7 @@ # that means we should start a new flat if self.current_flat < self.dimension: # iterate this flat by create a new sub_iter - self.sub_iter = nd_iter_space(self.dimension-1, self.current_cube) + self.sub_iter = nd_iter_space(self.dimension - 1, self.current_cube) try: return self.get_sub_iter_next() except StopIteration: @@ -404,7 +429,7 @@ self.current_flat = 0 if self.current_cube <= self.limit or self.limit == 0: if self.current_flat < self.dimension: - self.sub_iter = nd_iter_space(self.dimension-1, self.current_cube) + self.sub_iter = nd_iter_space(self.dimension - 1, self.current_cube) try: return self.get_sub_iter_next() except StopIteration: @@ -414,7 +439,7 @@ raise StopIteration def get_sub_iter_next(self): - #print "get_sub_iter_next: current_flat - %d, current_cube - %d, sub_iter - (%d, %d)" % (self.current_flat, self.current_cube, self.sub_iter.dimension, self.sub_iter.limit) + # print "get_sub_iter_next: current_flat - %d, current_cube - %d, sub_iter - (%d, %d)" % (self.current_flat, self.current_cube, self.sub_iter.dimension, self.sub_iter.limit) try: self.value = copy.copy(self.sub_iter.next()) # the returned value should be a combination of the sub-value list @@ -426,15 +451,19 @@ self.current_flat += 1 raise + def jsvar(orig_str): return str(orig_str).replace('-', '_') -def escapejs(val): # escape strings for javascript + +def escapejs(val): # escape strings for javascript return json.dumps(str(val)) + def dict_combine(orig_dict, new_dict): return dict(orig_dict.items() + new_dict.items()) + def dict_reduce(orig_dict, keys): if type(keys) is not list: keys = [keys] @@ -444,6 +473,7 @@ del new_dict[each_key] return new_dict + # get the current language code def get_current_lang(): lang = get_language() @@ -451,7 +481,8 @@ return 'en' else: return lang - + + class HiveEnvironment(Environment): def __init__(self, *args, **kwargs): super(HiveEnvironment, self).__init__(*args, extensions=['jinja2.ext.i18n'], **kwargs) @@ -471,6 +502,7 @@ self.filters['json_dumps'] = json_dumps self.filters['escape_url'] = urllib.parse.quote + def convert_keys_to_string(data): if isinstance(data, str): return str(data) @@ -481,6 +513,7 @@ else: return data + class SortedListEncoder(json.JSONEncoder): def encode(self, obj): def sort_lists(item): @@ -490,61 +523,69 @@ return {k: sort_lists(v) for k, v in item.items()} else: return item + return super(SortedListEncoder, self).encode(sort_lists(obj)) + + # dump the dict into a fixed ordered string def dump_dict(src_dict): if not src_dict: return '' new_dict = convert_keys_to_string(src_dict) # we do not need to sort list, as the list has its own order - #return json.dumps(new_dict, sort_keys=True, cls=SortedListEncoder) + # return json.dumps(new_dict, sort_keys=True, cls=SortedListEncoder) return json.dumps(new_dict, sort_keys=True) + def url2obj(url_str): if type(url_str) not in (str, str): return url_str return json.loads(urllib.parse.unquote(url_str)) - + + def obj2url(obj): return urllib.parse.quote(json.dumps(obj)) + def json_dumps(obj): return json.dumps(obj, skipkeys=True, default=repr) - -def getcallargs(args, varargname, kwname, varargs, keywords): + + +def getcallargs(args, varargname, kwname, varargs, keywords): dctArgs = {} varargs = tuple(varargs) keywords = dict(keywords) - + argcount = len(args) varcount = len(varargs) callvarargs = None - + if argcount <= varcount: for n, argname in enumerate(args): dctArgs[argname] = varargs[n] - - callvarargs = varargs[-(varcount-argcount):] - + + callvarargs = varargs[-(varcount - argcount):] + else: for n, var in enumerate(varargs): dctArgs[args[n]] = var - - for argname in args[-(argcount-varcount):]: + + for argname in args[-(argcount - varcount):]: if argname in keywords: dctArgs[argname] = keywords.pop(argname) - + callvarargs = () - + if varargname is not None: dctArgs[varargname] = callvarargs - + if kwname is not None: dctArgs[kwname] = keywords - + dctArgs.update(keywords) return dctArgs + def is_ipv4(value): val_list = value.split('.') if len(val_list) != 4: @@ -553,6 +594,8 @@ if not each.isdigit(): return False return True + + def is_ipv6(value): val_list = value.split(':') if len(val_list) < 3: @@ -564,7 +607,8 @@ except: return False return True - + + def get_sysctl_int(name): libc = ctypes.CDLL("/lib/libc.so.7") size = ctypes.c_uint(0) @@ -584,31 +628,36 @@ raise ValueError('Fail to get value from sysctl: %s' % name) return v.value - + + _tzmap_cache = None from hive.tzmap import tzmap_orig + + def get_system_tzmap(): global _tzmap_cache # find the cache from the memory if _tzmap_cache is not None: return _tzmap_cache - + tzmap = [OrderedDict(sorted([ - ('GMT', {'parent':None, 'type':'item'}), - ])), OrderedDict([]), OrderedDict([])] + ('GMT', {'parent': None, 'type': 'item'}), + ])), OrderedDict([]), OrderedDict([])] for continent_name, continent in tzmap_orig.iteritems(): - tzmap[0][continent_name] = {'parent':None, 'type':'menu'} # no other info for now + tzmap[0][continent_name] = {'parent': None, 'type': 'menu'} # no other info for now for country_name, country in continent.iteritems(): - tzmap[1][country_name] = {'parent':continent_name, 'type':'menu'} + tzmap[1][country_name] = {'parent': continent_name, 'type': 'menu'} for zone_name, zone in country.iteritems(): - tzmap[2][zone_name] = {'parent':country_name, 'type':'item'} + tzmap[2][zone_name] = {'parent': country_name, 'type': 'item'} _tzmap_cache = tzmap return tzmap + def escapeshellarg(cmd_str): ret = '"' - esacpe_chars = ['#', '&', ';', '`', '|', '*', '?', '~', '<', '>', '^', '(', ')', '[', ']', '{', '}', '$', '\\', ',', '"'] + esacpe_chars = ['#', '&', ';', '`', '|', '*', '?', '~', '<', '>', '^', '(', ')', '[', ']', '{', '}', '$', '\\', ',', + '"'] for each in cmd_str: if each in esacpe_chars: ret += '\\' @@ -616,7 +665,8 @@ ret += '"' return ret -def upload_receive( request ): + +def upload_receive(request): """ Returns the file(s) uploaded by the user. """ @@ -631,7 +681,7 @@ return None -class JFUResponse( HttpResponse ): +class JFUResponse(HttpResponse): """ Returns an HTTP response with its data encoded in JSON format. @@ -645,17 +695,17 @@ declare a JSON MIME-type. """ - def __init__( self, request, data = True, *args, **kwargs ): - data = json.dumps( data ) - j = "application/json" + def __init__(self, request, data=True, *args, **kwargs): + data = json.dumps(data) + j = "application/json" accept = request.META.get('HTTP_ACCEPT') mime = j if accept and j in accept else 'text/plain' - super( JFUResponse, self ).__init__( data, mime, *args, **kwargs ) - + super(JFUResponse, self).__init__(data, mime, *args, **kwargs) -class UploadResponse( JFUResponse ): + +class UploadResponse(JFUResponse): """ Takes a dictionary containing the required jQuery File Upload fileupload response data and returns a JFUResponse. @@ -674,11 +724,12 @@ [https://github.com/blueimp/jQuery-File-Upload/wiki/Setup] """ - def __init__( self, request, file_dict, *args, **kwargs ): - files = file_dict if isinstance( file_dict, list ) else [ file_dict ] - data = { 'files' : files } - super( UploadResponse, self ).__init__( request, data, *args, **kwargs ) + def __init__(self, request, file_dict, *args, **kwargs): + files = file_dict if isinstance(file_dict, list) else [file_dict] + data = {'files': files} + super(UploadResponse, self).__init__(request, data, *args, **kwargs) + class HiveWebPipe(object): _pipe_pool = {} @@ -692,7 +743,7 @@ self._post_data = '' HiveWebPipe._pipe_pool[self._pipeid] = self - #left stands for client end, right stands for server end. + # left stands for client end, right stands for server end. def right_write(self, new_data): self.data.put(new_data) @@ -704,8 +755,8 @@ else: break return data - - def left_write(self,new_data): + + def left_write(self, new_data): self._post_data = new_data def right_read(self): @@ -743,7 +794,7 @@ is_finished = pipe.is_finished() msg = pipe.left_read() data = { - 'is_finished': is_finished, + 'is_finished': is_finished, 'msg': msg, } if pipe.is_finished(): @@ -752,14 +803,15 @@ else: # for network issue, pipe get request may come before HiveWebPipe is registered data = { - 'is_finished': False, + 'is_finished': False, 'msg': '', } return HttpResponse(json.dumps(data)) -def hive_webpipe_write_handler(request,id): + +def hive_webpipe_write_handler(request, id): pipe = HiveWebPipe.get_pipe_by_id(id) - + if pipe: cmd = request.POST['interactive_msg'] if cmd == '__eND_': @@ -769,7 +821,8 @@ pipe.left_write(cmd) return HttpResponse(json.dumps({})) - + + # add style classes to text # classes should be a string like "fa-sign-in text-warning" # multiple class names could be splitted by spaces @@ -797,7 +850,7 @@ files = [] for item in items: path = os.path.join(root, item) - #basic module has been compiled together with framework, should be ignored here. + # basic module has been compiled together with framework, should be ignored here. if path.find('client/app/modules/main') != -1: continue if path.find('client/app/modules/common') != -1: @@ -812,7 +865,7 @@ filename = f.split('/')[-1] tmp = filename.split('.') if len(tmp) == 1: - #ignore those files whose filename without suffix + # ignore those files whose filename without suffix continue if len(tmp) == 2: if tmp[1] == suffix: @@ -826,6 +879,7 @@ for d in dirs: get_frontend_file_path(result, d, suffix) + def update_frontend_index_html(): css_list = [] locales_path = djproject.an_settings.webui_locales_path @@ -859,7 +913,8 @@ with open(os.path.join(locales_path, "main/zh-cn.js"), 'w') as f: main_zh_str = "var mmcn = " + json.dumps(main_zh) f.write(main_zh_str) - js_list = ['app/modules/main/sidebar.js', 'app/modules/main/profile.js', 'app/locales/main/zh-cn.js', 'app/locales/main/en.js'] + js_list = ['app/modules/main/sidebar.js', 'app/modules/main/profile.js', 'app/locales/main/zh-cn.js', + 'app/locales/main/en.js'] get_frontend_file_path(css_list, djproject.an_settings.frontend_modules_path, 'css') get_frontend_file_path(js_list, djproject.an_settings.frontend_modules_path, 'js') html = '' @@ -875,15 +930,16 @@ m.update(str(time.time()).encode('utf-8')) random = m.hexdigest() for each in css_list: - new += ' \n'% (each, random) + new += ' \n' % (each, random) for each in js_list: - new += ' \n'% (each, random) - new = ''+ new + new += ' \n' % (each, random) + new = '' + new new += ' ' html = html.replace(to_replace, new) with open(djproject.an_settings.index_html_path, 'wb+') as f: f.write(html.encode('utf-8')) + def get_device_type(device_type): _type = None for each in device_type.split(): @@ -898,11 +954,13 @@ return "ag" elif "netgate" in _type: return "netgate" + return None elif _type in WAF_TYPE_LIST: return _type if _type[0] != "v" else _type[1:] else: return _type + def standard_model_type(type_list): std_list = [] for each in type_list: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/__init__.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/__init__.py (working copy) @@ -21,7 +21,7 @@ def to_json(self): ret = {} for k, v in self.__dict__.iteritems(): - if isinstance(v, (str, unicode, int, long, float, bool, dict)): + if isinstance(v, (str, int, float, bool, dict)): ret[k] = v ret['id'] = self.id return json.dumps(ret, skipkeys=True) @@ -47,14 +47,14 @@ def to_json(self): ret = {} for k, v in self.__dict__.iteritems(): - if isinstance(v, (str, unicode, int, long, float, bool, dict, list)): + if isinstance(v, (str, int, float, bool, dict, list)): ret[k] = v elif k == 'help_text': # in this case, v should be a lazy translation object # XXX I18N make the translation here! ret[k] = v.encode('utf-8') ret['id'] = self.id - ret['label'] = unicode(self.label) + ret['label'] = str(self.label) return json.dumps(ret, skipkeys=True, default=repr) @@ -70,10 +70,10 @@ def to_json(self): ret = {} for k, v in self.__dict__.iteritems(): - if isinstance(v, (str, unicode, int, long, float, bool)): + if isinstance(v, (str, int, float, bool)): ret[k] = v ret['id'] = self.id - ret['label'] = unicode(self.label) + ret['label'] = str(self.label) return json.dumps(ret, skipkeys=True) @@ -88,8 +88,8 @@ def to_json(self): ret = {} for k, v in self.__dict__.iteritems(): - if isinstance(v, (str, unicode, int, long, float, bool, dict)): + if isinstance(v, (str, int, float, bool, dict)): ret[k] = v ret['id'] = self.id - ret['label'] = unicode(self.label) + ret['label'] = str(self.label) return json.dumps(ret, skipkeys=True) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/form.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/form.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/form.py (working copy) @@ -1,24 +1,30 @@ +from hive import tzmap +from hive.utils import get_system_tzmap from hive.widgets import FormWidget -from hive.utils import andebug, get_system_tzmap -import json + class TextInput(FormWidget): _macro_name = 'textinput' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=64, unit_name='', required=False, default=None, editable=True, style='default',note='',**kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=64, unit_name='', + required=False, default=None, editable=True, style='default', note='', **kwargs): super(TextInput, self).__init__(name, label, help_text, validators, value, editable) self.length = length self.required = required self.unit_name = unit_name - self.note=note + self.note = note if default is None: self.default = "" else: self.default = str(default) self.style = style + class TextArea(FormWidget): _macro_name = 'textarea' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=512, required=False, default=None, editable=True, style='default', **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=512, required=False, + default=None, editable=True, style='default', **kwargs): super(TextArea, self).__init__(name, label, help_text, validators, value, editable) self.length = length self.required = required @@ -27,10 +33,13 @@ else: self.default = str(default) self.style = style - + + class PasswordInput(FormWidget): _macro_name = 'passwordinput' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=64, unit_name='', required=False, default=None, editable=True, confirm=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, length=64, unit_name='', + required=False, default=None, editable=True, confirm=True, **kwargs): super(PasswordInput, self).__init__(name, label, help_text, validators, value, editable) self.length = length self.required = required @@ -40,126 +49,168 @@ self.default = "" else: self.default = str(default) - + + class Select(FormWidget): _macro_name = 'select' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, choices=[], choices_url='', default=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, choices=[], choices_url='', + default=None, editable=True, **kwargs): super(Select, self).__init__(name, label, help_text, validators, value, editable) self.choices = choices self.default = default self.choices_url = choices_url - + + class MultiSelect(FormWidget): _macro_name = 'multiselect' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, choices=[], choices_url='', default=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, choices=[], choices_url='', + default=None, editable=True, **kwargs): super(MultiSelect, self).__init__(name, label, help_text, validators, value, editable) self.choices = choices self.default = default self.choices_url = choices_url - + + class Switch(FormWidget): _macro_name = 'switch' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, default=None, lexical=None, display=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, default=None, lexical=None, + display=None, editable=True, **kwargs): super(Switch, self).__init__(name, label, help_text, validators, value, editable) self.lexical = lexical - self.display= display + self.display = display self.default = default - + + class CheckboxInput(FormWidget): _macro_name = 'checkboxinput' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, default=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, default=None, editable=True, + **kwargs): super(CheckboxInput, self).__init__(name, label, help_text, validators, value, editable) self.default = default - + + class TransferBox(FormWidget): _macro_name = 'transferbox' - def __init__(self, name=None, label='', help_text='', choices_url='', target_model=None, value=None, editable=True, popup_add_url='', **kwargs): + + def __init__(self, name=None, label='', help_text='', choices_url='', target_model=None, value=None, editable=True, + popup_add_url='', **kwargs): super(TransferBox, self).__init__(name, label, help_text, value=value, editable=editable) self.choices_url = choices_url self.popup_add_url = popup_add_url self.target_model = target_model + class AssoBox(FormWidget): _macro_name = 'assobox' - def __init__(self, name=None, label='', help_text='', value=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', value=None, editable=True, **kwargs): super(AssoBox, self).__init__(name, label, help_text, value=value, editable=editable) for param_name, param_value in kwargs.iteritems(): setattr(self, param_name, param_value) - + + class AssoListBox(FormWidget): _macro_name = 'assolist' - def __init__(self, name=None, label='', help_text='', value=None, editable=True, **kwargs): + + def __init__(self, name=None, label='', help_text='', value=None, editable=True, **kwargs): super(AssoListBox, self).__init__(name, label, help_text, value=value, editable=editable) for param_name, param_value in kwargs.iteritems(): setattr(self, param_name, param_value) - + + class UnionWidget(FormWidget): _macro_name = 'unionwidget' - def __init__(self, name=None, label='', help_text='', value=None, editable=True, field_list=[], required=False, model_instance=None, manager=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', value=None, editable=True, field_list=[], required=False, + model_instance=None, manager=None, **kwargs): super(UnionWidget, self).__init__(name, label, help_text, value=value, editable=editable) self.field_list = field_list self.model_instance = model_instance self.manager = manager self.required = required + class GroupWidget(FormWidget): _macro_name = 'groupwidget' - def __init__(self, name=None, label='', help_text='', value=None, editable=True, field_list=[], required=False, model_instance=None, manager=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', value=None, editable=True, field_list=[], required=False, + model_instance=None, manager=None, **kwargs): super(GroupWidget, self).__init__(name, label, help_text, value=value, editable=editable) self.field_list = field_list self.model_instance = model_instance self.manager = manager self.required = required - + + class ListWidget(FormWidget): _macro_name = 'listwidget' - def __init__(self, name=None, label='', help_text='', value=None, editable=True, sub_field=None, required=False, model_instance=None, manager=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', value=None, editable=True, sub_field=None, required=False, + model_instance=None, manager=None, **kwargs): super(ListWidget, self).__init__(name, label, help_text, value=value, editable=editable) self.sub_field = sub_field self.model_instance = model_instance self.manager = manager self.required = required + class StaticFormWidget(FormWidget): _macro_name = 'staticwidget' - def __init__(self, name=None, label='', help_text='', asso_idx=0, context_value=None, context_repr=None, context_obj_dict=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', asso_idx=0, context_value=None, context_repr=None, + context_obj_dict=None, **kwargs): super(StaticFormWidget, self).__init__(name, label, help_text) self.asso_idx = asso_idx self.context_repr = context_repr self.context_value = context_value self.context_obj_dict = context_obj_dict + class ReferenceWidget(FormWidget): _macro_name = 'referencewidget' + def __init__(self, name=None, label='', help_text='', value=None, link=None, **kwargs): super(ReferenceWidget, self).__init__(name, label, help_text, value=value) self.link = link + class EnumOrder(FormWidget): _macro_name = 'enumorder' + def __init__(self, name=None, label='', help_text='', values=None, **kwargs): super(EnumOrder, self).__init__(name, label, help_text) self.values = values + class DateTimeInput(FormWidget): _macro_name = 'datetimeinput' + def __init__(self, name=None, label='', help_text='', validators=None, value=None, editable=True, **kwargs): super(DateTimeInput, self).__init__(name, label, help_text, validators, value, editable) + class TimezoneSelect(FormWidget): _macro_name = 'timezoneselect' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, editable=True, tzdict=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, editable=True, tzdict=None, + **kwargs): super(TimezoneSelect, self).__init__(name, label, help_text, validators, value, editable) if tzdict: - self.tzmap = tzmap + self.tzmap = tzmap else: - self.tzmap = get_system_tzmap() + self.tzmap = get_system_tzmap() + class ImportLocalFile(FormWidget): _macro_name = 'importlocalfile' - def __init__(self, name=None, label='', help_text='', validators=None, value=None, editable=True, button_text='', maxFileSize=5000000, acceptFileSuffix='', **kwargs): + + def __init__(self, name=None, label='', help_text='', validators=None, value=None, editable=True, button_text='', + maxFileSize=5000000, acceptFileSuffix='', **kwargs): super(ImportLocalFile, self).__init__(name, label, help_text, validators, value, editable) self.button_text = button_text self.maxFileSize = maxFileSize self.acceptFileSuffix = acceptFileSuffix - \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/statistics.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/statistics.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/statistics.py (working copy) @@ -1,12 +1,14 @@ -from hive.widgets import StatisticsWidget -from hive.utils import andebug import json +from hive.widgets import StatisticsWidget + + # Statistics Widgets class LineChart(StatisticsWidget): _macro_name = 'linechart' - def __init__(self, name=None, label='', help_text='', ajax_url=None, samp_freq=None, style=None, error=False, unit_name='', options_json={}, **kwargs): + def __init__(self, name=None, label='', help_text='', ajax_url=None, samp_freq=None, style=None, error=False, + unit_name='', options_json={}, **kwargs): super(LineChart, self).__init__(name, label, help_text, ajax_url) self.samp_freq = samp_freq self.style = style Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/status.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/status.py (revision 2575) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/widgets/status.py (working copy) @@ -1,15 +1,17 @@ from hive.widgets import StatusWidget -from hive.utils import andebug -import json + # Status Widgets class Stat(StatusWidget): _macro_name = 'stat' + def __init__(self, name=None, label='', help_text='', **kwargs): super(Stat, self).__init__(name, label, help_text) - + + class ActionStatus(StatusWidget): _macro_name = 'actionstatus' + def __init__(self, name=None, label='', help_text='', value=None, lexical='', status_values=[], actions=[], instance=None, style='table', **kwargs): super(ActionStatus, self).__init__(name, label, help_text) @@ -19,43 +21,56 @@ self.style = style self.value = value self.lexical = lexical - + @property def value_str(self): if self.value is None: return '' else: return str(self.value) - + + class Gauge(StatusWidget): _macro_name = 'gauge' - def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', num_unit_name='', **kwargs): + + def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', + num_unit_name='', **kwargs): super(Gauge, self).__init__(name, label, help_text, ajax_url, refresh_freq) self.unit_name = unit_name self.num_unit_name = num_unit_name + class BarStatus(StatusWidget): _macro_name = 'barstatus' - def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', num_unit_name='', **kwargs): + + def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', + num_unit_name='', **kwargs): super(BarStatus, self).__init__(name, label, help_text, ajax_url, refresh_freq) self.unit_name = unit_name self.num_unit_name = num_unit_name - + + class SquareStatus(StatusWidget): _macro_name = 'squarestatus' - def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', fields=None, **kwargs): + + def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=3000, unit_name='', fields=None, + **kwargs): super(SquareStatus, self).__init__(name, label, help_text, ajax_url, refresh_freq) self.unit_name = unit_name self.fields = fields + class PlainStatus(StatusWidget): _macro_name = 'plainstatus' + def __init__(self, name=None, label='', help_text='', ajax_url='', refresh_freq=0, unit_name='', **kwargs): super(PlainStatus, self).__init__(name, label, help_text, ajax_url, refresh_freq) self.unit_name = unit_name - + + class PlainInfo(StatusWidget): _macro_name = 'plaininfo' + def __init__(self, name=None, label='', help_text='', value=None, lexical='', instance=None, style='table', **kwargs): super(PlainInfo, self).__init__(name, label, help_text) @@ -63,4 +78,3 @@ self.style = style self.value = value self.lexical = lexical -