Index: /branches/amp_4_0/extensions/license_server/webui/model/cm_device/__init__.py =================================================================== --- /branches/amp_4_0/extensions/license_server/webui/model/cm_device/__init__.py (revision 2665) +++ /branches/amp_4_0/extensions/license_server/webui/model/cm_device/__init__.py (working copy) @@ -125,12 +125,7 @@ option_fields = ( DynamicMultiEnumField(name='device_list', verbose_name=_('Device List'), values=['__field:apv.device_mgmt.device.Device.name']), ) - - class Meta: - verbose_name = 'Registered Devices' - button_submit_text = lambda options: __('Register a new device') show_im_export_button = False - cache_expire_time = 5 list_config_options = { 'delete_confirm_msg': 'If you perform the deletion, it will disable the deivces which has been enabled.', 'columns':[ @@ -144,6 +139,11 @@ {'name': 'perbw'}, {'name': 'rel_bandwith'}, ]} + + class Meta: + verbose_name = 'Registered Devices' + button_submit_text = lambda options: __('Register a new device') + cache_expire_time = 5 help_text = _("Please make sure RESTful API service has been turned on 'https' on target devices and a RESTful API account has already been configured as well. If not, please refer to CLI handbook, using command 'restapi on https [port]' and 'user [user_name] [password] api' to configure them first." ) class Manager(CLIManager): Index: /branches/amp_4_0/extensions/vpn_license_server/webui/model/vpn_device/__init__.py =================================================================== --- /branches/amp_4_0/extensions/vpn_license_server/webui/model/vpn_device/__init__.py (revision 2665) +++ /branches/amp_4_0/extensions/vpn_license_server/webui/model/vpn_device/__init__.py (working copy) @@ -137,12 +137,7 @@ option_fields = ( DynamicMultiEnumField(name='device_list', verbose_name=_('Device List'), values=['__field:apv.device_mgmt.device.Device.name']), ) - - class Meta: - verbose_name = 'Registered Devices' - button_submit_text = lambda options: __('Register a new device') show_im_export_button = False - cache_expire_time = 5 list_config_options = { 'delete_confirm_msg': 'If you perform the deletion, it will disable the deivces which has been enabled.', 'columns':[ @@ -156,6 +151,11 @@ {'name': 'perbw'}, {'name': 'rel_bandwith'}, ]} + + class Meta: + verbose_name = 'Registered Devices' + button_submit_text = lambda options: __('Register a new device') + cache_expire_time = 5 help_text = _("Please make sure RESTful API service has been turned on 'https' on target devices and a RESTful API account has already been configured as well. If not, please refer to CLI handbook, using command 'restapi on https [port]' and 'user [user_name] [password] api' to configure them first." ) class Manager(CLIManager): Index: /branches/amp_4_0/platform/config/init_db.sql =================================================================== --- /branches/amp_4_0/platform/config/init_db.sql (revision 2665) +++ /branches/amp_4_0/platform/config/init_db.sql (working copy) @@ -196,7 +196,8 @@ -- Create a file_type table CREATE TABLE IF NOT EXISTS file_type ( - name varchar(32) primary key, + id SERIAL primary key, + name varchar(32), extend jsonb ); @@ -218,9 +219,11 @@ create_time varchar(64) NOT NULL, modify_time varchar(64) DEFAULT NULL, type varchar(32) NOT NULL, + file_type_id int NOT NULL, device_type varchar(32) DEFAULT NULL, + device_id varchar(64), comment text, - FOREIGN KEY (type) REFERENCES file_type (name) ON DELETE CASCADE, + FOREIGN KEY (file_type_id) REFERENCES file_type(id) ON DELETE CASCADE, extend jsonb ); Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/__init__.py (working copy) @@ -0,0 +1 @@ + Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/apps.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/apps.py (nonexistent) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/apps.py (working copy) @@ -0,0 +1,8 @@ +# cm/apps.py + +from django.apps import AppConfig + + +class CmConfig(AppConfig): + name = 'cm' + verbose_name = 'CM Module' Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/communication.py (working copy) @@ -1,8 +1,10 @@ import http.client import threading +import io from io import StringIO from ipaddress import ip_address, IPv4Address from xml.dom.minidom import parseString +import base64 import dicttoxml import pycurl @@ -90,9 +92,12 @@ def send_https_rest_request(method, url, params, ip, restapi_port, restapi_username, restapi_password, proto='https', cookie='', save_cookie=False, lock=True): - authorization = 'Basic %s' % (('%s:%s' % (restapi_username, restapi_password)).encode('base64').replace('\n', '')) - buffer = StringIO() + auth_str = f"{restapi_username}:{restapi_password}" + auth_bytes = auth_str.encode("utf-8") + authorization = 'Basic ' + base64.b64encode(auth_bytes).decode('ascii') + buffer = io.BytesIO() curl = pycurl.Curl() + curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.CONNECTTIMEOUT, 5) if lock: global L @@ -120,16 +125,16 @@ curl.perform() status = curl.getinfo(pycurl.RESPONSE_CODE) if status == 200: - cookie = buffer.getvalue().split("Set-Cookie: ")[1].split("\r\n")[0] + cookie = buffer.getvalue().decode('utf-8', errors='replace').split("Set-Cookie: ")[1].split("\r\n")[0] data = { 'status': status, - 'body': buffer.getvalue().split("\r\n\r\n")[1], + 'body': buffer.getvalue().decode('utf-8', errors='replace').split("\r\n\r\n")[1], 'cookie': cookie, } else: data = { 'status': status, - 'body': buffer.getvalue(), + 'body': buffer.getvalue().decode('utf-8', errors='replace'), } curl.close() except Exception as e: @@ -156,7 +161,7 @@ data = { 'status': curl.getinfo(pycurl.RESPONSE_CODE), # 'reason': response.reason, - 'body': buffer.getvalue(), + 'body': buffer.getvalue().decode('utf-8', errors='replace'), } curl.close() except Exception as e: @@ -182,7 +187,7 @@ curl.perform() data = { 'status': curl.getinfo(pycurl.RESPONSE_CODE), - 'body': buffer.getvalue(), + 'body': buffer.getvalue().decode('utf-8', errors='replace'), } curl.close() except Exception as e: @@ -196,8 +201,9 @@ def send_https_xmlrpc_request(method, url, params, ip, xmlrpc_port, proto='https'): if not check_ipv4(ip): ip = "[%s]" % ip - buffer = StringIO() + buffer = io.BytesIO() curl = pycurl.Curl() + curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.CONNECTTIMEOUT, 5) if method == 'POST': curl.setopt(pycurl.POST, 1) @@ -271,8 +277,9 @@ authorization = 'Basic %s' % (('%s:%s' % (restapi_username, restapi_password)).encode('base64').replace('\n', '')) if not check_ipv4(ip): ip = "[%s]" % ip - buffer = StringIO() + buffer = io.BytesIO() curl = pycurl.Curl() + curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.VERBOSE, 1) # DEBUG MODE curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json', 'Authorization: %s' % authorization, 'Expect: ']) curl.setopt(pycurl.CONNECTTIMEOUT, 5) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/__init__.py (working copy) @@ -1,2 +1,11 @@ verbose_name = u'CM Root' child_seq = ['system'] +from .device_mgmt import * +from .audit import * +from configuration import * +from ha import * +from monitor import * +from network import * +from system import * +from tasking import * +from virtualization import * Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/audit/user/__init__.py (working copy) @@ -12,12 +12,17 @@ from djproject.an_settings import * from hive.imports.model import * from hive.utils import get_device_type +from hive.model.base import ANModel +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.manager import CLIManager __ = _ # ToDo: Fix the syntax error in the following code. class AuditUser(ANModel): + cache_expire_time = 1 id = IntegerField(primary_key=True, configurable=False) user_name = CharField(verbose_name='User Name', length='1..16') vsite_name = CharField(verbose_name='Vsite name', length='1..64') @@ -26,7 +31,6 @@ class Meta: verbose_name = 'Audit User' - cache_expire_time = 1 class LogList(Action): verbose_name = _('Log List') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/config_file/__init__.py (working copy) @@ -2,6 +2,8 @@ import datetime import shutil from importlib import reload +import json +import os from apscheduler.schedulers.background import BackgroundScheduler from django.db.models.query import QuerySet @@ -12,15 +14,15 @@ from cm.lib.libbasic_operation import get_ip_address from cm.lib.parse_configfile import parse_vsite_from_configfile, parse_vsite_config_from_configfile from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import config_init_device, add_job_into_database, remove_job_from_database, config_batch_cli, \ - config_compliance_check, config_backup, config_custom, config_recover, config_compliance_check_all, \ - config_compliance_check_custom, config_clone, config_summary_diff -from cm.models.device_mgmt.device import get_rest_info_from_device -from cm.models.tasking import GLOBAL_TASK from djproject.an_settings import * from djproject.settings import CONFIG_UPLOAD_STORE_DIR from hive.imports.model import * from hive.utils import get_current_session +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ """ @@ -78,10 +80,10 @@ cli_content = TextField(verbose_name='CLI Content', optional=True, editable=True) upload_path = CharField(verbose_name='File Path', optional=True, editable=False) upload_type = BooleanField(verbose_name='Upload Type', optional=True, editable=False, default=False) + show_im_export_button = False class Meta: verbose_name = 'Config File' - show_im_export_button = False class Sync_from_device(Action): verbose_name = _('Sync From Device') @@ -352,6 +354,9 @@ return def _insert(self, instance): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_custom + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() data = instance.get_field_dict() file_type = data['file_type'] @@ -375,8 +380,8 @@ raise ModelQueryException(CLICmdError(__('The customzied file name is empty!'))) task_name = data['name'] + ".cfg" - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) if data["upload_type"]: @@ -427,7 +432,7 @@ sched = BackgroundScheduler() sched.add_job(config_custom, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) else: os.mknod(file_path + '/' + data['name']) @@ -465,6 +470,7 @@ def send_commands_to_device(self, device_name, cli_list): # return None + from cm.models.device_mgmt.device import get_rest_info_from_device device_info = get_rest_info_from_device(device_name)[0] sync_config = json.dumps({'config': '\n'.join(cli_list)}) try: @@ -517,13 +523,17 @@ return command_result def _perform_Backup(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_backup + from cm.models.device_mgmt.device import get_rest_info_from_device + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() pk_list = options['__pk_list'][0] device_name = pk_list['name'] date = datetime.datetime.now() + datetime.timedelta(seconds=3) device_info = get_rest_info_from_device(device_name)[0] task_name = device_name + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Backup config create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' @@ -535,12 +545,15 @@ sched = BackgroundScheduler() sched.add_job(config_backup, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return def _perform_Onekey_Backup(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_backup + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() result = [] if "device" in options["devices"]: if "ALL" in options["devices"]["device"]: @@ -573,8 +586,8 @@ date = datetime.datetime.now() + datetime.timedelta(seconds=3) device_info = get_rest_info_from_device(item)[0] task_name = item + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Backup config create time: " + datetime.datetime.now().strftime( @@ -621,11 +634,15 @@ id=task_name) else: sched.add_job(config_backup, trigger, next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return def _perform_Recover(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_recover + from cm.models.device_mgmt.device import get_rest_info_from_device + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() pk_list = options['__pk_list'][0] backup = True if "backup" in options and options["backup"] else False file_type = pk_list['file_type'] @@ -648,8 +665,8 @@ raise ModelQueryException( CLICmdError(__("Sorry, can not find this file(%s)." % os.path.basename(file_path)))) device_info = get_rest_info_from_device(device_name)[0] - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Recover config create time: " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n' @@ -661,7 +678,7 @@ sched = BackgroundScheduler() sched.add_job(config_recover, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return @@ -691,6 +708,7 @@ return def _perform_Sync_from_device(self, options): + from cm.models.device_mgmt.device import get_rest_info_from_device # sync from device {u'device': [u'\u5a01\u9707\u5929', u'\u64ce\u5929\u67f1'], # perform sync from device {u'device': [u'\u64ce\u5929\u67f1'], u'__pk_list': []} # print "perform sync from device", options @@ -765,12 +783,15 @@ return sorted(device_backup_list)[-1] def _perform_Clone_file(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_clone + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() pk_list = options['__pk_list'][0] date_now = datetime.datetime.now() date = date_now + datetime.timedelta(seconds=1) task_name = pk_list['name'] + '-' + date_now.strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Clone config create time: " + date_now.strftime('%Y-%m-%d %H:%M:%S') + '\n' @@ -784,11 +805,15 @@ sched = BackgroundScheduler() sched.add_job(config_clone, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return def _perform_Init_device(self, options): + from cm.lib.task_scheduler import config_init_device, remove_job_from_database + from cm.models.device_mgmt.device import get_rest_info_from_device + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() pk_list = options['__pk_list'][0] device_list = [] if 'group' in options['devices']: @@ -852,8 +877,8 @@ device_info['scope'] = options['scope'] if 'scope' in options and options['scope'] else None task_name = pk_list['name'] + '-' + device_str + '_' + date_now.strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} @@ -867,12 +892,16 @@ sched = BackgroundScheduler() sched.add_job(config_init_device, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return def check_config_from_device(self, file_name, diff_file_path, task_name, options, file_type, source_deivce=None): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_compliance_check, \ + config_compliance_check_all + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() if not os.path.exists(os.path.dirname(diff_file_path)): os.mkdir(os.path.dirname(diff_file_path)) if options['option'] == 'immediate': @@ -886,8 +915,8 @@ __('Time(%s) has passed, Now %s' % (date.strftime('%Y-%m-%d %H:%M:%S'), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} @@ -949,7 +978,7 @@ device_list=options["device"]) sched.add_job(config_compliance_check, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return @@ -984,6 +1013,10 @@ None if ":" in source_deivce and "-" in source_deivce else source_deivce) def _perform_Custom_Compliance_check(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, \ + config_compliance_check_custom, config_summary_diff + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() file_type = "backup" target_type = options["target_type"] source = options["source_device"] @@ -1053,8 +1086,8 @@ os.mkdir(os.path.dirname(diff_file_path)) date = datetime.datetime.now() + datetime.timedelta(seconds=3) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} @@ -1071,7 +1104,7 @@ date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) sched.add_job(config_compliance_check_custom, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) if target_type == 'batch': diff_file_name = 'summary_diff' @@ -1081,8 +1114,8 @@ if not os.path.exists(os.path.dirname(diff_file_path)): os.mkdir(os.path.dirname(diff_file_path)) date = datetime.datetime.now() + datetime.timedelta(seconds=4) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} @@ -1097,7 +1130,7 @@ add_job_into_database(task_name, 'summary_diff', description, 'date', 'waiting', date.strftime('%Y-%m-%d %H:%M:%S'), new_schedule_str, device_list=diff_file_name) sched.add_job(config_summary_diff, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return @@ -1123,6 +1156,10 @@ return def _perform_Batch_CLI(self, options): + from cm.lib.task_scheduler import add_job_into_database, config_batch_cli + from cm.models.device_mgmt.device import get_rest_info_from_device + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() pk_list = options['__pk_list'][0] if not pk_list['file_type'] == 'command': raise ModelQueryException(CLICmdError(__("Just command files could be used!"))) @@ -1146,8 +1183,8 @@ date_now = datetime.datetime.now() device_info = get_rest_info_from_device(device)[0] task_name = '%s-%s' % (pk_list['name'], date_now.strftime('%Y_%m_%d_%H:%M:%S')) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Send batch commands to device create time: " + date_now.strftime( @@ -1162,7 +1199,7 @@ sched = BackgroundScheduler() sched.add_job(config_batch_cli, 'date', next_run_time=real_date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/update/__init__.py (working copy) @@ -5,11 +5,13 @@ from django.utils.translation import gettext_lazy as _ from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, get_file_md5, compare_md5, mv_file, \ - upload_job, update_updatelist, upload_to_device -from cm.models.tasking import GLOBAL_TASK from djproject.settings import FILE_UPLOAD_DIR from hive.imports.model import * +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ @@ -23,19 +25,19 @@ md5_value = CharField(verbose_name='MD5', length='1..64', editable=True) download_link = CharField(verbose_name='Download Link', optional=True, length='1..128', configurable=False) location = CharField(verbose_name='Location', length='1..128', optional=True, configurable=False) + show_im_export_button = False + list_config_options = { + 'columns': [ + {'name': 'app_name'}, + {'name': 'build_version'}, + {'name': 'md5_value'}, + {'name': 'file_size'}, + {'name': 'download_link'}, + {'name': 'location'} + ]} class Meta: verbose_name = _('Upgrade') - show_im_export_button = False - list_config_options = { - 'columns': [ - {'name': 'app_name'}, - {'name': 'build_version'}, - {'name': 'md5_value'}, - {'name': 'file_size'}, - {'name': 'download_link'}, - {'name': 'location'} - ]} class Upload(Action): verbose_name = _('Build Upload') @@ -60,7 +62,7 @@ forever = False instance_mul = '1' option_fields = ( - # DynamicMultiEnumField(name='device', verbose_name='devices', + # DynamicMultiEnumField(name='device', verbose_name='devices', # values=['__value:---Please Select---', '__field:cm.device_mgmt.device.Device.name']), UnionField(name='devices', verbose_name='devices', optional=True, fields={ 'group': DynamicMultiEnumField(verbose_name='device_group', values=['__value:---Please Select---', @@ -153,6 +155,10 @@ return def _perform_Upload(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, get_file_md5, \ + compare_md5, mv_file, upload_job, update_updatelist, TaskScheduler + TASK = TaskScheduler() + def update_download_link(id, url): update_sql = "UPDATE update_list SET download_link='%s' WHERE id='%d'" % (url, id) db = DB.get_connected_db() @@ -182,8 +188,8 @@ date = date_now + datetime.timedelta(seconds=3) name = options['__pk_list'][0]['app_name'] + '-' + options['__pk_list'][0][ 'build_version'] + '-' + date_now.strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(name): - GLOBAL_TASK.remove(name) + if TASK.get_schedule(name): + TASK.remove(name) remove_job_from_database(name) @@ -198,11 +204,15 @@ sched = BackgroundScheduler() sched.add_job(upload_job, 'date', next_run_time=date, args=[int(options['__pk_list'][0]['id']), name, tmp_url], id=name) - GLOBAL_TASK.add(name, sched) + TASK.add(name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return def _perform_System_update(self, options): + from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_batch_cli, \ + upload_to_device, TaskScheduler + TASK = TaskScheduler() + def get_data_from_device(device_name): db = DB.get_connected_db() fetchall_sql = '''SELECT ip_address, restapi_port, restapi_username, restapi_password, type FROM device where name='%s' ''' % device_name @@ -242,8 +252,8 @@ device_info = get_data_from_device(device)[0] if not device_info: continue - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -272,6 +282,6 @@ sched = BackgroundScheduler() sched.add_job(upload_to_device, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/__init__.py (working copy) @@ -1,14 +1,19 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ +import json from cm.lib.communication import send_https_rest_request from cm.lib.parse_configfile import parse_specific_config, \ change_config_file, INSERT, DELETE, add_vsite_to_config, remove_vsite_from_config from cm.lib.postgres_db import DB -from cm.models.device_mgmt.device import get_rest_info_from_device from hive.imports.model import * from hive.model.legacycli import MATCHALL from hive.utils import get_current_session +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ @@ -30,6 +35,7 @@ def vs_insert_delete(asso_data, cli_str, specific_line, insert_or_delete): + from cm.models.device_mgmt.device import get_rest_info_from_device device_name = asso_data['device_name'] vsite_name = asso_data['vs_name'] device_info = get_rest_info_from_device(device_name) @@ -62,6 +68,7 @@ def CM_check(new_url, running_params_list, parse_list, src_str): + from cm.models.device_mgmt.device import get_rest_info_from_device result_diff = {} for params in running_params_list: for device in params['device_list']: @@ -221,18 +228,18 @@ tgt='configuration.vsite_config.acl.ACLRule.asso', mul='1', pos='left', optional=True), }) + list_config_options = {'columns': [ + {'name': 'vs_name'}, + {'name': 'device_name'}, + {'name': 'site_FQDM'}, + {'name': 'ip'}, + {'name': 'site_type'}, + {'name': 'description'}, + {'name': 'parent_site'}, + ]} class Meta: verbose_name = 'Vsite config' - list_config_options = {'columns': [ - {'name': 'vs_name'}, - {'name': 'device_name'}, - {'name': 'site_FQDM'}, - {'name': 'ip'}, - {'name': 'site_type'}, - {'name': 'description'}, - {'name': 'parent_site'}, - ]} class VPN_Network_Group_Sync(Action): verbose_name = _('Sync to Device') @@ -1027,6 +1034,7 @@ return result def _delete_instance(self, instance): + from cm.models.device_mgmt.device import get_rest_info_from_device vs_name = instance.vs_name device_name = instance.device_name file_path = DEFAULT_VSITE_PATH + device_name + '-' + vs_name @@ -1051,6 +1059,7 @@ return ret def _insert(self, instance): + from cm.models.device_mgmt.device import get_rest_info_from_device def insert_failed(vs_name, device_info): cli_str = "no virtual site name '%s' \n YES" % vs_name data = {'cmd': cli_str} @@ -1146,6 +1155,7 @@ # cli_str = 'virtual site ip ' def _perform_Refresh(self, options): + from cm.models.device_mgmt.device import get_rest_info_from_device # configuration refresh {u'__pk_list': [{u'vs_name': u'bug', u'device_name': u'liyi_2017'}]} pk_list = options['__pk_list'][0] vs_name = pk_list['vs_name'] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/acl/__init__.py (working copy) @@ -1,4 +1,5 @@ import datetime +import json from apscheduler.schedulers.background import BackgroundScheduler from django.db.models.query import QuerySet @@ -8,12 +9,14 @@ from cm.lib.parse_configfile import parse_specific_config, delete_specific_info_from_config, \ sync_vsite_to_config from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ - update_cm_role_devicelist from cm.models.configuration.vsite_config import vs_insert_delete, CM_check -from cm.models.tasking import GLOBAL_TASK from hive.imports.model import * from hive.model.legacycli import MATCHALL +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ @@ -24,14 +27,14 @@ group_type = CharField(verbose_name='ACL Group Type', length='1..32', primary_key=True) name = CharField(verbose_name='ACL Group Name', length='1..32', primary_key=True) description = TextField(verbose_name='ACL Group Description', length='1..1024', optional=True) + list_config_options = {'columns': [ + {'name': 'group_type'}, + {'name': 'name'}, + {'name': 'description'}, + ]} class Meta: verbose_name = 'ACL Group' - list_config_options = {'columns': [ - {'name': 'group_type'}, - {'name': 'name'}, - {'name': 'description'}, - ]} class Manager(CLIManager): def _filter(self, filter_dict): @@ -81,13 +84,13 @@ mul='*', pos='right', optional=True) group_name = CharField(verbose_name='ACL Group Name', length='1..32', primary_key=True) resource = CharField(verbose_name='ACL Resource', length='1..128', primary_key=True) + list_config_options = {'columns': [ + {'name': 'group_name'}, + {'name': 'resource'}, + ]} class Meta: verbose_name = 'ACL Resource' - list_config_options = {'columns': [ - {'name': 'group_name'}, - {'name': 'resource'}, - ]} class Manager(CLIManager): def _filter(self, filter_dict): @@ -131,16 +134,16 @@ priority = Int32Field(verbose_name=_('Acl Priority'), scope='1..1000') target_type = CharField(verbose_name='Target Type', length='1..8', default='R', primary_key=True, help_text='[R(role) | U(user) | G(group)]') + list_config_options = {'columns': [ + {'name': 'target_name'}, + {'name': 'group_name'}, + {'name': 'action'}, + {'name': 'priority'}, + {'name': 'target_type'}, + ]} class Meta: verbose_name = 'ACL Rule' - list_config_options = {'columns': [ - {'name': 'target_name'}, - {'name': 'group_name'}, - {'name': 'action'}, - {'name': 'priority'}, - {'name': 'target_type'}, - ]} class Manager(CLIManager): def _filter(self, filter_dict): @@ -271,6 +274,8 @@ return def _perform_Sync(self, options): + from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, TaskScheduler + TASK = TaskScheduler() sess = get_current_session() cm_acl_model = get_model('cm', ['configuration', 'vsite_config', 'acl', 'CMACLResource']) @@ -305,8 +310,8 @@ {'params': json.dumps({'config': cli_str}), 'device_list': device[0]['device_list']}) task_name = '%s-%s' % (cm_manager[0].description, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -335,7 +340,7 @@ device_list=','.join(service_list)) sched = BackgroundScheduler() sched.add_job(sync_to_device, 'date', next_run_time=date, args=new_schedule['args']) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return @@ -373,6 +378,7 @@ return result_diff def _perform_Init_DeviceList(self, options): + from cm.lib.task_scheduler import update_cm_role_devicelist if options['option'] not in ['delete', 'insert']: raise ModelQueryException(CLICmdError(__('Internal error.'))) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/role/__init__.py (working copy) @@ -8,12 +8,14 @@ from cm.lib.parse_configfile import parse_specific_config, delete_specific_info_from_config, \ sync_vsite_to_config from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ - update_cm_role_devicelist from cm.models.configuration.vsite_config import vs_insert_delete, CM_check -from cm.models.tasking import GLOBAL_TASK from hive.imports.model import * from hive.model.legacycli import MATCHALL +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ @@ -390,6 +392,9 @@ return def _perform_Sync(self, options): + from cm.lib.task_scheduler import sync_to_device, add_job_into_database, remove_job_from_database, \ + TaskScheduler + TASK = TaskScheduler() sess = get_current_session() cm_role_model = get_model('cm', ['configuration', 'vsite_config', 'role', 'CMRoleGroup']) # pk_list = options['__pk_list'][0] @@ -425,8 +430,8 @@ {'params': json.dumps({'config': cli_str}), 'device_list': device[0]['device_list']}) task_name = '%s-%s' % (cm_manager[0].description, datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S')) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) # data_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -455,7 +460,7 @@ sched = BackgroundScheduler() sched.add_job(sync_to_device, 'date', next_run_time=date, args=new_schedule['args']) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) return @@ -491,6 +496,7 @@ return result_diff def _perform_Init_DeviceList(self, options): + from cm.lib.task_scheduler import update_cm_role_devicelist if options['option'] not in ['delete', 'insert']: raise ModelQueryException(CLICmdError(__('Internal error.'))) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/configuration/vsite_config/vpn/__init__.py (working copy) @@ -27,14 +27,14 @@ (2, 'Application'), (0, 'Both'), )) + list_config_options = {'columns': [ + {'name': 'vpn_group_name'}, + {'name': 'net_resource'}, + {'name': 'net_type'}, + ], 'click_enable': False} class Meta: verbose_name = 'Network-type VPN Resource Item' - list_config_options = {'columns': [ - {'name': 'vpn_group_name'}, - {'name': 'net_resource'}, - {'name': 'net_type'}, - ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): @@ -84,14 +84,14 @@ (2, 'Application'), (0, 'Both'), )) + list_config_options = {'columns': [ + {'name': 'vpn_group_name'}, + {'name': 'net_resource'}, + {'name': 'net_type'}, + ], 'click_enable': False} class Meta: verbose_name = 'Network-type VPN Resource Item' - list_config_options = {'columns': [ - {'name': 'vpn_group_name'}, - {'name': 'net_resource'}, - {'name': 'net_type'}, - ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): @@ -136,6 +136,9 @@ tgt='configuration.vsite_config.VsiteConfig.asso_vpn_resource_group', mul='*', pos='right', optional=True) name = CharField(verbose_name='VPN Resource Group', primary_key=True, length='1..64') + list_config_options = {'columns': [ + {'name': 'name'}, + ]} # asso_net_item = AssoField2(verbose_name=_('Network-type VPN Resource Item'), # tgt='configuration.vsite_config.vpn.NetGroupItem.asso', mul='1', pos='left', optional=True) @@ -144,9 +147,6 @@ class Meta: verbose_name = 'VPN Resource Group' - list_config_options = {'columns': [ - {'name': 'name'}, - ]} class Manager(CLIManager): def _filter(self, filter_dict): @@ -199,14 +199,14 @@ first_ip = IPAddressField(verbose_name=_('First IP Address'), primary_key=True) last_ip = IPAddressField(verbose_name=_('Last IP Address'), primary_key=True) unit_name = CharField(verbose_name='Unit Name', length='1..64', optional=True) + list_config_options = {'columns': [ + {'name': 'netpool_name'}, + {'name': 'first_ip'}, + {'name': 'last_ip'}, + ], 'click_enable': False} class Meta: verbose_name = 'IP Address Range' - list_config_options = {'columns': [ - {'name': 'netpool_name'}, - {'name': 'first_ip'}, - {'name': 'last_ip'}, - ], 'click_enable': False} class Manager(CLIManager): def _get(self, pk_dict=None): @@ -255,6 +255,10 @@ mul='*', pos='right', optional=True) name = CharField(verbose_name=_('VPN Netpool name'), primary_key=True, length='1..31') enable_nat = BooleanField(verbose_name='Enable Nat') + list_config_options = {'columns': [ + {'name': 'name'}, + {'name': 'enable_nat'} + ]} # ip_range = FieldGroup(writable=False, verbose_name='IP Range', level=BASIC, editable=True, fields={ # 'asso_ip_address' : AssoField2(verbose_name=_('IP Address'), @@ -263,10 +267,6 @@ class Meta: verbose_name = 'VPN Netpool' - list_config_options = {'columns': [ - {'name': 'name'}, - {'name': 'enable_nat'} - ]} class Manager(CLIManager): def _filter(self, filter_dict): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (working copy) @@ -18,7 +18,6 @@ from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_backup from cm.models.audit.user.user_body import body_login_detail, body_device_vsite from cm.models.system.license import check_license -from cm.models.tasking import GLOBAL_TASK from djproject.an_settings import * from hive.imports.model import * from hive.model.loading import get_model @@ -171,29 +170,29 @@ }) device_group = AssoField2(verbose_name=_('Device Group'), tgt='device_mgmt.device_group.DeviceGroup.device_list', mul='*', pos='right') + show_im_export_button = False + list_config_options = { + 'columns': [ + {'name': 'name'}, + {'name': 'type'}, + {'name': 'ip'}, + {'name': 'restapi_port'}, + {'name': 'restapi_account'}, + {'name': 'connection'}, + {'name': 'status'}, + {'name': 'license_key'}, + {'name': 'asso_ip_pool'}, + {'name': 'log_enable'}, + # {'name':'snmp_enable'}, + {'name': 'gateway_domain'}, + {'name': 'location'}, + {'name': 'firewall_ip'}, + {'name': 'intranet_ip'}, + {'name': 'webui_port'}, + ], 'select_button_enable': True, 'export_button_enable': True} class Meta: verbose_name = 'Managed Devices' - show_im_export_button = False - list_config_options = { - 'columns': [ - {'name': 'name'}, - {'name': 'type'}, - {'name': 'ip'}, - {'name': 'restapi_port'}, - {'name': 'restapi_account'}, - {'name': 'connection'}, - {'name': 'status'}, - {'name': 'license_key'}, - {'name': 'asso_ip_pool'}, - {'name': 'log_enable'}, - # {'name':'snmp_enable'}, - {'name': 'gateway_domain'}, - {'name': 'location'}, - {'name': 'firewall_ip'}, - {'name': 'intranet_ip'}, - {'name': 'webui_port'}, - ], 'select_button_enable': True, 'export_button_enable': True} help_text = _( "When registering, please make sure a RESTful API account has already been configured on target device and the RESTful API service has been turned on 'https' as well. If not, please refer to CLI handbook, using command 'restapi on [protocol] [port]' and 'user [password] [level]' to configure them first.") @@ -372,7 +371,7 @@ each['license_limit'] = rtn[12][0] if rtn[12] else '' each['license_date'] = rtn[-1][0] if rtn[-1] else '' each['license_feature'] = re.sub("\s+", ", ", rtn[-2][0].split('License Key')[0]).strip(', ') if \ - rtn[-2] else '' + rtn[-2] else '' del each['version'] if each['snmp_general']: @@ -460,9 +459,12 @@ return False def _insert(self, instance): + from cm.lib.task_scheduler import remove_job_from_database, TaskScheduler + TASK = TaskScheduler() data = instance.get_field_dict() - check_license(data["type"]) + # check_license(data["type"]) + def insert_ip_pool(device, device_id): response_data = send_cli_to_device(device, 'show running all "iprange dynamic"', mode='enable') if response_data['message'] == 'success': @@ -473,7 +475,7 @@ db = DB.get_connected_db() for value in rtn[0]: ip_pool_range = value['first_ip'] + '-' + value['last_ip'] - save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES " + "('%s', '%s')" % ( + save_sql = "INSERT INTO ip_pool(ip_pool_range, device_id) VALUES ('%s', '%s')" % ( ip_pool_range, device_id) db.execute_sql(save_sql) db.close() @@ -483,44 +485,61 @@ raise ModelQueryException(CLICmdError(__('Activation Failed'))) return - data['id'] = uuid.uuid1() + import uuid + import datetime + + data['id'] = str(uuid.uuid1()) data['zone'] = "default" - data['ip'] = data['ip'].values()[0] - data['device_group'] = data['device_group'][0]['name'] - if data['group_name_from'] == 'new': + # Defensive: 'ip', 'firewall_ip', 'intranet_ip' are dicts with a single value + for k in ['ip', 'firewall_ip', 'intranet_ip']: + val = data.get(k, {}) + if val and isinstance(val, dict): + data[k] = list(val.values())[0] if val else None + else: + data[k] = None + + # Defensive: device_group is a list of dicts + if 'device_group' in data and data['device_group']: + data['device_group'] = data['device_group'][0].get('name', None) + else: + data['device_group'] = None + + if data.get('group_name_from') == 'new' and data['device_group']: db = DB.get_connected_db() fetchall_sql = "SELECT name FROM device_group WHERE name = '%s'" % data['device_group'] res_data = db.fetchall(fetchall_sql) if res_data and res_data[0]: raise ModelQueryException(CLICmdError(__('Device Group "%s" is exist.' % data['device_group']))) - insert_sql = "INSERT INTO device_group(name) values ('%s')" % data['device_group'] db.execute_sql(insert_sql) db.close() - data['firewall_ip'] = data['firewall_ip'].values()[0] - data['intranet_ip'] = data['intranet_ip'].values()[0] - data['restapi_username'] = data['restapi_account']['restapi_username'] - data['restapi_password'] = data['restapi_account']['restapi_password'] - data['console_username'] = data['console_account']['console_username'] - data['console_password'] = data['console_account']['console_password'] - if self.check_ip_name_from_device(data['ip'], data['name']): + # Defensive: .get() for nested keys + for account, keys in [('restapi_account', ['restapi_username', 'restapi_password']), + ('console_account', ['console_username', 'console_password'])]: + account_dict = data.get(account, {}) + for key in keys: + data[key] = account_dict.get(key, "") + + if self.check_ip_name_from_device(data['ip'], data.get('name')): raise ModelQueryException(CLICmdError(__('Sorry IP or Name has exists.'))) L.add_lock(data['ip']) try: - if data['protocol'] == 'xmlrpc': + if data.get('protocol') == 'xmlrpc': rest_response_data = send_cli_to_device(data, "show version", mode='enable') else: - new_url = modify_url('/rest/device_type/system/SystemInfo/version', data['type']) - rest_response_data = send_https_rest_request('GET', new_url, '', data['ip'], data['restapi_port'], - data['restapi_username'], data['restapi_password']) + new_url = modify_url('/rest/device_type/system/SystemInfo/version', data.get('type')) + rest_response_data = send_https_rest_request('GET', new_url, '', data['ip'], + data.get('restapi_port'), + data.get('restapi_username'), + data.get('restapi_password')) except Exception as e: if 'timed out' in str(e): raise ModelQueryException(CLICmdError( __('Register timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) raise ModelQueryException(CLICmdError(__('Register Failed.'))) else: - if data['protocol'] == 'xmlrpc': + if data.get('protocol') == 'xmlrpc': if rest_response_data['message'] == 'success': data['version'] = rest_response_data['data'] elif rest_response_data['message'] == 'requestfailed': @@ -541,60 +560,43 @@ RegexParser('License Key : (.+?)\n', MATCHONE), RegexParser('Model : (.+?)\n', MATCHONE), ]) - if rtn[0]: + if rtn and rtn[0]: data['license_key'] = rtn[0][0] data['status'] = 'active' else: data['license_key'] = None data['status'] = 'new' - model_str = rtn[1][0] if rtn[1] else None + model_str = rtn[1][0] if rtn and rtn[1] else None def get_device_model(model_string): if model_string: - # Array AVX vAPV - # Array AVX vxAG - # Array APV xxx - # Array vAPV - # Array AG xxx - # Array vxAG - if model_string.find('Array APV') != -1: + if 'Array APV' in model_string: return 'APV' - if model_string.find('Array AG') != -1: + if 'Array AG' in model_string: return 'AG' - if model_string.find('vAPV') != -1: + if 'vAPV' in model_string: return 'vAPV' - if model_string.find('vxAG') != -1: + if 'vxAG' in model_string: return 'vxAG' - # INFOSEC Ccypher vNSAE - # INFOSEC Ccypher vNetOpti - # INFOSEC Ccypher vNetGate - # INFOSEC vNSAE - # INFOSEC vNetOpti - # INFOSEC vNetGate - # INFOSEC NSAE xxx - # INFOSEC NetOpti xxx - # INFOSEC NetGate xxx - if model_string.find('vNetOpti') != -1: + if 'vNetOpti' in model_string: return 'vNetOpti' - if model_string.find('vNSAE') != -1: + if 'vNSAE' in model_string: return 'vNSAE' - if model_string.find('vNetGate') != -1: + if 'vNetGate' in model_string: return 'vNetGate' - if model_string.find('INFOSEC NetOpti') != -1: + if 'INFOSEC NetOpti' in model_string: return 'NetOpti' - if model_string.find('INFOSEC NSAE') != -1: + if 'INFOSEC NSAE' in model_string: return 'NSAE' - if model_string.find('INFOSEC NetGate') != -1: + if 'INFOSEC NetGate' in model_string: return 'NetGate' - # Array vASF - # Array ASF - if model_string.find('Array ASF') != -1: + if 'Array ASF' in model_string: return 'ASF' - if model_string.find('vASF') != -1: + if 'vASF' in model_string: return 'vASF' - if model_string.find('Infosec NetWAF') != -1: + if 'Infosec NetWAF' in model_string: return 'NetWAF' - if model_string.find('vNetWAF') != -1: + if 'vNetWAF' in model_string: return 'vNetWAF' return None @@ -602,24 +604,30 @@ if not model: raise ModelQueryException(CLICmdError( __('Register Failed! Invalid model! Only %s are supported.') % ', '.join(DEVICE_STD_LIST))) - if model != data['type']: + if model != data.get('type'): raise ModelQueryException(CLICmdError( __('Register Failed! Current device model is %s but you choose %s.') % (model, data['type']))) - # else: - # raise ModelQueryException(CLICmdError(__('Register Failed! Please make sure device model is right.'))) db = DB.get_connected_db() - save_sql = "INSERT INTO device(id, zone, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, console_username, console_password, connection, status, license_key, gateway_domain, location, firewall_ip, intranet_ip, version, type, own, log_enable, webui_port, device_group, enable_password) Values " + "('%(id)s', '%(zone)s', '%(name)s', '%(ip)s', '%(protocol)s', '%(restapi_port)s', '%(restapi_username)s', '%(restapi_password)s', '%(console_username)s', '%(console_password)s', 'connected', '%(status)s', '%(license_key)s', '%(gateway_domain)s', '%(location)s', '%(firewall_ip)s', '%(intranet_ip)s', '%(version)s', '%(type)s', '%(own)s', '%(log_enable)d', '%(webui_port)d', '%(device_group)s', '%(enable_password)s')" % data + save_sql = ( + "INSERT INTO device(" + "id, zone, name, ip_address, protocol, restapi_port, restapi_username, restapi_password, " + "console_username, console_password, connection, status, license_key, gateway_domain, location, " + "firewall_ip, intranet_ip, version, type, own, log_enable, webui_port, device_group, enable_password" + ") Values ('%(id)s', '%(zone)s', '%(name)s', '%(ip)s', '%(protocol)s', '%(restapi_port)s', '%(restapi_username)s', '%(restapi_password)s', '%(console_username)s', '%(console_password)s', 'connected', '%(status)s', '%(license_key)s', '%(gateway_domain)s', '%(location)s', '%(firewall_ip)s', '%(intranet_ip)s', '%(version)s', '%(type)s', '%(own)s', '%(log_enable)d', '%(webui_port)d', '%(device_group)s', '%(enable_password)s')" + ) % data db.execute_sql(save_sql) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - insert_config_sql = "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type, device_id) values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( - data['name'], create_time, create_time, 'device', '', data['type'], data['id']) + insert_config_sql = ( + "INSERT INTO file_list(name, create_time, modify_time, type, file_type_id, comment, device_type, " + "device_id)" + "values('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( + data['name'], create_time, create_time, 'device', 1, '', data['type'], data['id']) + ) db.execute_sql(insert_config_sql) db.close() - # file_path = self.get_config_from_device(data['ip'], data['restapi_port'], data['restapi_username'], data['restapi_password'], data['type'], data['name']) - ### XXX : Need to check if ip pool is used in AMP 3.0 - if data['type'].lower() in VPN_TYPE_LIST: + if data["type"].lower() in VPN_TYPE_LIST: try: insert_ip_pool(data, data['id']) except Exception as e: @@ -628,18 +636,13 @@ db.execute_sql(delete_sql) db.close() raise ModelQueryException(CLICmdError(__('Get %s ip pool failed!' % data['type']))) - # parse vsite name, ip... from global config - # parse_vsite_config_from_configfile(file_path, data['name']) - # parse all vsite config from 'show running all', and write to vsite file - # parse_vsite_from_configfile(file_path, data['name']) - # backup the config file - if data["backup_enable"]: + if data.get("backup_enable"): date = datetime.datetime.now() + datetime.timedelta(seconds=1) device_info = get_rest_info_from_device(data['name'])[0] task_name = data['name'] + '-' + datetime.datetime.now().strftime('%Y_%m_%d_%H:%M:%S') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) + if TASK.get_schedule(task_name): + TASK.remove(task_name) remove_job_from_database(task_name) new_schedule = {} description = " Backup config create time: " + datetime.datetime.now().strftime( @@ -652,25 +655,13 @@ sched = BackgroundScheduler() sched.add_job(config_backup, 'date', next_run_time=date, args=new_schedule['args'], id=task_name) - GLOBAL_TASK.add(task_name, sched) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) mark_expire_all(get_model('cm', ['device_mgmt', 'device_group', 'DeviceGroup'])) - # insert device default template key-value insert_default_tmpkey(data['ip']) - - # create index template - # put_template(data['ip']) - # oper_log('info', 'device', 'Added a new device %s' % (data['name'])) return - # #set log host - # if data['log_enable']: - # cm = __import__("cm") - # self.set_log_host(data['ip'], data['restapi_port'], data['restapi_username'], data['restapi_password']) - - # return - # def set_log_host(ip_addr, ip_port, restapi_username, restapi_password): # "log http welf\n log option showlogidkey on\n log level debug \n vpn netpool trafficlog " # pass Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device_group/__init__.py (working copy) @@ -6,6 +6,11 @@ from cm.lib.communication import L from cm.lib.postgres_db import DB from hive.imports.model import * +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/ha/ha_cluster/__init__.py (working copy) @@ -5,9 +5,12 @@ from cm.lib.communication import send_https_rest_request, modify_url from cm.lib.postgres_db import DB -from cm.models.device_mgmt.device import get_rest_info_from_device from djproject.an_settings import * from hive.imports.model import * +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ @@ -123,6 +126,7 @@ return def get_cmd_result(self, cmd, device_name): + from cm.models.device_mgmt.device import get_rest_info_from_device device_info = get_rest_info_from_device(device_name)[0] data = {'cmd': cmd} config_params = json.dumps(data) @@ -190,6 +194,7 @@ return json.dumps({'device': device_name, 'data': result}) def _get_ha_condition(self, pk_dict): + from cm.models.device_mgmt.device import get_rest_info_from_device db = DB.get_connected_db() # select_sql = ''' SELECT name FROM device where ha_cluster='%s' ''' % pk_dict['name'] select_sql = "select name from hc_device left join device on d_id=id where g_id=%d" % pk_dict['id'] Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/monitor/__init__.py (working copy) @@ -5,9 +5,12 @@ from django.utils.translation import gettext_lazy as _ from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import monitor_alert, get_global_settings -from cm.models.tasking import GLOBAL_TASK -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import IntegerField, BooleanField, CharField, TextField, EnumField +from hive.model.action import Action +from hive.model.fields import ValueCondition +from hive.model.manager import UpdatingFields, CLIManager +from hive.log_utils import logging __ = _ @@ -144,6 +147,8 @@ @UpdatingFields(['sslcert_check_enable', 'assign', 'message']) def _update_sslcert_info(self, instance): + from cm.lib.task_scheduler import monitor_alert, TaskScheduler + TASK = TaskScheduler() monitor_id = self.update_insert_ssl_cert(instance.sslcert_check_enable, instance.assign, instance.message) if not monitor_id: raise ModelQueryException(CLICmdError(__('Update failed.'))) @@ -151,11 +156,11 @@ if instance.sslcert_check_enable: sched = BackgroundScheduler() sched.add_job(monitor_alert, 'interval', minutes=60 * 24, args=[10, "", monitor_id]) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) - GLOBAL_TASK.add(task_name, sched) + if TASK.get_schedule(task_name): + TASK.remove(task_name) + TASK.add(task_name, sched) else: - GLOBAL_TASK.remove(task_name) + TASK.remove(task_name) class MonitorMsg(ANModel): @@ -168,20 +173,20 @@ target_type = IntegerField(verbose_name='Target_type') extend = TextField(verbose_name='Extend Message') note = TextField(verbose_name='Note', optional=True, editable=True) + list_config_options = { + 'columns': [ + {'name': 'alert_time'}, + {'name': 'deal_time'}, + {'name': 'alert_name'}, + {'name': 'extend'}, + {'name': 'target_name'}, + {'name': 'target_type'}, + {'name': 'alert_status'} + ], 'add_button_hide': True + } class Meta: verbose_name = 'Monitor Message' - list_config_options = { - 'columns': [ - {'name': 'alert_time'}, - {'name': 'deal_time'}, - {'name': 'alert_name'}, - {'name': 'extend'}, - {'name': 'target_name'}, - {'name': 'target_type'}, - {'name': 'alert_status'} - ], 'add_button_hide': True - } class SearchAlert(Action): verbose_name = _('Search Alert') @@ -380,24 +385,24 @@ (1, 'Major'), (2, 'Minor') )) + list_config_options = { + 'columns': [ + {'name': 'name'}, + {'name': 'target_type'}, + {'name': 'target_list'}, + {'name': 'type_id'}, + {'name': 'interval'}, + {'name': 'condition'}, + {'name': 'message'}, + {'name': 'enable'}, + {'name': 'modify_time'}, + {'name': 'assign'}, + {'name': 'level'} + ] + } class Meta: verbose_name = 'Monitor' - list_config_options = { - 'columns': [ - {'name': 'name'}, - {'name': 'target_type'}, - {'name': 'target_list'}, - {'name': 'type_id'}, - {'name': 'interval'}, - {'name': 'condition'}, - {'name': 'message'}, - {'name': 'enable'}, - {'name': 'modify_time'}, - {'name': 'assign'}, - {'name': 'level'} - ] - } class DealAlert(Action): verbose_name = _('Deal Alert') @@ -489,15 +494,19 @@ return None def start_monitor(self, task_name, args, interval): + from cm.lib.task_scheduler import monitor_alert, TaskScheduler + TASK = TaskScheduler() sched = BackgroundScheduler() sched.add_job(monitor_alert, 'interval', minutes=interval, args=args) - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) - GLOBAL_TASK.add(task_name, sched) + if TASK.get_schedule(task_name): + TASK.remove(task_name) + TASK.add(task_name, sched) def remove_monitor(self, task_name): + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() if task_name: - GLOBAL_TASK.remove(task_name) + TASK.remove(task_name) return def get_device_ip(self, device_list): @@ -528,6 +537,7 @@ del data['extend'] def _insert(self, instance): + from cm.lib.task_scheduler import get_global_settings # {'enable': True, 'name': u'alert', 'device_list': u'device', 'interval': 2, 'email': '', # 'send_email': False, 'modify_time': '', 'message': u'heiheihei', 'id': 0, 'condition': u'cpu>=100'} now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') @@ -588,6 +598,7 @@ ''' def _update(self, instance): + from cm.lib.task_scheduler import get_global_settings now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') data = instance.get_field_dict() data['now'] = now Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/dns/__init__.py (working copy) @@ -1,17 +1,23 @@ +import re + from django.utils.translation import gettext_lazy as _ -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import IPAddressField +from hive.model.manager import UpdatingFields, CLIManager +from hive.model.legacycli import CLICmdError, RegexParser, BlankParser, MATCHALL +from hive.model.query import QuerySet __ = _ class DNSServer(ANModel): server_ip = IPAddressField(verbose_name=_('IP'), primary_key=True) + show_im_export_button = False + list_config_options = {'click_enable': False} class Meta: verbose_name = _('DNS Server') - show_im_export_button = False - list_config_options = {'click_enable': False} class Manager(CLIManager): def _get_query_set(self): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/interface/__init__.py (working copy) @@ -1,8 +1,16 @@ +import re + from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -from hive.imports.model import * -from hive.node.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import (IPAddressField, AssoField2, UnionField, NetmaskField, Uint32Field, + MACAddressField, IntegerField, BooleanField, CharField, TextField, EnumField) +from hive.model.manager import UpdatingFields, CLIManager +from hive.model.legacycli import CLICmdError, RegexParser, BlankParser, MATCHALL, MATCHONE, CLICmdWarning +from hive.model.query import QuerySet, mark_expire_all, DelayedQuery +from hive.model.loading import get_model +from hive.model.action import Action __ = _ @@ -73,15 +81,15 @@ monitorable=True, mul='1', pos='left', optional=True) mac = MACAddressField(verbose_name=_('MAC'), editable=False, optional=True) mtu = Uint32Field(verbose_name=_('MTU'), editable=False) + list_config_options = { + 'columns': [ + {'name': 'interface_name'}, + {'name': 'ip'}, + {'name': 'mask_prefix'}, + ]} class Meta: verbose_name = _('System Interface') - list_config_options = { - 'columns': [ - {'name': 'interface_name'}, - {'name': 'ip'}, - {'name': 'mask_prefix'}, - ]} class Modify_IP(Action): verbose_name = _('Modify IP') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/network/route/__init__.py (working copy) @@ -1,6 +1,13 @@ +import re from django.utils.translation import gettext_lazy as _ -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import GroupField, IPv4AddressField, IPv6AddressField +from hive.model.action import Action +from hive.model.manager import CLIManager +from hive.model.legacycli import RegexParser, MATCHALL, CLICmdError, BlankParser +from hive.utils import is_ipv4, is_ipv6 +from hive.model.query import QuerySet __ = _ @@ -9,6 +16,7 @@ gateway_ip = GroupField(verbose_name=_('Gateway IP'), fields={'ipv4': IPv4AddressField(verbose_name=_('IPv4'), optional=True), 'ipv6': IPv6AddressField(verbose_name=_('IPv6'), optional=True)}, optional=True) + show_im_export_button = False class Clear(Action): verbose_name = _('Clear') @@ -22,7 +30,6 @@ class Meta: profile = True verbose_name = _('Default Route') - show_im_export_button = False class Manager(CLIManager): def _get_query_set(self): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/access.py (working copy) @@ -1,7 +1,15 @@ +import re + from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import BooleanField, PortField, CLITextField, UnionField, GroupField, ImportLocalFileField, URLField, TextField, EnumField, AssoField2, CharField +from hive.model.action import Action +from hive.model.fields import FieldGroup, EmptyValueCondition, NonemptyValueCondition, BASIC, ValueCondition +from hive.model.manager import QueryingGroups, UpdatingFields, CLIManager +from hive.model.legacycli import RegexParser, MATCHALL, CLICmdError, BlankParser, EasyParser, MATCHONE, CLICmdNormal __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/config_mgmt/__init__.py (working copy) @@ -1,7 +1,11 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.action import Action +from hive.model.manager import CLIManager +from hive.model.legacycli import CLICmdError, BlankParser +from hive.model.query import QuerySet class BackupConfig(ANModel): @@ -16,11 +20,11 @@ total_time = 15 full_progress = False forever = False + show_im_export_button = False class Meta: profile = True verbose_name = _('Configuration Backup') - show_im_export_button = False class Manager(CLIManager): def _get_query_set(self): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/host.py (working copy) @@ -1,8 +1,16 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ +import os import cm.conf -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import CharField +from hive.model.action import Action +from hive.model.legacycli import CLICmdError, BlankParser, CLICmdNormal +from hive.model.manager import CLIManager +from djproject.an_settings import CM_ConfigFile_PATH +from hive.exceptions import ModelQueryException +from hive.model.query import mark_expire_all __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/license.py (working copy) @@ -4,9 +4,15 @@ from django.utils.translation import gettext_lazy as _ from cm.lib.postgres_db import DB -from djproject.an_settings import * -from hive.imports.model import * from hive.session import check_license_expire +from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import IntegerField, PortField, CLITextField, UnionField, GroupField, ImportLocalFileField, URLField, TextField, EnumField, AssoField2, CharField +from hive.model.manager import QueryingGroups, UpdatingFields, CLIManager +from hive.model.legacycli import RegexParser, cli_parse, CLICmdError, BlankParser, MATCHONE +from hive.utils import get_current_session +from djproject.an_settings import ADC_TYPE_LIST, DEVICE_STD_LIST, VPN_TYPE_LIST, WAF_TYPE_LIST +from hive.exceptions import ModelQueryException __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/ntp.py (working copy) @@ -1,7 +1,16 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ +import re + from hive.imports.model import * +from hive.model.action import Action +from hive.model.base import ANModel +from hive.model.fields.builtin import (BooleanField, Uint64Field, StorableField, IntegerField, IPAddressField, + CLITextField, TextField, EnumField, AssoField2) +from hive.model.manager import QueryingGroups, UpdatingFields, CLIManager +from hive.model.legacycli import RegexParser, cli_parse, CLICmdError, BlankParser, MATCHONE, EasyParser +from hive.model.fields import FieldGroup, EmptyValueCondition, NonemptyValueCondition, BASIC, ValueCondition, STATS from hive.model.query import mark_expire_all Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/system_mgmt/__init__.py (working copy) @@ -1,9 +1,15 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ -from cm.router import _extension_log -from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import EnumField, UnionField, ImportLocalFileField, ImportURLField +from hive.model.action import Action +from hive.model.manager import CLIManager +from hive.model.legacycli import RegexParser, MATCHALL, CLICmdError, BlankParser +from hive.model.query import QuerySet +from hive.exceptions import ModelQueryException from hive.utils import andebug, update_frontend_index_html +from cm.router import _extension_log __ = _ @@ -31,11 +37,11 @@ 'url': ImportURLField(verbose_name=_('URL')) }), ) + show_im_export_button = False class Meta: profile = True verbose_name = _('System Update') - show_im_export_button = False class Manager(CLIManager): def _get_query_set(self): @@ -70,11 +76,11 @@ 'url': ImportURLField(verbose_name=_('URL')) }), ) + show_im_export_button = False class Meta: profile = True verbose_name = _('Extension Update') - show_im_export_button = False class Manager(CLIManager): def _get_query_set(self): @@ -122,11 +128,11 @@ alert_msg = "This WebUI session will no longer be active if system reboot is successful." process_title = "System Rebooting..." forever = True + show_im_export_button = False class Meta: profile = True verbose_name = _('System Shutdown & Reboot') - show_im_export_button = False class Manager(CLIManager): def _get_query_set(self): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/time.py (working copy) @@ -4,6 +4,11 @@ from django.utils.translation import gettext_lazy as _ from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.action import Action +from hive.model.fields.builtin import DateTimeField, TimezoneField +from hive.model.manager import QueryingGroups, UpdatingFields, CLIManager +from hive.model.legacycli import RegexParser, cli_parse, CLICmdError, BlankParser, MATCHONE, EasyParser class TimeSettings(ANModel): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/system/user_mgmt/__init__.py (working copy) @@ -1,4 +1,5 @@ import datetime +import json from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ @@ -7,6 +8,14 @@ from hive.imports.model import * from hive.model.query import clear_cache_all from hive.session import ANSession +from hive.model.base import ANModel +from hive.model.fields.builtin import FieldGroup, CharField, PasswordField, EnumField, IntegerField, BooleanField, JSONField +from hive.model.fields import FieldGroup +from hive.model.action import Action +from hive.model.manager import CLIManager +from hive.model.legacycli import RegexParser, MATCHALL, CLICmdError, BlankParser, EasyParser, MATCHONE +from hive.exceptions import ModelQueryException +from hive.model.loading import get_model __ = _ @@ -30,14 +39,14 @@ 'role_name': CharField(length='1..16', optional=True), 'root': BooleanField(verbose_name=_('Root'), default=False) }) + list_config_options = {'columns': [ + {'name': 'username'}, + {'name': 'level'}, + ]} + show_im_export_button = True class Meta: verbose_name = _('System Administrator') - list_config_options = {'columns': [ - {'name': 'username'}, - {'name': 'level'}, - ]} - show_im_export_button = True class Manager(CLIManager): def _get_query_set(self): @@ -153,6 +162,11 @@ 'role_name': CharField(length='1..16', optional=True), 'root': BooleanField(verbose_name=_('Root'), default=False) }) + show_im_export_button = True + list_config_options = {'columns': [ + {'name': 'username'}, + {'name': 'auth'}, + ]} class Meta: verbose_name = _('User Authorization') @@ -160,7 +174,6 @@ {'name': 'username'}, {'name': 'auth'}, ]} - show_im_export_button = True class GetRoleByUsername(Action): verbose_name = _('Get Role By Username') @@ -282,14 +295,14 @@ 'id': IntegerField(primary_key=True, verbose_name='Role Id'), 'role_name': CharField(verbose_name=_('Role Name'), length='1..16') }) + list_config_options = {'columns': [ + {'name': 'id'}, + {'name': 'role_name'}, + ]} + show_im_export_button = False class Meta: verbose_name = _('Role Management') - list_config_options = {'columns': [ - {'name': 'id'}, - {'name': 'role_name'}, - ]} - show_im_export_button = False class Manager(CLIManager): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/tasking/__init__.py (working copy) @@ -1,17 +1,21 @@ import datetime import shutil +import os +import json from apscheduler.schedulers.background import BackgroundScheduler from django.db.models.query import QuerySet from django.utils.translation import gettext_lazy as _ from cm.lib.postgres_db import DB -from cm.lib.task_scheduler import TaskScheduler, update_job_runtime, append_job_description, \ - Scheduler_func_list, update_job_state from hive.imports.model import * +from hive.model.base import ANModel +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ -GLOBAL_TASK = TaskScheduler() class Tasks(ANModel): @@ -44,20 +48,20 @@ result_field = TextField(verbose_name='Result Field', length='1..1024', optional=True) device_list = TextField(verbose_name='Device List', length='1..1024', optional=True) scheduler = TextField(verbose_name='Scheduler', optional=True) + show_im_export_button = False + list_config_options = { + 'columns': [ + {'name': 'name'}, + {'name': 'type'}, + {'name': 'state'}, + {'name': 'next_run_time'}, + {'name': 'description'}, + {'name': 'result_field'}, + {'name': 'device_list'} + ], 'add_button_hide': True, 'click_enable': False} class Meta: verbose_name = 'Tasking' - show_im_export_button = False - list_config_options = { - 'columns': [ - {'name': 'name'}, - {'name': 'type'}, - {'name': 'state'}, - {'name': 'next_run_time'}, - {'name': 'description'}, - {'name': 'result_field'}, - {'name': 'device_list'} - ], 'add_button_hide': True, 'click_enable': False} class Modify(Action): verbose_name = _('Modify Job') @@ -113,6 +117,8 @@ return QuerySet(self._model, res) def _delete_instance(self, instance): + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() data = instance.get_field_dict() db = DB.get_connected_db() @@ -133,17 +139,19 @@ delete_sql = "DELETE FROM task WHERE id=%d;" % data['id'] db.execute_sql(delete_sql) db.close() - if GLOBAL_TASK.get_schedule(data['name']): - GLOBAL_TASK.remove(data['name']) + if TASK.get_schedule(data['name']): + TASK.remove(data['name']) return def _perform_Clear(self): + from cm.lib.task_scheduler import TaskScheduler + TASK = TaskScheduler() select_sql = "SELECT id, name from task" db = DB.get_connected_db() result = db.fetchall(select_sql) for each in result: - if GLOBAL_TASK.get_schedule(each[1]): - GLOBAL_TASK.remove(each[1]) + if TASK.get_schedule(each[1]): + TASK.remove(each[1]) delete_sql = "DELETE from task" db.execute_sql(delete_sql) db.close() @@ -157,6 +165,8 @@ return def _perform_Modify(self, options): + from cm.lib.task_scheduler import TaskScheduler, update_job_runtime + TASK = TaskScheduler() pk_list = options['__pk_list'][0] if pk_list['state'] == 'waiting': time_str = options['expire_time'][:19] # '2014-03-28T06:59:52' @@ -167,7 +177,7 @@ datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))))) task_name = pk_list['name'] - sched = GLOBAL_TASK.get_schedule(task_name) + sched = TASK.get_schedule(task_name) if sched: # todo if not find the sched, restart it immediate jobs = sched.get_jobs()[-1] @@ -180,6 +190,10 @@ mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) def _perform_Restart(self, options): + from cm.lib.task_scheduler import TaskScheduler, update_job_runtime, append_job_description, \ + Scheduler_func_list, update_job_state + TASK = TaskScheduler() + def get_scheduler(id): db = DB.get_connected_db() fetchall_sql = '''SELECT scheduler FROM task where id='%d';''' % id @@ -222,7 +236,7 @@ append_job_description(task_name, description) update_job_state(task_name, 'waiting') - if GLOBAL_TASK.get_schedule(task_name): - GLOBAL_TASK.remove(task_name) - GLOBAL_TASK.add(task_name, sched) + if TASK.get_schedule(task_name): + TASK.remove(task_name) + TASK.add(task_name, sched) mark_expire_all(get_model('cm', ['tasking', 'Tasks'])) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/virtualization/__init__.py (working copy) @@ -4,6 +4,11 @@ from cm.lib.communication import call_restapi from cm.lib.postgres_db import DB from hive.imports.model import * +from hive.model.base import * +from hive.model.fields.builtin import * +from hive.model.action import Action +from hive.model.fields import * +from hive.model.manager import CLIManager __ = _ Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (working copy) @@ -14,8 +14,6 @@ from cm.lib.libbasic_operation import get_rest_info_from_device, send_command_to_device, oper_log, send_cli_to_device, \ get_ip_address from cm.lib.parse_configfile import TarFiles -from cm.lib.task_scheduler import remove_job_from_database -from cm.models.tasking import GLOBAL_TASK from djproject.an_settings import * from hive.log_utils import * from hive.model.legacycli import cli_parse, RegexParser, MATCHALL, MATCHONE @@ -1061,13 +1059,15 @@ def clear_schedule_backup_all(request): + from cm.lib.task_scheduler import remove_job_from_database, TaskScheduler + TASK = TaskScheduler() task_name_list = json.loads(request.POST.get('task_name_list', None)) if task_name_list: db = DB.get_connected_db() for task in task_name_list: - if GLOBAL_TASK.get_schedule(task): - GLOBAL_TASK.remove(task) + if TASK.get_schedule(task): + TASK.remove(task) remove_job_from_database(task) delete_sql = "DELETE FROM schedule_backup_all WHERE task_name = '%s'" % task db.execute_sql(delete_sql) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/settings.py (working copy) @@ -117,6 +117,9 @@ ) INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'cm.apps.CmConfig' # 'django.contrib.auth', # 'django.contrib.contenttypes', # 'django.contrib.sessions', Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/urls.py (working copy) @@ -16,6 +16,7 @@ from hive.report.generate_report import handle_report_generation from hive.controller.backup_controller import handle_backup_req from hive.controller.restore_controller import handle_restore_req +import cm.models.device_mgmt.device.Device js_info_dict = { #'packages': ('your.app.package',), Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/ajax.py (working copy) @@ -37,7 +37,7 @@ def render(self): if hasattr(self, self.action): - request_dict = dict(self.request.REQUEST) + request_dict = self.request.POST.copy() request_dict.pop('csrfmiddlewaretoken', None) return getattr(self, self.action)(**request_dict) elif self.action.startswith('box_'): @@ -160,6 +160,8 @@ def add(self, **kwargs): try: json_str = kwargs['post_data'] + if isinstance(json_str, list): + json_str = json_str[0] json_dict = json.loads(json_str) new_obj = self.model(**json_dict) except KeyError: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/base.py (working copy) @@ -15,6 +15,8 @@ MultipleObjectsReturned) from django.db.models.fields.related import ManyToOneRel from django.utils.encoding import smart_str, force_str +from django.db import models +from django.db.models.base import ModelBase from djproject.an_settings import PSEUDO_MANAGER_TEST_MODE from hive.exceptions import ValidationError \ No newline at end of file @@ -41,9 +43,9 @@ return super_new(cls, name, bases, attrs) # Create the class. - module = attrs.pop('__module__') - new_class = super_new(cls, name, bases, {'__module__': module}) - attr_meta = attrs.pop('Meta', None) + module = attrs['__module__'] + new_class = super_new(cls, name, bases, attrs) + attr_meta = attrs.get('Meta', None) abstract = getattr(attr_meta, 'abstract', False) meta_type = getattr(attr_meta, 'meta_type', None) if not meta_type: \ No newline at end of file @@ -64,8 +66,8 @@ new_class.add_to_class('_meta', meta_type(meta, **kwargs)) # Append the manager class - attr_manager = attrs.pop('Manager', None) - super_manager = attrs.pop('SuperManager', None) + attr_manager = attrs.get('Manager', None) + super_manager = attrs.get('SuperManager', None) # Pseudo manager if not attr_manager or PSEUDO_MANAGER_TEST_MODE: \ No newline at end of file @@ -111,7 +113,11 @@ # uninteresting parents. continue - parent_field_groups = base._meta.field_groups + if hasattr(base._meta, "field_groups"): + parent_field_groups = base._meta.field_groups + # ... use it + else: + continue # or handle gracefully # XXX We DO NOT support inheriting from non-abstract classes for now if base._meta.abstract: \ No newline at end of file @@ -210,11 +216,14 @@ signals.class_prepared.send(sender=cls) -class ANModel(object): +class ANModel(metaclass=ANModelBase): """ Parent class of all models """ - __metaclass__ = ANModelBase + + class Meta: + abstract = True + _deferred = False _manager_cache = {} \ No newline at end of file @@ -297,9 +306,7 @@ return smart_str(u'<%s: %s>' % (self.__class__.__name__, u)) def __str__(self): - if hasattr(self, '__unicode__'): - return force_str(self).encode('utf-8') - return '%s object' % self.__class__.__name__ + return self.__unicode__() def __eq__(self, other): return isinstance(other, self.__class__) and self._get_pk_val() == other._get_pk_val() \ No newline at end of file @@ -376,9 +383,9 @@ @classmethod def get_manager(cls, session): """ Get the manager object of the model. - - Called at model loading phase to instanciate manager classes of - every model and setup a cache for later visiting. There is one + + Called at model loading phase to instanciate manager classes of + every model and setup a cache for later visiting. There is one manager instance for every model bond to every session. :param session: ``ANSession`` \ No newline at end of file @@ -457,7 +464,7 @@ # is responsible for making sure they have a valid value. # if f.optional and raw_value in validators.EMPTY_VALUES: # continue - # + # if f.is_empty_value(raw_value): if f.optional: # !!! set the default value here \ No newline at end of file @@ -495,7 +502,7 @@ # 2. Check per-field conditions and make necessary marking for f in grp.fields: if self.get_attr_raw(f.attname) is DelayedQuery: - # if the field value is DelayedQuery, this means we will get the value for this + # if the field value is DelayedQuery, this means we will get the value for this # field later, so will also do condtion check for this field later. continue field_condition_pass = True \ No newline at end of file @@ -637,7 +644,7 @@ return cls._meta.repr_by_pk(pk) else: return ' - '.join([smart_str(cls._meta.get_field(pk_field).value_to_display(val=pk_value, style=False)) for - pk_field, pk_value in pk.iteritems() if pk_value]) + pk_field, pk_value in pk.items() if pk_value]) def pk_str(self): if self._meta.profile: \ No newline at end of file @@ -679,7 +686,7 @@ return asso_ref def url_params(self): - return '&'.join(['%s=%s' % (name, obj2url(value)) for name, value in self.pk_dict().iteritems()]) + return '&'.join(['%s=%s' % (name, obj2url(value)) for name, value in self.pk_dict().items()]) def get_statistics(self, start_time=None, end_time=None, field=None, cal_method=None): # Get statistics for this module or one field in this module. \ No newline at end of file @@ -689,14 +696,14 @@ instance_pk=self.pk_str()) else: ret = {} - for group_name, g in self._meta.field_groups.iteritems(): + for group_name, g in self._meta.field_groups.items(): if g.isstats: for f in g.fields: ret[f.attname] = getattr(self, f.attname, None) return ret def get_ajax_url(self, action, param_dict={}): - param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.iteritems()]) + param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.items()]) if param_str != '': param_str = '?' + param_str \ No newline at end of file @@ -712,7 +719,7 @@ return '/api/' + self._meta.app_label + '/' + '/'.join(self._meta.path) + '/_' + action + '/' + '/'.join( pk_item_list) + param_str - # + # def tree_node_str(self, status=0): colour = ['#00FA00', '#19E100', '#32C800', '#4B9600', '#647D00', '#7D6400', '#964B00', '#C83200', '#E11900', '#FA0000'] \ No newline at end of file @@ -737,7 +744,7 @@ def parse_module_path(module): - """ Parse a Python module like ``apv.models.loadbalancing.slb`` into (app_label, package_path) like + """ Parse a Python module like ``apv.models.loadbalancing.slb`` into (app_label, package_path) like ('apv', ['loadbalancing','slb']) Notice that the ``module`` must contain 'models' in its path \ No newline at end of file @@ -771,3 +778,6 @@ """ Parent class of all custom list """ + + class Meta: + abstract = True \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/__init__.py (working copy) @@ -154,6 +154,10 @@ messages.update(error_messages or {}) self.error_messages = messages + def __lt__(self, other): + # Compare creation order for correct field ordering + return self.creation_counter < other.creation_counter + def __cmp__(self, other): # This is needed because bisect does not take a comparison function. return cmp(self.creation_counter, other.creation_counter) @@ -817,6 +821,8 @@ FieldGroup.creation_counter += 1 self.visible_edit = visible_edit self._actions = [] + self.is_relation = False + self.many_to_many = False def __cmp__(self, other): return (self.creation_counter > other.creation_counter) - (self.creation_counter < other.creation_counter) @@ -860,11 +866,11 @@ if old_group: for field in old_group._fields: self._fields.insert(bisect(self._fields, field), field) - for name, field in self._field_dict.iteritems(): + for name, field in self._field_dict.items(): field.set_group(self) cls.add_to_class(name, field) self._fields.insert(bisect(self._fields, field), field) - cls._meta.add_field_group(self) + cls._meta.add_field_group(self) # changed from add_field_group to add_field # @cached_property @property @@ -1003,6 +1009,16 @@ if each_act.condition(instance): tmp_acts.append(each_act) return tmp_acts + + def __lt__(self, other): + if not isinstance(other, FieldGroup): + return NotImplemented + return self.creation_counter < other.creation_counter + + def __eq__(self, other): + if not isinstance(other, FieldGroup): + return NotImplemented + return self.creation_counter == other.creation_counter class DefaultValueCondition(object): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/fields/builtin.py (working copy) @@ -39,6 +39,8 @@ self.lexical = lexical self.display = display self.style = style + self.is_relation = False + self.many_to_many = False Field.__init__(self, verbose_name, name, **kwargs) def to_python(self, value): @@ -138,6 +140,9 @@ pattern=None, pattern_msg=None, examples=None, style=None, style_class={}, **kwargs): super(CharField, self).__init__(verbose_name, name, **kwargs) + self.is_relation = False + self.many_to_many = False + # setup length validator length_str_list = [length, self.__class__.type_arguments.get('length', None)] + \ [parent_class.type_arguments.get('length', None) for parent_class in @@ -188,7 +193,7 @@ rtn = value rtn = html.escape(rtn) if style and self.style_class: - for (regex, class_str) in self.style_class.iteritems(): + for (regex, class_str) in self.style_class.items(): if re.match(regex, value) is not None: rtn = text_add_style(rtn, class_str) break @@ -258,6 +263,8 @@ self.examples = examples self.style = style + self.is_relation = False + self.many_to_many = False def to_python(self, value): if value is None: @@ -338,6 +345,8 @@ if not values: raise ValueError('values argument is mandatory for EnumFields') self._values = values + self.is_relation = False + self.many_to_many = False @property def values(self): @@ -581,6 +590,9 @@ def __init__(self, verbose_name=None, name=None, scope=None, unit_name='', **kwargs): super(IntegerField, self).__init__(verbose_name, name, **kwargs) + + self.is_relation = False + self.many_to_many = False # setup scope validator scope_str_list = [scope, self.__class__.type_arguments.get('scope', None)] + \ @@ -769,7 +781,7 @@ # copy it into a new dict new_value = {} - for field_name, field in self.fields.iteritems(): + for field_name, field in self.fields.items(): # Only one option among the sub fields is enough if field_name in value: new_value[field_name] = field.to_python(value[field_name]) @@ -778,7 +790,7 @@ raise exceptions.ValidationError('UnionField does not have one of the sub field key') def validate(self, value, model_instance): - for field_name, field in self.fields.iteritems(): + for field_name, field in self.fields.items(): if field_name in value: field.validate(value[field_name], model_instance) return @@ -787,7 +799,7 @@ def init_sub_fields(self): # will be called by "contribute_to_class" self.field_list = [] - for sub_field_name, sub_field in self.fields.iteritems(): + for sub_field_name, sub_field in self.fields.items(): sub_field._set_attributes_from_name(sub_field_name) sub_field.nesting_field = self sub_field.init_sub_fields() @@ -798,7 +810,7 @@ def get_pseudo_value(self): result = {} - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): result[key] = field.get_pseudo_value() # only one option is enough break @@ -807,7 +819,7 @@ def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) rtn_str = '' - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): if key in value: rtn_str = field.value_to_display(val=value[key], style=style, text=text) break @@ -815,7 +827,7 @@ return rtn_str def value_split(self, value): - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): if key in value: rtn_str = field.value_to_display(val=value[key]) break @@ -874,19 +886,7 @@ def type_default(self, obj): result = {} - """ - !!! type_default should not query the value of itself which may cause dead loop !!! - if obj is not None: - value = getattr(obj, self.attname, None) - if value: - value_key = value.keys() - for key, field in self.fields.iteritems(): - if key == value_key[0]: - result[key] = field.get_default(None) - break - else: - """ - (first_key, first_field) = self.fields.iteritems().next() + (first_key, first_field) = next(iter(self.fields.items())) result[first_key] = first_field.get_default(None) return result @@ -976,11 +976,12 @@ def is_empty_value(self, value): if value: - if type(value) in (str, str): + if isinstance(value, str): return False - elif type(value) is dict: - data = value.values() - if data[0]: + elif isinstance(value, dict): + data = list(value.values()) + # If dict is empty, data will be [], so this will be True + if data and data[0]: return False else: return True @@ -1506,7 +1507,7 @@ def unit_name(self): # all fields should have the same unit_names try: - first_field = self.fields.itervalues().next() + first_field = self.fields.values().next() if hasattr(first_field, 'unit_name'): return first_field.unit_name else: @@ -1519,7 +1520,7 @@ def value_limit(self): # all fields should have the same value_limit try: - first_field = self.fields.itervalues().next() + first_field = self.fields.values().next() if hasattr(first_field, 'value_limit'): return first_field.value_limit else: @@ -1540,7 +1541,7 @@ # copy it into a new dict new_value = {} - for field_name, field in self.fields.iteritems(): + for field_name, field in self.fields.items(): if field.condition: if field_name in value: new_value[field_name] = field.to_python(value[field_name]) @@ -1551,7 +1552,7 @@ return new_value def validate(self, value, model_instance): - for field_name, field in self.fields.iteritems(): + for field_name, field in self.fields.items(): if field.condition: self.optional = True if field_name in value: @@ -1563,7 +1564,7 @@ def init_sub_fields(self): # will be called by "contribute_to_class" self.field_list = [] - for sub_field_name, sub_field in self.fields.iteritems(): + for sub_field_name, sub_field in self.fields.items(): sub_field._set_attributes_from_name(sub_field_name) sub_field.nesting_field = self sub_field.init_sub_fields() @@ -1574,14 +1575,14 @@ def get_pseudo_value(self): result = {} - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): result[key] = field.get_pseudo_value() return result def _value_to_display(self, obj=None, val=None, style=True, text=True): value = self._get_val_from_obj_or_val(obj, val) rtn_str = '' - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): rtn_str += key + ': ' + field.value_to_display(val=value[key]) + ' ' rtn_str = html.escape(rtn_str) return rtn_str @@ -1592,7 +1593,7 @@ def value_split(self, value): result = [] if value: - for key, field in self.fields.iteritems(): + for key, field in self.fields.items(): sub_val = field.value_split(value[key]) for each in sub_val: result.append([each[0], each[1]]) @@ -1631,7 +1632,7 @@ rtn_dict = {} i = 0 - for sub_field_name, sub_field in self.fields.iteritems(): + for sub_field_name, sub_field in self.fields.items(): rtn_dict[sub_field_name] = sub_field.get_iterable_value(nd_value[i]) i += 1 @@ -2216,7 +2217,7 @@ for ins in instance_list: value = get_complex_field(ins, field_name, manager) if value: - for field, sub_value in value.iteritems(): + for field, sub_value in value.items(): ret.append([sub_value, field.value_to_display(val=sub_value)]) if key_type == '__ins_field': @@ -2225,7 +2226,7 @@ instance = base_manager.get(pk_dict) value = get_complex_field(instance, field_name, base_manager) if value: - for field, sub_value in value.iteritems(): + for field, sub_value in value.items(): if type(sub_value) in (str, str): sub_value_list = sub_value.split(', ') for each in sub_value_list: @@ -2330,7 +2331,7 @@ for ins in instance_list: value = get_complex_field(ins, field_name, manager) if value: - for field, sub_value in value.iteritems(): + for field, sub_value in value.items(): ret.append([sub_value, field.value_to_display(val=sub_value)]) if key_type == '__ins_field': @@ -2339,7 +2340,7 @@ instance = base_manager.get(pk_dict) value = get_complex_field(instance, field_name, base_manager) if value: - for field, sub_value in value.iteritems(): + for field, sub_value in value.items(): if type(sub_value) in (str, str): sub_value_list = sub_value.split(', ') for each in sub_value_list: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/legacycli.py (working copy) @@ -36,7 +36,8 @@ session_id = self.session_id cmd_len = 1024 - len(session_id.encode('utf-8')) - len(s._username.encode('utf-8')) - 3 - cmd_str = cmd.encode('utf-8') + if isinstance(cmd, str): + cmd_str = cmd.encode('utf-8') while len(cmd_str) > 0: each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode( 'utf-8') + F_EOS @@ -52,7 +53,10 @@ self.pipe.left_write("") if not cmd[-1] == '\n': cmd = cmd + '\n' - cmd_str = cmd.encode('utf-8') + + if isinstance(cmd, str): + cmd_str = cmd.encode('utf-8') + each_cmd = cmd_str[:cmd_len] + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode( 'utf-8') + F_EOS _thread_locals._hive_socket.settimeout(self.timeout) @@ -70,7 +74,10 @@ if data.find(F_EOP) != -1: data = data.replace(F_EOP, '') cmd = "quit\n" - cmd_str = cmd.encode('utf-8') + + if isinstance(cmd, str): + cmd_str = cmd.encode('utf-8') + each_cmd = cmd_str + F_EOC + s._username.encode('utf-8') + F_EOU + session_id.encode('utf-8') + F_EOS s._send(each_cmd) s._disconnect() @@ -296,7 +303,6 @@ cmd = escapeshellarg(cmd) cmd = '/ca/bin/backend -u ' + username + ' -c ' + cmd - cmd = cmd.encode('utf-8') + f_eop (status, output) = subprocess.getstatusoutput(cmd) output = output.strip(f_eop) if len(args) == 0: @@ -335,7 +341,9 @@ # XXX check first \n less than cmd_len - cmd_str = cmd.encode('utf-8') + if isinstance(cmd, str): + cmd_str = cmd.encode('utf-8') + while len(cmd_str) > 0: each_cmd = ( cmd_str[:cmd_len] + @@ -413,7 +421,8 @@ self._enable_pass = enable_pass def _user2enable(self): - result = self.cmd('en\n' + self._enable_pass + '\n') + b_data_output = self.cmd('en\n' + self._enable_pass + '\n') + result = b_data_output.decode('utf-8') if not result.strip() == 'Enable password:': raise CLIEnablePassError @@ -684,7 +693,7 @@ def __init__(self, regex, mode=MATCHONE, result=AUTO, match_exception=None, unmatch_exception=None, reflags=0, match_msg=None, unmatch_msg=None, **kwargs): super(RegexParser, self).__init__(**kwargs) - self._regex = re.compile(regex, reflags) if type(regex) in (str, unicode) else regex + self._regex = re.compile(regex, reflags) if type(regex) in (str, str) else regex self._match_exception = match_exception self._unmatch_exception = unmatch_exception self._match_msg = match_msg Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/loading.py (working copy) @@ -12,6 +12,7 @@ from djproject import an_settings from hive.model.package import ANPackage from hive.utils import anerror, andebug +from hive.exceptions import * __all__ = ('get_apps', 'get_app', 'get_package', 'get_model', 'register_models', 'get_app_errors', 'app_cache_ready') @@ -43,6 +44,7 @@ def __init__(self): self.__dict__ = self.__shared_state + self.unresolved_associations = [] def _populate(self): """ @@ -50,6 +52,7 @@ sense that every caller will see the same state upon return, and if the cache is already initialised, it does no work. """ + andebug('an.model.cli', 'Inside _populate') if self.loaded: return self.write_lock.acquire() @@ -65,6 +68,7 @@ self._load_app(app_name) self.loaded = True finally: + self._resolve_all_unresolved_associations(self) self.write_lock.release() def _label_for(self, app_name): @@ -101,7 +105,7 @@ # register the models to the package package_key = self._label_for(app_name) + '.'.join(new_package._meta.path) if package_key in self.app_models: - for each_name, each_model in self.app_models[package_key].iteritems(): + for each_name, each_model in self.app_models[package_key].items(): new_package.append_model(each_name, each_model) return new_package @@ -164,6 +168,7 @@ Returns the module containing the models for the given app_name. Returns None if it's not found. """ + andebug('an.model.cli', 'Inside get_app') self._populate() try: return self.app_packages[app_name] @@ -240,16 +245,12 @@ return model._meta.get_field(rest_path[0]) def register_models(self, app_label, package_path, *models): - """ - Register a set of models as belonging to an package. - """ + print('Inside register_models') package_key = app_label + '.'.join(package_path) for model in models: # Store as 'name: model' pair in a dictionary - # in the app_models dictionary model_name = model._meta.object_name model_dict = self.app_models.setdefault(package_key, OrderedDict()) - model_dict = OrderedDict(sorted(model_dict.items())) if model_name in model_dict: continue model_dict[model_name] = model @@ -261,16 +262,46 @@ continue asso_idx = 0 for each_tgt in each_field.tgt: - # find the field from existing registered models (not yet finished loading) tgt_field = self.get_field_from_full_path(app_label, each_tgt.split('.')) - if tgt_field: - each_field.register_target_asso_while_loading(tgt_field, asso_idx) + try: + if tgt_field: + each_field.register_target_asso_while_loading(tgt_field, asso_idx) + else: + raise ModelDefinitionError('Could not find target matching association' + ', %s, %s' % (each_field.path, each_tgt) + ) + except ModelDefinitionError: + self.unresolved_associations.append((each_field, app_label, each_tgt, asso_idx)) asso_idx += 1 + print(f'app models:{self.app_models}') + def _resolve_all_unresolved_associations(self): + still_unresolved = [] + for each_field, app_label, each_tgt, asso_idx in self.unresolved_associations: + tgt_field = self.get_field_from_full_path(app_label, each_tgt.split('.')) + if tgt_field: + try: + each_field.register_target_asso_while_loading(tgt_field, asso_idx) + except Exception as e: + # Log, or optionally collect for a later hard error + print(f"Error registering association {each_field} -> {each_tgt}: {e}") + still_unresolved.append((each_field, app_label, each_tgt, asso_idx)) + else: + still_unresolved.append((each_field, app_label, each_tgt, asso_idx)) + if still_unresolved: + # Either raise, or just print/log: + for each_field, app_label, each_tgt, asso_idx in still_unresolved: + print(f"Unresolved association: {each_field} -> {each_tgt}") + # You can optionally raise an error if you want to be strict: + # raise ModelDefinitionError("Some associations could not be resolved!") + self.unresolved_associations = still_unresolved cache = AppCache() + + + # These methods were always module level, so are kept that way for backwards # compatibility. def get_apps(): Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/manager.py (working copy) @@ -2,6 +2,7 @@ import inspect import itertools as it import shelve +import json from django.db.models.query import QuerySet from django.utils.encoding import smart_str @@ -114,7 +115,7 @@ manager_cls._custom_updates_reverse = {} # register the _custom_gets - for method_name, method in manager_cls.__dict__.iteritems(): + for method_name, method in manager_cls.__dict__.items(): if hasattr(method, "_custom_group_gets"): field_list = [] for each_group_name in method._custom_group_gets: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/model/options.py (working copy) @@ -263,13 +263,13 @@ # self.duplicate_targets will map each duplicate field column to the # columns it duplicates. collections = {} - for column, target in self.duplicate_targets.iteritems(): + for column, target in self.duplicate_targets.items(): try: collections[target].add(column) except KeyError: collections[target] = set([column]) self.duplicate_targets = {} - for elt in collections.itervalues(): + for elt in collections.values(): if len(elt) == 1: continue for column in elt: @@ -282,9 +282,13 @@ 'field group added into the sorted dict: %s %d bisec %d' % (group.name, group.creation_counter, bisect(self.field_groups.values(), group))) - self.field_groups.insert(bisect(self.field_groups.values(), group), group.name, group) + items = list(self.field_groups.items()) + insert_at = bisect(list(self.field_groups.values()), group) + items.insert(insert_at, (group.name, group)) + self.field_groups = OrderedDict(items) + if self.object_name == 'HTTPVirtualService': - for each_name, each_value in self.field_groups.iteritems(): + for each_name, each_value in self.field_groups.items(): andebug('hive.debug', 'field group now includes: %s %d' % (each_name, each_value.creation_counter)) def get_field_group(self, group_name): @@ -303,7 +307,7 @@ if 'default' not in self.field_groups: default_grp = fields.FieldGroup(name='default', verbose_name=_('Basic Settings'), editable=(True if self.profile else False)) - self.field_groups.insert(0, 'default', default_grp) + self.field_groups = OrderedDict([('default', default_grp)] + list(self.field_groups.items())) # the creation counter of 'default' field group is set to 0 # the default group won't call the contribute_to_class method, so we should set the class reference manually here default_grp.model = self._model @@ -772,7 +776,7 @@ def get_ajax_url(self, action, param_dict=None): if param_dict: - param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.iteritems()]) + param_str = '&'.join([str(key) + '=' + str(value) for key, value in param_dict.items()]) if param_str != '': param_str = '?' + param_str else: @@ -786,7 +790,7 @@ for field in action_cls.option_fields: field.register_related_action(action_cls) field.init_sub_fields() - self._actions.insert(bisect(self._actions, action_cls), action_cls) + self._actions.append(action_cls) @property def actions(self): @@ -892,7 +896,7 @@ left_fields.remove(each_field) # unique sets - for name, unique_set in self.unique_sets.iteritems(): + for name, unique_set in self.unique_sets.items(): if name is True: # this is a primary key set continue @@ -1018,7 +1022,7 @@ else: pk_dict = None if pk_dict: - for pk_field_name, instance_id in pk_dict.iteritems(): + for pk_field_name, instance_id in pk_dict.items(): pk_field = self.get_field(pk_field_name) pk_dict[pk_field_name] = pk_field.to_python(instance_id) return pk_dict @@ -1111,7 +1115,7 @@ else: # default: get all config fields name in base class - for gn, g in self.field_groups.iteritems(): + for gn, g in self.field_groups.items(): if g.isbasic: for f in g.fields: if f.hidden or f.hide_list: @@ -1146,7 +1150,7 @@ else: # default: get all config fields name in stats class - for gn, g in self.field_groups.iteritems(): + for gn, g in self.field_groups.items(): if g.isstats: for f in g.fields: if f.hidden: @@ -1309,7 +1313,7 @@ else: # default: get all config fields name in base class - for gn, g in self.field_groups.iteritems(): + for gn, g in self.field_groups.items(): if g.isbasic: for f in g.fields: if f.hidden or f.hide_list: @@ -1351,7 +1355,7 @@ else: # default: get all config fields name in stats class - for gn, g in self.field_groups.iteritems(): + for gn, g in self.field_groups.items(): if g.isstats: for f in g.fields: if f.hidden: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/router.py (working copy) @@ -1,13 +1,16 @@ # -*- coding: UTF-8 -*- +import json +import os import collections import csv import fcntl import logging import re import subprocess +from datetime import time from urllib import parse as urlparse -from django.http import HttpResponseRedirect +from django.http import HttpResponseRedirect, HttpResponse, Http404 from django.shortcuts import redirect from django.views.decorators.csrf import csrf_exempt @@ -32,10 +35,12 @@ from hive.rest_utils import get_rest_url_rule_num, get_rest_process_func from hive.search import * from hive.session import temp_session -from hive.utils import _thread_locals +from hive.utils import _thread_locals, get_current_session, url2obj from hive.utils import upload_receive, UploadResponse, dict_combine, \ get_device_type, update_frontend_index_html from lib.response import FileResponse + +from hive.exceptions import ValidationError, ModelQueryException, ManagerImplError try: # ToDo: fix this later - as localfile is an empty file. Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/search.py (working copy) @@ -9,6 +9,7 @@ import djproject.an_settings as an_settings from hive.model.loading import get_app, get_model from hive.utils import HiveEnvironment, Singleton, get_current_session +from hive.utils import andebug have_no_default_page = {'System Status': '/apv/index/dashboard'} Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/session.py (working copy) @@ -5,11 +5,12 @@ from datetime import datetime from datetime import timedelta -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect from django.template import RequestContext from django.views.decorators.csrf import csrf_exempt from jinja2 import PackageLoader +from django.middleware.csrf import get_token from cm.lib.libbasic_operation import oper_log from djproject import an_settings @@ -251,23 +252,21 @@ @csrf_exempt def app_login_handler(request): - django_ctx = RequestContext(request) if request.method == 'GET': session = None if 'hive_sess' in request.COOKIES: session = ANSession.find_session_by_id(request.COOKIES['hive_sess']) if session: - return HttpResponse(json.dumps({'error_code': 0, 'msg': 'You have logged in.'}), - content_type='application/json') + return JsonResponse({'error_code': 0, 'msg': 'You have logged in.'}) else: - return HttpResponse(json.dumps({'error_code': 1, 'msg': 'You need to log in.'}), - content_type='application/json') + return JsonResponse({'error_code': 1, 'msg': 'You need to log in.'}) elif request.method == 'POST': session = None username = '' if 'hive_sess' in request.COOKIES: session = ANSession.find_session_by_id(request.COOKIES['hive_sess']) + if 'username' in request.POST: username = request.POST['username'] password = request.POST['password'] @@ -276,78 +275,72 @@ username = session._username password = session._password else: - oper_log('error', 'login', 'User <%s> failed to login.' % username) - return HttpResponse(json.dumps({'error_code': 1, 'msg': 'Invalid request.'}), - content_type='application/json') + oper_log('error', 'login', f'User <{username}> failed to login.') + return JsonResponse({'error_code': 1, 'msg': 'Invalid request.'}) ret = getattr(auth, 'auth_' + current_app())(username, password) if ret: if not session: - # create session session = ANSession(username, password, ret) - # XXX APV enable password - if 'enable_password' in request.POST: - result = session.login(request.POST['enable_password']) - else: - result = session.login('') + result = session.login(request.POST.get('enable_password', '')) if result: session.enable_password_checked = True + csrf_token = get_token(request) + user_info = { "django_language": session.pref.get_default_lang(), - "csrf_token": str(django_ctx['csrf_token']), + "csrf_token": csrf_token, "hive_key": str(ret), "current_user": username } - check_license = check_license_expire(session) - if check_license[0]: - if session: - session.logout() + # TODO: Need to revist the license part later. Created AA-2982 to take care of licensing part + '''check_license_result = check_license_expire(session) + if check_license_result[0]: + session.logout() _thread_locals.session = None - response = HttpResponse(json.dumps({'error_code': 3, 'msg': 'License has been expired.'}), - content_type='application/json') - oper_log('error', 'login', - 'User <%s> failed to login because the license has been expired.' % username) - return response + oper_log('error', 'login', f'User <{username}> failed to login due to license expiry.') + return JsonResponse({'error_code': 3, 'msg': 'License has been expired.'})''' + send_notification('success', 'User logged in.', session=session) - response = HttpResponse(json.dumps( - {'error_code': 0, 'hive_key': ret, 'user_info': user_info, 'msg': 'Logging in successfully.', - 'check_license': check_license}), content_type='application/json') + response = JsonResponse({ + 'error_code': 0, + 'hive_key': ret, + 'user_info': user_info, + 'msg': 'Logging in successfully.', + 'check_license': False + }) response.set_cookie('django_language', session.pref.get_default_lang(), secure=True) response.set_cookie('hive_sess', session.sessid, secure=True) - # response.set_cookie('ComposerUICookie', 'f1f713c9e000f5d3f280adbd124df4f5') - response.set_cookie('csrf_token', str(django_ctx['csrf_token']), secure=True) + response.set_cookie('csrf_token', csrf_token, secure=True) response.set_cookie('hive_key', ret, secure=True) response.set_cookie('current_user', username, secure=True) - oper_log('info', 'login', 'User <%s> login successfully.' % username) + + oper_log('info', 'login', f'User <{username}> login successfully.') return response + else: - if 'enable_password' in request.POST: - send_notification('error', '%s is trying to login with an invalid enable password.' % username, - session=session) - error_msg = 'Invalid enable password' - else: - error_msg = '' - response = HttpResponse(json.dumps({'error_code': -1, 'msg': error_msg}), - content_type='application/json') + error_msg = 'Invalid enable password' if 'enable_password' in request.POST else '' + send_notification('error', f'{username} login failed: {error_msg}', session=session) + oper_log('error', 'login', f'User <{username}> failed to login with {error_msg}.') + response = JsonResponse({'error_code': -1, 'msg': error_msg}) response.set_cookie('django_language', session.pref.get_default_lang(), secure=True) response.set_cookie('hive_sess', session.sessid, secure=True) - oper_log('error', 'login', 'User <%s> failed to login with an invalid enable password.' % username) return response + else: if session: session.logout() - # clear session in thread here, just make it send notification correctly. _thread_locals.session = None - send_notification('error', '%s is trying to login with an invalid password.' % username) - oper_log('error', 'login', 'User <%s> failed to login with an invalid password.' % username) - return HttpResponse(json.dumps({'error_code': 2, 'msg': 'Invalid username or password.'}), - content_type='application/json') - return None + send_notification('error', f'{username} is trying to login with an invalid password.') + oper_log('error', 'login', f'User <{username}> failed to login with an invalid password.') + return JsonResponse({'error_code': 2, 'msg': 'Invalid username or password.'}) + return JsonResponse({'error_code': 99, 'msg': 'Invalid method.'}) + def app_logout_handler(request): response = HttpResponse(json.dumps({'error_code': 0, 'msg': 'Loging out successfully.'}), content_type='application/json') @@ -444,28 +437,40 @@ def _process_request(self, request): if getattr(_thread_locals, '_hive_thread_id', -1) == -1: + andebug('an.model.cli', 'process_request: {}'.format(request)) + andebug('an.model.cli', 'Inside getattr if process_request method') + andebug('an.model.cli', + 'process_request function call trace:\n{}'.format(''.join(traceback.format_stack()))) _thread_locals._hive_thread_id = HiveSessionMiddleware.global_thread_id + andebug('an.model.cli', 'Thread Id: {}'.format(_thread_locals._hive_thread_id)) HiveSessionMiddleware.global_thread_id += 1 + _thread_locals.request = request + # validate the session - if request.path_info in an_settings.PUBLIC_PATHS: + path = request.path_info + full_path = request.get_full_path() + + if path in an_settings.PUBLIC_PATHS: return None # got public path, continue the process to the router - if request.path_info.startswith("/restapi_on/"): - return None # as we send port after RESTful_API_ON, so could not match in PUBLIC_PATHS - if request.path_info.startswith("/ReloadAppNode/"): - return None # as we send app name after /ReloadAppNode/, so could not match in PUBLIC_PATHS - if request.path_info.startswith("/rest"): + + if path.startswith("/restapi_on/"): return None - if request.META['REQUEST_URI'].startswith('/rest'): - return None # restful api continue the process to the hive_rest_router - if request.META['REQUEST_URI'].startswith('/statistics'): - return None # restful api continue the process to the hive_rest_router - if request.META['REQUEST_URI'].startswith('/monitord'): - return None # restful api continue the process to the hive_rest_router - if request.META['REQUEST_URI'].startswith('/cm/device_change_result'): - return None # restful api continue the process to the hive_rest_router + if path.startswith("/ReloadAppNode/"): + return None + if path.startswith("/rest"): + return None + if full_path.startswith('/rest'): + return None + if full_path.startswith('/statistics'): + return None + if full_path.startswith('/monitord'): + return None + if full_path.startswith('/cm/device_change_result'): + return None if 'AUTH_TYPE' in request.META: - return None # restful api continue the process to the hive_rest_router + return None + # get session id from cookie sess = None if 'hive_sess' in request.COOKIES: @@ -476,7 +481,6 @@ _thread_locals.session = None sess = None else: - # here we should add the timeout mechanism by checking && updating timestamp in session timestamp = sess.timestamp difference = time.time() - timestamp if difference > 15 * 60 and timestamp != -1: @@ -484,36 +488,33 @@ _thread_locals.session = None sess = None else: - # for time change request, rest the timestamp - if request.path_info in ['/api/avx/system/TimeSettings/_update']: + if path in ['/api/avx/system/TimeSettings/_update']: sess.timestamp = -1 - # for request in white list, won't update the timestamp in session - elif not request.path_info in ['/poll_notification', '/cm/get_user_auth', '/cm/get_user_info']: + elif path not in ['/poll_notification', '/cm/get_user_auth', '/cm/get_user_info']: sess.timestamp = time.time() + if sess is None: # codes for new frontend - if request.path_info == '/': + if path == '/': response = redirect('/app/') else: - response = HttpResponse() - response.status_code = 401 + response = HttpResponse(status=401) return response - # codes for old frontend - # response = redirect('/login') - # reset the language cookie into system default + # reset language cookie to system default response.set_cookie('django_language', an_settings.SYS_DEFAULT_LANG) - if request.path_info and request.path_info.startswith( - '/' + current_app()) and 'ajax' not in request.path_info: - original_url = request.path_info - if len(request.GET): - original_url += ('?' + request.GET.urlencode()) + + if path.startswith('/' + current_app()) and 'ajax' not in path: + original_url = path + if request.GET: + original_url += '?' + request.GET.urlencode() response.set_cookie('hive_redirect', original_url, max_age=100) + return response else: try: - sess.client_ip = request.META['REMOTE_ADDR'] - except: + sess.client_ip = request.META.get('REMOTE_ADDR', '') + except Exception: pass _thread_locals.session = sess - return None # found session continue the process to the router + return None # session is valid, continue processing Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/utils.py (working copy) @@ -506,9 +506,9 @@ def convert_keys_to_string(data): if isinstance(data, str): return str(data) - elif isinstance(data, collections.Mapping): - return dict(map(convert_keys_to_string, data.iteritems())) - elif isinstance(data, collections.Iterable): + elif isinstance(data, collections.abc.Mapping): # FIXED HERE + return dict(map(convert_keys_to_string, data.items())) # FIXED .iteritems() to .items() + elif isinstance(data, collections.abc.Iterable): # FIXED HERE return type(data)(map(convert_keys_to_string, data)) else: return data Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/manage.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/manage.py (revision 2665) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/manage.py (working copy) @@ -13,7 +13,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djproject.settings") if 'compilejsi18n' not in sys.argv: print('Initializing whoosh indexes...') - hive_search_init(djproject.an_settings.AN_APPS[0]) + #hive_search_init(djproject.an_settings.AN_APPS[0]) if 'webui_on' in sys.argv: djproject.an_settings.WEBUI_ON_OFF = True if 'restapi_on' in sys.argv: