Index: /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py =================================================================== --- /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py (revision 9005) +++ /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py (working copy) @@ -9,10 +9,13 @@ from datetime import datetime from avx.company import * import cgi -try: - import xml.etree.cElementTree as ET -except ImportError: +try: + import xml.etree.cElementTree as ET +except ImportError: import xml.etree.ElementTree as ET +import uuid +from handler import VAInstanceHandle, SessionCounter + __ = _ child_seq = ['VAInstance'] @@ -140,192 +143,18 @@ class Manager(CLIManager): def _get_query_set(self): - self.cli.set_enable() - result = self.cli.cmd('show va instance', - EasyParser('va instance', ['?va_name', '?va_size', '?starting_port', '?domain_id', '?tgt_name']), - EasyParser('va custinstance', ['?va_name', '?cpu_thread', '?memory', '?domain_id', '?tgt_name'])) - cpu_thread_result = self.cli.cmd('show va resource', RegexParser('\s+(?P[^\n]+?)\s+(?P[0-9]+(\sshared)?)\s+(?P[0-9]+)G\s+[0-9]+\s+dedicated VF\s+', MATCHALL, reflags=re.S)) - - va_status_result = self.cli.cmd('show va status', RegexParser('(?P\S+)\s+(?Prunning|shutoff)\n', MATCHALL, reflags=re.S)) - - vnc_result = self.cli.cmd('show vncport', RegexParser('(?P\S+):\s*(?P[0-9]+)\n', MATCHALL, reflags=re.S)) - - mgmtip_result = self.cli.cmd('show va mgmtip', RegexParser('va mgmtip (?P\S+) (?P\S+)\n', MATCHALL, reflags=re.S)) - - vlan_result = self.cli.cmd('show va mgmtvlan' , RegexParser('va mgmtvlan (?P\S+) (?P\S+)\n', MATCHALL, reflags=re.S)) - - nic_type_result = self.cli.cmd('show va mgmttype', RegexParser('(?P\S+)\s+(?P\S+)\n', MATCHALL, reflags=re.S)) - - va_exdisk_result = self.cli.cmd('show va disk', - RegexParser('(?P\S+):\n(?P([\S]+\s*?[\S]*?\s*?[\S]*?\s*?[\S]*?\n)+)\n', MATCHALL, reflags=re.S)) - - res = [] - for each in va_exdisk_result: - disks = each["disk_name"].split("\n") - del disks[0] - for item in disks: - if item and item.split()[0] != "reserved": - each_disk = { - "disk_name": item.split()[0], - "va_name": each["va_name"], - "disk_size": item.split()[-2].split("(")[0], - "bus_type": item.split()[-1] - } - res.append(each_disk) - - if COMPANY_KEYWORD == "Array": - apv_type_result = self.cli.cmd('show va license apv', EasyParser('', ['?va_name', '?type'])) - - ag_type_result = self.cli.cmd('show va license ag', EasyParser('', ['?va_name', '?type'])) - - netgate_type_result = self.cli.cmd('show va license netgate', EasyParser('', ['?va_name', '?type'])) - - netiag_type_result = self.cli.cmd('show va license netiag', EasyParser('', ['?va_name', '?type'])) - - - if COMPANY_KEYWORD == "Infosec": - nsae_type_result = self.cli.cmd('show va license nsae', EasyParser('', ['?va_name', '?type'])) - - netgate_type_result = self.cli.cmd('show va license netgate', EasyParser('', ['?va_name', '?type'])) - - netopti_type_result = self.cli.cmd('show va license netopti', EasyParser('', ['?va_name', '?type'])) - - apv_type_result = self.cli.cmd('show va license apv', EasyParser('', ['?va_name', '?type'])) - - ag_type_result = self.cli.cmd('show va license ag', EasyParser('', ['?va_name', '?type'])) - - netiag_type_result = self.cli.cmd('show va license netiag', EasyParser('', ['?va_name', '?type'])) - - rst = result[0] + result[1] - for each in rst: - # for normal va instance - if "starting_port" in each and each["starting_port"] == 'null': - del each["starting_port"] - # for custom va instance - if "va_size" not in each: - each["va_size"] = "custom" - each["cpu_thread_shared"] = False - - # init the field - name = each["va_name"] - each["ip"] = "" - each["va_vncport"] = "" - each["vlan_tag"] = "" - each["nic_type"] = "e1000" - each["type"] = "" - each["attached_disk"] = [] - - # memory & cpu_thread & cpu_thread_shared - for item in cpu_thread_result: - if item["va_name"] == name: - each["memory"] = item["memory"] - if "shared" in item["cpu_thread"]: - cpu_thread = int(item["cpu_thread"].split("shared")[0]) - cpu_thread_shared = True - else: - cpu_thread = int(item["cpu_thread"]) - cpu_thread_shared = False - break - each["cpu_thread"] = cpu_thread - each["cpu_thread_shared"] = cpu_thread_shared - # vnc - for item in vnc_result: - if item["va_name"] == name: - each["va_vncport"] = item["vnc_port"] - break - # mgmt_ip - for item in mgmtip_result: - if item["va_name"] == name: - each["ip"] = item["ip"] - break - # mgmtvlan - for item in vlan_result: - if item["va_name"] == name: - each["vlan_tag"] = item["tag"] - break - # mgmttype - for item in nic_type_result: - if item["va_name"] == name: - each["nic_type"] = item["nic_type"] - break - # mgmtexdisk - - for item in res: - if item["va_name"] == name: - each_disk = { - "disk_name": item["disk_name"], - "va_name": item["va_name"], - "disk_size": item["disk_size"], - "bus_type": item["bus_type"] - } - each["attached_disk"].append(each_disk) - - if COMPANY_KEYWORD == "Array": - # ag/apv license type - for item in apv_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in ag_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in netgate_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in netiag_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - if COMPANY_KEYWORD == "Infosec": - # nsae/netopti/netgate license type - for item in nsae_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in netopti_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in netgate_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in apv_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in ag_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - for item in netiag_type_result: - if "va_name" in item and item["va_name"] == name: - each["type"] = item["type"] - break - self._model._meta.mark_delay_query(each) - - # query va image info for each va instance - image = each["tgt_name"] - sess = get_current_session() - image_model = get_model('avx', ['va', 'image', 'VAImageManagement']) - image_manager = image_model.get_manager(sess) - image_instance = image_manager.all().filter({"image_name": image}) - product_category = image_instance[0].product_category if image_instance else '' - if product_category == "Other": - product_category = image_instance[0].other_category - product_name = image_instance[0].product_name if image_instance else '' - vendor = image_instance[0].vendor if image_instance else '' - each["product_category"] = product_category - each["product_name"] = product_name - each["vendor"] = vendor - each["status"] = False - for item in va_status_result: - if item["va_name"] == each["va_name"]: - each["status"] = True if item["status"] == "running" else False - break - + vaHandler = VAInstanceHandle(self.cli) + current_time = int(time.time()) + counter = SessionCounter(vaHandler.get_id()) + counter_data = counter.get_counter_data() + if counter_data['timestamp'] <= current_time: # check newest request + request_id = str(uuid.uuid4()) + counter.cleanup_old_results(request_id) + counter.set_counter_data(counter_data['counter'] + 1, request_id, current_time) + vaHandler.get_data_from_cli(request_id, counter) + else: + counter.wait_for_result(counter_data['uuid']) + rst = counter.read_result_file(counter.get_counter_data()['uuid']) return QuerySet(self._model, rst) def _get_boottime(self, pk_dict): Index: /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py =================================================================== --- /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py (revision 0) +++ /branches/rel_avx_2_7_3/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py (working copy) @@ -0,0 +1,370 @@ +""" +This file is for counter of API: +Get list of VA Instance. +It will use file to save counter setting, +and check which request is latest by uuid and timestamp. +keep latest data is returned. +""" +from hive.imports.model import * +from hive.model.loading import get_model +from hive.utils import get_current_session, aninfo +from hive.model.legacycli import RegexParser, MATCHALL +from djproject.an_settings import VA_BOOT_TIME_INFO_PATH +from datetime import datetime +from avx.company import * +import json +import time +import os +import fcntl +import shutil + +TEMP_DIR = '/tmp' +VA_INSTANCE_COUNTER_DIR = '/va_instance_counter' +VA_INSTANCE_COUNTER_FILE = 'config.json' +EXPIRY_SECONDS = 600 # 10 min + +class SessionCounter(): + def __init__(self, session_id): + self.session_id = session_id + self.counter_dir = TEMP_DIR + VA_INSTANCE_COUNTER_DIR + self.session_dir = self.counter_dir + "/%s/" % self.session_id + if not os.path.exists(self.counter_dir): + os.mkdir(self.counter_dir) + if not os.path.exists(self.session_dir): + os.mkdir(self.session_dir) + self.counter_file = self.session_dir + VA_INSTANCE_COUNTER_FILE + + def get_result_file(self, uuid): + return (self.session_dir + 'result_%s.json' % uuid) + + def get_counter_data(self): + # structure: { "counter": 0, "uuid": "xxxx", "timestamp": 123456 } + if not os.path.exists(self.counter_file): + return {"counter": 0, "uuid": "", "timestamp": 0} + with open(self.counter_file, 'r') as f: + fcntl.flock(f, fcntl.LOCK_SH) + data = json.load(f) + fcntl.flock(f, fcntl.LOCK_UN) + return data + + def set_counter_data(self, counter, uuid, timestamp): + with open(self.counter_file, 'w') as f: + fcntl.flock(f, fcntl.LOCK_EX) + json.dump({"counter": counter, "uuid": uuid, "timestamp": timestamp}, f) + fcntl.flock(f, fcntl.LOCK_UN) + + def write_result_file(self, uuid, result): + final_path = self.get_result_file(uuid) + with tempfile.NamedTemporaryFile(mode='w', dir=self.session_dir, delete=False) as tmp_file: + json.dump(result, tmp_file) + tmp_file_path = tmp_file.name + os.rename(tmp_file_path, final_path) + + def read_result_file(self, uuid): + path = self.get_result_file(uuid) + if not os.path.exists(path): + return [] + with open(path, 'r') as f: + return json.load(f) + + def wait_for_result(self, uuid, timeout=10): + path = self.get_result_file(uuid) + waited = 0 + while not os.path.exists(path) and waited < timeout: + time.sleep(0.5) + waited += 0.5 + + def cleanup_old_results(self, current_uuid): + now = time.time() + for fname in os.listdir(self.counter_dir): + dir_path = os.path.join(self.counter_dir, fname) + if os.path.isdir(dir_path) and fname != self.session_id: + dir_mtime = os.path.getmtime(dir_path) + if now - dir_mtime > EXPIRY_SECONDS: + try: + shutil.rmtree(dir_path) + except: + aninfo('hive.debug', 'There is error occurred when cleanup old results directory for VA Instance list') + continue + if not fname.startswith('result_') or not fname.endswith('.json'): + continue + uuid = fname[7:-5] + if uuid == current_uuid: + continue + full_path = os.path.join(self.session_dir, fname) + try: + mtime = os.path.getmtime(full_path) + if now - mtime > EXPIRY_SECONDS: + os.remove(full_path) + except: + aninfo('hive.debug', 'There is error occurred when cleanup old results file for VA Instance list') + +class VAInstanceHandle(): + def __init__(self, cli): + self.sess = get_current_session() + self.cli = cli + + def get_id(self): + return self.sess._sessid + + def get_instance(self): + self.cli.set_enable() + return self.cli.cmd('show va instance', + EasyParser('va instance', ['?va_name', '?va_size', '?starting_port', '?domain_id', '?tgt_name']), + EasyParser('va custinstance', ['?va_name', '?cpu_thread', '?memory', '?domain_id', '?tgt_name'])) + + def get_resource(self): + self.cli.set_enable() + return self.cli.cmd('show va resource', RegexParser('\s+(?P[^\n]+?)\s+(?P[0-9]+(\sshared)?)\s+(?P[0-9]+)G\s+[0-9]+\s+dedicated VF\s+', MATCHALL, reflags=re.S)) + + def get_status(self): + self.cli.set_enable() + return self.cli.cmd('show va status', RegexParser('(?P\S+)\s+(?Prunning|shutoff)\n', MATCHALL, reflags=re.S)) + + def get_vncport(self): + self.cli.set_enable() + return self.cli.cmd('show vncport', RegexParser('(?P\S+):\s*(?P[0-9]+)\n', MATCHALL, reflags=re.S)) + + def get_mgmtip(self): + self.cli.set_enable() + return self.cli.cmd('show va mgmtip', RegexParser('va mgmtip (?P\S+) (?P\S+)\n', MATCHALL, reflags=re.S)) + + def get_vlan(self): + self.cli.set_enable() + return self.cli.cmd('show va mgmtvlan' , RegexParser('va mgmtvlan (?P\S+) (?P\S+)\n', MATCHALL, reflags=re.S)) + + def get_nic_type(self): + self.cli.set_enable() + return self.cli.cmd('show va mgmttype', RegexParser('(?P\S+)\s+(?P\S+)\n', MATCHALL, reflags=re.S)) + + def get_disk(self): + self.cli.set_enable() + result = self.cli.cmd('show va disk', + RegexParser('(?P\S+):\n(?P([\S]+\s*?[\S]*?\s*?[\S]*?\s*?[\S]*?\n)+)\n', MATCHALL, reflags=re.S)) + res = [] + for each in result: + disks = each["disk_name"].split("\n") + del disks[0] + for item in disks: + if item and item.split()[0] != "reserved": + each_disk = { + "disk_name": item.split()[0], + "va_name": each["va_name"], + "disk_size": item.split()[-2].split("(")[0], + "bus_type": item.split()[-1] + } + res.append(each_disk) + return res + + def get_apv_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license apv', EasyParser('', ['?va_name', '?type'])) + + def get_ag_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license ag', EasyParser('', ['?va_name', '?type'])) + + def get_netgate_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license netgate', EasyParser('', ['?va_name', '?type'])) + + def get_netiag_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license netiag', EasyParser('', ['?va_name', '?type'])) + + def get_nsae_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license nsae', EasyParser('', ['?va_name', '?type'])) + + def get_netopti_license(self): + self.cli.set_enable() + return self.cli.cmd('show va license netopti', EasyParser('', ['?va_name', '?type'])) + + def get_boottime(self, va_name): + boottime = 'N/A' + boottime_file = VA_BOOT_TIME_INFO_PATH + va_name +'/boottime' + if os.path.exists(boottime_file): + try: + destination = open(boottime_file) + date_str = destination.read() + destination.close() + except Exception as e: + if destination: + destination.close() + #here has a %Z bug of python 2.7.5 datetime + l = date_str.strip().split() + l = l[:4] + [l[-1]] + date = datetime.strptime(' '.join(l), '%a %b %d %H:%M:%S %Y') + boottime = date.isoformat() + return boottime + + def handle_parameter(self, done): + rst = done['get_instance'][0] + done['get_instance'][1] + image_model = get_model('avx', ['va', 'image', 'VAImageManagement']) + image_manager = image_model.get_manager(self.sess) + for each in rst: + # for normal va instance + if "starting_port" in each and each["starting_port"] == 'null': + del each["starting_port"] + # for custom va instance + if "va_size" not in each: + each["va_size"] = "custom" + each["cpu_thread_shared"] = False + + # init the field + name = each["va_name"] + each["ip"] = "" + each["va_vncport"] = "" + each["vlan_tag"] = "" + each["nic_type"] = "e1000" + each["type"] = "" + each["attached_disk"] = [] + + # memory & cpu_thread & cpu_thread_shared + for item in done['get_resource']: + if item["va_name"] == name: + each["memory"] = item["memory"] + if "shared" in item["cpu_thread"]: + cpu_thread = int(item["cpu_thread"].split("shared")[0]) + cpu_thread_shared = True + else: + cpu_thread = int(item["cpu_thread"]) + cpu_thread_shared = False + break + each["cpu_thread"] = cpu_thread + each["cpu_thread_shared"] = cpu_thread_shared + # vnc + for item in done['get_vncport']: + if item["va_name"] == name: + each["va_vncport"] = item["vnc_port"] + break + # mgmt_ip + for item in done['get_mgmtip']: + if item["va_name"] == name: + each["ip"] = item["ip"] + break + # mgmtvlan + for item in done['get_vlan']: + if item["va_name"] == name: + each["vlan_tag"] = item["tag"] + break + # mgmttype + for item in done['get_nic_type']: + if item["va_name"] == name: + each["nic_type"] = item["nic_type"] + break + # mgmtexdisk + + for item in done['get_disk']: + if item["va_name"] == name: + each_disk = { + "disk_name": item["disk_name"], + "va_name": item["va_name"], + "disk_size": item["disk_size"], + "bus_type": item["bus_type"] + } + each["attached_disk"].append(each_disk) + + if COMPANY_KEYWORD == "Array": + # ag/apv license type + for item in done['get_apv_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_ag_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_netgate_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_netiag_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + if COMPANY_KEYWORD == "Infosec": + # nsae/netopti/netgate license type + for item in done['get_nsae_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_netopti_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_netgate_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_apv_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_ag_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + for item in done['get_netiag_license']: + if "va_name" in item and item["va_name"] == name: + each["type"] = item["type"] + break + + # query va image info for each va instance + image = each["tgt_name"] + image_instance = image_manager.all().filter({"image_name": image}) + product_category = image_instance[0].product_category if image_instance else '' + if product_category == "Other": + product_category = image_instance[0].other_category + product_name = image_instance[0].product_name if image_instance else '' + vendor = image_instance[0].vendor if image_instance else '' + each["product_category"] = product_category + each["product_name"] = product_name + each["vendor"] = vendor + each["status"] = False + each['boottime'] = self.get_boottime(each["va_name"]) + for item in done['get_status']: + if item["va_name"] == each["va_name"]: + each["status"] = True if item["status"] == "running" else False + break + return rst + + def get_data_from_cli(self, uuid, counter): + function_map = { + 'get_instance': (self.get_instance, []), + 'get_resource': (self.get_resource, []), + 'get_status': (self.get_status, []), + 'get_vncport': (self.get_vncport, []), + 'get_mgmtip': (self.get_mgmtip, []), + 'get_vlan': (self.get_vlan, []), + 'get_nic_type': (self.get_nic_type, []), + 'get_disk': (self.get_disk, []), + } + if COMPANY_KEYWORD == "Array": + function_map['get_apv_license'] = (self.get_apv_license, []) + function_map['get_ag_license'] = (self.get_ag_license, []) + function_map['get_netgate_license'] = (self.get_netgate_license, []) + function_map['get_netiag_license'] = (self.get_netiag_license, []) + if COMPANY_KEYWORD == "Infosec": + function_map['get_nsae_license'] = (self.get_nsae_license, []) + function_map['get_netgate_license'] = (self.get_netgate_license, []) + function_map['get_netopti_license'] = (self.get_netopti_license, []) + function_map['get_apv_license'] = (self.get_apv_license, []) + function_map['get_ag_license'] = (self.get_ag_license, []) + function_map['get_netiag_license'] = (self.get_netiag_license, []) + done = {} + for key in function_map.keys(): + current = counter.get_counter_data() + if current['uuid'] != uuid: + break + try: + func, args = function_map[key] # call real function + result = func(*args) + done[key] = result + except Exception as e: + done[key] = "ERROR: %s" % str(e) + current = counter.get_counter_data() + if current['uuid'] != uuid: + return [] + result = self.handle_parameter(done) + counter.write_result_file(uuid, result)