Index: /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py =================================================================== --- /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py (revision 9088) +++ /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/__init__.py (working copy) @@ -145,7 +145,10 @@ def _get_query_set(self): vaHandler = VAInstanceHandle(self.cli) current_time = int(time.time()) - counter = SessionCounter(vaHandler.get_id()) + query_type = "" + if getattr(self, 'query_type', None): + query_type = getattr(self, 'query_type') + counter = SessionCounter(vaHandler.get_id(), query_type) counter_data = counter.get_counter_data() if counter_data['timestamp'] <= current_time: # check newest request request_id = str(uuid.uuid4()) Index: /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py =================================================================== --- /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py (revision 9088) +++ /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/avx/models/va/instance/handler.py (working copy) @@ -23,24 +23,36 @@ VA_INSTANCE_COUNTER_DIR = '/va_instance_counter' VA_INSTANCE_COUNTER_FILE = 'config.json' EXPIRY_SECONDS = 600 # 10 min +QUERY_TYPE_MAP_FILE = 'query_type_map.json' class SessionCounter(): - def __init__(self, session_id): + def __init__(self, session_id, query_type=None): self.session_id = session_id self.counter_dir = TEMP_DIR + VA_INSTANCE_COUNTER_DIR self.session_dir = self.counter_dir + "/%s/" % self.session_id + self.query_type_file = self.session_dir + QUERY_TYPE_MAP_FILE try: if not os.path.exists(self.counter_dir): - os.mkdir(self.counter_dir) + os.makedirs(self.counter_dir) if not os.path.exists(self.session_dir): - os.mkdir(self.session_dir) + os.makedirs(self.session_dir) except OSError as e: if e.errno != errno.EEXIST: raise - self.counter_file = self.session_dir + VA_INSTANCE_COUNTER_FILE + exist_type_id = self.get_type_id_by_query_type(query_type) + if exist_type_id: + type_id = exist_type_id + else: + type_id = self.get_type_id() + if query_type: + self.set_type_id(type_id, query_type) + else: + self.set_type_id(type_id, "") + self.result_dir = self.session_dir + "{type_id}/".format(type_id=type_id) + self.counter_file = self.result_dir + VA_INSTANCE_COUNTER_FILE def get_result_file(self, uuid): - return (self.session_dir + 'result_%s.json' % uuid) + return (self.result_dir + 'result_%s.json' % uuid) def get_counter_data(self): # structure: { "counter": 0, "uuid": "xxxx", "timestamp": 123456 } @@ -53,6 +65,9 @@ return data def set_counter_data(self, counter, uuid, timestamp): + parent_dir = os.path.dirname(self.counter_file) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) with open(self.counter_file, 'w') as f: fcntl.flock(f, fcntl.LOCK_EX) json.dump({"counter": counter, "uuid": uuid, "timestamp": timestamp}, f) @@ -60,7 +75,7 @@ def write_result_file(self, uuid, result): final_path = self.get_result_file(uuid) - with tempfile.NamedTemporaryFile(mode='w', dir=self.session_dir, delete=False) as tmp_file: + with tempfile.NamedTemporaryFile(mode='w', dir=self.result_dir, delete=False) as tmp_file: json.dump(result, tmp_file) tmp_file_path = tmp_file.name os.rename(tmp_file_path, final_path) @@ -96,7 +111,7 @@ uuid = fname[7:-5] if uuid == current_uuid: continue - full_path = os.path.join(self.session_dir, fname) + full_path = os.path.join(self.result_dir, fname) try: mtime = os.path.getmtime(full_path) if now - mtime > EXPIRY_SECONDS: @@ -104,6 +119,89 @@ except: aninfo('hive.debug', 'There is error occurred when cleanup old results file for VA Instance list') + def get_type_id(self): + """ + Get the new type id for this session counter. + This is used to identify the type of query being made. + file content ex: [{"type": 1, "query_type": ""}] + :return last type id + 1 + 1 is the first type id. + """ + type_id = 1 + content = [] + if os.path.exists(self.query_type_file): + with open(self.query_type_file, 'r') as f: + fcntl.flock(f, fcntl.LOCK_SH) + content = json.load(f) + fcntl.flock(f, fcntl.LOCK_UN) + if content: + last_type = max(item["type"] for item in content) + type_id = last_type + 1 + return type_id + + def set_type_id(self, type_id, query_type): + """ + Set the type id and query type to the file. + :param type_id: The new type id. + :param query_type: The query type. + """ + if not os.path.exists(self.query_type_file): + with open(self.query_type_file, 'w') as f: + json.dump([], f) + parent_dir = os.path.dirname(self.query_type_file) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + try: + with open(self.query_type_file, 'r+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + content = json.load(f) + content.append({"type": type_id, "query_type": query_type}) + f.seek(0) + json.dump(content, f) + f.truncate() + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + import traceback + aninfo("hive.debug", "set_type_id error: %s\n%s" % (e, traceback.format_exc())) + + def get_type_id_by_query_type(self, query_type): + """ + Get the type id by query type. + :param query_type: The query type. + :return: The type id if found, otherwise None. + """ + if not os.path.exists(self.query_type_file): + return None + with open(self.query_type_file, 'r') as f: + fcntl.flock(f, fcntl.LOCK_EX) + content = json.load(f) + norm_query_type = self.normalize_query_type(query_type) + for item in content: + item_norm = self.normalize_query_type(item["query_type"]) + if item_norm == norm_query_type: + fcntl.flock(f, fcntl.LOCK_UN) + return item["type"] + fcntl.flock(f, fcntl.LOCK_UN) + return None + + def normalize_query_type(self, query_type): + """ + Normalize the query type to a standard format. + :param query_type: The query type to normalize. + :return: The normalized query type. + """ + if not query_type: + return "" + qt = dict(query_type) + if "key" in qt: + if isinstance(qt["key"], str): + qt["key"] = json.loads(qt["key"]) + elif isinstance(qt["key"], unicode): + qt["key"] = json.loads(qt["key"]) + if "key" in qt and isinstance(qt["key"], list): + qt["key"] = sorted(qt["key"]) + return json.dumps(qt, sort_keys=True) + class VAInstanceHandle(): def __init__(self, cli): self.sess = get_current_session() Index: /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/hive/model/ajax.py =================================================================== --- /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/hive/model/ajax.py (revision 9088) +++ /branches/rel_avx_2_7_4/src/webui/webui/htdocs/new/src/hive/model/ajax.py (working copy) @@ -636,6 +636,7 @@ #otherwise, it will get all field values def get_list_data(self, **kwargs): rtn = [] + setattr(self.manager, 'query_type', kwargs) if hasattr(self.manager, '_get_cacheable') and not self.manager._get_cacheable(): qs = self.manager._get_query_set() else: