Index: /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/clean_elastic.py =================================================================== --- /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/clean_elastic.py (revision 2344) +++ /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/clean_elastic.py (working copy) @@ -12,7 +12,7 @@ def delete_indices(url_delete_date): # 删除30天前的索引 - url_delete = 'http://'+elasticsearch_url+'/%s?pretty' % (url_delete_date) + url_delete = 'http://' + elasticsearch_url + '/%s?pretty' % (url_delete_date) request_delete = urllib2.Request(url_delete) request_delete.get_method = lambda: 'DELETE' # 设置HTTP请求方式 response_delete = urllib2.urlopen(request_delete).read() @@ -20,7 +20,7 @@ def delete_indices_by_disk_usage(percent, index): - url_get = 'http://'+elasticsearch_url+'/_cat/allocation?format=json' + url_get = 'http://' + elasticsearch_url + '/_cat/allocation?format=json' request_get = urllib2.Request(url_get) response_get = urllib2.urlopen(request_get).read() allocations = json.loads(response_get) @@ -31,9 +31,9 @@ if 'disk.percent' in allocation: pct += float(allocation['disk.percent']) num += 1 - pct = pct/num + pct = pct / num - if pct > percent-3: + if pct > percent - 3: delete_indices(index) return False return True @@ -43,31 +43,43 @@ args = sys.argv[1:] if len(args) == 0: raise Exception("Invalid reporting args") - + if os.path.exists("/ca/extensions/auditing"): days = int(args[0]) percent = 80 + storage = 'primary' + if len(args) >= 2: percent = int(args[1]) + if len(args) > 2: + storage = str(args[2]) + + location = '/var/log/elasticsearch' + if storage == 'secondary': + location = '/secondary/elasticsearch' days_before_num = datetime.date.today() - datetime.timedelta(days=days) # 获取30天前的日期 - for abs_path, dir, file in os.walk('/var/log/elasticsearch'): + for abs_path, dir, file in os.walk(location): for f in file: if os.path.getmtime(os.path.join(abs_path, f)) < time.mktime(days_before_num.timetuple()): os.chdir(abs_path) os.remove(f) - url_get = 'http://'+elasticsearch_url+'/_cat/indices?format=json' + url_get = 'http://' + elasticsearch_url + '/_cat/indices?format=json' request_get = urllib2.Request(url_get) response_get = urllib2.urlopen(request_get).read() indices = json.loads(response_get) + def takeIndex(elem): return elem['index'] + + indices.sort(key=takeIndex, reverse=False) + def isprimer(index): if 'acm_syslog-' in index['index']: date_index = index['index'].replace('acm_syslog-', '') @@ -76,15 +88,16 @@ if date_time < days_before_num: date_format = 'acm_syslog-' + \ - date_time.__format__('%Y.%m.%d') # 对日期进行格式化输出，例：2018.04.23 + date_time.__format__('%Y.%m.%d') # 对日期进行格式化输出，例：2018.04.23 delete_indices(date_format) return False else: return False return True + indices = filter(isprimer, indices) for index in indices: if delete_indices_by_disk_usage(percent, index['index']): - break + break \ No newline at end of file Index: /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py =================================================================== --- /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py (revision 2354) +++ /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/custom_exceptions/generic_exception.py (working copy) @@ -1,4 +1,5 @@ -import subprocess +from django.http import HttpResponse +import json class GenericError(Exception): @@ -10,4 +11,25 @@ def __str__(self): - return "{}: {}".format(self.message, self.original_exception) \ No newline at end of file + return "{}: {}".format(self.message, self.original_exception) + + +def handle_exception(exc): + """Converts an exception into an HttpResponse""" + if isinstance(exc, GenericError): + response_data = { + 'error': exc.status_code, + 'message': exc.message, + 'details': str(exc) + } + status_code = exc.status_code + else: + # Default behavior for other exceptions + response_data = { + 'error': exc.status_code, + 'message': 'An unexpected error occurred', + 'details': str(exc) + } + status_code = 500 + + return HttpResponse(json.dumps(response_data), status=status_code, content_type='application/json') \ No newline at end of file Index: /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/log_location.py =================================================================== --- /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/log_location.py (revision 2354) +++ /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/log_location.py (working copy) @@ -31,18 +31,11 @@ except ge.GenericError as e: oper_log('error', 'system', e.message) - return HttpResponse(json.dumps({ - 'error': 500, - 'message': e.message, - 'data': str(e) - }), content_type='application/json') + return ge.handle_exception(e) except Exception as e: oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e)) - return HttpResponse(json.dumps({ - 'error': 500, - 'message': "An Unexpected error occurred", - 'data': str(e) - }), content_type='application/json') + e.message = 'An Unexpected error occurred, details: {}'.format(e) + return ge.handle_exception(e) else: return HttpResponse(json.dumps({ @@ -63,11 +56,8 @@ }), content_type='application/json') except Exception as e: oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e)) - return HttpResponse(json.dumps({ - 'error': 500, - 'message': "An Unexpected error occurred", - 'data': str(e) - }), content_type='application/json') + e.message = 'An Unexpected error occurred, details: {}'.format(e) + return ge.handle_exception(e) def change_log_location(request): @@ -89,24 +79,15 @@ }), content_type='application/json') except IOError as e: oper_log('error', 'system', 'Failed to write to config file') - return HttpResponse(json.dumps({ - 'error': 500, - 'message': "Failed to write to config file", - 'data': str(e) - }), content_type='application/json') + e.message = "Failed to write to config file" + return ge.handle_exception(e) except ge.GenericError as e: # Log the error (optional) oper_log('error', 'system', 'Error while creating partition: {}'.format(e)) - return HttpResponse(json.dumps({ - 'error': 500, - 'message': "Error while creating partition", - 'data': str(e) - }), content_type='application/json') + e.message = "Error while creating partition" + return ge.handle_exception(e) except Exception as e: # Handle unexpected exceptions oper_log('error', 'system', 'An unexpected error occurred: {}'.format(e)) - return HttpResponse(json.dumps({ - 'error': 500, - 'message': "An unexpected error occurred", - 'data': str(e) - }), content_type='application/json') \ No newline at end of file + e.message = 'An unexpected error occurred: {}'.format(e) + return ge.handle_exception(e) \ No newline at end of file Index: /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/storage.py =================================================================== --- /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/storage.py (revision 2344) +++ /branches/amp_3_6_0/src/webui/webui/htdocs/new/src/hive/storage.py (working copy) @@ -10,9 +10,9 @@ MonitorTree = [{ "name": - "Device Build", + "Device Build", "depend": - "system", + "system", "paths": [{ "name": "Device Build", "description": "Device Build", @@ -22,9 +22,9 @@ }] }, { "name": - "Configuration File", + "Configuration File", "depend": - "system", + "system", "paths": [{ "name": "Device Configuration Files", "description": "Device Configuration Files", @@ -34,9 +34,9 @@ }] }, { "name": - "Monitoring", + "Monitoring", "depend": - "monitoring", + "monitoring", "paths": [{ "name": "Reporting", "description": "Report Materials, Report Files", @@ -51,9 +51,9 @@ }] }, { "name": - "Auditing", + "Auditing", "depend": - "auditing", + "auditing", "paths": [{ "name": "Device Log", "description": "Device Log", @@ -69,7 +69,6 @@ def _runcmd(cmd, input=None): - if input is not None: p = subprocess.Popen(cmd, shell=True, @@ -114,7 +113,8 @@ path['size'] = getdirsize(path['path']) elif path['type'] == "es_query": try: - httpClient = httplib.HTTPConnection(CMDATA['ELASTICSEARCH_IP'], str(CMDATA['ELASTICSEARCH_PORT']), True, timeout=300) + httpClient = httplib.HTTPConnection(CMDATA['ELASTICSEARCH_IP'], str(CMDATA['ELASTICSEARCH_PORT']), + True, timeout=300) httpClient.request("GET", path['path']) response = httpClient.getresponse() body = response.read() @@ -140,10 +140,17 @@ elif key == "crontab": if request.method == 'GET': - obj = {'schedule': False, 'duration': '30', 'percent': '80'} + obj = {'schedule': False, 'duration': '30', 'percent': '80', 'storage': 'primary'} amp_cron = CronTab(user=True) + storage_type = request.GET.get('storage') + + comment = 'storage_daily_clean' + if storage_type == 'secondary': + comment = 'secondary_storage_daily_clean' + obj['storage'] = storage_type + + iter = amp_cron.find_comment(comment) - iter = amp_cron.find_comment('storage_daily_clean') for job in iter: obj['schedule'] = job.is_enabled() if len(job.command.split(' ')) >= 3: @@ -158,22 +165,37 @@ # andebug("hive.debug", request.body) obj = json.loads(request.body) amp_cron = CronTab(user=True) - iter = amp_cron.find_comment('storage_daily_clean') - - for job in iter: - job.delete() - job1 = amp_cron.new(command='/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s' % (obj['duration'], obj['percent'])) - job1.setall('0 1 * * *') - job1.set_comment("storage_daily_clean") - - job2 = amp_cron.new(command='/usr/bin/python /ca/bin/clean_oper_log.py %s %s' % (obj['duration'], obj['percent'])) - job2.setall('0 1 * * *') - job2.set_comment("storage_daily_clean") - - if not obj['schedule']: - job1.enable(False) - job2.enable(False) + storage = obj['storage'] + if len(storage) != 0 and storage == 'secondary': + iter = amp_cron.find_comment('secondary_storage_daily_clean') + for job in iter: + job.delete() + + job = amp_cron.new(command='/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s %s' % ( + obj['duration'], obj['percent'], storage)) + job.setall('0 1 * * *') + job.set_comment("secondary_storage_daily_clean") + if not obj['schedule']: + job.enable(False) + else: + iter = amp_cron.find_comment('storage_daily_clean') + for job in iter: + job.delete() + + job1 = amp_cron.new(command='/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s' % ( + obj['duration'], obj['percent'])) + job1.setall('0 1 * * *') + job1.set_comment("storage_daily_clean") + + job2 = amp_cron.new( + command='/usr/bin/python /ca/bin/clean_oper_log.py %s %s' % (obj['duration'], obj['percent'])) + job2.setall('0 1 * * *') + job2.set_comment("storage_daily_clean") + + if not obj['schedule']: + job1.enable(False) + job2.enable(False) amp_cron.write() return HttpResponse(json.dumps(obj), content_type='application/json') @@ -181,7 +203,8 @@ elif key == "clean": obj = json.loads(request.body) _runcmd("/usr/bin/python /ca/bin/clean_oper_log.py %s %s" % (obj['duration'], obj['percent'])) - _, _, content = _runcmd("/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s" % (obj['duration'], obj['percent'])) + _, _, content = _runcmd( + "/usr/bin/python /ca/webui/htdocs/new/src/clean_elastic.py %s %s" % (obj['duration'], obj['percent'])) return HttpResponse(content) - raise Http404('Has no such page for action %s' % key) + raise Http404('Has no such page for action %s' % key) \ No newline at end of file