Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py (revision 2490) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/djproject/urls.py (working copy) @@ -14,7 +14,8 @@ from hive.log_location import handle_log_location_app from hive.llb_stats import handle_llb_stats_req from hive.report.generate_report import handle_report_generation -from hive.controller.backup_controller import handle_backup_generation +from hive.controller.backup_controller import handle_backup_req +from hive.controller.restore_controller import handle_restore_req js_info_dict = { #'packages': ('your.app.package',), @@ -64,7 +65,8 @@ url(r'^llb/(?P.*)$', handle_llb_stats_req), url(r'^report/(?P.*)$', handle_report_generation), url(r'^log/(?P\w+)$', handle_log_location_app), - url(r'^backup/(?P.*)$', handle_backup_generation), + url(r'^backup/(?P.*)$', handle_backup_req), + url(r'^restore/(?P.*)$', handle_restore_req), url(r'^real_service$', real_service), url(r'^rs_block$', rs_block), url(r'^rs_block_import$', rs_block_import), Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (revision 2490) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (working copy) @@ -12,7 +12,7 @@ BACKUP_DOWNLOAD_FILE = "/var/backups" -def handle_backup_generation(request, path=None): +def handle_backup_req(request, path=None): try: if path == 'start' and request.method == 'POST': return start_backup(request) @@ -31,15 +31,16 @@ oper_log('error', 'system', e.message) return ge.handle_exception(e) except Exception as e: - oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(e.message)) - e.message = 'An Unexpected error occurred, details: {}'.format(e.message) - return ge.handle_exception(e) + oper_log('error', 'system', 'Exception while creating backup., details: {}'.format(e.message)) + e.message = 'Exception while creating backup., details: {}'.format(str(e.message)) + raise ge.GenericError(500, e.message) def start_backup(request): """ Starts the backup in the background.""" try: session = get_current_session() + expiry = request.session.get_expiry_date() session.cli.set_enable() session.cli.cmd("config terminal") output = session.cli.cmd("show run") @@ -49,15 +50,15 @@ ) # Creating another process so that it runs in background p1 = subprocess.Popen(["python", "-c", cmd], - stdin=subprocess.PIPE, - close_fds=True) + stdin=subprocess.PIPE, + close_fds=True) p1.stdin.write(output.encode()) p1.stdin.close() return HttpResponse(json.dumps({"message": "Backup started!"}), content_type='application/json') except Exception as e: oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(str(e))) e.message = 'Exception while creating backup., details: {}'.format(str(e.message)) - raise ge.GenericError(400, e.message) + raise ge.GenericError(500, e.message) def backup_status(request): @@ -68,14 +69,15 @@ except IOError as e: oper_log('error', 'system', 'No log found!, details: {}'.format(str(e))) e.message = 'No log found!, details: {}'.format(str(e)) - raise ge.GenericError(400, e.message) + raise ge.GenericError(204, e.message) try: for line in reversed(logs): if "Backup completed" in line in line: backup_filename = str(line).split("Backup completed:")[1].strip() scheme = "https" if request.is_secure() else "http" - download_url = (scheme + "://" + request.get_host() + "/backup/download?filename={}".format(backup_filename)) + download_url = (scheme + "://" + request.get_host() + "/backup/download?filename={}".format( + backup_filename)) return HttpResponse(json.dumps({"message": "Backup Completed!", "download_url": download_url}), content_type='application/json') elif "failed" in line: Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (revision 0) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (working copy) @@ -0,0 +1,60 @@ +import os +import json +import subprocess + +from django.http import HttpResponse +from hive.custom_exceptions import generic_exception as ge +from cm.lib.libbasic_operation import oper_log +from hive.utils import andebug, get_current_session +from hive.services.restore_service import RestoreConfig +from hive.session import ANSession + + +def handle_restore_req(request, path=None): + try: + if path == 'start' and request.method == 'POST': + if 'backup_file' not in request.FILES: + return HttpResponse(json.dumps({ + 'error': 405, + 'message': "No file uploaded" + }), content_type='application/json') + + uploaded_file = request.FILES['backup_file'] + filename = os.path.join("/tmp" + uploaded_file.name) + + # Save the uploaded file + with open(filename, 'wb') as f: + for chunk in uploaded_file.chunks(): + f.write(chunk) + return restore_backup(filename, request) + else: + return HttpResponse(json.dumps({ + 'error': 405, + 'message': "Invalid HTTP method" + }), content_type='application/json') + + except ge.GenericError as e: + oper_log('error', 'system', e.message) + return ge.handle_exception(e) + except Exception as e: + oper_log('error', 'system', 'Exception while restoring backup., details: {}'.format(e.message)) + e.message = 'Exception while restoring backup., details: {}'.format(str(e.message)) + raise ge.GenericError(500, e.message) + + +def restore_backup(file, request): + """ Restore Backup""" + restore_config = RestoreConfig() + try: + session = get_current_session() + # Since the file upload during restore is taking a long time _thread_locals in utils.py is not initialized + # with session id when we start the restore. So getting the session object specifically here in case the + # get_current_session does not return session object + if session is None: + session = ANSession.find_session_by_id(request.COOKIES['hive_sess']) + return restore_config.restore_backup_config(file, session) + + except Exception as e: + oper_log('error', 'system', 'Exception while restoring backup., details: {}'.format(str(e))) + e.message = 'Exception while restoring backup., details: {}'.format(str(e.message)) + raise ge.GenericError(500, "Exception while fetching backup status") \ No newline at end of file Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (revision 2490) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (working copy) @@ -5,12 +5,13 @@ import shutil import logging import sys +import json sys.path.append('/ca/webui/htdocs/new/src') from cm.lib.libbasic_operation import oper_log from service_utils import ServiceUtils from hive.utils import andebug -import json +from hive.custom_exceptions import generic_exception as ge BACKUP_TIMESTAMP = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") BACKUP_DIR = "/tmp/product_backup_" + BACKUP_TIMESTAMP @@ -34,7 +35,6 @@ def perform_backup(self): """ Performs the backup process """ - andebug('an.model.cli', 'Inside perform_backup') try: os.makedirs(BACKUP_DIR) @@ -92,8 +92,5 @@ backup_logger.info("Rolling backup changes") ServiceUtils.run_command("rm -rf /var/backups/{}".format(BACKUP_FILE), backup_logger) oper_log('error', 'system', "Exception while creating backup.") - return json.dumps({ - 'error': 500, - 'message': "Exception while creating backup.", - 'data': str(e) - }) \ No newline at end of file + e.message = 'Exception while creating backup, details: {}'.format(str(e.message)) + raise ge.GenericError(400, e.message) \ No newline at end of file Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/restore_service.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/restore_service.py (revision 0) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/restore_service.py (working copy) @@ -0,0 +1,111 @@ +import os +import subprocess +import tarfile +import shutil +import datetime +import sys +import logging +import json +from django.http import HttpResponse +from hive.utils import andebug, get_current_session +from hive.services.service_utils import ServiceUtils +from hive.custom_exceptions import generic_exception as ge + +sys.path.append('/ca/webui/htdocs/new/src') +from cm.lib.libbasic_operation import oper_log + +TIMESTAMP = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +RESTORE_DIR = "/tmp/restore" +RESTORE_LOG_FILE = "/var/log/restore.log" +POSTGRES_USER = "postgres" +POSTGRES_DB = "cm" +POSTGRES_HOST = "127.0.0.1" +POSTGRES_PORT = "5432" +INFLUX_DB = "composer" +SQLITE_FILE_PATH = "/var/opt/composer/ui/conf/composer.db" +INFLUX_DIR = "/usr/local/bin/composer/influxd" + +# Configure restore logger +restore_logger = logging.getLogger("restore_logger") +restore_logger.setLevel(logging.INFO) + +file_handler = logging.FileHandler(RESTORE_LOG_FILE) +file_handler.setFormatter(logging.Formatter("%(asctime)s - %(message)s")) + +restore_logger.addHandler(file_handler) + + +class RestoreConfig: + def __init__(self): + pass + + def restore_backup_config(self, backup_file, session): + """ Performs the restore process """ + try: + if not os.path.exists(RESTORE_DIR): + os.makedirs(RESTORE_DIR) + + restore_logger.info("Extracting Backup") + with tarfile.open(backup_file, "r:gz") as tar: + for member in tar.getmembers(): + member_path = os.path.join(RESTORE_DIR, member.name) + if not os.path.abspath(member_path).startswith(os.path.abspath(RESTORE_DIR)): + return json.dumps({ + 'error': 400, + 'message': "Exception while restoring backup. Not a valid backup file" + }) + tar.extractall(RESTORE_DIR) + + restore_logger.info("Restoring InfluxDB") + if os.path.exists(INFLUX_DIR): + cmd = "/usr/local/bin/composer/influx -execute 'DROP DATABASE composer'" + ServiceUtils.run_command(cmd, restore_logger) + influx_backup_path = os.path.join(RESTORE_DIR, "influxdb") + cmd = "/usr/local/bin/composer/influxd restore -portable -db composer {}".format(influx_backup_path) + ServiceUtils.run_command(cmd, restore_logger) + restore_logger.info("InfluxDB restored successfully.") + else: + restore_logger.info("InfluxDB backup not found!") + + restore_logger.info("Restoring Postgres DB") + pg_dump = os.path.join(RESTORE_DIR, "postgres_backup.dump") + if os.path.exists(pg_dump): + cmd = ("/usr/pgsql-10/bin/pg_restore -U {} -h {} -p {} -d {} -c {}".format(POSTGRES_USER, POSTGRES_HOST, + POSTGRES_PORT, POSTGRES_DB, + pg_dump)) + ServiceUtils.run_command(cmd, restore_logger) + restore_logger.info("Postgres DB restored successfully.") + else: + restore_logger.info("Postgres DB not found.") + + restore_logger.info("Restoring SQLite DB") + sqlite_backup = os.path.join(RESTORE_DIR, "composer.db") + if os.path.exists(sqlite_backup): + shutil.copy(sqlite_backup, SQLITE_FILE_PATH) + restore_logger.info("SQLite DB restored successfully.") + else: + restore_logger.info("SQLite DB not found.") + + restore_logger.info("Restoring Running configuration") + session.cli.set_enable() + session.cli.cmd("config terminal") + + for config_file in os.listdir(RESTORE_DIR): + if config_file.startswith("show_run_"): + file_path = os.path.join(RESTORE_DIR, config_file) + with open(file_path, "r") as show_run_file: + line = show_run_file.readline() + session.cli.cmd(line) + + # Backup system logs + restore_logger.info("Restore system logs") + ServiceUtils.run_command("rsync -av {}/logs/ /var/log/".format(RESTORE_DIR), restore_logger) + + restore_logger.info("Restore completed") + return HttpResponse(json.dumps({"message": "Restore completed."}), content_type='application/json') + + except Exception as e: + restore_logger.info("Restoring backup failed! " + str(e)) + oper_log('error', 'system', "Exception while creating restoring backup.") + e.message = 'Exception while restoring backup., details: {}'.format(str(e.message)) + raise ge.GenericError(400, e.message) \ No newline at end of file