Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py (revision 2486) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/cm/lib/postgres_db.py (working copy) @@ -507,6 +507,20 @@ );''' self.execute_sql(create_table_query) + def create_table_backup_schedule(self): + print('Creating table backup_schedule...') + create_table_sql = ''' + CREATE TABLE IF NOT EXISTS backup_schedule ( + id SERIAL PRIMARY KEY, + frequency varchar(64) NOT NULL, + time TIME NOT NULL, + day_of_the_week integer DEFAULT NULL, + day_of_the_month integer DEFAULT NULL, + month integer DEFAULT NULL CHECK (month BETWEEN 1 AND 12) + ); + ''' + self.execute_sql(create_table_sql) + if __name__ == '__main__': db = DB.get_connected_db() @@ -538,4 +552,5 @@ db.create_table_config_template() db.create_table_device_config_template() db.create_table_role_device_group() + db.create_table_backup_schedule() db.close() \ No newline at end of file Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (revision 2493) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/backup_controller.py (working copy) @@ -6,37 +6,14 @@ from hive.custom_exceptions import generic_exception as ge from cm.lib.libbasic_operation import oper_log from hive.utils import andebug, get_current_session +from hive.services.service_utils import ServiceUtils +from hive.services.schedule_backup import schedule_backup_job, fetch_schedule_backup_data, remove_backup_cron_job BACKUP_SCRIPT_FILE = "/ca/webui/htdocs/new/src/cm/backup/backup.sh" BACKUP_LOG_FILE = "/var/log/backup.log" BACKUP_DOWNLOAD_FILE = "/var/backups" -def handle_backup_req(request, path=None): - try: - if path == 'start' and request.method == 'POST': - return start_backup(request) - elif path == 'status' and request.method == 'GET': - return backup_status(request) - elif path == 'download' and request.method == 'GET': - filename = request.GET.get('filename') - return download_backup(filename) - else: - return HttpResponse(json.dumps({ - 'error': 405, - 'message': "Invalid HTTP method" - }), content_type='application/json') - - except ge.GenericError as e: - oper_log('error', 'system', e.message) - return ge.handle_exception(e) - except Exception as e: - message = str(e.message).replace("'", "") - oper_log('error', 'system', 'Exception while creating backup., details: {}'.format(message)) - e.message = 'Exception while creating backup., details: {}'.format(str(message)) - raise ge.GenericError(500, message) - - def start_backup(request): """ Starts the backup in the background.""" try: @@ -59,7 +36,7 @@ message = str(e.message).replace("'", "") oper_log('error', 'system', 'An Unexpected error occurred, details: {}'.format(message)) e.message = 'Exception while creating backup., details: {}'.format(message) - raise ge.GenericError(500, message) + raise ge.GenericError(500, e.message) def backup_status(request): @@ -87,7 +64,7 @@ except Exception as e: message = str(e).replace("'", "") oper_log('error', 'system', 'Exception while creating backup., details: {}'.format(message)) - e.message = 'Exception while creating backup., details: {}'.format(str(message)) + e.message = 'Exception while creating backup., details: {}'.format(message) raise ge.GenericError(500, e.message) return HttpResponse(json.dumps({"message": "Backup in Progress!"}), content_type='application/json') @@ -110,7 +87,7 @@ return HttpResponse(json.dumps({ "error": 400, "message": "Filename not provided. Please provide the Backup filename" - }), content_type='application/json') + }), content_type='application/json', status=400) file_path = os.path.join(BACKUP_DOWNLOAD_FILE, filename) @@ -118,7 +95,7 @@ return HttpResponse(json.dumps({ "error": 404, "message": "Backup file does not exist" - }), content_type='application/json') + }), content_type='application/json', status=404) # Using Stream Http to get the file response as chunks of data # instead of loading whole file in memory which is memory intensive @@ -128,4 +105,93 @@ except IOError as e: message = str(e).replace("'", "") oper_log('error', 'system', 'Error downloading backup file!, details: {}'.format(message)) - raise ge.GenericError(500, "Error downloading backup file!") \ No newline at end of file + raise ge.GenericError(500, "Error downloading backup file!") + + +def schedule_backup(request): + """ Handles user request to schedule backup """ + data = json.loads(request.body) + frequency = data.get("frequency") + time_str = data.get("time") + day_of_the_week = data.get("day_of_the_week") + day_of_the_month = data.get("day_of_the_month") + month = data.get("month") + + if not frequency: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Frequency need to be specified" + }), content_type='application/json', status=400) + + if not time_str: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Time need to be specified" + }), content_type='application/json', status=400) + + # Extracts the hours and minutes + try: + hours, minutes = time_str.split(":") + except ValueError as ve: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Invalid Time format. Use HH:MM" + }), content_type='application/json', status=400) + + if frequency == ServiceUtils.WEEKLY and day_of_the_week is None: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Day of the week need to be specified as the frequency is weekly." + }), content_type='application/json', status=400) + elif frequency == ServiceUtils.MONTHLY and day_of_the_month is None: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Day of the month need to be specified as the frequency is monthy." + }), content_type='application/json', status=400) + elif frequency == ServiceUtils.YEARLY and (day_of_the_month is None or month is None): + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Day of the month and month need to be specified as the frequency is yearly." + }), content_type='application/json', status=400) + + return schedule_backup_job(frequency, hours, minutes, day_of_the_week, day_of_the_month, month, request) + + +def get_schedule_backup_info(request): + """ Fetched the Backup schedule configured """ + return fetch_schedule_backup_data() + + +restore_handlers = { + ('start', 'POST'): start_backup, + ('status', 'GET'): backup_status, + ('download', 'GET'): download_backup, + ('schedule', 'POST'): schedule_backup, + ('schedule', 'GET'): get_schedule_backup_info, + ('schedule', 'DELETE'): remove_backup_cron_job, + +} + + +def handle_backup_req(request, path=None): + try: + handler = restore_handlers.get((path, request.method)) + if handler: + if handler is download_backup: + return download_backup(request.GET.get('filename')) + else: + return handler(request) + else: + return HttpResponse(json.dumps({ + 'error': 405, + 'message': "Invalid HTTP method" + }), content_type='application/json', status=405) + + except ge.GenericError as e: + oper_log('error', 'system', e.message) + return ge.handle_exception(e) + except Exception as e: + message = str(e.message).replace("'", "") + oper_log('error', 'system', 'Exception during backup or restore., details: {}'.format(message)) + message = 'Exception while creating backup or restoring backup., details: {}'.format(message) + raise ge.GenericError(500, message) \ No newline at end of file Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (revision 2493) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/controller/restore_controller.py (working copy) @@ -23,7 +23,7 @@ return HttpResponse(json.dumps({ 'error': 400, 'message': "Invalid file type. please upload file with extention tar.gz" - }), content_type='application/json') + }), content_type='application/json', status=400) # Save the uploaded file with open(filename, 'wb') as f: @@ -34,7 +34,7 @@ return HttpResponse(json.dumps({ 'error': 400, 'message': "No file uploaded" - }), content_type='application/json') + }), content_type='application/json', status=400) elif path == 'status' and request.method == 'GET': return restore_status(request) @@ -43,7 +43,7 @@ return HttpResponse(json.dumps({ 'error': 400, 'message': "Invalid HTTP method" - }), content_type='application/json') + }), content_type='application/json', status=400) except ge.GenericError as e: oper_log('error', 'system', e.message) @@ -104,8 +104,8 @@ e.message = 'Exception while restoring backup., details: {}'.format(str(message)) raise HttpResponse(json.dumps({ 'error': 500, - "status": "FAILED", - "message": "Error while restoring backup : {}".format(e.message)}), content_type='application/json') + "status": "Failed", + "message": "Error while restoring backup : {}".format(e.message)}), content_type='application/json', status=500) return HttpResponse(json.dumps({ "status": "In-Progress", Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (revision 2493) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (working copy) @@ -92,5 +92,10 @@ backup_logger.info("Rolling backup changes") ServiceUtils.run_command("rm -rf /var/backups/{}".format(BACKUP_FILE), backup_logger) oper_log('error', 'system', "Exception while creating backup.") - e.message = 'Exception while creating backup, details: {}'.format(str(e.message)) - raise ge.GenericError(400, e.message) \ No newline at end of file + e.message = 'Exception while creating backup, details: {}'.format(e.message) + raise ge.GenericError(500, e.message) + + +if __name__ == "__main__": + backup = BackupConfig() + backup.perform_backup() Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py (revision 0) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/schedule_backup.py (working copy) @@ -0,0 +1,215 @@ +import os +import json +import subprocess +import logging +import datetime + +from django.http import HttpResponse +from hive.custom_exceptions import generic_exception as ge +from cm.lib.libbasic_operation import oper_log +from hive.utils import andebug, get_current_session +from hive.services.service_utils import ServiceUtils +from hive.session import ANSession +from lib.crontab import CronTab +from cm.lib.postgres_db import DB + +# Configure Logging +backup_cron_logger = logging.getLogger("backup_cron") +backup_cron_logger.setLevel(logging.INFO) + +# Defining a file handler for the logs +file_handler = logging.FileHandler(ServiceUtils.BACKUP_CRON_LOG_FILE) +file_handler.setFormatter(logging.Formatter("%(asctime)s - %(message)s", "%Y-%m-%d %H:%M:%S")) + +backup_cron_logger.addHandler(file_handler) + + +def schedule_backup_job(frequency, hours, minutes, day_of_the_week, day_of_the_month, month, request): + """ Schedules the backup job """ + try: + # Persist Backup Schedule data in DB + db = DB.get_connected_db() + fetchall_sql = '''SELECT * FROM backup_schedule''' + data = db.fetchall(fetchall_sql) + time_str = hours + ':' + minutes + if not data: + insert_sql = """INSERT INTO backup_schedule + (frequency, time, day_of_the_month, day_of_the_week, month) + VALUES ('{}', '{}', {}, {}, {})""".format( + frequency, time_str, + "NULL" if day_of_the_month is None else day_of_the_month, + "NULL" if day_of_the_week is None else day_of_the_week, + "NULL" if month is None else month + ) + db.execute_sql(insert_sql) + else: + update_sql = """UPDATE backup_schedule set frequency='{}', time='{}', day_of_the_month={}, day_of_the_week={}, + month={} where id = 1""".format( + frequency, time_str, + "NULL" if day_of_the_month is None else day_of_the_month, + "NULL" if day_of_the_week is None else day_of_the_week, + "NULL" if month is None else month + ) + db.execute_sql(update_sql) + db.close() + + # Create cron expression based on frequency + if frequency == ServiceUtils.HOURLY: + cron_time = "{} * * * *".format(minutes) + elif frequency == ServiceUtils.DAILY: + cron_time = "{} {} * * *".format(minutes, hours) + elif frequency == ServiceUtils.WEEKLY: + cron_time = "{} {} * * {}".format(minutes, hours, day_of_the_week) + elif frequency == ServiceUtils.MONTHLY: + cron_time = "{} {} {} * *".format(minutes, hours, day_of_the_month) # Runs on specified day + elif frequency == ServiceUtils.YEARLY: + cron_time = "{} {} {} {} *".format(minutes, hours, day_of_the_month, month) # Runs on specified month & day + else: + return HttpResponse(json.dumps({ + 'error': 400, + 'message': "Invalid frequency!" + }), content_type='application/json') + + # Setup Cron Job + backup_job = cron_time + (' export DJANGO_SETTINGS_MODULE=djproject.settings && ' + '/usr/bin/python /ca/webui/htdocs/new/src/hive/services/backup_service.py') + deploy_cron_job(backup_job) + + return HttpResponse(json.dumps({"message": "Backup schedule updated successfully!", "cron_time": cron_time}), + content_type="application/json") + except Exception as e: + backup_cron_logger.error("Schedule Backup failed! " + str(e)) + oper_log('error', 'system', "Exception while creating scheduled backup.") + message = 'Exception while creating scheduled backup, details: {}'.format(e.message) + raise ge.GenericError(500, message) + + +def deploy_cron_job(backup_job): + """ Deploys the cron job in system""" + retcode, err, installed_content = _runcmd("crontab -l") + if retcode != 0 and 'no crontab for' not in err: + raise OSError("crontab not supported in your system") + cron_jobs = installed_content.splitlines() + updated_cron_jobs = [] + + if retcode == 0: + for job in cron_jobs: + if "/ca/webui/htdocs/new/src/hive/services/backup_service.py" not in job: + # Replace only this job + updated_cron_jobs.append(job) + + updated_cron_jobs.append(backup_job) + + if not updated_cron_jobs: + new_cron_content = "\n".join(cron_jobs) + "\n" + else: + new_cron_content = "\n".join(updated_cron_jobs) + "\n" + + # install back + retcode, err, out = _runcmd("crontab", new_cron_content) + if retcode != 0: + raise ValueError( + "failed to install crontab, check if crontab is valid") + + +def remove_backup_cron_job(request): + """ Removes the backup cron job in system""" + try: + retcode, err, installed_content = _runcmd("crontab -l") + if retcode != 0 and 'no crontab for' not in err: + raise OSError("crontab not supported in your system") + cron_jobs = installed_content.splitlines() + updated_cron_jobs = [] + + if retcode == 0: + for job in cron_jobs: + if "/ca/webui/htdocs/new/src/hive/services/backup_service.py" not in job: + # Replace only this job + updated_cron_jobs.append(job) + + if not updated_cron_jobs: + new_cron_content = "\n".join(cron_jobs) + "\n" + else: + new_cron_content = "\n".join(updated_cron_jobs) + "\n" + + # install back + retcode, err, out = _runcmd("crontab", new_cron_content) + if retcode != 0: + raise ValueError( + "failed to install crontab, check if crontab is valid") + + # Remove cron from DB + db = DB.get_connected_db() + fetchall_sql = '''SELECT * FROM backup_schedule''' + data = db.fetchall(fetchall_sql) + if not data: + return HttpResponse(json.dumps({"message": "No scheduled backup found!"}), + content_type="application/json") + + delete_cron_sql = '''DELETE FROM backup_schedule''' + db.execute_sql(delete_cron_sql) + db.close() + + return HttpResponse(json.dumps({"message": "Successfully removed the backup schedule!"}), + content_type="application/json") + + except Exception as e: + backup_cron_logger.error("Removing Schedule Backup failed! " + str(e)) + oper_log('error', 'system', "Exception while removing scheduled backup.") + message = 'Exception while removing scheduled backup, details: {}'.format(e.message) + raise ge.GenericError(500, message) + + +def _runcmd(cmd, input=None): + """ Runs Cron commands""" + if input is not None: + p = subprocess.Popen(cmd, + shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + preexec_fn=os.setsid) + else: + p = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + preexec_fn=os.setsid) + + stdoutdata, stderrdata = p.communicate(input) + return p.returncode, stderrdata, stdoutdata + + +def fetch_schedule_backup_data(): + """ Deploys the scheduled backup job details """ + try: + db = DB.get_connected_db() + fetchall_sql = '''SELECT * FROM backup_schedule''' + data = db.fetchall(fetchall_sql) + + if data: + key = ['frequency', 'time', 'day_of_the_month', 'day_of_the_week', 'month'] + values = list(data[0]) + + # Convert `datetime.time` to a string (HH:MM format) + if isinstance(values[2], datetime.time): + values[2] = values[2].strftime("%H:%M") # Convert to HH:MM string + + result = dict(zip(key, values)) + else: + result = None + + if result: + return HttpResponse(json.dumps(result), + content_type="application/json") + else: + return HttpResponse(json.dumps({"message": "No Backups scheduled!"}), + content_type="application/json") + + except Exception as e: + backup_cron_logger.error("Fetching Schedule Backup details failed! " + str(e)) + oper_log('error', 'system', "Exception while fetching scheduled backup details.") + message = 'Exception while fetching scheduled backup details, details: {}'.format(e.message) + raise ge.GenericError(500, message) \ No newline at end of file Index: /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/service_utils.py =================================================================== --- /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/service_utils.py (revision 2490) +++ /branches/amp_3_7/src/webui/webui/htdocs/new/src/hive/services/service_utils.py (working copy) @@ -6,12 +6,18 @@ class ServiceUtils: # Set Variables BACKUP_LOG_FILE = "/var/log/backup.log" + BACKUP_CRON_LOG_FILE = "/var/log/backup_cron.log" POSTGRES_USER = "postgres" POSTGRES_DB = "cm" POSTGRES_HOST = "127.0.0.1" POSTGRES_PORT = "5432" INFLUX_DB = "composer" SQLITE_FILE = "/var/opt/composer/ui/conf/composer.db" + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + MONTHLY = "monthly" + YEARLY = "yearly" def __init__(self): pass