Index: /branches/amp_3_7_2/amp.spec =================================================================== --- /branches/amp_3_7_2/amp.spec (revision 2751) +++ /branches/amp_3_7_2/amp.spec (working copy) @@ -202,6 +202,8 @@ install -Dm 0755 scripts/check_webui_root_user.py %{buildroot}/ca/bin/check_webui_root_user.py install -Dm 0755 scripts/check_crontab_add.py %{buildroot}/ca/bin/check_crontab_add.py install -Dm 0755 scripts/clean_oper_log.py %{buildroot}/ca/bin/clean_oper_log.py +install -Dm 0755 scripts/export_es_snapshot.sh %{buildroot}/ca/bin/export_es_snapshot.sh +install -Dm 0755 scripts/import_es_snapshot.sh %{buildroot}/ca/bin/import_es_snapshot.sh cp conf/dashboard.ndjson %{buildroot}/ca/bin/ cp scripts/import_kibana_dashboard.sh %{buildroot}/ca/bin/ cp lib/composer/composer-2.5.0.2.tgz %{buildroot}/ca/etc/ Index: /branches/amp_3_7_2/scripts/export_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/export_es_snapshot.sh (nonexistent) +++ /branches/amp_3_7_2/scripts/export_es_snapshot.sh (working copy) @@ -0,0 +1,188 @@ +#!/bin/bash +set -uo pipefail + +# export_es_snapshot.sh +# +# Usage: +# export_es_snapshot.sh [--local_path X] +# [--ip X --user X --pass X --remote_path X] +# +# Always creates local ES snapshot of acm_syslog-*. +# If --ip is provided → snapshot directory is rsynced to remote host. +# + +# ============================================================ +# DEFAULTS +# ============================================================ +SNAPSHOT_DIR="" # filled from --local_path OR defaults to /var/backups/es_snapshots +REPO_NAME="${REPO_NAME:-amp_backup_repo}" +ES_HOST="${ES_HOST:-localhost}" +ES_PORT="${ES_PORT:-9200}" + +REMOTE_IP="" +REMOTE_USER="" +REMOTE_PASS="" +REMOTE_PATH="" + +EXPORT_LOG="/var/log/es_export.log" +TIMESTAMP="$(date +'%Y%m%d_%H%M%S')" +SNAP_NAME="snap_${TIMESTAMP}" +RESTART_ES=false + +# ============================================================ +# LOGGING HELPERS +# ============================================================ +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') $1" | tee -a "$EXPORT_LOG" +} + +fail() { + log "ERROR: $1" + log "=== EXPORT FAILED ===" + exit 1 +} + +# ============================================================ +# ARGUMENT PARSER +# ============================================================ +while [[ $# -gt 0 ]]; do + case "$1" in + --local_path) + SNAPSHOT_DIR="$2" + shift 2 + ;; + --ip) + REMOTE_IP="$2" + shift 2 + ;; + --user) + REMOTE_USER="$2" + shift 2 + ;; + --pass) + REMOTE_PASS="$2" + shift 2 + ;; + --remote_path) + REMOTE_PATH="$2" + shift 2 + ;; + *) + echo "Unknown parameter: $1" + exit 1 + ;; + esac +done + +# APPLY DEFAULT LOCAL PATH IF NOTHING PROVIDED +if [[ -z "$SNAPSHOT_DIR" ]]; then + SNAPSHOT_DIR="/var/backups/es_snapshots" +fi + +# ============================================================ +# VALIDATIONS +# ============================================================ +if [[ -n "$REMOTE_IP" ]]; then + # Remote requires all 3 + remote_path + [[ -z "$REMOTE_USER" ]] && fail "Remote export requires --user" + [[ -z "$REMOTE_PASS" ]] && fail "Remote export requires --pass" + [[ -z "$REMOTE_PATH" ]] && fail "Remote export requires --remote_path" +fi + +# ============================================================ +# START LOG +# ============================================================ +log "===============================================" +log " STARTING ELASTICSEARCH SNAPSHOT EXPORT" +log " Snapshot directory : $SNAPSHOT_DIR" +log " Snapshot name : $SNAP_NAME" +if [[ -n "$REMOTE_IP" ]]; then + log " Remote transfer : $REMOTE_USER@$REMOTE_IP:$REMOTE_PATH" +else + log " Remote transfer : none" +fi +log "===============================================" + +# ============================================================ +# CREATE SNAPSHOT DIRECTORY +# ============================================================ +mkdir -p "$SNAPSHOT_DIR" || fail "Cannot create snapshot directory" +chown elasticsearch:elasticsearch "$SNAPSHOT_DIR" 2>/dev/null || true + + +# ============================================================ +# CONFIGURE PATH.REPO +# ============================================================ +log "Ensuring path.repo is configured..." + +if ! grep -q "path.repo" /etc/elasticsearch/elasticsearch.yml; then + echo -e "\npath.repo: [\"$SNAPSHOT_DIR\"]" >> /etc/elasticsearch/elasticsearch.yml + RESTART_ES=true +else + if ! grep -q "$SNAPSHOT_DIR" /etc/elasticsearch/elasticsearch.yml; then + sed -i "s#path.repo:.*#path.repo: [\"$SNAPSHOT_DIR\"]#g" /etc/elasticsearch/elasticsearch.yml + RESTART_ES=true + fi +fi + +# ============================================================ +# RESTART ELASTICSEARCH +# ============================================================ +log "[3/6] Restarting Elasticsearch if required..." +if [ "$RESTART_ES" = true ]; then + systemctl restart elasticsearch + sleep 5 +else + echo "Elasticsearch restart not required." | tee -a "$EXPORT_LOG" +fi + +# ============================================================ +# REGISTER SNAPSHOT REPO +# ============================================================ +curl -s -X PUT "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$SNAPSHOT_DIR\"}}" \ + >> "$EXPORT_LOG" 2>&1 || true + +# ============================================================ +# CREATE SNAPSHOT +# ============================================================ +log "Creating local snapshot: $SNAP_NAME (indices=acm_syslog-*)" + +curl -s -X PUT \ + "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}/${SNAP_NAME}?wait_for_completion=true" \ + -H "Content-Type: application/json" \ + -d "{\"indices\":\"acm_syslog-*\",\"ignore_unavailable\":true,\"include_global_state\":false}" \ + >> "$EXPORT_LOG" 2>&1 + +if tail -n 10 "$EXPORT_LOG" | grep -qi '"error"'; then + fail "Snapshot creation failed" +fi + +log "Snapshot created successfully: $SNAP_NAME" + +# ============================================================ +# REMOTE EXPORT (optional) +# ============================================================ +if [[ -n "$REMOTE_IP" ]]; then + command -v sshpass >/dev/null 2>&1 || fail "sshpass not installed" + command -v rsync >/dev/null 2>&1 || fail "rsync not installed" + + log "Transferring snapshot to ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PATH}" + + sshpass -p "$REMOTE_PASS" rsync -av \ + -e "ssh -o StrictHostKeyChecking=no" \ + "$SNAPSHOT_DIR/" "$REMOTE_USER@$REMOTE_IP:${REMOTE_PATH}/" \ + >> "$EXPORT_LOG" 2>&1 || fail "rsync failed" + + log "Snapshot transferred to remote successfully" +fi + +# ============================================================ +# DONE +# ============================================================ +log "===============================================" +log " SNAPSHOT CREATED SUCCESSFULLY" +log " Snapshot name: $SNAP_NAME" +log "===============================================" +exit 0 \ No newline at end of file Index: /branches/amp_3_7_2/scripts/import_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/import_es_snapshot.sh (nonexistent) +++ /branches/amp_3_7_2/scripts/import_es_snapshot.sh (working copy) @@ -0,0 +1,223 @@ +#!/bin/bash +set -uo pipefail + +# --------------------------------------------------------- +# import_es_snapshot.sh +# +# Usage: +# import_es_snapshot.sh --ip X --user X --pass X [--start X --end X] +# [--path X] +# +# Restores Elasticsearch snapshot from a remote AMP. +# If --path is provided → use primary / secondary / custom snapshot directory. +# --------------------------------------------------------- + +# ----------------------------------------------- +# Defaults +# ----------------------------------------------- +AMP_IP="" +AMP_USER="" +AMP_PASS="" +START_DATE="" +END_DATE="" +DIR="/var/backups/es_snapshots" +TMP_LIST="/tmp/es_idx_list.json" +LOG_FILE="/var/log/es_import.log" +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +LOCAL_SNAPSHOT_DIR="/var/backups/es_snapshots" + +log() { echo "$(date '+%F %T') $1" | tee -a "$LOG_FILE"; } +fail() { log "ERROR: $1"; exit 1; } + +# ----------------------------------------------- +# Parse CLI arguments +# ----------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --ip) AMP_IP="$2"; shift 2 ;; + --user) AMP_USER="$2"; shift 2 ;; + --pass) AMP_PASS="$2"; shift 2 ;; + --start) START_DATE="$2"; shift 2 ;; + --end) END_DATE="$2"; shift 2 ;; + --path) DIR="$2"; shift 2 ;; + *) + echo "Unknown parameter: $1" + exit 1 + ;; + esac +done + +# ----------------------------------------------- +# Validate mandatory args +# ----------------------------------------------- +if [[ -z "$AMP_IP" || -z "$AMP_USER" || -z "$AMP_PASS" ]]; then + fail "Missing required arguments: --ip, --user, --pass" +fi + +log "Restoring snapshot from remote AMP directory: $DIR" + +# ----------------------------------------------- +# STEP 1 — Detect snapshot name on remote AMP +# ----------------------------------------------- +log "[1/9] Detecting snapshot name on source AMP..." + +INDEX_FILE=$(sshpass -p "$AMP_PASS" ssh -o StrictHostKeyChecking=no \ + "$AMP_USER@$AMP_IP" "ls -1 $DIR/index-* 2>/dev/null | head -1" +) + +[[ -z "$INDEX_FILE" ]] && fail "No index-* file found on remote AMP" + +SNAP_NAME=$(sshpass -p "$AMP_PASS" ssh -o StrictHostKeyChecking=no \ + "$AMP_USER@$AMP_IP" "grep -Po '\"name\"\\s*:\\s*\"\\K[^\"]+' $INDEX_FILE | head -1" +) + +[[ -z "$SNAP_NAME" ]] && fail "Failed to extract snapshot name" + +log "Snapshot detected: $SNAP_NAME" + +# ---------------------------------------------------------- +# STEP 2 — Clean local snapshot directory +# ---------------------------------------------------------- +log "[2/9] Cleaning local snapshot directory..." + +if [[ -z "${LOCAL_SNAPSHOT_DIR:-}" || "$LOCAL_SNAPSHOT_DIR" == "/" ]]; then + fail "LOCAL_SNAPSHOT_DIR is invalid. Aborting to prevent system damage." +fi + +rm -rf "${LOCAL_SNAPSHOT_DIR:?}/"* 2>>"$LOG_FILE" || fail "Failed to clean local snapshot directory" +mkdir -p "$LOCAL_SNAPSHOT_DIR" +chown -R elasticsearch:elasticsearch "$LOCAL_SNAPSHOT_DIR" +chmod -R 755 "$LOCAL_SNAPSHOT_DIR" + +# ----------------------------------------------- +# STEP 3 — Copy snapshot directory from remote AMP +# ----------------------------------------------- +log "[3/9] Copying snapshot directory..." + +sshpass -p "$AMP_PASS" rsync -avz \ + -e "ssh -o StrictHostKeyChecking=no" \ + "$AMP_USER@$AMP_IP:$DIR/" "$LOCAL_SNAPSHOT_DIR/" >> "$LOG_FILE" 2>&1 || fail "rsync failed" + +log "Snapshot directory copied successfully" + +# ----------------------------------------------- +# STEP 4 — Register ES repository +# ----------------------------------------------- +log "[4/9] Registering snapshot repository in Elasticsearch" + +curl -s -XPUT "http://localhost:9200/_snapshot/amp_restore_repo" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$LOCAL_SNAPSHOT_DIR\"}}" \ + >> "$LOG_FILE" 2>&1 + +log "Repository registered" + +# ---------------------------------------------------------- +# STEP 5 — Set path.repo and restart Elasticsearch +# ---------------------------------------------------------- +log "[5/9] Setting path.repo and restarting Elasticsearch..." + +if ! grep -q "path.repo" /etc/elasticsearch/elasticsearch.yml 2>/dev/null; then + echo "path.repo: [\"$LOCAL_SNAPSHOT_DIR\"]" >> /etc/elasticsearch/elasticsearch.yml \ + || fail "Failed to write path.repo" + log "Added path.repo to elasticsearch.yml" +else + if ! grep -q "$LOCAL_SNAPSHOT_DIR" /etc/elasticsearch/elasticsearch.yml; then + sed -i "s#path.repo:.*#path.repo: [\"$LOCAL_SNAPSHOT_DIR\"]#g" /etc/elasticsearch/elasticsearch.yml + log "Updated path.repo to include $LOCAL_SNAPSHOT_DIR" + fi +fi + +systemctl restart elasticsearch >> "$LOG_FILE" 2>&1 || fail "Elasticsearch restart failed" +sleep 5 + +# ----------------------------------------------- +# STEP 6 — Extract index list +# ----------------------------------------------- +log "[6/9] Extracting index list..." + +curl -s "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME" \ + | sed -n 's/.*"indices":\[\(.*\)\].*/\1/p' \ + | tr ',' '\n' | tr -d '"' > "$TMP_LIST" + +[[ ! -s "$TMP_LIST" ]] && fail "Failed to retrieve index list" + +log "Index list extracted" + +# ----------------------------------------------- +# STEP 7 — DATE RANGE RESTORE (optional) +# ----------------------------------------------- +if [[ -n "$START_DATE" || -n "$END_DATE" ]]; then + [[ -n "$END_DATE" && -z "$START_DATE" ]] && fail "End date requires start date" + [[ -n "$START_DATE" && -z "$END_DATE" ]] && END_DATE=$(date +"%Y-%m-%d") + + log "[7/9] Date-range restore: $START_DATE → $END_DATE" + + while read INDEX; do + IDX_DATE=$(echo "$INDEX" | grep -o '[0-9]\{8\}') + [[ -z "$IDX_DATE" ]] && continue + + IDX_DATE_FMT="${IDX_DATE:0:4}.${IDX_DATE:4:2}.${IDX_DATE:6:2}" + + if [[ "$IDX_DATE_FMT" < "$START_DATE" || "$IDX_DATE_FMT" > "$END_DATE" ]]; then + continue + fi + + if curl -s -I "http://localhost:9200/${INDEX}_restored_${TIMESTAMP}" | grep -q "200"; then + log "Skipping (already exists): $INDEX" + continue + fi + + log "Restoring index: $INDEX" + + curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ + -H "Content-Type: application/json" \ + -d "{ + \"indices\": \"$INDEX\", + \"ignore_unavailable\": true, + \"include_global_state\": false, + \"rename_pattern\": \"(.+)\", + \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" + }" >> "$LOG_FILE" 2>&1 + + sleep 2 + done < "$TMP_LIST" + + log "DATE RANGE RESTORE COMPLETED" + exit 0 +fi + +# ----------------------------------------------- +# STEP 8 — FULL RESTORE PASS 1 — acm_* (only) +# ----------------------------------------------- +log "[8/9] FULL RESTORE PASS 1 — acm_*" + +curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ + -H "Content-Type: application/json" \ + -d "{ + \"indices\": \"acm_*\", + \"ignore_unavailable\": true, + \"include_global_state\": false, + \"rename_pattern\": \"(.+)\", + \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" + }" >> "$LOG_FILE" 2>&1 + +sleep 5 + +# ----------------------------------------------- +# STEP 9 — FULL RESTORE PASS 2 — everything except acm_* +# ----------------------------------------------- +log "[9/9] FULL RESTORE PASS 2 — non-acm" + +curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ + -H "Content-Type: application/json" \ + -d "{ + \"indices\": \"*,-acm_*\", + \"ignore_unavailable\": true, + \"include_global_state\": false, + \"rename_pattern\": \"(.+)\", + \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" + }" >> "$LOG_FILE" 2>&1 + +log "SNAPSHOT RESTORED SUCCESSFULLY" +exit 0 \ No newline at end of file Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/djproject/urls.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/djproject/urls.py (revision 2752) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/djproject/urls.py (working copy) @@ -18,6 +18,7 @@ from hive.controller.restore_controller import handle_restore_req from hive.controller.utils import handle_observability_status_req, handle_observability_restart_req from hive.controller.snmpv3_controller import handle_snmpv3_req +from hive.controller.snapshot_controller import handle_snapshot_req js_info_dict = { #'packages': ('your.app.package',), @@ -70,6 +71,7 @@ url(r'^backup/(?P.*)$', handle_backup_req), url(r'^restore/(?P.*)$', handle_restore_req), url(r'^snmpv3/(?P.*)$', handle_snmpv3_req), + url(r'^snapshot/(?P.*)$', handle_snapshot_req), url(r'^real_service$', real_service), url(r'^rs_block$', rs_block), url(r'^rs_block_import$', rs_block_import), Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py (nonexistent) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py (working copy) @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- + +import json +import os +import datetime +from django.http import HttpResponse + +from hive.services.snapshot_service import SnapshotService +from cm.lib.libbasic_operation import oper_log +from hive.utils import andebug + +LOG_FILE_IMPORT = "/var/log/es_import.log" +LOG_FILE_EXPORT = "/var/log/es_export.log" + + +def json_response(data, status=200): + return HttpResponse(json.dumps(data), content_type="application/json", status=status) + + +# ============================================================ +# ROUTER +# ============================================================ +def handle_snapshot_req(request, path=None): + andebug('an.model.cli', 'Inside handle_snapshot_req') + try: + if request.method == "POST": + if path == "export": + return export_snapshot(request) + elif path == "import": + andebug('an.model.cli', 'Inside path == import') + return import_snapshot(request) + + elif request.method == "GET": + if path == "import/status": + return get_import_status() + elif path == "export/status": + return get_export_status() + + return json_response({"error": 400, "message": "Invalid HTTP method or path"}, 400) + + except Exception as e: + oper_log("error", "system", "Snapshot handler exception: {}".format(str(e))) + return json_response({"error": 500, "message": "Internal Snapshot Controller Error"}, 500) + + +# ============================================================ +# EXPORT +# ============================================================ +def export_snapshot(request): + if request.method != "POST": + return json_response({"error": "POST required"}, 405) + + try: + body = json.loads(request.body) if request.body else {} + except: + return json_response({"error": "Invalid JSON"}, 400) + + result = SnapshotService.run_export( + location=body.get("location", "primary"), + ip=body.get("ip"), + username=body.get("username"), + password=body.get("password"), + path=body.get("path") + ) + + if result.get("return_code") == 0: + return json_response({"message": "Snapshot export completed successfully"}) + elif result.get("return_code") == 1: + return json_response({"message": "Error exporting snapshot: {}".format(result)}) + else: + oper_log("error", "system", "Error exporting snapshot: {}".format(result)) + return json_response({"error": "Error exporting snapshot"}, 500) + + +# ============================================================ +# IMPORT +# ============================================================ +def import_snapshot(request): + andebug("an.model.cli", "Inside import_snapshot") + + try: + body = json.loads(request.body) + except: + return json_response({"error": "Invalid JSON"}, 400) + + start_date = body.get("start_date") + end_date = body.get("end_date") + andebug("an.model.cli", "start_date: {}".format(start_date)) + andebug("an.model.cli", "end_date: {}".format(end_date)) + + # DATE RULES + if end_date and not start_date: + return json_response({"error": "End date requires start date"}, 400) + + if start_date and not end_date: + end_date = datetime.datetime.now().strftime("%Y-%m-%d") + + result = SnapshotService.run_import( + location=body.get("location", "primary"), + ip=body.get("ip"), + username=body.get("username"), + password=body.get("password"), + path=body.get("path"), + start_date=start_date, + end_date=end_date + ) + + if result.get("return_code") == 0: + return json_response({"message": "Snapshot import completed successfully"}) + elif result.get("return_code") == 1: + return json_response({"message": "Error exporting snapshot: {}".format(result)}) + else: + oper_log("error", "system", "Error importing snapshot: {}".format(result)) + return json_response({"error": "Error importing snapshot"}, 500) + + +# ============================================================ +# STATUS HANDLERS +# ============================================================ +def get_import_status(): + if not os.path.exists(LOG_FILE_IMPORT): + return json_response({"error": "Import log not found"}, 404) + + try: + with open(LOG_FILE_IMPORT) as f: + text = "".join(f.readlines()[-10:]) + + if "ERROR" in text or "exception" in text.lower(): + return json_response({"status": "Error"}) + + if "SNAPSHOT RESTORED SUCCESSFULLY" in text: + return json_response({"status": "Successful"}) + + return json_response({"status": "In Progress"}) + + except Exception as e: + oper_log("error", "system", "Import status exception: {}".format(str(e))) + return json_response({"error": "Error fetching import status"}, 500) + + +def get_export_status(): + if not os.path.exists(LOG_FILE_EXPORT): + return json_response({"status": "unknown", "message": "Export log not found"}, 404) + + try: + with open(LOG_FILE_EXPORT, "r") as f: + text = "".join(f.readlines()[-10:]) + + if "ERROR" in text: + return json_response({"status": "Error"}) + + if "SNAPSHOT CREATED SUCCESSFULLY" in text: + return json_response({"status": "Successful"}) + + return json_response({"status": "In Progress"}) + + except Exception as e: + oper_log("error", "system", "Export status exception: {}".format(str(e))) + return json_response({"error": "Error fetching export status"}, 500) \ No newline at end of file Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/backup_service.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (revision 2751) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/backup_service.py (working copy) @@ -84,7 +84,7 @@ # Backup system logs backup_logger.info("Backing up system logs...") exclude_dirs = ["elasticsearch", "composer*.log", "vmware-network*.log", "yum*.log", "backup.log" - , "restore.log"] + , "restore.log"] exclude_flags = " ".join(["--exclude=" + d for d in exclude_dirs]) ServiceUtils.run_command("rsync -av {} /var/log/ {}/logs/".format(exclude_flags, TMP_BACKUP_DIR), backup_logger) @@ -121,7 +121,6 @@ # Update success status in DB self.update_backups_db(BACKUP_FILE, "Completed", 'update') - # Backup PostgreSQL """ Taking backup of postgres DB after updating DB with the Success message. So that during restore the DB contains correct data """ @@ -247,7 +246,7 @@ return HttpResponse(json.dumps({"status": "Completed", "details": "No Backup in progress!"}), - content_type="application/json") + content_type="application/json") except Exception as e: oper_log('error', 'system', "Exception while fetching backup status.") message = str(e.message).replace("'", "") @@ -265,4 +264,4 @@ args = parser.parse_args() backup = BackupConfig() - backup.perform_backup(destination=args.backup_target) \ No newline at end of file + backup.perform_backup(destination=args.backup_target) Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/snapshot_service.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/snapshot_service.py (nonexistent) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/services/snapshot_service.py (working copy) @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +import subprocess +import datetime + +from hive.services.log_location_service import get_secondary_drive_info +from hive.utils import andebug +from hive.util import constants as const + + +class SnapshotService(object): + + # ============================================================ + # DATE NORMALIZATION + # ============================================================ + @staticmethod + def normalize_date(date_str): + if not date_str: + return None + + ds = date_str.replace("-", "").replace("/", "").replace(".", "") + if len(ds) != 8 or not ds.isdigit(): + raise ValueError("Invalid date format") + + year = int(ds[:4]) + + # DDMMYYYY → swap + if year < 1900: + ds = ds[4:] + ds[2:4] + ds[:2] + + return "{}.{}.{}".format(ds[0:4], ds[4:6], ds[6:8]) + + # ============================================================ + # INTERNAL RUNNER + # ============================================================ + @staticmethod + def _run(cmd): + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + + try: + out = out.decode() + except: + pass + try: + err = err.decode() + except: + pass + + return {"return_code": proc.returncode, "stdout": out, "stderr": err} + + except Exception as e: + return {"return_code": 1, "error": str(e)} + + # ============================================================ + # EXPORT + # ============================================================ + @staticmethod + def run_export(location, ip=None, username=None, password=None, path=None): + + andebug("an.model.cli", "Inside run_export") + + cmd = ["/ca/bin/export_es_snapshot.sh"] + + # PRIMARY + if location == "primary": + cmd += ["--local_path", const.PRIMARY_ES_SNAPSHOT_LOC] + return SnapshotService._run(cmd) + + # SECONDARY + if location == "secondary": + info = get_secondary_drive_info() + + if not info.get("is_disk_available"): + return {"return_code": 1, "error": "Secondary disk unavailable"} + if not info.get("is_mounted"): + return {"return_code": 1, "error": "Secondary disk not mounted"} + + cmd += ["--local_path", const.SECONDARY_ES_SNAPSHOT_LOC] + return SnapshotService._run(cmd) + + # CUSTOM: remote server export + if location == "custom": + # Must provide remote credentials + if not (ip and username and password): + return {"return_code": 1, "error": "ip/username/password required for remote"} + + # User should provide custom remote path + if not path: + return {"return_code": 1, "error": "Path required for custom location"} + cmd += [ + "--ip", ip, + "--user", username, + "--pass", password, + "--remote_path", path + ] + return SnapshotService._run(cmd) + + return {"return_code": -1, "error": "Invalid location"} + + # ============================================================ + # IMPORT + # ============================================================ + @staticmethod + def run_import(location, ip, username, password, path=None, + start_date=None, end_date=None): + + if not (ip and username and password): + return {"return_code": 1, "error": "ip/username/password required"} + + # LOCATION HANDLING + if location == "primary": + andebug("an.model.cli", "Using PRIMARY location") + path = const.PRIMARY_ES_SNAPSHOT_LOC + + elif location == "secondary": + info = get_secondary_drive_info() + if not info.get("is_disk_available"): + return {"return_code": 1, "error": "Secondary disk unavailable"} + if not info.get("is_mounted"): + return {"return_code": 1, "error": "Secondary disk not mounted"} + + path = const.SECONDARY_ES_SNAPSHOT_LOC + + elif location == "custom": + if not path: + return {"return_code": 1, "error": "Path required for custom location"} + else: + return {"return_code": 1, "error": "Invalid location"} + + andebug("an.model.cli", "Resolved path: {}".format(path)) + + # DATE NORMALIZATION + try: + start_norm = SnapshotService.normalize_date(start_date) if start_date else None + end_norm = SnapshotService.normalize_date(end_date) if end_date else None + andebug("an.model.cli", "start_norm: {}".format(start_norm)) + andebug("an.model.cli", "end_norm: {}".format(end_norm)) + except: + return {"return_code": 1, "error": "Invalid date format"} + + # BUILD COMMAND + cmd = [ + "/ca/bin/import_es_snapshot.sh", + "--ip", ip, + "--user", username, + "--pass", password, + "--path", path, + ] + + if start_norm: + cmd += ["--start", start_norm] + if end_norm: + cmd += ["--end", end_norm] + + andebug("an.model.cli", "Final import command: {}".format(cmd)) + + return SnapshotService._run(cmd) Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/util/constants.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/util/constants.py (revision 2751) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/util/constants.py (working copy) @@ -6,4 +6,8 @@ SECONDARY_LOG_LOCATION = '/secondary/elasticsearch' -ELASTIC_SEARCH_CONFIG_FILE = '/etc/elasticsearch/elasticsearch.yml' \ No newline at end of file +ELASTIC_SEARCH_CONFIG_FILE = '/etc/elasticsearch/elasticsearch.yml' + +PRIMARY_ES_SNAPSHOT_LOC = '/var/backups/es_snapshots' + +SECONDARY_ES_SNAPSHOT_LOC = '/secondary/es_snapshots' \ No newline at end of file