Index: /branches/amp_3_7_2/extensions/auditing/webui/engine/export/export.controller.js =================================================================== --- /branches/amp_3_7_2/extensions/auditing/webui/engine/export/export.controller.js (revision 2841) +++ /branches/amp_3_7_2/extensions/auditing/webui/engine/export/export.controller.js (working copy) @@ -69,17 +69,25 @@ } engineService.exportSnapshot(payload).then(function (response) { - var msg = response.data && response.data.message; - $scope.isExporting = false; - if (msg && (msg.indexOf("Error") !== -1 || msg.indexOf("ERROR") !== -1)) { - $scope.error = msg; - } else { - $scope.message = msg; + if (response.data && response.data.status === 'Successful') { + $scope.message = response.data.message; startStatusPolling(); + } else { + $scope.isExporting = false; + var msg = response.data && response.data.message ? response.data.message : 'Export failed'; + if (response.data && response.data.details) { + msg += ': ' + response.data.details; + } + $scope.error = msg; } }, function (error) { - $scope.isExporting = false; - $scope.error = error.data && error.data.message ? error.data.message : 'Export failed'; + if (error.status === -1 || error.status === 0 || error.status === 504) { + $scope.message = 'Request is taking longer than expected. Checking status in background...'; + startStatusPolling(); + } else { + $scope.isExporting = false; + $scope.error = error.data && error.data.message ? error.data.message : 'Export failed'; + } }); } }; Index: /branches/amp_3_7_2/extensions/auditing/webui/engine/import/import.controller.js =================================================================== --- /branches/amp_3_7_2/extensions/auditing/webui/engine/import/import.controller.js (revision 2841) +++ /branches/amp_3_7_2/extensions/auditing/webui/engine/import/import.controller.js (working copy) @@ -64,17 +64,25 @@ } engineService.importSnapshot(payload).then(function (response) { - var msg = response.data && response.data.message; - $scope.isImporting = false; - if (msg && (msg.indexOf("Error") !== -1 || msg.indexOf("ERROR") !== -1)) { - $scope.error = msg; - } else { - $scope.message = msg; + if (response.data && response.data.status === 'Successful') { + $scope.message = response.data.message; startStatusPolling(); + } else { + $scope.isImporting = false; + var msg = response.data && response.data.message ? response.data.message : 'Import failed'; + if (response.data && response.data.details) { + msg += ': ' + response.data.details; + } + $scope.error = msg; } }, function (error) { - $scope.isImporting = false; - $scope.error = error.data && error.data.message ? error.data.message : 'Import failed'; + if (error.status === -1 || error.status === 0 || error.status === 504) { + $scope.message = 'Request is taking longer than expected. Checking status in background...'; + startStatusPolling(); + } else { + $scope.isImporting = false; + $scope.error = error.data && error.data.message ? error.data.message : 'Import failed'; + } }); } }; Index: /branches/amp_3_7_2/scripts/export_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/export_es_snapshot.sh (revision 2841) +++ /branches/amp_3_7_2/scripts/export_es_snapshot.sh (working copy) @@ -107,7 +107,8 @@ # CREATE SNAPSHOT DIRECTORY # ============================================================ mkdir -p "$SNAPSHOT_DIR" || fail "Cannot create snapshot directory" -chown elasticsearch:elasticsearch "$SNAPSHOT_DIR" 2>/dev/null || true +chown -R elasticsearch:elasticsearch "$SNAPSHOT_DIR" 2>/dev/null || true +chmod -R u+rwX,g+rwX,o+rX "$SNAPSHOT_DIR" 2>/dev/null || true # ============================================================ @@ -139,23 +140,33 @@ # ============================================================ # REGISTER SNAPSHOT REPO # ============================================================ -curl -s -X PUT "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}" \ +REPO_RESPONSE=$(curl -s -X PUT "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}" \ -H "Content-Type: application/json" \ - -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$SNAPSHOT_DIR\"}}" \ - >> "$EXPORT_LOG" 2>&1 || true + -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$SNAPSHOT_DIR\"}}") +echo "$REPO_RESPONSE" >> "$EXPORT_LOG" + +if echo "$REPO_RESPONSE" | grep -qi '"error"'; then + log "ERROR: Failed to register snapshot repository" + echo "Elasticsearch Response: $REPO_RESPONSE" + fail "Repository registration failed" +fi + # ============================================================ # CREATE SNAPSHOT # ============================================================ log "Creating local snapshot: $SNAP_NAME (indices=acm_syslog-*)" -curl -s -X PUT \ +SNAP_RESPONSE=$(curl -s -X PUT \ "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}/${SNAP_NAME}?wait_for_completion=true" \ -H "Content-Type: application/json" \ - -d "{\"indices\":\"acm_syslog-*\",\"ignore_unavailable\":true,\"include_global_state\":false}" \ - >> "$EXPORT_LOG" 2>&1 + -d "{\"indices\":\"acm_syslog-*\",\"ignore_unavailable\":true,\"include_global_state\":false}") -if tail -n 10 "$EXPORT_LOG" | grep -qi '"error"'; then +echo "$SNAP_RESPONSE" >> "$EXPORT_LOG" + +if echo "$SNAP_RESPONSE" | grep -qi '"error"'; then + log "ERROR: Snapshot creation failed" + echo "Elasticsearch Response: $SNAP_RESPONSE" fail "Snapshot creation failed" fi @@ -170,12 +181,35 @@ log "Transferring snapshot to ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PATH}" - sshpass -p "$REMOTE_PASS" rsync -av \ + RSYNC_OUTPUT=$(sshpass -p "$REMOTE_PASS" rsync -av \ -e "ssh -o StrictHostKeyChecking=no" \ - "$SNAPSHOT_DIR/" "$REMOTE_USER@$REMOTE_IP:${REMOTE_PATH}/" \ - >> "$EXPORT_LOG" 2>&1 || fail "rsync failed" + "$SNAPSHOT_DIR/" "$REMOTE_USER@$REMOTE_IP:${REMOTE_PATH}/" 2>&1) + + RSYNC_RET=$? + echo "$RSYNC_OUTPUT" >> "$EXPORT_LOG" - log "Snapshot transferred to remote successfully" + if [ $RSYNC_RET -eq 0 ]; then + log "Snapshot transferred to remote successfully (rsync)" + else + log "ERROR: rsync failed. Attempting fallback to SCP..." + echo "Rsync Output: $RSYNC_OUTPUT" + + # FALLBACK TO SCP + # recursive copy of contents + SCP_OUTPUT=$(sshpass -p "$REMOTE_PASS" scp -r -o StrictHostKeyChecking=no \ + "$SNAPSHOT_DIR"/* "$REMOTE_USER@$REMOTE_IP:${REMOTE_PATH}/" 2>&1) + + SCP_RET=$? + echo "$SCP_OUTPUT" >> "$EXPORT_LOG" + + if [ $SCP_RET -eq 0 ]; then + log "Snapshot transferred to remote successfully (scp)" + else + log "ERROR: SCP fallback failed" + echo "SCP Output: $SCP_OUTPUT" + fail "Remote transfer failed (rsync and scp)" + fi + fi fi # ============================================================ Index: /branches/amp_3_7_2/scripts/import_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/import_es_snapshot.sh (revision 2841) +++ /branches/amp_3_7_2/scripts/import_es_snapshot.sh (working copy) @@ -92,14 +92,45 @@ # ----------------------------------------------- # STEP 3 — Copy snapshot directory from remote AMP # ----------------------------------------------- +# ----------------------------------------------- +# STEP 3 — Copy snapshot directory from remote AMP +# ----------------------------------------------- log "[3/9] Copying snapshot directory..." -sshpass -p "$AMP_PASS" rsync -avz \ +RSYNC_OUTPUT=$(sshpass -p "$AMP_PASS" rsync -avz \ -e "ssh -o StrictHostKeyChecking=no" \ - "$AMP_USER@$AMP_IP:$DIR/" "$LOCAL_SNAPSHOT_DIR/" >> "$LOG_FILE" 2>&1 || fail "rsync failed" + "$AMP_USER@$AMP_IP:$DIR/" "$LOCAL_SNAPSHOT_DIR/" 2>&1) -log "Snapshot directory copied successfully" +RSYNC_RET=$? +echo "$RSYNC_OUTPUT" >> "$LOG_FILE" +if [ $RSYNC_RET -eq 0 ]; then + log "Snapshot directory copied successfully (rsync)" +else + log "ERROR: rsync failed. Attempting fallback to SCP..." + echo "Rsync Output: $RSYNC_OUTPUT" + + # FALLBACK TO SCP + # recursive copy of contents + # Note: scp -r user@host:dir/* local_dir/ might behave differently depending on glob expansion on remote + # safer to scp -r user@host:dir local_dir and handle nesting or just ensure DIR ends with / + + # Trying specific scp command: + SCP_OUTPUT=$(sshpass -p "$AMP_PASS" scp -r -o StrictHostKeyChecking=no \ + "$AMP_USER@$AMP_IP:$DIR/"* "$LOCAL_SNAPSHOT_DIR/" 2>&1) + + SCP_RET=$? + echo "$SCP_OUTPUT" >> "$LOG_FILE" + + if [ $SCP_RET -eq 0 ]; then + log "Snapshot directory copied successfully (scp)" + else + log "ERROR: SCP fallback failed" + echo "SCP Output: $SCP_OUTPUT" + fail "Remote copy failed (rsync and scp)" + fi +fi + # ---------------------------------------------------------- # STEP 4 — Set path.repo and restart Elasticsearch # ---------------------------------------------------------- @@ -124,11 +155,18 @@ # ----------------------------------------------- log "[5/9] Registering snapshot repository in Elasticsearch" -curl -s -XPUT "http://localhost:9200/_snapshot/amp_restore_repo" \ +REPO_RESPONSE=$(curl -s -XPUT "http://localhost:9200/_snapshot/amp_restore_repo" \ -H "Content-Type: application/json" \ - -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$LOCAL_SNAPSHOT_DIR\"}}" \ - >> "$LOG_FILE" 2>&1 + -d "{\"type\":\"fs\",\"settings\":{\"location\":\"$LOCAL_SNAPSHOT_DIR\"}}") +echo "$REPO_RESPONSE" >> "$LOG_FILE" + +if echo "$REPO_RESPONSE" | grep -qi '\"error\"'; then + log "ERROR: Failed to register snapshot repository" + echo "Elasticsearch Response: $REPO_RESPONSE" + fail "Repository registration failed" +fi + log "Repository registered" # ----------------------------------------------- @@ -170,7 +208,7 @@ log "Restoring index: $INDEX" - curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ + RESTORE_RESPONSE=$(curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ -H "Content-Type: application/json" \ -d "{ \"indices\": \"$INDEX\", @@ -178,8 +216,16 @@ \"include_global_state\": false, \"rename_pattern\": \"(.+)\", \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" - }" >> "$LOG_FILE" 2>&1 + }") + + echo "$RESTORE_RESPONSE" >> "$LOG_FILE" + if echo "$RESTORE_RESPONSE" | grep -qi '\"error\"'; then + log "ERROR: Failed to restore index $INDEX" + echo "Elasticsearch Response: $RESTORE_RESPONSE" + # Continue or fail? Usually safer to continue best-effort but log error + fi + sleep 2 done < "$TMP_LIST" @@ -192,7 +238,7 @@ # ----------------------------------------------- log "[8/9] FULL RESTORE PASS 1 — acm_*" -curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ +RESTORE_P1=$(curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ -H "Content-Type: application/json" \ -d "{ \"indices\": \"acm_*\", @@ -200,8 +246,15 @@ \"include_global_state\": false, \"rename_pattern\": \"(.+)\", \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" - }" >> "$LOG_FILE" 2>&1 + }") +echo "$RESTORE_P1" >> "$LOG_FILE" +if echo "$RESTORE_P1" | grep -qi '\"error\"'; then + log "ERROR: Full restore pass 1 failed" + echo "Elasticsearch Response: $RESTORE_P1" + fail "Restore pass 1 failed" +fi + sleep 5 # ----------------------------------------------- @@ -209,7 +262,7 @@ # ----------------------------------------------- log "[9/9] FULL RESTORE PASS 2 — non-acm" -curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ +RESTORE_P2=$(curl -s -XPOST "http://localhost:9200/_snapshot/amp_restore_repo/$SNAP_NAME/_restore" \ -H "Content-Type: application/json" \ -d "{ \"indices\": \"*,-acm_*\", @@ -217,7 +270,14 @@ \"include_global_state\": false, \"rename_pattern\": \"(.+)\", \"rename_replacement\": \"\$1_restored_${TIMESTAMP}\" - }" >> "$LOG_FILE" 2>&1 + }") +echo "$RESTORE_P2" >> "$LOG_FILE" +if echo "$RESTORE_P2" | grep -qi '\"error\"'; then + log "ERROR: Full restore pass 2 failed" + echo "Elasticsearch Response: $RESTORE_P2" + fail "Restore pass 2 failed" +fi + log "SNAPSHOT RESTORED SUCCESSFULLY" exit 0 \ No newline at end of file Index: /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py =================================================================== --- /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py (revision 2842) +++ /branches/amp_3_7_2/src/webui/webui/htdocs/new/src/hive/controller/snapshot_controller.py (working copy) @@ -8,7 +8,6 @@ from hive.services.snapshot_service import SnapshotService from cm.lib.libbasic_operation import oper_log -from hive.utils import andebug LOG_FILE_IMPORT = "/var/log/es_import.log" LOG_FILE_EXPORT = "/var/log/es_export.log" @@ -159,34 +158,31 @@ try: with open(LOG_FILE_IMPORT) as f: - lines = f.readlines()[-10:] # read last 10 lines + lines = f.readlines()[-100:] # read last 100 lines - text = "".join(lines) + result = {"status": "In Progress"} - for line in reversed(lines): - if "error" in line.lower() or "exception" in line.lower(): + for line in lines: + line_lower = line.lower() + # Check for failures - look for "error:" to avoid matching "errors": false in JSON + # Also check for "exception" for stack traces + if "error:" in line_lower or "exception" in line_lower: ts = extract_timestamp(line) - if ts is not None: - return json_response({ - "status": "Failed", - "failed_at": ts - }) - else: - return json_response({"status": "Failed"}) + result = { + "status": "Failed", + "failed_at": ts + } - for line in reversed(lines): + # Check for success (overrides failure if it comes later) if ("SNAPSHOT RESTORED SUCCESSFULLY" in line or "DATE RANGE RESTORE COMPLETED" in line): ts = extract_timestamp(line) - if ts is not None: - return json_response({ - "status": "Successful", - "completed_at": ts - }) - else: - return json_response({"status": "Successful"}) + result = { + "status": "Successful", + "completed_at": ts + } - return json_response({"status": "In Progress"}) + return json_response(result) except Exception as e: oper_log("error", "system", "Import status exception: {}".format(str(e))) @@ -199,33 +195,29 @@ try: with open(LOG_FILE_EXPORT, "r") as f: - lines = f.readlines()[-10:] # read last 10 lines + lines = f.readlines()[-100:] # read last 100 lines - text = "".join(lines) + result = {"status": "In Progress"} - for line in reversed(lines): - if "error" in line.lower() or "exception" in line.lower(): + for line in lines: + line_lower = line.lower() + # Check for failures + if "error:" in line_lower or "exception" in line_lower: ts = extract_timestamp(line) - if ts is not None: - return json_response({ - "status": "Failed", - "failed_at": ts - }) - else: - return json_response({"status": "Failed"}) + result = { + "status": "Failed", + "failed_at": ts + } - for line in reversed(lines): + # Check for success (overrides failure if it comes later) if "SNAPSHOT CREATED SUCCESSFULLY" in line: ts = extract_timestamp(line) - if ts is not None: - return json_response({ - "status": "Successful", - "completed_at": ts - }) - else: - return json_response({"status": "Successful"}) + result = { + "status": "Successful", + "completed_at": ts + } - return json_response({"status": "In Progress"}) + return json_response(result) except Exception as e: oper_log("error", "system", "Export status exception: {}".format(str(e)))