Index: /branches/amp_3_7_2/scripts/export_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/export_es_snapshot.sh (revision 2895) +++ /branches/amp_3_7_2/scripts/export_es_snapshot.sh (working copy) @@ -194,14 +194,27 @@ log "Snapshot created successfully: $SNAP_NAME" +# Verify snapshot state +SNAP_STATE=$(curl -s "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}/${SNAP_NAME}" | \ + tr -d '\n' | grep -oE '"state"\s*:\s*"[^"]+"' | grep -oE '"[A-Z]+"' | tr -d '"' | head -1) + +if [[ -n "$SNAP_STATE" ]]; then + log "Snapshot state: $SNAP_STATE" + if [[ "$SNAP_STATE" == "FAILED" ]]; then + fail "Snapshot creation failed (state=FAILED)" + elif [[ "$SNAP_STATE" != "SUCCESS" ]]; then + log "WARNING: Snapshot state is '$SNAP_STATE' (expected SUCCESS)" + fi +fi + # ============================================================ # CLEANUP OLD SNAPSHOTS (keep only the latest one) # ============================================================ log "Cleaning up old snapshots (keeping only: $SNAP_NAME)..." -# Get list of all snapshots in the repository +# Get list of all snapshots in the repository (handle multi-line JSON) ALL_SNAPS=$(curl -s "http://${ES_HOST}:${ES_PORT}/_snapshot/${REPO_NAME}/_all" | \ - grep -oP '"snapshot"\s*:\s*"\K[^"]+' 2>/dev/null) + tr -d '\n' | grep -oE '"snapshot"\s*:\s*"[^"]+"' | grep -oE ':"[^"]+"' | tr -d ':"' 2>/dev/null) # Delete all snapshots except the current one DELETED_COUNT=0 @@ -231,27 +244,53 @@ log "Transferring snapshot to ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_PATH}" + # Ensure remote directory exists + log "Creating remote directory if needed..." + sshpass -p "$REMOTE_PASS" ssh -o StrictHostKeyChecking=no \ + -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa \ + "$REMOTE_USER@$REMOTE_IP" "mkdir -p ${REMOTE_PATH}" 2>/dev/null + # PRIMARY: Use tar+ssh (much faster for many small files) log "Using tar+ssh for transfer (optimized for large file counts)..." - - TAR_OUTPUT=$(cd "$SNAPSHOT_DIR" && tar czf - . 2>/dev/null | \ + + # Create temp files for error capture + SSH_ERR_FILE=$(mktemp) + TAR_ERR_FILE=$(mktemp) + + # Run tar+ssh and capture PIPESTATUS properly + (cd "$SNAPSHOT_DIR" && tar czf - . 2>"$TAR_ERR_FILE") | \ sshpass -p "$REMOTE_PASS" ssh -o StrictHostKeyChecking=no \ + -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa \ -o ServerAliveInterval=60 -o ServerAliveCountMax=5 \ - "$REMOTE_USER@$REMOTE_IP" "cd ${REMOTE_PATH} && tar xzf -" 2>&1) - - TAR_RET=$? - echo "$TAR_OUTPUT" >> "$EXPORT_LOG" + "$REMOTE_USER@$REMOTE_IP" "cd ${REMOTE_PATH} && tar xzf -" 2>"$SSH_ERR_FILE" - if [ $TAR_RET -eq 0 ]; then + # Capture PIPESTATUS immediately + PIPE_STATUS=("${PIPESTATUS[@]}") + LOCAL_TAR_RET=${PIPE_STATUS[0]} + REMOTE_TAR_RET=${PIPE_STATUS[1]} + + # Read error files + TAR_ERRORS=$(cat "$TAR_ERR_FILE" 2>/dev/null) + SSH_ERRORS=$(cat "$SSH_ERR_FILE" 2>/dev/null) + rm -f "$TAR_ERR_FILE" "$SSH_ERR_FILE" + + # Log any errors + [[ -n "$TAR_ERRORS" ]] && echo "Local tar errors: $TAR_ERRORS" >> "$EXPORT_LOG" + [[ -n "$SSH_ERRORS" ]] && echo "Remote SSH/tar errors: $SSH_ERRORS" >> "$EXPORT_LOG" + + log "tar+ssh exit codes: local=$LOCAL_TAR_RET, remote=$REMOTE_TAR_RET" + + if [ $LOCAL_TAR_RET -eq 0 ] && [ $REMOTE_TAR_RET -eq 0 ]; then log "Snapshot transferred to remote successfully (tar+ssh)" else - log "ERROR: tar+ssh failed. Attempting fallback to rsync..." - echo "Tar Output: $TAR_OUTPUT" + log "ERROR: tar+ssh failed (local=$LOCAL_TAR_RET, remote=$REMOTE_TAR_RET). Attempting fallback to rsync..." + [[ -n "$TAR_ERRORS" ]] && echo "TAR Errors: $TAR_ERRORS" + [[ -n "$SSH_ERRORS" ]] && echo "SSH Errors: $SSH_ERRORS" # FALLBACK TO RSYNC (slower but more robust) if command -v rsync >/dev/null 2>&1; then RSYNC_OUTPUT=$(sshpass -p "$REMOTE_PASS" rsync -avzh --progress \ - -e "ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=5" \ + -e "ssh -o StrictHostKeyChecking=no -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa -o ServerAliveInterval=60 -o ServerAliveCountMax=5" \ "$SNAPSHOT_DIR/" "$REMOTE_USER@$REMOTE_IP:${REMOTE_PATH}/" 2>&1) RSYNC_RET=$? Index: /branches/amp_3_7_2/scripts/import_es_snapshot.sh =================================================================== --- /branches/amp_3_7_2/scripts/import_es_snapshot.sh (revision 2895) +++ /branches/amp_3_7_2/scripts/import_es_snapshot.sh (working copy) @@ -60,8 +60,6 @@ log "===============================================" log " STARTING ELASTICSEARCH SNAPSHOT IMPORT " -echo $$ > "$PID_FILE" - # ----------------------------------------------- # Parse CLI arguments # ----------------------------------------------- @@ -163,8 +161,9 @@ if [ $TAR_RET -eq 0 ] && [ $TAR_EXTRACT_RET -eq 0 ]; then log "Snapshot directory copied successfully (tar+ssh)" else - log "ERROR: tar+ssh failed. Attempting fallback to rsync..." - echo "Tar Output: $TAR_OUTPUT" + log "ERROR: tar+ssh failed (ssh=$TAR_RET, tar=$TAR_EXTRACT_RET). Attempting fallback to rsync..." + [[ -n "$SSH_ERRORS" ]] && echo "SSH Errors: $SSH_ERRORS" + [[ -n "$TAR_ERRORS" ]] && echo "TAR Errors: $TAR_ERRORS" # FALLBACK TO RSYNC (slower but more robust) if command -v rsync >/dev/null 2>&1; then @@ -190,25 +189,82 @@ log "Fixing snapshot directory ownership..." chown -R elasticsearch:elasticsearch "$IMPORT_DIR" chmod -R 755 "$IMPORT_DIR" + +# ---------------------------------------------------------- +# STEP 3.5 — Validate imported snapshot directory +# ---------------------------------------------------------- +log "[3.5/9] Validating imported snapshot directory..." + +# Check for required snapshot files +IMPORT_INDEX_FILE=$(ls -1 "$IMPORT_DIR"/index-* 2>/dev/null | head -1) +IMPORT_INDICES_DIR="$IMPORT_DIR/indices" +IMPORT_SNAP_FILE=$(ls -1 "$IMPORT_DIR"/snap-*.dat 2>/dev/null | head -1) + +# Log what was found +log "Checking import directory contents:" +log " - index-* file: ${IMPORT_INDEX_FILE:-NOT FOUND}" +log " - indices/ dir: $([ -d "$IMPORT_INDICES_DIR" ] && echo "EXISTS" || echo "NOT FOUND")" +log " - snap-*.dat: ${IMPORT_SNAP_FILE:-NOT FOUND}" + +# Validate required files exist +if [[ -z "$IMPORT_INDEX_FILE" ]]; then + log "ERROR: No index-* file found in import directory" + log "This usually means the source snapshot directory is empty or corrupted" + ls -la "$IMPORT_DIR" >> "$LOG_FILE" 2>&1 + fail "Invalid snapshot: missing index-* file in $IMPORT_DIR" +fi + +if [[ ! -d "$IMPORT_INDICES_DIR" ]]; then + log "ERROR: No indices/ directory found in import directory" + log "This means the snapshot has no index data" + fail "Invalid snapshot: missing indices/ directory in $IMPORT_DIR" +fi + +if [[ -z "$IMPORT_SNAP_FILE" ]]; then + log "ERROR: No snap-*.dat file found in import directory" + fail "Invalid snapshot: missing snap-*.dat file in $IMPORT_DIR" +fi + +# Verify the snapshot name matches what we extracted from remote +IMPORT_SNAP_NAME=$(grep -Po '"name"\s*:\s*"\K[^"]+' "$IMPORT_INDEX_FILE" | tail -1) +if [[ "$IMPORT_SNAP_NAME" != "$SNAP_NAME" ]]; then + log "WARNING: Snapshot name mismatch!" + log " Expected (from remote): $SNAP_NAME" + log " Found (in import): $IMPORT_SNAP_NAME" + # Use the name from the imported files + SNAP_NAME="$IMPORT_SNAP_NAME" + log "Using imported snapshot name: $SNAP_NAME" +fi + +log "Import directory validated successfully" # ---------------------------------------------------------- # STEP 4 — Set path.repo and restart Elasticsearch # ---------------------------------------------------------- log "[4/9] Setting path.repo and restarting Elasticsearch..." +RESTART_ES=false + if ! grep -q "path.repo" /etc/elasticsearch/elasticsearch.yml 2>/dev/null; then echo "path.repo: [\"$LOCAL_SNAPSHOT_DIR\"]" >> /etc/elasticsearch/elasticsearch.yml \ || fail "Failed to write path.repo" log "Added path.repo to elasticsearch.yml" + RESTART_ES=true else if ! grep -q "$LOCAL_SNAPSHOT_DIR" /etc/elasticsearch/elasticsearch.yml; then sed -i "s#path.repo:.*#path.repo: [\"$LOCAL_SNAPSHOT_DIR\"]#g" /etc/elasticsearch/elasticsearch.yml log "Updated path.repo to include $LOCAL_SNAPSHOT_DIR" + RESTART_ES=true fi fi -systemctl restart elasticsearch >> "$LOG_FILE" 2>&1 || fail "Elasticsearch restart failed" -sleep 5 +if [ "$RESTART_ES" = true ]; then + log "Restarting Elasticsearch (path.repo changed)..." + systemctl restart elasticsearch >> "$LOG_FILE" 2>&1 || fail "Elasticsearch restart failed" + sleep 5 +else + log "Elasticsearch restart not required (path.repo already configured)" +fi # ---------------------------------------------------------- # STEP 5 — Register snapshot repository @@ -217,15 +273,22 @@ # Wait for Elasticsearch to be ready log "Waiting for Elasticsearch cluster to be ready..." +ES_READY=0 for i in {1..30}; do HEALTH=$(curl -s "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s") if echo "$HEALTH" | grep -q '"status":"yellow"\|"status":"green"'; then log "Elasticsearch cluster is ready" + ES_READY=1 break fi sleep 5 done +if [[ "$ES_READY" -eq 0 ]]; then + log "ERROR: Elasticsearch cluster did not become ready within 150 seconds" + fail "Elasticsearch cluster health check timeout" +fi + # Retry repository registration (up to 5 attempts) REPO_REGISTERED=0 for attempt in {1..5}; do @@ -266,19 +329,88 @@ log "Cluster settings updated" # ----------------------------------------------- -# STEP 6 — Extract index list +# STEP 5.6 — Verify available snapshots # ----------------------------------------------- -log "[6/9] Extracting index list..." +log "[5.6/9] Verifying available snapshots in repository..." -curl -s "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME" \ - | sed -n 's/.*"indices":\[\(.*\)\].*/\1/p' \ - | tr ',' '\n' | tr -d '"' > "$TMP_LIST" +# Query Elasticsearch for all snapshots in the repository +ALL_SNAPS_RESPONSE=$(curl -s "http://localhost:9200/_snapshot/$REPO_NAME/_all") +echo "All snapshots response: $ALL_SNAPS_RESPONSE" >> "$LOG_FILE" -[[ ! -s "$TMP_LIST" ]] && fail "Failed to retrieve index list" +# Extract snapshot names (handle multi-line JSON) +AVAILABLE_SNAPS=$(echo "$ALL_SNAPS_RESPONSE" | tr -d '\n' | \ + grep -oE '"snapshot"\s*:\s*"[^"]+"' | grep -oE ':"[^"]+"' | tr -d ':"' | head -5) -log "Index list extracted" +if [[ -z "$AVAILABLE_SNAPS" ]]; then + log "ERROR: No snapshots found in repository" + log "The repository may be empty or corrupted" + echo "$ALL_SNAPS_RESPONSE" + fail "No snapshots available in repository '$REPO_NAME'" +fi +log "Available snapshots in repository:" +for snap in $AVAILABLE_SNAPS; do + log " - $snap" +done + +# Check if our detected snapshot is actually available +SNAP_FOUND=0 +for snap in $AVAILABLE_SNAPS; do + if [[ "$snap" == "$SNAP_NAME" ]]; then + SNAP_FOUND=1 + break + fi +done + +if [[ "$SNAP_FOUND" -eq 0 ]]; then + # Use the first available snapshot instead + FIRST_SNAP=$(echo "$AVAILABLE_SNAPS" | head -1) + log "WARNING: Detected snapshot '$SNAP_NAME' not found in repository!" + log "Using first available snapshot: $FIRST_SNAP" + SNAP_NAME="$FIRST_SNAP" +fi + +log "Will restore snapshot: $SNAP_NAME" + # ----------------------------------------------- +# STEP 6 — Extract index list +# ----------------------------------------------- +log "[6/9] Extracting index list..." + +# Get snapshot info and store for debugging +SNAP_INFO=$(curl -s "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME") +echo "Snapshot API Response: $SNAP_INFO" >> "$LOG_FILE" + +# Check if snapshot exists +if echo "$SNAP_INFO" | grep -qi '"error"'; then + log "ERROR: Snapshot not found or inaccessible" + echo "Elasticsearch Response: $SNAP_INFO" + fail "Snapshot '$SNAP_NAME' not found in repository '$REPO_NAME'" +fi + +# Extract indices - convert to single line first, then parse +# The JSON response spans multiple lines, so we need tr to flatten it +echo "$SNAP_INFO" \ + | tr -d '\n' \ + | sed -n 's/.*"indices":\[\([^]]*\)\].*/\1/p' \ + | tr ',' '\n' \ + | tr -d '"' \ + | tr -d ' ' \ + | grep -v '^$' > "$TMP_LIST" + +# Count and log indices +INDEX_COUNT=$(wc -l < "$TMP_LIST" 2>/dev/null | tr -d ' ') +log "Found $INDEX_COUNT indices in snapshot" + +if [[ ! -s "$TMP_LIST" ]] || [[ "$INDEX_COUNT" -eq 0 ]]; then + log "ERROR: No indices found in snapshot" + log "Tip: Check if snapshot contains any data. Raw response logged above." + fail "Failed to retrieve index list - snapshot may be empty" +fi + +log "Index list extracted successfully ($INDEX_COUNT indices)" + +# ----------------------------------------------- # STEP 7 — DATE RANGE RESTORE (optional) # ----------------------------------------------- if [[ -n "$START_DATE" || -n "$END_DATE" ]]; then @@ -291,7 +423,8 @@ IDX_DATE=$(echo "$INDEX" | grep -o '[0-9]\{4\}\.[0-9]\{2\}\.[0-9]\{2\}' | head -1) [[ -z "$IDX_DATE" ]] && continue - IDX_DATE_FMT="${IDX_DATE:0:4}.${IDX_DATE:4:2}.${IDX_DATE:6:2}" + # IDX_DATE is already in YYYY.MM.DD format from the regex, use directly + IDX_DATE_FMT="$IDX_DATE" if [[ "$IDX_DATE_FMT" < "$START_DATE" || "$IDX_DATE_FMT" > "$END_DATE" ]]; then continue @@ -342,7 +475,7 @@ # ----------------------------------------------- log "[8/9] FULL RESTORE PASS 1 — acm_*" -RESTORE_P1=$(curl -s -X POST "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME/_restore" \ +RESTORE_P1=$(curl -s -X POST "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME/_restore?wait_for_completion=true" \ -H "Content-Type: application/json" \ -d '{ "indices": "acm_*", @@ -356,7 +489,7 @@ if echo "$RESTORE_P1" | grep -qi '\"error\"'; then log "ERROR: Full restore pass 1 failed" - echo "Elasticsearch Response: RESTORE_P1" + echo "Elasticsearch Response: $RESTORE_P1" fail "Restore PASS 1 failed" fi @@ -367,7 +500,7 @@ # ----------------------------------------------- log "[9/9] FULL RESTORE PASS 2 — non-acm" -RESTORE_P2=$(curl -s -X POST "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME/_restore" \ +RESTORE_P2=$(curl -s -X POST "http://localhost:9200/_snapshot/$REPO_NAME/$SNAP_NAME/_restore?wait_for_completion=true" \ -H "Content-Type: application/json" \ -d '{ "indices": "-acm_*", @@ -385,6 +518,40 @@ fail "Restore PASS 2 failed" fi +# ----------------------------------------------- +# STEP 9.5 — Validate restore results +# ----------------------------------------------- +log "[9.5/10] Validating restore results..." + +# Count indices restored in each pass +P1_INDICES=$(echo "$RESTORE_P1" | tr -d '\n' | grep -oE '"indices"\s*:\s*\[[^\]]*\]' | grep -oE '\[[^\]]*\]' | tr -cd ',' | wc -c) +P1_INDICES=$((P1_INDICES + 1)) # Count commas + 1 = number of items (unless empty) +if echo "$RESTORE_P1" | grep -q '"indices":\[\]'; then P1_INDICES=0; fi + +P2_INDICES=$(echo "$RESTORE_P2" | tr -d '\n' | grep -oE '"indices"\s*:\s*\[[^\]]*\]' | grep -oE '\[[^\]]*\]' | tr -cd ',' | wc -c) +P2_INDICES=$((P2_INDICES + 1)) +if echo "$RESTORE_P2" | grep -q '"indices":\[\]'; then P2_INDICES=0; fi + +# Count shards +P1_SHARDS=$(echo "$RESTORE_P1" | tr -d '\n' | grep -oE '"successful"\s*:\s*[0-9]+' | grep -oE '[0-9]+' | head -1) +P2_SHARDS=$(echo "$RESTORE_P2" | tr -d '\n' | grep -oE '"successful"\s*:\s*[0-9]+' | grep -oE '[0-9]+' | head -1) + +TOTAL_INDICES=$((P1_INDICES + P2_INDICES)) +TOTAL_SHARDS=$((${P1_SHARDS:-0} + ${P2_SHARDS:-0})) + +log "Restore results: PASS 1 = $P1_INDICES indices, $P1_SHARDS shards" +log "Restore results: PASS 2 = $P2_INDICES indices, $P2_SHARDS shards" +log "Total restored: $TOTAL_INDICES indices, $TOTAL_SHARDS shards" + +if [[ "$TOTAL_INDICES" -eq 0 ]] && [[ "$TOTAL_SHARDS" -eq 0 ]]; then + log "WARNING: No indices or shards were restored!" + log "This could mean:" + log " 1. The snapshot was empty" + log " 2. All indices already exist with the same names" + log " 3. There was a permission or path.repo issue" + log "Check Elasticsearch logs for more details: /var/log/elasticsearch/" +fi + # ---------------------------------------------------------- # STEP 10 — CLEANUP # ----------------------------------------------------------