Index: /branches/amp_4_0/platform/scripts/create_user.sh =================================================================== --- /branches/amp_4_0/platform/scripts/create_user.sh (nonexistent) +++ /branches/amp_4_0/platform/scripts/create_user.sh (working copy) @@ -0,0 +1,17 @@ +#!/bin/bash + +# Configuration +NEW_USER="admin" +NEW_PASS="admin" + +# 1. Create the user with a home directory +useradd -m "$NEW_USER" + +# 2. Set the password +# We use chpasswd for non-interactive password setting +echo "$NEW_USER:$NEW_PASS" | chpasswd + +# 3. Add user to the 'wheel' group for sudo/su access +usermod -aG wheel "$NEW_USER" + +echo "User '$NEW_USER' created successfully and added to the 'wheel' group." Index: /branches/amp_4_0/platform/tools/README.md =================================================================== --- /branches/amp_4_0/platform/tools/README.md (revision 2855) +++ /branches/amp_4_0/platform/tools/README.md (nonexistent) @@ -1,45 +0,0 @@ -### Tools directory contains the scripts to install the AMP tools. - -### Tool & Version - -* Python -> 3.13 -* Django -> 5 -* Java -> 21 -* PostgresSQL -> 17.4 -* Elastic -> 8.18.0-1 -* Logstash -> 1:8.18.0-1 -* Kibana -> 8.18.0-1 -* MetricBeats -> 8.18.0-1 -* Filebeat -> 8.18.0-1 -* InfluxDB -> 2.7.11-1 -* InfluxDB-CLI -> 2.7.5-1 -* Telegraf -> 1.34.1-1 -* Nginx -> 1.20.1 - -### Installation order - -1. Python -2. PSQL -3. ELK - 1. Configure ELK -4. INFLUX -5. TELEGRAF - -### ToDo (Best practices) - -* Use random passwords or better alternatives for the stored passwords from the script. - -### Debug - -#### InfluxDB - -By default, influxdb GUI is not exposed to the outside network; we can allow the 8086 to access the InfluxDB GUI using -the following firewall rules—it's strictly for the debugging purpose. - -sudo firewall-cmd --permanent --add-port=8086/tcp -sudo firewall-cmd --permanent --add-port=8086/udp - -sudo firewall-cmd --reload - -### Configure HyperTables for Device Metrics -psql "host=localhost port=5432 dbname=amp_ts user=amp_ts_user" -v ON_ERROR_STOP=1 -f /opt/telegraf_snmp_timescale.sql Index: /branches/amp_4_0/platform/tools/configure_elk.sh =================================================================== --- /branches/amp_4_0/platform/tools/configure_elk.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/configure_elk.sh (nonexistent) @@ -1,228 +0,0 @@ -#!/bin/bash -set -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Please run as root" - exit 1 -fi - -# --- File Configuration --- -LOG_FILE="/var/log/configure_elk.log" -SYSTEM_MODULE_FILE="/etc/filebeat/modules.d/system.yml" - -# --- Host & Credentials --- -KIBANA_HOST="http://localhost:5601" - -ELASTIC_SUPER_USER="elastic" -ELASTIC_PASSWORD="Arr@y2050" - -# --- Logging Helpers --- -log_info() { - echo "[INFO] $(date +'%Y-%m-%d %H:%M:%S') $1" | tee -a "$LOG_FILE" -} - -log_error() { - echo "[ERROR] $(date +'%Y-%m-%d %H:%M:%S') $1" >&2 | tee -a "$LOG_FILE" -} - -create_dashboard_index_patterns() { - local service="$1" - log_info "setting up the dashboards & index management for - '$service'..." - sudo "$service" setup --dashboards --index-management\ - -E setup.ilm.overwrite=true \ - -E setup.kibana.host="$KIBANA_HOST" \ - -E setup.kibana.username="$ELASTIC_SUPER_USER" \ - -E setup.kibana.password="$ELASTIC_PASSWORD" \ - -E output.elasticsearch.username="$ELASTIC_SUPER_USER" \ - -E output.elasticsearch.password="$ELASTIC_PASSWORD" \ - | tee -a "$LOG_FILE" -} - -create_data_view_logstash() { - log_info "Creating Kibana data view for Logstash..." - - local response - response=$(curl -s -X POST "$KIBANA_HOST/api/saved_objects/index-pattern" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -H 'Content-Type: application/json' \ - -H 'kbn-xsrf: true' \ - -d "{ - \"attributes\": { - \"title\": \"logstash-*\", - \"timeFieldName\": \"@timestamp\" - } - }") - - echo "$response" >> "$LOG_FILE" - - if echo "$response" | jq . >/dev/null 2>&1; then - log_info "✅ Successfully created Logstash data view." - else - log_error "❌ Failed to create Logstash data view. Raw output: $response" - exit 1 - fi -} - -create_data_view_acm() { - log_info "Creating Kibana data view for ACM..." - - local response - response=$(curl -s -X POST "$KIBANA_HOST/api/saved_objects/index-pattern" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -H 'Content-Type: application/json' \ - -H 'kbn-xsrf: true' \ - -d "{ - \"attributes\": { - \"title\": \"acm-*\", - \"timeFieldName\": \"@timestamp\" - } - }") - - echo "$response" >> "$LOG_FILE" - - if echo "$response" | jq . >/dev/null 2>&1; then - log_info "✅ Successfully created acm data view." - else - log_error "❌ Failed to create acm data view. Raw output: $response" - exit 1 - fi -} - -make_data_view_acm_default() { - CURL_AUTH_ARGS=(-u "${ELASTIC_SUPER_USER}:${ELASTIC_PASSWORD}") # Basic Auth for curl - CURL_COMMON_ARGS=(-H "kbn-xsrf: true" -H "Content-Type: application/json" --silent --fail --show-error) - - TARGET_DATAVIEW_NAME="acm-*" - TARGET_DATAVIEW_ID="acm-*" - - SET_DEFAULT_RESPONSE=$(curl -X POST "${KIBANA_HOST}/api/data_views/default" \ - "${CURL_AUTH_ARGS[@]}" \ - "${CURL_COMMON_ARGS[@]}" \ - -d '{ "data_view_id": "'"${TARGET_DATAVIEW_ID}"'", "force": true }') - - # Check if curl command itself failed - if [ $? -ne 0 ]; then - log_error "❌ Failed to set default data view. Check Kibana URL, credentials, and connectivity." >&2 - log_info "Response: ${SET_DEFAULT_RESPONSE}" >&2 - exit 1 - fi - - # Verify success from the response content using jq - ACKNOWLEDGED=$(echo "${SET_DEFAULT_RESPONSE}" | jq -r ".acknowledged") - - if [ "${ACKNOWLEDGED}" == "true" ]; then - log_info "Success: Default data view set to '${TARGET_DATAVIEW_NAME}' (ID: ${TARGET_DATAVIEW_ID})." - else - log_error "❌ Kibana API reported failure to set default data view." >&2 - log_info "Response: ${SET_DEFAULT_RESPONSE}" >&2 - exit 1 - fi -} - -clean_start_filebeat() { - log_info "Cleaning up and safely starting Filebeat..." - - systemctl stop filebeat || true | tee -a "$LOG_FILE" - - LOCK_FILE="/var/lib/filebeat/filebeat.lock" - if [[ -f "$LOCK_FILE" ]]; then - log_info "Removing stale lock file: $LOCK_FILE" - rm -f "$LOCK_FILE" | tee -a "$LOG_FILE" - fi - - SYSTEM_YML="/etc/filebeat/modules.d/system.yml" - - log_info "Overwriting $SYSTEM_YML with valid configuration..." - tee "$SYSTEM_YML" > /dev/null < MAX_HEAP_MB )) && OS_HEAP_MB=$MAX_HEAP_MB -(( DP_HEAP_MB > MAX_HEAP_MB )) && DP_HEAP_MB=$MAX_HEAP_MB - - -echo "🧠 Total RAM: ${TOTAL_MEM_MB}MB" -echo "📦 OpenSearch heap: ${OS_HEAP_MB}MB" -echo "🚰 Data Prepper heap: ${DP_HEAP_MB}MB" -echo "🖥️ Dashboards max-old-space-size: ${DASHBOARDS_MB}MB" - -# --- OpenSearch --- -if [[ -f "$OPENSEARCH_JVM_OPTS" ]]; then - cp "$OPENSEARCH_JVM_OPTS" "$OPENSEARCH_JVM_OPTS.bak.$(date +%s)" - sed -i "s/^-Xms.*/-Xms${OS_HEAP_MB}m/" "$OPENSEARCH_JVM_OPTS" - sed -i "s/^-Xmx.*/-Xmx${OS_HEAP_MB}m/" "$OPENSEARCH_JVM_OPTS" - echo "✅ OpenSearch heap updated." -else - echo "❌ OpenSearch JVM config not found." -fi - -# --- Data Prepper --- -DP_SERVICE=$(systemctl show -p FragmentPath data-prepper | cut -d'=' -f2) -if [[ -n "$DP_SERVICE" && -f "$DP_SERVICE" ]]; then - cp "$DP_SERVICE" "$DP_SERVICE.bak.$(date +%s)" - sed -i '/^Environment="DATA_PREPPER_JAVA_OPTS=/d' "$DP_SERVICE" - sed -i "/^\[Service\]/a Environment=\"DATA_PREPPER_JAVA_OPTS=-Xms${DP_HEAP_MB}m -Xmx${DP_HEAP_MB}m\"" "$DP_SERVICE" - echo "✅ Data Prepper systemd unit updated." -else - echo "❌ Data Prepper service unit not found." -fi - -if [[ -f "$DATAPREPPER_SCRIPT" ]]; then - if ! grep -q 'export JAVA_OPTS="$JAVA_OPTS $DATA_PREPPER_JAVA_OPTS"' "$DATAPREPPER_SCRIPT"; then - cp "$DATAPREPPER_SCRIPT" "$DATAPREPPER_SCRIPT.bak.$(date +%s)" - sed -i '/^#!\/bin\/bash/a export JAVA_OPTS="$JAVA_OPTS $DATA_PREPPER_JAVA_OPTS"' "$DATAPREPPER_SCRIPT" - echo "✅ Patched Data Prepper launcher script." - else - echo "✅ Data Prepper launcher already patched." - fi -else - echo "❌ Data Prepper script not found." -fi - -# --- OpenSearch Dashboards --- -DASHBOARDS_UNIT=$(systemctl show -p FragmentPath "$DASHBOARDS_SERVICE" | cut -d'=' -f2) -if [[ -n "$DASHBOARDS_UNIT" && -f "$DASHBOARDS_UNIT" ]]; then - cp "$DASHBOARDS_UNIT" "$DASHBOARDS_UNIT.bak.$(date +%s)" - # The original sed command was sufficient, but we'll ensure it overrides any existing - # NODE_OPTIONS for max-old-space-size and adds the new value. - # We should also retain other NODE_OPTIONS if they exist, but for --max-old-space-size, - # we want to ensure only our new value is present. - sed -i '/^Environment="NODE_OPTIONS=/d' "$DASHBOARDS_UNIT" # Remove any existing NODE_OPTIONS - sed -i "/^\[Service\]/a Environment=\"NODE_OPTIONS=--max-old-space-size=${DASHBOARDS_MB}\"" "$DASHBOARDS_UNIT" - echo "✅ Dashboards memory setting updated." -else - echo "❌ OpenSearch Dashboards service unit not found." -fi - -# --- Reload + Restart --- -systemctl daemon-reexec -systemctl daemon-reload - -echo "♻️ Restarting services..." -systemctl restart opensearch && echo "🚀 OpenSearch restarted." -systemctl restart data-prepper && echo "🚀 Data Prepper restarted." -systemctl restart "$DASHBOARDS_SERVICE" && echo "🚀 Dashboards restarted." - -echo "✅ All heap/memory updates applied successfully." \ No newline at end of file Index: /branches/amp_4_0/platform/tools/configure_psql.sh =================================================================== --- /branches/amp_4_0/platform/tools/configure_psql.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/configure_psql.sh (nonexistent) @@ -1,12 +0,0 @@ -#!/bin/bash - -# Source custom path changes -source /etc/profile.d/custom-path.sh - -AN_DB_NAME="cm" -AN_USER="amp_admin" -AN_USER_PASSWORD="Array@123$" - -# Import PSQL Tables for the AN provided database -PGPASSWORD="$AN_USER_PASSWORD" psql -U "$AN_USER" -d "$AN_DB_NAME" -f ./config/init_db.sql - Index: /branches/amp_4_0/platform/tools/configure_telegraf_timescale.sh =================================================================== --- /branches/amp_4_0/platform/tools/configure_telegraf_timescale.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/configure_telegraf_timescale.sh (nonexistent) @@ -1,241 +0,0 @@ -#!/bin/bash -# configure_telegraf_timescale.sh -# - No install, no repos, no tokens. -# - Validates at least one SNMP input exists in telegraf.d (apv/ag/asf). -# - Writes telegraf.conf with PostgreSQL output. -# - Restarts Telegraf. - -set -euo pipefail - -LOG_FILE="/var/log/install_telegraf.log" -TELEGRAF_CONFIG_DIR="/etc/telegraf/telegraf.d" -TELEGRAF_CONFIG="/etc/telegraf/telegraf.conf" - -# === Timescale/PostgreSQL target (adjust if needed) === -PG_HOST="127.0.0.1" -PG_PORT="5432" -PG_DB="amp_ts" -PG_USER="amp_ts_user" -PG_PASSWORD="Array@123$" # put this somewhere safer in production - -log() { echo "[$(date +'%F %T')] $*" | tee -a "$LOG_FILE" ; } -need() { command -v "$1" >/dev/null 2>&1 || { log "ERROR: $1 not found"; exit 1; }; } - -# --- sanity checks --- -for c in sudo grep sed awk paste telegraf systemctl; do need "$c"; done -for d in "/var/log" "/etc/telegraf" "$TELEGRAF_CONFIG_DIR"; do - if [[ ! -d "$d" ]]; then - log "WARN: $d missing, creating..." - sudo mkdir -p "$d" - fi -done - -# --- require at least one SNMP input file in telegraf.d --- -SNMP_FILES=( - "/etc/telegraf/telegraf.d/apv.conf" - "/etc/telegraf/telegraf.d/ag.conf" - "/etc/telegraf/telegraf.d/asf.conf" -) - -FOUND_SNMP=false -for f in "${SNMP_FILES[@]}"; do - if [[ -f "$f" ]]; then - FOUND_SNMP=true - # quick validation (best-effort) - grep -qE '^[[:space:]]*agents[[:space:]]*=' "$f" || { log "ERROR: 'agents = [...]' not found in $f"; exit 1; } - grep -qm1 -E '^[[:space:]]*community[[:space:]]*=' "$f" || { log "ERROR: 'community = \"...\"' not found in $f"; exit 1; } - grep -qm1 -E '^[[:space:]]*timeout[[:space:]]*=' "$f" || { log "ERROR: 'timeout = \"...\"' not found in $f"; exit 1; } - log "$(basename "$f") looks OK (agents/community/timeout present)." - fi -done -$FOUND_SNMP || { log "ERROR: no SNMP input file found (expected one of: ${SNMP_FILES[*]})"; exit 1; } - -# --- backup any existing main config --- -if [[ -f "$TELEGRAF_CONFIG" ]]; then - sudo cp -a "$TELEGRAF_CONFIG" "${TELEGRAF_CONFIG}.bak.$(date +%s)" - log "Backed up ${TELEGRAF_CONFIG}." -fi - -# --- write a clean main config that targets Timescale/PostgreSQL --- -log "Writing ${TELEGRAF_CONFIG}..." -sudo tee "$TELEGRAF_CONFIG" >/dev/null </dev/null 2>&1; then - log "ERROR: telegraf --test failed. Inspect output with:" - log " telegraf --config $TELEGRAF_CONFIG --config-directory $TELEGRAF_CONFIG_DIR --test" - exit 1 -fi -log "telegraf --test passed." - -# --- restart telegraf --- -log "Restarting Telegraf..." -sudo systemctl restart telegraf || true -sudo systemctl status telegraf --no-pager -l | sed -n '1,40p' || true - -if systemctl is-active --quiet telegraf; then - log "Telegraf is active." -else - log "ERROR: Telegraf failed to start. See: journalctl -u telegraf -n 200 --no-pager" - exit 1 -fi - -log "Done." Index: /branches/amp_4_0/platform/tools/container/.env =================================================================== --- /branches/amp_4_0/platform/tools/container/.env (revision 2855) +++ /branches/amp_4_0/platform/tools/container/.env (working copy) @@ -1,11 +1,19 @@ -# .env file for AMP Docker Stack +# ------------------------------------------------------------------ +# Brand: Array Networks (AN) +# Product: Array Management Platform (AMP) +# Purpose: Environment Configuration for Docker Swarm Stack +# ------------------------------------------------------------------ -# OpenSearch Credentials +# --- OpenSearch (AMP Analytics Store) --- OPENSEARCH_INITIAL_ADMIN_PASSWORD=Arr@y2050 -OPENSEARCH_JAVA_OPTS="-Xms1g -Xmx1g" OPENSEARCH_JWT_SECRET="Arr@y2050" -# TimescaleDB / Postgres Credentials +# Memory Allocation: +# By default, manage_amp.sh calculates this based on 50% of Host RAM. +# Uncomment to override (e.g. for testing constraints). +# OPENSEARCH_JAVA_OPTS="-Xms1g -Xmx1g" + +# --- TimescaleDB / Postgres (AMP Metrics Store) --- POSTGRES_USER=postgres POSTGRES_PASSWORD=Arr@y2050 POSTGRES_DB=postgres @@ -13,14 +21,30 @@ AMP_DB_USER=amp_ts_user AMP_DB_PASSWORD=Array@123$ -# Grafana +# --- Grafana (AMP Visualization) --- GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=GArr@y2050 GF_SERVER_ROOT_URL=https://localhost/monitoring/ GF_SERVER_SERVE_FROM_SUB_PATH=true -# Logstash -LOGSTASH_JAVA_OPTS="-Xms1g -Xmx1g" +# --- Logstash (AMP Ingest Pipeline) --- +# Memory Allocation: +# By default, manage_amp.sh calculates this based on 25% of Host RAM. +# Uncomment to override. +# LOGSTASH_JAVA_OPTS="-Xms1g -Xmx1g" +# --- Ports Configuration --- +AMP_NGINX_HTTP_PORT=80 +AMP_NGINX_HTTPS_PORT=443 +AMP_GRAFANA_PORT=3000 +AMP_TIMESCALEDB_PORT=5432 + +# --- Network Configuration --- # Host IP for Nginx to reach backend on host HOST_BACKEND_IP=host.docker.internal + +# --- General Domain/IP Configuration --- +# The IP address or Domain Name used to access AMP from the browser. +# This MUST match the URL in your browser to avoid Grafana auth loops. +# Uncomment to override auto-detection (e.g. for specific domain or IP). +# AMP_DOMAIN_OR_IP=192.168.162.139 Index: /branches/amp_4_0/platform/tools/container/ARCHITECTURE.md =================================================================== --- /branches/amp_4_0/platform/tools/container/ARCHITECTURE.md (nonexistent) +++ /branches/amp_4_0/platform/tools/container/ARCHITECTURE.md (working copy) @@ -0,0 +1,89 @@ +# System Architecture: AMP Platform on Docker Swarm + +## 1. Executive Summary + +This document details the architecture of the AMP Platform deployed on a 3-Node Docker Swarm cluster. The current design utilizes a **Hybrid High Availability (HA)** model: it provides full redundancy for network ingress and stateless processing, while pinning stateful storage to a designated primary node to ensure data consistency without external shared storage dependencies (NFS/SAN). + +## 2. Deployment Topology + +The cluster consists of three nodes participating in a Docker Swarm. + +### Node Roles + +| Node | Hostname | Swarm Role | Labels | Primary Responsibility | +| :--- | :--- | :--- | :--- | :--- | +| **Node 1** | `amp-node-1` | **Manager (Leader)** | `type=storage` | **Storage + Compute**. Hosts the database files (OpenSearch, TimescaleDB, Registry) on local high-speed disk. | +| **Node 2** | `amp-node-2` | **Manager/Worker** | `type=compute` | **Compute Only**. Runs stateless services (Logstash, Nginx, Dashboards). | +| **Node 3** | `amp-node-3` | **Manager/Worker** | `type=compute` | **Compute Only**. Runs stateless services (Logstash, Nginx, Dashboards). | + +### Component Diagram + +```mermaid +graph TD + Client["Client / Log Sources"] --> VIP["Virtual IP (Keepalived)"] + + subgraph cluster_swarm ["Docker Swarm Cluster"] + VIP --> Nginx_Service["Nginx (Web Port 443)"] + VIP --> Logstash["Logstash (Syslog Port 514)"] + + subgraph cluster_stateless ["Stateless Layer (Any Node)"] + Nginx_Service --> Dashboards + Nginx_Service --> Grafana + end + + subgraph cluster_stateful ["Stateful Layer (Node 1 Only)"] + Logstash --> OpenSearch[("OpenSearch Data")] + Grafana --> Timescale[("TimescaleDB Data")] + end + end +``` + +## 3. High Availability (HA) Strategy + +### 3.1 Network Layer (Ingress) + +* **Technology**: `Keepalived` (VRRP). +* **Mechanism**: A floating Virtual IP (VIP) is assigned to the active Leader (Node 1). If Node 1 fails, the VIP automatically migrates to Node 2 or Node 3 within seconds. +* **Benefit**: External systems (Syslog senders, User Browsers) never need to reconfigure IPs. The "Front Door" is always open. + +### 3.2 Compute Layer (Stateless Services) + +* **Services**: `nginx`, `logstash`, `opensearch-dashboards`, `grafana`. +* **Mechanism**: Docker Swarm Orchestration. +* **Behavior**: These services are not pinned. If a node fails, Swarm reschedules replicas to remaining healthy nodes. +* **Benefit**: Continuous request processing. Dashboards and Ingestion endpoints remain accessible. + +### 3.3 Storage Layer (Stateful Services) + +* **Services**: `opensearch`, `timescaledb`. +* **Mechanism**: Pinned placement (`constraints: - node.labels.type == storage`) using Local Docker Volumes (`driver: local`). +* **Behavior**: These services MUST run on Node 1 because their data files exist physically on Node 1's disk. They cannot start on other nodes. + +## 4. Architectural Analysis + +### 4.1 Advantages + +1. **High Performance (I/O)**: By using local disks (NVMe/SSD) on Node 1 instead of NFS, database write speeds are maximized. This is critical for high-volume log ingestion. +2. **Stateless Scalability**: Heavy processing tasks (Log Parsing via Logstash, SSL Termination via Nginx) are distributed across 3 nodes. Node 1 is not overwhelmed by CPU tasks, leaving it free to handle Database I/O. +3. **Simplicity**: No requirement for external NAS, SAN, or complex Ceph/GlusterFS configurations. Reduced maintenance overhead. +4. **Partial Failover**: In the event of a Node 1 failure, the UI remains accessible (with connection errors) and VIP remains pingable, preventing "Connection Refused" errors on client side. + +### 4.2 Limitations & Risks + +1. **Single Point of Failure (Data)**: If Node 1 fails, **OpenSearch and TimescaleDB go offline**. No new logs can be written, and no historical data can be queried. +2. **Data Loss Risk (Buffer Overflow)**: While Logstash (on surviving nodes) will buffer incoming logs in memory, it will eventually fill up and start dropping data if Node 1 does not recover quickly. +3. **Manual Recovery**: If Node 1 has a hardware failure, recovering the data requires restoring from backups, as the data does not exist on Nodes 2/3. + +## 5. Solution: Path to Full HA + +To mitigate the storage limitation and achieve **Zero Downtime**, the cluster can be upgraded to use **Shared Storage**. + +### Implementation: NFS Migration + +* **Dependency**: An external NFS Server accessible by all 3 nodes. +* **Configuration**: + 1. Update `stack.yml` volumes to use `driver_opts: type: nfs`. + 2. Remove `placement.constraints` from database services. +* **Result**: Database containers effectively become stateless. If Node 1 fails, Swarm simply restarts OpenSearch on Node 2, which mounts the same NFS path and resumes operations immediately. + +This upgrade can be performed without reinstalling the cluster, by modifying the `stack.yml` as detailed in `README.md`. Index: /branches/amp_4_0/platform/tools/container/README.md =================================================================== --- /branches/amp_4_0/platform/tools/container/README.md (revision 2855) +++ /branches/amp_4_0/platform/tools/container/README.md (working copy) @@ -1,158 +1,540 @@ -# Container Tools +# Docker Swarm Deployment Guide -This directory contains the Docker container configurations and tools for the AMP platform. +## 1. Introduction -## Directory Structure +This directory contains the configurations and scripts to deploy the AMP platform on **Docker Swarm**. -The environment is modularized to allow for easier management and verification of individual services. +### Prerequisites -- **`compose/`**: Contains individual Docker Compose files for each service definition (e.g., `opensearch.yml`, `timescaledb.yml`) and a `base.yml` for shared networks and volumes. -- **`services/`**: Contains configuration files and build contexts for specific services (e.g., `services/opensearch/`, `services/nginx/`). -- **`manage_service.sh`**: The primary script to manage the lifecycle and health of these services. -- **`install_prerequisites.sh`**: Script to install host dependencies (`rsync`, `docker`, `python3`, `java`) and configure system limits. +1. **Docker Engine**: Installed on all nodes. +2. **Swarm Mode**: Active on the manager node. +3. **Ports**: Ensure Swarm ports are open between nodes (TCP 2377, 7946; UDP 7946, 4789). -## File Locations & Persistence +## 2. Architecture Overview -The AMP platform is designed to be stateless in its container definitions, with configuration and persistent data decoupled from the images. +Unlike standard Docker Compose (which runs everything on a single host with local bind-mounts), this Swarm implementation is designed for **orchestration** across one or more nodes. -### 1. Configuration Files +- **Stack**: `stack.yml` defines the services, replicating the Compose setup but adapted for Swarm (Overlay networks, Configs, Secrets). +- **Registry**: A local Docker Registry (`localhost:5000`) is used to share images between nodes. +- **Configs/Secrets**: Configuration files are immutable and injected into containers managed by Swarm. +- **Pinning**: Stateful services (`opensearch`, `timescaledb`) are pinned to specific nodes using labels to ensure they can access their data volumes. -All service configurations (e.g., `nginx.conf`, `telegraf.conf`, certificates) are stored in the host repository and mounted into the containers as read-only volumes. +### Architecture Summary -- **Path**: `platform/tools/container/services//` -- **Updates**: Changes made to files in these directories are typically picked up by the services after a restart via `./manage_service.sh up`. +| Feature | OpenSearch (Logs) | TimescaleDB (Metrics) | +| :--- | :--- | :--- | +| **Scaling** | Multi-Node (Active-Active) | Single-Node (Active) | +| **Data Location** | Distributed (Shards on all nodes) | Pinned (Storage Node only) | +| **HA Strategy** | Application-Level Replication | Storage Node Reliability | +| **Node Failure** | Logs accessible from other nodes | Metrics unavailable until node recovers | -### 2. Persistent Data (Volumes) +## 3. Single Node Deployment -Stateful data (databases, logs, indices) is stored in **Named Docker Volumes**. This ensures data survives container updates and accidental removals. +For a development or single-server production environment: -| Volume Name | Description | -| :--- | :--- | -| **`opensearch-data`** | OpenSearch indices and cluster state. | -| **`timescaledb-data`** | TimescaleDB (PostgreSQL) relational data and logs. | -| **`grafana-data`** | Grafana sqlite database (dashboards, users). | -| **`certs-vol`** | Shared SSL certificates generated during `setup`. | -| **`security-config-vol`** | OpenSearch Security plugin configurations. | -| **`*-logs`** | Log volumes for each service (e.g., `opensearch-logs`, `nginx-logs`). | +1. **Prerequisites**: + Run the helper script (from parent directory) to install Docker and configure firewall ports: -### 3. Accessing Persistent Data & Logs + ```bash + ./install_prerequisites.sh + ``` -Since data and logs are stored in Docker volumes (not directly on the host filesystem), you cannot browse them with a standard file explorer. Instead, use Docker tools or helper containers. +2. **Initialize Swarm**: -#### Accessing Logs + ```bash + ./manage_amp.sh init + ``` -Logs are stored in named volumes (e.g., `opensearch-logs`). To view or export them: + *This initializes Swarm and labels the current node as `type=storage` so databases can run on it.* -**Quick View (Last 100 lines):** +3. **Generate Certificates**: + ```bash + ./manage_amp.sh setup + ``` + + *This generates self-signed certificates and places them in the `certs-vol` volume required by services.* + +4. **Build & Deploy**: + + ```bash + ./manage_amp.sh build + ./manage_amp.sh deploy + ``` + + *This builds images, pushes them to the local registry, and deploys the `amp` stack.* + +5. **Initialize Security**: + + ```bash + ./manage_amp.sh security_init + ``` + + *This initializes the OpenSearch security index.* + +6. **Configure OpenSearch**: + + ```bash + ./manage_amp.sh configurator + ``` + + *This applies security roles and creates index patterns.* + +7. **Verify**: + + ```bash + ./manage_amp.sh status + ``` + +## 4. Multi-Node Deployment + +To run on multiple nodes (e.g., Manager + Workers): + +### 1. Prerequisite Setup (All Nodes) + +On **Manager** and **all Worker** nodes, ensure Docker is installed and ports are open: + ```bash -docker run --rm -v opensearch-logs:/logs alpine tail -n 100 /logs/opensearch.log +# Copy install_prerequisites.sh to the node if needed, then run: +./install_prerequisites.sh ``` -**Interactive Shell:** +### 2. Configure Hostnames (Critical for Swarm) +Docker Swarm uses hostnames to identify nodes. If all nodes are named `localhost`, it will be very confusing. + +Run this on **each node** to give it a unique name: + ```bash -docker run --rm -it -v nginx-logs:/logs alpine sh -# cd /logs && ls -l +# On Manager +hostnamectl set-hostname manager-1 + +# On Worker 1 +hostnamectl set-hostname worker-1 + +# On Worker 2 +hostnamectl set-hostname worker-2 ``` -#### Accessing Data +*Re-login to the shell for the change to take effect.* -Database files are similarly protected. To inspect raw data files (debug only): +### 3. Manager Setup +On the **Manager** node: + ```bash -docker run --rm -v timescaledb-data:/data alpine ls -lR /data +./manage_amp.sh init ``` -#### Accessing Configuration +This prints a token to join workers. -Configuration files are **mounted from the host** and are directly editable: +### 3. Join Workers -- Go to: `platform/tools/container/services//` -- Edit the files (e.g., `nginx/conf.d/app.conf`). -- Apply changes: `./manage_service.sh up` +On each **Worker** node: -> [!TIP] -> To find the physical location of a volume on your host system (Linux only), use: +```bash +docker swarm join --token :2377 +``` + +### 4. Configure Storage Node + +Choose **one** node (usually the manager or a specific high-disk node) to host the databases. Find its Node ID and label it: + +```bash +docker node ls +docker node update --label-add type=storage +``` + +### 5. Generate Certificates + +On the **Manager** node: + +```bash +./manage_amp.sh setup +``` + +> [!IMPORTANT] +> **Multi-Node Volume Warning**: Since we are using standard (local) Docker volumes, the `certs-vol` created on the Manager is **not** automatically available on Worker nodes. +> You must either: > -> ```bash -> docker volume inspect -> ``` +> 1. Use a shared volume plugin (NFS, etc.) for `certs-vol`. +> 2. Or manually copy the contents of `certs-vol` from the Manager to the corresponding volume on every Worker node. > -> On macOS/Windows (Docker Desktop), the physical files are inside the Docker VM and not directly accessible on the host OS without mounting them as shown above. +> If you skip this, services on Worker nodes will fail to find certificates. -#### Physical Locations (Linux / Docker VM) +### 6. Deploy -If running on a standard Linux Docker host (or inside the Docker VM), the logs are located at `/var/lib/docker/volumes/_/_data`. -Assuming the project name is `compose` (default), the locations are: +On the **Manager** node, specify the **Manager IP** for the registry so workers can reach it: -| Service | Host Path (Linux) | -| :--- | :--- | -| **OpenSearch** | `/var/lib/docker/volumes/compose_opensearch-logs/_data/` | -| **OpenSearch Dashboards** | `/var/lib/docker/volumes/compose_opensearch-dashboards-logs/_data/` | -| **Nginx** | `/var/lib/docker/volumes/compose_nginx-logs/_data/` | -| **Grafana** | `/var/lib/docker/volumes/compose_grafana-logs/_data/` | -| **Telegraf** | `/var/lib/docker/volumes/compose_telegraf-logs/_data/` | -| **Logstash** | `/var/lib/docker/volumes/compose_logstash-logs/_data/` | -| **PgBouncer** | `/var/lib/docker/volumes/compose_pgbouncer-logs/_data/` | -| **TimescaleDB** | `/var/lib/docker/volumes/compose_timescaledb-data/_data/pg_log/` | +```bash +export REGISTRY=:5000 +./manage_amp.sh build +./manage_amp.sh deploy +``` -## Setup & Prerequisites +Wait for services to start. -Before running services, ensure your system has the necessary tools. OpenSearch requires `vm.max_map_count` >= 262144. +### 7. Configure OpenSearch -Run the helper script to attempt automatic host setup: +Once OpenSearch is running (check `docker service ls`), run: ```bash +./manage_amp.sh configurator +``` + +Wait for the services to stabilize. Images will be pulled from the manager's local registry by all workers. + +## 5. Multi-Node Deployment with Manager and Worker (Small Clusters) + +This guide is for small clusters (1-5 nodes) where nodes must act as both **Managers** and **Workers** to maximize resource utilization, often requiring high availability (HA) for the control plane. + +### 1. Prerequisite Setup (Small Cluster) + +On **every node** (Managers and future Workers), run the installation script: + +```bash +# Copy install_prerequisites.sh to the node ./install_prerequisites.sh ``` -## Usage +# Copy install_prerequisites.sh to the node -Use `manage_service.sh` to manage and verify services. +./install_prerequisites.sh -### 1. Basic Usage +``` +### 2. Configure Hostnames + +Since all nodes act as both Managers and Workers, give them generic numbered names: + ```bash -./manage_service.sh [service_name] [action] +# Node 1 +hostnamectl set-hostname amp-node-1 + +# Node 2 +hostnamectl set-hostname amp-node-2 + +# Node 3 +hostnamectl set-hostname amp-node-3 ``` -- **`service_name`**: Name of the service (filename in `compose/` without .yml). -- **`action`** (optional): - - `up` (default): Start service and run health checks. - - `down`: Stop and remove the service. - - `purge`: Stop service and remove its volumes (fresh start). - - `validate`: Run health checks on a running service. +*Re-login to the shell for the change to take effect.* -### 2. Recommended Startup Order +### 3. Initialize First Manager -1. **Setup & Core Storage** +On the **first node** (Node 1): - ```bash - ./manage_service.sh setup - ./manage_service.sh opensearch - ./manage_service.sh timescaledb - ``` +```bash +./manage_amp.sh init +``` -2. **Backend & Middleware** +*This initializes the Swarm and generates the join token.* - ```bash - ./manage_service.sh configurator - ./manage_service.sh pgbouncer - ``` +### 3. Join & Promote Other Nodes -3. **Frontend & Observability** +1. **Join Node 2 & 3**: Run the join command (output from Step 2) on the other nodes. - ```bash - ./manage_service.sh opensearch-dashboards - ./manage_service.sh grafana - ./manage_service.sh telegraf - ./manage_service.sh logstash - ./manage_service.sh nginx - ``` + ```bash + docker swarm join --token :2377 + ``` -## Development Notes +2. **Promote to Manager** (On Node 1): + To ensure HA for the Swarm control plane, promote the new nodes to Managers. -- **Modifying Configuration**: Service configurations are located in `services/`. -- **Modifying Compose Definitions**: Docker compose definitions are in `compose/.yml`. -- Volumes in `compose/*.yml` reference `../services//...`. + ```bash + docker node promote + docker node promote + ``` + +### 4. Configure Mixed Roles (Active) + +By default, we want these managers to also run workloads (containers). Ensure they are set to `Active` availability (not `Drain`). + +On a Manager node: + +```bash +# Repeat for all node IDs +docker node update --availability active +``` + +### 5. Configure Storage Node + +Choose **ONE** specific node to host the database data (pinning). + +```bash +docker node ls +docker node update --label-add type=storage +``` + +### 6. High Availability: Traffic Routing (VIP) + +Since any node might go down, do not point users to a specific Node IP. Instead, configure a Floating VIP using the provided script. + +1. **Run `configure_vip.sh`** on **EACH** node with a unique priority: + + ```bash + # Node 1 (Primary) + ./configure_vip.sh --vip 192.168.200.100 --priority 102 + + # Node 2 (Backup) + ./configure_vip.sh --vip 192.168.200.100 --priority 101 + + # Node 3 (Backup) + ./configure_vip.sh --vip 192.168.200.100 --priority 100 + ``` + + *Replace `192.168.200.100` with your desired VIP.* + +2. **DNS**: Point your domain to this VIP. + +### 7. Deploy Stack + +On **Node 1** (or any Manager): + +1. **Generate Certificates**: + + ```bash + ./manage_amp.sh setup + ``` + +> [!IMPORTANT] + > **Manual Certificate Distribution**: Since `certs-vol` is a local volume, other nodes cannot access certificates generated on Node 1. You must copy them manually. + + **Steps to distribute certificates:** + + 1. **Locate the Volume Path** (On Node 1): + ```bash + SOURCE_PATH=$(docker volume inspect certs-vol --format '{{.Mountpoint}}') + echo "Certs are at: $SOURCE_PATH" + ``` + + 2. **Copy to Other Nodes** (Run on Node 1): + *Replace `amp-node-2`, `amp-node-3` with your other node IPs/Hostnames.* + + ```bash + # Loop through other nodes + for NODE in amp-node-2 amp-node-3; do + echo "--- Processing $NODE ---" + + # 1. Create volume on destination + ssh root@$NODE "docker volume create certs-vol" + + # 2. Get destination path + DEST_PATH=$(ssh root@$NODE "docker volume inspect certs-vol --format '{{.Mountpoint}}'") + + # 3. Copy files + scp -r $SOURCE_PATH/* root@$NODE:$DEST_PATH/ + + echo "✅ Copied to $NODE" + done + ``` + +2. **Deploy**: + + You cannot use `127.0.0.1` because other nodes will try to pull from *their own* localhost and fail. + + Use the **Physical IP** of Node 1 (where the registry is running). + + ```bash + # Replace with Node 1's actual IP (e.g., 192.168.162.139) + export REGISTRY=192.168.162.139:5000 + + ./manage_amp.sh build + ./manage_amp.sh deploy + ./manage_amp.sh security_init + ``` + +3. **Configure OpenSearch**: + + The OpenSearch configuration (including admin certificates) is managed in `services/opensearch/opensearch.yml`. + If you change environment variables or certificate names, update this file and redeploy. + + ```bash + ./manage_amp.sh configurator + ``` + +## 6. Full High Availability (HA) with NFS + +To achieve **Full Failover** (where services restart on another node if the leader fails), you must use **Shared Storage** (NFS). + +> [!IMPORTANT] +> Without NFS, database services (`opensearch`, `timescaledb`) are pinned to Node 1 (`type=storage`). If Node 1 fails, these services go down. +> With NFS, services can float to **any node**. + +### 1. Prerequisites (All Nodes) + +Install NFS client on **ALL Swarm Nodes**: + +```bash +# RedHat/Rocky/AlmaLinux +dnf install -y nfs-utils + +# Ubuntu/Debian +apt-get install -y nfs-common +``` + +### 2. Configure `stack.yml` + +You must edit `stack.yml` to: + +1. **Mount NFS Volumes**: Replace `driver: local` with NFS config. +2. **Unpin Services**: Remove `constraints: - node.labels.type == storage`. + +#### Example `stack.yml` Modification + +```yaml +services: + opensearch: + deploy: + # REMOVE constraints for HA + # placement: + # constraints: + # - node.labels.type == storage + replicas: 1 + +volumes: + # Example for OpenSearch Data + opensearch-data: + driver: local + driver_opts: + type: nfs + o: addr=192.168.1.100,rw,nfsvers=4 + device: ":/volume1/nfs/amp/opensearch" + + # Repeat for: timescaledb-data, grafana-data, certs-vol, etc. +``` + +### 3. Deploy + +1. **Destroy** the old stack (required to remove old volumes): + + ```bash + ./manage_amp.sh destroy + ``` + +2. **Prune** old volumes (WARNING: Deletes Data!): + + ```bash + docker volume prune -f + ``` + +3. **Deploy New Stack**: + + ```bash + ./manage_amp.sh deploy + ``` + +## 7. Limitations and Options + +### Data Persistence & Locations + +Understanding where your data lives is critical for backup and recovery. + +- **Storage Node**: + - In a **Single Node** setup, the manager is effectively the storage node. + - In a **Multi-Node** setup, only the node labeled `type=storage` holds the data (unless using NFS). + +- **Data Locations (Local Volumes)**: + `/var/lib/docker/volumes/` + + | Volume Name | Content | Service | + | :--- | :--- | :--- | + | `amp_opensearch-data` | OpenSearch Indices & Shards | `opensearch` | + | `amp_timescaledb-data` | Time-series Metrics Data | `timescaledb` | + | `amp_grafana-data` | Dashboards, Users, Alerts | `grafana` | + +### Data Lifecycle (Crash vs Delete) + +- **Service Crash / Restart**: ✅ **Safe**. + If a container crashes or is manually restarted, it re-mounts the **same** volume. No data is lost. + +- **Stack Removal (`./manage_swarm.sh rm`)**: ✅ **Safe**. + Docker Swarm **does not** delete named volumes when a stack is removed. The volumes (`amp_opensearch-data`, etc.) remain on the disk. + When you re-deploy (`./manage_swarm.sh deploy`), the new services will attach to the **existing** volumes and resume with all old data. + +- **Node Restart**: ✅ **Safe**. + Data persists on the disk. When the node boots up, Docker mounts the volume again. + +- **Manual Volume Prune**: ⚠️ **DANGER**. + Data is only deleted if you explicitly run `docker volume rm ` or `docker volume prune`. + +### Addressing TimescaleDB Limitations (HA Options) + +Since TimescaleDB runs in a single container, here are 3 ways to improve its reliability: + +1. **Option A: Infrastructure HA (Recommended)** + - Run the Storage Node as a VM. Rely on Hypervisor (VMware/Hyper-V) to restart the VM if hardware fails. +2. **Option B: Shared Storage (NFS)** + - Decouple data from the node using NFS (see Section 6). Swarm can then reschedule the DB to any node. +3. **Option C: External Managed Database** + - Use AWS RDS or Google Cloud SQL. + +## 8. Advanced Config + +### Management Commands + +Use the `manage_amp.sh` script for common tasks. + +| Action | Description | +| :--- | :--- | +| `setup` | Generate SSL certificates. | +| `build` | Build images and push to local registry. | +| `deploy` | Deploy the stack. | +| `configurator` | Configure OpenSearch roles/indices. | + +### Configs & Secrets + +Modifying files in `../services/` requires a **re-deploy**. Swarm Configs are immutable. + +## 9. Troubleshooting + +### "Volume 'certs-vol' was just created and is empty" + +Run `./manage_swarm.sh setup` immediately. + +### Connection Refused (ECONNREFUSED) + +Normal during startup. Wait for OpenSearch to be ready. + +### Services Stuck at 0/1 + +Use `docker service ps --no-trunc`. Common causes: missing secrets, missing certificates, or placement constraints. + +## 10. Monitoring & Alerting + +### Self-Healing + +Docker Swarm automatically restarts containers if they crash. We have configured **Healthchecks** (`HEALTHCHECK`) for critical services: + +- **OpenSearch**: Checks cluster health status. +- **TimescaleDB**: Checks Postgres connectivity. +- **Nginx**: Checks HTTP responsiveness. +- **Grafana**: Checks API availability. + +If a service becomes unresponsive (even if the process is running), Swarm will mark it as unhealthy and restart it automatically. + +### Alerting (Grafana) + +The platform uses Grafana for alerting. + +1. **Access Grafana**: `https:///monitoring/`. +2. **Contact Points**: Go to **Alerting > Contact points** to define where to send alerts (Email, Slack, PagerDuty, Webhook). +3. **Notification Policies**: Route alerts to specific contact points. +4. **Alert Rules**: Create rules based on Telegraf metrics. + - *Example CPU Alert*: `mean(cpu_usage_system) > 90` for 5 minutes. + - *Example Disk Alert*: `disk_used_percent > 85`. + - *Example OpenSearch*: `opensearch_cluster_status != green`. + +Telegraf is pre-configured to collect: + +- Host Metrics (CPU, Memory, Disk, Net) +- Docker Metrics (Container status) +- Postgres Metrics +- OpenSearch Metrics (via `logstash` pipeline or configured plugin) + +### Logging + +Logs are centralized in OpenSearch. + +- **View Logs**: Go to **OpenSearch Dashboards** -> **Discover**. +- **Index Pattern**: Create an index pattern for `logstash-*`. Index: /branches/amp_4_0/platform/tools/container/compose/backend.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/backend.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/backend.yml (nonexistent) @@ -1,23 +0,0 @@ -services: - backend: - build: - context: .. - dockerfile: services/backend/Dockerfile - container_name: backend - environment: - - DJANGO_SETTINGS_MODULE=djproject.settings - - POSTGRES_HOST=pgbouncer - - POSTGRES_PORT=6432 - - POSTGRES_DB=cm - - POSTGRES_USER=amp_admin - - POSTGRES_PASSWORD=Array@123$ - - OPENSEARCH_HOSTS=https://opensearch-node1:9200 - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - volumes: - # Mount application source code - - ../../../../src/webui/webui/htdocs/new/src:/app:rw - - ../../../../extensions:/extensions:rw - command: ["python", "manage.py", "runserver", "0.0.0.0:8888"] - networks: - - amp-network - - timescaledb Index: /branches/amp_4_0/platform/tools/container/compose/base.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/base.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/base.yml (nonexistent) @@ -1,19 +0,0 @@ -volumes: - opensearch-data: - timescaledb-data: - grafana-data: - certs-vol: - external: true - security-config-vol: - opensearch-logs: - nginx-logs: - timescaledb-logs: - grafana-logs: - telegraf-logs: - logstash-logs: - opensearch-dashboards-logs: - pgbouncer-logs: - -networks: - amp-network: - driver: bridge Index: /branches/amp_4_0/platform/tools/container/compose/configurator.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/configurator.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/configurator.yml (nonexistent) @@ -1,16 +0,0 @@ -services: - configurator: - image: rockylinux:9 - container_name: amp-configurator - environment: - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - volumes: - - certs-vol:/usr/share/opensearch/config/certs:ro - - ../services/setup:/setup:ro - - ../services/setup/install_curl.sh:/usr/local/bin/install_curl.sh - - ../services/setup/configure_opensearch.sh:/usr/local/bin/configure_opensearch.sh - - ../services/opensearch/amplog_template.json:/usr/share/opensearch/config/amplog_template.json - - ../services/opensearch-dashboards/export.ndjson:/usr/share/opensearch/config/export.ndjson - command: ["/bin/sh", "-c", "bash /usr/local/bin/install_curl.sh && bash /usr/local/bin/configure_opensearch.sh"] - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/grafana.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/grafana.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/grafana.yml (nonexistent) @@ -1,20 +0,0 @@ -services: - grafana: - image: grafana/grafana:latest - container_name: grafana - environment: - GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER} - GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD} - GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL} - GF_SERVER_SERVE_FROM_SUB_PATH: ${GF_SERVER_SERVE_FROM_SUB_PATH} - GF_PLUGINS_PREINSTALL: grafana-opensearch-datasource - GF_LOG_MODE: console,file - GF_PATHS_LOGS: /var/log/grafana - volumes: - - grafana-data:/var/lib/grafana - - grafana-logs:/var/log/grafana - - ../services/grafana/provisioning:/etc/grafana/provisioning - ports: - - "3000:3000" - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/logstash.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/logstash.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/logstash.yml (nonexistent) @@ -1,40 +0,0 @@ -services: - logstash: - image: opensearchproject/logstash-oss-with-opensearch-output-plugin:latest - container_name: logstash - command: > - bash -c " - if [ ! -f /usr/share/logstash/config/certs/node.pem ]; then - echo 'Waiting for certs...'; sleep 5; - fi - - # Auto-download JDBC driver if missing - DRIVER_PATH=/usr/share/logstash/drivers/postgresql.jar - if [ ! -f $$DRIVER_PATH ]; then - echo 'Downloading PostgreSQL JDBC Driver...' - curl -L -o $$DRIVER_PATH https://jdbc.postgresql.org/download/postgresql-42.7.3.jar - fi - - logstash-plugin install logstash-integration-jdbc - logstash-plugin install logstash-filter-geoip - bin/logstash - " - environment: - LS_JAVA_OPTS: ${LOGSTASH_JAVA_OPTS} - OPENSEARCH_URL: https://opensearch-node1:9200 - OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - volumes: - - ../services/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - - ../services/logstash/pipeline:/usr/share/logstash/pipeline:ro - - ../services/logstash/drivers:/usr/share/logstash/drivers:ro - - certs-vol:/usr/share/logstash/config/certs:ro - - logstash-logs:/usr/share/logstash/logs - ports: - - "5044:5044" - - "5514:5514/udp" - - "5514:5514/tcp" - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/nginx.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/nginx.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/nginx.yml (nonexistent) @@ -1,15 +0,0 @@ -services: - nginx: - image: nginx:alpine - container_name: nginx - volumes: - - ../services/nginx/conf.d:/etc/nginx/conf.d - - certs-vol:/etc/nginx/certs:ro - - nginx-logs:/var/log/nginx - ports: - - "80:80" - - "443:443" - extra_hosts: - - "host.docker.internal:host-gateway" - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/opensearch-dashboards.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/opensearch-dashboards.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/opensearch-dashboards.yml (nonexistent) @@ -1,42 +0,0 @@ -services: - opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:2.11.0 - container_name: opensearch-dashboards - user: root - - environment: - OPENSEARCH_HOSTS: '["https://opensearch-node1:9200"]' - OPENSEARCH_SSL_VERIFICATIONMODE: certificate - OPENSEARCH_USERNAME: admin - OPENSEARCH_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - OPENSEARCH_SSL_CERTIFICATEAUTHORITIES: '["/usr/share/opensearch-dashboards/config/certs/root-ca.pem"]' - - SERVER_SSL_ENABLED: "true" - SERVER_SSL_KEY: /usr/share/opensearch-dashboards/config/certs/node-key.pem - SERVER_SSL_CERTIFICATE: /usr/share/opensearch-dashboards/config/certs/node.pem - LOGGING_DEST: /usr/share/opensearch-dashboards/logs/osd.log - - SERVER_BASEPATH: /visualization - SERVER_REWRITEBASEPATH: "true" - - ports: - - "5601:5601" - - volumes: - - certs-vol:/usr/share/opensearch-dashboards/config/certs:ro - - /bin/yq:/usr/local/bin/yq:ro - - ../services/opensearch-dashboards/assets:/usr/share/opensearch-dashboards/src/core/server/core_app/assets/custom:ro - - opensearch-dashboards-logs:/usr/share/opensearch-dashboards/logs - - command: > - bash -c " - chown -R 1000:1000 /usr/share/opensearch-dashboards/logs && - runuser -u opensearch-dashboards -- /usr/share/opensearch-dashboards/opensearch-dashboards-docker-entrypoint.sh - " - - networks: - - amp-network - -volumes: - certs-vol: - external: true Index: /branches/amp_4_0/platform/tools/container/compose/opensearch.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/opensearch.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/opensearch.yml (nonexistent) @@ -1,59 +0,0 @@ -services: - opensearch: - image: opensearchproject/opensearch:2.11.0 - container_name: opensearch-node1 - environment: - - cluster.name=opensearch-cluster - - node.name=opensearch-node1 - - discovery.seed_hosts=opensearch-node1 - - cluster.initial_master_nodes=opensearch-node1 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}" - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - # Security Plugin Configuration - - plugins.security.ssl.http.enabled=true - - plugins.security.ssl.http.pemkey_filepath=/usr/share/opensearch/config/certs/node-key.pem - - plugins.security.ssl.http.pemcert_filepath=/usr/share/opensearch/config/certs/node.pem - - plugins.security.ssl.http.pemtrustedcas_filepath=/usr/share/opensearch/config/certs/root-ca.pem - - plugins.security.ssl.transport.enabled=true - - plugins.security.ssl.transport.pemkey_filepath=/usr/share/opensearch/config/certs/node-key.pem - - plugins.security.ssl.transport.pemcert_filepath=/usr/share/opensearch/config/certs/node.pem - - plugins.security.ssl.transport.pemtrustedcas_filepath=/usr/share/opensearch/config/certs/root-ca.pem - - plugins.security.restapi.roles_enabled=["all_access", "security_rest_api_access"] - - plugins.security.allow_default_init_securityindex=true - - plugins.security.nodes_dn=CN=node-1,O=OpenSearchNode,L=Bengaluru,ST=Karnataka,C=IN - - plugins.security.authcz.admin_dn=CN=admin,O=OpenSearchAdmin,L=Bengaluru,ST=Karnataka,C=IN - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - volumes: - - opensearch-data:/usr/share/opensearch/data - - certs-vol:/usr/share/opensearch/config/certs:ro - - security-config-vol:/usr/share/opensearch/config/opensearch-security-mount:ro - - opensearch-logs:/usr/share/opensearch/logs - command: > - bash -c " - set -e - # 1. Run demo installer to generate default security files (roles, users, etc.) - # This writes to /usr/share/opensearch/config/opensearch-security - chmod +x /usr/share/opensearch/plugins/opensearch-security/tools/install_demo_configuration.sh - /usr/share/opensearch/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i - - # 2. Overwrite config.yml with our custom version (JWT, etc.) - echo '--- Overwriting config.yml ---' - cp /usr/share/opensearch/config/opensearch-security-mount/config.yml /usr/share/opensearch/config/opensearch-security/config.yml - - echo '--- Overwriting internal_users.yml ---' - cp /usr/share/opensearch/config/opensearch-security-mount/internal_users.yml /usr/share/opensearch/config/opensearch-security/internal_users.yml - - # 3. Start OpenSearch - # Disable demo config in entrypoint so it doesn't overwrite our work - export DISABLE_INSTALL_DEMO_CONFIG=true - ./opensearch-docker-entrypoint.sh - " - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/pgbouncer.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/pgbouncer.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/pgbouncer.yml (nonexistent) @@ -1,15 +0,0 @@ -services: - pgbouncer: - image: edoburu/pgbouncer:latest - container_name: pgbouncer - environment: - - DB_USER=amp_admin - - DB_PASSWORD=Array@123$ - - DB_HOST=timescaledb - - DB_NAME=cm - volumes: - - ../services/pgbouncer/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini:ro - - ../services/pgbouncer/userlist.txt:/etc/pgbouncer/userlist.txt:ro - - pgbouncer-logs:/var/log/pgbouncer - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/setup.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/setup.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/setup.yml (nonexistent) @@ -1,13 +0,0 @@ -services: - setup: - image: rockylinux:9 - container_name: amp-setup - working_dir: /setup - environment: - - OPENSEARCH_JWT_SECRET=${OPENSEARCH_JWT_SECRET} - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} - volumes: - - ../services/setup:/setup:rw - - certs-vol:/certs:rw - - security-config-vol:/security-config:rw - command: ["/bin/sh", "-c", "dnf install -y openssl httpd-tools && chmod +x /setup/setup.sh && /setup/setup.sh /certs /security-config"] Index: /branches/amp_4_0/platform/tools/container/compose/telegraf.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/telegraf.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/telegraf.yml (nonexistent) @@ -1,22 +0,0 @@ -services: - telegraf: - image: telegraf:latest - container_name: telegraf - user: "0:${DOCKER_GID:-0}" - group_add: - - "${DOCKER_GID:-0}" - environment: - PG_PASSWORD: ${POSTGRES_PASSWORD} - volumes: - - ../services/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro - - ../services/telegraf/telegraf.d:/etc/telegraf/telegraf.d:ro - - telegraf-logs:/var/log/telegraf - - /var/run/docker.sock:/var/run/docker.sock:ro - - /dev:/dev:ro - - /sys:/rootfs/sys:ro - - /proc:/rootfs/proc:ro - - /etc:/rootfs/etc:ro - - /run/udev:/run/udev:ro - command: ["telegraf", "--config", "/etc/telegraf/telegraf.conf", "--config-directory", "/etc/telegraf/telegraf.d"] - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/compose/timescaledb.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/compose/timescaledb.yml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/compose/timescaledb.yml (nonexistent) @@ -1,15 +0,0 @@ -services: - timescaledb: - image: timescale/timescaledb:latest-pg16 - container_name: timescaledb - environment: - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - volumes: - - timescaledb-data:/var/lib/postgresql/data - - ../services/postgres/initdb.d:/docker-entrypoint-initdb.d - ports: - - "5432:5432" - networks: - - amp-network Index: /branches/amp_4_0/platform/tools/container/configure_vip.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/configure_vip.sh (nonexistent) +++ /branches/amp_4_0/platform/tools/container/configure_vip.sh (working copy) @@ -0,0 +1,115 @@ +#!/bin/bash + +# configure_vip.sh +# Usage: ./configure_vip.sh --vip --priority [--interface ] +# Example (Node 1): ./configure_vip.sh --vip 192.168.1.100 --priority 101 +# Example (Node 2): ./configure_vip.sh --vip 192.168.1.100 --priority 100 + +set -e + +VIP="" +PRIORITY="" +INTERFACE="" +ROUTER_ID=51 +AUTH_PASS="array_amp" + +usage() { + echo "Usage: $0 --vip --priority [--interface ]" + echo " --vip : The Floating Virtual IP (e.g., 192.168.1.100)" + echo " --priority : Higher number = Higher preference (Master). Use 101, 100, 99..." + echo " --interface : (Optional) Network interface. Auto-detected if omitted." + exit 1 +} + +# Parse Arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + --vip) VIP="$2"; shift ;; + --priority) PRIORITY="$2"; shift ;; + --interface) INTERFACE="$2"; shift ;; + *) echo "Unknown parameter: $1"; usage ;; + esac + shift +done + +if [ -z "$VIP" ] || [ -z "$PRIORITY" ]; then + usage +fi + +# Auto-detect interface if not specified +if [ -z "$INTERFACE" ]; then + # Get the interface used for the default route + INTERFACE=$(ip route get 8.8.8.8 | awk '{print $5; exit}') + + if [ -z "$INTERFACE" ]; then + echo "❌ Could not auto-detect network interface. Please specify with --interface." + exit 1 + fi + echo "ℹ️ Auto-detected interface: $INTERFACE" +fi + +CONFIG_FILE="/etc/keepalived/keepalived.conf" +BACKUP_FILE="/etc/keepalived/keepalived.conf.bak.$(date +%s)" + +echo "--- Configuring Keepalived ---" +echo "VIP: $VIP" +echo "Interface: $INTERFACE" +echo "Priority: $PRIORITY" + +# Backup existing config +if [ -f "$CONFIG_FILE" ]; then + echo "Backing up existing config to $BACKUP_FILE" + cp "$CONFIG_FILE" "$BACKUP_FILE" +fi + +# Generate Config +# We use 'state BACKUP' for all nodes. +# The one with the highest priority will be elected Master. +# This avoids preemption flaps if we used MASTER/BACKUP explicitly without nopreempt. + +cat > "$CONFIG_FILE" </dev/null 2>&1; then + echo "Detected firewalld. Opening Swarm ports..." + sudo firewall-cmd --permanent --add-port=2377/tcp + sudo firewall-cmd --permanent --add-port=7946/tcp + sudo firewall-cmd --permanent --add-port=7946/udp + sudo firewall-cmd --permanent --add-port=4789/udp + # Keepalived VRRP + sudo firewall-cmd --permanent --add-protocol=vrrp + sudo firewall-cmd --reload + echo "✅ Firewalld updated." + elif command -v ufw >/dev/null 2>&1; then + echo "Detected UFW. Opening Swarm ports..." + sudo ufw allow 2377/tcp + sudo ufw allow 7946/tcp + sudo ufw allow 7946/udp + sudo ufw allow 4789/udp + # Keepalived VRRP + sudo ufw allow vrrp + sudo ufw reload + echo "✅ UFW updated." + else + echo "⚠️ No supported firewall manager (firewalld/ufw) detected. Ensure ports 2377/tcp, 7946/tcp+udp, 4789/udp AND protocol VRRP are open manually." + fi + + echo "--- Prerequisites Check Complete ---" + Index: /branches/amp_4_0/platform/tools/container/legacy/README_old.md =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/README_old.md (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/README_old.md (working copy) @@ -0,0 +1,184 @@ +# Container Tools + +This directory contains the Docker container configurations and tools for the AMP platform. + +## Directory Structure + +The environment is modularized to allow for easier management and verification of individual services. + +- **`compose/`**: Contains individual Docker Compose files for each service definition (e.g., `opensearch.yml`, `timescaledb.yml`) and a `base.yml` for shared networks and volumes. +- **`services/`**: Contains configuration files and build contexts for specific services (e.g., `services/opensearch/`, `services/nginx/`). +- **`manage_service.sh`**: The primary script to manage the lifecycle and health of these services. +- **`install_prerequisites.sh`**: Script to install host dependencies (`rsync`, `docker`, `python3`, `java`) and configure system limits. + +## File Locations & Persistence + +The AMP platform is designed to be stateless in its container definitions, with configuration and persistent data decoupled from the images. + +### 1. Configuration Files + +All service configurations (e.g., `nginx.conf`, `telegraf.conf`, certificates) are stored in the host repository and mounted into the containers as read-only volumes. + +- **Path**: `platform/tools/container/services//` +- **Updates**: Changes made to files in these directories are typically picked up by the services after a restart via `./manage_service.sh up`. + +### 2. Persistent Data (Volumes) + +Stateful data (databases, logs, indices) is stored in **Named Docker Volumes**. This ensures data survives container updates and accidental removals. + +| Volume Name | Description | +| :--- | :--- | +| **`opensearch-data`** | OpenSearch indices and cluster state. | +| **`timescaledb-data`** | TimescaleDB (PostgreSQL) relational data and logs. | +| **`grafana-data`** | Grafana sqlite database (dashboards, users). | +| **`certs-vol`** | Shared SSL certificates generated during `setup`. | +| **`security-config-vol`** | OpenSearch Security plugin configurations. | +| **`*-logs`** | Log volumes for each service (e.g., `opensearch-logs`, `nginx-logs`). | + +### 3. Accessing Persistent Data & Logs + +Since data and logs are stored in Docker volumes (not directly on the host filesystem), you cannot browse them with a standard file explorer. Instead, use Docker tools or helper containers. + +#### Accessing Logs + +Logs are stored in named volumes (e.g., `opensearch-logs`). To view or export them: + +**Quick View (Last 100 lines):** + +```bash +docker run --rm -v opensearch-logs:/logs alpine tail -n 100 /logs/opensearch.log +``` + +**Interactive Shell:** + +```bash +docker run --rm -it -v nginx-logs:/logs alpine sh +# cd /logs && ls -l +``` + +#### Accessing Data + +Database files are similarly protected. To inspect raw data files (debug only): + +```bash +docker run --rm -v timescaledb-data:/data alpine ls -lR /data +``` + +#### Accessing Configuration + +Configuration files are **mounted from the host** and are directly editable: + +- Go to: `platform/tools/container/services//` +- Edit the files (e.g., `nginx/conf.d/app.conf`). +- Apply changes: `./manage_service.sh up` + +> [!TIP] +> To find the physical location of a volume on your host system (Linux only), use: +> +> ```bash +> docker volume inspect +> ``` +> +> On macOS/Windows (Docker Desktop), the physical files are inside the Docker VM and not directly accessible on the host OS without mounting them as shown above. + +#### Physical Locations (Linux / Docker VM) + +If running on a standard Linux Docker host (or inside the Docker VM), the logs are located at `/var/lib/docker/volumes/_/_data`. +Assuming the project name is `compose` (default), the locations are: + +| Service | Host Path (Linux) | +| :--- | :--- | +| **OpenSearch** | `/var/lib/docker/volumes/compose_opensearch-logs/_data/` | +| **OpenSearch Dashboards** | `/var/lib/docker/volumes/compose_opensearch-dashboards-logs/_data/` | +| **Nginx** | `/var/lib/docker/volumes/compose_nginx-logs/_data/` | +| **Grafana** | `/var/lib/docker/volumes/compose_grafana-logs/_data/` | +| **Telegraf** | `/var/lib/docker/volumes/compose_telegraf-logs/_data/` | +| **Logstash** | `/var/lib/docker/volumes/compose_logstash-logs/_data/` | +| **PgBouncer** | `/var/lib/docker/volumes/compose_pgbouncer-logs/_data/` | +| **TimescaleDB** | `/var/lib/docker/volumes/compose_timescaledb-data/_data/pg_log/` | + +## Setup & Prerequisites + +Before running services, ensure your system has the necessary tools. OpenSearch requires `vm.max_map_count` >= 262144. + +Run the helper script to attempt automatic host setup: + +```bash +./install_prerequisites.sh +``` + +## Usage + +Use `manage_service.sh` to manage and verify services. + +### 1. Basic Usage + +```bash +./manage_service.sh [service_name] [action] +``` + +- **`service_name`**: Name of the service (filename in `compose/` without .yml). +- **`action`** (optional): + - `up` (default): Start service and run health checks. + - `down`: Stop and remove the service. + - `purge`: Stop service and remove its volumes (fresh start). + - `validate`: Run health checks on a running service. + +### 2. Recommended Startup Order + +1. **Setup & Core Storage** + + ```bash + ./manage_service.sh setup + ./manage_service.sh opensearch + ./manage_service.sh timescaledb + ``` + +2. **Backend & Middleware** + + ```bash + ./manage_service.sh configurator + ./manage_service.sh pgbouncer + ``` + +3. **Frontend & Observability** + + ```bash + ./manage_service.sh opensearch-dashboards + ./manage_service.sh grafana + ./manage_service.sh telegraf + ./manage_service.sh logstash + ./manage_service.sh nginx + ``` + +## Development Notes + +- **Modifying Configuration**: Service configurations are located in `services/`. +- **Modifying Compose Definitions**: Docker compose definitions are in `compose/.yml`. +- Volumes in `compose/*.yml` reference `../services//...`. + +## Docker Swarm Support + +To run the platform in Docker Swarm mode (Single or Multi-Node): + +### 1. Initialization + +Initialize Swarm mode and label the current node for storage pinning: + +```bash +./swarm/manage_swarm.sh init +``` + +### 2. Build & Deploy + +This will start a local registry, build all images, push them, and deploy the stack. + +```bash +./swarm/manage_swarm.sh build +./swarm/manage_swarm.sh deploy +``` + +### 3. Management + +- **Status**: `./swarm/manage_swarm.sh status` +- **Remove**: `./swarm/manage_swarm.sh rm` Index: /branches/amp_4_0/platform/tools/container/legacy/compose/backend.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/backend.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/backend.yml (working copy) @@ -0,0 +1,23 @@ +services: + backend: + build: + context: .. + dockerfile: services/backend/Dockerfile + container_name: backend + environment: + - DJANGO_SETTINGS_MODULE=djproject.settings + - POSTGRES_HOST=pgbouncer + - POSTGRES_PORT=6432 + - POSTGRES_DB=cm + - POSTGRES_USER=amp_admin + - POSTGRES_PASSWORD=Array@123$ + - OPENSEARCH_HOSTS=https://opensearch-node1:9200 + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + volumes: + # Mount application source code + - ../../../../src/webui/webui/htdocs/new/src:/app:rw + - ../../../../extensions:/extensions:rw + command: ["python", "manage.py", "runserver", "0.0.0.0:8888"] + networks: + - amp-network + - timescaledb Index: /branches/amp_4_0/platform/tools/container/legacy/compose/base.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/base.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/base.yml (working copy) @@ -0,0 +1,19 @@ +volumes: + opensearch-data: + timescaledb-data: + grafana-data: + certs-vol: + external: true + security-config-vol: + opensearch-logs: + nginx-logs: + timescaledb-logs: + grafana-logs: + telegraf-logs: + logstash-logs: + opensearch-dashboards-logs: + pgbouncer-logs: + +networks: + amp-network: + driver: bridge Index: /branches/amp_4_0/platform/tools/container/legacy/compose/configurator.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/configurator.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/configurator.yml (working copy) @@ -0,0 +1,16 @@ +services: + configurator: + image: rockylinux:9 + container_name: amp-configurator + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + volumes: + - certs-vol:/usr/share/opensearch/config/certs:ro + - ../services/setup:/setup:ro + - ../services/setup/install_curl.sh:/usr/local/bin/install_curl.sh + - ../services/setup/configure_opensearch.sh:/usr/local/bin/configure_opensearch.sh + - ../services/opensearch/amplog_template.json:/usr/share/opensearch/config/amplog_template.json + - ../services/opensearch-dashboards/export.ndjson:/usr/share/opensearch/config/export.ndjson + command: ["/bin/sh", "-c", "bash /usr/local/bin/install_curl.sh && bash /usr/local/bin/configure_opensearch.sh"] + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/grafana.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/grafana.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/grafana.yml (working copy) @@ -0,0 +1,20 @@ +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + environment: + GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD} + GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL} + GF_SERVER_SERVE_FROM_SUB_PATH: ${GF_SERVER_SERVE_FROM_SUB_PATH} + GF_PLUGINS_PREINSTALL: grafana-opensearch-datasource + GF_LOG_MODE: console,file + GF_PATHS_LOGS: /var/log/grafana + volumes: + - grafana-data:/var/lib/grafana + - grafana-logs:/var/log/grafana + - ../services/grafana/provisioning:/etc/grafana/provisioning + ports: + - "3000:3000" + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/logstash.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/logstash.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/logstash.yml (working copy) @@ -0,0 +1,40 @@ +services: + logstash: + image: opensearchproject/logstash-oss-with-opensearch-output-plugin:latest + container_name: logstash + command: > + bash -c " + if [ ! -f /usr/share/logstash/config/certs/node.pem ]; then + echo 'Waiting for certs...'; sleep 5; + fi + + # Auto-download JDBC driver if missing + DRIVER_PATH=/usr/share/logstash/drivers/postgresql.jar + if [ ! -f $$DRIVER_PATH ]; then + echo 'Downloading PostgreSQL JDBC Driver...' + curl -L -o $$DRIVER_PATH https://jdbc.postgresql.org/download/postgresql-42.7.3.jar + fi + + logstash-plugin install logstash-integration-jdbc + logstash-plugin install logstash-filter-geoip + bin/logstash + " + environment: + LS_JAVA_OPTS: ${LOGSTASH_JAVA_OPTS} + OPENSEARCH_URL: https://opensearch-node1:9200 + OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - ../services/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - ../services/logstash/pipeline:/usr/share/logstash/pipeline:ro + - ../services/logstash/drivers:/usr/share/logstash/drivers:ro + - certs-vol:/usr/share/logstash/config/certs:ro + - logstash-logs:/usr/share/logstash/logs + ports: + - "5044:5044" + - "5514:5514/udp" + - "5514:5514/tcp" + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/nginx.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/nginx.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/nginx.yml (working copy) @@ -0,0 +1,15 @@ +services: + nginx: + image: nginx:alpine + container_name: nginx + volumes: + - ../services/nginx/conf.d:/etc/nginx/conf.d + - certs-vol:/etc/nginx/certs:ro + - nginx-logs:/var/log/nginx + ports: + - "80:80" + - "443:443" + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch-dashboards.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch-dashboards.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch-dashboards.yml (working copy) @@ -0,0 +1,42 @@ +services: + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2.11.0 + container_name: opensearch-dashboards + user: root + + environment: + OPENSEARCH_HOSTS: '["https://opensearch-node1:9200"]' + OPENSEARCH_SSL_VERIFICATIONMODE: certificate + OPENSEARCH_USERNAME: admin + OPENSEARCH_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + OPENSEARCH_SSL_CERTIFICATEAUTHORITIES: '["/usr/share/opensearch-dashboards/config/certs/root-ca.pem"]' + + SERVER_SSL_ENABLED: "true" + SERVER_SSL_KEY: /usr/share/opensearch-dashboards/config/certs/node-key.pem + SERVER_SSL_CERTIFICATE: /usr/share/opensearch-dashboards/config/certs/node.pem + LOGGING_DEST: /usr/share/opensearch-dashboards/logs/osd.log + + SERVER_BASEPATH: /visualization + SERVER_REWRITEBASEPATH: "true" + + ports: + - "5601:5601" + + volumes: + - certs-vol:/usr/share/opensearch-dashboards/config/certs:ro + - /bin/yq:/usr/local/bin/yq:ro + - ../services/opensearch-dashboards/assets:/usr/share/opensearch-dashboards/src/core/server/core_app/assets/custom:ro + - opensearch-dashboards-logs:/usr/share/opensearch-dashboards/logs + + command: > + bash -c " + chown -R 1000:1000 /usr/share/opensearch-dashboards/logs && + runuser -u opensearch-dashboards -- /usr/share/opensearch-dashboards/opensearch-dashboards-docker-entrypoint.sh + " + + networks: + - amp-network + +volumes: + certs-vol: + external: true Index: /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/opensearch.yml (working copy) @@ -0,0 +1,59 @@ +services: + opensearch: + image: opensearchproject/opensearch:2.11.0 + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1 + - cluster.initial_master_nodes=opensearch-node1 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + # Security Plugin Configuration + - plugins.security.ssl.http.enabled=true + - plugins.security.ssl.http.pemkey_filepath=/usr/share/opensearch/config/certs/node-key.pem + - plugins.security.ssl.http.pemcert_filepath=/usr/share/opensearch/config/certs/node.pem + - plugins.security.ssl.http.pemtrustedcas_filepath=/usr/share/opensearch/config/certs/root-ca.pem + - plugins.security.ssl.transport.enabled=true + - plugins.security.ssl.transport.pemkey_filepath=/usr/share/opensearch/config/certs/node-key.pem + - plugins.security.ssl.transport.pemcert_filepath=/usr/share/opensearch/config/certs/node.pem + - plugins.security.ssl.transport.pemtrustedcas_filepath=/usr/share/opensearch/config/certs/root-ca.pem + - plugins.security.restapi.roles_enabled=["all_access", "security_rest_api_access"] + - plugins.security.allow_default_init_securityindex=true + - plugins.security.nodes_dn=CN=node-1,O=OpenSearchNode,L=Bengaluru,ST=Karnataka,C=IN + - plugins.security.authcz.admin_dn=CN=admin,O=OpenSearchAdmin,L=Bengaluru,ST=Karnataka,C=IN + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data:/usr/share/opensearch/data + - certs-vol:/usr/share/opensearch/config/certs:ro + - security-config-vol:/usr/share/opensearch/config/opensearch-security-mount:ro + - opensearch-logs:/usr/share/opensearch/logs + command: > + bash -c " + set -e + # 1. Run demo installer to generate default security files (roles, users, etc.) + # This writes to /usr/share/opensearch/config/opensearch-security + chmod +x /usr/share/opensearch/plugins/opensearch-security/tools/install_demo_configuration.sh + /usr/share/opensearch/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i + + # 2. Overwrite config.yml with our custom version (JWT, etc.) + echo '--- Overwriting config.yml ---' + cp /usr/share/opensearch/config/opensearch-security-mount/config.yml /usr/share/opensearch/config/opensearch-security/config.yml + + echo '--- Overwriting internal_users.yml ---' + cp /usr/share/opensearch/config/opensearch-security-mount/internal_users.yml /usr/share/opensearch/config/opensearch-security/internal_users.yml + + # 3. Start OpenSearch + # Disable demo config in entrypoint so it doesn't overwrite our work + export DISABLE_INSTALL_DEMO_CONFIG=true + ./opensearch-docker-entrypoint.sh + " + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/pgbouncer.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/pgbouncer.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/pgbouncer.yml (working copy) @@ -0,0 +1,15 @@ +services: + pgbouncer: + image: edoburu/pgbouncer:latest + container_name: pgbouncer + environment: + - DB_USER=amp_admin + - DB_PASSWORD=Array@123$ + - DB_HOST=timescaledb + - DB_NAME=cm + volumes: + - ../services/pgbouncer/pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini:ro + - ../services/pgbouncer/userlist.txt:/etc/pgbouncer/userlist.txt:ro + - pgbouncer-logs:/var/log/pgbouncer + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/setup.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/setup.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/setup.yml (working copy) @@ -0,0 +1,13 @@ +services: + setup: + image: rockylinux:9 + container_name: amp-setup + working_dir: /setup + environment: + - OPENSEARCH_JWT_SECRET=${OPENSEARCH_JWT_SECRET} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + volumes: + - ../services/setup:/setup:rw + - certs-vol:/certs:rw + - security-config-vol:/security-config:rw + command: ["/bin/sh", "-c", "dnf install -y openssl httpd-tools && chmod +x /setup/setup.sh && /setup/setup.sh /certs /security-config"] Index: /branches/amp_4_0/platform/tools/container/legacy/compose/telegraf.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/telegraf.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/telegraf.yml (working copy) @@ -0,0 +1,22 @@ +services: + telegraf: + image: telegraf:latest + container_name: telegraf + user: "0:${DOCKER_GID:-0}" + group_add: + - "${DOCKER_GID:-0}" + environment: + PG_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ../services/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro + - ../services/telegraf/telegraf.d:/etc/telegraf/telegraf.d:ro + - telegraf-logs:/var/log/telegraf + - /var/run/docker.sock:/var/run/docker.sock:ro + - /dev:/dev:ro + - /sys:/rootfs/sys:ro + - /proc:/rootfs/proc:ro + - /etc:/rootfs/etc:ro + - /run/udev:/run/udev:ro + command: ["telegraf", "--config", "/etc/telegraf/telegraf.conf", "--config-directory", "/etc/telegraf/telegraf.d"] + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/compose/timescaledb.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/compose/timescaledb.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/compose/timescaledb.yml (working copy) @@ -0,0 +1,15 @@ +services: + timescaledb: + image: timescale/timescaledb:latest-pg16 + container_name: timescaledb + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - timescaledb-data:/var/lib/postgresql/data + - ../services/postgres/initdb.d:/docker-entrypoint-initdb.d + ports: + - "5432:5432" + networks: + - amp-network Index: /branches/amp_4_0/platform/tools/container/legacy/manage_service.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/legacy/manage_service.sh (nonexistent) +++ /branches/amp_4_0/platform/tools/container/legacy/manage_service.sh (working copy) @@ -0,0 +1,303 @@ +#!/bin/bash + +# validate_service.sh +# Usage: ./validate_service.sh [service_name] + +SERVICE=$1 + +# REPO_ROOT Resolution +# Assuming script is in platform/tools/container/ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../" && pwd)" + +SPLIT_DIR_NAME="compose" +COMPOSE_DIR="$SCRIPT_DIR/$SPLIT_DIR_NAME" + +# Detect docker-compose command +if command -v docker-compose &> /dev/null; then + DOCKER_COMPOSE_CMD="docker-compose" +elif docker compose version &> /dev/null; then + DOCKER_COMPOSE_CMD="docker compose" +else + echo "Error: neither 'docker-compose' nor 'docker compose' found." + exit 1 +fi +echo "Using Docker Compose command: $DOCKER_COMPOSE_CMD" +export COMPOSE_IGNORE_ORPHANS=True + +# Detect Docker GID for Telegraf and other services needing socket access +if [ -S /var/run/docker.sock ]; then + # Compatibility: stat flags differ between GNU and BSD (macOS) + if [[ "$OSTYPE" == "darwin"* ]]; then + DOCKER_GID=$(stat -f '%g' /var/run/docker.sock) + else + DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) + fi + export DOCKER_GID + echo "Detected Docker GID: $DOCKER_GID" +fi + +if [ -z "$SERVICE" ]; then + echo "Usage: ./manage_service.sh [service_name] [action]" + echo "AMP Service Manager" + echo "" + echo "Available services:" + echo " base, setup, opensearch, configurator, opensearch-dashboards, timescaledb," + echo " pgbouncer, grafana, telegraf, logstash, nginx" + echo "" + echo "Actions:" + echo " up - Start/Build service (default)" + echo " down - Stop/Remove service" + echo " purge - Stop/Remove service AND volumes (Fresh Start)" + echo " validate - Run health checks only" + exit 1 +fi + +ACTION=${2:-up} + +# Check Docker Permissions +if ! docker ps >/dev/null 2>&1; then + echo "❌ Error: Docker permission denied or Docker is not running." + exit 1 +fi + +echo "--- Processing Service: $SERVICE ($ACTION) ---" + +# Ensure external volume 'certs-vol' exists +if ! docker volume ls -q | grep -q "^certs-vol$"; then + echo "Creating external volume: certs-vol" + docker volume create certs-vol +fi + +# Move to the Compose Directory +cd "$COMPOSE_DIR" || exit 1 + +# Load .env if it exists in the script directory +if [ -f "$SCRIPT_DIR/.env" ]; then + # Note: Docker Compose loads .env automatically if it's in the same dir as the yaml files. + # If it's in the parent, we might need to symlink it or specify --env-file + if [ ! -f ".env" ]; then + ln -s "$SCRIPT_DIR/.env" .env + echo "DEBUG: Symlinked .env from parent directory." + fi +fi + +if [ "$ACTION" == "down" ] || [ "$ACTION" == "purge" ]; then + if [ "$ACTION" == "purge" ]; then + echo "Stopping $SERVICE and removing volumes..." + $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE.yml" down -v + else + $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE.yml" down + fi + echo "Service $SERVICE stopped." + exit 0 +fi + +if [ "$SERVICE" == "base" ]; then + echo "Validating Base configuration..." + # 'base' only has volumes/networks, 'up' fails in v2 if no services. + # We trust that subsequent service runs (which include base.yml) will create them. + # We just check config validity here. + $DOCKER_COMPOSE_CMD -f base.yml config >/dev/null + if [ $? -eq 0 ]; then + echo "✅ Base configuration is valid. Networks/Volumes will be created when starting services." + else + echo "❌ Base configuration is invalid." + exit 1 + fi + exit 0 +fi + +SERVICE_FILE="$SERVICE.yml" + +if [ ! -f "$SERVICE_FILE" ]; then + echo "Error: Service file $SERVICE_FILE not found in workspace." + exit 1 +fi + +if [ "$ACTION" == "up" ]; then + # Bring up the service (along with base) + # We use base.yml from the temp dir as well + $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE_FILE" up -d --build + + echo "Waiting for service to initialize..." + sleep 5 +fi + +# Validation Checks +case $SERVICE in + setup) + echo "Checking Setup exit code..." + EXIT_CODE=$(docker inspect amp-setup --format='{{.State.ExitCode}}') + if [ "$EXIT_CODE" == "0" ]; then + echo "✅ Setup finished successfully." + echo "--- Certificates in 'certs-vol' ---" + # List contents of /certs in the volume using a temporary container (using setup service definition) + $DOCKER_COMPOSE_CMD -f base.yml -f setup.yml run --rm --entrypoint ls setup -lR /certs + echo "-----------------------------------" + echo "Note: Certificates are stored in the Docker volume 'certs-vol'." + else + echo "❌ Setup failed with exit code $EXIT_CODE." + fi + ;; + opensearch) + echo "Checking OpenSearch health..." + if docker ps | grep -q opensearch-node1; then + echo "✅ OpenSearch container is running." + + # Retries for Health Check + MAX_RETRIES=30 + COUNT=0 + PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} + + echo "Waiting for OpenSearch API to be ready (up to 60s)..." + while [ $COUNT -lt $MAX_RETRIES ]; do + docker exec opensearch-node1 curl -k -s -u admin:$PASSWORD https://localhost:9200/_cluster/health > /dev/null + if [ $? -eq 0 ]; then + echo "✅ OpenSearch API is healthy." + break + fi + echo " ... waiting for API ($((COUNT*2))s)" + sleep 2 + COUNT=$((COUNT+1)) + done + + if [ $COUNT -eq $MAX_RETRIES ]; then + echo "❌ OpenSearch API failed to respond after 60 seconds." + echo " Debugging info:" + docker logs --tail 20 opensearch-node1 + fi + else + echo "❌ OpenSearch container is not running." + fi + ;; + timescaledb) + echo "Checking TimescaleDB readiness..." + sleep 5 + if docker exec timescaledb pg_isready; then + echo "✅ TimescaleDB is ready." + else + echo "❌ TimescaleDB is not ready." + echo " Debugging info:" + docker logs --tail 20 timescaledb + fi + ;; + pgbouncer) + echo "Checking PgBouncer..." + if docker ps | grep -q pgbouncer; then + echo "✅ PgBouncer container is running." + # Need a way to check port inside? + # Using /bin/sh to check port open with timeout (nc might not be installed) + # docker exec pgbouncer nc -z localhost 6432 + else + echo "❌ PgBouncer container is not running." + fi + ;; + grafana) + echo "Checking Grafana..." + + MAX_RETRIES=30 + COUNT=0 + + echo "Waiting for Grafana to be ready (up to 60s)..." + while [ $COUNT -lt $MAX_RETRIES ]; do + curl -s -I http://localhost:3000/login > /dev/null + if [ $? -eq 0 ]; then + echo "✅ Grafana login page is accessible." + break + fi + echo " ... waiting for Grafana ($((COUNT*2))s)" + sleep 2 + COUNT=$((COUNT+1)) + done + + if [ $COUNT -eq $MAX_RETRIES ]; then + echo "❌ Grafana login page check failed." + echo " Debugging info:" + docker logs --tail 20 grafana + fi + ;; + opensearch-dashboards) + echo "Checking OpenSearch Dashboards (waiting for startup)..." + + MAX_RETRIES=30 + COUNT=0 + + echo "Waiting for OpenSearch Dashboards to be ready (up to 60s)..." + while [ $COUNT -lt $MAX_RETRIES ]; do + # 5601 might not be exposed to host in docker-compose.yml (networks only) + # So we use docker exec to check inside + docker exec opensearch-dashboards curl -s -I -k https://localhost:5601 > /dev/null + if [ $? -eq 0 ]; then + echo "✅ OpenSearch Dashboards is running." + break + fi + echo " ... waiting for Dashboards ($((COUNT*2))s)" + sleep 2 + COUNT=$((COUNT+1)) + done + + if [ $COUNT -eq $MAX_RETRIES ]; then + echo "❌ OpenSearch Dashboards check failed." + echo " Debugging info:" + docker logs --tail 20 opensearch-dashboards + fi + ;; + + nginx) + echo "Checking Nginx..." + + MAX_RETRIES=30 + COUNT=0 + + echo "Waiting for Nginx to be ready (up to 60s)..." + while [ $COUNT -lt $MAX_RETRIES ]; do + curl -s -k -I https://localhost > /dev/null + if [ $? -eq 0 ]; then + echo "✅ Nginx is reachable." + break + fi + echo " ... waiting for Nginx ($((COUNT*2))s)" + sleep 2 + COUNT=$((COUNT+1)) + done + + if [ $COUNT -eq $MAX_RETRIES ]; then + echo "❌ Nginx check failed." + echo " Debugging info:" + docker logs --tail 20 nginx + fi + ;; + configurator) + echo "Checking Configurator..." + # Configurator is a job that requires OpenSearch. In isolated validation, it will likely fail. + # We check if it attempted to run. + sleep 5 + STATUS=$(docker inspect amp-configurator --format='{{.State.Status}}') + EXIT_CODE=$(docker inspect amp-configurator --format='{{.State.ExitCode}}') + + echo "Configurator Status: $STATUS (Exit Code: $EXIT_CODE)" + + if [ "$STATUS" == "exited" ]; then + # It's okay if it fails (non-zero) because OpenSearch isn't here. + echo "✅ Configurator container was created and attempted execution." + if [ "$EXIT_CODE" != "0" ]; then + echo " (Note: Exit code $EXIT_CODE is expected in isolation as OpenSearch is unreachable)" + fi + elif [ "$STATUS" == "running" ]; then + echo "✅ Configurator is running." + else + echo "❌ Configurator status is $STATUS." + fi + ;; + *) + echo "No specific validation script for $SERVICE, verifying container status..." + if docker ps -a | grep -q $SERVICE; then + echo "✅ Service $SERVICE appears to be created/running." + else + echo "⚠️ Service $SERVICE container not found." + fi + ;; +esac + +echo "--- Done ---" Property changes on: platform/tools/container/legacy/manage_service.sh ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Index: /branches/amp_4_0/platform/tools/container/manage_amp.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/manage_amp.sh (nonexistent) +++ /branches/amp_4_0/platform/tools/container/manage_amp.sh (working copy) @@ -0,0 +1,458 @@ +#!/bin/bash + +# manage_swarm.sh +# Usage: ./manage_swarm.sh [action] + +ACTION=${1:-status} +STACK_NAME="amp" +REGISTRY="localhost:5000" + +# REPO_ROOT Resolution +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Resolve SERVICES_DIR relative to the script +SERVICES_DIR="$(cd "$SCRIPT_DIR/services" && pwd)" +STACK_FILE="$SCRIPT_DIR/stack.yml" + +echo "Using Services Directory: $SERVICES_DIR" + +# Load .env file +ENV_FILE="$SCRIPT_DIR/.env" +if [ -f "$ENV_FILE" ]; then + echo "Loading environment variables from $ENV_FILE" + set -a + source "$ENV_FILE" + set +a +else + echo "⚠️ .env file not found at $ENV_FILE" +fi + + +# Function: Calculate Dynamic Heap Size (50% of Host RAM) +calculate_heap_size() { + # If explicitly set in .env or shell, skip auto-calc + if [ -n "$OPENSEARCH_JAVA_OPTS" ]; then + echo "Using Memory Override from .env: $OPENSEARCH_JAVA_OPTS" + return + fi + + echo "--- Calculating Memory Allocation ---" + TOTAL_MEM_MB=0 + + # Detect OS to get Total Memory + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + TOTAL_MEM_BYTES=$(sysctl -n hw.memsize) + TOTAL_MEM_MB=$((TOTAL_MEM_BYTES / 1024 / 1024)) + else + # Linux + if [ -f /proc/meminfo ]; then + TOTAL_MEM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}') + TOTAL_MEM_MB=$((TOTAL_MEM_KB / 1024)) + else + echo "⚠️ Cannot detect memory (neither macOS nor Linux /proc/meminfo). Defaulting to 1GB." + TOTAL_MEM_MB=2048 # Fallback assumes at least 2GB machine -> 1GB heap + fi + fi + + echo "Host Total Memory: ${TOTAL_MEM_MB}MB" + + # 50% Rule + HEAP_SIZE_MB=$((TOTAL_MEM_MB / 2)) + + # Min/Max Clamping + # Min: 1GB (1024MB) + # Max: 32GB (32768MB) - JVM compressed oops limit + MIN_HEAP=1024 + MAX_HEAP=32768 + + if [ "$HEAP_SIZE_MB" -lt "$MIN_HEAP" ]; then + echo "⚠️ Calculated Heap (${HEAP_SIZE_MB}MB) is below minimum. Setting to 1GB." + HEAP_SIZE_MB=$MIN_HEAP + elif [ "$HEAP_SIZE_MB" -gt "$MAX_HEAP" ]; then + echo "⚠️ Calculated Heap (${HEAP_SIZE_MB}MB) exceeds 32GB. Clamping to 32GB." + HEAP_SIZE_MB=$MAX_HEAP + fi + + export OPENSEARCH_JAVA_OPTS="-Xms${HEAP_SIZE_MB}m -Xmx${HEAP_SIZE_MB}m" + echo "✅ Auto-Allocated OpenSearch Heap: $OPENSEARCH_JAVA_OPTS" + + # --- Logstash Memory (25% of Host RAM) --- + if [ -n "$LOGSTASH_JAVA_OPTS" ]; then + echo "Using Logstash Memory Override from .env: $LOGSTASH_JAVA_OPTS" + else + LS_HEAP_SIZE_MB=$((TOTAL_MEM_MB / 4)) + # Clamp Logstash: Min 1GB, Max 8GB + MIN_LS=1024 + MAX_LS=8192 + + if [ "$LS_HEAP_SIZE_MB" -lt "$MIN_LS" ]; then + LS_HEAP_SIZE_MB=$MIN_LS + elif [ "$LS_HEAP_SIZE_MB" -gt "$MAX_LS" ]; then + LS_HEAP_SIZE_MB=$MAX_LS + fi + export LOGSTASH_JAVA_OPTS="-Xms${LS_HEAP_SIZE_MB}m -Xmx${LS_HEAP_SIZE_MB}m" + echo "✅ Auto-Allocated Logstash Heap: $LOGSTASH_JAVA_OPTS" + fi +} + + +# Function: Generate Dynamic Configuration Files +generate_configs() { + echo "--- Generating Dynamic Configurations ---" + + # Generate PgBouncer Config from Template + PGB_DIR="$SERVICES_DIR/pgbouncer" + TEMPLATE="$PGB_DIR/pgbouncer.ini.template" + OUTPUT="$PGB_DIR/pgbouncer.ini" + USERLIST="$PGB_DIR/userlist.txt" + + if [ -f "$TEMPLATE" ]; then + echo "Generating pgbouncer.ini from template..." + cp "$TEMPLATE" "$OUTPUT" + + # Helper for OS-aware sed + replace_var() { + local search=$1 + local replace=$2 + local file=$3 + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i "" "s|${search}|${replace}|g" "$file" + else + sed -i "s|${search}|${replace}|g" "$file" + fi + } + + replace_var "__AMP_ADMIN_PASSWORD__" "$POSTGRES_PASSWORD" "$OUTPUT" + replace_var "__AMP_DB_NAME__" "$AMP_DB_NAME" "$OUTPUT" + replace_var "__AMP_DB_USER__" "$AMP_DB_USER" "$OUTPUT" + replace_var "__AMP_DB_PASSWORD__" "$AMP_DB_PASSWORD" "$OUTPUT" + + echo "✅ Generated $OUTPUT" + fi + + # Generate Userlist (MD5 format: "md5" + md5(password + username)) + echo "Generating userlist.txt..." + + # Helper to calc md5 + calc_md5() { + local user=$1 + local pass=$2 + if [[ "$OSTYPE" == "darwin"* ]]; then + echo -n "${pass}${user}" | md5 -q + else + echo -n "${pass}${user}" | md5sum | awk '{print $1}' + fi + } + + # Admin User (amp_admin) using POSTGRES_PASSWORD + MD5_ADMIN=$(calc_md5 "amp_admin" "$POSTGRES_PASSWORD") + # App User (amp_ts_user) + MD5_APP=$(calc_md5 "$AMP_DB_USER" "$AMP_DB_PASSWORD") + + # Write file + echo "\"amp_admin\" \"md5$MD5_ADMIN\"" > "$USERLIST" + echo "\"$AMP_DB_USER\" \"md5$MD5_APP\"" >> "$USERLIST" + + echo "✅ Generated $USERLIST" +} + +cleanup() { + echo "" + echo "⚠️ Caught Signal. Cleaning up ephemeral containers..." + docker rm -f amp-setup-ephemeral amp-configurator-ephemeral 2>/dev/null + exit 1 +} +trap cleanup INT TERM + +check_swarm() { + if ! docker info | grep -q "Swarm: active"; then + echo "❌ Docker Swarm is NOT active." + echo "Run './manage_swarm.sh init' to initialize." + exit 1 + fi +} + +init_swarm() { + if docker info | grep -q "Swarm: active"; then + echo "✅ Swarm is already active." + else + echo "Initializing Docker Swarm..." + docker swarm init + fi + + # Label this node as storage node for pinning + NODE_ID=$(docker info --format '{{.Swarm.NodeID}}') + echo "Labeling node $NODE_ID with type=storage..." + docker node update --label-add type=storage $NODE_ID +} + +ensure_registry() { + if ! docker ps | grep -q "registry"; then + echo "Starting Local Registry ($REGISTRY)..." + docker run -d -p 5000:5000 --restart=always --name registry registry:2 + else + echo "✅ Local Registry is running." + fi +} + +build_and_push() { + ensure_registry + echo "--- Mirroring Images (Pull -> Tag -> Push) ---" + + # Map service names to upstream images + # Format: "service_name:upstream_image" + IMAGES=( + "opensearch:opensearchproject/opensearch:2.11.0" + "timescaledb:timescale/timescaledb:latest-pg16" + "pgbouncer:edoburu/pgbouncer:latest" + "grafana:grafana/grafana:latest" + "telegraf:telegraf:latest" + "logstash:opensearchproject/logstash-oss-with-opensearch-output-plugin:latest" + "nginx:nginx:alpine" + "opensearch-dashboards:opensearchproject/opensearch-dashboards:2.11.0" + ) + + for INFO in "${IMAGES[@]}"; do + SERVICE_NAME="${INFO%%:*}" + UPSTREAM="${INFO#*:}" + LOCAL_TAG="$REGISTRY/amp/$SERVICE_NAME:latest" + + echo "Processing $SERVICE_NAME..." + echo " - Pulling $UPSTREAM" + docker pull -q $UPSTREAM + + echo " - Tagging as $LOCAL_TAG" + docker tag $UPSTREAM $LOCAL_TAG + + echo " - Pushing to Registry..." + docker push -q $LOCAL_TAG + echo " ✅ Done." + done +} + +create_secrets() { + echo "--- strict Checking/Creating Secrets ---" + + # Helper to create secret if missing + create_secret_if_missing() { + SECRET_NAME=$1 + SECRET_VALUE=$2 + if ! docker secret ls | grep -q "$SECRET_NAME"; then + printf "$SECRET_VALUE" | docker secret create $SECRET_NAME - + echo "Created secret: $SECRET_NAME" + else + echo "Secret $SECRET_NAME exists." + fi + } + + # Passwords from .env + create_secret_if_missing "opensearch_initial_admin_password" "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin}" + create_secret_if_missing "pg_password" "${POSTGRES_PASSWORD:-postgres}" + create_secret_if_missing "opensearch_jwt_secret" "${OPENSEARCH_JWT_SECRET:-supersecretjwtkey}" +} + +deploy_stack() { + # Detect Host IP for Grafana Domain if not set + detect_host_ip() { + if [ -n "$AMP_DOMAIN_OR_IP" ]; then + echo "Using configured AMP_DOMAIN_OR_IP: $AMP_DOMAIN_OR_IP" + return + fi + + echo "--- Detecting Host IP ---" + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + HOST_IP=$(ipconfig getifaddr en0) + if [ -z "$HOST_IP" ]; then HOST_IP=$(ipconfig getifaddr en1); fi + else + # Linux (hostname -I | first ip) + HOST_IP=$(hostname -I | awk '{print $1}') + fi + + if [ -z "$HOST_IP" ]; then + echo "⚠️ Could not auto-detect IP. Defaulting to localhost." + HOST_IP="localhost" + fi + + export AMP_DOMAIN_OR_IP="$HOST_IP" + echo "✅ Auto-Detected Host IP: $AMP_DOMAIN_OR_IP" + } + + check_swarm + create_secrets + detect_host_ip + create_secrets + calculate_heap_size + generate_configs + + # Allow user to override REGISTRY for multi-node (e.g. export REGISTRY=192.168.1.5:5000) + export REGISTRY=${REGISTRY:-127.0.0.1:5000} + echo "--- Deploying Stack: $STACK_NAME (Registry: $REGISTRY) ---" + + # Create external volumes if missing + CERTS_VOL_CREATED=false + for vol in certs-vol security-config-vol; do + if ! docker volume ls | grep -q "$vol"; then + docker volume create $vol + echo "Created volume: $vol" + if [ "$vol" == "certs-vol" ]; then + CERTS_VOL_CREATED=true + fi + fi + done + + if $CERTS_VOL_CREATED; then + echo "⚠️ Volume 'certs-vol' was just created and is empty." + echo "⚠️ You MUST run './manage_swarm.sh setup' to generate certificates, otherwise services will fail." + fi + + docker stack deploy -c "$STACK_FILE" $STACK_NAME +} + +rm_stack() { + echo "Removing Stack $STACK_NAME..." + docker stack rm $STACK_NAME +} + +run_setup() { + echo "--- Running Setup (Generating Certificates) ---" + # Ensure volumes exist + if ! docker volume ls | grep -q "certs-vol"; then + docker volume create certs-vol + fi + if ! docker volume ls | grep -q "security-config-vol"; then + docker volume create security-config-vol + fi + + # Run setup container (ephemeral) + # We mount the local services/setup dir to /setup + docker run --rm --name amp-setup-ephemeral \ + -v "$SERVICES_DIR/setup:/setup:rw" \ + -v "certs-vol:/certs:rw" \ + -v "security-config-vol:/security-config:rw" \ + -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin}" \ + -e OPENSEARCH_JWT_SECRET="${OPENSEARCH_JWT_SECRET:-supersecretjwtkey}" \ + rockylinux:9 \ + /bin/sh -c "dnf install -y openssl httpd-tools && chmod +x /setup/setup.sh && /setup/setup.sh /certs /security-config && chown -R 1000:1000 /certs && chown -R 1000:1000 /security-config" & + + PID=$! + wait $PID + + + echo "✅ Setup complete. Certificates generated in 'certs-vol' (Ownership set to 1000:1000)." +} + +security_init() { + echo "--- Initializing OpenSearch Security Index ---" + # Find running opensearch container (exclude dashboards) + # We use --format to check names but output ID + CONTAINER_ID=$(docker ps --format '{{.ID}} {{.Names}}' | grep "amp_opensearch" | grep -v "dashboards" | awk '{print $1}' | head -n 1) + + if [ -z "$CONTAINER_ID" ]; then + echo "❌ OpenSearch container not found. Please run 'deploy' and wait for it to start." + exit 1 + fi + + echo "Found OpenSearch container: $CONTAINER_ID" + echo "Running securityadmin.sh..." + + # 1. Copy default configs to /tmp/sec-config + # 2. Overwrite with custom configs from setup (config.yml, internal_users.yml) + # 3. Run securityadmin.sh + docker exec "$CONTAINER_ID" bash -c " + mkdir -p /tmp/sec-config && \ + cp /usr/share/opensearch/config/opensearch-security/* /tmp/sec-config/ && \ + cp /usr/share/opensearch/config/opensearch-security-mount/* /tmp/sec-config/ && \ + chmod +x /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh && \ + /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ + -cd /tmp/sec-config \ + -icl \ + -nhnv \ + --accept-red-cluster \ + -h localhost \ + -p 9200 \ + -cacert /usr/share/opensearch/config/certs/root-ca.pem \ + -cert /usr/share/opensearch/config/certs/admin.pem \ + -key /usr/share/opensearch/config/certs/admin-key.pem + " & + + PID=$! + wait $PID + + + if [ $? -eq 0 ]; then + echo "✅ Security Index Initialized Successfully." + else + echo "❌ Security Initialization Failed." + exit 1 + fi +} + +run_configurator() { + echo "--- Running Configurator (OpenSearch Setup) ---" + + # Needs to run on the overlay network to reach 'opensearch' service + # and mount certs to create the security roles using curl + + check_swarm + + # We use docker run attached to the overlay network + # Note: overlay networks are only attachable if 'attachable: true' is set in stack.yml (it is). + + echo "Waiting for OpenSearch to be reachable..." + # A simple check before running configurator could go here, or let the script fail if not ready. + + docker run --rm --name amp-configurator-ephemeral \ + --network=${STACK_NAME}_amp-overlay \ + -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin}" \ + -e OPENSEARCH_URL="https://opensearch:9200" \ + -e OPENSEARCH_DASHBOARDS_URL="https://opensearch-dashboards:5601" \ + -v "certs-vol:/usr/share/opensearch/config/certs:ro" \ + -v "$SERVICES_DIR/setup:/setup:ro" \ + -v "$SERVICES_DIR/setup/install_curl.sh:/usr/local/bin/install_curl.sh" \ + -v "$SERVICES_DIR/setup/configure_opensearch.sh:/usr/local/bin/configure_opensearch.sh" \ + -v "$SERVICES_DIR/opensearch/amplog_template.json:/usr/share/opensearch/config/amplog_template.json" \ + -v "$SERVICES_DIR/opensearch-dashboards/export.ndjson:/usr/share/opensearch/config/export.ndjson" \ + rockylinux:9 \ + /bin/sh -c "bash /usr/local/bin/install_curl.sh && bash /usr/local/bin/configure_opensearch.sh" & + + PID=$! + wait $PID + + + echo "✅ Configurator finished." +} + +case $ACTION in + init) + init_swarm + ensure_registry + ;; + build) + build_and_push + ;; + registry) + ensure_registry + ;; + setup) + run_setup + ;; + configurator) + run_configurator + ;; + security_init) + security_init + ;; + deploy) + deploy_stack + ;; + rm|remove) + rm_stack + ;; + status) + docker stack services $STACK_NAME + ;; + *) + echo "Usage: $0 {init|build|setup|deploy|security_init|status|configurator|rm}" + exit 1 +esac Property changes on: platform/tools/container/manage_amp.sh ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Index: /branches/amp_4_0/platform/tools/container/manage_service.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/manage_service.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/container/manage_service.sh (nonexistent) @@ -1,303 +0,0 @@ -#!/bin/bash - -# validate_service.sh -# Usage: ./validate_service.sh [service_name] - -SERVICE=$1 - -# REPO_ROOT Resolution -# Assuming script is in platform/tools/container/ -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../../../" && pwd)" - -SPLIT_DIR_NAME="compose" -COMPOSE_DIR="$SCRIPT_DIR/$SPLIT_DIR_NAME" - -# Detect docker-compose command -if command -v docker-compose &> /dev/null; then - DOCKER_COMPOSE_CMD="docker-compose" -elif docker compose version &> /dev/null; then - DOCKER_COMPOSE_CMD="docker compose" -else - echo "Error: neither 'docker-compose' nor 'docker compose' found." - exit 1 -fi -echo "Using Docker Compose command: $DOCKER_COMPOSE_CMD" -export COMPOSE_IGNORE_ORPHANS=True - -# Detect Docker GID for Telegraf and other services needing socket access -if [ -S /var/run/docker.sock ]; then - # Compatibility: stat flags differ between GNU and BSD (macOS) - if [[ "$OSTYPE" == "darwin"* ]]; then - DOCKER_GID=$(stat -f '%g' /var/run/docker.sock) - else - DOCKER_GID=$(stat -c '%g' /var/run/docker.sock) - fi - export DOCKER_GID - echo "Detected Docker GID: $DOCKER_GID" -fi - -if [ -z "$SERVICE" ]; then - echo "Usage: ./manage_service.sh [service_name] [action]" - echo "AMP Service Manager" - echo "" - echo "Available services:" - echo " base, setup, opensearch, configurator, opensearch-dashboards, timescaledb," - echo " pgbouncer, grafana, telegraf, logstash, nginx" - echo "" - echo "Actions:" - echo " up - Start/Build service (default)" - echo " down - Stop/Remove service" - echo " purge - Stop/Remove service AND volumes (Fresh Start)" - echo " validate - Run health checks only" - exit 1 -fi - -ACTION=${2:-up} - -# Check Docker Permissions -if ! docker ps >/dev/null 2>&1; then - echo "❌ Error: Docker permission denied or Docker is not running." - exit 1 -fi - -echo "--- Processing Service: $SERVICE ($ACTION) ---" - -# Ensure external volume 'certs-vol' exists -if ! docker volume ls -q | grep -q "^certs-vol$"; then - echo "Creating external volume: certs-vol" - docker volume create certs-vol -fi - -# Move to the Compose Directory -cd "$COMPOSE_DIR" || exit 1 - -# Load .env if it exists in the script directory -if [ -f "$SCRIPT_DIR/.env" ]; then - # Note: Docker Compose loads .env automatically if it's in the same dir as the yaml files. - # If it's in the parent, we might need to symlink it or specify --env-file - if [ ! -f ".env" ]; then - ln -s "$SCRIPT_DIR/.env" .env - echo "DEBUG: Symlinked .env from parent directory." - fi -fi - -if [ "$ACTION" == "down" ] || [ "$ACTION" == "purge" ]; then - if [ "$ACTION" == "purge" ]; then - echo "Stopping $SERVICE and removing volumes..." - $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE.yml" down -v - else - $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE.yml" down - fi - echo "Service $SERVICE stopped." - exit 0 -fi - -if [ "$SERVICE" == "base" ]; then - echo "Validating Base configuration..." - # 'base' only has volumes/networks, 'up' fails in v2 if no services. - # We trust that subsequent service runs (which include base.yml) will create them. - # We just check config validity here. - $DOCKER_COMPOSE_CMD -f base.yml config >/dev/null - if [ $? -eq 0 ]; then - echo "✅ Base configuration is valid. Networks/Volumes will be created when starting services." - else - echo "❌ Base configuration is invalid." - exit 1 - fi - exit 0 -fi - -SERVICE_FILE="$SERVICE.yml" - -if [ ! -f "$SERVICE_FILE" ]; then - echo "Error: Service file $SERVICE_FILE not found in workspace." - exit 1 -fi - -if [ "$ACTION" == "up" ]; then - # Bring up the service (along with base) - # We use base.yml from the temp dir as well - $DOCKER_COMPOSE_CMD -f base.yml -f "$SERVICE_FILE" up -d --build - - echo "Waiting for service to initialize..." - sleep 5 -fi - -# Validation Checks -case $SERVICE in - setup) - echo "Checking Setup exit code..." - EXIT_CODE=$(docker inspect amp-setup --format='{{.State.ExitCode}}') - if [ "$EXIT_CODE" == "0" ]; then - echo "✅ Setup finished successfully." - echo "--- Certificates in 'certs-vol' ---" - # List contents of /certs in the volume using a temporary container (using setup service definition) - $DOCKER_COMPOSE_CMD -f base.yml -f setup.yml run --rm --entrypoint ls setup -lR /certs - echo "-----------------------------------" - echo "Note: Certificates are stored in the Docker volume 'certs-vol'." - else - echo "❌ Setup failed with exit code $EXIT_CODE." - fi - ;; - opensearch) - echo "Checking OpenSearch health..." - if docker ps | grep -q opensearch-node1; then - echo "✅ OpenSearch container is running." - - # Retries for Health Check - MAX_RETRIES=30 - COUNT=0 - PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-admin} - - echo "Waiting for OpenSearch API to be ready (up to 60s)..." - while [ $COUNT -lt $MAX_RETRIES ]; do - docker exec opensearch-node1 curl -k -s -u admin:$PASSWORD https://localhost:9200/_cluster/health > /dev/null - if [ $? -eq 0 ]; then - echo "✅ OpenSearch API is healthy." - break - fi - echo " ... waiting for API ($((COUNT*2))s)" - sleep 2 - COUNT=$((COUNT+1)) - done - - if [ $COUNT -eq $MAX_RETRIES ]; then - echo "❌ OpenSearch API failed to respond after 60 seconds." - echo " Debugging info:" - docker logs --tail 20 opensearch-node1 - fi - else - echo "❌ OpenSearch container is not running." - fi - ;; - timescaledb) - echo "Checking TimescaleDB readiness..." - sleep 5 - if docker exec timescaledb pg_isready; then - echo "✅ TimescaleDB is ready." - else - echo "❌ TimescaleDB is not ready." - echo " Debugging info:" - docker logs --tail 20 timescaledb - fi - ;; - pgbouncer) - echo "Checking PgBouncer..." - if docker ps | grep -q pgbouncer; then - echo "✅ PgBouncer container is running." - # Need a way to check port inside? - # Using /bin/sh to check port open with timeout (nc might not be installed) - # docker exec pgbouncer nc -z localhost 6432 - else - echo "❌ PgBouncer container is not running." - fi - ;; - grafana) - echo "Checking Grafana..." - - MAX_RETRIES=30 - COUNT=0 - - echo "Waiting for Grafana to be ready (up to 60s)..." - while [ $COUNT -lt $MAX_RETRIES ]; do - curl -s -I http://localhost:3000/login > /dev/null - if [ $? -eq 0 ]; then - echo "✅ Grafana login page is accessible." - break - fi - echo " ... waiting for Grafana ($((COUNT*2))s)" - sleep 2 - COUNT=$((COUNT+1)) - done - - if [ $COUNT -eq $MAX_RETRIES ]; then - echo "❌ Grafana login page check failed." - echo " Debugging info:" - docker logs --tail 20 grafana - fi - ;; - opensearch-dashboards) - echo "Checking OpenSearch Dashboards (waiting for startup)..." - - MAX_RETRIES=30 - COUNT=0 - - echo "Waiting for OpenSearch Dashboards to be ready (up to 60s)..." - while [ $COUNT -lt $MAX_RETRIES ]; do - # 5601 might not be exposed to host in docker-compose.yml (networks only) - # So we use docker exec to check inside - docker exec opensearch-dashboards curl -s -I -k https://localhost:5601 > /dev/null - if [ $? -eq 0 ]; then - echo "✅ OpenSearch Dashboards is running." - break - fi - echo " ... waiting for Dashboards ($((COUNT*2))s)" - sleep 2 - COUNT=$((COUNT+1)) - done - - if [ $COUNT -eq $MAX_RETRIES ]; then - echo "❌ OpenSearch Dashboards check failed." - echo " Debugging info:" - docker logs --tail 20 opensearch-dashboards - fi - ;; - - nginx) - echo "Checking Nginx..." - - MAX_RETRIES=30 - COUNT=0 - - echo "Waiting for Nginx to be ready (up to 60s)..." - while [ $COUNT -lt $MAX_RETRIES ]; do - curl -s -k -I https://localhost > /dev/null - if [ $? -eq 0 ]; then - echo "✅ Nginx is reachable." - break - fi - echo " ... waiting for Nginx ($((COUNT*2))s)" - sleep 2 - COUNT=$((COUNT+1)) - done - - if [ $COUNT -eq $MAX_RETRIES ]; then - echo "❌ Nginx check failed." - echo " Debugging info:" - docker logs --tail 20 nginx - fi - ;; - configurator) - echo "Checking Configurator..." - # Configurator is a job that requires OpenSearch. In isolated validation, it will likely fail. - # We check if it attempted to run. - sleep 5 - STATUS=$(docker inspect amp-configurator --format='{{.State.Status}}') - EXIT_CODE=$(docker inspect amp-configurator --format='{{.State.ExitCode}}') - - echo "Configurator Status: $STATUS (Exit Code: $EXIT_CODE)" - - if [ "$STATUS" == "exited" ]; then - # It's okay if it fails (non-zero) because OpenSearch isn't here. - echo "✅ Configurator container was created and attempted execution." - if [ "$EXIT_CODE" != "0" ]; then - echo " (Note: Exit code $EXIT_CODE is expected in isolation as OpenSearch is unreachable)" - fi - elif [ "$STATUS" == "running" ]; then - echo "✅ Configurator is running." - else - echo "❌ Configurator status is $STATUS." - fi - ;; - *) - echo "No specific validation script for $SERVICE, verifying container status..." - if docker ps -a | grep -q $SERVICE; then - echo "✅ Service $SERVICE appears to be created/running." - else - echo "⚠️ Service $SERVICE container not found." - fi - ;; -esac - -echo "--- Done ---" Property changes on: platform/tools/container/manage_service.sh ___________________________________________________________________ Deleted: svn:executable ## -1 +0,0 ## -* \ No newline at end of property Index: /branches/amp_4_0/platform/tools/container/services/grafana/provisioning/datasources/datasources.yaml =================================================================== --- /branches/amp_4_0/platform/tools/container/services/grafana/provisioning/datasources/datasources.yaml (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/grafana/provisioning/datasources/datasources.yaml (working copy) @@ -4,23 +4,35 @@ - name: PostgreSQL type: postgres url: timescaledb:5432 - user: postgres # Using superuser or custom read-only user + user: ${DS_POSTGRES_USER} secureJsonData: - password: postgres # Should match .env + password: ${DS_POSTGRES_PASSWORD} jsonData: - database: postgres + database: cm sslmode: "disable" timescaledb: true version: 16 isDefault: true + - name: TimescaleDB + type: postgres + url: timescaledb:5432 + user: ${DS_AMP_DB_USER} + secureJsonData: + password: ${DS_AMP_DB_PASSWORD} + jsonData: + database: ${DS_AMP_DB_NAME} + sslmode: "disable" + timescaledb: true + version: 16 + - name: OpenSearch type: grafana-opensearch-datasource access: proxy - url: https://opensearch-node1:9200 + url: https://opensearch:9200 basicAuth: true - basicAuthUser: admin - basicAuthPassword: Arr@y2050 # Should match .env + basicAuthUser: ${DS_OS_USER} + basicAuthPassword: ${DS_OS_PASSWORD} jsonData: timeField: "@timestamp" tlsSkipVerify: true Index: /branches/amp_4_0/platform/tools/container/services/nginx/conf.d/app.conf =================================================================== --- /branches/amp_4_0/platform/tools/container/services/nginx/conf.d/app.conf (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/nginx/conf.d/app.conf (working copy) @@ -10,9 +10,16 @@ server { listen 80; listen [::]:80; - server_name localhost; + server_name _; - return 301 https://$host$request_uri; + location /health { + return 200 "OK\n"; + add_header Content-Type text/plain; + } + + location / { + return 301 https://$host$request_uri; + } } # --- HTTPS Server Block --- Index: /branches/amp_4_0/platform/tools/container/services/opensearch/opensearch.yml =================================================================== --- /branches/amp_4_0/platform/tools/container/services/opensearch/opensearch.yml (nonexistent) +++ /branches/amp_4_0/platform/tools/container/services/opensearch/opensearch.yml (working copy) @@ -0,0 +1,29 @@ +cluster.name: opensearch-cluster +network.host: 0.0.0.0 + +# Bind to all interfaces +transport.host: 0.0.0.0 + +# Disable memory lock (swarm usually doesn't allow memlock unless configured) +bootstrap.memory_lock: false + +# Single node discovery for now (as per stack.yml) +discovery.type: single-node + +# --- Security Configuration --- +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemkey_filepath: /usr/share/opensearch/config/certs/node-key.pem +plugins.security.ssl.http.pemcert_filepath: /usr/share/opensearch/config/certs/node.pem +plugins.security.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca.pem + +plugins.security.ssl.transport.enabled: true +plugins.security.ssl.transport.pemkey_filepath: /usr/share/opensearch/config/certs/node-key.pem +plugins.security.ssl.transport.pemcert_filepath: /usr/share/opensearch/config/certs/node.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: /usr/share/opensearch/config/certs/root-ca.pem + +# Allowed Admin DNs (Fixing the parsing issue) +plugins.security.authcz.admin_dn: + - "CN=admin,O=OpenSearchAdmin,L=Bengaluru,ST=Karnataka,C=IN" + +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.allow_default_init_securityindex: true Index: /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini =================================================================== --- /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini (nonexistent) @@ -1,22 +0,0 @@ -[databases] -cm = host=timescaledb port=5432 dbname=cm user=amp_admin password=Array@123$ -amp_ts = host=timescaledb port=5432 dbname=amp_ts user=amp_ts_user password=Array@123$ - -[pgbouncer] -listen_addr = 0.0.0.0 -logfile = /var/log/pgbouncer/pgbouncer.log -listen_port = 6432 -auth_type = md5 -auth_file = /etc/pgbouncer/userlist.txt -pool_mode = transaction -max_client_conn = 1000 -default_pool_size = 50 -reserve_pool_size = 10 -log_connections = 1 -log_disconnections = 1 -log_pooler_errors = 1 -stats_period = 60 -verbose = 0 -admin_users = amp_admin -stats_users = amp_admin -ignore_startup_parameters = extra_float_digits Index: /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini.template =================================================================== --- /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini.template (nonexistent) +++ /branches/amp_4_0/platform/tools/container/services/pgbouncer/pgbouncer.ini.template (working copy) @@ -0,0 +1,18 @@ +[databases] +cm = host=timescaledb port=5432 dbname=cm user=amp_admin password=__AMP_ADMIN_PASSWORD__ +amp_ts = host=timescaledb port=5432 dbname=__AMP_DB_NAME__ user=__AMP_DB_USER__ password=__AMP_DB_PASSWORD__ + +[pgbouncer] +listen_port = 6432 +listen_addr = * +auth_type = md5 +auth_file = /etc/pgbouncer/userlist.txt +pool_mode = session +max_client_conn = 100 +default_pool_size = 20 + +# Admin users +admin_users = amp_admin +stats_users = amp_admin + +ignore_startup_parameters = extra_float_digits Index: /branches/amp_4_0/platform/tools/container/services/pgbouncer/userlist.txt =================================================================== --- /branches/amp_4_0/platform/tools/container/services/pgbouncer/userlist.txt (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/pgbouncer/userlist.txt (nonexistent) @@ -1,2 +0,0 @@ -"amp_admin" "md507b5a1b329433696700cb7444c9b1392" -"amp_ts_user" "md5dfc2c68ba0156715b3d09d4351336c53" Index: /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sh (nonexistent) +++ /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sh (working copy) @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Dynamically create user and DB using Environment Variables +# Variables are injected by Docker from .env + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER $AMP_DB_USER WITH PASSWORD '$AMP_DB_PASSWORD'; + CREATE DATABASE $AMP_DB_NAME; + GRANT ALL PRIVILEGES ON DATABASE $AMP_DB_NAME TO $AMP_DB_USER; + + -- Explicitly create 'cm' database for configuration management + CREATE DATABASE cm; + GRANT ALL PRIVILEGES ON DATABASE cm TO $AMP_DB_USER; +EOSQL Property changes on: platform/tools/container/services/postgres/initdb.d/01_basics.sh ___________________________________________________________________ Added: svn:executable ## -0,0 +1 ## +* \ No newline at end of property Index: /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sql =================================================================== --- /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sql (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/01_basics.sql (nonexistent) @@ -1,15 +0,0 @@ --- 01_basics.sql --- Create secondary user and db (Timescale/Postgres image creates 'postgres' user by default) -CREATE USER amp_ts_user WITH PASSWORD 'Array@123$'; -CREATE DATABASE amp_ts; -CREATE DATABASE cm; -GRANT ALL PRIVILEGES ON DATABASE amp_ts TO amp_ts_user; --- GRANT ALL PRIVILEGES ON DATABASE cm TO amp_ts_user; -- Optional, main user is postgres usually - -\c amp_ts -CREATE EXTENSION IF NOT EXISTS timescaledb; - --- Basic schema for Logstash lookup --- Rely on init_db.sql to create the 'device' table with the correct schema -\c postgres --- CREATE TABLE IF NOT EXISTS device ... (REMOVED due to conflict with init_db.sql) Index: /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/02_telegraf_snmp.sql =================================================================== --- /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/02_telegraf_snmp.sql (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/02_telegraf_snmp.sql (working copy) @@ -737,3 +737,8 @@ SELECT add_retention_policy('asf_http_service', INTERVAL '180 days'); SELECT add_retention_policy('asf_https_service', INTERVAL '180 days'); COMMIT; + +-- Grant permissions to amp_ts_user so Grafana can see the tables +GRANT USAGE ON SCHEMA public TO amp_ts_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO amp_ts_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO amp_ts_user; Index: /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/03_docker_metrics.sql =================================================================== --- /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/03_docker_metrics.sql (nonexistent) +++ /branches/amp_4_0/platform/tools/container/services/postgres/initdb.d/03_docker_metrics.sql (working copy) @@ -0,0 +1,162 @@ +-- 03_docker_metrics.sql +-- Schema for Telegraf Docker Input Plugin +-- Tables: docker_container_mem, docker_container_cpu, docker_container_net, docker_container_blkio + +BEGIN; + +\c amp_ts + +-- 1) docker_container_mem +CREATE TABLE IF NOT EXISTS docker_container_mem ( + time TIMESTAMPTZ NOT NULL, + container_name TEXT, + container_image TEXT, + container_status TEXT, + engine_host TEXT, + server_address TEXT, + usage_percent DOUBLE PRECISION, + "usage" DOUBLE PRECISION, + "limit" DOUBLE PRECISION, + max_usage DOUBLE PRECISION, + active_anon DOUBLE PRECISION, + active_file DOUBLE PRECISION, + cache DOUBLE PRECISION, + hierarchical_memory_limit DOUBLE PRECISION, + inactive_anon DOUBLE PRECISION, + inactive_file DOUBLE PRECISION, + mapped_file DOUBLE PRECISION, + pgfault DOUBLE PRECISION, + pgmajfault DOUBLE PRECISION, + pgpgin DOUBLE PRECISION, + pgpgout DOUBLE PRECISION, + rss DOUBLE PRECISION, + rss_huge DOUBLE PRECISION, + unevictable DOUBLE PRECISION, + total_active_anon DOUBLE PRECISION, + total_active_file DOUBLE PRECISION, + total_cache DOUBLE PRECISION, + total_inactive_anon DOUBLE PRECISION, + total_inactive_file DOUBLE PRECISION, + total_mapped_file DOUBLE PRECISION, + total_pgfault DOUBLE PRECISION, + total_pgmajfault DOUBLE PRECISION, + total_pgpgin DOUBLE PRECISION, + total_pgpgout DOUBLE PRECISION, + total_rss DOUBLE PRECISION, + total_rss_huge DOUBLE PRECISION, + total_unevictable DOUBLE PRECISION, + total_writeback DOUBLE PRECISION, + writeback DOUBLE PRECISION +); +ALTER TABLE docker_container_mem OWNER TO amp_ts_user; +SELECT create_hypertable('docker_container_mem', 'time', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_docker_mem_container ON docker_container_mem (container_name, time DESC); + +-- 2) docker_container_cpu +CREATE TABLE IF NOT EXISTS docker_container_cpu ( + time TIMESTAMPTZ NOT NULL, + container_name TEXT, + container_image TEXT, + container_status TEXT, + engine_host TEXT, + server_address TEXT, + usage_percent DOUBLE PRECISION, + usage_total DOUBLE PRECISION, + usage_system DOUBLE PRECISION, + usage_user DOUBLE PRECISION, + throttling_periods DOUBLE PRECISION, + throttling_throttled_periods DOUBLE PRECISION, + throttling_throttled_time DOUBLE PRECISION +); +ALTER TABLE docker_container_cpu OWNER TO amp_ts_user; +SELECT create_hypertable('docker_container_cpu', 'time', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_docker_cpu_container ON docker_container_cpu (container_name, time DESC); + +-- 3) docker_container_net +CREATE TABLE IF NOT EXISTS docker_container_net ( + time TIMESTAMPTZ NOT NULL, + container_name TEXT, + container_image TEXT, + container_status TEXT, + engine_host TEXT, + server_address TEXT, + network TEXT, + rx_bytes DOUBLE PRECISION, + rx_dropped DOUBLE PRECISION, + rx_errors DOUBLE PRECISION, + rx_packets DOUBLE PRECISION, + tx_bytes DOUBLE PRECISION, + tx_dropped DOUBLE PRECISION, + tx_errors DOUBLE PRECISION, + tx_packets DOUBLE PRECISION +); +ALTER TABLE docker_container_net OWNER TO amp_ts_user; +SELECT create_hypertable('docker_container_net', 'time', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_docker_net_container ON docker_container_net (container_name, time DESC); + +-- 4) docker_container_blkio +CREATE TABLE IF NOT EXISTS docker_container_blkio ( + time TIMESTAMPTZ NOT NULL, + container_name TEXT, + container_image TEXT, + container_status TEXT, + engine_host TEXT, + server_address TEXT, + device TEXT, + io_service_bytes_recursive_async DOUBLE PRECISION, + io_service_bytes_recursive_read DOUBLE PRECISION, + io_service_bytes_recursive_sync DOUBLE PRECISION, + io_service_bytes_recursive_total DOUBLE PRECISION, + io_service_bytes_recursive_write DOUBLE PRECISION, + io_serviced_recursive_async DOUBLE PRECISION, + io_serviced_recursive_read DOUBLE PRECISION, + io_serviced_recursive_sync DOUBLE PRECISION, + io_serviced_recursive_total DOUBLE PRECISION, + io_serviced_recursive_write DOUBLE PRECISION +); +ALTER TABLE docker_container_blkio OWNER TO amp_ts_user; +SELECT create_hypertable('docker_container_blkio', 'time', if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_docker_blkio_container ON docker_container_blkio (container_name, time DESC); + +-- 5) docker (summary stats depending on telegraf version/config, sometimes just 'docker') +CREATE TABLE IF NOT EXISTS docker ( + time TIMESTAMPTZ NOT NULL, + engine_host TEXT, + server_address TEXT, + n_containers DOUBLE PRECISION, + n_containers_paused DOUBLE PRECISION, + n_containers_running DOUBLE PRECISION, + n_containers_stopped DOUBLE PRECISION, + n_images DOUBLE PRECISION, + n_goroutines DOUBLE PRECISION, + n_listener_events DOUBLE PRECISION, + memory_total DOUBLE PRECISION, + pool_blocksize DOUBLE PRECISION +); +ALTER TABLE docker OWNER TO amp_ts_user; +SELECT create_hypertable('docker', 'time', if_not_exists => TRUE); + +-- Compression Policies (7 days) +ALTER TABLE docker_container_mem SET (timescaledb.compress, timescaledb.compress_segmentby = 'container_name', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE docker_container_cpu SET (timescaledb.compress, timescaledb.compress_segmentby = 'container_name', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE docker_container_net SET (timescaledb.compress, timescaledb.compress_segmentby = 'container_name', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE docker_container_blkio SET (timescaledb.compress, timescaledb.compress_segmentby = 'container_name', timescaledb.compress_orderby = 'time DESC'); + +SELECT add_compression_policy('docker_container_mem', INTERVAL '7 days'); +SELECT add_compression_policy('docker_container_cpu', INTERVAL '7 days'); +SELECT add_compression_policy('docker_container_net', INTERVAL '7 days'); +SELECT add_compression_policy('docker_container_blkio', INTERVAL '7 days'); + +-- Retention Policies (30 days for raw metrics) +SELECT add_retention_policy('docker_container_mem', INTERVAL '30 days'); +SELECT add_retention_policy('docker_container_cpu', INTERVAL '30 days'); +SELECT add_retention_policy('docker_container_net', INTERVAL '30 days'); +SELECT add_retention_policy('docker_container_blkio', INTERVAL '30 days'); +SELECT add_retention_policy('docker', INTERVAL '30 days'); + +COMMIT; + +-- Grant permissions (Crucial) +GRANT USAGE, CREATE ON SCHEMA public TO amp_ts_user; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO amp_ts_user; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO amp_ts_user; Index: /branches/amp_4_0/platform/tools/container/services/setup/configure_opensearch.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/services/setup/configure_opensearch.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/setup/configure_opensearch.sh (working copy) @@ -2,18 +2,57 @@ # configure_opensearch.sh # Waits for OpenSearch to be ready, then applies security roles/mappings -OPENSEARCH_URL="https://opensearch-node1:9200" +OPENSEARCH_URL="${OPENSEARCH_URL:-https://opensearch:9200}" +OPENSEARCH_DASHBOARDS_URL="${OPENSEARCH_DASHBOARDS_URL:-https://opensearch-dashboards:5601}" ADMIN_USER="admin" ADMIN_PASS="${OPENSEARCH_INITIAL_ADMIN_PASSWORD}" log() { echo "[$(date -Iseconds)] $1"; } -log "Waiting for OpenSearch at $OPENSEARCH_URL..." -until curl -s -k -u "$ADMIN_USER:$ADMIN_PASS" "$OPENSEARCH_URL/_cluster/health" > /dev/null; do - log "OpenSearch not ready yet... sleeping 5s" +MAX_RETRIES=60 +COUNT=0 +log "Waiting for OpenSearch at $OPENSEARCH_URL (timeout: 300s)..." + +# --- DNS DEBUGGING --- +log "🔍 Debugging Network/DNS..." +log "Contents of /etc/resolv.conf:" +cat /etc/resolv.conf + +log "Testing DNS resolution:" +for HOST in opensearch amp_opensearch tasks.amp_opensearch tasks.opensearch; do + if nslookup "$HOST" >/dev/null 2>&1; then + log " ✅ nslookup $HOST: SUCCESS" + nslookup "$HOST" + else + log " ❌ nslookup $HOST: FAILED" + fi +done +# --------------------- + +while [ $COUNT -lt $MAX_RETRIES ]; do + # Capture output and exit code to debug failure + OUTPUT=$(curl -v -k -u "$ADMIN_USER:$ADMIN_PASS" "$OPENSEARCH_URL/_cluster/health" 2>&1) + RET=$? + + if [ $RET -eq 0 ]; then + log "✅ OpenSearch is UP." + break + fi + + log "OpenSearch not ready yet at $OPENSEARCH_URL... (Exit Code: $RET)" + # Print the error message (last few lines where the error usually is) + log "Curl Error Output:" + echo "$OUTPUT" | tail -n 5 | while read -r line; do log " $line"; done + sleep 5 + COUNT=$((COUNT+1)) done +if [ $COUNT -eq $MAX_RETRIES ]; then + log "❌ Timeout waiting for OpenSearch." + exit 1 +fi + log "OpenSearch is UP. Applying configuration..." # 1. Create jwt_users Role @@ -53,23 +92,46 @@ RESPONSE=$(curl -s -k -u "$ADMIN_USER:$ADMIN_PASS" -w "%{http_code}" -X PUT "$OPENSEARCH_URL/_template/acm_template" -H 'Content-Type: application/json' -d @/usr/share/opensearch/config/amplog_template.json) log "Done. HTTP Response: $RESPONSE" -log "Waiting for OpenSearch Dashboards (https://opensearch-dashboards:5601/visualization/api/status)..." -until STATUS_RES=$(curl -s -k -u "$ADMIN_USER:$ADMIN_PASS" "https://opensearch-dashboards:5601/visualization/api/status") && echo "$STATUS_RES" | grep -q '"state":"green"'; do - # Fallback for state extraction if jq is missing - if command -v jq >/dev/null 2>&1; then - CURRENT_STATE=$(echo "$STATUS_RES" | jq -r '.status.overall.state // "Unknown"') - else - CURRENT_STATE=$(echo "$STATUS_RES" | grep -o '"state":"[^"]*"' | head -1 | cut -d'"' -f4 || echo "Unknown") +MAX_RETRIES=60 +COUNT=0 +log "Waiting for OpenSearch Dashboards ($OPENSEARCH_DASHBOARDS_URL/visualization/api/status) (timeout: 300s)..." +while [ $COUNT -lt $MAX_RETRIES ]; do + STATUS_RES=$(curl -s -k -u "$ADMIN_USER:$ADMIN_PASS" "$OPENSEARCH_DASHBOARDS_URL/visualization/api/status") + if echo "$STATUS_RES" | grep -q '"state":"green"'; then + log "✅ Dashboards is Green/Ready." + break fi - log "Dashboards not ready yet (Current: $CURRENT_STATE). Raw response: $STATUS_RES" - log "Sleeping 5s..." + + # Fallback for state extraction if jq is missing or response is not JSON + # Check if response looks like JSON (starts with {) + case "$STATUS_RES" in + \{*) + if command -v jq >/dev/null 2>&1; then + CURRENT_STATE=$(echo "$STATUS_RES" | jq -r '.status.overall.state // "Unknown"') + else + CURRENT_STATE=$(echo "$STATUS_RES" | grep -o '"state":"[^"]*"' | head -1 | cut -d'"' -f4) + fi + ;; + *) + CURRENT_STATE="Not JSON (Starting...)" + ;; + esac + + if [ -z "$CURRENT_STATE" ]; then CURRENT_STATE="Unknown"; fi + log "Dashboards not ready yet (Current: $CURRENT_STATE). Sleeping 5s..." sleep 5 + COUNT=$((COUNT+1)) done +if [ $COUNT -eq $MAX_RETRIES ]; then + log "❌ Timeout waiting for Dashboards." + exit 1 +fi + log "Importing Saved Objects (Index Patterns, Dashboards)..." # Using -w %{http_code} to catch errors and -o to avoid dumping large JSON to logs HTTP_CODE=$(curl -s -k -u "$ADMIN_USER:$ADMIN_PASS" -o /tmp/import_res.json -w "%{http_code}" \ - -X POST "https://opensearch-dashboards:5601/visualization/api/saved_objects/_import?overwrite=true" \ + -X POST "$OPENSEARCH_DASHBOARDS_URL/visualization/api/saved_objects/_import?overwrite=true" \ -H "osd-xsrf: true" \ --form file=@/usr/share/opensearch/config/export.ndjson) Index: /branches/amp_4_0/platform/tools/container/services/setup/install_curl.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/services/setup/install_curl.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/setup/install_curl.sh (working copy) @@ -12,7 +12,7 @@ echo "Detected RHEL-based distribution: $ID" # --allowerasing is key for Rocky Linux 9 to replace curl-minimal dnf install -y epel-release - dnf install -y --allowerasing curl jq + dnf install -y --allowerasing curl jq bind-utils iputils elif [[ "$ID" == "debian" || "$ID" == "ubuntu" ]]; then echo "Detected Debian-based distribution: $ID" apt-get update && apt-get install -y curl Index: /branches/amp_4_0/platform/tools/container/services/setup/install_python.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/services/setup/install_python.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/setup/install_python.sh (working copy) @@ -6,7 +6,7 @@ exec > >(tee -a "$LOGFILE") 2>&1 echo "------------------------------------------------------------------" -echo " Installing Python 3.13.0 on Rocky Linux 9.5 (Non-Interactive)" +echo " Installing Python 3.13.0 on Rocky Linux 9.x (Non-Interactive)" echo " Log file: $LOGFILE" echo "------------------------------------------------------------------" Index: /branches/amp_4_0/platform/tools/container/services/setup/setup.sh =================================================================== --- /branches/amp_4_0/platform/tools/container/services/setup/setup.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/container/services/setup/setup.sh (working copy) @@ -18,7 +18,7 @@ SAN_HOSTS="DNS:localhost,IP:127.0.0.1,DNS:opensearch-node1,DNS:opensearch-dashboards" ADMIN_KEY="$OUTPUT_DIR/admin-key.pem" ADMIN_CERT="$OUTPUT_DIR/admin.pem" -ADMIN_DN="/CN=admin/O=OpenSearchAdmin/L=Bengaluru/ST=Karnataka/C=IN" +ADMIN_DN="/C=IN/ST=Karnataka/L=Bengaluru/O=OpenSearchAdmin/CN=admin" log() { echo "[$(date -Iseconds)] $1"; } @@ -122,6 +122,69 @@ description: "Kibana server user" EOF +# --- 4. OpenSearch Main Config (opensearch.yml) --- +# We generate this to avoid Env Var parsing issues with Lists (nodes_dn, admin_dn) +OS_CONF="$OUTPUT_DIR/opensearch.yml" +log "Generating opensearch.yml..." +cat > "$OS_CONF" < "$ROLES_MAPPING" < + bash -c "export OPENSEARCH_PASSWORD=\$$(cat /run/secrets/opensearch_initial_admin_password) && /usr/share/opensearch-dashboards/opensearch-dashboards-docker-entrypoint.sh opensearch-dashboards" + secrets: + - opensearch_initial_admin_password + volumes: + - certs-vol:/usr/share/opensearch-dashboards/config/certs:ro + networks: + - amp-overlay + ports: + - "5601:5601" + + grafana: + image: ${REGISTRY:-127.0.0.1:5000}/amp/grafana:latest + deploy: + replicas: 1 + placement: + constraints: + - node.labels.type == storage # Pinned for sqlite db + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:3000/api/health || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + environment: + - GF_LOG_LEVEL=debug + - GF_SERVER_ROOT_URL=https://${AMP_DOMAIN_OR_IP:-localhost}/monitoring/ + - GF_SERVER_SERVE_FROM_SUB_PATH=true + - GF_SECURITY_COOKIE_SECURE=true + - GF_SECURITY_COOKIE_SAMESITE=lax + - GF_SERVER_DOMAIN=${AMP_DOMAIN_OR_IP:-localhost} + - GF_SERVER_ENFORCE_DOMAIN=false + - GF_SECURITY_COOKIE_PATH=/ + - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=1440 + - GF_AUTH_LOGIN_MAXIMUM_INACTIVE_LIFETIME_DAYS=30 + - GF_AUTH_LOGIN_MAXIMUM_LIFETIME_DAYS=30 + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-GArr@y2050} + - GF_INSTALL_PLUGINS=grafana-opensearch-datasource + - DS_POSTGRES_USER=${POSTGRES_USER:-postgres} + - DS_POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-Arr@y2050} + - DS_AMP_DB_NAME=${AMP_DB_NAME:-amp_ts} + - DS_AMP_DB_USER=${AMP_DB_USER:-amp_ts_user} + - DS_AMP_DB_PASSWORD=${AMP_DB_PASSWORD:-Array@123$} + - DS_OS_USER=admin + - DS_OS_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-Arr@y2050} + configs: + - source: grafana_datasources + target: /etc/grafana/provisioning/datasources/datasources.yaml + volumes: + - grafana-data:/var/lib/grafana + networks: + - amp-overlay + ports: + - "${AMP_GRAFANA_PORT:-3000}:3000" + + telegraf: + image: ${REGISTRY:-127.0.0.1:5000}/amp/telegraf:latest + user: root + security_opt: + - label=disable + deploy: + mode: global # Run on EVERY node to monitor docker + entrypoint: ["telegraf"] + environment: + PG_PASSWORD_FILE: /run/secrets/pg_password + secrets: + - pg_password + configs: + - source: telegraf_main_conf + target: /etc/telegraf/telegraf.conf + - source: telegraf_ag_conf + target: /etc/telegraf/telegraf.d/ag.toml + - source: telegraf_apv_conf + target: /etc/telegraf/telegraf.d/apv.toml + - source: telegraf_asf_conf + target: /etc/telegraf/telegraf.d/asf.toml + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /dev:/dev:ro + - /proc:/rootfs/proc:ro + - /sys:/rootfs/sys:ro + - /etc:/rootfs/etc:ro + networks: + - amp-overlay + + logstash: + image: ${REGISTRY:-127.0.0.1:5000}/amp/logstash:latest + deploy: + replicas: 1 + ports: + - "514:5514/tcp" + - "514:5514/udp" + environment: + OPENSEARCH_URL: https://opensearch:9200 + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + POSTGRES_USER: postgres + POSTGRES_DB: cm + command: > + bash -c "export OPENSEARCH_INITIAL_ADMIN_PASSWORD=\$$(cat /run/secrets/opensearch_initial_admin_password) && /usr/share/logstash/bin/logstash" + + secrets: + - opensearch_initial_admin_password + - pg_password + configs: + - source: logstash_pipeline_conf + target: /usr/share/logstash/pipeline/syslog.conf + - source: logstash_config_yml + target: /usr/share/logstash/config/logstash.yml + volumes: + - certs-vol:/usr/share/logstash/config/certs:ro + - ./services/logstash/drivers:/usr/share/logstash/drivers:ro + networks: + - amp-overlay + +volumes: + opensearch-data: + timescaledb-data: + grafana-data: + certs-vol: + external: true + security-config-vol: + external: true + opensearch-logs: Index: /branches/amp_4_0/platform/tools/enable_opensearch_jwt_auth.sh =================================================================== --- /branches/amp_4_0/platform/tools/enable_opensearch_jwt_auth.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/enable_opensearch_jwt_auth.sh (nonexistent) @@ -1,88 +0,0 @@ -#!/bin/bash - -OPENSEARCH_CONFIG_DIR="/etc/opensearch/opensearch-security" -CONFIG_YML="$OPENSEARCH_CONFIG_DIR/config.yml" -ROLES_YML="$OPENSEARCH_CONFIG_DIR/roles.yml" -ROLES_MAPPING_YML="$OPENSEARCH_CONFIG_DIR/roles_mapping.yml" -CERTS_DIR="/etc/opensearch/certs" -BACKUP_CONFIG_YML="$CONFIG_YML.bak-$(date +%F-%H%M%S)" -SECURITY_ADMIN_SCRIPT="/usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh" - -sudo dnf install epel-release -y -sudo dnf install yq -y - -pip3.13 install pyyaml - -echo "🔐 Generating a new JWT signing key..." -BASE64_SIGNING_KEY=$(openssl rand -base64 32 | tr -d '\n"' | tr -d "'") - -if [ -z "$BASE64_SIGNING_KEY" ]; then - echo "❌ Failed to generate signing key. Exiting." - exit 1 -fi - -echo "📝 Creating backup of config.yml..." -cp "$CONFIG_YML" "$BACKUP_CONFIG_YML" - -echo "✅ Updating config.yml using yq..." -command -v yq >/dev/null 2>&1 || { echo "❌ yq not installed."; exit 1; } - -# Enable HTTP for JWT domain and set key -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_enabled = true' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_authenticator.config.signing_key = "'"$BASE64_SIGNING_KEY"'"' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_authenticator.config.jwt_url_parameter = "access_token"' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_authenticator.config.roles_key = "roles"' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_authenticator.config.subject_key = "sub"' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.http_authenticator.config.issuer = "amp.com"' "$CONFIG_YML" -yq eval -i '.config.dynamic.authc.jwt_auth_domain.order = 1' "$CONFIG_YML" - -echo "✅ Ensuring roles.yml contains jwt_users role..." -if ! grep -q '^jwt_users:' "$ROLES_YML"; then - cat >> "$ROLES_YML" <> "$ROLES_MAPPING_YML" </dev/null; then - JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | cut -d'.' -f1) - if (( JAVA_VERSION >= 17 )); then - log_success "Java ${JAVA_VERSION} found. Compatible with OpenSearch 3.x." - if [ -z "${JAVA_HOME_PATH}" ]; then - DETECTED_JAVA_BIN=$(readlink -f $(which java)) - if [ -n "${DETECTED_JAVA_BIN}" ]; then - JAVA_HOME_PATH=$(dirname $(dirname "${DETECTED_JAVA_BIN}")) - log_info "Auto-detected JAVA_HOME_PATH: ${JAVA_HOME_PATH}" - else - log_error "Failed to determine JAVA_HOME_PATH. Please set it manually." - fi - fi - else - log_error "Java version ${JAVA_VERSION} found. OpenSearch 3.x requires Java 17 or higher." - fi -else - log_error "Java not found. Please install Java (OpenJDK 17 or higher)." -fi - -# 3. Download OpenSearch repository file -log_info "Downloading OpenSearch 3.x repository file: ${OPENSEARCH_REPO_FILE}..." -sudo curl -o "${OPENSEARCH_REPO_FILE}" "${OPENSEARCH_REPO_DOWNLOAD_URL}" || log_error "Failed to download OpenSearch repository file." - -# Configure OpenSearch repository -if ! grep -q "gpgcheck=1" "${OPENSEARCH_REPO_FILE}"; then - log_warn "Adding 'gpgcheck=1' to ${OPENSEARCH_REPO_FILE}." - echo "gpgcheck=1" | sudo tee -a "${OPENSEARCH_REPO_FILE}" > /dev/null -fi -if ! grep -q "gpgkey=" "${OPENSEARCH_REPO_FILE}"; then - log_warn "Adding 'gpgkey=${OPENSEARCH_GPG_KEY_URL}' to ${OPENSEARCH_REPO_FILE}." - echo "gpgkey=${OPENSEARCH_GPG_KEY_URL}" | sudo tee -a "${OPENSEARCH_REPO_FILE}" > /dev/null -fi -if ! grep -q "repo_gpgcheck=" "${OPENSEARCH_REPO_FILE}"; then - log_info "Adding 'repo_gpgcheck=0' to ${OPENSEARCH_REPO_FILE}." - echo "repo_gpgcheck=0" | sudo tee -a "${OPENSEARCH_REPO_FILE}" > /dev/null -fi - -# 4. Download OpenSearch Dashboards repository file -log_info "Downloading OpenSearch Dashboards 3.x repository file: ${DASHBOARDS_REPO_FILE}..." -sudo curl -o "${DASHBOARDS_REPO_FILE}" "${DASHBOARDS_REPO_DOWNLOAD_URL}" || log_error "Failed to download OpenSearch Dashboards repository file." - -# Configure Dashboards repository -if ! grep -q "gpgcheck=1" "${DASHBOARDS_REPO_FILE}"; then - log_warn "Adding 'gpgcheck=1' to ${DASHBOARDS_REPO_FILE}." - echo "gpgcheck=1" | sudo tee -a "${DASHBOARDS_REPO_FILE}" > /dev/null -fi -if ! grep -q "gpgkey=" "${DASHBOARDS_REPO_FILE}"; then - log_warn "Adding 'gpgkey=${OPENSEARCH_GPG_KEY_URL}' to ${DASHBOARDS_REPO_FILE}." - echo "gpgkey=${OPENSEARCH_GPG_KEY_URL}" | sudo tee -a "${DASHBOARDS_REPO_FILE}" > /dev/null -fi -if ! grep -q "repo_gpgcheck=" "${DASHBOARDS_REPO_FILE}"; then - log_info "Adding 'repo_gpgcheck=0' to ${DASHBOARDS_REPO_FILE}." - echo "repo_gpgcheck=0" | sudo tee -a "${DASHBOARDS_REPO_FILE}" > /dev/null -fi - -# 5. Enable SHA1 for OpenSearch repositories (Rocky Linux 9 workaround) -log_info "Temporarily enabling SHA1 crypto policy for repository verification..." -sudo update-crypto-policies --set DEFAULT:SHA1 || log_error "Failed to enable SHA1 crypto policy." -log_success "SHA1 crypto policy enabled." - -# 6. Clean DNF cache -log_info "Cleaning DNF cache..." -sudo dnf clean all || log_error "Failed to clean DNF cache." -sudo dnf makecache -y || log_error "Failed to make DNF cache." -log_success "DNF cache updated." - -# 7. Install OpenSearch and OpenSSL -log_info "Installing OpenSearch 3.x and OpenSSL..." -sudo dnf -y install opensearch openssl || log_error "Failed to install OpenSearch or OpenSSL." -log_success "OpenSearch and OpenSSL installed." - -# 8. Install OpenSearch Dashboards -log_info "Installing OpenSearch Dashboards ${DASHBOARDS_VERSION}..." -sudo dnf -y install opensearch-dashboards || log_error "Failed to install OpenSearch Dashboards." -log_success "OpenSearch Dashboards installed." - -# 9. Revert SHA1 crypto policy -log_info "Reverting SHA1 crypto policy to default..." -sudo update-crypto-policies --set DEFAULT || log_error "Failed to revert crypto policy." -log_success "Crypto policy reverted to default." - -# 10. Set vm.max_map_count -log_info "Setting vm.max_map_count for OpenSearch..." -if ! grep -q "vm.max_map_count=262144" /etc/sysctl.d/99-opensearch.conf 2>/dev/null; then - echo "vm.max_map_count=262144" | sudo tee -a /etc/sysctl.d/99-opensearch.conf > /dev/null - sudo sysctl --system || log_error "Failed to apply sysctl changes." - log_success "vm.max_map_count set to 262144." -else - log_info "vm.max_map_count already configured." -fi - -# 11. Create certificate directory -log_info "Creating certificate directory: ${CERT_DIR}..." -sudo mkdir -p "${CERT_DIR}" || log_error "Failed to create directory ${CERT_DIR}." - -# 12. Generate Self-Signed SSL Certificates -log_info "Generating self-signed SSL certificates..." - -# Generate CA Key -sudo openssl genrsa -out "${CA_KEY}" 2048 || log_error "Failed to generate CA key." -sudo chmod 400 "${CA_KEY}" - -# Generate CA Certificate -sudo openssl req -new -x509 -key "${CA_KEY}" -out "${CA_CERT}" -days 3650 -subj "${CA_DN}" || log_error "Failed to generate CA certificate." -sudo chmod 444 "${CA_CERT}" - -# Create SAN config for Node certificate -NODE_EXTFILE_TEMP="/tmp/node_san.cnf" -echo "subjectAltName=${SAN_HOSTS}" | sudo tee "${NODE_EXTFILE_TEMP}" > /dev/null || log_error "Failed to create node SAN file." -sudo chmod 644 "${NODE_EXTFILE_TEMP}" - -# Generate Node Key -sudo openssl genrsa -out "${NODE_KEY}" 2048 || log_error "Failed to generate node key." -sudo chmod 400 "${NODE_KEY}" - -# Generate Node CSR -sudo openssl req -new -key "${NODE_KEY}" -out "${CERT_DIR}/node.csr" -subj "${NODE_DN}" || log_error "Failed to generate node CSR." - -# Sign Node Certificate -sudo openssl x509 -req -in "${CERT_DIR}/node.csr" -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out "${NODE_CERT}" -days 3650 -extfile "${NODE_EXTFILE_TEMP}" || log_error "Failed to sign node certificate." -sudo chmod 444 "${NODE_CERT}" - -# Generate Admin Key -sudo openssl genrsa -out "${ADMIN_KEY}" 2048 || log_error "Failed to generate admin key." -sudo chmod 400 "${ADMIN_KEY}" - -# Generate Admin CSR -sudo openssl req -new -key "${ADMIN_KEY}" -out "${CERT_DIR}/admin.csr" -subj "${ADMIN_DN}" || log_error "Failed to generate admin CSR." - -# Sign Admin Certificate -sudo openssl x509 -req -in "${CERT_DIR}/admin.csr" -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out "${ADMIN_CERT}" -days 3650 || log_error "Failed to sign admin certificate." -sudo chmod 444 "${ADMIN_CERT}" - -# Clean up -sudo rm -f "${CERT_DIR}/node.csr" "${CERT_DIR}/admin.csr" "${NODE_EXTFILE_TEMP}" -log_success "SSL certificates generated." - -# 13. Set certificate permissions for OpenSearch -log_info "Setting certificate permissions for OpenSearch..." -sudo chown -R opensearch:opensearch "${CERT_DIR}" || log_error "Failed to chown certificate directory." -sudo chmod 755 "${CERT_DIR}" -sudo chmod 640 "${NODE_KEY}" "${ADMIN_KEY}" -sudo chmod 644 "${NODE_CERT}" "${CA_CERT}" "${ADMIN_CERT}" -log_success "Certificate permissions set for OpenSearch." - -# 14. Set certificate permissions for OpenSearch Dashboards -log_info "Setting certificate permissions for OpenSearch Dashboards..." -sudo chown opensearch:opensearch "${NODE_KEY}" "${NODE_CERT}" "${CA_CERT}" || log_error "Failed to set certificate ownership for Dashboards." -log_success "Certificate permissions set for Dashboards." - -## Add opensearch-dashboards user to the opensearch group -sudo usermod -aG opensearch opensearch-dashboards - -sudo chown -R opensearch:opensearch /etc/opensearch/ -sudo chmod -R u+rwX,go-w /etc/opensearch/ - -# 15. Verify certificate files -check_cert_files - -# 16. Configure OpenSearch -log_info "Configuring OpenSearch..." -sudo cp "${OPENSEARCH_CONFIG_FILE}" "${OPENSEARCH_CONFIG_FILE}.bak_$(date +%Y%m%d%H%M%S)" - -cat << EOF | sudo tee "${OPENSEARCH_CONFIG_FILE}" > /dev/null -# Managed by OpenSearch installation script - -# Basic Configuration -cluster.name: opensearch-cluster -node.name: node-1 -network.host: 0.0.0.0 - -# Single-node discovery -discovery.seed_hosts: ["127.0.0.1"] -cluster.initial_master_nodes: ["node-1"] - -# Security Plugin (TLS/SSL) -plugins.security.ssl.http.enabled: true -plugins.security.ssl.http.pemcert_filepath: ${NODE_CERT} -plugins.security.ssl.http.pemkey_filepath: ${NODE_KEY} -plugins.security.ssl.http.pemtrustedcas_filepath: ${CA_CERT} -plugins.security.ssl.http.clientauth_mode: OPTIONAL - -plugins.security.ssl.transport.enabled: true -plugins.security.ssl.transport.pemcert_filepath: ${NODE_CERT} -plugins.security.ssl.transport.pemkey_filepath: ${NODE_KEY} -plugins.security.ssl.transport.pemtrustedcas_filepath: ${CA_CERT} -plugins.security.ssl.transport.enforce_hostname_verification: false - -plugins.security.allow_default_init_securityindex: true - -# Allow REST API access to these roles -plugins.security.restapi.roles_enabled: ["security_rest_api_access", "all_access"] -plugins.security.restapi.admin.enabled: true - -# Admin DN -plugins.security.authcz.admin_dn: - - C=IN,ST=Karnataka,L=Bengaluru,O=OpenSearchAdmin,CN=admin - -# Node DN -plugins.security.nodes_dn: - - CN=node-1,O=OpenSearchNode,L=Bengaluru,ST=Karnataka,C=IN - -# Disable system call filter for dev/test -bootstrap.system_call_filter: false -EOF -log_success "OpenSearch configuration updated." - -# 17. Configure OpenSearch Dashboards -log_info "Configuring OpenSearch Dashboards..." -sudo cp "${DASHBOARDS_CONFIG_FILE}" "${DASHBOARDS_CONFIG_FILE}.bak_$(date +%Y%m%d%H%M%S)" - -cat << EOF | sudo tee "${DASHBOARDS_CONFIG_FILE}" > /dev/null -# Managed by OpenSearch installation script - -server.host: 0.0.0.0 -server.port: ${DASHBOARDS_PORT} -opensearch.hosts: ["https://127.0.0.1:9200"] -opensearch.ssl.verificationMode: certificate -opensearch.username: "admin" -opensearch.password: "${NEW_ADMIN_PASSWORD}" -opensearch.requestHeadersWhitelist: ["securitytenant","Authorization"] - -# SSL Configuration -server.ssl.enabled: true -server.ssl.certificate: ${NODE_CERT} -server.ssl.key: ${NODE_KEY} -opensearch.ssl.certificateAuthorities: ["${CA_CERT}"] - -server.basePath: "/visualization" -server.rewriteBasePath: true - -EOF - -# Set permissions -sudo chown -R opensearch-dashboards:opensearch-dashboards "${DASHBOARDS_CONFIG_FILE}" || log_error "Failed to set permissions for Dashboards config." -sudo chmod 600 "${DASHBOARDS_CONFIG_FILE}" -log_success "OpenSearch Dashboards configuration updated." - -# 18. Configure OpenSearch systemd service -log_info "Generating OpenSearch systemd service file..." -sudo rm -f "${OPENSEARCH_SERVICE_FILE}" - -cat << EOF | sudo tee "${OPENSEARCH_SERVICE_FILE}" > /dev/null -[Unit] -Description=OpenSearch -Documentation=https://opensearch.org/ -Wants=network-online.target -After=network-online.target - -[Service] -Environment="JAVA_HOME=${JAVA_HOME_PATH}" -Environment="OPENSEARCH_INITIAL_ADMIN_PASSWORD=dummy_password" -Type=notify -RuntimeDirectory=opensearch -EnvironmentFile=-/etc/default/opensearch -EnvironmentFile=-/etc/sysconfig/opensearch - -WorkingDirectory=/usr/share/opensearch - -User=opensearch -Group=opensearch - -ExecStartPre=/bin/mkdir -p /dev/shm/performanceanalyzer -ExecStartPre=/bin/chown opensearch:opensearch /dev/shm/performanceanalyzer - -ExecStart=/usr/share/opensearch/bin/systemd-entrypoint -p \${PID_DIR}/opensearch.pid --quiet - -StandardOutput=journal -StandardError=inherit -SyslogIdentifier=opensearch - -LimitNOFILE=65535 -LimitNPROC=4096 -LimitAS=infinity -LimitFSIZE=infinity - -TimeoutStopSec=0 -KillSignal=SIGTERM -KillMode=process -SendSIGKILL=no -SuccessExitStatus=143 - -TimeoutStartSec=75 - -ProtectControlGroups=true -ProtectKernelModules=true -ProtectKernelTunables=true -DevicePolicy=closed -ProtectProc=invisible -ProtectSystem=full -LockPersonality=true -NoNewPrivileges=true -RestrictSUID SGID=true -RestrictRealtime=true -ProtectHostname=true -ProtectKernelLogs=true -ProtectClock=true - -SystemCallFilter=madvise mincore mlock mlock2 munlock get_mempolicy sched_getaffinity sched_setaffinity fcntl @system-service -SystemCallFilter=~@reboot ~@swap -SystemCallErrorNumber=EPERM - -CapabilityBoundingSet=~CAP_BLOCK_SUSPEND ~CAP_LEASE ~CAP_SYS_PACCT ~CAP_SYS_TTY_CONFIG ~CAP_SYS_ADMIN ~CAP_SYS_PTRACE ~CAP_NET_ADMIN - -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX - -ReadWritePaths=/var/log/opensearch -ReadWritePaths=/var/lib/opensearch -ReadWritePaths=/dev/shm/ -ReadWritePaths=-/etc/opensearch -ReadWritePaths=-/mnt/snapshots - -ReadOnlyPaths=-/etc/os-release -/usr/lib/os-release -/etc/system-release -ReadOnlyPaths=/proc/self/mountinfo /proc/diskstats -ReadOnlyPaths=/proc/self/cgroup /sys/fs/cgroup/cpu /sys/fs/cgroup/cpuacct /sys/fs/cgroup/memory - -[Install] -WantedBy=multi-user.target -EOF - -sudo chmod 644 "${OPENSEARCH_SERVICE_FILE}" -sudo chown root:root "${OPENSEARCH_SERVICE_FILE}" -log_success "OpenSearch service configured." - -# 19. Configure OpenSearch Dashboards systemd service -log_info "Configuring OpenSearch Dashboards systemd service..." -DASHBOARDS_SERVICE_FILE="/usr/lib/systemd/system/${DASHBOARDS_SERVICE}.service" - -sudo rm -f "${DASHBOARDS_SERVICE_FILE}" - -cat << EOF | sudo tee "${DASHBOARDS_SERVICE_FILE}" > /dev/null -[Unit] -Description=OpenSearch Dashboards -Documentation=https://opensearch.org/docs/latest/ -Wants=network-online.target -After=network-online.target opensearch.service - -[Service] -Type=simple -User=opensearch-dashboards -Group=opensearch-dashboards -ExecStart=/usr/share/opensearch-dashboards/bin/opensearch-dashboards -Restart=on-failure -WorkingDirectory=/usr/share/opensearch-dashboards -StandardOutput=journal -StandardError=inherit -SyslogIdentifier=opensearch-dashboards -Environment=NODE_ENV=production -Environment=CONFIG_PATH=${DASHBOARDS_CONFIG_FILE} - -LimitNOFILE=65535 -LimitNPROC=4096 -LimitAS=infinity -LimitFSIZE=infinity - -TimeoutStopSec=0 -KillSignal=SIGTERM -KillMode=process -SendSIGKILL=no - -[Install] -WantedBy=multi-user.target -EOF - -sudo chmod 644 "${DASHBOARDS_SERVICE_FILE}" -sudo chown root:root "${DASHBOARDS_SERVICE_FILE}" -log_success "OpenSearch Dashboards service configured." - -# 20. Start OpenSearch -log_info "Enabling and starting OpenSearch service..." -sudo systemctl daemon-reload || log_error "Failed to reload systemd daemon." -sudo systemctl enable "${OPENSEARCH_SERVICE}" || log_error "Failed to enable OpenSearch service." -sudo systemctl restart "${OPENSEARCH_SERVICE}" || log_error "Failed to start OpenSearch service." - -# 21. Wait for OpenSearch -log_info "Waiting for OpenSearch to start..." -ATTEMPTS=60 -for i in $(seq 1 $ATTEMPTS); do - if curl -s -k -u admin:admin "https://localhost:9200/_plugins/_security/health?pretty" | grep -q "status.*UP"; then - log_success "OpenSearch service is up!" - break - else - log_info "Attempt $i/$ATTEMPTS: OpenSearch not available. Waiting 10 seconds..." - sleep 10 - fi - if [ $i -eq $ATTEMPTS ]; then - log_error "OpenSearch did not start. Check logs." - fi -done - -# 22. Initialize Security Plugin -log_info "Initializing OpenSearch Security Plugin..." - -# Apply default security configuration with reload -if JAVA_HOME="${JAVA_HOME_PATH}" sudo -E /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd "${SECURITY_CONFIG_DIR}" \ - -icl -nhnv \ - -cacert "${CA_CERT}" -cert "${ADMIN_CERT}" -key "${ADMIN_KEY}" \ - -rl; then - - log_success "Security Plugin initialized." - - log_info "Waiting for security index to stabilize..." - sleep 10 # Give it a moment after plugin initialization - - # New steps to update password and roles - log_info "Updating admin user password and roles based on manual steps..." - # Dynamically generate the password hash, filtering out the warning - log_info "Dynamically generating hash for the admin password..." - ADMIN_HASH=$(sudo -u opensearch /usr/share/opensearch/plugins/opensearch-security/tools/hash.sh -p "${NEW_ADMIN_PASSWORD}" | grep -o '\$2y\$12\$[a-zA-Z0-9./]*' | tr -d '\n') - if [ -z "${ADMIN_HASH}" ]; then - log_error "Failed to generate password hash. Ensure OpenSearch is running and paths are correct." - fi - log_success "Password hash generated." - - # Update internal_users.yml with the newly generated hash - log_info "Updating internal_users.yml with new admin password hash..." - # Using a more robust sed command with a different delimiter '@' - sudo sed -i -E "s@^\s*hash:.*@ hash: \"${ADMIN_HASH}\"@" "${SECURITY_CONFIG_DIR}/internal_users.yml" || log_error "Failed to update admin hash in internal_users.yml." - log_success "internal_users.yml updated with new admin hash." - - # Re-run securityadmin.sh to apply the new internal_users.yml with the password change. - log_info "Applying the updated security configuration..." - if JAVA_HOME="${JAVA_HOME_PATH}" sudo -E /usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh \ - -cd "${SECURITY_CONFIG_DIR}" \ - -icl -nhnv \ - -cacert "${CA_CERT}" -cert "${ADMIN_CERT}" -key "${ADMIN_KEY}"; then - log_success "Updated security configuration applied successfully." - else - log_error "Failed to apply updated security configuration." - fi - - # 23. Verify admin user roles after changes - log_info "Verifying admin user roles after password and configuration update..." - sleep 10 # A final delay to ensure changes are live - ADMIN_ROLES_VERIFY=$(curl -s -k -u admin:"${NEW_ADMIN_PASSWORD}" "https://localhost:9200/_plugins/_security/api/roles?pretty") - # This curl command directly verifies the roles that are available and accessible with the new credentials. - if echo "$ADMIN_ROLES_VERIFY" | grep -q "all_access" && echo "$ADMIN_ROLES_VERIFY" | grep -q "security_rest_api_full_access"; then - log_success "Admin user can access expected roles with the new password. Validation successful." - else - log_error "Validation failed. Admin user could not access all expected roles with the new password. Response: $ADMIN_ROLES_VERIFY" - fi - -else - log_error "Failed to initialize Security Plugin. Check logs." -fi - -# 24. Start OpenSearch Dashboards -log_info "Enabling and starting OpenSearch Dashboards service..." -sudo systemctl daemon-reload || log_error "Failed to reload systemd daemon." -sudo systemctl enable "${DASHBOARDS_SERVICE}" || log_error "Failed to enable OpenSearch Dashboards service." -sudo systemctl restart "${DASHBOARDS_SERVICE}" || log_error "Failed to start OpenSearch Dashboards service." - -# 25. Wait for OpenSearch Dashboards -log_info "Waiting for OpenSearch Dashboards to start..." -ATTEMPTS=60 -for i in $(seq 1 $ATTEMPTS); do - if curl -s -k -u admin:"${NEW_ADMIN_PASSWORD}" "https://localhost:${DASHBOARDS_PORT}/api/status" | grep -q "green"; then - log_success "OpenSearch Dashboards is up!" - break - else - log_info "Attempt $i/$ATTEMPTS: Dashboards not available. Waiting 10 seconds..." - sleep 10 - fi - if [ $i -eq $ATTEMPTS ]; then - log_error "OpenSearch Dashboards did not start. Check logs." - fi -done - -# 26. Configure Firewall -log_info "Configuring firewall for ports 9200, 9600, and ${DASHBOARDS_PORT}..." -if systemctl is-active --quiet firewalld; then - sudo firewall-cmd --add-port=9200/tcp --permanent || log_warn "Failed to add port 9200/tcp." - sudo firewall-cmd --add-port=9600/tcp --permanent || log_warn "Failed to add port 9600/tcp." - sudo firewall-cmd --add-port=${DASHBOARDS_PORT}/tcp --permanent || log_warn "Failed to add port ${DASHBOARDS_PORT}/tcp." - sudo firewall-cmd --reload || log_warn "Failed to reload firewall." - log_success "Firewall rules updated." -else - log_warn "Firewalld not running. Configure firewall manually for ports 9200, 9600, and ${DASHBOARDS_PORT}." -fi - -# 27. Final Output -log_success "OpenSearch 3.x and Dashboards installation complete!" -log_info "Access OpenSearch at: https://$(hostname -I | awk '{print $1}'):9200 or https://127.0.0.1:9200" -log_info "Access Dashboards at: https://$(hostname -I | awk '{print $1}'):${DASHBOARDS_PORT} or https://127.0.0.1:${DASHBOARDS_PORT}" -log_info "Username: admin, Password: ${NEW_ADMIN_PASSWORD}" -log_warn "Import CA certificate (${CA_CERT}) into your browser's trust store." -log_warn "For production, use trusted CA certificates and secure passwords." Index: /branches/amp_4_0/platform/tools/install_elk.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_elk.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/install_elk.sh (nonexistent) @@ -1,534 +0,0 @@ -#!/bin/bash -set -e - -if [[ "$EUID" -ne 0 ]]; then - echo "Please run as root" - exit 1 -fi - -# --- Log File Configuration --- -LOG_FILE="/var/log/install_elk.log" - -# --- Version Configuration --- -ELASTIC_MAJOR="8.x" -KIBANA_MAJOR="8.x" -BEATS_MAJOR="8.x" -LOGSTASH_MAJOR="8.x" -FILEBEAT_MAJOR="8.x" - -# --- Credentials --- -ELASTIC_SUPER_USER="elastic" -ELASTIC_PASSWORD="Arr@y2050" - -KIBANA_ELASTIC_USERNAME="kibana_internal" -KIBANA_ELASTIC_PASSWORD="KArr@y2050" -ES_DATA_READER_ROLE="data_reader" -ES_USERNAME_1="admin" -ES_PASSWORD_1="Array@123" -METRICBEAT_ELASTIC_USERNAME="metricbeat_internal" -METRICBEAT_ELASTIC_PASSWORD="MArr@y2050" -FILEBEAT_ELASTIC_USERNAME="filebeat_internal" -FILEBEAT_ELASTIC_PASSWORD="FArr@y2050" -LOGSTASH_ELASTIC_USERNAME="logstash_internal" -LOGSTASH_ELASTIC_PASSWORD="LArr@y2050" - -KIBANA_HOST="http://localhost:5601" - -# --- Server IP Configuration --- -# Get the primary IP address -SERVER_IP=$(hostname -I | awk '{print $1}') -if [[ -z "$SERVER_IP" ]]; then - echo "Failed to automatically determine server IP address. Please set SERVER_IP manually." | tee -a "$LOG_FILE" - exit 1 -fi -echo "Detected server IP address: $SERVER_IP" | tee -a "$LOG_FILE" - -# --- Logging Helpers --- -log_info() { - echo "[INFO] $(date +'%Y-%m-%d %H:%M:%S') $1" | tee -a "$LOG_FILE" -} - -log_error() { - echo "[ERROR] $(date +'%Y-%m-%d %H:%M:%S') $1" >&2 | tee -a "$LOG_FILE" -} - -# --- Command Check --- -check_command() { - if ! command -v "$1" &>/dev/null; then - log_error "$1 not found. Please install it." - exit 1 - fi -} - -log_info "Verifying required commands..." -for cmd in curl sudo rpm tee dnf sed systemctl ip firewall-cmd jq; do check_command "$cmd"; done - -# --- Add 8.x Repositories --- -log_info "Adding Elastic 8.x repositories..." -sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch -repo_config=$(cat </dev/null 2>&1 #check the json - if [[ $? -ne 0 ]]; then - log_error "Failed to set $ELASTIC_SUPER_USER password. Output: $ELASTIC_PW_RESULT" - exit 1 - fi - - log_info "Creating metricbeat writer role" - METRICBEAT_WRITER_ROLE_RESULT=$(curl -s -X PUT "http://localhost:9200/_security/role/metricbeat_writer" \ - -H "Content-Type: application/json" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -d '{ - "cluster": [ "monitor", "manage_ilm", "manage_ingest_pipelines", "manage_index_templates"], - "indices": [ - { - "names": [ "metricbeat-*" ], - "privileges": [ "write", "create_index", "auto_configure", "view_index_metadata"] - } - ] - }') - echo "$METRICBEAT_WRITER_ROLE_RESULT" | tee -a "$LOG_FILE" - echo "$METRICBEAT_WRITER_ROLE_RESULT" | jq . >/dev/null 2>&1 #check the json - if [[ $? -ne 0 ]]; then - log_error "Failed to create metricbeat_writer. Output: $METRICBEAT_WRITER_ROLE_RESULT" - exit 1 - fi - - log_info "Creating filebeat writer role" - FILEBEAT_WRITER_ROLE_RESULT=$(curl -s -X PUT "http://localhost:9200/_security/role/filebeat_writer" \ - -H "Content-Type: application/json" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -d '{ - "cluster": [ "monitor", "manage_ilm", "manage_ingest_pipelines", "manage_index_templates"], - "indices": [ - { - "names": [ "filebeat-*" ], - "privileges": [ "write", "create_index", "auto_configure", "view_index_metadata" ] - } - ] - }') - echo "$FILEBEAT_WRITER_ROLE_RESULT" | tee -a "$LOG_FILE" - echo "$FILEBEAT_WRITER_ROLE_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create filebeat_writer. Output: $FILEBEAT_WRITER_ROLE_RESULT" - exit 1 - fi - - log_info "Creating logstash writer role" - LOGSTASH_WRITER_ROLE_RESULT=$(curl -s -X PUT "http://localhost:9200/_security/role/logstash_writer" \ - -H "Content-Type: application/json" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -d '{ - "cluster": [ "monitor", "manage_ilm", "manage_ingest_pipelines", "manage_index_templates", "manage_pipeline" ], - "indices": [ - { - "names": [ "logstash-*", "acm-*" ], - "privileges": [ "write", "create_index", "auto_configure", "manage", "view_index_metadata" ] - }, - { - "names" : [ ".monitoring-logstash-*" ], - "privileges" : [ "create", "write", "auto_configure", "manage", "create_index" ] - } - ] - }') - echo "$LOGSTASH_WRITER_ROLE_RESULT" | tee -a "$LOG_FILE" - echo "$LOGSTASH_WRITER_ROLE_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create filebeat_writer. Output: $LOGSTASH_WRITER_ROLE_RESULT" - exit 1 - fi - - log_info "Creating $KIBANA_ELASTIC_USERNAME user..." - KIBANA_PW_RESULT=$(curl -s -X PUT -u $ELASTIC_SUPER_USER:"$ELASTIC_PASSWORD" -H 'Content-Type: application/json' \ - -d "{ \"password\": \"$KIBANA_ELASTIC_PASSWORD\", \"roles\": [\"kibana_system\"]}" \ - http://localhost:9200/_security/user/$KIBANA_ELASTIC_USERNAME) - echo "$KIBANA_PW_RESULT" | tee -a "$LOG_FILE" - echo "$KIBANA_PW_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to set $KIBANA_ELASTIC_USERNAME password. Output: $KIBANA_PW_RESULT" - exit 1 - fi - - log_info "Creating $LOGSTASH_ELASTIC_USERNAME user..." - LOGSTASH_PW_RESULT=$(curl -s -X PUT -u $ELASTIC_SUPER_USER:"$ELASTIC_PASSWORD" -H 'Content-Type: application/json' \ - -d "{ \"password\": \"$LOGSTASH_ELASTIC_PASSWORD\", \"roles\": [\"logstash_writer\"], \"full_name\": \"Internal Logstash User\" }" \ - http://localhost:9200/_security/user/$LOGSTASH_ELASTIC_USERNAME) - echo "$LOGSTASH_PW_RESULT" | tee -a "$LOG_FILE" - echo "$LOGSTASH_PW_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to set $LOGSTASH_ELASTIC_USERNAME password. Output: $LOGSTASH_PW_RESULT" - exit 1 - fi - - log_info "Creating $FILEBEAT_ELASTIC_USERNAME user..." - FILEBEAT_PW_RESULT=$(curl -s -X PUT -u $ELASTIC_SUPER_USER:"$ELASTIC_PASSWORD" -H 'Content-Type: application/json' \ - -d "{ \"password\": \"$FILEBEAT_ELASTIC_PASSWORD\", \"roles\" : [ \"filebeat_writer\" ], \"full_name\" : \"Internal Filebeat User\"}" \ - http://localhost:9200/_security/user/$FILEBEAT_ELASTIC_USERNAME) - echo "$FILEBEAT_PW_RESULT" | tee -a "$LOG_FILE" - echo "$FILEBEAT_PW_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create $FILEBEAT_ELASTIC_USERNAME user. Output: $FILEBEAT_PW_RESULT" - exit 1 - fi - - log_info "Creating $METRICBEAT_ELASTIC_USERNAME user..." - METRICBEAT_PW_RESULT=$(curl -s -X PUT -u $ELASTIC_SUPER_USER:"$ELASTIC_PASSWORD" -H 'Content-Type: application/json' \ - -d "{ \"password\": \"$METRICBEAT_ELASTIC_PASSWORD\", \"roles\" : [ \"metricbeat_writer\" ], \"full_name\" : \"Internal Metricbeat User\"}" \ - http://localhost:9200/_security/user/$METRICBEAT_ELASTIC_USERNAME) - echo "$METRICBEAT_PW_RESULT" | tee -a "$LOG_FILE" - echo "$METRICBEAT_PW_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create $METRICBEAT_ELASTIC_USERNAME user. Output: $METRICBEAT_PW_RESULT" - exit 1 - fi - - # --- Create the custom 'data_reader' role with necessary privileges --- - log_info "Creating the '$ES_DATA_READER_ROLE' role with necessary data access privileges..." - DATA_READER_ROLE_RESULT=$(curl -s -X POST "http://localhost:9200/_security/role/$ES_DATA_READER_ROLE" \ - -H "Content-Type: application/json" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -d "{ - \"indices\": [ - { - \"names\": [\"*\"], - \"privileges\": [\"read\", \"view_index_metadata\"] - } - ], - \"run_as\": [], - \"cluster\": [\"monitor\", \"manage_ilm\", \"manage_ingest_pipelines\", \"manage_index_templates\"] - }") - echo "$DATA_READER_ROLE_RESULT" | tee -a "$LOG_FILE" - echo "$DATA_READER_ROLE_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create the '$ES_DATA_READER_ROLE' role. Output: $DATA_READER_ROLE_RESULT" - exit 1 - fi - log_info "Successfully created the '$ES_DATA_READER_ROLE' role." - - # --- Create the 'admin' user --- - log_info "Creating the '$ES_USERNAME_1' user for Kibana..." - ADMIN_USER_RESULT=$(curl -s -X POST "http://localhost:9200/_security/user/$ES_USERNAME_1" \ - -H "Content-Type: application/json" \ - -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ - -d "{ - \"password\" : \"$ES_PASSWORD_1\", - \"roles\" : [ \"kibana_admin\", \"$ES_DATA_READER_ROLE\" ], - \"full_name\" : \"Admin User\" - }") - echo "$ADMIN_USER_RESULT" | tee -a "$LOG_FILE" - echo "$ADMIN_USER_RESULT" | jq . >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - log_error "Failed to create the '$ES_USERNAME_1' user. Output: $ADMIN_USER_RESULT" - exit 1 - fi - log_info "Successfully created the '$ES_USERNAME_1' user." -else - log_error "Failed to extract initial password from logs." - exit 1 -fi - -# --- Configure Kibana --- -KB_YML="/etc/kibana/kibana.yml" -log_info "Backing up and updating Kibana config..." -sudo cp "$KB_YML" "${KB_YML}.bak" | tee -a "$LOG_FILE" - -# use sed to remove -sudo sed -i '/^server.host/d' "$KB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^elasticsearch.hosts/d' "$KB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^elasticsearch.username/d' "$KB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^elasticsearch.password/d' "$KB_YML" | tee -a "$LOG_FILE" -# Append to the file. -echo "server.host: 0.0.0.0" | sudo tee -a "$KB_YML" -echo "elasticsearch.hosts: ['http://localhost:9200']" | sudo tee -a "$KB_YML" -echo "elasticsearch.username: '$KIBANA_ELASTIC_USERNAME'" | sudo tee -a "$KB_YML" -echo "elasticsearch.password: '$KIBANA_ELASTIC_PASSWORD'" | sudo tee -a "$KB_YML" - -cat "$KB_YML" | tee -a "$LOG_FILE" # Log the content of the file - -# --- Configure Metricbeat --- -MB_YML="/etc/metricbeat/metricbeat.yml" -log_info "Backing up and updating Metricbeat config..." -sudo cp "$MB_YML" "${MB_YML}.bak" | tee -a "$LOG_FILE" -#use sed to remove -sudo sed -i '/^output.elasticsearch/d' "$MB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ hosts:/d' "$MB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ username:/d' "$MB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ password:/d' "$MB_YML" | tee -a "$LOG_FILE" -#append -echo "output.elasticsearch:" | sudo tee -a "$MB_YML" -echo " hosts: ['localhost:9200']" | sudo tee -a "$MB_YML" -echo " username: '$METRICBEAT_ELASTIC_USERNAME'" | sudo tee -a "$MB_YML" -echo " password: '$METRICBEAT_ELASTIC_PASSWORD'" | sudo tee -a "$MB_YML" - -# Add systemd metricset -echo "metricbeat.modules:" | sudo tee -a "$MB_YML" -echo "- module: system" | sudo tee -a "$MB_YML" -echo " metricsets: [cpu, load, memory, network, process, process_summary, socket_summary, uptime, filesystem, diskio]" | sudo tee -a "$MB_YML" -echo " enabled: true" | sudo tee -a "$MB_YML" -echo " period: 10s" | sudo tee -a "$MB_YML" # Collect every 10 seconds - -cat "$MB_YML" | tee -a "$LOG_FILE" # Log the content of the file - -# --- Configure Logstash --- -LS_YML="/etc/logstash/logstash.yml" -LS_CONF_DIR="/etc/logstash/conf.d" -LS_PIPELINE_CONF="$LS_CONF_DIR/logstash.conf" - -log_info "Backing up and updating Logstash config..." -sudo cp "$LS_YML" "${LS_YML}.bak" | tee -a "$LOG_FILE" - -# use sed to remove monitoring settings -sudo sed -i '/^xpack.monitoring.enabled/d' "$LS_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^xpack.monitoring.elasticsearch.hosts/d' "$LS_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^xpack.monitoring.elasticsearch.username/d' "$LS_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^xpack.monitoring.elasticsearch.password/d' "$LS_YML" - -# Add the configuration settings -echo "xpack.monitoring.enabled: true" | sudo tee -a "$LS_YML" -echo "xpack.monitoring.elasticsearch.hosts: ['http://localhost:9200']" | sudo tee -a "$LS_YML" -echo "xpack.monitoring.elasticsearch.username: '$LOGSTASH_ELASTIC_USERNAME'" | sudo tee -a "$LS_YML" -echo "xpack.monitoring.elasticsearch.password: '$LOGSTASH_ELASTIC_PASSWORD'" | sudo tee -a "$LS_YML" - -cat "$LS_YML" | tee -a "$LOG_FILE" - -# --- Create a basic Logstash pipeline configuration --- -log_info "Creating a basic Logstash pipeline configuration..." -LS_PIPELINE_CONFIG=$(cat < ["http://localhost:9200"] - index => "logstash-%{+YYYY.MM.dd}" - user => "$LOGSTASH_ELASTIC_USERNAME" - password => "$LOGSTASH_ELASTIC_PASSWORD" - } - stdout { codec => rubydebug } -} -EOF -) -echo "$LS_PIPELINE_CONFIG" | sudo tee "$LS_PIPELINE_CONF" | tee -a "$LOG_FILE" - -# --- Configure Filebeat --- -FB_YML="/etc/filebeat/filebeat.yml" -log_info "Backing up and updating Filebeat config..." -sudo cp "$FB_YML" "${FB_YML}.bak" | tee -a "$LOG_FILE" - -# use sed to remove -sudo sed -i '/^output.elasticsearch/d' "$FB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ hosts:/d' "$FB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ username:/d' "$FB_YML" | tee -a "$LOG_FILE" -sudo sed -i '/^ password:/d' "$FB_YML" | tee -a "$LOG_FILE" - -# Add filebeat config -echo "output.elasticsearch:" | sudo tee -a "$FB_YML" -echo " hosts: ['localhost:9200']" | sudo tee -a "$FB_YML" -echo " username: '$FILEBEAT_ELASTIC_USERNAME'" | sudo tee -a "$FB_YML" -echo " password: '$FILEBEAT_ELASTIC_PASSWORD'" | sudo tee -a "$FB_YML" - -cat "$FB_YML" | tee -a "$LOG_FILE" - -# --- Open Firewall for Kibana, Logstash, and Filebeat --- -if systemctl is-active --quiet firewalld; then - log_info "Opening firewall for Kibana (port 5601), Logstash (port 9600), and Filebeat (port 5601)..." - sudo firewall-cmd --add-port={5601,9200,5044}/tcp --permanent | tee -a "$LOG_FILE" - sudo firewall-cmd --reload | tee -a "$LOG_FILE" - sudo firewall-cmd --permanent --zone=public \ - --add-masquerade \ - --add-port=514/udp \ - --add-port=5514/udp \ - --add-port=514/tcp \ - --add-port=5514/tcp \ - --add-forward-port=port=514:proto=udp:toport=5514 \ - --add-forward-port=port=514:proto=tcp:toport=5514 | tee -a "$LOG_FILE" - sudo firewall-cmd --reload | tee -a "$LOG_FILE" -fi - -# --- Start Elasticsearch, Kibana, Metricbeat, Logstash, and Filebeat --- -log_info "Enabling and starting Elasticsearch, Kibana, Metricbeat, Logstash, and Filebeat..." -sudo systemctl enable --now elasticsearch kibana metricbeat logstash filebeat | tee -a "$LOG_FILE" - -# --- Final Verification --- -log_info "Verifying services..." -for svc in elasticsearch kibana metricbeat logstash filebeat; do - if systemctl is-active --quiet "$svc"; then - log_info "$svc is running." - else - log_error "$svc failed to start. See logs: journalctl -u $svc" - fi -done - -log_info "✅ ELK Stack 8.x single-node setup complete." -log_info "Installation logs are available in $LOG_FILE." -log_info "Access Kibana at http://${SERVER_IP}:5601 (admin / ${ES_PASSWORD_1})" - -exit 0 Index: /branches/amp_4_0/platform/tools/install_fluentbit_dataprepper.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_fluentbit_dataprepper.sh (revision 2855) +++ /branches/amp_4_0/platform/tools/install_fluentbit_dataprepper.sh (nonexistent) @@ -1,566 +0,0 @@ -#!/bin/bash - -# Script to install and configure Data Prepper, Fluent Bit, and rsyslog for OpenSearch. -# Assumes OpenSearch and OpenSearch Dashboards are already installed and configured -# (including SSL certificates and the 'admin' user with password) from a previous script run. - -# --- Configuration --- -OPENSEARCH_HOST="localhost" -OPENSEARCH_PORT=9200 -NEW_ADMIN_PASSWORD="Arr@y2050" # OpenSearch admin password -CERT_DIR="/etc/opensearch/certs" -CA_CERT="${CERT_DIR}/root-ca.pem" -NODE_KEY="${CERT_DIR}/node-key.pem" -NODE_CERT="${CERT_DIR}/node.pem" -ADMIN_KEY="${CERT_DIR}/admin-key.pem" -ADMIN_CERT="${CERT_DIR}/admin.pem" -KEYSTORE="${CERT_DIR}/dataprepper-keystore.p12" # PKCS12 keystore -JAVA_HOME_PATH="/usr/lib/jvm/java-21-openjdk-21.0.7.0.6-1.el9.x86_64" # JDK 21 -INDEX_PATTERN="acm-*" - -# Data Prepper Configuration -DATA_PREPPER_INSTALL_DIR="/usr/share/data-prepper" -DATA_PREPPER_CONFIG_DIR="${DATA_PREPPER_INSTALL_DIR}/config" -DATA_PREPPER_PIPELINES_DIR="${DATA_PREPPER_INSTALL_DIR}/pipelines" -DATA_PREPPER_LOG_DIR="/var/log/data-prepper" -DATA_PREPPER_SERVICE="data-prepper" -DATA_PREPPER_SERVICE_FILE="/usr/lib/systemd/system/${DATA_PREPPER_SERVICE}.service" -DATA_PREPPER_USER="data-prepper" -DATA_PREPPER_GROUP="data-prepper" -DATA_PREPPER_HTTP_PORT=2021 -DATA_PREPPER_SOURCE_USER="data_prepper_user" -DATA_PREPPER_SOURCE_PASSWORD="data_prepper_password" - -# Fluent Bit Configuration -FLUENT_BIT_CONFIG_DIR="/etc/fluent-bit" -FLUENT_BIT_LOG_DIR="/var/log/fluent-bit" -FLUENT_BIT_SERVICE="fluent-bit" -SYSLOG_TCP_PORT=514 -SYSLOG_UDP_PORT=514 - -# rsyslog Configuration -RSYSLOG_CONFIG_DIR="/etc/rsyslog.d" -RSYSLOG_SERVICE="rsyslog" - -# --- Functions --- - -log_info() { echo -e "\n\033[0;34m[INFO]\033[0m $1"; } -log_success() { echo -e "\n\033[0;32m[SUCCESS]\033[0m $1"; } -log_warn() { echo -e "\n\033[0;33m[WARNING]\033[0m $1"; } -log_error() { echo -e "\n\033[0;31m[ERROR]\033[0m $1"; exit 1; } - -create_data_prepper_user() { - log_info "Creating Data Prepper user and group..." - if ! id "${DATA_PREPPER_USER}" &>/dev/null; then - sudo groupadd --system "${DATA_PREPPER_GROUP}" || log_error "Failed to create group ${DATA_PREPPER_GROUP}." - sudo useradd --system -g "${DATA_PREPPER_GROUP}" -s /sbin/nologin -d "${DATA_PREPPER_INSTALL_DIR}" "${DATA_PREPPER_USER}" || log_error "Failed to create user ${DATA_PREPPER_USER}." - log_success "Data Prepper user '${DATA_PREPPER_USER}' and group '${DATA_PREPPER_GROUP}' created." - else - log_info "Data Prepper user '${DATA_PREPPER_USER}' already exists." - fi -} - -convert_pem_to_keystore() { - log_info "Converting PEM certificates to PKCS12 keystore..." - if [ ! -f "${KEYSTORE}" ]; then - sudo openssl pkcs12 -export -in "${NODE_CERT}" -inkey "${NODE_KEY}" -out "${KEYSTORE}" -name dataprepper -passout pass: || log_error "Failed to create PKCS12 keystore." - sudo chown ${DATA_PREPPER_USER}:${DATA_PREPPER_GROUP} "${KEYSTORE}" - sudo chmod 640 "${KEYSTORE}" - log_success "PKCS12 keystore created at ${KEYSTORE}." - else - log_info "PKCS12 keystore already exists at ${KEYSTORE}." - fi -} - -configure_data_prepper_server_config() { - log_info "Configuring Data Prepper server settings..." - sudo mkdir -p "${DATA_PREPPER_CONFIG_DIR}" "${DATA_PREPPER_LOG_DIR}" || log_error "Failed to create Data Prepper directories." - sudo chown -R ${DATA_PREPPER_USER}:${DATA_PREPPER_GROUP} "${DATA_PREPPER_CONFIG_DIR}" "${DATA_PREPPER_LOG_DIR}" - sudo chmod 750 "${DATA_PREPPER_LOG_DIR}" - sudo usermod -aG opensearch ${DATA_PREPPER_USER} - cat << EOF | sudo tee "${DATA_PREPPER_CONFIG_DIR}/data-prepper-config.yaml" > /dev/null -ssl: true -key_store_file_path: "${KEYSTORE}" -key_store_password: "" -private_key_password: "" -server_port: 4900 -authentication: - http_basic: - username: "data_prepper_server_user" - password: "data_prepper_server_password" -EOF - sudo chown ${DATA_PREPPER_USER}:${DATA_PREPPER_GROUP} "${DATA_PREPPER_CONFIG_DIR}/data-prepper-config.yaml" - sudo chmod 640 "${DATA_PREPPER_CONFIG_DIR}/data-prepper-config.yaml" - log_success "Data Prepper server configuration created." -} - -configure_data_prepper_pipeline() { - log_info "Creating Data Prepper pipeline configuration..." - sudo mkdir -p "${DATA_PREPPER_PIPELINES_DIR}" || log_error "Failed to create ${DATA_PREPPER_PIPELINES_DIR}." - sudo chown -R ${DATA_PREPPER_USER}:${DATA_PREPPER_GROUP} "${DATA_PREPPER_PIPELINES_DIR}" - cat << EOF | sudo tee "${DATA_PREPPER_PIPELINES_DIR}/opensearch-pipeline.yaml" > /dev/null -version: "2" -opensearch-pipeline: - workers: 2 - delay: 100 - source: - http: - ssl: true - ssl_certificate_file: "${NODE_CERT}" - ssl_key_file: "${NODE_KEY}" - port: ${DATA_PREPPER_HTTP_PORT} - path: / - processor: - - date: - from_time_received: true - sink: - - opensearch: - hosts: ["https://localhost:9200"] - insecure: true - cert: "${CA_CERT}" - username: "${DATA_PREPPER_SOURCE_USER}" - password: "${DATA_PREPPER_SOURCE_PASSWORD}" - index: "acm-%{yyyy.MM.dd}" -EOF - sudo chown ${DATA_PREPPER_USER}:${DATA_PREPPER_GROUP} "${DATA_PREPPER_PIPELINES_DIR}/opensearch-pipeline.yaml" - sudo chmod 640 "${DATA_PREPPER_PIPELINES_DIR}/opensearch-pipeline.yaml" - log_success "Data Prepper pipeline configuration created." -} - -configure_data_prepper_service() { - log_info "Configuring Data Prepper systemd service..." - sudo rm -f "${DATA_PREPPER_SERVICE_FILE}" - cat << EOF | sudo tee "${DATA_PREPPER_SERVICE_FILE}" > /dev/null -[Unit] -Description=OpenSearch Data Prepper -Documentation=https://opensearch.org/docs/latest/data-prepper/ -Wants=network-online.target -After=network-online.target opensearch.service - -[Service] -Environment="JAVA_HOME=${JAVA_HOME_PATH}" -Type=simple -User=${DATA_PREPPER_USER} -Group=${DATA_PREPPER_GROUP} -WorkingDirectory=${DATA_PREPPER_INSTALL_DIR} -ExecStart=${DATA_PREPPER_INSTALL_DIR}/bin/data-prepper -Restart=on-failure -StandardOutput=append:${DATA_PREPPER_LOG_DIR}/data-prepper-service.log -StandardError=append:${DATA_PREPPER_LOG_DIR}/data-prepper-service.log -SyslogIdentifier=data-prepper -LimitNOFILE=65535 -LimitNPROC=4096 -LimitAS=infinity -LimitFSIZE=infinity -TimeoutStopSec=0 -KillSignal=SIGTERM -KillMode=process -SendSIGKILL=no - -[Install] -WantedBy=multi-user.target -EOF - sudo chmod 644 "${DATA_PREPPER_SERVICE_FILE}" - sudo chown root:root "${DATA_PREPPER_SERVICE_FILE}" - log_success "Data Prepper service configured." -} - -configure_rsyslog() { - [ "$EUID" -eq 0 ] || { log_error "Must run as root."; return 1; } - : "${RSYSLOG_CONFIG_DIR:=/etc/rsyslog.d}" - : "${SYSLOG_UDP_PORT:=514}" - : "${SYSLOG_TCP_PORT:=514}" - : "${RSYSLOG_SERVICE:=rsyslog}" - log_info "Configuring rsyslog to forward to Fluent Bit..." - sudo mkdir -p "${RSYSLOG_CONFIG_DIR}" || { log_error "Failed to create rsyslog config directory."; return 1; } - cat << EOF | sudo tee "${RSYSLOG_CONFIG_DIR}/fluent-bit.conf" > /dev/null || { log_error "Failed to write config."; return 1; } -module(load="omfwd") -*.* @127.0.0.1:${SYSLOG_UDP_PORT} -*.* @@127.0.0.1:${SYSLOG_TCP_PORT} -EOF - sudo chmod 644 "${RSYSLOG_CONFIG_DIR}/fluent-bit.conf" || { log_error "Failed to set permissions."; return 1; } - sudo cp /etc/rsyslog.conf /etc/rsyslog.conf.bak || log_error "Failed to backup rsyslog.conf." - sudo sed -i '/module(load="imudp")/s/^/#/' /etc/rsyslog.conf || { log_error "Failed to disable imudp module."; return 1; } - sudo sed -i '/module(load="imtcp")/s/^/#/' /etc/rsyslog.conf || { log_error "Failed to disable imtcp module."; return 1; } - sudo sed -i '/input(type="imudp" port="514")/s/^/#/' /etc/rsyslog.conf || { log_error "Failed to disable imudp input."; return 1; } - sudo sed -i '/input(type="imtcp" port="514")/s/^/#/' /etc/rsyslog.conf || { log_error "Failed to disable imtcp input."; return 1; } - sudo systemctl is-active --quiet "${RSYSLOG_SERVICE}" || { log_error "rsyslog service not running."; return 1; } - sudo systemctl restart "${RSYSLOG_SERVICE}" || { log_error "Failed to restart rsyslog service."; return 1; } - log_info "Ensuring firewall ports are open..." - if systemctl is-active --quiet firewalld; then - sudo firewall-cmd --add-port=514/tcp --permanent || log_error "Failed to open TCP 514." - sudo firewall-cmd --add-port=514/udp --permanent || log_error "Failed to open UDP 514." - sudo firewall-cmd --reload || log_error "Failed to reload firewall." - else - log_warn "Firewalld not running. Ensure ports 514 TCP/UDP are open." - fi - log_info "Testing rsyslog to Fluent Bit connectivity..." - sudo systemctl start fluent-bit || { log_error "Failed to start Fluent Bit service."; return 1; } - sudo systemctl is-active --quiet fluent-bit || { log_error "Fluent Bit service not running."; return 1; } - if ! command -v nc >/dev/null 2>&1; then - log_error "netcat (nc) not installed. Install with 'dnf install nc' and rerun." - return 1 - fi - echo "Test rsyslog message" | logger -t "test_rsyslog" - sleep 2 - for attempt in {1..3}; do - if timeout 2 nc -zv 127.0.0.1 ${SYSLOG_TCP_PORT} &>/dev/null && (echo -n "test" | timeout 2 nc -u -w 1 127.0.0.1 ${SYSLOG_UDP_PORT} &>/dev/null); then - log_success "rsyslog can connect to Fluent Bit on TCP/UDP 514." - break - elif [ $attempt -eq 3 ]; then - log_error "rsyslog cannot connect to Fluent Bit after $attempt attempts. Check firewall, SELinux, or Fluent Bit service." - return 1 - fi - log_info "Attempt $attempt failed. Retrying in 2 seconds..." - sleep 2 - done - log_success "rsyslog configured to forward to Fluent Bit." -} - -create_index_pattern() { - log_info "Creating index pattern '${INDEX_PATTERN}' in OpenSearch..." - local INDEX_PATTERN_JSON=$(cat < /dev/null -[SERVICE] - flush 1 - log_level debug - log_file ${FLUENT_BIT_LOG_DIR}/fluent-bit.log - daemon off - parsers_file parsers.conf - http_server on - http_listen 0.0.0.0 - http_port 2020 - -[INPUT] - Name syslog - Mode tcp - Port ${SYSLOG_TCP_PORT} - Parser syslog-rfc5424 - Listen 0.0.0.0 - -[INPUT] - Name syslog - Mode udp - Port ${SYSLOG_UDP_PORT} - Parser syslog-rfc5424 - Listen 0.0.0.0 - -[OUTPUT] - Name http - Match * - Host localhost - Port ${DATA_PREPPER_HTTP_PORT} - URI / - Format json - tls On - tls.verify Off - tls.ca_file ${CA_CERT} - header Content-Type application/json - http_user ${DATA_PREPPER_SOURCE_USER} - http_passwd ${DATA_PREPPER_SOURCE_PASSWORD} -EOF - cat < /dev/null -[PARSER] - Name syslog-rfc5424 - Format regex - Regex /^\<(?[0-9]+)\>(?