Index: /branches/amp_4_0/platform/tools/install_telegraf.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_telegraf.sh (revision 2695) +++ /branches/amp_4_0/platform/tools/install_telegraf.sh (working copy) @@ -1,143 +1,44 @@ #!/bin/bash +# install_telegraf.sh +# - Installs Telegraf from system repos. +# - Leaves ALL configuration to configure_telegraf_timescale.sh -set -e +set -euo pipefail -# --- Configuration --- LOG_FILE="/var/log/install_telegraf.log" -CONFIG_FILE="/opt/influxdb3_token.toml" TELEGRAF_CONFIG_DIR="/etc/telegraf/telegraf.d" TELEGRAF_CONFIG="/etc/telegraf/telegraf.conf" -INFLUXDB_HOST="http://localhost:8086" -INFLUXDB_ORG="AN" -TELEGRAF_BUCKET="AMP" TELEGRAF_VERSION="1.34.1" -# Function to log messages -log_message() { - local level="$1" - local message="$2" - local timestamp="$(date +'%Y-%m-%d %H:%M:%S')" - local log_line="[$level] $timestamp - $message" - echo "$log_line" - echo "$log_line" >> "$LOG_FILE" -} +# --- Logging helpers --- +log() { echo "[$(date +'%F %T')] $*"; echo "[$(date +'%F %T')] $*" >> "$LOG_FILE"; } +need() { command -v "$1" >/dev/null 2>&1 || { log "ERROR: $1 not found"; exit 1; }; } -log_info() { log_message "INFO" "$1"; } -log_error() { log_message "ERROR" "$1" >&2; } -log_warning() { log_message "WARNING" "$1"; } - -check_command() { - if ! command -v "$1" &>/dev/null; then - log_error "$1 command not found. Please install it." - exit 1 - fi -} - -# --- Check directory permissions --- -log_info "Checking directory permissions..." +# --- Check directories --- +log "Checking directory permissions..." for dir in "/var/log" "/etc/telegraf" "$TELEGRAF_CONFIG_DIR"; do if [[ ! -d "$dir" ]]; then - log_warning "$dir directory does not exist. Creating it..." + log "WARN: $dir does not exist. Creating..." sudo mkdir -p "$dir" - [[ $? -ne 0 ]] && log_error "Failed to create directory $dir." && exit 1 fi - [[ ! -w "$dir" ]] && log_error "$dir directory is not writable. Please check permissions." && exit 1 + [[ ! -w "$dir" ]] && { log "ERROR: $dir not writable"; exit 1; } done -log_info "Directory permissions check complete." -# --- Prerequisites --- -log_info "Checking for required commands..." -for cmd in curl sudo dnf grep sed; do check_command "$cmd"; done -log_info "All required commands are present." +# --- Prereqs --- +log "Checking required commands..." +for cmd in sudo dnf grep sed; do need "$cmd"; done +log "All required commands are present." -# --- Add InfluxData Repo --- -log_info "Adding InfluxData repository..." -REPO_URL="https://repos.influxdata.com/centos/9/x86_64/stable" -REPO_FILE="/etc/yum.repos.d/influxdata.repo" - -if [[ ! -f "$REPO_FILE" ]]; then - cat < /dev/null <&2; } + +require() { command -v "$1" &>/dev/null || { error "Missing command: $1"; exit 1; } } + +[[ $EUID -eq 0 ]] || { error "Run as root (sudo)."; exit 1; } +require dnf +require systemctl +require sed +require tee + +log "Adding TimescaleDB repository…" +tee /etc/yum.repos.d/timescale_timescaledb.repo > /dev/null <<'EOF' +[timescale_timescaledb] +name=timescale_timescaledb +baseurl=https://packagecloud.io/timescale/timescaledb/el/9/$basearch +repo_gpgcheck=1 +enabled=1 +gpgcheck=0 +gpgkey=https://packagecloud.io/timescale/timescaledb/gpgkey +sslverify=1 +metadata_expire=300 +EOF + +dnf clean all -y +dnf makecache -y + +log "Installing TimescaleDB CE for PostgreSQL ${PG_VERSION}…" +dnf -y install "timescaledb-2-postgresql-${PG_VERSION}" + +# Optional tuning tool +dnf -y install timescaledb-tools || true + +# Ensure shared_preload_libraries includes timescaledb +if ! grep -q "^shared_preload_libraries.*timescaledb" "$CONF_FILE"; then + if grep -q "^shared_preload_libraries" "$CONF_FILE"; then + sed -i "s/^shared_preload_libraries.*/shared_preload_libraries = 'timescaledb'/" "$CONF_FILE" + else + echo "shared_preload_libraries = 'timescaledb'" >> "$CONF_FILE" + fi +fi + +# Run timescaledb-tune if available (non-interactive) +if command -v timescaledb-tune &>/dev/null; then + log "Running timescaledb-tune…" + timescaledb-tune --quiet --yes --pg-config "${PG_BIN_DIR}/pg_config" || true +fi + +log "Restarting ${PG_SERVICE}…" +systemctl restart "${PG_SERVICE}" + +# Wait for server +log "Waiting for PostgreSQL to accept connections…" +for i in {1..20}; do + sleep 2 + PGPASSWORD="$DB_SUPERUSER_PASSWORD" "${PG_BIN_DIR}/psql" -U "$DB_SUPERUSER" -h 127.0.0.1 -c "SELECT 1" &>/dev/null && break + [[ $i -eq 20 ]] && { error "PostgreSQL not accepting connections."; exit 1; } +done + +# Create role if missing +export PGPASSWORD="$DB_SUPERUSER_PASSWORD" +sudo -E -u postgres "${PG_BIN_DIR}/psql" -v ON_ERROR_STOP=1 </dev/null || true)" +if [[ "${DB_EXISTS:-}" != "1" ]]; then + log "Creating database ${NEW_DB_NAME}…" + sudo -E -u postgres "${PG_BIN_DIR}/psql" -c "CREATE DATABASE ${NEW_DB_NAME}" +else + log "Database ${NEW_DB_NAME} already exists." +fi + +# Enable extension + grants +sudo -E -u postgres "${PG_BIN_DIR}/psql" -v ON_ERROR_STOP=1 < 4, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_metrics_agent_time ON an_device_metrics (agent_host, time DESC); + +-- 2) MEASUREMENT: an_device_performance +CREATE TABLE IF NOT EXISTS an_device_performance ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT NOT NULL, + ssl_ae_core_utilization DOUBLE PRECISION, + ssl_se_core_utilization DOUBLE PRECISION, + PRIMARY KEY (time, agent_host) +); +SELECT create_hypertable('an_device_performance','time','agent_host', number_partitions => 4, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_perf_agent_time ON an_device_performance (agent_host, time DESC); + +-- 3) TABLE: an_device_storage (HOST-RESOURCES-MIB hrStorageTable) +-- hrStorage values are in allocation units; store alloc_unit to compute bytes later. +CREATE TABLE IF NOT EXISTS an_device_storage ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT NOT NULL, + prefix TEXT NOT NULL, + size BIGINT, -- hrStorageSize + used BIGINT, -- hrStorageUsed + alloc_unit BIGINT, -- hrStorageAllocationUnits + PRIMARY KEY (time, agent_host, prefix) +); +SELECT create_hypertable('an_device_storage','time','agent_host', number_partitions => 4, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_storage_prefix ON an_device_storage (prefix); +CREATE INDEX IF NOT EXISTS idx_storage_agent_time ON an_device_storage (agent_host, time DESC); + +-- 4) TABLE: apv_virtual_stats +CREATE TABLE IF NOT EXISTS apv_virtual_stats ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT NOT NULL, + serverid TEXT NOT NULL, + addr TEXT NOT NULL, + port TEXT NOT NULL, + protocol TEXT NOT NULL, + url_hits BIGINT, + hostname_hits BIGINT, + perstnt_cookie_hits BIGINT, + qos_cookie_hits BIGINT, + default_hits BIGINT, + perstnt_url_hits BIGINT, + static_hits BIGINT, + qos_network_hits BIGINT, + qos_url_hits BIGINT, + backup_hits BIGINT, + cache_hits BIGINT, + regex_hits BIGINT, + rcookie_hits BIGINT, + icookie_hits BIGINT, + conn_cnt BIGINT, + qos_client_port_hits BIGINT, + qos_body_hits BIGINT, + header_hits BIGINT, + hash_url_hits BIGINT, + redirect_hits BIGINT, + conn_per_sec DOUBLE PRECISION, + in_byte_per_sec DOUBLE PRECISION, + out_byte_per_sec DOUBLE PRECISION, + in_packet_per_sec DOUBLE PRECISION, + out_packet_per_sec DOUBLE PRECISION, + health_status TEXT, + PRIMARY KEY (time, agent_host, serverid, addr, port, protocol) +); +SELECT create_hypertable('apv_virtual_stats','time','agent_host', number_partitions => 8, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_virtual_serverid ON apv_virtual_stats (serverid); +CREATE INDEX IF NOT EXISTS idx_virtual_agent_time ON apv_virtual_stats (agent_host, time DESC); +CREATE INDEX IF NOT EXISTS idx_virtual_server_time ON apv_virtual_stats (serverid, time DESC); + +-- 5) TABLE: apv_real_stats +CREATE TABLE IF NOT EXISTS apv_real_stats ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT NOT NULL, + real_server_id TEXT NOT NULL, + addr TEXT NOT NULL, + port TEXT NOT NULL, + protocol TEXT NOT NULL, + status TEXT NOT NULL, + rs_cnt_of_req BIGINT, + rs_conn_cnt BIGINT, + rs_total_hits BIGINT, + rs_conn_per_sec DOUBLE PRECISION, + rs_in_byte_per_sec DOUBLE PRECISION, + rs_out_byte_per_sec DOUBLE PRECISION, + rs_in_packet_per_sec DOUBLE PRECISION, + rs_out_packet_per_sec DOUBLE PRECISION, + PRIMARY KEY (time, agent_host, real_server_id, addr, port, protocol) +); +SELECT create_hypertable('apv_real_stats','time','agent_host', number_partitions => 8, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_real_realserverid ON apv_real_stats (real_server_id); +CREATE INDEX IF NOT EXISTS idx_real_agent_time ON apv_real_stats (agent_host, time DESC); +CREATE INDEX IF NOT EXISTS idx_real_server_time ON apv_real_stats (real_server_id, time DESC); + +-- 6) TABLE: apv_llb_stats +-- Note: several fields arrive as strings (e.g., "7.502ms", "7+00:33:52"); keep TEXT. +CREATE TABLE IF NOT EXISTS apv_llb_stats ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT NOT NULL, + link_index BIGINT, + link_name TEXT NOT NULL, + link_gateway TEXT, + link_status TEXT NOT NULL, + link_resp_time TEXT, + link_up_time TEXT, + link_down_time TEXT, + link_down_count BIGINT, + link_down_event TEXT, + link_bandwid_in BIGINT, + link_bandwid_out BIGINT, + link_thresh BIGINT, + link_hits BIGINT, + link_conn BIGINT, + link_usage BIGINT, + PRIMARY KEY (time, agent_host, link_name) +); +SELECT create_hypertable('apv_llb_stats','time','agent_host', number_partitions => 4, if_not_exists => TRUE); +CREATE INDEX IF NOT EXISTS idx_llb_linkname ON apv_llb_stats (link_name); +CREATE INDEX IF NOT EXISTS idx_llb_agent_time ON apv_llb_stats (agent_host, time DESC); + +-- 7) Ensure NOT using columnstore policies (ignore if function/policy doesn’t exist) +DO $$ BEGIN PERFORM remove_columnstore_policy('apv_virtual_stats'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_columnstore_policy('apv_real_stats'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_columnstore_policy('apv_llb_stats'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_columnstore_policy('an_device_metrics'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_columnstore_policy('an_device_performance'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_columnstore_policy('an_device_storage'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; + +-- 8) Standard row-compression (NOT columnstore) +ALTER TABLE IF EXISTS apv_virtual_stats SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host, serverid', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE IF EXISTS apv_real_stats SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host, real_server_id', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE IF EXISTS apv_llb_stats SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host, link_name', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE IF EXISTS an_device_metrics SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE IF EXISTS an_device_performance SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host', timescaledb.compress_orderby = 'time DESC'); +ALTER TABLE IF EXISTS an_device_storage SET (timescaledb.compress, timescaledb.compress_segmentby = 'agent_host, prefix', timescaledb.compress_orderby = 'time DESC'); + +-- 9) Compression policies (compress after 7 days) +SELECT add_compression_policy('apv_virtual_stats', INTERVAL '7 days'); +SELECT add_compression_policy('apv_real_stats', INTERVAL '7 days'); +SELECT add_compression_policy('apv_llb_stats', INTERVAL '7 days'); +SELECT add_compression_policy('an_device_metrics', INTERVAL '7 days'); +SELECT add_compression_policy('an_device_performance', INTERVAL '7 days'); +SELECT add_compression_policy('an_device_storage', INTERVAL '7 days'); + +-- 10) Retention policies (drop after 180 days) +SELECT add_retention_policy('apv_virtual_stats', INTERVAL '180 days'); +SELECT add_retention_policy('apv_real_stats', INTERVAL '180 days'); +SELECT add_retention_policy('apv_llb_stats', INTERVAL '180 days'); +SELECT add_retention_policy('an_device_metrics', INTERVAL '180 days'); +SELECT add_retention_policy('an_device_performance', INTERVAL '180 days'); +SELECT add_retention_policy('an_device_storage', INTERVAL '180 days'); + +-- 11) Continuous aggregates (5-minute) — WITH NO DATA +-- 11.1) Virtual stats +CREATE MATERIALIZED VIEW IF NOT EXISTS cag_apv_virtual_stats_5m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('5 minutes', time) AS bucket, + agent_host, + serverid, + avg(conn_per_sec) AS avg_connps, + avg(in_byte_per_sec) AS avg_in_bps, + avg(out_byte_per_sec) AS avg_out_bps, + avg(in_packet_per_sec) AS avg_in_pps, + avg(out_packet_per_sec) AS avg_out_pps +FROM apv_virtual_stats +GROUP BY bucket, agent_host, serverid +WITH NO DATA; + +-- 11.2) Real stats +CREATE MATERIALIZED VIEW IF NOT EXISTS cag_apv_real_stats_5m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('5 minutes', time) AS bucket, + agent_host, + real_server_id, + avg(rs_conn_per_sec) AS avg_connps, + avg(rs_in_byte_per_sec) AS avg_in_bps, + avg(rs_out_byte_per_sec) AS avg_out_bps, + avg(rs_in_packet_per_sec) AS avg_in_pps, + avg(rs_out_packet_per_sec) AS avg_out_pps +FROM apv_real_stats +GROUP BY bucket, agent_host, real_server_id +WITH NO DATA; + +-- 11.3) LLB stats (link_resp_time is TEXT → parse number before averaging, assume milliseconds) +CREATE MATERIALIZED VIEW IF NOT EXISTS cag_apv_llb_stats_5m +WITH (timescaledb.continuous) AS +SELECT + time_bucket('5 minutes', time) AS bucket, + agent_host, + link_name, + avg( NULLIF(regexp_replace(link_resp_time, '[^0-9.]', '', 'g'), '')::double precision ) AS avg_resp_time_ms, + avg(link_usage) AS avg_usage, + avg(link_bandwid_in) AS avg_bandwid_in, + avg(link_bandwid_out) AS avg_bandwid_out, + max(link_down_count) AS max_down_count +FROM apv_llb_stats +GROUP BY bucket, agent_host, link_name +WITH NO DATA; + +-- 12) Refresh policies (remove if present, then add) +DO $$ BEGIN PERFORM remove_continuous_aggregate_policy('cag_apv_virtual_stats_5m'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_continuous_aggregate_policy('cag_apv_real_stats_5m'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; +DO $$ BEGIN PERFORM remove_continuous_aggregate_policy('cag_apv_llb_stats_5m'); EXCEPTION WHEN undefined_function THEN NULL; WHEN others THEN NULL; END $$; + +SELECT add_continuous_aggregate_policy('cag_apv_virtual_stats_5m', start_offset => INTERVAL '30 days', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '15 minutes'); +SELECT add_continuous_aggregate_policy('cag_apv_real_stats_5m', start_offset => INTERVAL '30 days', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '15 minutes'); +SELECT add_continuous_aggregate_policy('cag_apv_llb_stats_5m', start_offset => INTERVAL '30 days', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '15 minutes'); + +COMMIT; + +-- 13) Optional immediate backfill (outside tx) +CALL refresh_continuous_aggregate('cag_apv_virtual_stats_5m', now() - INTERVAL '30 days', now()); +CALL refresh_continuous_aggregate('cag_apv_real_stats_5m', now() - INTERVAL '30 days', now()); +CALL refresh_continuous_aggregate('cag_apv_llb_stats_5m', now() - INTERVAL '30 days', now()); Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py (revision 2695) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/hive/storage.py (working copy) @@ -114,6 +114,7 @@ if path['type'] == "file_system": path['size'] = getdirsize(path['path']) elif path['type'] == "es_query": + httpClient = None try: httpClient = http.client.HTTPConnection(CMDATA['ELASTICSEARCH_IP'], str(CMDATA['ELASTICSEARCH_PORT']),