Index: /branches/amp_4_0/scripts/check_adc_ssl.py =================================================================== --- /branches/amp_4_0/scripts/check_adc_ssl.py (revision 2771) +++ /branches/amp_4_0/scripts/check_adc_ssl.py (working copy) @@ -5,19 +5,16 @@ import logging import os import sys -from io import StringIO +from io import BytesIO import psycopg2 import pycurl -import toml -from influxdb_client import InfluxDBClient, Point -from influxdb_client.client.write_api import SYNCHRONOUS + # --- Logging Setup --- LOG_FILE = '/var/log/check_devices_ssl_certificate.log' -LOG_LEVEL = logging.INFO # Set to logging.DEBUG for more verbose output +LOG_LEVEL = logging.INFO -# Ensure the log directory exists and is writable (best practice for production) log_dir = os.path.dirname(LOG_FILE) if not os.path.exists(log_dir): try: @@ -25,7 +22,7 @@ except OSError as e: logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s') logging.error(f"Failed to create log directory {log_dir}: {e}. Logging to stderr.") - sys.exit(1) # Exit if logging setup is critical + sys.exit(1) try: logging.basicConfig( @@ -36,8 +33,6 @@ except Exception as e: logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s') logging.error(f"Failed to set up file logging to {LOG_FILE}: {e}. Logging to stderr.") - # If logging to the file fails, we can choose to continue to stderr or exit. - # For a critical script, exiting might be safer. sys.exit(1) logger = logging.getLogger(__name__) @@ -50,49 +45,8 @@ f"Failed to import djproject.an_settings: {e}. Please ensure the path is correct and the module exists.") sys.exit(1) -# --- InfluxDB 2.x Configuration --- -# IMPORTANT: Load these from environment variables in production for security. -INFLUXDB_URL = os.getenv("INFLUXDB_V2_URL", "http://localhost:8086") -INFLUXDB_TOKEN = os.getenv("INFLUXDB_V2_TOKEN", "YOUR_INFLUXDB_V2_TOKEN") -# --- NEW: Read InfluxDB Token from a TOML file --- -INFLUXDB_TOKEN_FILE = "/opt/influxdb2_token.toml" -try: - with open(INFLUXDB_TOKEN_FILE, 'r') as f: - config = toml.load(f) - INFLUXDB_TOKEN = config['influxdb']['token'] - logger.info(f"Successfully loaded InfluxDB token from {INFLUXDB_TOKEN_FILE}") -except FileNotFoundError: - logger.critical(f"InfluxDB token file not found: {INFLUXDB_TOKEN_FILE}. Exiting.") - sys.exit(1) -except KeyError as e: - logger.critical( - f"Key '{e}' not found in InfluxDB token file {INFLUXDB_TOKEN_FILE}. Ensure '[influxdb]' and 'token = ...' exist. Exiting.") - sys.exit(1) -except Exception as e: - logger.critical(f"Failed to read InfluxDB token from {INFLUXDB_TOKEN_FILE}: {e}. Exiting.") - sys.exit(1) -# Ensure the token was loaded -if not INFLUXDB_TOKEN: - logger.critical("InfluxDB token is empty after attempting to load from file. Exiting.") - sys.exit(1) - -INFLUXDB_ORG = os.getenv("INFLUXDB_V2_ORG", "AN") # <<< REPLACE THIS or set env var -INFLUXDB_BUCKET_CERTS_METADATA = os.getenv("INFLUXDB_V2_BUCKET_CERTS_METADATA", "AMP") # Bucket for cert metadata -INFLUXDB_BUCKET_EVENTS = os.getenv("INFLUXDB_V2_BUCKET_EVENTS", "event_detection") # Bucket for event data - -# Initialize InfluxDB 2.x client -try: - influx_client = InfluxDBClient(url=INFLUXDB_URL, token=INFLUXDB_TOKEN, org=INFLUXDB_ORG) - influx_write_api = influx_client.write_api(write_options=SYNCHRONOUS) - influx_query_api = influx_client.query_api() - logger.info("Successfully initialized InfluxDB 2.x client.") -except Exception as e: - logger.critical(f"Failed to initialize InfluxDB 2.x client: {e}. Exiting.") - sys.exit(1) - - def modify_url(url, device_type, vsite_name=None): """ Dynamically modifies a base URL based on device type and optional vsite name. @@ -100,7 +54,6 @@ _type = None new_url = None - # Query device type from string like: 'Array APV 3600 v5' -> 'APV' for each in device_type.split(): if each.lower() in DEVICE_TYPE_LIST: _type = each.lower() @@ -158,7 +111,7 @@ authorization = 'Basic %s' % base64.b64encode(auth_string).decode('utf-8') curl = pycurl.Curl() - curl.body = StringIO() + curl.body = BytesIO() curl.setopt(pycurl.VERBOSE, 0) curl.setopt(pycurl.CONNECTTIMEOUT, 5) @@ -174,14 +127,8 @@ curl.setopt(pycurl.HEADER, 0) if proto == 'https': - # --- SECURITY WARNING --- - # Disabling SSL verification (SSL_VERIFYPEER, SSL_VERIFYHOST) is a SECURITY RISK. - # It makes your connection vulnerable to Man-in-the-Middle attacks. - # In production, set SSL_VERIFYPEER to 1 and SSL_CACERT to a trusted CA bundle. - # If your devices use self-signed certificates, import their CAs into a trust store - # and point SSL_CACERT to it. - curl.setopt(pycurl.SSL_VERIFYPEER, 0) # Set to 1 in production and configure SSL_CACERT - curl.setopt(pycurl.SSL_VERIFYHOST, 0) # Set to 2 in production + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.URL, f"https://{ip}:{restapi_port}{url}") else: curl.setopt(pycurl.URL, f"http://{ip}:{restapi_port}{url}") @@ -202,8 +149,6 @@ multi.select(0.1) except pycurl.error as e: logger.error(f"pycurl multi.perform error: {e}. Continuing with other handles.") - # It's difficult to identify which specific handle failed here. - # Detailed error handling for each handle is done below. for i, curl in enumerate(handles): current_option = option_list[i] @@ -237,59 +182,144 @@ return result -def insert_certificate_event(time_range, agent_host, server_id, product, event_name, event_type): +def ensure_event_detection_table_exists(conn): """ - Inserts or updates certificate event data in InfluxDB 2.x using Flux for checking existence. + Ensures the event_detection table exists in PostgreSQL with TimescaleDB hypertable. """ - logger.debug(f"Checking InfluxDB for existing event: {event_name} on {agent_host} ({server_id})") + create_table_sql = """ + CREATE TABLE IF NOT EXISTS event_detection ( + time TIMESTAMPTZ NOT NULL, + agent_host TEXT, + server_id TEXT, + product TEXT, + event_name TEXT, + event_type TEXT, + quantity INTEGER, + details JSONB + ); + """ + enable_extension_sql = "CREATE EXTENSION IF NOT EXISTS timescaledb;" + create_hypertable_sql = "SELECT create_hypertable('event_detection', 'time', if_not_exists => TRUE);" + create_hypertable_fallback_sql = "SELECT create_hypertable('event_detection', 'time');" - # Flux query to check for existing events - # We query the 'event_detection' bucket for events within the specified time_range - # and filter by tags for uniqueness. - flux_query = f''' - from(bucket: "{INFLUXDB_BUCKET_EVENTS}") - |> range(start: -{time_range}) - |> filter(fn: (r) => r._measurement == "event_detection" and - r.agent_host == "{agent_host}" and - r.server_id == "{server_id}" and - r.product == "{product}" and - r.event_name == "{event_name}") - |> count() - |> yield(name: "result") - ''' - try: - tables = influx_query_api.query(flux_query, org=INFLUXDB_ORG) - event_exists = False - for table in tables: - for record in table.records: - if record.get_value() > 0: # If count is > 0, event exists - event_exists = True - break - if event_exists: - break + with conn.cursor() as cur: + try: + cur.execute(enable_extension_sql) + except psycopg2.Error as e: + # might fail if not superuser, but usually required for timescaledb + logger.warning(f"Could not enable timescaledb extension: {e}") + conn.rollback() + else: + conn.commit() - if not event_exists: - # Create a Point for writing data - point = Point("event_detection") \ - .tag("agent_host", agent_host) \ - .tag("server_id", server_id) \ - .tag("product", product) \ - .tag("event_name", event_name) \ - .tag("event_type", event_type) \ - .field("quantity", 1) # InfluxDB 2.x often prefers integer fields explicitly (1i) for quantities + cur.execute(create_table_sql) + conn.commit() + + check_hypertable_sql = "SELECT count(*) FROM timescaledb_information.hypertables WHERE hypertable_name = 'event_detection';" + cur.execute(check_hypertable_sql) + if cur.fetchone()[0] == 0: + try: + cur.execute(create_hypertable_sql) + conn.commit() + logger.info("Created hypertable for event_detection.") + except psycopg2.Error as e: + logger.warning(f"Failed to create hypertable with if_not_exists: {e}. Trying fallback.") + conn.rollback() + try: + cur.execute(create_hypertable_fallback_sql) + conn.commit() + logger.info("Created hypertable for event_detection (fallback).") + except psycopg2.Error as e2: + logger.error(f"Failed to create hypertable (fallback): {e2}") + conn.rollback() + else: + logger.info("event_detection is already a hypertable.") - influx_write_api.write(bucket=INFLUXDB_BUCKET_EVENTS, org=INFLUXDB_ORG, record=point) - logger.info(f"Inserting new event into InfluxDB: {event_name} for {agent_host}.") - logger.debug(f"Event successfully written to InfluxDB for {event_name} on {agent_host}.") - else: - logger.debug( - f"Event {event_name} for {agent_host} already exists in InfluxDB within {time_range}. Skipping write.") + logger.info("Ensured event_detection table exists.") + except psycopg2.Error as e: + logger.error(f"Failed to ensure event_detection table exists: {e}") + conn.rollback() - except Exception as e: - logger.error(f"Error communicating with InfluxDB 2.x for event insertion: {e}", exc_info=True) + +def ensure_adc_vs_ssl_info_table_exists(conn): + """ + Ensures the adc_vs_ssl_info table exists in PostgreSQL. + """ + create_table_sql = """ + CREATE TABLE IF NOT EXISTS adc_vs_ssl_info ( + device_id INTEGER, + device_name TEXT, + vs_name TEXT, + vhost_name TEXT, + cert_type TEXT, + expiration INTEGER, + PRIMARY KEY (device_id, device_name, vs_name, vhost_name, cert_type) + ); + """ + try: + with conn.cursor() as cur: + cur.execute(create_table_sql) + conn.commit() + logger.info("Ensured adc_vs_ssl_info table exists.") + except psycopg2.Error as e: + logger.error(f"Failed to ensure adc_vs_ssl_info table exists: {e}") + conn.rollback() + + +def insert_certificate_event(conn, time_val, tenant_id, event_name, cert_serial, cert_issuer, cert_subject, + cert_not_before, cert_not_after, cert_status, extra_tags): + """ + Inserts certificate event data into PostgreSQL event_detection table. + """ + agent_host = extra_tags.get('agent_host') + server_id = tenant_id # Using tenant_id/device_id as server_id + product = extra_tags.get('product') + event_type = extra_tags.get('event_type') + vhost_name = extra_tags.get('vhost_name') + logger.debug(f"Checking PostgreSQL for existing event: {event_name} on {agent_host} ({server_id})") + check_sql = """ + SELECT count(*) FROM event_detection + WHERE time > %s::timestamptz - INTERVAL '1 hour' + AND agent_host = %s + AND server_id = %s + AND product = %s + AND event_name = %s + """ + + try: + with conn.cursor() as cur: + cur.execute(check_sql, (time_val, agent_host, str(server_id), product, event_name)) + count = cur.fetchone()[0] + + if count == 0: + details = { + 'cert_serial': cert_serial, + 'cert_issuer': cert_issuer, + 'cert_subject': cert_subject, + 'cert_not_before': cert_not_before, + 'cert_not_after': cert_not_after, + 'cert_status': cert_status, + 'vhost_name': vhost_name + } + + insert_sql = """ + INSERT INTO event_detection (time, agent_host, server_id, product, event_name, event_type, quantity, details) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """ + cur.execute(insert_sql, (time_val, agent_host, str(server_id), product, event_name, event_type, 1, json.dumps(details))) + conn.commit() + logger.info(f"Inserting new event into PostgreSQL: {event_name} for {agent_host}.") + else: + logger.debug(f"Event {event_name} for {agent_host} already exists in PostgreSQL. Skipping write.") + + except psycopg2.Error as e: + logger.error(f"Error communicating with PostgreSQL for event insertion: {e}") + conn.rollback() + + def get_expire(data): """Helper function for sorting certificate expiration data.""" if data[4] == 'Expired': @@ -317,6 +347,9 @@ cur = conn.cursor() logger.info("Connected to PostgreSQL database 'cm'.") + ensure_event_detection_table_exists(conn) + ensure_adc_vs_ssl_info_table_exists(conn) + logger.info("Deleting all existing data from adc_vs_ssl_info table.") cur.execute('delete from adc_vs_ssl_info') # This line truncates the table every run conn.commit() @@ -332,8 +365,6 @@ restapi_options = [] result = [] ssl_cli_data = json.dumps({'cmd': 'show ssl host'}) - - # Prepare the first set of API calls (show ssl host) for row in rows: device_name, ip_address, restapi_port, restapi_username, restapi_password, connection, device_type, device_id = row @@ -354,11 +385,9 @@ logger.info("No connected ADC devices found to query 'show ssl host'. Exiting early.") return - # Execute the first set of API calls restapi_result = call_restapi_multi(restapi_options) cert_restapi_options = [] - # Process 'show ssl host' results and prepare second set of API calls (show ssl certificate) logger.info("Processing 'show ssl host' results and preparing 'show ssl certificate' calls.") for index, rst in enumerate(restapi_result): original_options = restapi_options[index] @@ -402,10 +431,8 @@ logger.info("No virtual hosts found requiring 'show ssl certificate' calls. Exiting early.") return - # Execute the second set of API calls cert_restapi_result = call_restapi_multi(cert_restapi_options) - # Process certificate details and insert into DB/InfluxDB logger.info("Processing 'show ssl certificate' results and inserting into database.") for index, cert_rst in enumerate(cert_restapi_result): original_cert_options = cert_restapi_options[index] @@ -459,6 +486,7 @@ expiration_days = -1 cert_status = "Expired" insert_certificate_event( + conn, datetime.datetime.utcnow().isoformat("T") + "Z", # Current UTC time for event device_id, # Using device_id as tenant_id for event 'CertKeyExpired', @@ -495,6 +523,7 @@ if event_type_suffix: insert_certificate_event( + conn, datetime.datetime.utcnow().isoformat("T") + "Z", # Current UTC time for event device_id, # Using device_id as tenant_id for event @@ -512,12 +541,8 @@ cert_type = "" # Reset for next cert in block continue - # After parsing all lines for a certificate, if we have data, insert/update DB if expiration_days is not None: try: - # Using ON CONFLICT (device_id, device_name, vs_name, vhost_name, cert_type) - # for a proper UPSERT logic, as 'id' is a serial. - # Ensure the combination of these fields is UNIQUE in your table schema. cur.execute( """ INSERT INTO adc_vs_ssl_info @@ -563,9 +588,6 @@ if conn: conn.close() logger.info("PostgreSQL connection closed.") - if influx_client: - influx_client.close() # Close InfluxDB client connection - logger.info("InfluxDB client connection closed.") # Sort and print the final result result.sort(key=get_expire, reverse=False) Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/router.py (working copy) @@ -1023,12 +1023,12 @@ def check_adc_ssl_cert(requset): - ret = os.popen("/usr/bin/python /ca/bin/check_adc_ssl.py").read() + ret = os.popen("python3 /ca/bin/check_adc_ssl.py").read() return HttpResponse(ret) def check_build_info(request): - ret = os.popen("/usr/bin/python /ca/bin/check_build.py").read() + ret = os.popen("python3 /ca/bin/check_build.py").read() return HttpResponse(ret, content_type='application/json') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/apv-ssl-management/check-apv-vs-certificate-expiry.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/apv-ssl-management/check-apv-vs-certificate-expiry.html (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/apv-ssl-management/check-apv-vs-certificate-expiry.html (working copy) @@ -2,11 +2,6 @@
- - - - - @@ -18,19 +13,19 @@ - + - + - + - + @@ -42,10 +37,7 @@ - - + \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/add-device-groups.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/add-device-groups.html (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/add-device-groups.html (working copy) @@ -1,37 +1,26 @@

Add a Device Group

- +
- + + @if (deviceGroupForm.get('groupName')?.hasError('required')) { + + Group name is required. + + }
- - - + \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.html (revision 2773) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.html (working copy) @@ -8,7 +8,7 @@
- +
No. Service Name {{ element[0] }} {{ element[1] }} SSL Hostname {{ element[1] }} {{ element[2] }} Certificate Type{{ element[0]}}{{ element[3]}} Days before expiration {{ element[1] }} {{ element[4] }}
Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.ts =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.ts (revision 2773) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/device-groups/device-groups.ts (working copy) @@ -5,7 +5,7 @@ import { MatTableDataSource } from '@angular/material/table'; import { SharedModule } from '../../../shared/shared-module'; import { Confirmation } from '../../../services/confirmation'; -import { MatDialog, MatDialogRef } from '@angular/material/dialog'; +import { MatDialog, MatDialogConfig, MatDialogRef } from '@angular/material/dialog'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { MatPaginator } from '@angular/material/paginator'; import { GlobalSerialPipe } from '../../../pipes/global-serial-pipe'; @@ -23,6 +23,7 @@ displayedColumns: string[] = ['serial', 'name', 'devices', 'action']; dataSource = new MatTableDataSource(this.deviceGroups); dialog = inject(MatDialog); + dialogConfig = new MatDialogConfig(); totalRecords = 0; @ViewChild(MatPaginator) paginator!: MatPaginator; @@ -82,10 +83,10 @@ confirmMsg = "Deleting this device group will also remove all devices associated with it. Are you sure you want to proceed?" } this._confirmation.openConfirmDialog({ - title: `Delete ${groupName}?`, + title: `Delete Device Group - ${groupName}?`, message: confirmMsg, - confirmButtonText: 'Yes, Delete It', - cancelButtonText: 'No, Keep It', + confirmButtonText: 'Delete', + cancelButtonText: 'Cancel', confirmButtonColor: 'warn', cancelButtonColor: 'primary' }).subscribe(result => { @@ -101,7 +102,7 @@ if (!result[0]) { this._notification.showError(`${result[1]}`); } else { - this._notification.showSuccess(`${groupName} deleted successfully!`); + this._notification.showSuccess(`The device group ${groupName} deleted successfully!`); this.getDeviceGroups(); } } @@ -109,21 +110,20 @@ error: (err) => { if (err.status === 200) { // ToDo: Backend fix required. - this._notification.showSuccess(`${groupName} deleted successfully!`); + this._notification.showSuccess(`The device group ${groupName} deleted successfully!`); this.getDeviceGroups(); } else { - this._notification.showError('Deletion Failed.'); + this._notification.showError('Failed to delete the device group.'); } } }) - } else { - this._notification.showSuccess('Deletion cancelled.'); } }); } addDeviceGroup() { - const dialogRef = this.dialog.open(AddDeviceGroupsDialog); + this.dialogConfig.disableClose = true; + const dialogRef = this.dialog.open(AddDeviceGroupsDialog, this.dialogConfig); dialogRef.afterClosed().subscribe(result => { if (result) { let rawPayload = new FormData(); @@ -132,11 +132,11 @@ .pipe(take(1)) .subscribe({ next: () => { - this._notification.showSuccess(`${result?.groupName} added successfully!`); + this._notification.showSuccess(`The device group ${result?.groupName} added successfully!`); this.getDeviceGroups(); }, error: () => { - this._notification.showError('Creation Failed.'); + this._notification.showError('Failed to create the device group.'); } }) } Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/device-license-overview.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/device-license-overview.html (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/device-license-overview.html (working copy) @@ -15,10 +15,5 @@
- - + + \ No newline at end of file Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.html (revision 2773) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.html (working copy) @@ -8,7 +8,8 @@
- + Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.ts =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.ts (revision 2773) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/devices.ts (working copy) @@ -91,8 +91,8 @@ this._confirmation.openConfirmDialog({ title: `Delete ${deviceName}?`, message: confirmMsg, - confirmButtonText: 'Yes, Delete It', - cancelButtonText: 'No, Keep It', + confirmButtonText: 'Delete', + cancelButtonText: 'Cancel', confirmButtonColor: 'warn', cancelButtonColor: 'primary' }).subscribe(result => { @@ -127,8 +127,6 @@ } } }) - } else { - this._notification.showSuccess('Deletion cancelled.'); } }); } @@ -148,7 +146,7 @@ this._confirmation.openConfirmDialog({ title: `Save configuration - ${deviceName}?`, message: confirmMsg, - confirmButtonText: 'Yes, Save It', + confirmButtonText: 'Save', cancelButtonText: 'Cancel', confirmButtonColor: 'warn', cancelButtonColor: 'primary' @@ -187,8 +185,6 @@ } } }) - } else { - this._notification.showSuccess('Save cancelled.'); } }); } @@ -196,6 +192,7 @@ addDevice() { this.dialogConfig.width = '65%'; this.dialogConfig.height = '80%'; + this.dialogConfig.disableClose = true; this.dialogConfig.data = { deviceType: JSON.parse(this.deviceType)?.DEVICE_STD_LIST, groups: this.groups,