Index: /branches/amp_4_0/platform/config/README.md =================================================================== --- /branches/amp_4_0/platform/config/README.md (revision 0) +++ /branches/amp_4_0/platform/config/README.md (working copy) @@ -0,0 +1,22 @@ +## Directory - platform/config + +Contains the configurations to support our system services + +### Telegraf + +#### Configuration + +We can use the `telegraf.conf` as core telegraf configuration, location - `/etc/telegraf/` + +The telegraf configuration file `input_snmp.conf` should be placed in `/etc/telegraf/telefraf.d` directory. + +### Logstash + +#### Configuration + +The logstash configuration file `syslog.conf` should be placed in `/etc/telegraf/conf.d` directory. + +### Postgresql + +The `init_db.sql` file contains the SQL (DDL) for the initial AMP setup. The same can be loaded to Postgresql using +`psql -U amp_admin -d cm -f /path/to/your/init_db.sql` command. Index: /branches/amp_4_0/platform/config/init_db.sql =================================================================== --- /branches/amp_4_0/platform/config/init_db.sql (revision 0) +++ /branches/amp_4_0/platform/config/init_db.sql (working copy) @@ -0,0 +1,367 @@ +-- 'cm' should be created as part of the install_psql.sh script. + +-- Create a cm_role table +CREATE TABLE IF NOT EXISTS cm_role +( + name varchar(64) primary key, + description varchar(32) NOT NULL, + priority integer DEFAULT 1, + status integer DEFAULT 0, + time varchar(64) NOT NULL, + device_list varchar(3328) DEFAULT NULL, + extend jsonb +); + +-- Create a zone table +CREATE TABLE IF NOT EXISTS zone +( + name varchar(32) NOT NULL, + PRIMARY KEY (name) +); + +-- Create a cm_acl_resource table +CREATE TABLE IF NOT EXISTS cm_acl_resource +( + id serial primary key, + group_name varchar(64) NOT NULL, + resource varchar(512) NOT NULL, + description varchar(32) NOT NULL, + status integer DEFAULT 0, + time varchar(64) NOT NULL, + device_list varchar(3328) DEFAULT NULL, + extend jsonb +); + +-- Create a cm_vpn_resource table +CREATE TABLE IF NOT EXISTS cm_vpn_resource +( + id serial primary key, + group_name varchar(64) NOT NULL, + resource varchar(128) NOT NULL, + type integer default 1, + group_type varchar(32) DEFAULT 'included', + description varchar(32) NOT NULL, + status integer DEFAULT 0, + time varchar(64) NOT NULL, + device_list varchar(3328) DEFAULT NULL, + extend jsonb +); + +-- Create a device_group table +CREATE TABLE IF NOT EXISTS device_group +( + name character varying(128) COLLATE pg_catalog."default" NOT NULL, + CONSTRAINT device_group_pkey PRIMARY KEY (name) +); + +-- Create a ha_cluster table +CREATE TABLE IF NOT EXISTS ha_cluster +( + id serial primary key, + name varchar(64) NOT NULL, + device_type varchar(16) default NULL, + extend_fields varchar(2048) default null +); + +-- Create a vsite_list table +CREATE TABLE IF NOT EXISTS vsite_list +( + id serial primary key, + vs_name varchar(64) NOT NULL, + device_name varchar(32) NOT NULL, + site_FQDM varchar(128) DEFAULT NULL, + ip varchar(128) NOT NULL, + site_type varchar(32) NOT NULL, + description varchar(128) DEFAULT NULL, + parent_site varchar(64) DEFAULT NULL, + extend jsonb +); + +-- Create a tar_file table +CREATE TABLE IF NOT EXISTS tar_file +( + id serial primary key, + name varchar(64) NOT NULL, + vsite_name varchar(64) DEFAULT NULL, + extend jsonb +); + +-- Create a proxy_cm table +CREATE TABLE IF NOT EXISTS proxy_cm +( + name varchar(32) NOT NULL, + ip varchar(64) NOT NULL, + PRIMARY KEY (name), + extend jsonb +); + +-- Insert default values for the proxy_cm table +INSERT INTO proxy_cm(name, ip) values ('default', '0.0.0.0'); + +-- Create a device table +CREATE TABLE IF NOT EXISTS device +( + id varchar(64) NOT NULL, + zone varchar(32) NOT NULL, + name varchar(32) NOT NULL UNIQUE, + ip_address varchar(64) NOT NULL, + restapi_port INTEGER DEFAULT 9997, + restapi_username varchar(16) DEFAULT NULL, + restapi_password varchar(256) DEFAULT NULL, + console_username varchar(16) DEFAULT NULL, + console_password varchar(256) DEFAULT NULL, + connection varchar(16) NOT NULL DEFAULT 'unconnected', + status varchar(16) NOT NULL DEFAULT 'new', + version varchar(8192) DEFAULT NULL, + license_key varchar(128) DEFAULT NULL, + gateway_domain varchar(128) DEFAULT NULL, + location varchar(128) NOT NULL, + firewall_ip varchar(64) NOT NULL, + intranet_ip varchar(64) NOT NULL, + type varchar(16) NOT NULL, + extend_fields varchar(8192) DEFAULT NULL, + own varchar(64) DEFAULT NULL, + log_enable INTEGER DEFAULT 0, + webui_port INTEGER DEFAULT 8888, + device_group varchar(64) NOT NULL, + extend jsonb, + snmp_general varchar(256) NOT NULL DEFAULT '{"snmp_enable": false}', + enable_password varchar(32) DEFAULT NULL, + create_time varchar(40) DEFAULT (now()), + protocol varchar(16) DEFAULT 'restapi', + PRIMARY KEY (id), + FOREIGN KEY (zone) REFERENCES proxy_cm(name) ON DELETE CASCADE, + FOREIGN KEY (device_group) REFERENCES device_group(name) ON DELETE CASCADE +); + +-- Create a hc_device table +CREATE TABLE IF NOT EXISTS hc_device +( + g_id serial references ha_cluster (id) on delete cascade, + d_id varchar(64) references device (id) on delete cascade, + unique (d_id, g_id) +); + +-- Create a service table +CREATE TABLE IF NOT EXISTS service +( + id serial primary key, + name varchar(64), + device_type varchar(32) NOT NULL, + service_name varchar(128) NOT NULL, + service_type varchar(32) NOT NULL, + extend jsonb +); + +-- Create an ip_pool table +CREATE TABLE IF NOT EXISTS ip_pool +( + id serial primary key, + ip_pool_range varchar(128) NOT NULL, + device_id varchar(64) NOT NULL REFERENCES device (id) ON DELETE CASCADE, + extend jsonb +); + +-- Create an update_list table +CREATE TABLE IF NOT EXISTS update_list +( + id serial primary key, + app_name varchar(32) NOT NULL, + build_version varchar(64) NOT NULL, + file_size INTEGER DEFAULT 0, + md5_value varchar(64) DEFAULT NULL, + download_link varchar(128) DEFAULT NULL, + location varchar(128) DEFAULT NULL, + extend jsonb +); + +-- Create a task table +CREATE TABLE IF NOT EXISTS task +( + id serial primary key, + name varchar(256) NOT NULL, + type varchar(16) NOT NULL, + trigger varchar(16) NOT NULL, + state varchar(16) NOT NULL, + next_run_time varchar(64) NOT NULL, + description text, + failed_times integer default 3, + scheduler varchar(10240) default NULL, + custom_fields varchar(1024) default NULL, + result_field varchar(1024) default NULL, + device_list varchar(1024) default NULL, + extend jsonb +); + +-- Create a file_type table +CREATE TABLE IF NOT EXISTS file_type +( + name varchar(32) primary key, + extend jsonb +); + +-- Insert supported file types +INSERT INTO file_type (name) +VALUES ('device'), + ('command'), + ('diff'), + ('system'), + ('backup'), + ('customize'), + ('template'), + ('vs'); + +-- Create a file_list table +CREATE TABLE IF NOT EXISTS file_list +( + name varchar(128) primary key, + create_time varchar(64) NOT NULL, + modify_time varchar(64) DEFAULT NULL, + type varchar(32) NOT NULL, + device_type varchar(32) DEFAULT NULL, + comment text, + FOREIGN KEY (type) REFERENCES file_type (name) ON DELETE CASCADE, + extend jsonb +); + +-- Create an audit_user table +CREATE TABLE IF NOT EXISTS audit_user +( + id serial primary key, + user_name varchar(16) NOT NULL, + vsite_name varchar(64) NOT NULL, + host_name varchar(64) NOT NULL, + extend jsonb, + lasted timestamp with time zone +); + +-- Create an ext_log table +CREATE TABLE IF NOT EXISTS ext_log +( + id BIGSERIAL, + name varchar(64) NOT NULL, + action varchar(20) NOT NULL, + time varchar(30) NOT NULL, + result varchar(20) NOT NULL, + PRIMARY KEY (id) +); + +-- Create a role table +CREATE TABLE IF NOT EXISTS role +( + id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ), + role_name varchar(128), + created_at timestamp without time zone, + CONSTRAINT role_primary_key PRIMARY KEY (id) +); + +-- Create a user_authorization table +CREATE TABLE IF NOT EXISTS user_authorization +( + id serial primary key, + username varchar(128) NOT NULL UNIQUE, + auth varchar(8192) DEFAULT NULL, + root INTEGER DEFAULT 0, + role_id INTEGER, + user_type varchar(32), + FOREIGN KEY (role_id) REFERENCES role (id) +); + +-- Create a operation_log table +CREATE TABLE IF NOT EXISTS operation_log +( + id serial primary key, + username varchar(128) NOT NULL, + client_ip varchar(64) NOT NULL, + time varchar(30) NOT NULL, + level varchar(20), + module varchar(20), + operation varchar(1024) +); + +-- Create a schedule_backup_all table +CREATE TABLE IF NOT EXISTS schedule_backup_all +( + task_name varchar(128) NOT NULL UNIQUE, + device_name varchar(128) NOT NULL, + interval INTEGER NOT NULL, + unit varchar(10) NOT NULL +); + +-- Create a log_settings table +CREATE TABLE IF NOT EXISTS log_settings +( + enable integer default 1, + level INTEGER default 2 +); + +-- Create a log_host table +CREATE TABLE IF NOT EXISTS log_host +( + host varchar(128), + port integer NOT NULL, + protocol integer default 1, + PRIMARY KEY (host, port, protocol) +); + +-- Create a config_template table +CREATE TABLE IF NOT EXISTS config_template +( + key varchar(128) primary key, + default_value varchar(128) NOT NULL, + description varchar(256) +); + +-- Create a device_config_template table +CREATE TABLE IF NOT EXISTS device_config_template +( + id serial NOT NULL, + device_id varchar(64) NOT NULL, + key varchar(64) NOT NULL, + value varchar(64) NOT NULL, + PRIMARY KEY (id) +); + +-- Create a role_device_group table +CREATE TABLE IF NOT EXISTS role_device_group +( + id serial primary key, + role_id integer, + device_group_name varchar(128), + created_at timestamp without time zone, + FOREIGN KEY (role_id) REFERENCES role (id) ON DELETE CASCADE, + FOREIGN KEY (device_group_name) REFERENCES device_group (name) ON DELETE CASCADE +); + +-- Create a backup_schedule table +CREATE TABLE IF NOT EXISTS backup_schedule +( + id SERIAL PRIMARY KEY, + frequency varchar(64) NOT NULL, + time TIME NOT NULL, + day_of_the_week integer DEFAULT NULL, + day_of_the_month integer DEFAULT NULL, + month integer DEFAULT NULL CHECK (month BETWEEN 1 AND 12) +); + +-- Create a remote_storage table +CREATE TABLE IF NOT EXISTS remote_storage +( + id SERIAL PRIMARY KEY, + ip varchar(64) NOT NULL, + username varchar(64) NOT NULL, + password varchar(64) NOT NULL, + path varchar(64) DEFAULT NULL +); + +-- Create a backups table +CREATE TABLE IF NOT EXISTS backups +( + id SERIAL PRIMARY KEY, + filename varchar(64) NOT NULL, + status varchar(64) DEFAULT NULL, + destination varchar(64) DEFAULT NULL, + time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- psql -U amp_admin -d cm -f /path/to/your/init_db.sql Index: /branches/amp_4_0/platform/config/input_snmp.conf =================================================================== --- /branches/amp_4_0/platform/config/input_snmp.conf (revision 0) +++ /branches/amp_4_0/platform/config/input_snmp.conf (working copy) @@ -0,0 +1,322 @@ +############################################################################### +# SNMP INPUT PLUGIN +############################################################################### +[[inputs.snmp]] + agents = ["192.168.85.101:161"] + community = "public" + agent_host_tag = "source" + timeout = "2s" + name = "snmp" + + [[inputs.snmp.field]] + name = "cpu_usage" + oid = ".1.3.6.1.4.1.7564.30.1.0" + + [[inputs.snmp.field]] + name = "mem_usage" + oid = ".1.3.6.1.4.1.7564.4.5.0" + + [[inputs.snmp.field]] + name = "net_mem_usage" + oid = ".1.3.6.1.4.1.7564.4.2.0" + + [[inputs.snmp.field]] + name = "totalOpenSSLConns" + oid = ".1.3.6.1.4.1.7564.20.2.1.0" + + [[inputs.snmp.field]] + name = "connections" + oid = ".1.3.6.1.4.1.7564.30.2.0" + + [[inputs.snmp.field]] + name = "requests" + oid = ".1.3.6.1.4.1.7564.30.3.0" + + [[inputs.snmp.field]] + name = "total_in" + oid = ".1.3.6.1.4.1.7564.23.2.0" + + [[inputs.snmp.field]] + name = "total_out" + oid = ".1.3.6.1.4.1.7564.23.3.0" + + # Commented out due to error (empty value causes float conversion failure) + # Uncomment only if the OID starts returning a valid numeric value. + [[inputs.snmp.field]] + # conversion = "float" + name = "sslAECoreUtilization" + oid = ".1.3.6.1.4.1.7564.30.9.0" + + [[inputs.snmp.field]] + # conversion = "float" + name = "sslSECoreUtilization" + oid = ".1.3.6.1.4.1.7564.30.10.0" + + [[inputs.snmp.table]] + name = "snmp_storage" + + [[inputs.snmp.table.field]] + is_tag = true + name = "prefix" + oid = ".1.3.6.1.2.1.25.2.3.1.3" + + [[inputs.snmp.table.field]] + name = "size" + oid = ".1.3.6.1.2.1.25.2.3.1.5" + + [[inputs.snmp.table.field]] + name = "used" + oid = ".1.3.6.1.2.1.25.2.3.1.6" + + [[inputs.snmp.table]] + name = "virtualStats" + + [[inputs.snmp.table.field]] + is_tag = true + name = "ServerId" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.2" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Addr" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.3" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Port" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.4" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Protocol" + oid = ".1.3.6.1.4.1.7564.19.1.2.2.1.3" + + [[inputs.snmp.table.field]] + name = "URLHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.5" + + [[inputs.snmp.table.field]] + name = "HostnameHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.6" + + [[inputs.snmp.table.field]] + name = "PerstntCookieHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.7" + + [[inputs.snmp.table.field]] + name = "QosCookieHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.8" + + [[inputs.snmp.table.field]] + name = "DefaultHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.9" + + [[inputs.snmp.table.field]] + name = "PerstntURLHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.10" + + [[inputs.snmp.table.field]] + name = "StaticHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.11" + + [[inputs.snmp.table.field]] + name = "QosNetworkHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.12" + + [[inputs.snmp.table.field]] + name = "QosURLHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.13" + + [[inputs.snmp.table.field]] + name = "BackupHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.14" + + [[inputs.snmp.table.field]] + name = "CacheHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.15" + + [[inputs.snmp.table.field]] + name = "RegexHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.16" + + [[inputs.snmp.table.field]] + name = "RCookieHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.17" + + [[inputs.snmp.table.field]] + name = "ICookieHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.18" + + [[inputs.snmp.table.field]] + name = "ConnCnt" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.19" + + [[inputs.snmp.table.field]] + name = "QosClientPortHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.22" + + [[inputs.snmp.table.field]] + name = "QosBodyHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.23" + + [[inputs.snmp.table.field]] + name = "HeaderHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.24" + + [[inputs.snmp.table.field]] + name = "HashURLHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.25" + + [[inputs.snmp.table.field]] + name = "RedirectHits" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.26" + + [[inputs.snmp.table.field]] + name = "ConnPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.31" + + [[inputs.snmp.table.field]] + name = "InBytePerSec" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.32" + + [[inputs.snmp.table.field]] + name = "OutBytePerSec" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.33" + + [[inputs.snmp.table.field]] + name = "InPacketPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.34" + + [[inputs.snmp.table.field]] + name = "OutPacketPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.35" + + [[inputs.snmp.table.field]] + is_tag = true + name = "HealthStatus" + oid = ".1.3.6.1.4.1.7564.19.2.2.1.1.36" + + [[inputs.snmp.table]] + name = "realStats" + + [[inputs.snmp.table.field]] + is_tag = true + name = "realServerId" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.2" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Addr" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.3" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Port" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.4" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Protocol" + oid = ".1.3.6.1.4.1.7564.19.1.1.2.1.3" + + [[inputs.snmp.table.field]] + is_tag = true + name = "Status" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.8" + + [[inputs.snmp.table.field]] + name = "rsCntOfReq" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.5" + + [[inputs.snmp.table.field]] + name = "rsConnCnt" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.6" + + [[inputs.snmp.table.field]] + name = "rsTotalHits" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.7" + + [[inputs.snmp.table.field]] + name = "rsConnPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.13" + + [[inputs.snmp.table.field]] + name = "rsInBytePerSec" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.14" + + [[inputs.snmp.table.field]] + name = "rsOutBytePerSec" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.15" + + [[inputs.snmp.table.field]] + name = "rsInPacketPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.16" + + [[inputs.snmp.table.field]] + name = "rsOutPacketPerSec" + oid = ".1.3.6.1.4.1.7564.19.2.1.1.1.17" + + [[inputs.snmp.table]] + name = "llbStats" + + [[inputs.snmp.table.field]] + name = "linkIndex" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.1" + + [[inputs.snmp.table.field]] + is_tag = true + name = "linkName" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.2" + + [[inputs.snmp.table.field]] + name = "linkGateway" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.3" + + [[inputs.snmp.table.field]] + is_tag = true + name = "linkStatus" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.4" + + [[inputs.snmp.table.field]] + name = "linkRespTime" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.5" + + [[inputs.snmp.table.field]] + name = "linkUpTime" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.6" + + [[inputs.snmp.table.field]] + name = "linkDownTime" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.7" + + [[inputs.snmp.table.field]] + name = "linkDownCount" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.8" + + [[inputs.snmp.table.field]] + name = "linkDownEvent" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.9" + + [[inputs.snmp.table.field]] + name = "linkBandwidIn" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.10" + + [[inputs.snmp.table.field]] + name = "linkBandwidOut" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.11" + + [[inputs.snmp.table.field]] + name = "linkThresh" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.12" + + [[inputs.snmp.table.field]] + name = "linkHits" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.13" + + [[inputs.snmp.table.field]] + name = "linkConn" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.14" + + [[inputs.snmp.table.field]] + name = "linkUsage" + oid = ".1.3.6.1.4.1.7564.34.2.1.2.1.15" Index: /branches/amp_4_0/platform/config/syslog.conf =================================================================== --- /branches/amp_4_0/platform/config/syslog.conf (revision 0) +++ /branches/amp_4_0/platform/config/syslog.conf (working copy) @@ -0,0 +1,50 @@ +input { + udp { + port => 5514 + type => "syslog" + } + tcp { + port => 5514 + type => "syslog" + } +} + +filter { + kv { + source => "message" + field_split => " " + value_split => "=" + remove_field => ["message"] + } + + date { + match => ["date", "YYYY-MM-dd"] + target => "@timestamp" + } + + if [time] { + mutate { + update => { "date" => "%{date} %{time}" } + remove_field => ["time"] + } + date { + match => ["date", "YYYY-MM-dd HH:mm:ss"] + target => "@timestamp" + } + } +} + +output { + elasticsearch { + hosts => ["http://localhost:9200"] + index => "acm-%{+YYYY.MM.dd}" + user => "logstash_internal" + password => "LArr@y2050" + template_name => "amplog" + manage_template => false + } + + stdout { + codec => rubydebug + } +} Index: /branches/amp_4_0/platform/config/syslog/amplog_template.json =================================================================== --- /branches/amp_4_0/platform/config/syslog/amplog_template.json (revision 0) +++ /branches/amp_4_0/platform/config/syslog/amplog_template.json (working copy) @@ -0,0 +1,82 @@ +{ + "index_patterns": ["acm-*"], + "template": { + "settings": { + "index": { + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "5000" + } + }, + "refresh_interval": "10s", + "auto_expand_replicas": "0-1" + } + }, + "mappings": { + "dynamic_templates": [ + { + "tags": { + "path_match": "tag.*", + "mapping": { + "ignore_above": 512, + "type": "keyword" + }, + "match_mapping_type": "string" + } + }, + { + "metrics_long": { + "mapping": { + "index": false, + "type": "float" + }, + "match_mapping_type": "long" + } + }, + { + "metrics_double": { + "mapping": { + "index": false, + "type": "float" + }, + "match_mapping_type": "double" + } + }, + { + "metrics_geo": { + "path_match": "*.geoip.location", + "mapping": { + "type": "geo_point" + } + } + }, + { + "message_fields": { + "path_match": "*message", + "mapping": { + "type": "text" + } + } + }, + { + "keyword_fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "measurement_name": { + "type": "keyword" + } + } + }, + "aliases": {} + } +} Index: /branches/amp_4_0/platform/config/telegraf.conf =================================================================== --- /branches/amp_4_0/platform/config/telegraf.conf (revision 0) +++ /branches/amp_4_0/platform/config/telegraf.conf (working copy) @@ -0,0 +1,30 @@ +[agent] + interval = "60s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "60s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "/var/log/telegraf/telegraf.log" + hostname = "" + omit_hostname = false + +[[outputs.influxdb_v2]] + urls = ["http://localhost:8086"] + token = "5vTtwB8fWzP9rkaLU8aaNnkM-RJzPl9yHdm1N28NdOtIyVvJTyb9JgyMjnAsHTLBlVtEQ7UMOencnJgHMvFabQ==" # verify the local device token if not working. + organization = "AN" + bucket = "AMP" + +############################################################################### +# OUTPUT TO ELASTICSEARCH +############################################################################### +; [[outputs.elasticsearch]] +; urls = ["http://localhost:9200"] +; index_name = "telegraf-snmp-%Y.%m.%d" +; type = "snmp" +; pipeline = "" +; timeout = "5s" Index: /branches/amp_4_0/platform/tools/README.md =================================================================== --- /branches/amp_4_0/platform/tools/README.md (revision 2554) +++ /branches/amp_4_0/platform/tools/README.md (working copy) @@ -5,7 +5,7 @@ * Python -> 3.13 * Django -> 5 * Java -> 21 -* PostgresSQL -> 13.20 +* PostgresSQL -> 17.4 * Elastic -> 8.18.0-1 * Logstash -> 1:8.18.0-1 * Kibana -> 8.18.0-1 @@ -16,13 +16,28 @@ * Telegraf -> 1.34.1-1 ### Installation order + 1. Python 2. PSQL 3. ELK - 1. Configure ELK + 1. Configure ELK 4. INFLUX 5. TELEGRAF ### ToDo (Best practices) * Use random passwords or better alternatives for the stored passwords from the script. + +### Debug + +#### InfluxDB + +By default, influxdb GUI is not exposed to the outside network, we can allow the 8086 to access the InfluxDB GUI using +the following firewall rules - its strictly for the debugging purpose. + +sudo firewall-cmd --permanent --add-port=8086/tcp + +sudo firewall-cmd --permanent --add-port=8086/udp + +sudo firewall-cmd --reload + Index: /branches/amp_4_0/platform/tools/configure_elk.sh =================================================================== --- /branches/amp_4_0/platform/tools/configure_elk.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/configure_elk.sh (working copy) @@ -63,6 +63,31 @@ fi } +create_data_view_acm() { + log_info "Creating Kibana data view for ACM..." + + local response + response=$(curl -s -X POST "$KIBANA_HOST/api/saved_objects/index-pattern" \ + -u "$ELASTIC_SUPER_USER:$ELASTIC_PASSWORD" \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: true' \ + -d "{ + \"attributes\": { + \"title\": \"acm-*\", + \"timeFieldName\": \"@timestamp\" + } + }") + + echo "$response" >> "$LOG_FILE" + + if echo "$response" | jq . >/dev/null 2>&1; then + log_info "✅ Successfully created acm data view." + else + log_error "❌ Failed to create acm data view. Raw output: $response" + exit 1 + fi +} + clean_start_filebeat() { log_info "Cleaning up and safely starting Filebeat..." @@ -116,6 +141,8 @@ clean_start_metricbeat create_data_view_logstash + + create_data_view_acm } configure_elk() { Index: /branches/amp_4_0/platform/tools/configure_psql.sh =================================================================== --- /branches/amp_4_0/platform/tools/configure_psql.sh (revision 0) +++ /branches/amp_4_0/platform/tools/configure_psql.sh (working copy) @@ -0,0 +1,12 @@ +#!/bin/bash + +# Source custom path changes +source /etc/profile.d/custom-path.sh + +AN_DB_NAME="cm" +AN_USER="amp_admin" +AN_USER_PASSWORD="Array@123$" + +# Import PSQL Tables for the AN provided database +PGPASSWORD="$AN_USER_PASSWORD" psql -U "$AN_USER" -d "$AN_DB_NAME" -f ./init_db.sql + Index: /branches/amp_4_0/platform/tools/install_elk.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_elk.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/install_elk.sh (working copy) @@ -290,8 +290,12 @@ "cluster": [ "monitor", "manage_ilm", "manage_ingest_pipelines", "manage_index_templates", "manage_pipeline" ], "indices": [ { - "names": [ "logstash-*" ], - "privileges": [ "write", "create_index", "view_index_metadata" ] + "names": [ "logstash-*", "acm-*" ], + "privileges": [ "write", "create_index", "auto_configure", "manage", "view_index_metadata" ] + }, + { + "names" : [ ".monitoring-logstash-*" ], + "privileges" : [ "create", "write", "auto_configure", "manage", "create_index" ] } ] }') @@ -498,6 +502,15 @@ log_info "Opening firewall for Kibana (port 5601), Logstash (port 9600), and Filebeat (port 5601)..." sudo firewall-cmd --add-port={5601,9200,5044}/tcp --permanent | tee -a "$LOG_FILE" sudo firewall-cmd --reload | tee -a "$LOG_FILE" + sudo firewall-cmd --permanent --zone=public \ + --add-masquerade \ + --add-port=514/udp \ + --add-port=5514/udp \ + --add-port=514/tcp \ + --add-port=5514/tcp \ + --add-forward-port=port=514:proto=udp:toport=5514 \ + --add-forward-port=port=514:proto=tcp:toport=5514 | tee -a "$LOG_FILE" + sudo firewall-cmd --reload | tee -a "$LOG_FILE" fi # --- Start Elasticsearch, Kibana, Metricbeat, Logstash, and Filebeat --- Index: /branches/amp_4_0/platform/tools/install_grafana.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_grafana.sh (revision 0) +++ /branches/amp_4_0/platform/tools/install_grafana.sh (working copy) @@ -0,0 +1,168 @@ +#!/bin/bash + +#--------------------------- +# Configuration Variables +#--------------------------- +GRAFANA_USER="admin" +GRAFANA_PASSWORD="GArr@y2050" +INFLUXDB_HOST="localhost" +INFLUXDB_PORT="8086" +INFLUXDB_ORG="AN" +INFLUXDB_BUCKET="AMP" +INFLUXDB_TOKEN_FILE="/opt/influxdb2_token.toml" +LOG_FILE="/var/log/configure_grafana.log" + +#--------------------------- +# Logging and Helpers +#--------------------------- +log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" +} + +info() { log "[INFO] $1"; } +error() { log "[ERROR] $1"; exit 1; } +command_exists() { command -v "$1" >/dev/null 2>&1; } + +#--------------------------- +# Read InfluxDB Token +#--------------------------- +read_influxdb_token() { + [[ -f "$INFLUXDB_TOKEN_FILE" ]] || error "InfluxDB token file not found: $INFLUXDB_TOKEN_FILE" + INFLUXDB_TOKEN=$(awk -F'=' '/token/ {gsub(/[ "\r]/, "", $2); print $2}' "$INFLUXDB_TOKEN_FILE") + [[ -n "$INFLUXDB_TOKEN" ]] || error "Failed to read InfluxDB token from file." + info "Read InfluxDB token." +} + +#--------------------------- +# Install Grafana +#--------------------------- +install_grafana() { + if ! command_exists grafana-server; then + info "Installing Grafana..." + cat </dev/null +[grafana] +name=Grafana OSS +baseurl=https://packages.grafana.com/oss/rpm +enabled=1 +gpgcheck=1 +gpgkey=https://packages.grafana.com/gpg.key +sslverify=1 +EOF + rpm --import https://packages.grafana.com/gpg.key || error "Failed to import GPG key." + dnf install -y grafana || error "Failed to install Grafana" + systemctl enable --now grafana-server || error "Failed to start Grafana" + info "Grafana installed and started." + else + info "Grafana is already installed." + fi +} + +#--------------------------- +# Reset Admin Password via CLI +#--------------------------- +reset_admin_password() { + if command_exists grafana-cli; then + info "Resetting Grafana admin password via grafana-cli..." + grafana-cli admin reset-admin-password "$GRAFANA_PASSWORD" >> "$LOG_FILE" 2>&1 || error "Failed to reset admin password via CLI." + info "Grafana admin password set to '$GRAFANA_PASSWORD'" + else + warn "grafana-cli not found, skipping admin password reset." + fi +} + +#--------------------------- +# Create API Token +#--------------------------- +create_api_token() { + info "Creating Grafana API token..." + RESPONSE=$(curl -s -X POST http://localhost:3000/api/auth/keys \ + -H "Content-Type: application/json" \ + -H "Authorization: Basic $(echo -n "$GRAFANA_USER:$GRAFANA_PASSWORD" | base64)" \ + -d '{"name": "InfluxDB API Token", "role": "Admin"}') + + TOKEN=$(echo "$RESPONSE" | jq -r '.token') + if [[ "$TOKEN" == "null" || -z "$TOKEN" ]]; then + error "Failed to create API token. Response: $RESPONSE" + fi + + echo "$TOKEN" | tee /opt/grafana_token >> "$LOG_FILE" + + # Secure the token file with restrictive permissions + chmod 600 /opt/grafana_token + chown root:root /opt/grafana_token + info "API token created and stored in /opt/grafana_token." +} + +#--------------------------- +# Add or Update InfluxDB Datasource +#--------------------------- +add_influxdb_datasource() { + command_exists curl || error "curl is required but not installed." + read_influxdb_token + + # Wait for Grafana + info "Waiting for Grafana API to be ready..." + until curl -s http://localhost:3000/api/health | grep -q "database"; do + sleep 2 + echo -n "." + done + echo + info "Grafana is ready." + + # Prepare JSON + DATA_SOURCE_JSON=$(cat <> "$LOG_FILE" + error "Failed to configure datasource (HTTP $CODE). Check $LOG_FILE for details." + fi + + # Adding firewall rule for Grafana + info "Configuring firewall to allow access to Grafana on port 3000..." + firewall-cmd --permanent --add-port=3000/tcp >> "$LOG_FILE" 2>&1 || error "Failed to add firewall rule." + firewall-cmd --reload >> "$LOG_FILE" 2>&1 || error "Failed to reload firewall." + info "Firewall configured." +} + +#--------------------------- +# Main +#--------------------------- +[[ $EUID -ne 0 ]] && error "This script must be run as root." +for cmd in curl systemctl rpm awk dnf tee firewall-cmd; do command_exists "$cmd" || error "$cmd is required."; done +mkdir -p "$(dirname "$LOG_FILE")" +install_grafana +reset_admin_password +add_influxdb_datasource +info "Script completed successfully." Index: /branches/amp_4_0/platform/tools/install_influx.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_influx.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/install_influx.sh (working copy) @@ -3,18 +3,15 @@ set -e # --- Configuration --- -LOG_FILE="/var/log/influxdb2_install.log" +LOG_FILE="/var/log/install_influxdb.log" INFLUXDB_USERNAME="array" -INFLUXDB_PASSWORD="Arr@y20250" +INFLUXDB_PASSWORD="Arr@y2050" INFLUXDB_ORG="AN" INFLUXDB_BUCKET="AMP" INFLUXDB_TOKEN_DESCRIPTION="Admin Token - Initial Setup" CONFIG_FILE="/opt/influxdb2_token.toml" INFLUXDB_HOST="http://localhost:8086" -# Redirect all script output to the log file -exec > "$LOG_FILE" 2>&1 - log_info() { echo "[INFO] $(date +'%Y-%m-%d %H:%M:%S') $1" | tee -a "$LOG_FILE" } Index: /branches/amp_4_0/platform/tools/install_psql.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_psql.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/install_psql.sh (working copy) @@ -1,64 +1,132 @@ #!/bin/bash +# Script to install and configure PostgreSQL on Rocky Linux 9.5 (Non-Interactive) +# =============================================================================== -set -e +set -Eeuo pipefail -LOG_FILE="/var/log/psql_install_$(date +%F-%T | tr ':' '-').log" +# --- Configuration --- +LOG_FILE="/var/log/install_psql.log" DB_USER="postgres" DB_PASSWORD="Arr@y2050" +PG_VERSION="17" +PGDATA="/var/lib/pgsql/${PG_VERSION}/data" +CONF_FILE="${PGDATA}/postgresql.conf" +HBA_FILE="${PGDATA}/pg_hba.conf" +NEW_DB_NAME="cm" +NEW_USER="amp_admin" +NEW_USER_PASSWORD="Array@123$" +PG_LOG_DIR="$PGDATA/log" + +# --- Helper Functions --- +log() { + local message="$1" + echo "$(date +'%Y-%m-%d %H:%M:%S') - $message" | tee -a "$LOG_FILE" +} + +execute() { + local command="$1" + log "Executing: $command" + bash -c "$command" + if [ $? -ne 0 ]; then + log "ERROR: Command '$command' failed with exit code $?" + exit 1 + fi +} -# Ensure we are root +# --- Main Script --- if [[ "$EUID" -ne 0 ]]; then - echo "This script must be run as root." | tee -a "$LOG_FILE" + log "ERROR: This script must be run as root." exit 1 fi -echo ">>> Logging PostgreSQL installation to $LOG_FILE" exec > >(tee -a "$LOG_FILE") 2>&1 -echo ">>> Starting PostgreSQL installation on Rocky Linux 9.5..." +log ">>> Starting PostgreSQL installation on Rocky Linux 9.5..." -echo ">>> Adding PostgreSQL repository..." -dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm +# Add PostgreSQL repository +execute "dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm" -echo ">>> Disabling built-in PostgreSQL module..." -dnf -y module disable postgresql +# Disable built-in PostgreSQL module +execute "dnf -y module disable postgresql" -echo ">>> Installing PostgreSQL server and client..." -dnf -y install postgresql-server postgresql - -echo ">>> Initializing PostgreSQL database..." -/usr/bin/postgresql-setup --initdb - -echo ">>> Enabling and starting PostgreSQL service..." -systemctl enable postgresql -systemctl start postgresql - -echo ">>> Configuring PostgreSQL for remote access..." -CONF_FILE="/var/lib/pgsql/data/postgresql.conf" -HBA_FILE="/var/lib/pgsql/data/pg_hba.conf" - -# Enable remote connections and use scram-sha-256 -sed -i "s/^#\?listen_addresses\s*=.*/listen_addresses = '*'/" "$CONF_FILE" -sed -i "s/^#\?password_encryption\s*=.*/password_encryption = 'scram-sha-256'/" "$CONF_FILE" - -# Add remote access rules to pg_hba.conf if they don't exist -if ! grep -q "host all all 0.0.0.0/0 scram-sha-256" "$HBA_FILE"; then - echo "host all all 0.0.0.0/0 scram-sha-256" >> "$HBA_FILE" -fi -if ! grep -q "host all all ::/0 scram-sha-256" "$HBA_FILE"; then - echo "host all all ::/0 scram-sha-256" >> "$HBA_FILE" +# Install PostgreSQL server and client +execute "dnf -y install postgresql${PG_VERSION}-server postgresql${PG_VERSION}" + +# Initialize PostgreSQL database +execute "/usr/pgsql-${PG_VERSION}/bin/postgresql-${PG_VERSION}-setup initdb" + +# Start PostgreSQL temporarily to set initial password +execute "systemctl start postgresql-${PG_VERSION}" + +# Set password for 'postgres' using UNIX socket (peer auth) +execute "su - postgres -c \"/usr/pgsql-${PG_VERSION}/bin/psql -c \\\"ALTER USER postgres WITH PASSWORD '$DB_PASSWORD';\\\"\"" + +# Stop PostgreSQL to apply configuration +execute "systemctl stop postgresql-${PG_VERSION}" + +# Configure PostgreSQL for remote access and scram auth +execute "sed -i 's/^#\\?listen_addresses\\s*=.*/listen_addresses = '\''*'\''/' \"$CONF_FILE\"" +execute "sed -i 's/^#\\?password_encryption\\s*=.*/password_encryption = '\''scram-sha-256'\''/' \"$CONF_FILE\"" + +# Update pg_hba.conf +execute "sed -i 's/^\(local\\s\\+all\\s\\+all\\s\\+\\).*$/\\1scram-sha-256/' \"$HBA_FILE\"" +execute "sed -i 's/^\(host\\s\\+all\\s\\+all\\s\\+127\\.0\\.0\\.1\\/32\\s\\+\\).*$/\\1scram-sha-256/' \"$HBA_FILE\"" +execute "sed -i 's/^\(host\\s\\+all\\s\\+all\\s\\+::1\\/128\\s\\+\\).*$/\\1scram-sha-256/' \"$HBA_FILE\"" +execute "sed -i 's/^\(local\\s\\+replication\\s\\+all\\s\\+\\).*$/\\1scram-sha-256/' \"$HBA_FILE\"" +grep -q "0.0.0.0/0" "$HBA_FILE" || echo "host all all 0.0.0.0/0 scram-sha-256" >> "$HBA_FILE" +grep -q "::/0" "$HBA_FILE" || echo "host all all ::/0 scram-sha-256" >> "$HBA_FILE" + +# Enable and start PostgreSQL service +execute "systemctl enable postgresql-${PG_VERSION}" +execute "systemctl start postgresql-${PG_VERSION}" + +# Wait for PostgreSQL to accept connections +log ">>> Waiting for PostgreSQL to accept connections..." +CONNECTION_CHECK_ATTEMPTS=20 +for ((i=1; i<=CONNECTION_CHECK_ATTEMPTS; i++)); do + sleep 2 + PGPASSWORD="$DB_PASSWORD" /usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -c "SELECT 1" > /dev/null 2>&1 + CONNECTION_CHECK_RESULT=$? + log ">>> Connection attempt $i: result=$CONNECTION_CHECK_RESULT" + if [ $CONNECTION_CHECK_RESULT -eq 0 ]; then + log ">>> PostgreSQL is accepting connections." + break + elif [ $i -eq $CONNECTION_CHECK_ATTEMPTS ]; then + log "ERROR: PostgreSQL failed to accept connections after $CONNECTION_CHECK_ATTEMPTS attempts." + PGPASSWORD="$DB_PASSWORD" /usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -c "SELECT 1" 2>>"$LOG_FILE" + log ">>> Performing diagnostic checks..." + execute "systemctl status postgresql-${PG_VERSION}" + execute "netstat -tulnp | grep 5432" + if [ -d "$PG_LOG_DIR" ]; then + execute "tail -n 20 \"$PG_LOG_DIR\"/postgresql-*.log" + else + log "WARNING: PostgreSQL log directory $PG_LOG_DIR does not exist." + fi + execute "firewall-cmd --state" + execute "firewall-cmd --list-ports" + exit 1 + else + log ">>> Connection attempt $i failed. Retrying..." + fi +done + +# Create database and user +export PGPASSWORD="$DB_PASSWORD" +execute "/usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -c \"CREATE DATABASE $NEW_DB_NAME;\"" +execute "/usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -c \"CREATE USER $NEW_USER WITH PASSWORD '$NEW_USER_PASSWORD';\"" +execute "/usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -c \"GRANT ALL PRIVILEGES ON DATABASE $NEW_DB_NAME TO $NEW_USER;\"" +execute "/usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -d $NEW_DB_NAME -c \"GRANT USAGE ON SCHEMA public TO $NEW_USER;\"" +execute "/usr/pgsql-${PG_VERSION}/bin/psql -U $DB_USER -h 127.0.0.1 -d $NEW_DB_NAME -c \"GRANT CREATE ON SCHEMA public TO $NEW_USER;\"" +unset PGPASSWORD + +# Open PostgreSQL port +if ! firewall-cmd --list-ports --permanent | grep -q "5432/tcp"; then + execute "firewall-cmd --add-port=5432/tcp --permanent" fi +execute "firewall-cmd --reload" -echo ">>> Restarting PostgreSQL to apply config..." -systemctl restart postgresql - -echo ">>> Setting password for 'postgres' user..." -sudo -u postgres psql -c "ALTER USER $DB_USER WITH PASSWORD '$DB_PASSWORD';" +log ">>> PostgreSQL installation complete!" +log ">>> Database '$NEW_DB_NAME' and user '$NEW_USER' created." +log ">>> Log saved to $LOG_FILE" -echo ">>> Opening PostgreSQL port 5432 in firewalld..." -firewall-cmd --add-port=5432/tcp --permanent -firewall-cmd --reload - -echo ">>> PostgreSQL installation complete with SCRAM-SHA-256 authentication!" -echo ">>> Password for user '$DB_USER' has been securely set." -echo ">>> Log saved to $LOG_FILE" +exit 0 Index: /branches/amp_4_0/platform/tools/install_python.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_python.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/install_python.sh (working copy) @@ -2,59 +2,103 @@ set -e -LOGFILE="/var/log/install_python_$(date +%F-%T | tr ':' '-').log" +LOGFILE="/var/log/install_python.log" exec > >(tee -a "$LOGFILE") 2>&1 +echo "------------------------------------------------------------------" +echo " Installing Python 3.13.0 on Rocky Linux 9.5 (Non-Interactive)" +echo " Log file: $LOGFILE" +echo "------------------------------------------------------------------" + # Ensure we are root if [[ "$EUID" -ne 0 ]]; then - echo "This script must be run as root." | tee -a "$LOGFILE" + echo "Error: This script must be run as root." exit 1 fi -echo "🔧 Updating system..." +echo "Step 1: Updating system packages..." dnf update -y -echo "📦 Installing required dependencies..." -dnf install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel wget make tar +echo "Step 2: Installing necessary build dependencies..." +dnf install -y \ + tar \ + wget \ + gcc \ + make \ + zlib-devel \ + bzip2-devel \ + openssl-devel \ + ncurses-devel \ + sqlite-devel \ + readline-devel \ + tk-devel \ + libffi-devel \ + xz-devel + +PYTHON_VERSION="3.13.0" +PYTHON_SRC_DIR="/tmp/Python-${PYTHON_VERSION}" +INSTALL_PREFIX="/usr/local" + +echo "Step 3: Downloading and extracting Python ${PYTHON_VERSION} source..." +cd /tmp +if [ ! -d "$PYTHON_SRC_DIR" ]; then + wget "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz" + tar -xf "Python-${PYTHON_VERSION}.tgz" +else + echo "Python source directory already exists: $PYTHON_SRC_DIR" +fi +cd "$PYTHON_SRC_DIR" + +echo "Step 4: Configuring the build..." +./configure --enable-optimizations --enable-shared --prefix="$INSTALL_PREFIX" -echo "☕ Installing Python 3.13..." -cd /usr/src -wget https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tgz -tar xzf Python-3.13.0.tgz -cd Python-3.13.0 +echo "Step 5: Building Python..." +make -j "$(nproc)" -./configure --enable-optimizations --enable-shared +echo "Step 6: Installing Python (using altinstall to avoid overwriting system python)..." make altinstall -echo "🔗 Creating symbolic links for 'python3.13' and 'python'..." -if [[ ! -f /usr/bin/python3.13 ]]; then - ln -s /usr/local/bin/python3.13 /usr/bin/python3.13 +echo "Step 7: Configuring shared library path for Python 3.13..." +echo "/usr/local/lib" > /etc/ld.so.conf.d/python3.13.conf +ldconfig + +echo "Step 8: Installing pip and creating symbolic links..." + +"$INSTALL_PREFIX/bin/python3.13" -m ensurepip --upgrade + +# Confirm pip3.13 exists before symlinking +if [ -f "$INSTALL_PREFIX/bin/pip3.13" ]; then + ln -sf "$INSTALL_PREFIX/bin/pip3.13" /usr/local/bin/pip3 +else + echo "Error: pip3.13 was not found after ensurepip. Aborting." + exit 1 fi -if [[ -f /usr/bin/python ]]; then - mv /usr/bin/python /usr/bin/python_backup_$(date +%F-%T | tr ':' '-') - echo "⚠️ Existing 'python' binary backed up." +# Symlink python3 if not already present +if [ ! -f "/usr/local/bin/python3" ]; then + ln -s "$INSTALL_PREFIX/bin/python3.13" /usr/local/bin/python3 fi -ln -sf /usr/local/bin/python3.13 /usr/bin/python - -echo 'export PATH=$PATH:/usr/local/bin' >> ~/.bashrc -source ~/.bashrc - -echo "🐍 Verifying Python installation..." -python --version - -echo "📦 Installing pip for Python 3.13..." -/usr/local/bin/python3.13 -m ensurepip --upgrade -/usr/local/bin/python3.13 -m pip install --upgrade pip - -echo "🌐 Installing Django 5..." -/usr/local/bin/python3.13 -m pip install "django==5.*" - -echo "🔎 Verifying Django installation..." -/usr/local/bin/python3.13 -m django --version +echo "Step 9: Adding /usr/local/bin to PATH (system-wide)..." +echo "export PATH=$INSTALL_PREFIX/bin:\$PATH" > /etc/profile.d/custom-path.sh +chmod +x /etc/profile.d/custom-path.sh +source /etc/profile.d/custom-path.sh + +echo "Step 10: Verifying Python installation..." +echo "Python path: $(which python3 || echo 'Not found')" +python3 --version || echo "python3 failed" + +echo "pip path: $(which pip3 || echo 'Not found')" +pip3 --version || echo "pip3 failed" + +echo "Step 11: Cleaning up source files..." +rm -rf "/tmp/Python-${PYTHON_VERSION}" "/tmp/Python-${PYTHON_VERSION}.tgz" + +echo "------------------------------------------------------------------" +echo " Python 3.13 installation completed successfully!" +echo " Python 3 executable: /usr/local/bin/python3" +echo " pip3 executable: /usr/local/bin/pip3" +echo " Log file: $LOGFILE" +echo "------------------------------------------------------------------" -echo "✅ Python 3.13, pip, and Django 5 installation completed!" -echo "➡️ Python version: $(python --version)" -echo "➡️ Django version: $(/usr/local/bin/python3.13 -m django --version)" -echo "📁 Installation log saved to: $LOGFILE" +exit 0 Index: /branches/amp_4_0/platform/tools/install_telegraf.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_telegraf.sh (revision 2554) +++ /branches/amp_4_0/platform/tools/install_telegraf.sh (working copy) @@ -3,7 +3,7 @@ set -e # --- Configuration --- -LOG_FILE="/var/log/telegraf_install.log" +LOG_FILE="/var/log/install_telegraf.log" CONFIG_FILE="/opt/influxdb2_token.toml" TELEGRAF_CONFIG_DIR="/etc/telegraf/telegraf.d" TELEGRAF_CONFIG="/etc/telegraf/telegraf.conf" Index: /branches/amp_4_0/platform/utils/expand-root.sh =================================================================== --- /branches/amp_4_0/platform/utils/expand-root.sh (revision 0) +++ /branches/amp_4_0/platform/utils/expand-root.sh (working copy) @@ -0,0 +1,50 @@ +#!/bin/bash + +LOGFILE="/var/log/expand-root.log" +exec > >(tee -a "$LOGFILE") 2>&1 +set -e + +# ============================================================================= +# WARNING: This script will modify disk partitions and extend your root volume. +# Make absolutely sure that: +# - You have unallocated space on /dev/sda (~150GB). +# - /dev/sda3 does NOT already exist. +# - You have a full backup or snapshot of the system. +# - The root filesystem uses XFS (default on RHEL/CentOS/Rocky Linux 7+). +# +# USE AT YOUR OWN RISK. This script is intended for experienced users/admins. +# ============================================================================= + +echo "===== $(date) Starting root volume expansion =====" + +read -p "Are you absolutely sure you want to proceed? (yes/NO): " confirm +if [[ "$confirm" != "yes" ]]; then + echo "[ABORTED] Operation cancelled by user." + exit 1 +fi + +echo "[INFO] Creating new partition on /dev/sda..." +parted /dev/sda --script mkpart primary 50GB 100% +parted /dev/sda --script set 3 lvm on + +echo "[INFO] Waiting for partition to be recognized..." +sleep 2 +partprobe +sleep 2 + +echo "[INFO] Creating physical volume on /dev/sda3..." +pvcreate /dev/sda3 + +echo "[INFO] Extending volume group 'rl'..." +vgextend rl /dev/sda3 + +echo "[INFO] Extending logical volume 'rl-root' with all free space..." +lvextend -l +100%FREE /dev/rl/root + +echo "[INFO] Growing XFS filesystem on /..." +xfs_growfs / + +echo "[SUCCESS] Root filesystem successfully extended!" +df -h / + +echo "===== $(date) Expansion complete. Log saved to $LOGFILE ====="