Index: /branches/amp_4_0/platform/config/syslog/syslog.conf =================================================================== --- /branches/amp_4_0/platform/config/syslog/syslog.conf (revision 2771) +++ /branches/amp_4_0/platform/config/syslog/syslog.conf (working copy) @@ -2,23 +2,25 @@ udp { port => 5514 type => "syslog" - codec => plain { - charset => "UTF-8" - } + codec => plain { charset => "UTF-8" } } tcp { port => 5514 type => "syslog" - codec => plain { - charset => "UTF-8" - } + codec => plain { charset => "UTF-8" } } + beats { + port => 5044 + type => "beats" + } } filter { - # Stage 1: Initial Cleaning + # ============================================================ + # Stage 1: Cleaning & Strict Device Authorization + # ============================================================ + if [message] { - # Remove null characters (NUL) which are common in syslog (represented as \x00 or \u0000) ruby { code => ' if event.get("message").is_a?(String) @@ -31,63 +33,130 @@ } } - if [type] == "syslog" { - # Stage 2: Parsing Attempts (Cascading Grok) + # --- Step 1: Aggressive IP Extraction --- + ruby { + code => " + host_data = event.get('host') + device_ip = nil - # Attempt 1: RFC 5424 (Standard Syslog Header) - grok { - match => { - "message" => "^<%{NUMBER:syslog_pri}>%{NUMBER:syslog_protocol_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{DATA:syslog_msgid} (?-|\\[.*?\\])(?:\\s+)?%{GREEDYDATA:syslog_message}" - } - tag_on_failure => ["_grokparsefailure_rfc5424"] - add_tag => ["rfc5424_attempt"] + # Case A: Syslog (String) + if host_data.is_a?(String) + device_ip = host_data + + # Case B: Beats (Hash) + elsif host_data.is_a?(Hash) && host_data.key?('ip') + ip_val = host_data['ip'] + if ip_val.is_a?(Array) + device_ip = ip_val[0] + else + device_ip = ip_val + end + end + + # Trim whitespace if found + if device_ip + event.set('device_ip', device_ip.strip) + else + # Critical: If no IP found, tag for immediate drop + event.tag('_missing_source_ip') + end + " + } + + # --- Step 2: Database Lookup (No Cache) --- + if [device_ip] { + jdbc_streaming { + jdbc_driver_library => "/etc/logstash/jdbc_drivers/postgresql-42.7.5.jar" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_connection_string => "jdbc:postgresql://127.0.0.1:5432/cm" + jdbc_user => "amp_admin" + jdbc_password => "Array@123$" + # FIX: Cast to text to prevent Type Mismatch (INET vs String) + statement => "SELECT name, type, device_group FROM device WHERE ip_address::text = :device_ip" + parameters => { "device_ip" => "device_ip" } + target => "device_info" + # FIX: Disable caching to ensure immediate updates apply + use_cache => false + tag_on_failure => ["_device_lookup_failure"] } + } - # FIX 1: Clean the syslog_message field before further parsing - if [syslog_message] { - mutate { - strip => ["syslog_message"] - } + # --- Step 3: The Gatekeeper (Fail Closed) --- + ruby { + code => " + # 1. Check if IP was missing + if event.get('tags') && event.get('tags').include?('_missing_source_ip') + event.cancel # DROP EVENT IMMEDIATELY + + # 2. Check DB Lookup Result + elsif event.get('device_ip') + info = event.get('device_info') + + # Validation: Must be an array, and NOT empty + if info && info.is_a?(Array) && !info.empty? + # AUTHORIZED + device = info[0] + event.set('device_name', device['name']) + event.set('device_type', device['type']) + event.set('device_group', device['device_group']) + event.set('auth_status', 'authorized') # Debug flag + else + # UNAUTHORIZED -> DROP + event.cancel # DROP EVENT IMMEDIATELY + end + + # Cleanup + event.remove('device_info') + end + " + } + + # --- Step 4: Final Safety Net --- + # If the Ruby script failed to cancel for some reason, this catches it. + if ![device_name] { + drop {} + } + + # Clean up host field now that validation is done + if [host] { + ruby { + code => " + if event.get('host').is_a?(String) + event.set('[host][name]', event.get('host')) + end + " } + } - # FIX 2: Parse HTTP Security OR Access Message (Key-Value) - if !("_grokparsefailure_rfc5424" in [tags]) and ([syslog_message] =~ /^HTTP security:/ or [syslog_message] =~ /^HTTP access:/) { + # ============================================================ + # Stage 2: Parsing Strategy (Unchanged) + # ============================================================ - # 1. Strip the prefix and sanitize the message - mutate { - # Removes either prefix - gsub => [ - "syslog_message", "^HTTP security:", "", - "syslog_message", "^HTTP access:", "", - "syslog_message", "\\\"", "" - ] - strip => ["syslog_message"] - } + if [type] == "syslog" { + # Attempt 1: RFC 5424 + grok { + match => { "message" => "^<%{NUMBER:syslog_pri}>%{NUMBER:syslog_protocol_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{DATA:syslog_msgid} (?-|\\[.*?\\])(?:\\s+)?%{GREEDYDATA:syslog_message}" } + tag_on_failure => ["_grokparsefailure_rfc5424"] + add_tag => ["rfc5424_header_parsed"] + } - # 2. Use KV filter to extract all security details - kv { - source => "syslog_message" - field_split => " " - value_split => "=" - target => "security_details" - add_tag => ["security_kv_parsed"] - } + if [syslog_message] { mutate { strip => ["syslog_message"] } } - # 3. Move and enrich key fields from the 'security_details' sub-field + # Parser A: HTTP Security / Access + if [syslog_message] =~ /^(HTTP security:|HTTP access:)/ { + mutate { gsub => [ "syslog_message", "^HTTP security:", "", "syslog_message", "^HTTP access:", "", "syslog_message", "\\\"", "" ] strip => ["syslog_message"] } + kv { source => "syslog_message" field_split => " " value_split => "=" target => "security_details" add_tag => ["security_kv_parsed"] } if "security_kv_parsed" in [tags] { mutate { - # Rename core fields rename => { "[security_details][time]" => "event_time" } rename => { "[security_details][severity]" => "security_severity" } - rename => { "[security_details][sip]" => "source_ip" } + rename => { "[security_details][sip]" => "client_ip" } rename => { "[security_details][sport]" => "source_port" } rename => { "[security_details][dip]" => "destination_ip" } rename => { "[security_details][dport]" => "destination_port" } rename => { "[security_details][atkType]" => "attack_type" } rename => { "[security_details][description]" => "attack_description" } rename => { "[security_details][atkData]" => "attack_data_raw" } - - # **NEW FIELDS: HTTP/Session Details** rename => { "[security_details][reqMethod]" => "http_request_method" } rename => { "[security_details][statusCode]" => "http_status_code" } rename => { "[security_details][protocol]" => "network_protocol" } @@ -96,388 +165,85 @@ rename => { "[security_details][usr]" => "user_name" } rename => { "[security_details][srv]" => "server_name" } rename => { "[security_details][host]" => "requested_host" } - - # Convert numerical fields to integers - convert => { - "source_port" => "integer" - "destination_port" => "integer" - "http_status_code" => "integer" - } + convert => { "source_port" => "integer" "destination_port" => "integer" "http_status_code" => "integer" } } - - # 4. Further parse the complex 'atkData' field if [attack_data_raw] { - grok { - match => { - "attack_data_raw" => "Matched Data: %{DATA:matched_data} found within REQUEST_FILENAME: %{GREEDYDATA:request_filename}" - } - tag_on_failure => ["_grokparsefailure_attack_data"] - } + grok { match => { "attack_data_raw" => "Matched Data: %{DATA:matched_data} found within REQUEST_FILENAME: %{GREEDYDATA:request_filename}" } tag_on_failure => ["_grokparsefailure_attack_data"] } } - - # 5. Parse the embedded event time - date { - match => ["event_time", "YYYY-MM-dd HH:mm:ss"] - target => "event_time" - add_tag => ["_security_event_time_parsed"] - } - - # 6. Clean up temporary fields and add final tag - mutate { - remove_field => ["security_details"] - remove_field => ["attack_data_raw"] - remove_tag => ["_grokparsefailure_an_welf_log_rfc5424"] - add_tag => ["an_http_log_parsed"] - } + date { match => ["event_time", "YYYY-MM-dd HH:mm:ss"] target => "event_time" } + mutate { remove_field => ["security_details", "attack_data_raw"] add_tag => ["an_http_log_parsed"] } } } - # *** CORRECTED BLOCK START *** - # FIX 3: Parse APP-HTTP Access Log (from the vAPV device) - # - # THIS IS THE FIX: Changed 'elsif' to 'if' - # - if !("_grokparsefailure_rfc5424" in [tags]) and [syslog_message] =~ /^APP-HTTP/ { - grok { - match => { - # This pattern matches the specific format of your APP-HTTP log - "syslog_message" => "^APP-HTTP %{IP:client_ip} %{WORD:http_request_method} %{URI:http_request_url} %{NUMBER:http_status_code:int} %{NUMBER:bytes_sent:int} %{NUMBER:bytes_received:int} %{NUMBER:http_version} %{IPORHOST:destination_ip} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:http_referrer} %{WORD:cache_status} %{IPORHOST:next_hop_ip} %{WORD:virtual_server} %{NUMBER:duration_seconds:float} %{INT:unknown_field1} %{INT:unknown_field2} %{INT:unknown_field3}$" - } - add_tag => ["app_http_log_parsed"] - tag_on_failure => ["_grokparsefailure_app_http"] - } - - if "app_http_log_parsed" in [tags] { - # Clean up the double quotes from the QUOTEDSTRING pattern - mutate { - gsub => [ - "user_agent", "\"", "", - "http_referrer", "\"", "" - ] - } - # Remove the now-redundant syslog_message and unneeded fields - mutate { - remove_field => ["syslog_message", "unknown_field1", "unknown_field2", "unknown_field3"] - } - } + # Parser B: APP-HTTP + if [syslog_message] =~ /^APP-HTTP/ or [message] =~ /^APP-HTTP/ { + if [syslog_message] { mutate { add_field => { "[@metadata][parse_source]" => "%{syslog_message}" } } } else { mutate { add_field => { "[@metadata][parse_source]" => "%{message}" } } } + grok { match => { "[@metadata][parse_source]" => "^APP-HTTP %{IP:client_ip} %{WORD:http_request_method} %{URI:http_request_url} %{NUMBER:http_status_code:int} %{NUMBER:bytes_sent:int} %{NUMBER:bytes_received:int} %{NUMBER:http_version} %{IPORHOST:destination_ip} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:http_referrer} %{WORD:cache_status} %{IPORHOST:next_hop_ip} %{WORD:virtual_server} %{NUMBER:duration_seconds:float} %{INT:unknown_field1} %{INT:unknown_field2} %{INT:unknown_field3}$" } add_tag => ["app_http_log_parsed"] tag_on_failure => ["_grokparsefailure_app_http"] } + if "app_http_log_parsed" in [tags] { mutate { gsub => [ "user_agent", "\"", "", "http_referrer", "\"", "" ] remove_field => ["unknown_field1", "unknown_field2", "unknown_field3"] } } } - # *** CORRECTED BLOCK END *** - - # --- REMAINING FALLBACK GROK ATTEMPTS (UNCHANGED) --- - # Attempt 2: AN_WELF_LOG (WITH SYSLOG HEADER) + # Fallback Parsers if "_grokparsefailure_rfc5424" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{POSINT:syslog_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{INT:event_id} - AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:src_ip} dstname=%{IP:destination_name} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:destination_data_raw}\"" - } - tag_on_failure => ["_grokparsefailure_an_welf_log"] - add_tag => ["an_welf_log_attempt"] - } - if !("_grokparsefailure_an_welf_log" in [tags]) { - mutate { - add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } - remove_field => ["year", "month", "day", "hour", "minute", "second"] - } - mutate { - gsub => ["destination_data_raw", "\\s+", " "] - } - } + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{POSINT:syslog_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{INT:event_id} - AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:client_ip} dstname=%{IP:destination_ip} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:destination_data_raw}\"" } tag_on_failure => ["_grokparsefailure_an_welf_log"] } + if !("_grokparsefailure_an_welf_log" in [tags]) { mutate { add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } remove_field => ["year", "month", "day", "hour", "minute", "second"] gsub => ["destination_data_raw", "\\s+", " "] } } } - - # Attempt 3: AN_WELF_LOG (without SYSLOG HEADER) if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] { - grok { - match => { - "message" => "^AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:src_ip} dstname=%{IP:destination_name} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:message_detail_raw}\"" - } - tag_on_failure => ["_grokparsefailure_an_welf_log_no_header"] - add_tag => ["an_welf_log_no_header_attempt"] - } - if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { - mutate { - add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } - remove_field => ["year", "month", "day", "hour", "minute", "second"] - } - mutate { - gsub => ["message_detail_raw", "\\s+", " "] - } - mutate { - add_field => { "syslog_priority" => "%{priority}" } - add_field => { "syslog_timestamp" => "%{log_time}" } - } - } + grok { match => { "message" => "^AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:client_ip} dstname=%{IP:destination_ip} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:message_detail_raw}\"" } tag_on_failure => ["_grokparsefailure_an_welf_log_no_header"] } + if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { mutate { add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } add_field => { "syslog_priority" => "%{priority}" } add_field => { "syslog_timestamp" => "%{log_time}" } remove_field => ["year", "month", "day", "hour", "minute", "second"] gsub => ["message_detail_raw", "\\s+", " "] } } } - - # Attempt 4: Custom/Non-Standard if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{MONTH:syslog_month} +%{MONTHDAY:syslog_day} %{YEAR:syslog_year} %{TIME:syslog_time} %{IPORHOST:syslog_hostname} %{GREEDYDATA:syslog_content_kv}" - } - tag_on_failure => ["_grokparsefailure_custom_nonstandard"] - add_tag => ["custom_nonstandard_attempt"] - } + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{MONTH:syslog_month} +%{MONTHDAY:syslog_day} %{YEAR:syslog_year} %{TIME:syslog_time} %{IPORHOST:syslog_hostname} %{GREEDYDATA:syslog_content_kv}" } tag_on_failure => ["_grokparsefailure_custom_nonstandard"] } if !("_grokparsefailure_custom_nonstandard" in [tags]) { - date { - match => ["syslog_month syslog_day syslog_year syslog_time", "MMM ddYYYY HH:mm:ss"] - target => "syslog_timestamp" - add_tag => ["_dateparseok"] - tag_on_failure => ["_dateparsefailure"] - } - kv { - source => "syslog_content_kv" - field_split => " " - value_split => "=" - target => "syslog_kv_data" - } - mutate { - add_field => { "syslog_appname" => "%{[syslog_kv_data][id]}" } - add_field => { "syslog_msgid" => "%{[syslog_kv_data][type]}" } - add_field => { "syslog_message" => "%{[syslog_kv_data][msg]}" } - remove_field => ["syslog_month", "syslog_day", "syslog_year", "syslog_time", "syslog_content_kv"] - } - - if [syslog_kv_data][fw] { - mutate { add_field => { "virtual_ip" => "%{[syslog_kv_data][fw]}" } } - } - if [syslog_kv_data][src] { - mutate { add_field => { "client_ip" => "%{[syslog_kv_data][src]}" } } - } - if [syslog_kv_data][dst] { - mutate { add_field => { "destination_ip" => "%{[syslog_kv_data][dst]}" } } - } - if [syslog_kv_data][dport] { - mutate { - add_field => { "destination_port" => "%{[syslog_kv_data][dport]}" } - convert => { "destination_port" => "integer" } - } - } - if [syslog_kv_data][time] { - date { - match => ["syslog_kv_data.time", "YYYY-M-d HH:mm:ss"] - target => "log_message_timestamp" - tag_on_failure => ["_kvtimeparsefailure"] - } - } + date { match => ["syslog_month syslog_day syslog_year syslog_time", "MMM ddYYYY HH:mm:ss"] target => "syslog_timestamp" } + kv { source => "syslog_content_kv" field_split => " " value_split => "=" target => "syslog_kv_data" } + mutate { add_field => { "syslog_appname" => "%{[syslog_kv_data][id]}" } add_field => { "syslog_msgid" => "%{[syslog_kv_data][type]}" } add_field => { "syslog_message" => "%{[syslog_kv_data][msg]}" } rename => { "[syslog_kv_data][fw]" => "virtual_ip" } rename => { "[syslog_kv_data][src]" => "client_ip" } rename => { "[syslog_kv_data][dst]" => "destination_ip" } rename => { "[syslog_kv_data][dport]" => "destination_port" } convert => { "destination_port" => "integer" } remove_field => ["syslog_month", "syslog_day", "syslog_year", "syslog_time", "syslog_content_kv"] } } } - - # Attempt 5: Traditional BSD Syslog if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_hostname} %{DATA:syslog_appname}(?:\\[%{POSINT:syslog_procid}\\])?:(?: %{DATA:syslog_msgid})? %{GREEDYDATA:syslog_message}" - } - tag_on_failure => ["_grokparsefailure_bsd"] - add_tag => ["bsd_attempt"] - } + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_hostname} %{DATA:syslog_appname}(?:\\[%{POSINT:syslog_procid}\\])?:(?: %{DATA:syslog_msgid})? %{GREEDYDATA:syslog_message}" } tag_on_failure => ["_grokparsefailure_bsd"] add_tag => ["bsd_attempt"] } } + if "_grokparsefailure_bsd" in [tags] and "_grokparsefailure_rfc5424" in [tags] { syslog_pri { } mutate { add_tag => ["fallback_parsed"] } } + } + # ============================================================ + # Stage 3: Normalization + # ============================================================ + if [client_ip] { mutate { copy => { "client_ip" => "src_ip" } } } + if [user_agent] { useragent { source => "user_agent" target => "useragent" remove_field => ["user_agent"] } } + + ruby { + code => " + if event.get('syslog_priority') + priority = event.get('syslog_priority').to_i + level = priority % 8 + facility = priority / 8 + level_map = { 0=>'Emergency', 1=>'Alert', 2=>'Critical', 3=>'Error', 4=>'Warning', 5=>'Notice', 6=>'Informational', 7=>'Debug' } + facility_map = { 0=>'Kernel', 1=>'User', 2=>'Mail', 3=>'System Daemons', 4=>'Security/Auth', 5=>'Syslog', 16=>'Local0', 23=>'Local7' } + event.set('severity_numeric', level) + event.set('severity', level_map[level] || 'Unknown') + event.set('log_facility', facility_map[facility] || 'Unknown') + end + " + } + + if [security_severity] { ruby { code => "orig = event.get('security_severity'); event.set('severity', orig.capitalize) if orig" } } - # --- Stage 3: Common Post-Parsing Processing (PATCH APPLIED HERE) --- + # GeoIP + if [client_ip] { geoip { source => "client_ip" target => "client_geoip" } } + if [client_ip] and ![client_geoip][location] { + mutate { add_field => { "[client_geoip][city_name]" => "Unknown" "[client_geoip][country_name]" => "Unresolved" } } + ruby { code => "event.set('[client_geoip][location]', {'lat' => 12.925415, 'lon' => 77.631492})" } + mutate { add_tag => ["_geoip_client_unresolved"] } + } + if [destination_ip] { geoip { source => "destination_ip" target => "destination_geoip" } } + if [destination_ip] and ![destination_geoip][location] { + mutate { add_field => { "[destination_geoip][city_name]" => "Unknown" "[destination_geoip][country_name]" => "Unresolved" } } + ruby { code => "event.set('[destination_geoip][location]', {'lat' => 0.0, 'lon' => 0.0})" } + mutate { add_tag => ["_geoip_destination_unresolved"] } + } - if !("_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] and "_grokparsefailure_bsd" in [tags]) { - if [host][ip] { - mutate { - copy => { "[host][ip]" => "device_ip" } - add_tag => ["device_ip_fallback"] - } - } - - if [user_agent] { - useragent { - source => "user_agent" - target => "useragent" - remove_field => ["user_agent"] - } - } - - if [syslog_appname] == "-" { mutate { remove_field => ["syslog_appname"] } } - if [syslog_procid] == "-" { mutate { remove_field => ["syslog_procid"] } } - if [syslog_structured_data] == "-" { - mutate { remove_field => ["syslog_structured_data"] } - } else if [syslog_structured_data] =~ /^\\[.+\\]$/ { - - } - - mutate { - rename => { - "syslog_pri" => "syslog_priority" - "syslog_hostname" => "device_hostname" - "syslog_procid" => "process_id" - "syslog_msgid" => "message_id" - } - rename => { "syslog_protocol_version" => "syslog_version" } - } - - ruby { - code => " - if event.get('syslog_priority') - priority = event.get('syslog_priority').to_i - level = priority % 8 - facility = priority / 8 - - level_map = { - 0 => 'Emergency', 1 => 'Alert', 2 => 'Critical', 3 => 'Error', - 4 => 'Warning', 5 => 'Notice', 6 => 'Informational', 7 => 'Debug' - } - facility_map = { - 0 => 'Kernel', 1 => 'User', 2 => 'Mail', 3 => 'System Daemons', - 4 => 'Security/Authorization', 5 => 'Syslog', 6 => 'LPR Subsystem', - 7 => 'NNTP Subsystem', 8 => 'UUCP Subsystem', 9 => 'Clock Daemon', - 10 => 'Security/Authorization', 11 => 'FTP Daemon', 12 => 'NTP Subsystem', - 13 => 'Log Audit', 14 => 'Log Alert', 15 => 'Clock Daemon', - 16 => 'Local0', 17 => 'Local1', 18 => 'Local2', 19 => 'Local3', - 20 => 'Local4', 21 => 'Local5', 22 => 'Local6', 23 => 'Local7' - } - - event.set('severity_numeric', level) - event.set('severity', level_map[level] || 'Unknown') - event.set('log_facility_numeric', facility) - event.set('log_facility', facility_map[facility] || 'Unknown') - end - " - } - - # *** CORRECTED PATCH START *** - # This patch overwrites the calculated 'severity' (e.g., 'Error') - # with the more accurate value from the log message (e.g., 'critical'), - # and capitalizes it for consistency. - if [security_severity] { - ruby { - code => " - # Get the security_severity value (e.g., 'critical') - original_severity = event.get('security_severity') - - # Capitalize it (e.g., 'Critical') and set it as the main 'severity' - event.set('severity', original_severity.capitalize) - " - } - } - # *** CORRECTED PATCH END *** - - mutate { - copy => { "[host][ip]" => "device_ip" } - rename => { "src_ip" => "client_ip" } - rename => { "source_ip" => "client_ip" } - rename => { "destination_name" => "destination_ip" } - } - - # GEOIP LOOKUPS - if [client_ip] { - geoip { - source => "client_ip" - target => "client_geoip" - tag_on_failure => ["_geoip_client_failed"] - } - } - if [client_ip] and ![client_geoip][location] { - mutate { - add_field => { - "[client_geoip][city_name]" => "Unknown" - "[client_geoip][country_name]" => "Unresolved" - "[client_geoip][location][lat]" => "12.9254143647407" - "[client_geoip][location][lon]" => "77.63149239905859" - } - } - mutate { - add_tag => ["_geoip_client_unresolved"] - } - } - - if [destination_ip] { - geoip { - source => "destination_ip" - target => "destination_geoip" - tag_on_failure => ["_geoip_destination_failed"] - } - } - if [destination_ip] and ![destination_geoip][location] { - mutate { - add_field => { - "[destination_geoip][city_name]" => "Unknown" - "[destination_geoip][country_name]" => "Unresolved" - "[destination_geoip][location][lat]" => "0.0" - "[destination_geoip][location][lon]" => "0.0" - } - } - mutate { - add_tag => ["_geoip_destination_unresolved"] - } - } - - # JDBC LOOKUP (DEVICE INFO) - if [device_ip] { - jdbc_streaming { - jdbc_driver_library => "/etc/logstash/jdbc_drivers/postgresql-42.7.5.jar" - jdbc_driver_class => "org.postgresql.Driver" - jdbc_connection_string => "jdbc:postgresql://127.0.0.1:5432/cm" - jdbc_user => "amp_admin" - jdbc_password => "Array@123$" - statement => "SELECT name, type, device_group FROM device WHERE ip_address = :device_ip" - parameters => { "device_ip" => "device_ip" } - target => "device_info" - tag_on_failure => ["_device_lookup_failure"] - } - - if [device_info] { - ruby { - code => " - if event.get('device_info') && event.get('device_info')[0] - device = event.get('device_info')[0] - event.set('device_name', device['name']) - event.set('device_type', device['type']) - event.set('device_group', device['device_group']) - else - event.tag('_device_info_not_found') - end - event.remove('device_info') - " - } - } else { - mutate { - add_tag => ["_device_info_not_found"] - } - } - } - - # Tag Cleanup based on successful parse - if !("_grokparsefailure_rfc5424" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "rfc5424"] - } - } else if !("_grokparsefailure_an_welf_log" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "an_welf_log_headered"] - } - } else if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "an_welf_log_no_header"] - } - } else if !("_grokparsefailure_custom_nonstandard" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "custom_nonstandard_log"] - } - } else if !("_grokparsefailure_bsd" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard"] - add_tag => ["syslog_parsed", "bsd_syslog"] - } - } - - # Stage 4: Handle Unparsed Logs - if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] and "_grokparsefailure_bsd" in [tags] { - mutate { - add_tag => ["_parsefailure_syslog_unhandled"] - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - } - } - } + if "rfc5424_header_parsed" in [tags] or "an_http_log_parsed" in [tags] or "app_http_log_parsed" in [tags] { + mutate { remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] add_tag => ["log_fully_parsed"] } } } Index: /branches/amp_4_0/platform/tools/install_logstash_oss.sh =================================================================== --- /branches/amp_4_0/platform/tools/install_logstash_oss.sh (revision 2771) +++ /branches/amp_4_0/platform/tools/install_logstash_oss.sh (working copy) @@ -448,21 +448,24 @@ udp { port => 5514 type => "syslog" - codec => plain { - charset => "UTF-8" - } + codec => plain { charset => "UTF-8" } } tcp { port => 5514 type => "syslog" - codec => plain { - charset => "UTF-8" - } + codec => plain { charset => "UTF-8" } + } + beats { + port => 5044 + type => "beats" } } filter { - # Stage 1: Initial Cleaning + # ============================================================ + # Stage 1: Cleaning & Strict Device Authorization + # ============================================================ + if [message] { ruby { code => ' @@ -476,342 +479,217 @@ } } - if [type] == "syslog" { - # Stage 2: Parsing Attempts (Cascading Grok) + # --- Step 1: Aggressive IP Extraction --- + ruby { + code => " + host_data = event.get('host') + device_ip = nil + # Case A: Syslog (String) + if host_data.is_a?(String) + device_ip = host_data + + # Case B: Beats (Hash) + elsif host_data.is_a?(Hash) && host_data.key?('ip') + ip_val = host_data['ip'] + if ip_val.is_a?(Array) + device_ip = ip_val[0] + else + device_ip = ip_val + end + end + + # Trim whitespace if found + if device_ip + event.set('device_ip', device_ip.strip) + else + # Critical: If no IP found, tag for immediate drop + event.tag('_missing_source_ip') + end + " + } + + # --- Step 2: Database Lookup (No Cache) --- + if [device_ip] { + jdbc_streaming { + jdbc_driver_library => "/etc/logstash/jdbc_drivers/postgresql-42.7.5.jar" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_connection_string => "jdbc:postgresql://127.0.0.1:5432/cm" + jdbc_user => "amp_admin" + jdbc_password => "Array@123$" + # FIX: Cast to text to prevent Type Mismatch (INET vs String) + statement => "SELECT name, type, device_group FROM device WHERE ip_address::text = :device_ip" + parameters => { "device_ip" => "device_ip" } + target => "device_info" + # FIX: Disable caching to ensure immediate updates apply + use_cache => false + tag_on_failure => ["_device_lookup_failure"] + } + } + + # --- Step 3: The Gatekeeper (Fail Closed) --- + ruby { + code => " + # 1. Check if IP was missing + if event.get('tags') && event.get('tags').include?('_missing_source_ip') + event.cancel # DROP EVENT IMMEDIATELY + + # 2. Check DB Lookup Result + elsif event.get('device_ip') + info = event.get('device_info') + + # Validation: Must be an array, and NOT empty + if info && info.is_a?(Array) && !info.empty? + # AUTHORIZED + device = info[0] + event.set('device_name', device['name']) + event.set('device_type', device['type']) + event.set('device_group', device['device_group']) + event.set('auth_status', 'authorized') # Debug flag + else + # UNAUTHORIZED -> DROP + event.cancel # DROP EVENT IMMEDIATELY + end + + # Cleanup + event.remove('device_info') + end + " + } + + # --- Step 4: Final Safety Net --- + # If the Ruby script failed to cancel for some reason, this catches it. + if ![device_name] { + drop {} + } + + # Clean up host field now that validation is done + if [host] { + ruby { + code => " + if event.get('host').is_a?(String) + event.set('[host][name]', event.get('host')) + end + " + } + } + + # ============================================================ + # Stage 2: Parsing Strategy (Unchanged) + # ============================================================ + + if [type] == "syslog" { # Attempt 1: RFC 5424 grok { - match => { - "message" => "^<%{NUMBER:syslog_pri}>%{NUMBER:syslog_protocol_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{DATA:syslog_msgid} (?-|\\[.*?\\])(?:\\s+)?%{GREEDYDATA:syslog_message}" - } + match => { "message" => "^<%{NUMBER:syslog_pri}>%{NUMBER:syslog_protocol_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{DATA:syslog_msgid} (?-|\\[.*?\\])(?:\\s+)?%{GREEDYDATA:syslog_message}" } tag_on_failure => ["_grokparsefailure_rfc5424"] - add_tag => ["rfc5424_attempt"] + add_tag => ["rfc5424_header_parsed"] } - # Parse syslog_message as AN_WELF_LOG - if !("_grokparsefailure_rfc5424" in [tags]) and [syslog_message] { - mutate { - strip => ["syslog_message"] - } - grok { - match => { - "syslog_message" => "^AN_WELF_LOG:id=%{WORD:log_id} time=\"%{TIMESTAMP_ISO8601:log_time}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:src_ip} dstname=%{IP:destination_name} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:message_detail_raw}\"" - } - tag_on_failure => ["_grokparsefailure_an_welf_log_rfc5424"] - add_tag => ["an_welf_log_rfc5424_subparsed"] - } - if !("_grokparsefailure_an_welf_log_rfc5424" in [tags]) and [message_detail_raw] { + if [syslog_message] { mutate { strip => ["syslog_message"] } } + + # Parser A: HTTP Security / Access + if [syslog_message] =~ /^(HTTP security:|HTTP access:)/ { + mutate { gsub => [ "syslog_message", "^HTTP security:", "", "syslog_message", "^HTTP access:", "", "syslog_message", "\\\"", "" ] strip => ["syslog_message"] } + kv { source => "syslog_message" field_split => " " value_split => "=" target => "security_details" add_tag => ["security_kv_parsed"] } + if "security_kv_parsed" in [tags] { mutate { - gsub => ["message_detail_raw", "\\s+", " "] + rename => { "[security_details][time]" => "event_time" } + rename => { "[security_details][severity]" => "security_severity" } + rename => { "[security_details][sip]" => "client_ip" } + rename => { "[security_details][sport]" => "source_port" } + rename => { "[security_details][dip]" => "destination_ip" } + rename => { "[security_details][dport]" => "destination_port" } + rename => { "[security_details][atkType]" => "attack_type" } + rename => { "[security_details][description]" => "attack_description" } + rename => { "[security_details][atkData]" => "attack_data_raw" } + rename => { "[security_details][reqMethod]" => "http_request_method" } + rename => { "[security_details][statusCode]" => "http_status_code" } + rename => { "[security_details][protocol]" => "network_protocol" } + rename => { "[security_details][reqURL]" => "http_request_url" } + rename => { "[security_details][sessionID]" => "session_id" } + rename => { "[security_details][usr]" => "user_name" } + rename => { "[security_details][srv]" => "server_name" } + rename => { "[security_details][host]" => "requested_host" } + convert => { "source_port" => "integer" "destination_port" => "integer" "http_status_code" => "integer" } } - grok { - match => { - "message_detail_raw" => "^cache:%{WORD:cache_status} peer:%{WORD:peer_type}/%{IP:peer_ip}$" - } - tag_on_failure => ["_grokparsefailure_msg"] - add_tag => ["msg_subparsed"] + if [attack_data_raw] { + grok { match => { "attack_data_raw" => "Matched Data: %{DATA:matched_data} found within REQUEST_FILENAME: %{GREEDYDATA:request_filename}" } tag_on_failure => ["_grokparsefailure_attack_data"] } } + date { match => ["event_time", "YYYY-MM-dd HH:mm:ss"] target => "event_time" } + mutate { remove_field => ["security_details", "attack_data_raw"] add_tag => ["an_http_log_parsed"] } } } - # Attempt 2: AN_WELF_LOG (WITH SYSLOG HEADER) - if "_grokparsefailure_rfc5424" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{POSINT:syslog_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{INT:event_id} - AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:src_ip} dstname=%{IP:destination_name} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:destination_data_raw}\"" - } - tag_on_failure => ["_grokparsefailure_an_welf_log"] - add_tag => ["an_welf_log_attempt"] - } - if !("_grokparsefailure_an_welf_log" in [tags]) { - mutate { - add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } - remove_field => ["year", "month", "day", "hour", "minute", "second"] - } - mutate { - gsub => ["destination_data_raw", "\\s+", " "] - } - } + # Parser B: APP-HTTP + if [syslog_message] =~ /^APP-HTTP/ or [message] =~ /^APP-HTTP/ { + if [syslog_message] { mutate { add_field => { "[@metadata][parse_source]" => "%{syslog_message}" } } } else { mutate { add_field => { "[@metadata][parse_source]" => "%{message}" } } } + grok { match => { "[@metadata][parse_source]" => "^APP-HTTP %{IP:client_ip} %{WORD:http_request_method} %{URI:http_request_url} %{NUMBER:http_status_code:int} %{NUMBER:bytes_sent:int} %{NUMBER:bytes_received:int} %{NUMBER:http_version} %{IPORHOST:destination_ip} %{QUOTEDSTRING:user_agent} %{QUOTEDSTRING:http_referrer} %{WORD:cache_status} %{IPORHOST:next_hop_ip} %{WORD:virtual_server} %{NUMBER:duration_seconds:float} %{INT:unknown_field1} %{INT:unknown_field2} %{INT:unknown_field3}$" } add_tag => ["app_http_log_parsed"] tag_on_failure => ["_grokparsefailure_app_http"] } + if "app_http_log_parsed" in [tags] { mutate { gsub => [ "user_agent", "\"", "", "http_referrer", "\"", "" ] remove_field => ["unknown_field1", "unknown_field2", "unknown_field3"] } } } - # Attempt 3: AN_WELF_LOG (without SYSLOG HEADER) + # Fallback Parsers + if "_grokparsefailure_rfc5424" in [tags] { + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{POSINT:syslog_version} %{TIMESTAMP_ISO8601:syslog_timestamp} %{HOSTNAME:syslog_hostname} %{DATA:syslog_appname} %{DATA:syslog_procid} %{INT:event_id} - AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:client_ip} dstname=%{IP:destination_ip} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:destination_data_raw}\"" } tag_on_failure => ["_grokparsefailure_an_welf_log"] } + if !("_grokparsefailure_an_welf_log" in [tags]) { mutate { add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } remove_field => ["year", "month", "day", "hour", "minute", "second"] gsub => ["destination_data_raw", "\\s+", " "] } } + } if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] { - grok { - match => { - "message" => "^AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:src_ip} dstname=%{IP:destination_name} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:message_detail_raw}\"" - } - tag_on_failure => ["_grokparsefailure_an_welf_log_no_header"] - add_tag => ["an_welf_log_no_header_attempt"] - } - if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { - mutate { - add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } - remove_field => ["year", "month", "day", "hour", "minute", "second"] - } - mutate { - gsub => ["message_detail_raw", "\\s+", " "] - } - mutate { - add_field => { "syslog_priority" => "%{priority}" } - add_field => { "syslog_timestamp" => "%{log_time}" } - } - } + grok { match => { "message" => "^AN_WELF_LOG:id=%{WORD:log_id} time=\"%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}\" fw=%{IP:virtual_ip} pri=%{POSINT:priority} proto=%{WORD:protocol} src=%{IP:client_ip} dstname=%{IP:destination_ip} arg=%{URIPATH:arg} op=%{WORD:operation} agent=\"%{DATA:user_agent}\" result=%{INT:http_result_code} sent=%{INT:bytes_sent} duration=%{NUMBER:duration_seconds} msg=\"%{GREEDYDATA:message_detail_raw}\"" } tag_on_failure => ["_grokparsefailure_an_welf_log_no_header"] } + if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { mutate { add_field => { "log_time" => "%{year}-%{month}-%{day} %{hour}:%{minute}:%{second}" } add_field => { "syslog_priority" => "%{priority}" } add_field => { "syslog_timestamp" => "%{log_time}" } remove_field => ["year", "month", "day", "hour", "minute", "second"] gsub => ["message_detail_raw", "\\s+", " "] } } } - - # Attempt 4: Custom/Non-Standard if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{MONTH:syslog_month} +%{MONTHDAY:syslog_day} %{YEAR:syslog_year} %{TIME:syslog_time} %{IPORHOST:syslog_hostname} %{GREEDYDATA:syslog_content_kv}" - } - tag_on_failure => ["_grokparsefailure_custom_nonstandard"] - add_tag => ["custom_nonstandard_attempt"] - } + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{MONTH:syslog_month} +%{MONTHDAY:syslog_day} %{YEAR:syslog_year} %{TIME:syslog_time} %{IPORHOST:syslog_hostname} %{GREEDYDATA:syslog_content_kv}" } tag_on_failure => ["_grokparsefailure_custom_nonstandard"] } if !("_grokparsefailure_custom_nonstandard" in [tags]) { - date { - match => ["syslog_month syslog_day syslog_year syslog_time", "MMM ddYYYY HH:mm:ss"] - target => "syslog_timestamp" - add_tag => ["_dateparseok"] - tag_on_failure => ["_dateparsefailure"] - } - kv { - source => "syslog_content_kv" - field_split => " " - value_split => "=" - target => "syslog_kv_data" - } - mutate { - add_field => { "syslog_appname" => "%{[syslog_kv_data][id]}" } - add_field => { "syslog_msgid" => "%{[syslog_kv_data][type]}" } - add_field => { "syslog_message" => "%{[syslog_kv_data][msg]}" } - remove_field => ["syslog_month", "syslog_day", "syslog_year", "syslog_time", "syslog_content_kv"] - } - - # Map fields from syslog_kv_data to common field names - if [syslog_kv_data][fw] { - mutate { add_field => { "virtual_ip" => "%{[syslog_kv_data][fw]}" } } - } - if [syslog_kv_data][src] { - mutate { add_field => { "client_ip" => "%{[syslog_kv_data][src]}" } } - } - if [syslog_kv_data][dst] { - mutate { add_field => { "destination_ip" => "%{[syslog_kv_data][dst]}" } } - } - if [syslog_kv_data][dport] { - mutate { - add_field => { "destination_port" => "%{[syslog_kv_data][dport]}" } - convert => { "destination_port" => "integer" } - } - } - # Optional: Parse log_time and timezone from KV data if needed for @timestamp - if [syslog_kv_data][time] { - date { - match => ["syslog_kv_data.time", "YYYY-M-d HH:mm:ss"] - target => "log_message_timestamp" - tag_on_failure => ["_kvtimeparsefailure"] - } - } + date { match => ["syslog_month syslog_day syslog_year syslog_time", "MMM ddYYYY HH:mm:ss"] target => "syslog_timestamp" } + kv { source => "syslog_content_kv" field_split => " " value_split => "=" target => "syslog_kv_data" } + mutate { add_field => { "syslog_appname" => "%{[syslog_kv_data][id]}" } add_field => { "syslog_msgid" => "%{[syslog_kv_data][type]}" } add_field => { "syslog_message" => "%{[syslog_kv_data][msg]}" } rename => { "[syslog_kv_data][fw]" => "virtual_ip" } rename => { "[syslog_kv_data][src]" => "client_ip" } rename => { "[syslog_kv_data][dst]" => "destination_ip" } rename => { "[syslog_kv_data][dport]" => "destination_port" } convert => { "destination_port" => "integer" } remove_field => ["syslog_month", "syslog_day", "syslog_year", "syslog_time", "syslog_content_kv"] } } } - - # Attempt 5: Traditional BSD Syslog if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] { - grok { - match => { - "message" => "^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_hostname} %{DATA:syslog_appname}(?:\\[%{POSINT:syslog_procid}\\])?:(?: %{DATA:syslog_msgid})? %{GREEDYDATA:syslog_message}" - } - tag_on_failure => ["_grokparsefailure_bsd"] - add_tag => ["bsd_attempt"] - } + grok { match => { "message" => "^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_hostname} %{DATA:syslog_appname}(?:\\[%{POSINT:syslog_procid}\\])?:(?: %{DATA:syslog_msgid})? %{GREEDYDATA:syslog_message}" } tag_on_failure => ["_grokparsefailure_bsd"] add_tag => ["bsd_attempt"] } } + if "_grokparsefailure_bsd" in [tags] and "_grokparsefailure_rfc5424" in [tags] { syslog_pri { } mutate { add_tag => ["fallback_parsed"] } } + } - # Stage 3: Common Post-Parsing Processing - if !("_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] and "_grokparsefailure_bsd" in [tags]) { - # USERAGENT FILTER - if [user_agent] { - useragent { - source => "user_agent" - target => "useragent" - remove_field => ["user_agent"] - } - } + # ============================================================ + # Stage 3: Normalization + # ============================================================ + if [client_ip] { mutate { copy => { "client_ip" => "src_ip" } } } + if [user_agent] { useragent { source => "user_agent" target => "useragent" remove_field => ["user_agent"] } } + + ruby { + code => " + if event.get('syslog_priority') + priority = event.get('syslog_priority').to_i + level = priority % 8 + facility = priority / 8 + level_map = { 0=>'Emergency', 1=>'Alert', 2=>'Critical', 3=>'Error', 4=>'Warning', 5=>'Notice', 6=>'Informational', 7=>'Debug' } + facility_map = { 0=>'Kernel', 1=>'User', 2=>'Mail', 3=>'System Daemons', 4=>'Security/Auth', 5=>'Syslog', 16=>'Local0', 23=>'Local7' } + event.set('severity_numeric', level) + event.set('severity', level_map[level] || 'Unknown') + event.set('log_facility', facility_map[facility] || 'Unknown') + end + " + } + + if [security_severity] { ruby { code => "orig = event.get('security_severity'); event.set('severity', orig.capitalize) if orig" } } - if [syslog_appname] == "-" { mutate { remove_field => ["syslog_appname"] } } - if [syslog_procid] == "-" { mutate { remove_field => ["syslog_procid"] } } - if [syslog_structured_data] == "-" { - mutate { remove_field => ["syslog_structured_data"] } - } else if [syslog_structured_data] =~ /^\\[.+\\]$/ { - # Handle structured data parsing if needed - } + # GeoIP + if [client_ip] { geoip { source => "client_ip" target => "client_geoip" } } + if [client_ip] and ![client_geoip][location] { + mutate { add_field => { "[client_geoip][city_name]" => "Unknown" "[client_geoip][country_name]" => "Unresolved" } } + ruby { code => "event.set('[client_geoip][location]', {'lat' => 12.925415, 'lon' => 77.631492})" } + mutate { add_tag => ["_geoip_client_unresolved"] } + } + if [destination_ip] { geoip { source => "destination_ip" target => "destination_geoip" } } + if [destination_ip] and ![destination_geoip][location] { + mutate { add_field => { "[destination_geoip][city_name]" => "Unknown" "[destination_geoip][country_name]" => "Unresolved" } } + ruby { code => "event.set('[destination_geoip][location]', {'lat' => 0.0, 'lon' => 0.0})" } + mutate { add_tag => ["_geoip_destination_unresolved"] } + } - mutate { - rename => { - "syslog_pri" => "syslog_priority" - "syslog_hostname" => "device_hostname" - "syslog_appname" => "application_name" - "syslog_procid" => "process_id" - "syslog_msgid" => "message_id" - } - rename => { "syslog_protocol_version" => "syslog_version" } - } - - ruby { - code => " - if event.get('syslog_priority') - level = event.get('syslog_priority').to_i % 8 - level_map = { - 0 => 'Emergency', 1 => 'Alert', 2 => 'Critical', 3 => 'Error', - 4 => 'Warning', 5 => 'Notice', 6 => 'Informational', 7 => 'Debug' - } - event.set('severity_numeric', level) - event.set('severity', level_map[level] || 'Unknown') - end - " - } - - ruby { - code => " - if event.get('syslog_priority') - facility = event.get('syslog_priority').to_i / 8 - facility_map = { - 0 => 'Kernel', 1 => 'User', 2 => 'Mail', 3 => 'System Daemons', - 4 => 'Security/Authorization', 5 => 'Syslog', 6 => 'LPR Subsystem', - 7 => 'NNTP Subsystem', 8 => 'UUCP Subsystem', 9 => 'Clock Daemon', - 10 => 'Security/Authorization', 11 => 'FTP Daemon', 12 => 'NTP Subsystem', - 13 => 'Log Audit', 14 => 'Log Alert', 15 => 'Clock Daemon', - 16 => 'Local0', 17 => 'Local1', 18 => 'Local2', 19 => 'Local3', - 20 => 'Local4', 21 => 'Local5', 22 => 'Local6', 23 => 'Local7' - } - event.set('log_facility_numeric', facility) - event.set('log_facility', facility_map[facility] || 'Unknown') - end - " - } - - # IP RENAMING LOGIC - mutate { - copy => { "[host][ip]" => "device_ip" } - } - - if [src_ip] { - mutate { - rename => { "src_ip" => "client_ip" } - } - } - - if [client_ip] { - geoip { - source => "client_ip" - target => "client_geoip" - tag_on_failure => ["_geoip_client_failed"] - } - } - if [client_ip] and ![client_geoip][location] { - mutate { - add_field => { - "[client_geoip][city_name]" => "Unknown" - "[client_geoip][country_name]" => "Unresolved" - "[client_geoip][location][lat]" => "12.9254143647407" - "[client_geoip][location][lon]" => "77.63149239905859" - } - } - mutate { - add_tag => ["_geoip_client_unresolved"] - } - } - - if [destination_ip] { - geoip { - source => "destination_ip" - target => "destination_geoip" - tag_on_failure => ["_geoip_destination_failed"] - } - } - if [destination_ip] and ![destination_geoip][location] { - mutate { - add_field => { - "[destination_geoip][city_name]" => "Unknown" - "[destination_geoip][country_name]" => "Unresolved" - "[destination_geoip][location][lat]" => "0.0" - "[destination_geoip][location][lon]" => "0.0" - } - } - mutate { - add_tag => ["_geoip_destination_unresolved"] - } - } - - if [device_ip] { - jdbc_streaming { - jdbc_driver_library => "/etc/logstash/jdbc_drivers/postgresql-42.7.5.jar" - jdbc_driver_class => "org.postgresql.Driver" - jdbc_connection_string => "jdbc:postgresql://127.0.0.1:5432/cm" - jdbc_user => "amp_admin" - jdbc_password => "Array@123$" - statement => "SELECT name, type, device_group FROM device WHERE ip_address = :device_ip" - parameters => { "device_ip" => "device_ip" } - target => "device_info" - tag_on_failure => ["_device_lookup_failure"] - } - - if [device_info] { - ruby { - code => " - if event.get('device_info') && event.get('device_info')[0] - device = event.get('device_info')[0] - event.set('device_name', device['name']) - event.set('device_type', device['type']) - event.set('device_group', device['device_group']) - else - event.tag('_device_info_not_found') - end - event.remove('device_info') - " - } - } else { - mutate { - add_tag => ["_device_info_not_found"] - } - } - } - - if !("_grokparsefailure_rfc5424" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "rfc5424"] - } - } else if !("_grokparsefailure_an_welf_log" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc542 - -4", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "an_welf_log_headered"] - } - } else if !("_grokparsefailure_an_welf_log_no_header" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "an_welf_log_no_header"] - } - } else if !("_grokparsefailure_custom_nonstandard" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_bsd"] - add_tag => ["syslog_parsed", "custom_nonstandard_log"] - } - } else if !("_grokparsefailure_bsd" in [tags]) { - mutate { - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard"] - add_tag => ["syslog_parsed", "bsd_syslog"] - } - } - - # Stage 4: Handle Unparsed Logs - if "_grokparsefailure_rfc5424" in [tags] and "_grokparsefailure_an_welf_log" in [tags] and "_grokparsefailure_an_welf_log_no_header" in [tags] and "_grokparsefailure_custom_nonstandard" in [tags] and "_grokparsefailure_bsd" in [tags] { - mutate { - add_tag => ["_parsefailure_syslog_unhandled"] - remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] - } - } - } + if "rfc5424_header_parsed" in [tags] or "an_http_log_parsed" in [tags] or "app_http_log_parsed" in [tags] { + mutate { remove_tag => ["_grokparsefailure_rfc5424", "_grokparsefailure_an_welf_log", "_grokparsefailure_an_welf_log_no_header", "_grokparsefailure_custom_nonstandard", "_grokparsefailure_bsd"] add_tag => ["log_fully_parsed"] } } } Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/libbasic_operation.py (working copy) @@ -421,3 +421,19 @@ file_content = file_content.replace("%%" + each["key"] + "%%", each["value"]) return file_content + + +def ping_device(ip): + """ + Ping a device to check if it is reachable. + Returns True if reachable, False otherwise. + """ + try: + # Use -c 1 for sending 1 packet, -W 2 for 2 seconds timeout + output = subprocess.check_output(['ping', '-c', '1', '-W', '2', ip], stderr=subprocess.STDOUT) + return True + except subprocess.CalledProcessError: + return False + except Exception as e: + logger.error('Ping device:%s failed, %s' % (ip, str(e))) + return False Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/lib/task_scheduler.py (working copy) @@ -18,7 +18,7 @@ from cm.lib.communication import send_https_rest_request, modify_url, call_restapi from cm.lib.libbasic_operation import oper_log, send_cli_to_device from cm.lib.libbasic_operation import send_command_to_device, get_rest_info_from_device, get_ip_address, \ - send_xmlrpc_to_device + send_xmlrpc_to_device, ping_device from cm.lib.libconfig import Backup_AG, Backup_APV, Recover_AG, Recover_APV, Backup_Custom, Apply_AG from cm.lib.libmonitor import Decision, SSLCertDecision from cm.lib.parse_configfile import parse_vsite_from_configfile, parse_vsite_config_from_configfile, parse_diff_config, \ @@ -31,7 +31,7 @@ from hive.notification import send_notification from hive.utils import get_device_type from djproject.an_settings import (extension_path, ADC_TYPE_LIST, DEVICE_STD_LIST, VPN_TYPE_LIST, WAF_TYPE_LIST, - DEFAULT_CONFIG_FILE_PATH, CM_AWK_CONF) + DEFAULT_CONFIG_FILE_PATH, CM_AWK_CONF, OTHER_CATEGORY_LIST) from hive.model.legacycli import cli_parse, RegexParser, MATCHONE logger = logging.getLogger('hive.debug') @@ -2124,7 +2124,14 @@ # begin_time = time.time() failed_time = 3 while failed_time: - if self.data['protocol'] == 'xmlrpc': + if self.data.get('type') in OTHER_CATEGORY_LIST: + if ping_device(self.data['ip']): + set_device_status(self.data['id'], 'connected') + break + else: + failed_time -= 1 + logger.error('Ping device<%s>(%s) failed.' % (self.data['name'], self.data['ip'])) + elif self.data['protocol'] == 'xmlrpc': response_data = send_cli_to_device(self.data, "show version", mode='enable') if response_data['message'] == 'success': set_device_status(self.data['id'], 'connected') Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/cm/models/device_mgmt/device/__init__.py (working copy) @@ -9,21 +9,21 @@ import elasticsearch from apscheduler.schedulers.background import BackgroundScheduler -from django.db.models.query import QuerySet +# from django.db.models.query import QuerySet from django.utils.encoding import force_bytes from django.utils.translation import gettext_lazy as _ from cm.lib.communication import send_https_rest_request, modify_url, L from cm.lib.elk import get_elk from cm.lib.libbasic_operation import send_command_to_device, send_cli_to_device, insert_default_tmpkey, \ - delete_all_tmpkey + delete_all_tmpkey, ping_device from cm.lib.postgres_db import DB from cm.lib.task_scheduler import add_job_into_database, remove_job_from_database, config_backup from cm.models.audit.user.user_body import body_login_detail, body_device_vsite from cm.models.system.license import check_license from djproject.an_settings import * from hive.model.loading import get_model -from hive.model.query import mark_expire_all +from hive.model.query import mark_expire_all, QuerySet from hive.utils import get_device_type, standard_model_type from hive.model.action import Action from hive.model.base import ANModel @@ -37,6 +37,8 @@ from djproject.an_settings import (ADC_TYPE_LIST, DEVICE_STD_LIST, VPN_TYPE_LIST, WAF_TYPE_LIST, DEFAULT_CONFIG_FILE_PATH, DEFAULT_VSITE_PATH) from hive.exceptions import ModelQueryException + + __ = _ @@ -148,7 +150,7 @@ 'community': CharField(verbose_name='Community', optional=True) }), 'license_key': CharField(verbose_name='License Key', editable=False, configurable=False, optional=True, - length='1..128'), + length='1..256'), 'webui_port': PortField(verbose_name='Webui Port', default=8888, editable=True), 'enable_password': PasswordField(verbose_name='Enable Password', length='1..32', optional=True), 'backup_enable': BooleanField(verbose_name='Backup Configuraion', display=('enable', 'disable'), default=False, @@ -538,13 +540,23 @@ if self.check_ip_name_from_device(data['ip'], data.get('name')): raise ModelQueryException(CLICmdError(__('Sorry IP or Name has exists.'))) - L.add_lock(data['ip']) + + # Extract IP string from dictionary if needed + ip_addr = list(data['ip'].values())[0] if isinstance(data['ip'], dict) else data['ip'] + L.add_lock(ip_addr) try: - if data.get('protocol') == 'xmlrpc': + if data.get('type') in OTHER_CATEGORY_LIST: + if ping_device(ip_addr): + rest_response_data = {'message': 'success', 'status': 200} + else: + raise ModelQueryException(CLICmdError(__('Register Failed. Device is unreachable.'))) + elif data.get('protocol') == 'xmlrpc': + # Ensure ip_address is set for send_cli_to_device + data['ip_address'] = ip_addr rest_response_data = send_cli_to_device(data, "show version", mode='enable') else: new_url = modify_url('/rest/device_type/system/SystemInfo/version', data.get('type')) - rest_response_data = send_https_rest_request('GET', new_url, '', data['ip'], + rest_response_data = send_https_rest_request('GET', new_url, '', ip_addr, data.get('restapi_port'), data.get('restapi_username'), data.get('restapi_password')) @@ -552,6 +564,8 @@ if 'timed out' in str(e): raise ModelQueryException(CLICmdError( __('Register timed out! Please check the ip address and make sure the device have already turned on the RESTful API service and the RESTful API port is correct!'))) + if data.get('type') in OTHER_CATEGORY_LIST: + raise ModelQueryException(CLICmdError(__('Register Failed. Device is unreachable.'))) raise ModelQueryException(CLICmdError(__('Register Failed.'))) else: if data.get('protocol') == 'xmlrpc': @@ -562,6 +576,9 @@ else: raise ModelQueryException( CLICmdError(__('Register Failed! Please make sure the XML RPC configuration is correct!'))) + elif data.get('type') in OTHER_CATEGORY_LIST: + # Skip body parsing for other devices + pass else: if rest_response_data['status'] == 200: rest_body = json.loads(rest_response_data['body']) @@ -571,10 +588,18 @@ __('Register Failed! Please make sure the RESTful API username/password is correct!'))) else: raise ModelQueryException(CLICmdError(__('Refresh Version Failed!'))) - rtn = cli_parse(data['version'], [ - RegexParser('License Key : (.+?)\n', MATCHONE), - RegexParser('Model : (.+?)\n', MATCHONE), - ]) + + if data.get('type') in OTHER_CATEGORY_LIST: + data['version'] = '' + data['license_key'] = 'None' + data['status'] = 'active' + # Skip parsing version for other devices + rtn = None + else: + rtn = cli_parse(data['version'], [ + RegexParser('License Key : (.+?)\n', MATCHONE), + RegexParser('Model : (.+?)\n', MATCHONE), + ]) if rtn and rtn[0]: data['license_key'] = rtn[0][0] data['status'] = 'active' @@ -616,12 +641,13 @@ return None model = get_device_model(model_str) - if not model: - raise ModelQueryException(CLICmdError( - __('Register Failed! Invalid model! Only %s are supported.') % ', '.join(DEVICE_STD_LIST))) - if model != data.get('type'): - raise ModelQueryException(CLICmdError( - __('Register Failed! Current device model is %s but you choose %s.') % (model, data['type']))) + if data.get('type') not in OTHER_CATEGORY_LIST: + if not model: + raise ModelQueryException(CLICmdError( + __('Register Failed! Invalid model! Only %s are supported.') % ', '.join(DEVICE_STD_LIST))) + if model != data.get('type'): + raise ModelQueryException(CLICmdError( + __('Register Failed! Current device model is %s but you choose %s.') % (model, data['type']))) db = DB.get_connected_db() save_sql = ( @@ -634,10 +660,9 @@ db.execute_sql(save_sql) create_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') insert_config_sql = ( - "INSERT INTO file_list(name, create_time, modify_time, type, file_type_id, comment, device_type, " - "device_id)" - "values('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % ( - data['name'], create_time, create_time, 'device', 1, '', data['type'], data['id']) + "INSERT INTO file_list(name, create_time, modify_time, type, comment, device_type) " + "values('%s', '%s', '%s', '%s', '%s', '%s')" % ( + data['name'], create_time, create_time, 'device', '', data['type']) ) db.execute_sql(insert_config_sql) db.close() @@ -1001,7 +1026,7 @@ return def _delete_instance(self, instance): - data = instance.get_field_dict() + # data = instance.get_field_dict() # check the task db = DB.get_connected_db() select_sql = 'SELECT name, state, device_list from task;' @@ -1009,23 +1034,24 @@ key = ['name', 'state', 'device_list'] result = [dict(zip(key, each)) for each in select_data] for item in result: - if data['name'] in item['device_list'] and item['state'] != 'failed' and item['state'] != 'done': + if instance.name in item['device_list'] and item['state'] != 'failed' and item['state'] != 'done': raise ModelQueryException(CLICmdError(__('Sorry, please delete the task:%s first.' % item['name']))) # get backup file list - select_sql = "SELECT name, device_type FROM file_list WHERE type='backup' and name='%s'" % data[ - 'name'] + select_sql = "SELECT name, device_type FROM file_list WHERE type='backup' and name='%s'" % instance.name select_data = db.fetchall(select_sql) key = ['name', 'device_type'] backup_result = [dict(zip(key, each)) for each in select_data] - delete_sql = "DELETE FROM device WHERE id = " + "'%s'" % data['id'] + delete_sql = "DELETE FROM device WHERE id = " + "'%s'" % instance.id db.execute_sql(delete_sql) - L.remove_lock(list(data['ip'].values())[0]) + # Extract IP string from dictionary + ip_addr = list(instance.ip.values())[0] if isinstance(instance.ip, dict) else instance.ip + L.remove_lock(ip_addr) # delete from file_list, delete device item and backup item - delete_sql = "DELETE FROM file_list WHERE name= '%s'" % data['name'] + delete_sql = "DELETE FROM file_list WHERE name= '%s'" % instance.name db.execute_sql(delete_sql) # remove the backup config file @@ -1037,28 +1063,31 @@ # need to remove both .all_cfg_tar file and dir path os.popen("rm -rf %s" % backup_path[0:-12]).read() - # delete vsite from vsite_list - select_sql = "SELECT vs_name FROM vsite_list WHERE device_name='%s'" % data['name'] - select_data = db.fetchall(select_sql) - key = ['vs_name'] - result = [dict(zip(key, each)) for each in select_data] - - delete_sql = "DELETE FROM vsite_list WHERE device_name='%s'" % data['name'] - db.execute_sql(delete_sql) - db.close() + if instance.type not in OTHER_CATEGORY_LIST: + # delete vsite from vsite_list + select_sql = "SELECT vs_name FROM vsite_list WHERE device_name='%s'" % instance.name + select_data = db.fetchall(select_sql) + key = ['vs_name'] + result = [dict(zip(key, each)) for each in select_data] - # remove the vsite config file - for item in result: - vsite_path = DEFAULT_VSITE_PATH + data['name'] + '-' + item['vs_name'] - if os.path.exists(vsite_path): - os.remove(vsite_path) - # remove the device config file - file_path = DEFAULT_CONFIG_FILE_PATH + 'device/' + data['name'] - if os.path.exists(file_path): - os.remove(file_path) + delete_sql = "DELETE FROM vsite_list WHERE device_name='%s'" % instance.name + db.execute_sql(delete_sql) + db.close() - # remove all device template key-value by device ip - delete_all_tmpkey(list(data['ip'].values())[0]) + # remove the vsite config file + for item in result: + vsite_path = DEFAULT_VSITE_PATH + instance.name + '-' + item['vs_name'] + if os.path.exists(vsite_path): + os.remove(vsite_path) + # remove the device config file + file_path = DEFAULT_CONFIG_FILE_PATH + 'device/' + instance.name + if os.path.exists(file_path): + os.remove(file_path) + + # remove all device template key-value by device ip + delete_all_tmpkey(ip_addr) + else: + db.close() """ for item in result: Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/djproject/an_settings.py (working copy) @@ -120,6 +120,8 @@ AN_WAF_LIST = CMDATA['PRODUCT_CATEGORY']['WAF'] OTHER_WAF_LIST = [] # just for oem, should be empty +OTHER_CATEGORY_LIST = ['Router', 'Switch', 'Windows', 'Linux', 'Others'] + VPN_TYPE_LIST = AN_VPN_LIST + OTHER_VPN_LIST ADC_TYPE_LIST = AN_ADC_LIST + OTHER_ADC_LIST WAF_TYPE_LIST = AN_WAF_LIST + OTHER_WAF_LIST Index: /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/add-device.html =================================================================== --- /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/add-device.html (revision 2771) +++ /branches/amp_4_0/src/webui/webui/htdocs/new/src/gui/src/app/components/sub-components/devices/add-device.html (working copy) @@ -1,29 +1,19 @@

Add Device

-