From 1389aae218719422b87052d73f421c5c5358b92c Mon Sep 17 00:00:00 2001 From: Roberto Rodriguez Date: Mon, 4 Mar 2019 10:03:39 -0500 Subject: [PATCH] [HOT FIX] 03042019 fix https://github.com/Cyb3rWard0g/HELK/issues/215 - Logstash plugins offline install (default) - Logstash mutate statements update - ES Memory Calculation fix - Compose files typo --- .../scripts/elasticsearch-entrypoint.sh | 2 +- docker/helk-kibana-analysis-basic.yml | 2 +- .../helk-kibana-notebook-analysis-basic.yml | 2 +- .../1523-winevent-process-name-filter.conf | 48 +++++++++----- .../1524-winevent-process-ids-filter.conf | 66 ++++++++++++------- .../scripts/logstash-entrypoint.sh | 9 ++- 6 files changed, 83 insertions(+), 46 deletions(-) diff --git a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh index be0c9aac..266a855a 100755 --- a/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh +++ b/docker/helk-elasticsearch/scripts/elasticsearch-entrypoint.sh @@ -18,7 +18,7 @@ if [[ -z "$ES_JAVA_OPTS" ]]; then ES_MEMORY="4g" else # Using GB instead of MB -- because plenty of RAM now - ES_MEMORY=$(( AVAILABLE_MEMORY / 1024 )) + ES_MEMORY=$(( AVAILABLE_MEMORY / 1024 / 2 )) if [ $ES_MEMORY -gt 31 ]; then ES_MEMORY="31g" else diff --git a/docker/helk-kibana-analysis-basic.yml b/docker/helk-kibana-analysis-basic.yml index 115592af..6e4a0c12 100644 --- a/docker/helk-kibana-analysis-basic.yml +++ b/docker/helk-kibana-analysis-basic.yml @@ -9,7 +9,7 @@ services: target: /usr/share/elasticsearch/config/elasticsearch.yml volumes: - esdata:/usr/share/elasticsearch/data - - ./helk-elasticsearch//scripts:/usr/share/elasticsearch/scripts + - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - cluster.name=helk-cluster diff --git a/docker/helk-kibana-notebook-analysis-basic.yml b/docker/helk-kibana-notebook-analysis-basic.yml index d58d70a6..d9e7b418 100644 --- a/docker/helk-kibana-notebook-analysis-basic.yml +++ b/docker/helk-kibana-notebook-analysis-basic.yml @@ -9,7 +9,7 @@ services: target: /usr/share/elasticsearch/config/elasticsearch.yml volumes: - esdata:/usr/share/elasticsearch/data - - ./helk-elasticsearch//scripts:/usr/share/elasticsearch/scripts + - ./helk-elasticsearch/scripts:/usr/share/elasticsearch/scripts entrypoint: /usr/share/elasticsearch/scripts/elasticsearch-entrypoint.sh environment: - cluster.name=helk-cluster diff --git a/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf b/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf index 84df954a..70f72ec3 100644 --- a/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf +++ b/docker/helk-logstash/pipeline/1523-winevent-process-name-filter.conf @@ -6,36 +6,52 @@ filter { if [event_id] { if [Image] { - mutate { add_field => { "z_logstash_pipeline" => "1523_1" } } - mutate { rename => { "Image" => "process_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_1" } + rename => { "Image" => "process_path" } + } } if [Application] { - mutate { add_field => { "z_logstash_pipeline" => "1523_2" } } - mutate { rename => { "Application" => "process_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_2" } + rename => { "Application" => "process_path" } + } } if [NewProcessName] { - mutate { add_field => { "z_logstash_pipeline" => "1523_3" } } - mutate { rename => { "NewProcessName" => "process_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_3" } + rename => { "NewProcessName" => "process_path" } + } } if [ProcessName] { - mutate { add_field => { "z_logstash_pipeline" => "1523_4" } } - mutate { rename => { "ProcessName" => "process_path" }} + mutate { + add_field => { "z_logstash_pipeline" => "1523_4" } + rename => { "ProcessName" => "process_path" } + } } if [ParentProcessName] { - mutate { add_field => { "z_logstash_pipeline" => "1523_5" } } - mutate { rename => { "ParentProcessName" => "process_parent_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_5" } + rename => { "ParentProcessName" => "process_parent_path" } + } } if [ParentImage] { - mutate { add_field => { "z_logstash_pipeline" => "1523_6" } } - mutate { rename => { "ParentImage" => "process_parent_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_6" } + rename => { "ParentImage" => "process_parent_path" } + } } if [TargetImage] { - mutate { add_field => { "z_logstash_pipeline" => "1523_7" } } - mutate { rename => { "TargetImage" => "process_target_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_7" } + rename => { "TargetImage" => "process_target_path" } + } } if [SourceImage] { - mutate { add_field => { "z_logstash_pipeline" => "1523_8" } } - mutate { rename => { "SourceImage" => "process_path" } } + mutate { + add_field => { "z_logstash_pipeline" => "1523_8" } + rename => { "SourceImage" => "process_path" } + } } if [ProdessName] { mutate { rename => { "ProdessName" => "process_path" } } diff --git a/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf b/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf index 1d800e71..a5560279 100644 --- a/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf +++ b/docker/helk-logstash/pipeline/1524-winevent-process-ids-filter.conf @@ -6,48 +6,70 @@ filter { if [event_id] { if [ProcessId] { - mutate { add_field => { "z_logstash_pipeline" => "1524_2" } } - mutate { rename => { "ProcessId" => "process_id" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_2" } + rename => { "ProcessId" => "process_id" } + } } if [NewProcessId] { - mutate { add_field => { "z_logstash_pipeline" => "1524_3" } } - mutate { rename => { "NewProcessId" => "process_id" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_3" } + rename => { "NewProcessId" => "process_id" } + } } if [ParentProcessId] { - mutate { add_field => { "z_logstash_pipeline" => "1524_5" } } - mutate { rename => { "ParentProcessId" => "process_parent_id" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_5" } + rename => { "ParentProcessId" => "process_parent_id" } + } } if [ProcessGuid] { - mutate { add_field => { "z_logstash_pipeline" => "1524_6" } } - mutate { rename => { "ProcessGuid" => "process_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_6" } + rename => { "ProcessGuid" => "process_guid" } + } } if [ParentProcessGuid] { - mutate { add_field => { "z_logstash_pipeline" => "1524_7" } } - mutate { rename => { "ParentProcessGuid" => "process_parent_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_7" } + rename => { "ParentProcessGuid" => "process_parent_guid" } + } } if [SourceProcessGuid] { - mutate { add_field => { "z_logstash_pipeline" => "1524_8" } } - mutate { rename => { "SourceProcessGuid" => "process_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_8" } + rename => { "SourceProcessGuid" => "process_guid" } + } } if [SourceProcessGUID] { - mutate { add_field => { "z_logstash_pipeline" => "1524_9" } } - mutate { rename => { "SourceProcessGUID" => "process_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_9" } + rename => { "SourceProcessGUID" => "process_guid" } + } } if [SourceProcessId] { - mutate { add_field => { "z_logstash_pipeline" => "1524_11" } } - mutate { rename => { "SourceProcessId" => "process_id" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_11" } + rename => { "SourceProcessId" => "process_id" } + } } if [TargetProcessGuid] { - mutate { add_field => { "z_logstash_pipeline" => "1524_12" } } - mutate { rename => { "TargetProcessGuid" => "process_target_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_12" } + rename => { "TargetProcessGuid" => "process_target_guid" } + } } if [TargetProcessGUID] { - mutate { add_field => { "z_logstash_pipeline" => "1524_13" } } - mutate { rename => { "TargetProcessGUID" => "process_target_guid" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_13" } + rename => { "TargetProcessGUID" => "process_target_guid" } + } } if [TargetProcessId] { - mutate { add_field => { "z_logstash_pipeline" => "1524_15" } } - mutate { rename => { "TargetProcessId" => "process_target_id" } } + mutate { + add_field => { "z_logstash_pipeline" => "1524_15" } + rename => { "TargetProcessId" => "process_target_id" } + } } } } \ No newline at end of file diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh index 36071460..2b9fd20b 100755 --- a/docker/helk-logstash/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/scripts/logstash-entrypoint.sh @@ -86,12 +86,11 @@ if ( logstash-plugin list 'prune' ) && ( logstash-plugin list 'i18n' ) && ( logs echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Plugins are already installed" else # logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-input-lumberjack && logstash-plugin install logstash-output-lumberjack && logstash-plugin install logstash-output-zabbix && logstash-plugin install logstash-filter-geoip && logstash-plugin install logstash-codec-cef && logstash-plugin install logstash-output-syslog && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-output-stdout && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-avro && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-snmptrap && logstash-plugin install logstash-input-snmp && logstash-plugin install logstash-input-jdbc && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone - echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install Logstash plugins over the Internet.." - if (logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-filter-geoip && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone); then - echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Logstash plugins installed via the Internet.." + if (logstash-plugin install file:///usr/share/logstash/logstash-offline-plugins-6.6.1.zip); then + echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Logstash plugins installed via offline package.." else - echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install logstash plugins from offline package.." - logstash-plugin install file:///usr/share/logstash/logstash-offline-plugins-6.6.1.zip + echo "[HELK-LOGSTASH-DOCKER-INSTALLATION-INFO] Trying to install logstash plugins over the Internet.." + logstash-plugin install logstash-filter-translate && logstash-plugin install logstash-filter-dns && logstash-plugin install logstash-filter-cidr && logstash-plugin install logstash-filter-geoip && logstash-plugin update logstash-filter-dissect && logstash-plugin install logstash-output-kafka && logstash-plugin install logstash-input-kafka && logstash-plugin install logstash-filter-alter && logstash-plugin install logstash-filter-fingerprint && logstash-plugin install logstash-filter-prune && logstash-plugin install logstash-codec-gzip_lines && logstash-plugin install logstash-codec-netflow && logstash-plugin install logstash-filter-i18n && logstash-plugin install logstash-filter-environment && logstash-plugin install logstash-filter-de_dot && logstash-plugin install logstash-input-wmi && logstash-plugin install logstash-filter-clone fi fi