From b9daa4c59291b479efa8d63e765f560653dcb3bb Mon Sep 17 00:00:00 2001 From: Roberto Rodriguez Date: Mon, 6 Aug 2018 22:32:51 -0700 Subject: [PATCH] v0.1.2-alpha08062018 Updated Logstash output templates to replace _doc mappings to doc. --- docker/docker-compose-elk-trial.yml | 2 +- .../output_templates/10-logs-all-default.json | 2 +- .../50-logs-winevent-all.json | 2 +- .../60-powershell-direct-template.json | 2 +- .../60-winevent-application-template.json | 2 +- .../60-winevent-powershell-template.json | 2 +- .../60-winevent-security-template.json | 2 +- .../60-winevent-sysmon-template.json | 2 +- .../output_templates/82-logs-not-ip.json | 2 +- .../output_templates/91-logs-ip-dst-nat.json | 2 +- .../output_templates/91-logs-ip-dst.json | 2 +- .../output_templates/91-logs-ip-src-nat.json | 2 +- .../output_templates/91-logs-ip-src.json | 2 +- .../93-logs-ipv6-dst-nat.json | 2 +- .../output_templates/93-logs-ipv6-dst.json | 2 +- .../93-logs-ipv6-src-nat.json | 2 +- .../output_templates/93-logs-ipv6-src.json | 2 +- .../output_templates/99-logs-any-fields.json | 2 +- .../scripts/logstash-entrypoint.sh | 2 +- .../trial/scripts/logstash-entrypoint.sh | 21 +++++++++++++------ docker/helk_install.sh | 2 +- 21 files changed, 35 insertions(+), 26 deletions(-) diff --git a/docker/docker-compose-elk-trial.yml b/docker/docker-compose-elk-trial.yml index 78300e1c..10b8eba2 100644 --- a/docker/docker-compose-elk-trial.yml +++ b/docker/docker-compose-elk-trial.yml @@ -34,7 +34,7 @@ services: entrypoint: /usr/share/logstash/scripts/logstash-entrypoint.sh restart: always depends_on: - - helk-elasticsearch + - helk-kibana networks: helk: aliases: diff --git a/docker/helk-logstash/output_templates/10-logs-all-default.json b/docker/helk-logstash/output_templates/10-logs-all-default.json index bb532509..5d67d030 100644 --- a/docker/helk-logstash/output_templates/10-logs-all-default.json +++ b/docker/helk-logstash/output_templates/10-logs-all-default.json @@ -13,7 +13,7 @@ "refresh_interval": "30s" }, "mappings": { - "_doc": { + "doc": { "dynamic": "true", "dynamic_templates": [ { diff --git a/docker/helk-logstash/output_templates/50-logs-winevent-all.json b/docker/helk-logstash/output_templates/50-logs-winevent-all.json index 5fd5825d..6f41d307 100644 --- a/docker/helk-logstash/output_templates/50-logs-winevent-all.json +++ b/docker/helk-logstash/output_templates/50-logs-winevent-all.json @@ -55,7 +55,7 @@ "refresh_interval": "30s" }, "mappings": { - "_doc":{ + "doc":{ "properties":{ "process_id":{"type":"integer"}, "event_id":{"type":"integer"}, diff --git a/docker/helk-logstash/output_templates/60-powershell-direct-template.json b/docker/helk-logstash/output_templates/60-powershell-direct-template.json index 50880362..01c319e8 100644 --- a/docker/helk-logstash/output_templates/60-powershell-direct-template.json +++ b/docker/helk-logstash/output_templates/60-powershell-direct-template.json @@ -3,7 +3,7 @@ "index_patterns" : "logs-endpoint-powershell-direct-*", "version": 2018080101, "mappings":{ - "_doc":{ + "doc":{ "properties":{ "process_id":{"type":"integer"} } diff --git a/docker/helk-logstash/output_templates/60-winevent-application-template.json b/docker/helk-logstash/output_templates/60-winevent-application-template.json index 3e05170f..9c0f46df 100644 --- a/docker/helk-logstash/output_templates/60-winevent-application-template.json +++ b/docker/helk-logstash/output_templates/60-winevent-application-template.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-endpoint-winevent-application-*" ], "version": 2018080101, "mappings":{ - "_doc":{ + "doc":{ "properties":{ "spp_restart_scheduled":{"type":"date"} } diff --git a/docker/helk-logstash/output_templates/60-winevent-powershell-template.json b/docker/helk-logstash/output_templates/60-winevent-powershell-template.json index 9eff63a5..b4cfe4f8 100644 --- a/docker/helk-logstash/output_templates/60-winevent-powershell-template.json +++ b/docker/helk-logstash/output_templates/60-winevent-powershell-template.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-endpoint-winevent-powershell-*" ], "version": 2018080201, "mappings":{ - "_doc": { + "doc": { "properties": { "powershell": { "dynamic": "false", diff --git a/docker/helk-logstash/output_templates/60-winevent-security-template.json b/docker/helk-logstash/output_templates/60-winevent-security-template.json index 1659e852..5a083756 100644 --- a/docker/helk-logstash/output_templates/60-winevent-security-template.json +++ b/docker/helk-logstash/output_templates/60-winevent-security-template.json @@ -3,7 +3,7 @@ "index_patterns": "logs-endpoint-winevent-security-*", "version": 2018080101, "mappings":{ - "_doc":{ + "doc":{ "properties":{ "@date_new_time":{"type":"date"}, "@date_previous_time":{"type":"date"}, diff --git a/docker/helk-logstash/output_templates/60-winevent-sysmon-template.json b/docker/helk-logstash/output_templates/60-winevent-sysmon-template.json index a4616216..fc9b83ef 100644 --- a/docker/helk-logstash/output_templates/60-winevent-sysmon-template.json +++ b/docker/helk-logstash/output_templates/60-winevent-sysmon-template.json @@ -6,7 +6,7 @@ "index.refresh_interval": "5s" }, "mappings":{ - "_doc":{ + "doc":{ "properties":{ "@date_creation":{"type":"date"}, "@date_creation_previous":{"type":"date"}, diff --git a/docker/helk-logstash/output_templates/82-logs-not-ip.json b/docker/helk-logstash/output_templates/82-logs-not-ip.json index 792353dc..e4434b83 100644 --- a/docker/helk-logstash/output_templates/82-logs-not-ip.json +++ b/docker/helk-logstash/output_templates/82-logs-not-ip.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "not_ip_dst": { "type": "keyword" diff --git a/docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json b/docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json index 278f5ed9..133e985a 100644 --- a/docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json +++ b/docker/helk-logstash/output_templates/91-logs-ip-dst-nat.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018052301, "mappings": { - "_doc": { + "doc": { "properties": { "dst_nat_ip_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/91-logs-ip-dst.json b/docker/helk-logstash/output_templates/91-logs-ip-dst.json index 8137e615..99846023 100644 --- a/docker/helk-logstash/output_templates/91-logs-ip-dst.json +++ b/docker/helk-logstash/output_templates/91-logs-ip-dst.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018052301, "mappings": { - "_doc": { + "doc": { "properties": { "dst_ip_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/91-logs-ip-src-nat.json b/docker/helk-logstash/output_templates/91-logs-ip-src-nat.json index 1f6ac226..32e721d5 100644 --- a/docker/helk-logstash/output_templates/91-logs-ip-src-nat.json +++ b/docker/helk-logstash/output_templates/91-logs-ip-src-nat.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018052301, "mappings": { - "_doc": { + "doc": { "properties": { "src_nat_ip_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/91-logs-ip-src.json b/docker/helk-logstash/output_templates/91-logs-ip-src.json index a1634b70..37514cd4 100644 --- a/docker/helk-logstash/output_templates/91-logs-ip-src.json +++ b/docker/helk-logstash/output_templates/91-logs-ip-src.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018052301, "mappings": { - "_doc": { + "doc": { "properties": { "src_ip_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json b/docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json index b71caada..f2976e42 100644 --- a/docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json +++ b/docker/helk-logstash/output_templates/93-logs-ipv6-dst-nat.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "dst_nat_ipv6_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/93-logs-ipv6-dst.json b/docker/helk-logstash/output_templates/93-logs-ipv6-dst.json index be703c5c..e301eb28 100644 --- a/docker/helk-logstash/output_templates/93-logs-ipv6-dst.json +++ b/docker/helk-logstash/output_templates/93-logs-ipv6-dst.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "dst_ipv6_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json b/docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json index 4d4eda0c..abe55b6f 100644 --- a/docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json +++ b/docker/helk-logstash/output_templates/93-logs-ipv6-src-nat.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "ipv6_src_nat_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/93-logs-ipv6-src.json b/docker/helk-logstash/output_templates/93-logs-ipv6-src.json index 930a1f10..7c93e410 100644 --- a/docker/helk-logstash/output_templates/93-logs-ipv6-src.json +++ b/docker/helk-logstash/output_templates/93-logs-ipv6-src.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "src_ipv6_addr": { "type": "ip", diff --git a/docker/helk-logstash/output_templates/99-logs-any-fields.json b/docker/helk-logstash/output_templates/99-logs-any-fields.json index 5f72a8c4..9162abe5 100644 --- a/docker/helk-logstash/output_templates/99-logs-any-fields.json +++ b/docker/helk-logstash/output_templates/99-logs-any-fields.json @@ -3,7 +3,7 @@ "index_patterns": [ "logs-*" ], "version": 2018080101, "mappings": { - "_doc": { + "doc": { "properties": { "any_ip_addr": { "type": "ip" diff --git a/docker/helk-logstash/scripts/logstash-entrypoint.sh b/docker/helk-logstash/scripts/logstash-entrypoint.sh index 785552a7..a1b6d882 100755 --- a/docker/helk-logstash/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/scripts/logstash-entrypoint.sh @@ -32,7 +32,7 @@ do done # ********** Install Plugin ***************** -echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstsh plugins.." +echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.." logstash-plugin install logstash-filter-prune # ********** Starting Logstash ***************** diff --git a/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh index ee7c81f8..a639c28c 100755 --- a/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh +++ b/docker/helk-logstash/trial/scripts/logstash-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# HELK script: logstash-setup.sh +# HELK script: logstash-entrypoint.sh # HELK script description: Pushes output templates to ES and starts Logstash # HELK build Stage: Alpha # Author: Roberto Rodriguez (@Cyb3rWard0g) @@ -16,20 +16,29 @@ else export LS_JAVA_OPTS="-Xms${LS_MEMORY}g -Xmx${LS_MEMORY}g" fi -ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200 # *********** Looking for ES *************** +ELASTICSEARCH_ACCESS=http://elastic:"elasticpassword"@helk-elasticsearch:9200 echo "[HELK-DOCKER-INSTALLATION-INFO] Waiting for elasticsearch URI to be accessible.." until curl -s $ELASTICSEARCH_ACCESS -o /dev/null; do sleep 1 done +echo "[HELK-DOCKER-INSTALLATION-INFO] Uploading templates to elasticsearch.." DIR=/usr/share/logstash/output_templates for file in ${DIR}/*.json - do - template_name=$(echo $file | sed -r ' s/^.*\/[0-9]+\-//'); - curl -H 'Content-Type: application/json' -XPUT "$ELASTICSEARCH_ACCESS/_template/$template_name" -d@${file}; +do + template_name=$(echo $file | sed -r ' s/^.*\/[0-9]+\-//'); + echo "[HELK-DOCKER-INSTALLATION-INFO] Uploading $template_name template to elasticsearch.."; + curl -s -H 'Content-Type: application/json' -XPUT $ELASTICSEARCH_ACCESS/_template/$template_name -d@${file}; + sleep 1 done -exec "$@" +# ********** Install Plugin ***************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Installing Logstash plugins.." +logstash-plugin install logstash-filter-prune + +# ********** Starting Logstash ***************** +echo "[HELK-DOCKER-INSTALLATION-INFO] Running docker-entrypoint script.." +/usr/local/bin/docker-entrypoint diff --git a/docker/helk_install.sh b/docker/helk_install.sh index f2d73f4f..dbe95a55 100755 --- a/docker/helk_install.sh +++ b/docker/helk_install.sh @@ -262,7 +262,7 @@ show_banner(){ echo "** HELK - THE HUNTING ELK **" echo "** **" echo "** Author: Roberto Rodriguez (@Cyb3rWard0g) **" - echo "** HELK build version: v0.1.2-alpha08032018 **" + echo "** HELK build version: v0.1.2-alpha08062018 **" echo "** HELK ELK version: 6.3.2 **" echo "** License: GPL-3.0 **" echo "**********************************************"