Filebeat+Logstash: Lowered flush size by significant amount to get logs into ES much quicker

filebeat/configmap.yaml

@@ -5,23 +5,25 @@ metadata:
   namespace: app-notifications
 data:
   filebeat.yml: |
-    filebeat.inputs:
-    - type: filestream
-      id: kubernetes-containers
-      paths:
-        - /var/log/containers/*_app-notifications_*.log
-      prospector:
-        scanner:
-          fingerprint.enabled: true
-          fingerprint.offset: 0
-          fingerprint.length: 512
-      parsers:
-        - container: ~
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: false
+          templates:
+            - condition:
+                equals:
+                  kubernetes.namespace: "app-notifications"
+              config:
+                - type: container
+                  paths:
+                    - /var/log/containers/*_app-notifications_*.log
+                  stream: stdout
+                  fingerprint.enabled: false
 
     processors:
       - add_kubernetes_metadata:
           in_cluster: true
-          host: ${NODE_NAME}
       - drop_event:
           when:
             not:
@@ -37,5 +39,14 @@ data:
 
     logging.level: info
 
+    queue.mem:
+      events: 4096
+      flush.min_events: 5
+      flush.timeout: 5s
+
     output.logstash:
-      hosts: ["logstash.app-notifications.svc.cluster.local:5044"]
+      hosts: ["logstash.app-notifications.svc.cluster.local:5044"]
+      bulk_max_size: 10
+      worker: 1
+      compression_level: 3
+      timeout: 30