Filebeat+Logstash: Lowered flush size by significant amount to get logs into ES much quicker

filebeat/configmap.yaml

@@ -5,23 +5,25 @@ metadata:
   namespace: app-notifications
 data:
   filebeat.yml: |
-    filebeat.inputs:
+    filebeat.autodiscover:
-    - type: filestream
+      providers:
-      id: kubernetes-containers
+        - type: kubernetes
-      paths:
+          node: ${NODE_NAME}
-        - /var/log/containers/*_app-notifications_*.log
+          hints.enabled: false
-      prospector:
+          templates:
-        scanner:
+            - condition:
-          fingerprint.enabled: true
+                equals:
-          fingerprint.offset: 0
+                  kubernetes.namespace: "app-notifications"
-          fingerprint.length: 512
+              config:
-      parsers:
+                - type: container
-        - container: ~
+                  paths:
+                    - /var/log/containers/*_app-notifications_*.log
+                  stream: stdout
+                  fingerprint.enabled: false
 
     processors:
       - add_kubernetes_metadata:
           in_cluster: true
-          host: ${NODE_NAME}
       - drop_event:
           when:
             not:
@@ -37,5 +39,14 @@ data:
 
     logging.level: info
 
+    queue.mem:
+      events: 4096
+      flush.min_events: 5
+      flush.timeout: 5s
+
     output.logstash:
-      hosts: ["logstash.app-notifications.svc.cluster.local:5044"]
+      hosts: ["logstash.app-notifications.svc.cluster.local:5044"]
+      bulk_max_size: 10
+      worker: 1
+      compression_level: 3
+      timeout: 30