No more debugging

Linked build commit: f51a66078e

README.md

@@ -0,0 +1,75 @@
+# ops-deployment
+
+This repository contains declarative Kubernetes configurations (deployments, services, persistent volumes) for all application services. Changes are automatically synchronized to the cluster by Flux.
+
+## Structure
+
+Each service directory contains:
+- `deployment.yaml` - Pod specifications, container images, environment variables, volume mounts
+- `service.yaml` - Service exposure (ClusterIP, NodePort)
+- Additional resources as needed (PVCs, ConfigMaps, etc.)
+
+## Workflow
+
+1. **CI/CD Pipeline**: Build process generates new container image
+2. **Automatic Update**: Pipeline commits updated image tag to this repository
+3. **Flux Synchronization**: Flux detects changes and applies to cluster
+4. **Rolling Deployment**: Kubernetes performs rolling update
+
+## Secrets Management
+
+Secrets are managed directly in Kubernetes using `kubectl` or sealed-secrets:
+- Never committed to this repository
+- Mounted as volumes at `/etc/secrets` in containers
+- Referenced via `secretName` in deployment manifests
+
+## Persistent Storage
+
+Services requiring data persistence (MySQL, RabbitMQ) use PersistentVolumeClaims:
+- Storage classes define volume provisioning
+- Data survives pod restarts and redeployments
+- Backups handled separately
+
+## Example Service
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend-push-notifications
+  namespace: app-notifications
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+        - name: backend-push-notifications
+          image: ghcr.io/user/service:2
+          ports:
+            - containerPort: 9000
+          volumeMounts:
+            - name: secrets
+              mountPath: /etc/secrets
+              readOnly: true
+      volumes:
+        - name: secrets
+          secret:
+            secretName: backend-push-notifications
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend-push-notifications
+spec:
+  selector:
+    app: backend-push-notifications
+  ports:
+    - port: 9000
+      targetPort: 9000
+      nodePort: 30904
+  type: NodePort
+```
+
+## Monitoring
+
+Services expose Prometheus metrics endpoints where applicable, on port 9000 and exposed via NodePort.

backend-api-internal/deployment.yaml

@@ -15,21 +15,41 @@ spec:
     spec:
       imagePullSecrets:
         - name: ghcr-secret
+      initContainers:
+        - name: copy-secrets
+          image: busybox
+          command:
+            - sh
+            - -c
+            - |
+              for f in /secrets/*; do
+                cp "$f" /etc/secrets/
+                chmod 666 "/etc/secrets/$(basename $f)"
+              done
+          volumeMounts:
+            - name: backend-api-internal-secrets
+              mountPath: /secrets
+            - name: writable-secrets
+              mountPath: /etc/secrets
       containers:
         - name: backend-api-internal
-          image: ghcr.io/gansejunge/app-notifications-backend-api-internal:2
+          image: ghcr.io/gansejunge/app-notifications-backend-api-internal:19
           ports:
             - containerPort: 8101
           env:
             - name: BACKEND_API_INTERNAL_RMQ_HOST
               value: "rabbitmq.app-notifications.svc.cluster.local"
-            - name: BACKEND_API_INTERNAL_DB_HOST
+            - name: DB_HOST
               value: "mysql.app-notifications.svc.cluster.local"
+            - name: LOG_LEVEL
+              value: "INFO"
           volumeMounts:
-            - name: backend-api-internal-secrets
+            - name: writable-secrets
               mountPath: /etc/secrets
-              readOnly: true
       volumes:
         - name: backend-api-internal-secrets
           secret:
             secretName: backend-api-internal
+        - name: writable-secrets
+          emptyDir: {}
+

backend-api/deployment.yaml

@@ -15,19 +15,38 @@ spec:
     spec:
       imagePullSecrets:
         - name: ghcr-secret
+      initContainers:
+        - name: copy-secrets
+          image: busybox
+          command:
+            - sh
+            - -c
+            - |
+              for f in /secrets/*; do
+                cp "$f" /etc/secrets/
+                chmod 666 "/etc/secrets/$(basename $f)"
+              done
+          volumeMounts:
+            - name: backend-api-secrets
+              mountPath: /secrets
+            - name: writable-secrets
+              mountPath: /etc/secrets
       containers:
         - name: backend-api
-          image: ghcr.io/gansejunge/app-notifications-backend-api:9
+          image: ghcr.io/gansejunge/app-notifications-backend-api:30
           ports:
             - containerPort: 8100
           env:
-            - name: BACKEND_API_DB_HOST
+            - name: DB_HOST
               value: "mysql.app-notifications.svc.cluster.local"
+            - name: LOG_LEVEL
+              value: "INFO"
           volumeMounts:
-            - name: backend-api-secrets
+            - name: writable-secrets
               mountPath: /etc/secrets
-              readOnly: true
       volumes:
         - name: backend-api-secrets
           secret:
             secretName: backend-api
+        - name: writable-secrets
+          emptyDir: {}

backend-push-notifications/deployment.yaml

@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend-push-notifications
+  namespace: app-notifications
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: backend-push-notifications
+  template:
+    metadata:
+      labels:
+        app: backend-push-notifications
+    spec:
+      imagePullSecrets:
+        - name: ghcr-secret
+      initContainers:
+        - name: copy-secrets
+          image: busybox
+          command:
+            - sh
+            - -c
+            - |
+              for f in /secrets/*; do
+                cp "$f" /etc/secrets/
+                chmod 666 "/etc/secrets/$(basename $f)"
+              done
+          volumeMounts:
+            - name: backend-push-notifications-secrets
+              mountPath: /secrets
+            - name: writable-secrets
+              mountPath: /etc/secrets
+      containers:
+        - name: backend-push-notifications
+          image: ghcr.io/gansejunge/app-notifications-backend-push-notifications:12
+          ports:
+            - containerPort: 9000
+              name: metrics
+          env:
+            - name: BACKEND_PN_RMQ_HOST
+              value: "rabbitmq.app-notifications.svc.cluster.local"
+            - name: BACKEND_PN_DB_HOST
+              value: "mysql.app-notifications.svc.cluster.local"
+            - name: LOG_LEVEL
+              value: "INFO"
+          volumeMounts:
+            - name: writable-secrets
+              mountPath: /etc/secrets
+      volumes:
+        - name: backend-push-notifications-secrets
+          secret:
+            secretName: backend-push-notifications
+        - name: writable-secrets
+          emptyDir: {}

backend-push-notifications/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend-push-notifications
+  namespace: app-notifications
+spec:
+  selector:
+    app: backend-push-notifications
+  ports:
+    - protocol: TCP
+      port: 9000
+      targetPort: 9000
+      nodePort: 30904
+      name: prometheus-metrics
+  type: NodePort

elasticsearch/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: app-notifications
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+        - name: elasticsearch
+          image: docker.elastic.co/elasticsearch/elasticsearch:8.12.1
+          ports:
+            - containerPort: 9200
+              name: http
+            - containerPort: 9300
+              name: transport
+          env:
+            - name: discovery.type
+              value: single-node
+            - name: xpack.security.enabled
+              value: "true"
+            - name: ES_JAVA_OPTS
+              value: "-Xms1g -Xmx1g"
+          volumeMounts:
+            - name: elastic-data
+              mountPath: /usr/share/elasticsearch/data
+      volumes:
+        - name: elastic-data
+          persistentVolumeClaim:
+            claimName: elasticsearch-data

elasticsearch/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: elasticsearch-data
+  namespace: app-notifications
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi

elasticsearch/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: app-notifications
+  labels:
+    app: elasticsearch
+spec:
+  clusterIP: None
+  selector:
+    app: elasticsearch
+  ports:
+    - port: 9200
+      name: http
+    - port: 9300
+      name: transport

filebeat/configmap.yaml

@@ -0,0 +1,52 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: app-notifications
+data:
+  filebeat.yml: |
+    filebeat.autodiscover:
+      providers:
+        - type: kubernetes
+          node: ${NODE_NAME}
+          hints.enabled: false
+          templates:
+            - condition:
+                equals:
+                  kubernetes.namespace: "app-notifications"
+              config:
+                - type: container
+                  paths:
+                    - /var/log/containers/*_app-notifications_*.log
+                  stream: stdout
+                  fingerprint.enabled: false
+
+    processors:
+      - add_kubernetes_metadata:
+          in_cluster: true
+      - drop_event:
+          when:
+            not:
+              or:
+                - regexp:
+                    kubernetes.pod.name: "^backend-.*"
+                - regexp:
+                    kubernetes.pod.name: "^service-.*"
+                - regexp:
+                    kubernetes.pod.name: "^mysql-.*"
+                - regexp:
+                    kubernetes.pod.name: "^rabbitmq-.*"
+
+    logging.level: info
+
+    queue.mem:
+      events: 4096
+      flush.min_events: 5
+      flush.timeout: 5s
+
+    output.logstash:
+      hosts: ["logstash.app-notifications.svc.cluster.local:5044"]
+      bulk_max_size: 10
+      worker: 1
+      compression_level: 3
+      timeout: 30

filebeat/daemonset.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: app-notifications
+  labels:
+    app: filebeat
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      serviceAccountName: filebeat
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:8.12.1
+        args: [
+          "-c", "/usr/share/filebeat/filebeat.yml",
+          "-e"
+        ]
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        volumeMounts:
+          - name: config
+            mountPath: /usr/share/filebeat/filebeat.yml
+            subPath: filebeat.yml
+          - name: varlog
+            mountPath: /var/log
+          - name: dockersock
+            mountPath: /var/lib/docker/containers
+            readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: filebeat-config
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: dockersock
+          hostPath:
+            path: /var/lib/docker/containers

filebeat/rbac.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: app-notifications
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: filebeat
+  namespace: app-notifications
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "namespaces"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: filebeat
+  namespace: app-notifications
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: app-notifications
+roleRef:
+  kind: Role
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io

kibana/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: app-notifications
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+        - name: kibana
+          image: docker.elastic.co/kibana/kibana:8.12.1
+          ports:
+            - containerPort: 5601
+          env:
+            - name: SERVER_PUBLICBASEURL
+              value: "https://kibana.gansejunge.com"
+            - name: ELASTICSEARCH_HOSTS
+              value: "http://elasticsearch:9200"
+            - name: ELASTICSEARCH_USERNAME
+              value: "kibana_system"
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: kibana-credentials
+                  key: password
+            - name: XPACK_SECURITY_ENCRYPTIONKEY
+              valueFrom:
+                secretKeyRef:
+                  name: kibana-encryption-keys
+                  key: xpack_security_encryptionKey
+            - name: XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY
+              valueFrom:
+                secretKeyRef:
+                  name: kibana-encryption-keys
+                  key: xpack_encryptedSavedObjects_encryptionKey
+            - name: XPACK_REPORTING_ENCRYPTIONKEY
+              valueFrom:
+                secretKeyRef:
+                  name: kibana-encryption-keys
+                  key: xpack_reporting_encryptionKey

kibana/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: app-notifications
+spec:
+  type: NodePort
+  ports:
+    - port: 5601
+      targetPort: 5601
+      nodePort: 30102
+      name: http
+  selector:
+    app: kibana

logstash/configmap.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logstash-config
+  namespace: app-notifications
+data:
+  logstash.yml: |
+    http.host: "0.0.0.0"
+    xpack.monitoring.enabled: false
+
+  logstash.conf: |
+    input {
+      beats {
+        port => 5044
+      }
+    }
+      filter {
+         if [level] == "DEBUG" { drop {} }
+      }
+    output {
+      elasticsearch {
+        hosts => ["http://elasticsearch:9200"]
+        user => "elastic"
+        password => "${ELASTIC_PASSWORD}"
+        ssl_verification_mode => none
+        index => "app-notifications-%{+YYYY.MM.dd}"
+      }
+    }

logstash/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: logstash
+  namespace: app-notifications
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: logstash
+  template:
+    metadata:
+      labels:
+        app: logstash
+    spec:
+      containers:
+      - name: logstash
+        image: docker.elastic.co/logstash/logstash:8.12.1
+        env:
+          - name: ELASTIC_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: elastic-credentials
+                key: ELASTIC_PASSWORD
+        ports:
+        - containerPort: 5044
+        volumeMounts:
+        - name: logstash-config
+          mountPath: /usr/share/logstash/config/logstash.yml
+          subPath: logstash.yml
+        - name: logstash-config
+          mountPath: /usr/share/logstash/pipeline/logstash.conf
+          subPath: logstash.conf
+      volumes:
+      - name: logstash-config
+        configMap:
+          name: logstash-config

logstash/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: logstash
+  namespace: app-notifications
+spec:
+  selector:
+    app: logstash
+  ports:
+    - protocol: TCP
+      port: 5044
+      targetPort: 5044

service-docker-repository-query/cron-check-pihole.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: check-pihole-updates
+  namespace: app-notifications
+spec:
+  schedule: "0 7 * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl
+            image: curlimages/curl:latest
+            command: ["curl"]
+            args:
+            - -s
+            - -X
+            - GET
+            - http://service-docker-repository-query-internal:5000/pihole
+          restartPolicy: OnFailure

service-docker-repository-query/cron-check-suwayomi.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: check-suwayomi-updates
+  namespace: app-notifications
+spec:
+  schedule: "0 8 * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl
+            image: curlimages/curl:latest
+            command: ["curl"]
+            args:
+            - -s
+            - -X
+            - GET
+            - http://service-docker-repository-query-internal:5000/suwayomi
+          restartPolicy: OnFailure

service-docker-repository-query/deployment.yaml

@@ -15,24 +15,41 @@ spec:
     spec:
       imagePullSecrets:
         - name: ghcr-secret
+      initContainers:
+        - name: copy-secrets
+          image: busybox
+          command:
+            - sh
+            - -c
+            - |
+              for f in /secrets/*; do
+                cp "$f" /etc/secrets/
+                chmod 666 "/etc/secrets/$(basename $f)"
+              done
+          volumeMounts:
+            - name: service-docker-repository-query-secrets
+              mountPath: /secrets
+            - name: writable-secrets
+              mountPath: /etc/secrets
       containers:
         - name: service-docker-repository-query
-          image: ghcr.io/gansejunge/app-notifications-service-docker-repository-query:3
+          image: ghcr.io/gansejunge/app-notifications-service-docker-repository-query:19
           ports:
             - containerPort: 5000
               name: fastapi
             - containerPort: 9000
               name: metrics
           env:
-            - name: SERVICE_DRQ_DB_HOST
+            - name: DB_HOST
               value: "mysql.app-notifications.svc.cluster.local"
             - name: BACKEND_API_URL
               value: "http://backend-api-internal.app-notifications.svc.cluster.local:8101/internal/receive-notifications"
           volumeMounts:
-            - name: service-docker-repository-query-secrets
+            - name: writable-secrets
               mountPath: /etc/secrets
-              readOnly: true
       volumes:
         - name: service-docker-repository-query-secrets
           secret:
             secretName: service-docker-repository-query
+        - name: writable-secrets
+          emptyDir: {}

service-royalroad-chapters/cron-check-new-royalroad-chapters.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: check-new-royalroad-chapters
+  namespace: app-notifications
+spec:
+  schedule: "5-59/10 * * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: curl
+            image: curlimages/curl:latest
+            command: ["curl"]
+            args:
+            - -s
+            - -X
+            - GET
+            - http://service-royalroad-chapters-internal:5000/royalroad
+          restartPolicy: OnFailure

service-royalroad-chapters/deployment.yaml

@@ -15,24 +15,41 @@ spec:
     spec:
       imagePullSecrets:
         - name: ghcr-secret
+      initContainers:
+        - name: copy-secrets
+          image: busybox
+          command:
+            - sh
+            - -c
+            - |
+              for f in /secrets/*; do
+                cp "$f" /etc/secrets/
+                chmod 666 "/etc/secrets/$(basename $f)"
+              done
+          volumeMounts:
+            - name: service-royalroad-chapters-secrets
+              mountPath: /secrets
+            - name: writable-secrets
+              mountPath: /etc/secrets
       containers:
         - name: service-royalroad-chapters
-          image: ghcr.io/gansejunge/app-notifications-service-royalroad-chapters:4
+          image: ghcr.io/gansejunge/app-notifications-service-royalroad-chapters:27
           ports:
             - containerPort: 5000
               name: fastapi
             - containerPort: 9000
               name: metrics
           env:
-            - name: SERVICE_RR_DB_HOST
+            - name: DB_HOST
               value: "mysql.app-notifications.svc.cluster.local"
             - name: BACKEND_API_URL
               value: "http://backend-api-internal.app-notifications.svc.cluster.local:8101/internal/receive-notifications"
           volumeMounts:
-            - name: service-royalroad-chapters-secrets
+            - name: writable-secrets
               mountPath: /etc/secrets
-              readOnly: true
       volumes:
         - name: service-royalroad-chapters-secrets
           secret:
             secretName: service-royalroad-chapters
+        - name: writable-secrets
+          emptyDir: {}