add fluentbit container to ship logfiles to ES

hifiscp/Chart.yaml

 apiVersion: v1
 name: hifiscp
-version: 0.1.0
+version: 0.1.1
 description: A Helm chart to deploy the HIFIS Cloud Portal Backend and Frontend
 keywords:
   - HIFIS

hifiscp/templates/cerebrum_deployment.yaml

@@ -33,7 +33,34 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
     spec:
+      {{- if .Values.cerebrum.logging.enabled }}
+      volumes:
+      - name: jettylog
+        emptyDir: {}
+      - name: fluentbit-config-volume
+        configMap:
+          name: {{ template "hifiscp.fullname" . }}-fluentbit-config
+      - name: fluentbit-output
+        secret:
+          secretName: {{ template "hifiscp.fullname" . }}-fluentbit-output-cerebrum
+      {{- end }}
       containers:
+      {{- if .Values.cerebrum.logging.enabled }}
+      - name: access-log
+        image: fluent/fluent-bit
+        volumeMounts:
+        - name: jettylog
+          mountPath: /var/log/jetty
+        - name: fluentbit-config-volume
+          mountPath: /fluent-bit/etc/fluent-bit.conf
+          subPath: fluent-bit.conf
+        - name: fluentbit-config-volume
+          mountPath: /fluent-bit/etc/parsers.conf
+          subPath: parsers.conf
+        - name: fluentbit-output
+          mountPath: /fluent-bit/etc/output.conf
+          subPath: output.conf
+      {{- end }}
       - name: cerebrum
         image: {{ .Values.cerebrum.image.repository }}:{{ .Values.cerebrum.image.tag }}
         imagePullPolicy: {{ .Values.cerebrum.image.pullPolicy }}
@@ -61,6 +88,11 @@ spec:
           failureThreshold: {{ .Values.cerebrum.readinessProbe.failureThreshold }}
           successThreshold: {{ .Values.cerebrum.readinessProbe.successThreshold }}
         {{- end }}
+        {{- if .Values.cerebrum.logging.enabled }}
+        volumeMounts:
+        - name: jettylog
+          mountPath: /var/log/jetty
+        {{- end }}
         env:
         {{- if .Values.cerebrum.config.mongodb.database }}
         - name: MONGODB_DB
@@ -82,6 +114,10 @@ spec:
         - name: MONGODB_USERNAME
           value: {{ .Values.cerebrum.config.mongodb.username | quote }}
         {{- end }}
+        {{- if .Values.cerebrum.logging.enabled }}
+        - name: ENABLE_LOGGING
+          value: "true"
+        {{- end }}
         - name: MONGODB_PASSWORD
           valueFrom:
             secretKeyRef:

hifiscp/templates/fluentbit-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "hifiscp.fullname" . }}-fluentbit-config
+  labels:
+    app: {{ template "hifiscp.name" . }}-fluentbit
+    chart: {{ template "hifiscp.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+      Flush        5
+      Daemon       Off
+      Log_Level    info
+      Parsers_File parsers.conf
+      Plugins_File plugins.conf
+      HTTP_Server  Off
+      HTTP_Listen  0.0.0.0
+      HTTP_Port    2020
+
+    [INPUT]
+      name tail
+      Tag accesslog
+      path /var/log/jetty/access.log
+      parser apache
+
+    @INCLUDE output.conf
+  parsers.conf: |+
+    [PARSER]
+      Name   apache
+      Format regex
+      Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+      Time_Key time
+      Time_Format %d/%b/%Y:%H:%M:%S %z
+      Types code:integer size:integer
\ No newline at end of file

hifiscp/templates/webapp_deployment.yaml

@@ -33,7 +33,35 @@ spec:
         release: {{ .Release.Name }}
         heritage: {{ .Release.Service }}
     spec:
+      {{- if .Values.webapp.logging.enabled }}
+      volumes:
+      - name: jettylog
+        emptyDir: {}
+      - name: fluentbit-config-volume
+        configMap:
+          name: {{ template "hifiscp.fullname" . }}-fluentbit-config
+      - name: fluentbit-output
+        secret:
+          secretName: {{ template "hifiscp.fullname" . }}-fluentbit-output-webapp
+      {{- end }}
       containers:
+      {{- if .Values.webapp.logging.enabled }}
+      - name: access-log
+        image: busybox
+        args: [/bin/sh, -c, 'tail -n+1 -f /var/log/jetty/access.log']
+        volumeMounts:
+        - name: jettylog
+          mountPath: /var/log/jetty
+        - name: fluentbit-config-volume
+          mountPath: /fluent-bit/etc/fluent-bit.conf
+          subPath: fluent-bit.conf
+        - name: fluentbit-config-volume
+          mountPath: /fluent-bit/etc/parsers.conf
+          subPath: parsers.conf
+        - name: fluentbit-output
+          mountPath: /fluent-bit/etc/output.conf
+          subPath: output.conf
+      {{- end }}
       - name: webapp
         image: {{ .Values.webapp.image.repository }}:{{ .Values.webapp.image.tag }}
         imagePullPolicy: {{ .Values.webapp.image.pullPolicy }}
@@ -61,6 +89,11 @@ spec:
           failureThreshold: {{ .Values.webapp.readinessProbe.failureThreshold }}
           successThreshold: {{ .Values.webapp.readinessProbe.successThreshold }}
         {{- end }}
+        {{- if .Values.webapp.logging.enabled }}
+        volumeMounts:
+        - name: jettylog
+          mountPath: /var/log/jetty
+        {{- end }}
         env:
         - name: CLIENT_SECRET
           valueFrom:
@@ -71,6 +104,10 @@ spec:
         - name: CONTEXT_PATH
           value: {{ .Values.webapp.config.contextPath | quote }}
         {{- end }}
+        {{- if .Values.webapp.logging.enabled }}
+        - name: ENABLE_LOGGING
+          value: "true"
+        {{- end }}
         resources:
 {{ toYaml .Values.cerebrum.resources | indent 10 }}
 {{- with .Values.cerebrum.nodeSelector }}

hifiscp/values.yaml

@@ -2,6 +2,9 @@
 cerebrum:
   replicas: 1
 
+  logging:
+    enabled: false
+
   availabilityChecker:
     enabled: 0
     schedule: "*/10 * * * *"
@@ -72,6 +75,9 @@ cerebrum:
 webapp:
   replicas: 1
 
+  logging:
+    enabled: false
+
   image:
     repository: registry.hzdr.de/hifis/cloud/access-layer/portal/webapp
     tag: latest