From 1ae3d5c3054cef5559f85c262b2c45830b108b55 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Sat, 3 Oct 2026 13:56:32 +0100 Subject: [PATCH 1/2] Feature: also send logs to VictoriaLogs --- .github/workflows/deploy.yaml | 1 + fluentbit/fluent-bit.yaml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index edaf19d..6bf61a7 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -148,5 +148,6 @@ jobs: --detach \ --restart on-failure \ --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \ + --env VICTORIALOGS_HOST=10.18.1.50 \ --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ ${DOCKER_IMAGE} diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index f02f8cc..83febfb 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -28,6 +28,15 @@ pipeline: tls: "On" logstash_format: "On" suppress_type_name: "On" + - name: http + match: "*" + host: ${VICTORIALOGS_HOST} + port: 9428 + uri: /insert/jsonline?_stream_fields=name,containerId&_msg_field=message,log&_time_field=date + format: json_lines + json_date_key: date + json_date_format: iso8601 + compress: gzip # # Testing only # inputs: # - name: dummy From df204d57e3ea683950fdd8560b7a9dfc7cf31430 Mon Sep 17 00:00:00 2001 From: Krishan Patel Date: Sat, 3 Oct 2026 14:12:57 +0100 Subject: [PATCH 2/2] Fiddle: only send logs to the outputs whose env vars are set --- fluentbit/Dockerfile | 8 +++++++- fluentbit/entrypoint.sh | 20 ++++++++++++++++++++ fluentbit/fluent-bit.yaml | 17 ----------------- fluentbit/outputs/opensearch.yaml | 7 +++++++ fluentbit/outputs/victorialogs.yaml | 9 +++++++++ 5 files changed, 43 insertions(+), 18 deletions(-) create mode 100755 fluentbit/entrypoint.sh create mode 100644 fluentbit/outputs/opensearch.yaml create mode 100644 fluentbit/outputs/victorialogs.yaml diff --git a/fluentbit/Dockerfile b/fluentbit/Dockerfile index 856adeb..b1e568d 100644 --- a/fluentbit/Dockerfile +++ b/fluentbit/Dockerfile @@ -1,6 +1,12 @@ FROM fluent/fluent-bit:5.1.1 +# NOTE(krishan711): the fluent-bit image has no shell, so add a static busybox for entrypoint.sh to pick the outputs +COPY --from=busybox:1.37.0-musl /bin/busybox /bin/busybox +RUN ["/bin/busybox", "--install", "-s", "/bin"] + COPY *.yaml /fluent-bit/etc/ COPY *.lua /fluent-bit/etc/ +COPY outputs /fluent-bit/etc/outputs +COPY entrypoint.sh /fluent-bit/entrypoint.sh -CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"] +ENTRYPOINT ["/fluent-bit/entrypoint.sh"] diff --git a/fluentbit/entrypoint.sh b/fluentbit/entrypoint.sh new file mode 100755 index 0000000..1adbdef --- /dev/null +++ b/fluentbit/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -e + +config=/fluent-bit/etc/fluent-bit.generated.yaml +cp /fluent-bit/etc/fluent-bit.yaml "$config" +echo " outputs:" >> "$config" +if [ -n "$OPENSEARCH_URL" ]; then + cat /fluent-bit/etc/outputs/opensearch.yaml >> "$config" + echo "Sending logs to OpenSearch at $OPENSEARCH_URL" +fi +if [ -n "$VICTORIALOGS_HOST" ]; then + cat /fluent-bit/etc/outputs/victorialogs.yaml >> "$config" + echo "Sending logs to VictoriaLogs at $VICTORIALOGS_HOST" +fi +if [ -z "$OPENSEARCH_URL" ] && [ -z "$VICTORIALOGS_HOST" ]; then + echo "Error: set OPENSEARCH_URL and/or VICTORIALOGS_HOST" + exit 1 +fi + +exec /fluent-bit/bin/fluent-bit --config="$config" diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index 83febfb..94b2881 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -20,23 +20,6 @@ pipeline: tag: docker_logs multiline.parser: docker path_key: containerLogPath - outputs: - - name: opensearch - match: "*" - host: ${OPENSEARCH_URL} - port: 443 - tls: "On" - logstash_format: "On" - suppress_type_name: "On" - - name: http - match: "*" - host: ${VICTORIALOGS_HOST} - port: 9428 - uri: /insert/jsonline?_stream_fields=name,containerId&_msg_field=message,log&_time_field=date - format: json_lines - json_date_key: date - json_date_format: iso8601 - compress: gzip # # Testing only # inputs: # - name: dummy diff --git a/fluentbit/outputs/opensearch.yaml b/fluentbit/outputs/opensearch.yaml new file mode 100644 index 0000000..f2c77eb --- /dev/null +++ b/fluentbit/outputs/opensearch.yaml @@ -0,0 +1,7 @@ + - name: opensearch + match: "*" + host: ${OPENSEARCH_URL} + port: 443 + tls: "On" + logstash_format: "On" + suppress_type_name: "On" diff --git a/fluentbit/outputs/victorialogs.yaml b/fluentbit/outputs/victorialogs.yaml new file mode 100644 index 0000000..2b771db --- /dev/null +++ b/fluentbit/outputs/victorialogs.yaml @@ -0,0 +1,9 @@ + - name: http + match: "*" + host: ${VICTORIALOGS_HOST} + port: 9428 + uri: /insert/jsonline?_stream_fields=name,containerId&_msg_field=message,log&_time_field=date + format: json_lines + json_date_key: date + json_date_format: iso8601 + compress: gzip