flb // k8s ingress-nginx

assuming ingress-nginx logs are customized as json already

we are splitting up stdout and stderr here

notice the alternate path to conf/ as it points to helm’s mounted volume

[SERVICE]
    flush 1
    parsers_file /fluent-bit/etc/parsers.conf
    parsers_file /fluent-bit/etc/conf/custom_parsers.conf
    streams_file /fluent-bit/etc/conf/streams.conf
    http_server on
    health_check on

[INPUT]
    name tail
    path /var/log/containers/ingress-nginx-controller*log
    parser cri
    tag kube
    mem_buf_limit 5MB
    skip_long_lines on

[FILTER]
    name expect
    match kube
    key_exists stream
    action exit

[FILTER]
    name modify
    match kube
    remove time

[FILTER]
    name modify
    match kube
    # stream processor refuses stream field
    rename stream streamfix

# https://docs.fluentbit.io/manual/pipeline/filters/parser
[FILTER]
    name parser
    match source.stdout
    key_name message
    # ingress-nginx logs are customized as json
    parser json
    reserve_data true

[FILTER]
    name parser
    match source.stdout
    key_name upstream_addr
    # provides upstream_ip and upstream_port
    parser custom_upstream
    reserve_data true

[FILTER]
    name modify
    match source.stdout
    remove time_local

[FILTER]
    name modify
    match source.stdout
    # elasticsearch refuses host field
    rename host vhost

[FILTER]
    name modify
    match source.*
    # revert back to original field name
    rename streamfix stream

#[OUTPUT]
#   name stdout
#   match source.*

as for custom parsers

[PARSER]
    name custom_upstream
    format regex
    regex ^(?<upstream_ip>[^:]*):(?<upstream_port>[^ ]*)$

as for streams

[STREAM_TASK]
    Name error_log
    Exec CREATE STREAM error WITH (tag='source.stderr') AS SELECT * from STREAM:tail.0 WHERE streamfix = 'stderr';

[STREAM_TASK]
    Name access_log
    Exec CREATE STREAM access WITH (tag='source.stdout') AS SELECT * from STREAM:tail.0 WHERE streamfix = 'stdout';