flb // cloudflare instant logs

this one is a bit tricky because the EdgeServerIP field doesn’t necessarily show up, making the IP mapping fail – in a nutshell

[INPUT]
    name tail
    tag cf.testsite
    path /data/cloudflare-instant-logs/*.json
    parser json

# source map
[FILTER]
    name modify
    match cf.*
    condition key_value_matches ClientIP [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+
    rename ClientIP source.ip

[FILTER]
    name geoip2
    match cf.*
    database /etc/fluent-bit/GeoLite2-City.mmdb
    lookup_key source.ip
    record source.geo.country_name         source.ip %{country.names.ru}
    record lat                             source.ip %{location.latitude}
    record lon                             source.ip %{location.longitude}

[FILTER]
    name nest
    match cf.*
    operation nest
    wildcard l*
    nest_under source.geo.location

# destination map
[FILTER]
    name modify
    match cf.*
    condition key_value_matches EdgeServerIP [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+
    rename EdgeServerIP destination.ip

[FILTER]
    name geoip2
    match cf.*
    database /etc/fluent-bit/GeoLite2-City.mmdb
    lookup_key destination.ip
    record destination.geo.country_name         destination.ip %{country.names.ru}
    record lat                                  destination.ip %{location.latitude}
    record lon                                  destination.ip %{location.longitude}

[FILTER]
    name nest
    match cf.*
    operation nest
    wildcard l*
    nest_under destination.geo.location

[OUTPUT]
    name file
    match cf.*
    path /var/log
    file fluent-bit.OUTPUT.log

besides, for high volume sites, some tuning might be required – not only on the flb side

[SERVICE]
    flush 5

...

[OUTPUT]
    name es
    ...
    buffer_size 1M

– but also on the log rotation

vi /etc/logrotate.d/fluent-bit

# warnings and errors can be pretty hefty therefore 1h rotation
# and no delaycompress
/var/log/fluent-bit*log {
        hourly
        rotate 1
        missingok
        compress
        create
}

vi /etc/logrotate.d/rsyslog-hourly

/var/log/syslog
/var/log/daemon.log
{
        rotate 0
        hourly
        missingok
        notifempty
        compress
        sharedscripts
        postrotate
                /usr/lib/rsyslog/rsyslog-rotate
        endscript
}

and don’t forget to copy logrotate daily cron to hourly