send nginx logs to opensearch with fluent-bit

tested on debian12

descr

we here separate access and error logs in two different indices

requirements

prepare an index template and eventually a user for that purpose – make sure you’ve got a template for pattern nginx-*.

# osearch
http://localhost:5601/app/opensearch_index_management_dashboards#/templates

make sure some data nodes are reachable

nmap -p 9200 10.1.0.30

# osearch
curl -k https://10.1.0.30:9200/?pretty -u admin:PASSWORD

setup

vi /etc/fluent-bit/fluent-bit.conf

# INDENTATION WITH SPACES NOT TABS

[SERVICE]
    flush 1
    daemon off
    parsers_file parsers.conf
    parsers_file custom_parsers.conf
    plugins_file plugins.conf
    log_file /var/log/fluent-bit.log

[INPUT]
    name tail
    path /var/log/nginx/*error*log
    tag nginx.error

# https://nginx.org/en/docs/http/ngx_http_log_module.html --> log_format escape=json
[INPUT]
    name tail
    path /var/log/nginx/*access*log
    tag nginx.access
    parser json_no_time

# @timestamp is enough
[FILTER]
    name modify
    match nginx.access
    remove time_iso8601

# https://elastic.co/guide/en/ecs/1.12/ecs-geo.html
[FILTER]
    name geoip2
    match nginx.access
    database /etc/fluent-bit/GeoLite2-City.mmdb
    lookup_key              remote_addr
    record geo.city_name    remote_addr %{city.names.en}
    record geo.country_name remote_addr %{country.names.en}
    record lat              remote_addr %{location.latitude}
    record lon              remote_addr %{location.longitude}
    log_level error

# catch lat lon (lat comes first)
[FILTER]
    name nest
    match nginx.access
    operation nest
    wildcard l*
    nest_under geo.location

# https://docs.fluentbit.io/manual/pipeline/filters/parser
[FILTER]
    name parser
    match nginx.access
    key_name request
    parser split_request
    reserve_data true

[FILTER]
    name parser
    match nginx.access
    key_name path
    parser strip_querystr
    reserve_data true
    preserve_key true

[FILTER]
    name modify
    match nginx.*
    add sensor nginx@HOSTNAME

# mimic k8s/cri
[FILTER]
    name modify
    match nginx.error
    add stream stderr

[FILTER]
    name modify
    match nginx.access
    add stream stdout

[OUTPUT]
    name file
    match nginx.*
    path /var/log
    file fluent-bit.nginx.log

chmod 600 fluent-bit.conf

vi /etc/fluent-bit/custom_parsers.conf

# @timestamp is enough - no time_key
# nginx's time_local is a pain to handle anyhow
# https://docs.fluentbit.io/manual/pipeline/parsers/configuring-parser
[PARSER]
    name   json_no_time
    format json

# split-up the request field
[PARSER]
    name split_request
    format regex
    regex ^(?<method>[^ ]*) (?<path>[^ ]*) HTTP/(?<http_version>[^ ]*)

# help differenciate web pages - strip out the query string
[PARSER]
    name strip_querystr
    format regex
    regex ^(?<page>[^?]*)

ready to go

tail -F /var/log/fluent-bit*log

systemctl restart fluent-bit
systemctl status fluent-bit

acceptance

access logs

curl -i localhost           # 200 access log
curl -i localhost/NO-EXIST      # 404 access log

error logs

easy-peasy error log generation

    # generate an error log - fcgiwrap not installed
        location ~ (\.cgi|\.sh)$ {
            fastcgi_pass unix:/var/run/fcgiwrap.socket;
        }

cat > /var/www/html/error.sh <EOF
#!/bin/bash

echo 'Content-Type: text/html'
echo
echo '<p>this message should NOT show up -- it should generate an nginx error log instead'
EOF

chmod +x error.sh
curl -i localhost/error.sh      # error log

troubleshooting

[ warn] [engine] failed to flush chunk '51468-1698409873.302155692.flb', retry in 9 seconds: task_id=0, input=tail.0
> output=opensearch.0 (out_id=0)
[error] [output:opensearch:opensearch.0] HTTP status=401 URI=/_bulk
[error] [engine] chunk '51468-1698409873.302155692.flb' cannot be retried: task_id=0, input=tail.0 > output=opensearch.0

resources

https://opensearch.org/blog/getting-started-with-fluent-bit-and-opensearch/

https://docs.fluentbit.io/manual/pipeline/outputs/opensearch

acceptance

https://opensearch.org/docs/latest/api-reference/index-apis/get-index/

https://opensearch.org/docs/latest/dashboards/discover/index-discover/

https://stackoverflow.com/questions/69617608/elasticsearch-8-errors-with-action-metadata-line-1-contains-an-unknown-paramet ==> nice example

https://docs.oracle.com/en-us/iaas/Content/search-opensearch/Tasks/ingestingfluentbit.htm

troubles

https://stackoverflow.com/questions/73867417/opensearch-401-for-bulk ==> don’t forget to authenticate

moar

https://docs.fluentbit.io/manual/pipeline/inputs/nginx