# CONFIGURED BY ANSIBLE

#
# parse nginx access and error logs
#

# https://nginx.org/en/docs/http/ngx_http_log_module.html --> log_format escape=json
[INPUT]
    name tail
    path /var/log/nginx/*access*log
    tag nginx.access
    parser json_no_time

[INPUT]
    name tail
    path /var/log/nginx/*error*log
    tag nginx.error

# @timestamp is enough for logs
[FILTER]
    name modify
    match nginx.access
    #remove time_local
    remove time_iso8601

# elasticsearch refuses host field
[FILTER]
    name modify
    match nginx.access
    rename host vhost

# no ssl fields if scheme is http
[FILTER]
    name modify
    match nginx.access
    remove_regex ^ssl_.*
    condition key_value_equals scheme http

# https://docs.fluentbit.io/manual/pipeline/filters/parser
[FILTER]
    name parser
    match nginx.access
    key_name request_uri
    parser strip_querystr
    reserve_data true
    preserve_key true

# https://elastic.co/guide/en/ecs/1.12/ecs-geo.html
[FILTER]
    name geoip2
    match nginx.access
    #database /usr/share/GeoIP/GeoLite2-City.mmdb
    database /etc/fluent-bit/GeoLite2-City.mmdb
    lookup_key                  remote_addr
    record source_city_name     remote_addr %{city.names.ru}
    record source_country_name  remote_addr %{country.names.ru}
    record nest_lat             remote_addr %{location.latitude}
    record nest_lon             remote_addr %{location.longitude}
    log_level error

# catch lat lon
[FILTER]
    name nest
    match nginx.access
    operation nest
    wildcard nest_*
    remove_prefix nest_
    nest_under source_location

[FILTER]
    name modify
    match nginx.access
    copy request_length           source_bytes
    copy bytes_sent               destination_bytes
    add destination_name          {{inventory_hostname}}
    # hard-coded variable in ansible inventory
    add destination_city_name     {{city}}
    add destination_country_name  {{country}}
    add nest_lat                  {{lat}}
    add nest_lon                  {{lon}}

[FILTER]
    name nest
    match nginx.access
    operation nest
    wildcard nest_*
    remove_prefix nest_
    nest_under destination_location

[FILTER]
    name modify
    match nginx.*
    add sensor nginx@{{inventory_hostname}}

#[OUTPUT]
#    name file
#    match nginx.*
#    path /var/log
#    file fluent-bit.log

[OUTPUT]
    name opensearch
    match nginx.access
    host {{log_host}}
    port {{log_port}}
    tls on
    tls.verify on
    http_user {{log_http_user}}
    http_passwd {{log_http_passwd}}
    index nginx-prod-access
    suppress_type_name on
    #replace_dots on
    trace_error on

[OUTPUT]
    name opensearch
    match nginx.error
    host {{log_host}}
    port {{log_port}}
    tls on
    tls.verify on
    http_user {{log_http_user}}
    http_passwd {{log_http_passwd}}
    index nginx-prod-error
    suppress_type_name on
    #replace_dots on
    trace_error on