# CONFIGURED BY ANSIBLE [INPUT] name tail tag ngenix alias ngenix path /data/ngenix/extracting-s3fs/*.log path_key filename parser ngenix_time #read_from_head true skip_empty_lines on # this is traffic seen by CDN, it's huge (~120k/minute hence 2000 ops/s) # cap to 5000 ops/s in average over 5 minutes #[FILTER] # name throttle # match ngenix # rate 5000 # window 300 # interval 1s # print_status true [FILTER] name modify match ngenix # @timestamp is enough for logs remove time_local # elasticsearch refuses host field rename host vhost # provides upstream_ip upstream_port [FILTER] name parser match ngenix key_name upstream_addr parser nge_custom_upstream reserve_data true # provides method path http_version [FILTER] name parser match ngenix key_name request parser nge_split_request reserve_data true # provides page [FILTER] name parser match ngenix key_name path parser nge_strip_querystr reserve_data true preserve_key true [FILTER] name modify match ngenix add sensor ngenix@{{inventory_hostname_short}} #[OUTPUT] # name file # match ngenix # path /var/log # file fluent-bit.log [OUTPUT] name opensearch match ngenix host {{log_host}} port {{log_port}} tls on tls.verify {{ssl_verify}} http_user {{log_http_cfuser}} http_passwd {{log_http_cfpasswd}} index ngenix-all suppress_type_name on replace_dots on trace_error on buffer_size 50MB compress gzip