Commit 3902286e authored by Dmytro Karpenko's avatar Dmytro Karpenko
Browse files

Initial import of logstash4arc files.

parent e6954899
This is the simple configuration and patterns files
for Logstash to parse and send ARC logs to an ES instance.
Feel free to modify and tailor it to your purposes.
arc_logs.conf can be put in /etc/logstash/conf.d
arc.patterns -- into /etc/logstash/patterns
In ARC 5.3.2+ it is possible to enable milliseconds granularity
in the logs, by inserting
export ARC_LOGGER_TIME_FORMAT=ELASTIC
into files under /etc/default/ or in the init scripts of ARC services.
This improves the order of the loglines in ES.
Please contact NT1 project (https://neic.no/activities/nt1/)
if you don't have your ES instance and want to use its.
# custom patterns for ARC
#
ARC_LOG_LINE \[%{TIMESTAMP_ISO8601:arc_timestamp}\] \[%{USERNAME:component}\] \[%{LOGLEVEL:level}\] \[%{POSINT:pid}/%{POSINT:thread}\] %{GREEDYDATA:msg}
GMJOBS_JOB_STATUS (?:Started|Finished)
ARC_JOB_ID (?:[a-zA-Z0-9]{54})
GMJOBS_LOG_LINE %{TIMESTAMP_ISO8601:arc_timestamp} %{GMJOBS_JOB_STATUS:status} - job id: %{ARC_JOB_ID:jobid}, unix user: %{POSINT:uid}:%{POSINT:gid}, name: \"%{DATA:jobname}\", owner: \"%{DATA:owner}\", lrms: %{USERNAME:lrms}, queue: %{USERNAME:queue}(, lrmsid: %{POSINT:lrmsid})?(, failure: \"%{DATA:failure_reason}\")?
PERFLOG_DURATION (?:[0-9]?*\.[0-9]?*)
BACKENDS_PERFLOGLINE \[%{TIMESTAMP_ISO8601:arc_timestamp}\] %{USERNAME:backend_script}, %{GREEDYDATA:backend_action}:\s*%{PERFLOG_DURATION:duration}
SYSTEM_PERFLOGLINE \[%{TIMESTAMP_ISO8601:arc_timestamp}\] %{DATA:metric}: %{GREEDYDATA:stats}
input {
file {
path => [ "/var/log/arc/grid-manager.log", "/var/log/arc/gridftpd.log", "/var/log/arc/gm-jobs.log", "/var/log/arc/ws-interface.log", "/var/log/arc/perfdata/*.perflog" ]
# data.perflog is inconsistent with the rest of perflogs
# - timestamp in UTC (localtime for others)
# - duration in nanoseconds (seconds for others)
# Thus excluding for now.
exclude => [ "data.perflog" ]
sincedb_path => "/var/log/arc/sincedb-arc"
# uncomment next line if you want to import existing data
# start_position => "beginning"
type => "arclog"
}
}
filter {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "%{ARC_LOG_LINE}" ]
match => [ "message", "%{GMJOBS_LOG_LINE}" ]
match => [ "message", "%{BACKENDS_PERFLOGLINE}" ]
match => [ "message", "%{SYSTEM_PERFLOGLINE}" ]
named_captures_only => true
remove_field => [ "message" ]
tag_on_failure => [ "_parse_arclog_failure00" ]
}
if [failure_reason] {
mutate {
replace => { "status" => "Failed" }
}
}
date {
match => [ "arc_timestamp", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSS" ]
timezone => "CET"
remove_field => [ "arc_timestamp" ]
}
}
output {
elasticsearch {
hosts => [ "arc-es-test.ndgf.org:9200" ]
index => "arclogs"
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment