|
Größe: 15595
Kommentar:
|
Größe: 20970
Kommentar:
|
| Gelöschter Text ist auf diese Art markiert. | Hinzugefügter Text ist auf diese Art markiert. |
| Zeile 120: | Zeile 120: |
| geoip{ source => "clientip" } mutate { tags => [ "geoip" ] # 'coords' will be kept, 'tmplat' is temporary. # Both of these new fields are strings. add_field => [ "coords", "%{geoip.longitude}", "tmplat", "%{geoip.latitude}" ] } mutate { tags => [ "geoip" ] # Merge 'tmplat' into 'coords' merge => [ "coords", "tmplat" ] |
##### neu ### geoip { source => "clientip" target => "geoip" # database => "/etc/logstash/GeoLiteCity.dat" add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] |
| Zeile 136: | Zeile 129: |
| mutate { tags => [ "geoip" ] # Convert our new array of strings back to float convert => [ "coords", "float" ] # Delete our temporary latitude field remove => [ "tmplat" ] } |
mutate { convert => [ "[geoip][coordinates]", "float"] } ############# neu #### #### alt ### # geoip{ # source => "clientip" # } # mutate { # # unsupported in 5 tags => [ "geoip" ] # # 'coords' will be kept, 'tmplat' is temporary. # # Both of these new fields are strings. # add_field => [ "coords", "%{geoip.longitude}", # "tmplat", "%{geoip.latitude}" ] # } # mutate { # #unsupported in 5 tags => [ "geoip" ] # # Merge 'tmplat' into 'coords' # merge => [ "coords", "tmplat" ] # } # mutate { # # unsupported in 5 tags => [ "geoip" ] # # Convert our new array of strings back to float # convert => [ "coords", "float" ] # # Delete our temporary latitude field # remove_field => [ "tmplat" ] # } ## alt ### |
Config
Inhaltsverzeichnis
Beats
input {
beats {
port => 5044
}
}
output {
elasticsearch {
hosts => "labor05:9200"
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
Dynatrace
input {
file {
path => "/root/dt/Server/WUM96184/dtserver/6592/logs/Server.0.0.log"
sincedb_path => "/dev/null"
start_position => "beginning"
}
}
filter {
mutate {
gsub => [
# replace all forward slashes with underscore
"message", "\t", " "
]
}
multiline {
pattern => "^[^0-9]"
what => "previous"
}
grok {
patterns_dir => ["/usr/local/logstash-1.5.4/etc/dynatrace_patterns"]
match => { "message" => "%{DATESTAMP_DT:datestamp_dt} %{WORD:loglevel} \[%{DATA:logger}\]%{GREEDYDATA:logmessage}" }
}
date {
# 2016-08-22 04:18:05
match => [ "datestamp_dt", "YYYY-MM-dd HH:mm:ss" ]
target => "@timestamp"
}
#kv{}
#mutate {
# convert => { "Revenue" => "float" }
#}
}
output {
# if "_grokparsefailure" in [tags] {
# stdout { codec => "rubydebug" }
# }
if "_grokparsefailure" not in [tags] {
elasticsearch { }
}
}
Apache
input{
# file { "path" => "/data/teufelsmuehle/access_log" # "/data/apache2/other_vhosts_access_all.log"
file { "path" => "/root/apfelreich_access.log" # "/data/apache2/other_vhosts_access_all.log"
"tags" => "apfelreich"
"start_position" => "beginning"
"sincedb_path" => "/dev/null"
}
#stdin {}
}
filter{
grok {
match => [ "message", "%{COMBINEDAPACHELOG}" ]
# www.apfelreich.net:80 127.0.0.1 - - [31/May/2015:06:27:28 +0200] "GET /index.html HTTP/1.1" 200 454 "-" "monit/5.4"
}
mutate {
gsub => [
# remove ? from referrer
"referrer", "\?", " "]
}
mutate {
gsub => [
# remove & from referrer
"referrer", "\&", " "]
}
# key-value-match in referrer
kv{ source => "referrer"}
date { locale => "en" match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] }
mutate {
convert => { "bytes" => "integer" }
}
##### neu ###
geoip {
source => "clientip"
target => "geoip"
# database => "/etc/logstash/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
############# neu ####
#### alt ###
# geoip{
# source => "clientip"
# }
# mutate {
# # unsupported in 5 tags => [ "geoip" ]
# # 'coords' will be kept, 'tmplat' is temporary.
# # Both of these new fields are strings.
# add_field => [ "coords", "%{geoip.longitude}",
# "tmplat", "%{geoip.latitude}" ]
# }
# mutate {
# #unsupported in 5 tags => [ "geoip" ]
# # Merge 'tmplat' into 'coords'
# merge => [ "coords", "tmplat" ]
# }
# mutate {
# # unsupported in 5 tags => [ "geoip" ]
# # Convert our new array of strings back to float
# convert => [ "coords", "float" ]
# # Delete our temporary latitude field
# remove_field => [ "tmplat" ]
# }
## alt ###
}
output{
#stdout { codec => rubydebug }
elasticsearch { host => "labor05" }
}
Jobdb
input{
file { path => "/root/jobdb/bhist_1.txt"
sincedb_path => "/dev/null"
start_position => "beginning"
type => "jobdb" }
}
filter {
if [message] == "" {
drop { }
}
# if [message] == "------------------------------------------------------------------------------" {
# drop { }
# }
multiline {
pattern => "^\s"
what => "previous"
}
multiline {
pattern => "^\w"
what => "previous"
}
mutate {
gsub => [
"message", "\n", ""
]
gsub => [
"message", " ", ""
]
gsub => [
"message", "------------------------------------------------------------------------------", ""
]
}
grok {
# Job <671106>, Job Name <VW324-PF-ND_MB41-URhDHGV_140_0-D36_270-29-15-0L47_1_1>,User <user>, Project <PROJECT>,
match => { "message" => "Job <%{NUMBER:jobid}>, Job Name <%{DATA:job_name}>,User <%{WORD}>, Project <%{DATA:project}>,%{GREEDYDATA}%{DATESTAMP_LSF:submitdate}%{GREEDYDATA:afterdate}Submitted from host <%{DATA:submithost}>, to Queue <%{DATA:queue}>%{GREEDYDATA:rest}" }
# "Job <%{NUMBER:jobid}>, Job Name <%{DATA:job_name}>,User <%{WORD}>, Project <%{DATA:project}>,%{GREEDYDATA}%{DATESTAMP_LSF:creationdate}%{GREEDYDATA:afterdate}-a %{WORD:appl}%{GREEDYDATA:afterappl}%{DATESTAMP_LSF_SHORT:submitdate}: Submitted from host <%{DATA:submithost}>, to Queue <%{DATA:queue}>%{GREEDYDATA:rest}"
}
#kv { trimkey => '<>' field_split => ";, "}
}
output {
stdout { codec => "rubydebug" }
}
Jobdb-Mysql
input{
file { path => "/root/jobdb/all_since_2015-06-26.txt"
sincedb_path => "/dev/null"
start_position => "beginning"
type => "jobdb_mysql" }
}
filter {
if [message] == "jobid\tsubmitepoch\tdispatchepoch\tendepoch\trequeueepoch\tjob_name\tuser\tproject\tfirstqueue\tlastqueue\tsubmithost\tncpus\tappl\tappl_res\tmode\tresources\tmem\thostlist\tcpulimit\trunl
imit\tstatus\texit_reason\treturn_value\tappl_err_string\tnumber_of_susps\tsignal\tpendtime\tpsusptime\truntime\tususptime\tunkwntime\ttotaltime\tjobdep\tpost_job_process\terr_num\terr_time\tnet_pend\tcomment\
tssusptime\thangup\tprec\tfemzip\tjob_cmd_ofl\tanalysis\tcpcpus\tuserlib\tuserlib_name\tbatch\tpost\tpam_parts_gt0\tpam_inpchkerr\tpam_penkin_ne0\tpam_errterm\tfpm\tfixthick\tdscal_min\tmpilist\tmax_appl_disk_
use\n" {
# "jobid submitepoch dispatchepoch endepoch requeueepoch job_name user project firstqueue lastqueue submithost ncpus appl appl_res mode resources
mem hostlist cpulimit runlimit status exit_reason return_value appl_err_string number_of_susps signal pendtime psusptime runtime ususptime unkwntime
totaltime jobdep post_job_process err_num err_time net_pend comment ssusptime hangup prec femzip job_cmd_ofl analysis cpcpus userlib userlib_name batch
post pam_parts_gt0 pam_inpchkerr pam_penkin_ne0 pam_errterm fpm fixthick dscal_min mpilist max_appl_disk_use" {
drop { }
}
grok {
# Job <671106>, Job Name <VW324-PF-ND_MB41-URhDHGV_140_0-D36_270-29-15-0L47_1_1>,User <u0zhb27>, Project <5ZA1606-36505>,
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "%{NUMBER:jobid}\t%{DATESTAMP_MYSQL:[@metadata][timestamp]}\t%{DATESTAMP_MYSQL_OR_ZERODATE:dispatchdate}\t%{DATESTAMP_MYSQL_OR_ZERODATE:enddate}\t%{DATESTAMP_MYSQL_OR_ZERODATE:requeuedate}\t%{DATA:jobname}\t%{DATA:user}\t%{DATA:project}\t%{DATA:firstqueue}\t%{DATA:lastqueue}\t%{DATA:submithost}\t%{DATA:ncpus}\t%{DATA:appl}\t%{DATA:appl_res}\t%{DATA:mode}\t%{DATA:ressources}\t%{DATA:mem}\t%{DATA:hostlist}\t%{DATA:cpulimit}\t%{DATA:runlimit}\t%{DATA:status}\t%{DATA:exit_reason}\t%{DATA:return_value}\t%{DATA:appl_err_string}\t%{DATA:number_of_susps}\t%{DATA:signal}\t%{DATA:pendtime}\t%{DATA:psusptime}\t%{DATA:runtime}\t%{DATA:ususptime}\t%{DATA:unkwntime}\t%{DATA:totaltime}\t%{DATA:jobdep}\t%{DATA:post_job_process}\t%{DATA:err_num}\t%{DATA:err_time}\t%{DATA:net_pend}\t%{DATA:comment}\t%{DATA:ssusptime}\t%{DATA:hangup}\t%{DATA:prec}\t%{DATA:femzip}\t%{DATA:job_cmd_ofl}\t%{DATA:analysis}\t%{DATA:cpcpus}\t%{DATA:userlib}\t%{DATA:userlib_name}\t%{DATA:batch}\t%{DATA:post}\t%{DATA:pam_parts_gt0}\t%{DATA:pam_inpchkerr}\t%{DATA:pam_penkin_ne0}\t%{DATA:pam_errterm}\t%{DATA:fpm}\t%{DATA:fixthick}\t%{DATA:dscal_min}\t%{DATA:mpilist}\t%{GREEDYDATA:max_appl_disk_use}"}}
# %{GREEDYDATA:rest}"
date {
locale => "en" # enforcing locale as date relies on the systems setting
timezone => "UTC" # as machine clock is in UTC
# 2015-06-26 00:00:09
match => [ "[@metadata][timestamp]", "YYYY-MM-dd HH:mm:ss" ] # updating directly the metadata
}
if [max_appl_disk_use] == "NULL" {
mutate{ replace => { "max_appl_disk_use" => "0" }}
}
mutate {
convert => { "ncpus" => "integer" }
convert => { "mem" => "integer" }
convert => { "cpulimit" => "integer" }
convert => { "runlimit" => "float" }
convert => { "number_of_susps" => "integer" }
convert => { "pendtime" => "integer" }
convert => { "psusptime" => "integer" }
convert => { "runtime" => "integer" }
convert => { "ususptime" => "integer" }
convert => { "unkwntime" => "integer" }
convert => { "totaltime" => "integer" }
convert => { "err_time" => "integer" }
convert => { "net_pend" => "integer" }
convert => { "ssusptime" => "integer" }
convert => { "cpcpus" => "integer" }
convert => { "max_appl_disk_use" => "integer" }
}
ruby {code => "event['core_h'] = event['ncpus'].to_f * event['runtime'].to_f / '3600'.to_f " }
#kv { trimkey => '<>' field_split => ";, "}
}
output {
# stdout { codec => "rubydebug" }
if "_grokparsefailure" not in [tags] {
elasticsearch { node_name => "jobdb" document_id => "%{jobid}"}
}
}
SMB
input {
# stdin { }
file {
path => "/var/log/samba/log.10*"
sincedb_path => "/dev/null" # remove for production
start_position => "beginning" # remove for production
type => "samba"
codec => plain { charset => "ISO-8859-1" } # necessary for ancient windows encoding
}
}
filter {
multiline {
pattern => "^\s" # line beginning with whitespace...
what => "previous" # ... belong to the previous line
}
mutate {
gsub => [
# remove linefeed
"message", "\n", ""]
}
grok{
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "\[%{SAMBADATE:[@metadata][timestamp]}\, %{INT:loglevel}\] %{PROG:process}:%{POSINT:pid}\(%{PROG:function}\) %{GREEDYDATA:rest}" }
}
date {
locale => "en" # enforcing locale as date relies on the systems setting
timezone => "UTC" # as machine clock is in UTC
match => [ "[@metadata][timestamp]", "YYYY-MM-dd HH:mm:ss,SSS", "YYYY/MM/dd HH:mm:ss.SSSSSS" ] # updating directly the metadata
}
if "service" in [rest]{
grok {
match => { "rest" => "%{DATA} service %{GREEDYDATA:service}" }
}
}
}
output {
# stdout { codec => rubydebug { } }
if "_grokparsefailure" not in [tags] { # don't consider lines without log message
elasticsearch { node_name => "samba" workers => 2 }
#host => "elkstack01" protocol => "http" user =>"es_admin" password => "password" } # name of this nodes & where to send data
}
}
SPM
input {
#stdin { }
file {
path => "/var/log/spm/*.log"
sincedb_path => "/dev/null" # remove for production
start_position => "beginning" # remove for production
type => "spm"
# #codec => plain { charset => "ISO-8859-1" } # necessary for ancient windows encoding
}
}
filter {
multiline {
pattern => "^\s" # line beginning with whitespace...
what => "previous" # ... belong to the previous line
}
multiline {
pattern => ".*---------------------- update resources ----------------------.*"
what => "next"
}
mutate {
gsub => [
# remove linefeed
"message", "\n", ""]
}
grok{
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "\[%{WORD:loglevel}%{SPACE}\] %{SPMDATE:[@metadata][timestamp]} \: %{DATA:logmessage} \[%{SENDER:sender}\]" }
}
date {
locale => "en" # enforcing locale as date relies on the systems setting
timezone => "UTC" # as machine clock is in UTC
match => [ "[@metadata][timestamp]", "YYYY-MM-dd HH:mm:ss,SSS", "YYYY/MM/dd HH:mm:ss.SSSSSS" ] # updating directly the metadata
}
}
output {
# stdout { codec => rubydebug { } }
if "_grokparsefailure" not in [tags] { # don't consider lines without log message
elasticsearch { node_name => "spm" }
# host => ["elkstack01", "elkstack02"] protocol => "http" user =>"es_admin" password => "password"} # name of this nodes & where to send data
}
}
Pattern
jobdb
DATESTAMP_LSF %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
DATESTAMP_LSF_SHORT %{DAY} %{MONTH} %{MONTHDAY} %{TIME}
DATESTAMP_MYSQL %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}
ZERODATE 0000-00-00 00:00:00
DATESTAMP_MYSQL_OR_ZERODATE (?:%{DATESTAMP_MYSQL}|%{ZERODATE})
smb
SAMBADATE %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}
spm
SPMDATE %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}
SENDER %{WORD}::%{DATA}
Dynatrace
DATESTAMP_DT (%{YEAR}-%{MONTHNUM2}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND})