Logstash日志收集

调整Logstash JVM

[root@db03 ~]# vim /etc/logstash/jvm.options
-Xms256m
-Xmx512m
Logstash配置
[root@db03 logstash]# grep '^[a-z]' logstash.yml
path.data: /var/lib/logstash
path.config: /etc/logstash/conf.d
path.logs: /var/log/logstash

将message日志收集到指定文件中

[root@db03 conf.d]# cat messages_to_file.conf 
input {
  file{
    type => "system_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }     
}

output{
  file{
    path => "/tmp/messages_%{+YYYY.MM.dd}"
  }
}

# 检测语法
## 检测语法
[root@db03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/messages_to_file.conf -t

## 启动logstash
[root@db03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/messages_to_file.conf &

Logstash收集多日志到文件中

[root@db03 conf.d]# cat system_to_file.conf 
input{
  file{
    type => "msg_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file{
    type => "sec_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}

output{
  if [type]  == "msg_log" {
    file{
       path => "/tmp/msg_%{+YYYY.MM.dd}"
    }
  }else if [type] == "sec_log" {
    file{
      path => "/tmp/sec_%{+YYYY.MM.dd}"
    }
  }
}

Logstash收集多个日志到ES中

[root@db03 conf.d]# cat system_to_es.conf 
input{
  file{
    type => "msg_log"
    path => "/var/log/messages"
    start_position => "beginning"
  }
  file{
    type => "sec_log"
    path => "/var/log/secure"
    start_position => "beginning"
  }
}

output{
    elasticsearch{
       hosts => ["10.0.0.51:9200"]
       index => "%{type}_%{+YYYY.MM.dd}"
    }
}

Logstash收集JAVA日志

# 1.启动tomcat
[root@db04 ~]# /app/tomcat/bin/startup.sh

# 2.编辑logstash配置文件
[root@db04 conf.d]# cat tomcat_to_es.conf 
input{
  file{
    type => "tomcat_catalina"
    path => "/app/tomcat/logs/catalina.out"
    start_position => "beginning"
  }
  file{
    type => "tomcat_access"
    path => "/app/tomcat/logs/localhost_access_log.*.txt"
    start_position => "beginning"

  }
}

output{
  elasticsearch{
    hosts => ["10.0.0.51:9200"]
    index => "%{type}_%{+YYYY.MM.dd}"
  }
}
# 3.检查语法
[root@db04 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_to_es.conf -t

# 4.启动logstash
[root@db04 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_to_es.conf &

将tomcat日志修改成json格式

[root@db04 conf.d]# vim /app/tomcat/conf/server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="access" suffix=".log"
               pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot
;AccessTime&quot;:&quot;%t&quot;,&quot;method&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Q
uery?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;AgentVersion&quot;:&quot;%{User-Agent}i&quot;}" />

收集tomcat json日志

[root@db04 conf.d]# vim tomcat_json_es.conf 
input{
  file{
    type => "tomcat_json"
    path => "/app/tomcat/logs/access.*.log"
    start_position => "beginning"

  }
}

output{
  elasticsearch{
    hosts => ["10.0.0.51:9200"]
    index => "%{type}_%{+YYYY.MM.dd}"
    codec => "json"
  }
}

[root@db04 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_json_es.conf -t
[root@db04 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_json_es.conf &

这个json不是我们想要的json 我们需要给logstash加规则

我们需要把获取到message中的KEY：VALUE将他解析成键值对的形式，展现出来

# 修改配置文件
	
#在Logstash的配置文件中，添加filter过滤规则
filter {
  json {
    source => "message"
    remove_field => ["message"]  # 删除message列
  }
}
#重新启动Logstash
[root@db04 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_json_es.conf &

去除黄色感叹号

黄色感叹号没有了

非JSON格式缺陷

1.无法画图

2.日志格式不清晰

3.搜索

Logstash收集nginx日志

修改nginx日志格式

# 在nginx主配置文件配置
[root@db04 conf.d]# vim /etc/nginx/nginx.conf
log_format access_json '{"@timestamp":"$time_iso8601",'
            '"host":"$server_addr",'
            '"clientip":"$remote_addr",'
            '"size":$body_bytes_sent,'
            '"responsetime":$request_time,'
            '"upstreamtime":"$upstream_response_time",'
            '"upstreamhost":"$upstream_addr",'
            '"http_host":"$host",'
            '"url":"$uri",'
            '"domain":"$host",'
            '"xff":"$http_x_forwarded_for",'
            '"referer":"$http_referer",'
            '"status":"$status"}';

# 在子配置文件添加这一行
[root@db04 conf.d]# cat /etc/nginx/conf.d/blog.abc.com.conf
access_log  /var/log/nginx/blog_abc_com_access.log  access_json;

编辑Logstash配置

[root@db04 conf.d]# cat nginx_to_es.conf 
input{
  file{
    type => "blog_abc_com_access"
    path => "/var/log/nginx/blog_abc_com_access.log"
    start_position => "beginning"
  }
}

filter {
  json {
    source => "message"
    remove_field => ["message"]  # 删除message列
  }
}

output{
  elasticsearch{
    hosts => ["10.0.0.51:9200"]
    index => "%{type}_%{+YYYY.MM.dd}"
    codec => "json"
  }
}

# 启动
[root@db04 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_to_es.conf &

原文地址：http://www.cnblogs.com/bloglee/p/16787779.html