ELFK-实时抓取NGINX日志

1、前言

本篇博客记录nginx访问日志、错误日志，通过filebeat进行数据结构化处理，输出到elasticsearch中，最后通过kibaka进行展示的完整过程，EK的安装及配置方法，请参考：ELK-Stack简介及安装手册

2、部署环境介绍

平台	IP	用途	E版本	F版本	K版本
CentOS 6.7 64Bit	192.168.1.241	ES+Cerebro+Kibana	6.7.0		6.7.0
CentOS 6.7 64Bit	192.168.1.130	Filebeat		6.7.0

3、Filebeat的安装与主配置文件详解

3.1、安装Filebeat

[root@test1 ~]# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.3.0-x86_64.rpm
[root@test1 ~]# rpm -vi filebeat-6.3.0-x86_64.rpm
[root@test1 ~]# chkconfig filebeat on
[root@test1 ~]# rpm -ql filebeat
    #filebeat启动脚本
    /usr/bin/filebeat
    /etc/init.d/filebeat
    #主程序路径
    /usr/share/filebeat/
    #filebeat模块配置文件路径
    /usr/share/filebeat/module
    #主程序安装路径
    /usr/share/kibana/
    #主配置文件路径
    /etc/filebeat
    #模块配置文件夹引用路径
    /etc/filebeat/modules.d/

3.2、主配置文件详解：/etc/filebeat/filebeat.yml

#============================= Filebeat modules ===============================

filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml
  # Set to true to enable config reloading
  reload.enabled: false
  # Period on which files under path should be checked for changes
  reload.period: 10s
#==================== Elasticsearch template setting ==========================
#如修改了默认索引名，此处必须配置
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.fields: "/etc/filebeat/fields.yml"
setup.template.overwrite: "true"
#kiabna展示默认索引名
setup.dashboards.index: "nginx-*"
setup.template.settings:
  index.number_of_shards: 2
  #index.codec: best_compression
  #_source.enabled: false
#============================== Kibana =====================================
setup.kibana:
  host: "192.168.1.241:5601"
#================================ Outputs =====================================
#-------------------------- Elasticsearch output ------------------------------
#when.contains配置判断输入字段是否包含特定内容，并创建特定的索引，主要用来区分日志
output.elasticsearch:
  hosts: ["192.168.1.241:9200"]
  index: "defaults-logs-%{+yyyy.MM.dd}"
  indices:
    - index: "nginx-130-errorlogs-%{+yyyy.MM.dd}"
      when.contains:
        fileset.name: "error"
    - index: "nginx-130-accesslogs-%{+yyyy.MM.dd}"
      when.contains:
        fileset.name: "access"
  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"
#多个工作线程开启负载模式  
  worker: "2"

3.3、开启filebeat nginx 模块，支持过滤日志

1	[root@test1 ~]# filebeat modules enable nginx

3.4、载入filebeat自带的仪表盘和可视化代码，通过kibana展示

1	[root@test1 ~]# filebeat setup --dashboards

3.5、修改filebeat nginx模块配置文件，input数据

[root@test1 ~]# vim /etc/filebeat/modules.d/nginx.yml
- module: nginx
  # Access logs
  access:
    enabled: true
    #新建access目录用来区分日志
    var.paths: ["/usr/local/nginx/logs/access/*"]

  error:
    enabled: true
    #新建error目录用来区分日志
    var.paths: ["/usr/local/nginx/logs/error/*"]

3.6、修改filebeat nginx模块access默认配置文件用来匹配自定义日志

#获取自定义NGINX日志格式
     log_format access '$remote_addr - $remote_user [$time_local] "$server_name" "$request" '
          '$status $body_bytes_sent "$request_body" "$http_referer" '
          '"$http_user_agent" "$http_x_forwarded_for" '
          '$connection $upstream_addr '
          '$upstream_response_time $request_time ';
          
#对模块源码进行调整，匹配自定义NGINX格式，注意数据类型
[root@test1 ~]# vim /usr/share/filebeat/module/nginx/access/ingest/default.json
"patterns":[
            "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.servername}\" \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.request_body}\" \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\" \"%{DATA:nginx.access.http_x_forwarded_for}\" %{NUMBER:nginx.access.connection} %{DATA:nginx.access.upstream_addr} %{DATA:nginx.access.upstream_response_time} %{DATA:nginx.access.request_time}"]

#默认的nginx模块只支持默认日志格式，如有变动需要在此处新增字段名，文章过长，只展示新增部分
[root@test1 ~]# vim /etc/filebeat/fields.yml
- key: nginx
  title: "Nginx"
  description: >
    Module for parsing the Nginx log files.
  short_config: true
  fields:
    ......
            - name: servername
              type: keyword
              description: >
                The http servername
            - name: request_body
              type: text
              format: bytes
              description: >
                The http request_body   
            - name: referrer
              type: keyword
              description: >
                The HTTP referrer.
            - name: http_x_forwarded_for
              type: text
              description: >
                The http http_x_forwarded_for  
            - name: connection
              type: keyword
              example: GET
              description: >
                The http connection
            - name: upstream_addr
              type: keyword
              example: GET
              description: >
                The http upstream_addr
            - name: upstream_response_time
              type: keyword
              example: GET
              description: >
                The http upstream_response_time
            - name: request_time
              type: keyword
              example: GET
              description: >
                The http request_time              
......

3.7、修改filebeat nginx模块error默认配置文件用来解决时区问题

[root@test1 ~]#vim /usr/share/filebeat/module/nginx/error/ingest
    "date": {
      "field": "nginx.error.time",
      "target_field": "@timestamp",
      "formats": ["YYYY/MM/dd H:m:s"],
      "timezone": "Asia/Shanghai"
    }