How to collect container log using filebeat

Installation

Docker

Pull image

1
docker pull docker.elastic.co/beats/filebeat:7.9.2

Example configuration file

1
curl -L -O https://raw.githubusercontent.com/elastic/beats/7.9/deploy/docker/filebeat.docker.yml

Run with docker-compose

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
version: "3"

services:
filebeat:
container_name: filebeat
image: docker.elastic.co/beats/filebeat:7.9.0
network_mode: bridge
restart: always
user: root
command: ["--strict.perms=false"]
environment:
- ELASTICSEARCH_HOST=${ELASTIC_HOST}
- ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
- ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
volumes:
- ./filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock:ro

Configuration

可以參考 example configuration file

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

filebeat.autodiscover:
providers:
- type: docker
hints.enabled: true

processors:
- add_cloud_metadata: ~

output.elasticsearch:
hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'
username: '${ELASTICSEARCH_USERNAME:}'
password: '${ELASTICSEARCH_PASSWORD:}'

Custom configuration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
## 開啟 auto reload 的功能 (非必要)
reload.enabled: true

# 設定 auto discover 可以參考,複雜的設定在此
## https://www.elastic.co/guide/en/beats/filebeat/current/configuration-autodiscover.html
filebeat.autodiscover:
providers:
- type: docker
templates:
- condition:
## 如果有 filebeat 的 label,filebeat 才會收集 log
has_fields: ["docker.container.labels.filebeat"]
config:
- type: container
paths:
- /var/lib/docker/containers/${data.docker.container.id}/*.log
exclude_lines: ["^\\s+[\\-`('.|_]"]

# 可以設定自己的 fields
fields:
env: prod

# 設定 log rotate
setup.template.name: "svc"
setup.template.pattern: "svc-*"
setup.ilm.enabled: true
setup.ilm.policy_name: "rotation-logs"
setup.ilm.rollover_alias: "svc"

setup.kibana:
host: '${KIBANA_HOST}'
username: '${KIBANA_USERNAME}'
password: '${KIBANA_PASSWORD}'

processors:
## 加入 host metadata
- add_host_metadata: ~
## 加入 cloud metadata
- add_cloud_metadata: ~
## 設定如何 decode json
- decode_json_fields:
fields: ["message"]
process_array: false
max_depth: 3
target: ""
overwrite_keys: false
add_error_key: true

output.elasticsearch:
hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}'
username: '${ELASTICSEARCH_USERNAME:}'
password: '${ELASTICSEARCH_PASSWORD:}'
index: "svc-%{[agent.version]}-%{+yyyy.MM.dd}"