Логирование: ELK Stack и Loki

1version: '3.8'
2services:
3  elasticsearch:
4    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
5    environment:
6      - discovery.type=single-node
7      - xpack.security.enabled=false
8    ports:
9      - "9200:9200"
10    volumes:
11      - es_data:/usr/share/elasticsearch/data
12
13  logstash:
14    image: docker.elastic.co/logstash/logstash:8.11.0
15    volumes:
16      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
17    depends_on:
18      - elasticsearch
19
20  kibana:
21    image: docker.elastic.co/kibana/kibana:8.11.0
22    ports:
23      - "5601:5601"
24    depends_on:
25      - elasticsearch
26
27volumes:
28  es_data:

1# logstash.conf
2input {
3  beats {
4    port => 5044
5  }
6  
7  tcp {
8    port => 5000
9    codec => json
10  }
11}
12
13filter {
14  if [type] == "nginx" {
15    grok {
16      match => { "message" => "%{COMBINEDAPACHELOG}" }
17    }
18    date {
19      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
20    }
21  }
22  
23  mutate {
24    remove_field => ["timestamp"]
25  }
26}
27
28output {
29  elasticsearch {
30    hosts => ["elasticsearch:9200"]
31    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
32  }
33}

1version: '3.8'
2services:
3  loki:
4    image: grafana/loki:2.9.0
5    ports:
6      - "3100:3100"
7    volumes:
8      - ./loki-config.yml:/etc/loki/local-config.yaml
9
10  promtail:
11    image: grafana/promtail:2.9.0
12    volumes:
13      - /var/log:/var/log
14      - ./promtail-config.yml:/etc/promtail/config.yml
15
16  grafana:
17    image: grafana/grafana:10.2.0
18    ports:
19      - "3000:3000"
20    environment:
21      - GF_SECURITY_ADMIN_PASSWORD=admin

1# promtail-config.yml
2server:
3  http_listen_port: 9080
4
5positions:
6  filename: /tmp/positions.yaml
7
8clients:
9  - url: http://loki:3100/loki/api/v1/push
10
11scrape_configs:
12  - job_name: system
13    static_configs:
14      - targets:
15          - localhost
16        labels:
17          job: system
18          __path__: /var/log/*.log
19  
20  - job_name: nginx
21    static_configs:
22      - targets:
23          - localhost
24        labels:
25          job: nginx
26          __path__: /var/log/nginx/access.log
27    pipeline_stages:
28      - regex:
29          expression: '^(?P<ip>\S+) .* "(?P<method>\S+) (?P<path>\S+) .* (?P<status>\d+)'
30      - labels:
31          method:
32          status:

1# Поиск по job
2{job="nginx"}
3
4# Поиск по тексту
5{job="nginx"} |= "error"
6
7# Регулярное выражение
8{job="nginx"} |~ "status=5[0-9]{2}"
9
10# Метрики
11rate({job="nginx"} |= "error" [5m])
12
13# Агрегация
14sum(rate({job="nginx"} | json [5m])) by (status)

1const pino = require('pino');
2const logger = pino({
3  level: 'info',
4  formatters: {
5    level: (label) => ({ level: label }),
6  },
7  timestamp: pino.stdTimeFunctions.isoTime,
8});
9
10logger.info({ userId: 123, action: 'login' }, 'User logged in');