Мониторинг: Prometheus + Grafana

1global:
2  scrape_interval: 15s
3  evaluation_interval: 15s
4
5alerting:
6  alertmanagers:
7    - static_configs:
8        - targets: ['alertmanager:9093']
9
10rule_files:
11  - 'alert_rules.yml'
12
13scrape_configs:
14  - job_name: 'prometheus'
15    static_configs:
16      - targets: ['localhost:9090']
17
18  - job_name: 'node-exporter'
19    static_configs:
20      - targets: ['node-exporter:9100']
21
22  - job_name: 'app'
23    static_configs:
24      - targets: ['app:3000']
25    metrics_path: '/metrics'

1version: '3.8'
2services:
3  prometheus:
4    image: prom/prometheus
5    volumes:
6      - ./prometheus.yml:/etc/prometheus/prometheus.yml
7      - prometheus_data:/prometheus
8    ports:
9      - "9090:9090"
10
11  grafana:
12    image: grafana/grafana
13    environment:
14      - GF_SECURITY_ADMIN_PASSWORD=admin
15    volumes:
16      - grafana_data:/var/lib/grafana
17    ports:
18      - "3000:3000"
19
20  node-exporter:
21    image: prom/node-exporter
22    volumes:
23      - /proc:/host/proc:ro
24      - /sys:/host/sys:ro
25    command:
26      - '--path.procfs=/host/proc'
27      - '--path.sysfs=/host/sys'
28
29volumes:
30  prometheus_data:
31  grafana_data:

1const client = require('prom-client');
2
3// Сбор метрик
4const collectDefaultMetrics = client.collectDefaultMetrics;
5collectDefaultMetrics();
6
7// Кастомные метрики
8const httpRequestDuration = new client.Histogram({
9  name: 'http_request_duration_seconds',
10  help: 'Duration of HTTP requests',
11  labelNames: ['method', 'route', 'status'],
12  buckets: [0.01, 0.05, 0.1, 0.5, 1, 5],
13});
14
15// Эндпоинт /metrics
16app.get('/metrics', async (req, res) => {
17  res.set('Content-Type', client.register.contentType);
18  res.end(await client.register.metrics());
19});

1using Prometheus;
2
3var app = WebApplication.CreateBuilder(args);
4
5builder.Services.AddMetricServer();
6
7var counter = Metrics.CreateCounter("http_requests_total", "Total HTTP requests");
8var histogram = Metrics.CreateHistogram("http_request_duration", "Request duration");
9
10app.MapGet("/", async context =>
11{
12    counter.Inc();
13    using (histogram.NewTimer())
14    {
15        await context.Response.WriteAsync("Hello World!");
16    }
17});
18
19app.Run();

1# Средняя загрузка CPU
2rate(node_cpu_seconds_total{mode="user"}[5m])
3
4# Использование памяти
5node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100
6
7# HTTP запросы в секунду
8rate(http_requests_total[5m])
9
10# 95-й перцентиль времени ответа
11histogram_quantile(0.95, rate(http_request_duration_bucket[5m]))

1groups:
2  - name: alerts
3    rules:
4      - alert: HighCPU
5        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
6        for: 5m
7        labels:
8          severity: warning
9        annotations:
10          summary: "High CPU usage on {{ $labels.instance }}"
11
12      - alert: ServiceDown
13        expr: up == 0
14        for: 1m
15        labels:
16          severity: critical
17        annotations:
18          summary: "Service {{ $labels.job }} is down"

1global:
2  slack_api_url: 'https://hooks.slack.com/services/xxx'
3
4route:
5  receiver: 'slack-notifications'
6
7receivers:
8  - name: 'slack-notifications'
9    slack_configs:
10      - channel: '#alerts'
11        send_resolved: true