Merge pull request #24 from igor04091968/codex/-e2e-grafana-+-prometheus
Harden Grafana Prometheus e2e readiness
This commit is contained in:
IgorRachkov
@@ -4,4 +4,9 @@ GRAFANA_ADMIN_PASSWORD=change_me_now
|
|||||||
GRAFANA_PORT=3000
|
GRAFANA_PORT=3000
|
||||||
PROMETHEUS_PORT=9090
|
PROMETHEUS_PORT=9090
|
||||||
SQL_EXPORTER_PORT=9399
|
SQL_EXPORTER_PORT=9399
|
||||||
|
AW_EXPORTER_PORT=9398
|
||||||
|
AW_SERVER_HOST=10.10.10.13
|
||||||
|
AW_SERVER_PORT=5600
|
||||||
|
AW_SERVER_SCHEME=http
|
||||||
|
AW_SCRAPE_INTERVAL_SECONDS=30
|
||||||
ONEC_DSN=postgres://onec_reader:change_me@10.10.10.20:5432/onec_db?sslmode=disable
|
ONEC_DSN=postgres://onec_reader:change_me@10.10.10.20:5432/onec_db?sslmode=disable
|
||||||
|
|||||||
+29
-21
@@ -3,44 +3,46 @@
|
|||||||
Готовый каркас для непрерывного сбора KPI из 1С и анализа в Grafana:
|
Готовый каркас для непрерывного сбора KPI из 1С и анализа в Grafana:
|
||||||
|
|
||||||
- `sql-exporter` читает SQL-представления KPI из БД 1С;
|
- `sql-exporter` читает SQL-представления KPI из БД 1С;
|
||||||
|
- `aw-exporter` собирает метрики ActivityWatch и отдает их Prometheus;
|
||||||
- `prometheus` собирает метрики и применяет alert-rules;
|
- `prometheus` собирает метрики и применяет alert-rules;
|
||||||
- `grafana` поднимает datasource и дашборд автоматически.
|
- `grafana` поднимает datasource и дашборд автоматически.
|
||||||
|
|
||||||
## Полные пути
|
## Полные пути
|
||||||
|
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/.env.example`
|
- `./.env.example`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/docker-compose.yml`
|
- `./docker-compose.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql-exporter/sql_exporter.yml`
|
- `./sql-exporter/sql_exporter.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql-exporter/collectors/onec_accounting_kpi.collector.yml`
|
- `./sql-exporter/collectors/onec_accounting_kpi.collector.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/prometheus.yml`
|
- `./prometheus/prometheus.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/alerts.yml`
|
- `./prometheus/alerts.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/recording_rules.yml`
|
- `./prometheus/recording_rules.yml`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/grafana/dashboards/1c-accounting-overview.json`
|
- `./grafana/dashboards/1c-accounting-overview.json`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/grafana/dashboards/1c-accounting-sre.json`
|
- `./grafana/dashboards/1c-accounting-sre.json`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/postgres_views_template.sql`
|
- `./sql/postgres_views_template.sql`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/mssql_views_template.sql`
|
- `./sql/mssql_views_template.sql`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/discover_postgres_1c.sh`
|
- `./tools/discover_postgres_1c.sh`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/validate_kpi_views.sh`
|
- `./tools/validate_kpi_views.sh`
|
||||||
- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline.sh`
|
- `./tools/check_pipeline.sh`
|
||||||
|
|
||||||
## Быстрый запуск
|
## Быстрый запуск
|
||||||
|
|
||||||
1. Подготовьте env:
|
1. Подготовьте env:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c
|
cd grafana-1c
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
```
|
```
|
||||||
|
|
||||||
2. В `.env` задайте:
|
2. В `.env` задайте:
|
||||||
|
|
||||||
- `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`;
|
- `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`;
|
||||||
- `ONEC_DSN` (DSN read-only пользователя в БД 1С).
|
- `ONEC_DSN` (DSN read-only пользователя в БД 1С);
|
||||||
|
- при необходимости `AW_SERVER_HOST`, `AW_SERVER_PORT`, `AW_SERVER_SCHEME`, `AW_EXPORTER_PORT` и `AW_SCRAPE_INTERVAL_SECONDS` для ActivityWatch exporter.
|
||||||
|
|
||||||
3. В БД 1С создайте KPI-представления:
|
3. В БД 1С создайте KPI-представления:
|
||||||
|
|
||||||
- для PostgreSQL возьмите `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/postgres_views_template.sql`;
|
- для PostgreSQL возьмите `./sql/postgres_views_template.sql`;
|
||||||
- для MS SQL возьмите `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/mssql_views_template.sql`.
|
- для MS SQL возьмите `./sql/mssql_views_template.sql`.
|
||||||
|
|
||||||
4. Поднимите стек:
|
4. Поднимите стек:
|
||||||
|
|
||||||
@@ -52,7 +54,9 @@ docker compose up -d
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsS http://127.0.0.1:9399/metrics | head
|
curl -fsS http://127.0.0.1:9399/metrics | head
|
||||||
|
curl -fsS http://127.0.0.1:9398/metrics | head
|
||||||
curl -fsS http://127.0.0.1:9090/-/healthy
|
curl -fsS http://127.0.0.1:9090/-/healthy
|
||||||
|
curl -fsS http://127.0.0.1:3000/api/health
|
||||||
```
|
```
|
||||||
|
|
||||||
Откройте Grafana: `http://<host>:3000`.
|
Откройте Grafana: `http://<host>:3000`.
|
||||||
@@ -62,23 +66,25 @@ curl -fsS http://127.0.0.1:9090/-/healthy
|
|||||||
Профилирование структуры 1С (PostgreSQL):
|
Профилирование структуры 1С (PostgreSQL):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/discover_postgres_1c.sh \
|
sh ./tools/discover_postgres_1c.sh \
|
||||||
"postgres://user:pass@db-host:5432/db?sslmode=disable"
|
"postgres://user:pass@db-host:5432/db?sslmode=disable"
|
||||||
```
|
```
|
||||||
|
|
||||||
Проверка KPI views:
|
Проверка KPI views:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/validate_kpi_views.sh \
|
sh ./tools/validate_kpi_views.sh \
|
||||||
"postgres://user:pass@db-host:5432/db?sslmode=disable"
|
"postgres://user:pass@db-host:5432/db?sslmode=disable"
|
||||||
```
|
```
|
||||||
|
|
||||||
Проверка end-to-end пайплайна:
|
Проверка end-to-end пайплайна:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline.sh
|
sh ./tools/check_pipeline.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Скрипт проверяет полный путь сбора данных: `sql-exporter` и `aw-exporter` отдают обязательные метрики, Prometheus успешно выполняет запросы по scrape-targets, а Grafana отвечает на health/API, видит datasource `prometheus` и provisioned dashboards. Если стек запущен не из каталога репозитория, передайте путь к каталогу `grafana-1c` первым аргументом.
|
||||||
|
|
||||||
## Что контролируется
|
## Что контролируется
|
||||||
|
|
||||||
- Непроведенные документы (`onec_unposted_documents_total`)
|
- Непроведенные документы (`onec_unposted_documents_total`)
|
||||||
@@ -86,6 +92,8 @@ sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline.
|
|||||||
- Просроченная дебиторка (`onec_overdue_receivables_total`)
|
- Просроченная дебиторка (`onec_overdue_receivables_total`)
|
||||||
- Ошибки проведения за 24ч (`onec_posting_errors_total`)
|
- Ошибки проведения за 24ч (`onec_posting_errors_total`)
|
||||||
- Свежесть данных из 1С (`onec_data_freshness_seconds`)
|
- Свежесть данных из 1С (`onec_data_freshness_seconds`)
|
||||||
|
- Доступность ActivityWatch API (`aw_up`)
|
||||||
|
- Количество bucket/events ActivityWatch (`aw_buckets_total`, `aw_bucket_events_count`)
|
||||||
|
|
||||||
## Принципы безопасности
|
## Принципы безопасности
|
||||||
|
|
||||||
|
|||||||
@@ -6,14 +6,15 @@ services:
|
|||||||
container_name: awrus-aw-exporter
|
container_name: awrus-aw-exporter
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
- AW_SERVER_HOST=10.10.10.13
|
- AW_SERVER_HOST=${AW_SERVER_HOST:-10.10.10.13}
|
||||||
- AW_SERVER_PORT=5600
|
- AW_SERVER_PORT=${AW_SERVER_PORT:-5600}
|
||||||
- AW_SERVER_SCHEME=http
|
- AW_SERVER_SCHEME=${AW_SERVER_SCHEME:-http}
|
||||||
- EXPORTER_PORT=9398
|
- EXPORTER_PORT=9398
|
||||||
|
- SCRAPE_INTERVAL_SECONDS=${AW_SCRAPE_INTERVAL_SECONDS:-30}
|
||||||
volumes:
|
volumes:
|
||||||
- ./sql-exporter/collectors/aw_activitywatch.py:/app/aw_activitywatch.py:ro
|
- ./sql-exporter/collectors/aw_activitywatch.py:/app/aw_activitywatch.py:ro
|
||||||
ports:
|
ports:
|
||||||
- "9398:9398"
|
- "${AW_EXPORTER_PORT:-9398}:9398"
|
||||||
command:
|
command:
|
||||||
- "python3"
|
- "python3"
|
||||||
- "/app/aw_activitywatch.py"
|
- "/app/aw_activitywatch.py"
|
||||||
|
|||||||
@@ -1,121 +1,209 @@
|
|||||||
{
|
{
|
||||||
"dashboard": {
|
"title": "ActivityWatch Overview",
|
||||||
"title": "ActivityWatch Overview",
|
"tags": [
|
||||||
"tags": ["activitywatch", "monitoring"],
|
"activitywatch",
|
||||||
"timezone": "browser",
|
"monitoring"
|
||||||
"panels": [
|
],
|
||||||
{
|
"timezone": "browser",
|
||||||
"id": 1,
|
"panels": [
|
||||||
"title": "Total Buckets",
|
{
|
||||||
"type": "stat",
|
"id": 1,
|
||||||
"targets": [
|
"title": "Total Buckets",
|
||||||
{
|
"type": "stat",
|
||||||
"expr": "aw_buckets_total",
|
"targets": [
|
||||||
"refId": "A",
|
{
|
||||||
"legendFormat": "Total Buckets"
|
"expr": "aw_buckets_total",
|
||||||
}
|
"refId": "A",
|
||||||
],
|
"legendFormat": "Total Buckets",
|
||||||
"options": {
|
"datasource": {
|
||||||
"colorMode": "value",
|
"type": "prometheus",
|
||||||
"graphMode": "area"
|
"uid": "prometheus"
|
||||||
},
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"min": 0
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area"
|
||||||
},
|
},
|
||||||
{
|
"fieldConfig": {
|
||||||
"id": 2,
|
"defaults": {
|
||||||
"title": "Events per Bucket",
|
"unit": "short",
|
||||||
"type": "table",
|
"min": 0
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"expr": "aw_bucket_events_count",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true,
|
|
||||||
"refId": "B"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"transformations": [
|
|
||||||
{
|
|
||||||
"id": "organize",
|
|
||||||
"options": {
|
|
||||||
"excludeByName": {
|
|
||||||
"Time": true,
|
|
||||||
"Value": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 3,
|
|
||||||
"title": "Collector Status",
|
|
||||||
"type": "stat",
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"expr": "aw_collector_status",
|
|
||||||
"format": "table",
|
|
||||||
"instant": true,
|
|
||||||
"refId": "C"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"colorMode": "value"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
"datasource": {
|
||||||
"id": 4,
|
"type": "prometheus",
|
||||||
"title": "Events Timeline",
|
"uid": "prometheus"
|
||||||
"type": "graph",
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"expr": "rate(aw_events_total[5m])",
|
|
||||||
"legendFormat": "{{bucket}} - {{event_type}}",
|
|
||||||
"refId": "D"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"custom": {
|
|
||||||
"lineWidth": 2,
|
|
||||||
"fillOpacity": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
{
|
"gridPos": {
|
||||||
"id": 5,
|
"h": 8,
|
||||||
"title": "Last Event Timestamp",
|
"w": 6,
|
||||||
"type": "gauge",
|
"x": 0,
|
||||||
"targets": [
|
"y": 0
|
||||||
{
|
|
||||||
"expr": "aw_events_last_timestamp",
|
|
||||||
"legendFormat": "{{bucket}}",
|
|
||||||
"refId": "E"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"orientation": "horizontal"
|
|
||||||
},
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "s",
|
|
||||||
"custom": {
|
|
||||||
"thresholds": {
|
|
||||||
"mode": "absolute",
|
|
||||||
"steps": [
|
|
||||||
{"color": "red", "value": 0},
|
|
||||||
{"color": "yellow", "value": 3600},
|
|
||||||
{"color": "green", "value": 86400}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
]
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "Events per Bucket",
|
||||||
|
"type": "table",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "aw_bucket_events_count",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"id": "organize",
|
||||||
|
"options": {
|
||||||
|
"excludeByName": {
|
||||||
|
"Time": true,
|
||||||
|
"Value": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 6,
|
||||||
|
"y": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "Collector Status",
|
||||||
|
"type": "stat",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "aw_collector_status",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "C",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value"
|
||||||
|
},
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 6,
|
||||||
|
"x": 18,
|
||||||
|
"y": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "Events Timeline",
|
||||||
|
"type": "graph",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "rate(aw_events_total[5m])",
|
||||||
|
"legendFormat": "{{bucket}} - {{event_type}}",
|
||||||
|
"refId": "D",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {
|
||||||
|
"lineWidth": 2,
|
||||||
|
"fillOpacity": 10
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 8
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Last Event Timestamp",
|
||||||
|
"type": "gauge",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "aw_events_last_timestamp",
|
||||||
|
"legendFormat": "{{bucket}}",
|
||||||
|
"refId": "E",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"orientation": "horizontal"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s",
|
||||||
|
"custom": {
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 3600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": 86400
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 8
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"uid": "activitywatch-overview",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 1,
|
||||||
|
"refresh": "30s",
|
||||||
|
"time": {
|
||||||
|
"from": "now-6h",
|
||||||
|
"to": "now"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,20 +1,11 @@
|
|||||||
apiVersion: 1
|
apiVersion: 1
|
||||||
|
|
||||||
providers:
|
providers:
|
||||||
- name: 1C-Buhgalteria
|
- name: awatch-rus
|
||||||
orgId: 1
|
orgId: 1
|
||||||
folder: "1C"
|
folder: "AWatch-rus"
|
||||||
type: file
|
type: file
|
||||||
disableDeletion: true
|
disableDeletion: true
|
||||||
editable: false
|
editable: false
|
||||||
options:
|
options:
|
||||||
path: /var/lib/grafana/dashboards
|
path: /var/lib/grafana/dashboards
|
||||||
|
|
||||||
- name: ActivityWatch
|
|
||||||
orgId: 2
|
|
||||||
folder: "ActivityWatch"
|
|
||||||
type: file
|
|
||||||
disableDeletion: false
|
|
||||||
editable: true
|
|
||||||
options:
|
|
||||||
path: /var/lib/grafana/dashboards
|
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
apiVersion: 1
|
|
||||||
|
|
||||||
datasources:
|
|
||||||
- name: Prometheus
|
|
||||||
type: prometheus
|
|
||||||
access: proxy
|
|
||||||
url: http://prometheus:9090
|
|
||||||
isDefault: true
|
|
||||||
editable: true
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
apiVersion: 1
|
|
||||||
|
|
||||||
datasources:
|
|
||||||
- name: Prometheus
|
|
||||||
type: prometheus
|
|
||||||
access: proxy
|
|
||||||
url: http://prometheus:9090
|
|
||||||
isDefault: true
|
|
||||||
editable: true
|
|
||||||
Regular → Executable
+102
-103
@@ -4,139 +4,138 @@ ActivityWatch Prometheus Exporter
|
|||||||
Собирает метрики из ActivityWatch API и экспонирует их в формате Prometheus.
|
Собирает метрики из ActivityWatch API и экспонирует их в формате Prometheus.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import time
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from prometheus_client import start_http_server, Gauge, Counter, Histogram, Info
|
from prometheus_client import Counter, Gauge, Info, start_http_server
|
||||||
from datetime import datetime, timedelta
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Configuration
|
# Configuration
|
||||||
AW_SERVER_HOST = "10.10.10.13"
|
AW_SERVER_HOST = os.getenv("AW_SERVER_HOST", "10.10.10.13")
|
||||||
AW_SERVER_PORT = 5600
|
AW_SERVER_PORT = int(os.getenv("AW_SERVER_PORT", "5600"))
|
||||||
AW_SERVER_SCHEME = "http"
|
AW_SERVER_SCHEME = os.getenv("AW_SERVER_SCHEME", "http")
|
||||||
AW_API_BASE = f"{AW_SERVER_SCHEME}://{AW_SERVER_HOST}:{AW_SERVER_PORT}/api/0"
|
AW_API_BASE = os.getenv(
|
||||||
EXPORTER_PORT = 9398
|
"AW_API_BASE",
|
||||||
|
f"{AW_SERVER_SCHEME}://{AW_SERVER_HOST}:{AW_SERVER_PORT}/api/0",
|
||||||
|
)
|
||||||
|
EXPORTER_PORT = int(os.getenv("EXPORTER_PORT", "9398"))
|
||||||
|
SCRAPE_INTERVAL_SECONDS = int(os.getenv("SCRAPE_INTERVAL_SECONDS", "30"))
|
||||||
|
|
||||||
# Metrics
|
# Metrics
|
||||||
aw_buckets_total = Gauge('aw_buckets_total', 'Total number of ActivityWatch buckets')
|
aw_up = Gauge("aw_up", "ActivityWatch API availability: 1 if the last scrape succeeded, 0 otherwise")
|
||||||
aw_events_total = Counter('aw_events_total', 'Total number of ActivityWatch events', ['bucket', 'event_type'])
|
aw_buckets_total = Gauge("aw_buckets_total", "Total number of ActivityWatch buckets")
|
||||||
aw_events_last_timestamp = Gauge('aw_events_last_timestamp', 'Timestamp of last event in bucket', ['bucket'])
|
aw_events_total = Counter("aw_events_total", "Total number of ActivityWatch events observed", ["bucket", "event_type"])
|
||||||
aw_bucket_events_count = Gauge('aw_bucket_events_count', 'Number of events in bucket', ['bucket'])
|
aw_events_last_timestamp = Gauge("aw_events_last_timestamp", "Timestamp of last event in bucket", ["bucket"])
|
||||||
aw_collector_status = Info('aw_collector_status', 'Status of ActivityWatch collectors')
|
aw_bucket_events_count = Gauge("aw_bucket_events_count", "Number of events sampled from bucket", ["bucket"])
|
||||||
aw_server_info = Info('aw_server_info', 'ActivityWatch server information')
|
aw_collector_status = Gauge(
|
||||||
|
"aw_collector_status",
|
||||||
|
"ActivityWatch bucket collector status: 1 if bucket was observed during the last scrape",
|
||||||
|
["bucket", "client", "hostname", "type"],
|
||||||
|
)
|
||||||
|
aw_server_info = Info("aw_server", "ActivityWatch server information")
|
||||||
|
|
||||||
|
|
||||||
class ActivityWatchExporter:
|
class ActivityWatchExporter:
|
||||||
def __init__(self, api_base):
|
def __init__(self, api_base):
|
||||||
self.api_base = api_base
|
self.api_base = api_base.rstrip("/")
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
self.session.headers.update({'Accept': 'application/json'})
|
self.session.headers.update({"Accept": "application/json"})
|
||||||
self.bucket_cache = {}
|
self.bucket_event_counts = {}
|
||||||
|
|
||||||
def get_buckets(self):
|
def get_buckets(self):
|
||||||
"""Get all buckets from ActivityWatch API."""
|
"""Get all buckets from ActivityWatch API."""
|
||||||
try:
|
response = self.session.get(f"{self.api_base}/buckets", timeout=10)
|
||||||
response = self.session.get(f"{self.api_base}/buckets", timeout=10)
|
response.raise_for_status()
|
||||||
response.raise_for_status()
|
return response.json()
|
||||||
return response.json()
|
|
||||||
except Exception as e:
|
def get_bucket_events(self, bucket_id, limit=1000):
|
||||||
logger.error(f"Failed to get buckets: {e}")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
def get_bucket_events(self, bucket_id, limit=1):
|
|
||||||
"""Get events from a specific bucket."""
|
"""Get events from a specific bucket."""
|
||||||
try:
|
response = self.session.get(
|
||||||
response = self.session.get(
|
f"{self.api_base}/buckets/{bucket_id}/events",
|
||||||
f"{self.api_base}/buckets/{bucket_id}/events",
|
params={"limit": limit},
|
||||||
params={'limit': limit},
|
timeout=10,
|
||||||
timeout=10
|
)
|
||||||
)
|
response.raise_for_status()
|
||||||
response.raise_for_status()
|
return response.json()
|
||||||
return response.json()
|
|
||||||
except Exception as e:
|
@staticmethod
|
||||||
logger.error(f"Failed to get events for {bucket_id}: {e}")
|
def event_type(event):
|
||||||
return []
|
data = event.get("data") or {}
|
||||||
|
return str(data.get("app") or data.get("title") or event.get("$schema") or "unknown")
|
||||||
def get_bucket_info(self, bucket_id):
|
|
||||||
"""Get detailed info about a bucket."""
|
@staticmethod
|
||||||
try:
|
def event_timestamp(event):
|
||||||
response = self.session.get(f"{self.api_base}/buckets/{bucket_id}", timeout=10)
|
timestamp = event.get("timestamp", 0)
|
||||||
response.raise_for_status()
|
if isinstance(timestamp, str):
|
||||||
return response.json()
|
return datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp()
|
||||||
except Exception as e:
|
return float(timestamp or 0)
|
||||||
logger.error(f"Failed to get info for {bucket_id}: {e}")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
def collect_metrics(self):
|
def collect_metrics(self):
|
||||||
"""Collect metrics from ActivityWatch."""
|
"""Collect metrics from ActivityWatch."""
|
||||||
buckets = self.get_buckets()
|
try:
|
||||||
|
buckets = self.get_buckets()
|
||||||
# Update bucket count
|
aw_up.set(1)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Failed to get buckets: %s", exc)
|
||||||
|
aw_up.set(0)
|
||||||
|
return
|
||||||
|
|
||||||
aw_buckets_total.set(len(buckets))
|
aw_buckets_total.set(len(buckets))
|
||||||
|
aw_server_info.info(
|
||||||
# Server info
|
{
|
||||||
aw_server_info.info({
|
"host": AW_SERVER_HOST,
|
||||||
'host': AW_SERVER_HOST,
|
"port": str(AW_SERVER_PORT),
|
||||||
'port': AW_SERVER_PORT,
|
"scheme": AW_SERVER_SCHEME,
|
||||||
'scheme': AW_SERVER_SCHEME,
|
"api_base": self.api_base,
|
||||||
'api_base': self.api_base
|
}
|
||||||
})
|
)
|
||||||
|
|
||||||
# Collector status
|
aw_collector_status.clear()
|
||||||
collectors = {}
|
|
||||||
for bucket_id, bucket_data in buckets.items():
|
for bucket_id, bucket_data in buckets.items():
|
||||||
client = bucket_data.get('client', 'unknown')
|
client = str(bucket_data.get("client", "unknown"))
|
||||||
hostname = bucket_data.get('hostname', 'unknown')
|
hostname = str(bucket_data.get("hostname", "unknown"))
|
||||||
bucket_type = bucket_data.get('type', 'unknown')
|
bucket_type = str(bucket_data.get("type", "unknown"))
|
||||||
|
|
||||||
# Count events
|
try:
|
||||||
events = self.get_bucket_events(bucket_id, limit=1000)
|
events = self.get_bucket_events(bucket_id)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Failed to get events for %s: %s", bucket_id, exc)
|
||||||
|
events = []
|
||||||
|
|
||||||
event_count = len(events)
|
event_count = len(events)
|
||||||
aw_bucket_events_count.labels(bucket=bucket_id).set(event_count)
|
aw_bucket_events_count.labels(bucket=bucket_id).set(event_count)
|
||||||
|
aw_collector_status.labels(bucket=bucket_id, client=client, hostname=hostname, type=bucket_type).set(1)
|
||||||
# Last event timestamp
|
|
||||||
|
previous_count = self.bucket_event_counts.get(bucket_id)
|
||||||
|
if previous_count is not None and event_count > previous_count:
|
||||||
|
for event in events[: event_count - previous_count]:
|
||||||
|
aw_events_total.labels(bucket=bucket_id, event_type=self.event_type(event)).inc()
|
||||||
|
self.bucket_event_counts[bucket_id] = event_count
|
||||||
|
|
||||||
if events:
|
if events:
|
||||||
last_event = events[0]
|
|
||||||
timestamp = last_event.get('timestamp', 0)
|
|
||||||
try:
|
try:
|
||||||
# Convert to Unix timestamp if needed
|
aw_events_last_timestamp.labels(bucket=bucket_id).set(self.event_timestamp(events[0]))
|
||||||
if isinstance(timestamp, str):
|
except Exception as exc:
|
||||||
dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
|
logger.warning("Failed to parse last event timestamp for %s: %s", bucket_id, exc)
|
||||||
unix_ts = dt.timestamp()
|
|
||||||
else:
|
|
||||||
unix_ts = timestamp
|
|
||||||
aw_events_last_timestamp.labels(bucket=bucket_id).set(unix_ts)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Collector status
|
|
||||||
collector_key = f"{hostname}_{client}"
|
|
||||||
collectors[collector_key] = {
|
|
||||||
'status': 'active',
|
|
||||||
'bucket': bucket_id,
|
|
||||||
'type': bucket_type,
|
|
||||||
'events': event_count
|
|
||||||
}
|
|
||||||
|
|
||||||
aw_collector_status.info(collectors)
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
exporter = ActivityWatchExporter(AW_API_BASE)
|
exporter = ActivityWatchExporter(AW_API_BASE)
|
||||||
|
|
||||||
# Initial collection
|
|
||||||
exporter.collect_metrics()
|
exporter.collect_metrics()
|
||||||
|
|
||||||
# Start HTTP server
|
|
||||||
start_http_server(EXPORTER_PORT)
|
start_http_server(EXPORTER_PORT)
|
||||||
logger.info(f"ActivityWatch exporter started on port {EXPORTER_PORT}")
|
logger.info("ActivityWatch exporter started on port %s", EXPORTER_PORT)
|
||||||
logger.info(f"Scraping ActivityWatch API at {AW_API_BASE}")
|
logger.info("Scraping ActivityWatch API at %s", AW_API_BASE)
|
||||||
|
|
||||||
# Collect metrics every 30 seconds
|
|
||||||
while True:
|
while True:
|
||||||
time.sleep(30)
|
time.sleep(SCRAPE_INTERVAL_SECONDS)
|
||||||
exporter.collect_metrics()
|
exporter.collect_metrics()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -1,20 +1,95 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
STACK_DIR="${1:-/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c}"
|
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
|
STACK_DIR="${1:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)}"
|
||||||
|
ENV_FILE="$STACK_DIR/.env"
|
||||||
|
|
||||||
echo "[*] Checking endpoints"
|
env_value() {
|
||||||
curl -fsS http://127.0.0.1:9399/metrics >/tmp/awrus-onec-metrics.out
|
key="$1"
|
||||||
curl -fsS http://127.0.0.1:9090/-/healthy >/tmp/awrus-prom-healthy.out
|
default="$2"
|
||||||
curl -fsS "http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22onec_sql_exporter%22%7D" >/tmp/awrus-prom-up.json
|
current=$(eval "printf '%s' \"\${$key:-}\"")
|
||||||
curl -fsS "http://127.0.0.1:9090/api/v1/query?query=onec_data_freshness_seconds" >/tmp/awrus-prom-freshness.json
|
if [ -n "$current" ]; then
|
||||||
|
printf '%s' "$current"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if [ -f "$ENV_FILE" ]; then
|
||||||
|
value=$(sed -n "s/^$key=//p" "$ENV_FILE" | tail -n 1)
|
||||||
|
if [ -n "$value" ]; then
|
||||||
|
printf '%s' "$value"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
printf '%s' "$default"
|
||||||
|
}
|
||||||
|
|
||||||
echo "[*] Checking container status"
|
GRAFANA_PORT=$(env_value GRAFANA_PORT 3000)
|
||||||
cd "$STACK_DIR"
|
PROMETHEUS_PORT=$(env_value PROMETHEUS_PORT 9090)
|
||||||
docker compose ps
|
SQL_EXPORTER_PORT=$(env_value SQL_EXPORTER_PORT 9399)
|
||||||
|
AW_EXPORTER_PORT=$(env_value AW_EXPORTER_PORT 9398)
|
||||||
|
GRAFANA_ADMIN_USER=$(env_value GRAFANA_ADMIN_USER admin)
|
||||||
|
GRAFANA_ADMIN_PASSWORD=$(env_value GRAFANA_ADMIN_PASSWORD change_me_now)
|
||||||
|
|
||||||
|
TMP_DIR="${TMPDIR:-/tmp}"
|
||||||
|
METRICS_OUT="$TMP_DIR/awrus-onec-metrics.out"
|
||||||
|
AW_METRICS_OUT="$TMP_DIR/awrus-aw-metrics.out"
|
||||||
|
PROM_HEALTH_OUT="$TMP_DIR/awrus-prom-healthy.out"
|
||||||
|
PROM_UP_OUT="$TMP_DIR/awrus-prom-up.json"
|
||||||
|
PROM_FRESHNESS_OUT="$TMP_DIR/awrus-prom-freshness.json"
|
||||||
|
GRAFANA_HEALTH_OUT="$TMP_DIR/awrus-grafana-health.json"
|
||||||
|
GRAFANA_DS_OUT="$TMP_DIR/awrus-grafana-datasources.json"
|
||||||
|
GRAFANA_DASH_OUT="$TMP_DIR/awrus-grafana-dashboards.json"
|
||||||
|
|
||||||
|
require_metric() {
|
||||||
|
metric_name="$1"
|
||||||
|
metrics_file="$2"
|
||||||
|
if ! grep -q "^$metric_name" "$metrics_file"; then
|
||||||
|
echo "[!] Required metric '$metric_name' was not found in $metrics_file" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
require_prometheus_success() {
|
||||||
|
file="$1"
|
||||||
|
if ! grep -q '"status":"success"' "$file"; then
|
||||||
|
echo "[!] Prometheus query did not return status=success: $file" >&2
|
||||||
|
cat "$file" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "[*] Checking exporter endpoints"
|
||||||
|
curl -fsS "http://127.0.0.1:$SQL_EXPORTER_PORT/metrics" >"$METRICS_OUT"
|
||||||
|
curl -fsS "http://127.0.0.1:$AW_EXPORTER_PORT/metrics" >"$AW_METRICS_OUT"
|
||||||
|
require_metric "onec_data_freshness_seconds" "$METRICS_OUT"
|
||||||
|
require_metric "aw_up" "$AW_METRICS_OUT"
|
||||||
|
|
||||||
|
echo "[*] Checking Prometheus health and scrape targets"
|
||||||
|
curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/-/healthy" >"$PROM_HEALTH_OUT"
|
||||||
|
curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/api/v1/query?query=up%7Bjob%3D~%22onec_sql_exporter%7Caw_activitywatch_exporter%22%7D" >"$PROM_UP_OUT"
|
||||||
|
curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/api/v1/query?query=onec_data_freshness_seconds" >"$PROM_FRESHNESS_OUT"
|
||||||
|
require_prometheus_success "$PROM_UP_OUT"
|
||||||
|
require_prometheus_success "$PROM_FRESHNESS_OUT"
|
||||||
|
|
||||||
|
echo "[*] Checking Grafana health, datasource and dashboards"
|
||||||
|
curl -fsS "http://127.0.0.1:$GRAFANA_PORT/api/health" >"$GRAFANA_HEALTH_OUT"
|
||||||
|
curl -fsS -u "$GRAFANA_ADMIN_USER:$GRAFANA_ADMIN_PASSWORD" "http://127.0.0.1:$GRAFANA_PORT/api/datasources/uid/prometheus" >"$GRAFANA_DS_OUT"
|
||||||
|
curl -fsS -u "$GRAFANA_ADMIN_USER:$GRAFANA_ADMIN_PASSWORD" "http://127.0.0.1:$GRAFANA_PORT/api/search?type=dash-db&query=" >"$GRAFANA_DASH_OUT"
|
||||||
|
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
echo "[*] Checking container status"
|
||||||
|
cd "$STACK_DIR"
|
||||||
|
docker compose ps
|
||||||
|
else
|
||||||
|
echo "[*] docker command not found; skipping container status"
|
||||||
|
fi
|
||||||
|
|
||||||
echo "[+] Pipeline health artifacts:"
|
echo "[+] Pipeline health artifacts:"
|
||||||
echo " /tmp/awrus-onec-metrics.out"
|
echo " $METRICS_OUT"
|
||||||
echo " /tmp/awrus-prom-healthy.out"
|
echo " $AW_METRICS_OUT"
|
||||||
echo " /tmp/awrus-prom-up.json"
|
echo " $PROM_HEALTH_OUT"
|
||||||
echo " /tmp/awrus-prom-freshness.json"
|
echo " $PROM_UP_OUT"
|
||||||
|
echo " $PROM_FRESHNESS_OUT"
|
||||||
|
echo " $GRAFANA_HEALTH_OUT"
|
||||||
|
echo " $GRAFANA_DS_OUT"
|
||||||
|
echo " $GRAFANA_DASH_OUT"
|
||||||
|
|||||||
Reference in New Issue
Block a user