From dc8988e5f7046c5e5beae9710310ca354332fb41 Mon Sep 17 00:00:00 2001 From: IgorRachkov <89467086+igor04091968@users.noreply.github.com> Date: Sat, 9 May 2026 03:58:52 +0300 Subject: [PATCH] Harden Grafana Prometheus e2e readiness --- grafana-1c/.env.example | 5 + grafana-1c/README.md | 50 +-- grafana-1c/docker-compose.yml | 9 +- .../grafana/dashboards/aw_overview.json | 310 +++++++++++------- .../provisioning/dashboards/dashboards.yml | 13 +- .../datasources/aw_prometheus.yml | 9 - .../provisioning/datasources/datasources.yml | 9 - .../collectors/aw_activitywatch.py | 205 ++++++------ grafana-1c/tools/check_pipeline.sh | 101 +++++- 9 files changed, 430 insertions(+), 281 deletions(-) delete mode 100644 grafana-1c/grafana/provisioning/datasources/aw_prometheus.yml delete mode 100644 grafana-1c/grafana/provisioning/datasources/datasources.yml mode change 100644 => 100755 grafana-1c/sql-exporter/collectors/aw_activitywatch.py diff --git a/grafana-1c/.env.example b/grafana-1c/.env.example index 2d2a13c..e13d3aa 100755 --- a/grafana-1c/.env.example +++ b/grafana-1c/.env.example @@ -4,4 +4,9 @@ GRAFANA_ADMIN_PASSWORD=change_me_now GRAFANA_PORT=3000 PROMETHEUS_PORT=9090 SQL_EXPORTER_PORT=9399 +AW_EXPORTER_PORT=9398 +AW_SERVER_HOST=10.10.10.13 +AW_SERVER_PORT=5600 +AW_SERVER_SCHEME=http +AW_SCRAPE_INTERVAL_SECONDS=30 ONEC_DSN=postgres://onec_reader:change_me@10.10.10.20:5432/onec_db?sslmode=disable diff --git a/grafana-1c/README.md b/grafana-1c/README.md index e415a25..df8ef61 100755 --- a/grafana-1c/README.md +++ b/grafana-1c/README.md @@ -3,44 +3,46 @@ Готовый каркас для непрерывного сбора KPI из 1С и анализа в Grafana: - `sql-exporter` читает SQL-представления KPI из БД 1С; +- `aw-exporter` собирает метрики ActivityWatch и отдает их Prometheus; - `prometheus` собирает метрики и применяет alert-rules; - `grafana` поднимает datasource и дашборд автоматически. ## Полные пути -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/.env.example` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/docker-compose.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql-exporter/sql_exporter.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql-exporter/collectors/onec_accounting_kpi.collector.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/prometheus.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/alerts.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/prometheus/recording_rules.yml` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/grafana/dashboards/1c-accounting-overview.json` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/grafana/dashboards/1c-accounting-sre.json` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/postgres_views_template.sql` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/mssql_views_template.sql` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/discover_postgres_1c.sh` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/validate_kpi_views.sh` -- `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline.sh` +- `./.env.example` +- `./docker-compose.yml` +- `./sql-exporter/sql_exporter.yml` +- `./sql-exporter/collectors/onec_accounting_kpi.collector.yml` +- `./prometheus/prometheus.yml` +- `./prometheus/alerts.yml` +- `./prometheus/recording_rules.yml` +- `./grafana/dashboards/1c-accounting-overview.json` +- `./grafana/dashboards/1c-accounting-sre.json` +- `./sql/postgres_views_template.sql` +- `./sql/mssql_views_template.sql` +- `./tools/discover_postgres_1c.sh` +- `./tools/validate_kpi_views.sh` +- `./tools/check_pipeline.sh` ## Быстрый запуск 1. Подготовьте env: ```bash -cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c +cd grafana-1c cp .env.example .env ``` 2. В `.env` задайте: - `GRAFANA_ADMIN_USER`, `GRAFANA_ADMIN_PASSWORD`; -- `ONEC_DSN` (DSN read-only пользователя в БД 1С). +- `ONEC_DSN` (DSN read-only пользователя в БД 1С); +- при необходимости `AW_SERVER_HOST`, `AW_SERVER_PORT`, `AW_SERVER_SCHEME`, `AW_EXPORTER_PORT` и `AW_SCRAPE_INTERVAL_SECONDS` для ActivityWatch exporter. 3. В БД 1С создайте KPI-представления: -- для PostgreSQL возьмите `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/postgres_views_template.sql`; -- для MS SQL возьмите `/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/sql/mssql_views_template.sql`. +- для PostgreSQL возьмите `./sql/postgres_views_template.sql`; +- для MS SQL возьмите `./sql/mssql_views_template.sql`. 4. Поднимите стек: @@ -52,7 +54,9 @@ docker compose up -d ```bash curl -fsS http://127.0.0.1:9399/metrics | head +curl -fsS http://127.0.0.1:9398/metrics | head curl -fsS http://127.0.0.1:9090/-/healthy +curl -fsS http://127.0.0.1:3000/api/health ``` Откройте Grafana: `http://:3000`. @@ -62,23 +66,25 @@ curl -fsS http://127.0.0.1:9090/-/healthy Профилирование структуры 1С (PostgreSQL): ```bash -sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/discover_postgres_1c.sh \ +sh ./tools/discover_postgres_1c.sh \ "postgres://user:pass@db-host:5432/db?sslmode=disable" ``` Проверка KPI views: ```bash -sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/validate_kpi_views.sh \ +sh ./tools/validate_kpi_views.sh \ "postgres://user:pass@db-host:5432/db?sslmode=disable" ``` Проверка end-to-end пайплайна: ```bash -sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline.sh +sh ./tools/check_pipeline.sh ``` +Скрипт проверяет полный путь сбора данных: `sql-exporter` и `aw-exporter` отдают обязательные метрики, Prometheus успешно выполняет запросы по scrape-targets, а Grafana отвечает на health/API, видит datasource `prometheus` и provisioned dashboards. Если стек запущен не из каталога репозитория, передайте путь к каталогу `grafana-1c` первым аргументом. + ## Что контролируется - Непроведенные документы (`onec_unposted_documents_total`) @@ -86,6 +92,8 @@ sh /mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c/tools/check_pipeline. - Просроченная дебиторка (`onec_overdue_receivables_total`) - Ошибки проведения за 24ч (`onec_posting_errors_total`) - Свежесть данных из 1С (`onec_data_freshness_seconds`) +- Доступность ActivityWatch API (`aw_up`) +- Количество bucket/events ActivityWatch (`aw_buckets_total`, `aw_bucket_events_count`) ## Принципы безопасности diff --git a/grafana-1c/docker-compose.yml b/grafana-1c/docker-compose.yml index 6206001..357fdc9 100755 --- a/grafana-1c/docker-compose.yml +++ b/grafana-1c/docker-compose.yml @@ -6,14 +6,15 @@ services: container_name: awrus-aw-exporter restart: unless-stopped environment: - - AW_SERVER_HOST=10.10.10.13 - - AW_SERVER_PORT=5600 - - AW_SERVER_SCHEME=http + - AW_SERVER_HOST=${AW_SERVER_HOST:-10.10.10.13} + - AW_SERVER_PORT=${AW_SERVER_PORT:-5600} + - AW_SERVER_SCHEME=${AW_SERVER_SCHEME:-http} - EXPORTER_PORT=9398 + - SCRAPE_INTERVAL_SECONDS=${AW_SCRAPE_INTERVAL_SECONDS:-30} volumes: - ./sql-exporter/collectors/aw_activitywatch.py:/app/aw_activitywatch.py:ro ports: - - "9398:9398" + - "${AW_EXPORTER_PORT:-9398}:9398" command: - "python3" - "/app/aw_activitywatch.py" diff --git a/grafana-1c/grafana/dashboards/aw_overview.json b/grafana-1c/grafana/dashboards/aw_overview.json index 953d191..f7b9733 100644 --- a/grafana-1c/grafana/dashboards/aw_overview.json +++ b/grafana-1c/grafana/dashboards/aw_overview.json @@ -1,121 +1,209 @@ { - "dashboard": { - "title": "ActivityWatch Overview", - "tags": ["activitywatch", "monitoring"], - "timezone": "browser", - "panels": [ - { - "id": 1, - "title": "Total Buckets", - "type": "stat", - "targets": [ - { - "expr": "aw_buckets_total", - "refId": "A", - "legendFormat": "Total Buckets" - } - ], - "options": { - "colorMode": "value", - "graphMode": "area" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0 + "title": "ActivityWatch Overview", + "tags": [ + "activitywatch", + "monitoring" + ], + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "Total Buckets", + "type": "stat", + "targets": [ + { + "expr": "aw_buckets_total", + "refId": "A", + "legendFormat": "Total Buckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" } } + ], + "options": { + "colorMode": "value", + "graphMode": "area" }, - { - "id": 2, - "title": "Events per Bucket", - "type": "table", - "targets": [ - { - "expr": "aw_bucket_events_count", - "format": "table", - "instant": true, - "refId": "B" - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - } - } - } - ] - }, - { - "id": 3, - "title": "Collector Status", - "type": "stat", - "targets": [ - { - "expr": "aw_collector_status", - "format": "table", - "instant": true, - "refId": "C" - } - ], - "options": { - "colorMode": "value" + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 } }, - { - "id": 4, - "title": "Events Timeline", - "type": "graph", - "targets": [ - { - "expr": "rate(aw_events_total[5m])", - "legendFormat": "{{bucket}} - {{event_type}}", - "refId": "D" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - } - } - } + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "id": 5, - "title": "Last Event Timestamp", - "type": "gauge", - "targets": [ - { - "expr": "aw_events_last_timestamp", - "legendFormat": "{{bucket}}", - "refId": "E" - } - ], - "options": { - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "custom": { - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": 0}, - {"color": "yellow", "value": 3600}, - {"color": "green", "value": 86400} - ] - } - } - } - } + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 } - ] + }, + { + "id": 2, + "title": "Events per Bucket", + "type": "table", + "targets": [ + { + "expr": "aw_bucket_events_count", + "format": "table", + "instant": true, + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + } + } + } + ], + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 0 + } + }, + { + "id": 3, + "title": "Collector Status", + "type": "stat", + "targets": [ + { + "expr": "aw_collector_status", + "format": "table", + "instant": true, + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "colorMode": "value" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + } + }, + { + "id": 4, + "title": "Events Timeline", + "type": "graph", + "targets": [ + { + "expr": "rate(aw_events_total[5m])", + "legendFormat": "{{bucket}} - {{event_type}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 5, + "title": "Last Event Timestamp", + "type": "gauge", + "targets": [ + { + "expr": "aw_events_last_timestamp", + "legendFormat": "{{bucket}}", + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "green", + "value": 86400 + } + ] + } + } + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + } + } + ], + "uid": "activitywatch-overview", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" } } diff --git a/grafana-1c/grafana/provisioning/dashboards/dashboards.yml b/grafana-1c/grafana/provisioning/dashboards/dashboards.yml index 940735e..b46d547 100755 --- a/grafana-1c/grafana/provisioning/dashboards/dashboards.yml +++ b/grafana-1c/grafana/provisioning/dashboards/dashboards.yml @@ -1,20 +1,11 @@ apiVersion: 1 providers: - - name: 1C-Buhgalteria + - name: awatch-rus orgId: 1 - folder: "1C" + folder: "AWatch-rus" type: file disableDeletion: true editable: false options: path: /var/lib/grafana/dashboards - - - name: ActivityWatch - orgId: 2 - folder: "ActivityWatch" - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards diff --git a/grafana-1c/grafana/provisioning/datasources/aw_prometheus.yml b/grafana-1c/grafana/provisioning/datasources/aw_prometheus.yml deleted file mode 100644 index 1a57b69..0000000 --- a/grafana-1c/grafana/provisioning/datasources/aw_prometheus.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true diff --git a/grafana-1c/grafana/provisioning/datasources/datasources.yml b/grafana-1c/grafana/provisioning/datasources/datasources.yml deleted file mode 100644 index 1a57b69..0000000 --- a/grafana-1c/grafana/provisioning/datasources/datasources.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true diff --git a/grafana-1c/sql-exporter/collectors/aw_activitywatch.py b/grafana-1c/sql-exporter/collectors/aw_activitywatch.py old mode 100644 new mode 100755 index fbe5efe..a3392cc --- a/grafana-1c/sql-exporter/collectors/aw_activitywatch.py +++ b/grafana-1c/sql-exporter/collectors/aw_activitywatch.py @@ -4,139 +4,138 @@ ActivityWatch Prometheus Exporter Собирает метрики из ActivityWatch API и экспонирует их в формате Prometheus. """ -import time import logging +import os +import time +from datetime import datetime + import requests -from prometheus_client import start_http_server, Gauge, Counter, Histogram, Info -from datetime import datetime, timedelta +from prometheus_client import Counter, Gauge, Info, start_http_server logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Configuration -AW_SERVER_HOST = "10.10.10.13" -AW_SERVER_PORT = 5600 -AW_SERVER_SCHEME = "http" -AW_API_BASE = f"{AW_SERVER_SCHEME}://{AW_SERVER_HOST}:{AW_SERVER_PORT}/api/0" -EXPORTER_PORT = 9398 +AW_SERVER_HOST = os.getenv("AW_SERVER_HOST", "10.10.10.13") +AW_SERVER_PORT = int(os.getenv("AW_SERVER_PORT", "5600")) +AW_SERVER_SCHEME = os.getenv("AW_SERVER_SCHEME", "http") +AW_API_BASE = os.getenv( + "AW_API_BASE", + f"{AW_SERVER_SCHEME}://{AW_SERVER_HOST}:{AW_SERVER_PORT}/api/0", +) +EXPORTER_PORT = int(os.getenv("EXPORTER_PORT", "9398")) +SCRAPE_INTERVAL_SECONDS = int(os.getenv("SCRAPE_INTERVAL_SECONDS", "30")) # Metrics -aw_buckets_total = Gauge('aw_buckets_total', 'Total number of ActivityWatch buckets') -aw_events_total = Counter('aw_events_total', 'Total number of ActivityWatch events', ['bucket', 'event_type']) -aw_events_last_timestamp = Gauge('aw_events_last_timestamp', 'Timestamp of last event in bucket', ['bucket']) -aw_bucket_events_count = Gauge('aw_bucket_events_count', 'Number of events in bucket', ['bucket']) -aw_collector_status = Info('aw_collector_status', 'Status of ActivityWatch collectors') -aw_server_info = Info('aw_server_info', 'ActivityWatch server information') +aw_up = Gauge("aw_up", "ActivityWatch API availability: 1 if the last scrape succeeded, 0 otherwise") +aw_buckets_total = Gauge("aw_buckets_total", "Total number of ActivityWatch buckets") +aw_events_total = Counter("aw_events_total", "Total number of ActivityWatch events observed", ["bucket", "event_type"]) +aw_events_last_timestamp = Gauge("aw_events_last_timestamp", "Timestamp of last event in bucket", ["bucket"]) +aw_bucket_events_count = Gauge("aw_bucket_events_count", "Number of events sampled from bucket", ["bucket"]) +aw_collector_status = Gauge( + "aw_collector_status", + "ActivityWatch bucket collector status: 1 if bucket was observed during the last scrape", + ["bucket", "client", "hostname", "type"], +) +aw_server_info = Info("aw_server", "ActivityWatch server information") + class ActivityWatchExporter: def __init__(self, api_base): - self.api_base = api_base + self.api_base = api_base.rstrip("/") self.session = requests.Session() - self.session.headers.update({'Accept': 'application/json'}) - self.bucket_cache = {} - + self.session.headers.update({"Accept": "application/json"}) + self.bucket_event_counts = {} + def get_buckets(self): """Get all buckets from ActivityWatch API.""" - try: - response = self.session.get(f"{self.api_base}/buckets", timeout=10) - response.raise_for_status() - return response.json() - except Exception as e: - logger.error(f"Failed to get buckets: {e}") - return {} - - def get_bucket_events(self, bucket_id, limit=1): + response = self.session.get(f"{self.api_base}/buckets", timeout=10) + response.raise_for_status() + return response.json() + + def get_bucket_events(self, bucket_id, limit=1000): """Get events from a specific bucket.""" - try: - response = self.session.get( - f"{self.api_base}/buckets/{bucket_id}/events", - params={'limit': limit}, - timeout=10 - ) - response.raise_for_status() - return response.json() - except Exception as e: - logger.error(f"Failed to get events for {bucket_id}: {e}") - return [] - - def get_bucket_info(self, bucket_id): - """Get detailed info about a bucket.""" - try: - response = self.session.get(f"{self.api_base}/buckets/{bucket_id}", timeout=10) - response.raise_for_status() - return response.json() - except Exception as e: - logger.error(f"Failed to get info for {bucket_id}: {e}") - return {} - + response = self.session.get( + f"{self.api_base}/buckets/{bucket_id}/events", + params={"limit": limit}, + timeout=10, + ) + response.raise_for_status() + return response.json() + + @staticmethod + def event_type(event): + data = event.get("data") or {} + return str(data.get("app") or data.get("title") or event.get("$schema") or "unknown") + + @staticmethod + def event_timestamp(event): + timestamp = event.get("timestamp", 0) + if isinstance(timestamp, str): + return datetime.fromisoformat(timestamp.replace("Z", "+00:00")).timestamp() + return float(timestamp or 0) + def collect_metrics(self): """Collect metrics from ActivityWatch.""" - buckets = self.get_buckets() - - # Update bucket count + try: + buckets = self.get_buckets() + aw_up.set(1) + except Exception as exc: + logger.error("Failed to get buckets: %s", exc) + aw_up.set(0) + return + aw_buckets_total.set(len(buckets)) - - # Server info - aw_server_info.info({ - 'host': AW_SERVER_HOST, - 'port': AW_SERVER_PORT, - 'scheme': AW_SERVER_SCHEME, - 'api_base': self.api_base - }) - - # Collector status - collectors = {} + aw_server_info.info( + { + "host": AW_SERVER_HOST, + "port": str(AW_SERVER_PORT), + "scheme": AW_SERVER_SCHEME, + "api_base": self.api_base, + } + ) + + aw_collector_status.clear() for bucket_id, bucket_data in buckets.items(): - client = bucket_data.get('client', 'unknown') - hostname = bucket_data.get('hostname', 'unknown') - bucket_type = bucket_data.get('type', 'unknown') - - # Count events - events = self.get_bucket_events(bucket_id, limit=1000) + client = str(bucket_data.get("client", "unknown")) + hostname = str(bucket_data.get("hostname", "unknown")) + bucket_type = str(bucket_data.get("type", "unknown")) + + try: + events = self.get_bucket_events(bucket_id) + except Exception as exc: + logger.error("Failed to get events for %s: %s", bucket_id, exc) + events = [] + event_count = len(events) aw_bucket_events_count.labels(bucket=bucket_id).set(event_count) - - # Last event timestamp + aw_collector_status.labels(bucket=bucket_id, client=client, hostname=hostname, type=bucket_type).set(1) + + previous_count = self.bucket_event_counts.get(bucket_id) + if previous_count is not None and event_count > previous_count: + for event in events[: event_count - previous_count]: + aw_events_total.labels(bucket=bucket_id, event_type=self.event_type(event)).inc() + self.bucket_event_counts[bucket_id] = event_count + if events: - last_event = events[0] - timestamp = last_event.get('timestamp', 0) try: - # Convert to Unix timestamp if needed - if isinstance(timestamp, str): - dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - unix_ts = dt.timestamp() - else: - unix_ts = timestamp - aw_events_last_timestamp.labels(bucket=bucket_id).set(unix_ts) - except: - pass - - # Collector status - collector_key = f"{hostname}_{client}" - collectors[collector_key] = { - 'status': 'active', - 'bucket': bucket_id, - 'type': bucket_type, - 'events': event_count - } - - aw_collector_status.info(collectors) + aw_events_last_timestamp.labels(bucket=bucket_id).set(self.event_timestamp(events[0])) + except Exception as exc: + logger.warning("Failed to parse last event timestamp for %s: %s", bucket_id, exc) + def main(): exporter = ActivityWatchExporter(AW_API_BASE) - - # Initial collection exporter.collect_metrics() - - # Start HTTP server + start_http_server(EXPORTER_PORT) - logger.info(f"ActivityWatch exporter started on port {EXPORTER_PORT}") - logger.info(f"Scraping ActivityWatch API at {AW_API_BASE}") - - # Collect metrics every 30 seconds + logger.info("ActivityWatch exporter started on port %s", EXPORTER_PORT) + logger.info("Scraping ActivityWatch API at %s", AW_API_BASE) + while True: - time.sleep(30) + time.sleep(SCRAPE_INTERVAL_SECONDS) exporter.collect_metrics() -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/grafana-1c/tools/check_pipeline.sh b/grafana-1c/tools/check_pipeline.sh index a282d87..bf9c7df 100755 --- a/grafana-1c/tools/check_pipeline.sh +++ b/grafana-1c/tools/check_pipeline.sh @@ -1,20 +1,95 @@ #!/bin/sh set -eu -STACK_DIR="${1:-/mnt/usb_hdd2/Projects/ActivityWatch-Russian/grafana-1c}" +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +STACK_DIR="${1:-$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)}" +ENV_FILE="$STACK_DIR/.env" -echo "[*] Checking endpoints" -curl -fsS http://127.0.0.1:9399/metrics >/tmp/awrus-onec-metrics.out -curl -fsS http://127.0.0.1:9090/-/healthy >/tmp/awrus-prom-healthy.out -curl -fsS "http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22onec_sql_exporter%22%7D" >/tmp/awrus-prom-up.json -curl -fsS "http://127.0.0.1:9090/api/v1/query?query=onec_data_freshness_seconds" >/tmp/awrus-prom-freshness.json +env_value() { + key="$1" + default="$2" + current=$(eval "printf '%s' \"\${$key:-}\"") + if [ -n "$current" ]; then + printf '%s' "$current" + return + fi + if [ -f "$ENV_FILE" ]; then + value=$(sed -n "s/^$key=//p" "$ENV_FILE" | tail -n 1) + if [ -n "$value" ]; then + printf '%s' "$value" + return + fi + fi + printf '%s' "$default" +} -echo "[*] Checking container status" -cd "$STACK_DIR" -docker compose ps +GRAFANA_PORT=$(env_value GRAFANA_PORT 3000) +PROMETHEUS_PORT=$(env_value PROMETHEUS_PORT 9090) +SQL_EXPORTER_PORT=$(env_value SQL_EXPORTER_PORT 9399) +AW_EXPORTER_PORT=$(env_value AW_EXPORTER_PORT 9398) +GRAFANA_ADMIN_USER=$(env_value GRAFANA_ADMIN_USER admin) +GRAFANA_ADMIN_PASSWORD=$(env_value GRAFANA_ADMIN_PASSWORD change_me_now) + +TMP_DIR="${TMPDIR:-/tmp}" +METRICS_OUT="$TMP_DIR/awrus-onec-metrics.out" +AW_METRICS_OUT="$TMP_DIR/awrus-aw-metrics.out" +PROM_HEALTH_OUT="$TMP_DIR/awrus-prom-healthy.out" +PROM_UP_OUT="$TMP_DIR/awrus-prom-up.json" +PROM_FRESHNESS_OUT="$TMP_DIR/awrus-prom-freshness.json" +GRAFANA_HEALTH_OUT="$TMP_DIR/awrus-grafana-health.json" +GRAFANA_DS_OUT="$TMP_DIR/awrus-grafana-datasources.json" +GRAFANA_DASH_OUT="$TMP_DIR/awrus-grafana-dashboards.json" + +require_metric() { + metric_name="$1" + metrics_file="$2" + if ! grep -q "^$metric_name" "$metrics_file"; then + echo "[!] Required metric '$metric_name' was not found in $metrics_file" >&2 + exit 1 + fi +} + +require_prometheus_success() { + file="$1" + if ! grep -q '"status":"success"' "$file"; then + echo "[!] Prometheus query did not return status=success: $file" >&2 + cat "$file" >&2 + exit 1 + fi +} + +echo "[*] Checking exporter endpoints" +curl -fsS "http://127.0.0.1:$SQL_EXPORTER_PORT/metrics" >"$METRICS_OUT" +curl -fsS "http://127.0.0.1:$AW_EXPORTER_PORT/metrics" >"$AW_METRICS_OUT" +require_metric "onec_data_freshness_seconds" "$METRICS_OUT" +require_metric "aw_up" "$AW_METRICS_OUT" + +echo "[*] Checking Prometheus health and scrape targets" +curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/-/healthy" >"$PROM_HEALTH_OUT" +curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/api/v1/query?query=up%7Bjob%3D~%22onec_sql_exporter%7Caw_activitywatch_exporter%22%7D" >"$PROM_UP_OUT" +curl -fsS "http://127.0.0.1:$PROMETHEUS_PORT/api/v1/query?query=onec_data_freshness_seconds" >"$PROM_FRESHNESS_OUT" +require_prometheus_success "$PROM_UP_OUT" +require_prometheus_success "$PROM_FRESHNESS_OUT" + +echo "[*] Checking Grafana health, datasource and dashboards" +curl -fsS "http://127.0.0.1:$GRAFANA_PORT/api/health" >"$GRAFANA_HEALTH_OUT" +curl -fsS -u "$GRAFANA_ADMIN_USER:$GRAFANA_ADMIN_PASSWORD" "http://127.0.0.1:$GRAFANA_PORT/api/datasources/uid/prometheus" >"$GRAFANA_DS_OUT" +curl -fsS -u "$GRAFANA_ADMIN_USER:$GRAFANA_ADMIN_PASSWORD" "http://127.0.0.1:$GRAFANA_PORT/api/search?type=dash-db&query=" >"$GRAFANA_DASH_OUT" + +if command -v docker >/dev/null 2>&1; then + echo "[*] Checking container status" + cd "$STACK_DIR" + docker compose ps +else + echo "[*] docker command not found; skipping container status" +fi echo "[+] Pipeline health artifacts:" -echo " /tmp/awrus-onec-metrics.out" -echo " /tmp/awrus-prom-healthy.out" -echo " /tmp/awrus-prom-up.json" -echo " /tmp/awrus-prom-freshness.json" +echo " $METRICS_OUT" +echo " $AW_METRICS_OUT" +echo " $PROM_HEALTH_OUT" +echo " $PROM_UP_OUT" +echo " $PROM_FRESHNESS_OUT" +echo " $GRAFANA_HEALTH_OUT" +echo " $GRAFANA_DS_OUT" +echo " $GRAFANA_DASH_OUT"