From 2e5242f5ba2305654a2d82e9f0460cb7b24506ac Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Wed, 13 May 2026 07:08:23 +0300 Subject: [PATCH] feat(dlp): add dedicated health check and grafana dashboard --- ansible/deploy_aw_server.yml | 18 + .../grafana/dashboards/dlp-dashboard.json | 314 +++++++++++++++++ scripts/diag_and_manual_restart.sh | 5 +- scripts/dlp-health-check.py | 318 ++++++++++++++++++ 4 files changed, 653 insertions(+), 2 deletions(-) create mode 100644 grafana-1c/grafana/dashboards/dlp-dashboard.json create mode 100644 scripts/dlp-health-check.py diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index 96826b8..3878298 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -650,6 +650,14 @@ group: root mode: "0755" + - name: Установить скрипт dlp-health-check + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/dlp-health-check.py" + dest: /usr/local/bin/dlp-health-check + owner: root + group: root + mode: "0755" + - name: Установить systemd unit AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.service" @@ -1312,6 +1320,16 @@ ansible.builtin.debug: msg: "{{ aw_post_deploy_health.stdout }}" + - name: Запустить /usr/local/bin/dlp-health-check --json + ansible.builtin.command: + cmd: /usr/local/bin/dlp-health-check --json + register: dlp_post_deploy_health + changed_when: false + + - name: Показать результат dlp-health-check + ansible.builtin.debug: + msg: "{{ dlp_post_deploy_health.stdout }}" + handlers: - name: Перезагрузить systemd ansible.builtin.systemd: diff --git a/grafana-1c/grafana/dashboards/dlp-dashboard.json b/grafana-1c/grafana/dashboards/dlp-dashboard.json new file mode 100644 index 0000000..2863298 --- /dev/null +++ b/grafana-1c/grafana/dashboards/dlp-dashboard.json @@ -0,0 +1,314 @@ +{ + "title": "AWatch DLP Overview", + "uid": "awatch-dlp-overview", + "tags": [ + "activitywatch", + "dlp", + "security" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-24h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "DLP Buckets Seen", + "type": "stat", + "targets": [ + { + "expr": "count(aw_collector_status{bucket=~\"aw-dlp-.*|aw-file-operations_.*\"})", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "title": "Endpoint Worst Age", + "type": "stat", + "targets": [ + { + "expr": "max(time() - aw_events_last_timestamp{bucket=~\"aw-dlp-endpoint-signals_.*\"})", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 900 + }, + { + "color": "red", + "value": 1800 + } + ] + } + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + } + }, + { + "id": 3, + "title": "FileOps Worst Age", + "type": "stat", + "targets": [ + { + "expr": "max(time() - aw_events_last_timestamp{bucket=~\"aw-file-operations_.*\"})", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 900 + }, + { + "color": "red", + "value": 1800 + } + ] + } + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + } + }, + { + "id": 4, + "title": "Incident Buckets Seen", + "type": "stat", + "targets": [ + { + "expr": "count(aw_collector_status{bucket=~\"aw-dlp-incidents_.*\"})", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "options": { + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + } + }, + { + "id": 5, + "title": "Endpoint Bucket Age", + "type": "table", + "targets": [ + { + "expr": "time() - aw_events_last_timestamp{bucket=~\"aw-dlp-endpoint-signals_.*\"}", + "format": "table", + "instant": true, + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 6, + "title": "FileOps Bucket Age", + "type": "table", + "targets": [ + { + "expr": "time() - aw_events_last_timestamp{bucket=~\"aw-file-operations_.*\"}", + "format": "table", + "instant": true, + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + } + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 7, + "title": "DLP Collector Status", + "type": "table", + "targets": [ + { + "expr": "aw_collector_status{bucket=~\"aw-dlp-.*|aw-file-operations_.*\"}", + "format": "table", + "instant": true, + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + } + } + } + ], + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + } + } + ] +} diff --git a/scripts/diag_and_manual_restart.sh b/scripts/diag_and_manual_restart.sh index 6b08955..8575362 100644 --- a/scripts/diag_and_manual_restart.sh +++ b/scripts/diag_and_manual_restart.sh @@ -14,7 +14,7 @@ Usage: scripts/diag_and_manual_restart.sh [--with-windows] [--yes] [--inventory ] Behavior: - 1) Runs remote diagnostics on aw_server using /usr/local/bin/aw-health-check + 1) Runs remote diagnostics on aw_server using /usr/local/bin/aw-health-check and /usr/local/bin/dlp-health-check 2) If diagnostics fail: - restarts required server services - optionally restarts Windows launch/recovery tasks (with --with-windows) @@ -40,7 +40,8 @@ command -v ansible-playbook >/dev/null 2>&1 || die "ansible-playbook not found" [[ -f "$INVENTORY" ]] || die "inventory not found: $INVENTORY" run_health_check() { - ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.command -a "/usr/local/bin/aw-health-check" + ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.command -a "/usr/local/bin/aw-health-check" && + ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.command -a "/usr/local/bin/dlp-health-check" } restart_server_components() { diff --git a/scripts/dlp-health-check.py b/scripts/dlp-health-check.py new file mode 100644 index 0000000..e22e37e --- /dev/null +++ b/scripts/dlp-health-check.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib import request + + +def _env(name: str, default: str) -> str: + value = os.environ.get(name) + return value if value not in (None, "") else default + + +def _http_json(url: str, timeout: int = 10) -> Any: + with request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def _parse_ts(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) + except ValueError: + return None + + +def _now_utc() -> datetime: + return datetime.now(UTC) + + +def _age_seconds(ts: datetime | None, now: datetime) -> int | None: + if ts is None: + return None + return max(0, int((now - ts).total_seconds())) + + +def _run_systemctl(*args: str) -> tuple[int, str]: + proc = subprocess.run( + ["systemctl", *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + return proc.returncode, proc.stdout.strip() + + +@dataclass +class CheckResult: + name: str + status: str + summary: str + details: dict[str, Any] + + +class HealthReport: + def __init__(self) -> None: + self.results: list[CheckResult] = [] + + def add(self, name: str, status: str, summary: str, **details: Any) -> None: + self.results.append(CheckResult(name=name, status=status, summary=summary, details=details)) + + @property + def ok(self) -> bool: + return not any(item.status == "fail" for item in self.results) + + def as_dict(self) -> dict[str, Any]: + counts = {"ok": 0, "warn": 0, "fail": 0} + for item in self.results: + counts[item.status] = counts.get(item.status, 0) + 1 + return { + "ok": self.ok, + "counts": counts, + "results": [ + { + "name": item.name, + "status": item.status, + "summary": item.summary, + "details": item.details, + } + for item in self.results + ], + } + + def render_text(self) -> str: + icon = {"ok": "OK", "warn": "WARN", "fail": "FAIL"} + lines = ["=== DLP Health Check ===", f"Timestamp: {_now_utc().isoformat().replace('+00:00', 'Z')}", ""] + for item in self.results: + lines.append(f"[{icon.get(item.status, item.status.upper())}] {item.name}: {item.summary}") + if item.details: + lines.append(f" details: {json.dumps(item.details, ensure_ascii=False, sort_keys=True)}") + lines.append("") + lines.append(f"Overall: {'OK' if self.ok else 'FAIL'}") + return "\n".join(lines) + + +def check_http_endpoint(report: HealthReport, name: str, url: str) -> None: + try: + payload = _http_json(url) + report.add(name, "ok", f"HTTP endpoint responded", url=url, payload=payload) + except Exception as exc: + report.add(name, "fail", f"HTTP endpoint failed: {exc}", url=url) + + +def check_systemd_unit(report: HealthReport, unit: str, kind: str) -> None: + active_rc, active_out = _run_systemctl("is-active", unit) + enabled_rc, enabled_out = _run_systemctl("is-enabled", unit) + exists_rc, _ = _run_systemctl("status", unit) + if exists_rc != 0 and active_rc != 0 and enabled_rc != 0: + report.add(f"systemd:{unit}", "warn", "unit not installed", kind=kind) + return + + if active_rc == 0 and enabled_rc == 0: + report.add(f"systemd:{unit}", "ok", "active and enabled", kind=kind) + return + + report.add( + f"systemd:{unit}", + "fail", + "unit is not active/enabled", + kind=kind, + active=active_out or str(active_rc), + enabled=enabled_out or str(enabled_rc), + ) + + +def _latest_bucket_ts(api_base: str, bucket_id: str, bucket_meta: dict[str, Any]) -> datetime | None: + meta = bucket_meta.get("metadata") or {} + ts = _parse_ts(meta.get("end")) + if ts is not None: + return ts + try: + events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1") + except Exception: + return None + if isinstance(events, list) and events: + return _parse_ts(events[0].get("timestamp")) + return None + + +def check_bucket_group( + report: HealthReport, + api_base: str, + buckets: dict[str, Any], + name: str, + prefix: str, + max_age_seconds: int, + severity_if_missing: str = "fail", + severity_if_stale: str = "fail", +) -> None: + now = _now_utc() + matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) + if not matched: + report.add( + f"buckets:{name}", + severity_if_missing, + f"no buckets matched prefix {prefix}", + prefix=prefix, + ) + return + + stale: list[dict[str, Any]] = [] + unknown: list[str] = [] + ages: dict[str, int] = {} + for bucket_id in matched: + ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) + age = _age_seconds(ts, now) + if age is None: + unknown.append(bucket_id) + continue + ages[bucket_id] = age + if age > max_age_seconds: + stale.append({"bucket": bucket_id, "age_seconds": age}) + + status = "ok" + summary = f"{len(matched)} buckets, freshest ok" + if stale: + status = severity_if_stale + summary = f"{len(stale)} stale buckets" + elif unknown: + status = "warn" + summary = f"{len(unknown)} buckets without timestamp" + + report.add( + f"buckets:{name}", + status, + summary, + prefix=prefix, + max_age_seconds=max_age_seconds, + bucket_count=len(matched), + max_observed_age_seconds=max(ages.values()) if ages else None, + stale=stale, + unknown=unknown, + ) + + +def check_endpoint_self_test_metrics(report: HealthReport, api_base: str, buckets: dict[str, Any]) -> None: + missing: list[str] = [] + expected = ("queueDepth", "eventsEnqueued", "eventsFlushed", "sendFailures") + for bucket_id in sorted(k for k in buckets if k.startswith("aw-dlp-endpoint-signals_")): + try: + events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit=20") + except Exception as exc: + report.add(f"endpoint-self-test:{bucket_id}", "warn", f"failed to read events: {exc}", bucket=bucket_id) + continue + found = False + if isinstance(events, list): + for event in events: + data = event.get("data") or {} + if data.get("signalType") == "self_test" and all(key in data for key in expected): + found = True + break + if not found: + missing.append(bucket_id) + + if missing: + report.add("endpoint-self-test-metrics", "warn", "missing transport metrics in recent self_test events", buckets=missing) + else: + report.add("endpoint-self-test-metrics", "ok", "recent self_test metrics present") + + +def check_compliance_reports(report: HealthReport, report_dir: Path, profiles: list[str], month: str) -> None: + missing: list[str] = [] + present: list[str] = [] + for profile in profiles: + for suffix in ("html", "json"): + path = report_dir / f"{profile}-{month}.{suffix}" + if path.exists(): + present.append(str(path)) + else: + missing.append(str(path)) + if missing: + report.add("compliance-reports", "fail", "missing expected compliance report artifacts", present=present, missing=missing) + else: + report.add("compliance-reports", "ok", "all expected compliance artifacts exist", present=present) + + +def main() -> int: + parser = argparse.ArgumentParser(description="AWatch DLP health check") + parser.add_argument("--aw-server", default=_env("AW_HEALTH_AW_SERVER", "http://127.0.0.1:5600")) + parser.add_argument("--policy-server", default=_env("AW_HEALTH_POLICY_SERVER", "http://127.0.0.1:5601")) + parser.add_argument("--case-server", default=_env("AW_HEALTH_CASE_SERVER", "http://127.0.0.1:5602")) + parser.add_argument("--max-age-seconds", type=int, default=int(_env("AW_HEALTH_MAX_AGE_SECONDS", "900"))) + parser.add_argument("--strict-fileops", action="store_true", default=_env("AW_HEALTH_STRICT_FILEOPS", "0").lower() in {"1", "true", "yes", "on"}) + parser.add_argument("--report-dir", default=_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports")) + parser.add_argument("--profiles", default=_env("AW_DLP_COMPLIANCE_PROFILES", "152-fz,pci-dss")) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + report = HealthReport() + aw_api_base = args.aw_server.rstrip("/") + "/api/0" + + check_http_endpoint(report, "http:aw", f"{aw_api_base}/info") + check_http_endpoint(report, "http:policy", args.policy_server.rstrip("/") + "/healthz") + check_http_endpoint(report, "http:cases", args.case_server.rstrip("/") + "/health") + + for unit in ( + "activitywatch-server", + "aw-dlp-policy-engine.service", + "aw-dlp-case-management.service", + "aw-worktime-api.service", + ): + check_systemd_unit(report, unit, "service") + + for unit in ( + "aw-dlp-report-scheduler.timer", + "aw-dlp-syslog-forwarder.timer", + "aw-dlp-webhook-sender.timer", + "aw-dlp-cef-exporter.timer", + "activitywatch-dlp-aggregator.timer", + "aw-dlp-ioc-refresh.timer", + "aw-worktime-ui-bridge.timer", + ): + check_systemd_unit(report, unit, "timer") + + try: + buckets = _http_json(f"{aw_api_base}/buckets") + if not isinstance(buckets, dict): + raise RuntimeError("bucket list is not a dict") + report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets)) + check_bucket_group(report, aw_api_base, buckets, "endpoint-signals", "aw-dlp-endpoint-signals_", args.max_age_seconds) + check_bucket_group( + report, + aw_api_base, + buckets, + "file-operations", + "aw-file-operations_", + args.max_age_seconds, + severity_if_missing="warn", + severity_if_stale="fail" if args.strict_fileops else "warn", + ) + check_bucket_group(report, aw_api_base, buckets, "incidents", "aw-dlp-incidents_", args.max_age_seconds * 24, severity_if_missing="warn", severity_if_stale="warn") + check_endpoint_self_test_metrics(report, aw_api_base, buckets) + except Exception as exc: + report.add("aw:buckets-index", "fail", f"failed to inspect bucket index: {exc}") + + month = _now_utc().strftime("%Y-%m") + profiles = [x.strip() for x in args.profiles.split(",") if x.strip()] + check_compliance_reports(report, Path(args.report_dir), profiles, month) + + payload = report.as_dict() + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(report.render_text()) + return 0 if payload["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main())