feat(detmir): add rust-first operations tooling

scripts/dlp-health-check.py

@@ -6,6 +6,7 @@ import json
 import os
 import subprocess
 import sys
+import time
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
@@ -18,9 +19,19 @@ def _env(name: str, default: str) -> str:
     return value if value not in (None, "") else default
 
 
-def _http_json(url: str, timeout: int = 10) -> Any:
-    with request.urlopen(url, timeout=timeout) as resp:
-        return json.loads(resp.read().decode("utf-8"))
+def _http_json(url: str, timeout: int = 15, attempts: int = 2, backoff_seconds: float = 0.5) -> Any:
+    last_exc: Exception | None = None
+    for attempt in range(max(1, attempts)):
+        try:
+            with request.urlopen(url, timeout=timeout) as resp:
+                return json.loads(resp.read().decode("utf-8"))
+        except Exception as exc:
+            last_exc = exc
+            if attempt + 1 >= max(1, attempts):
+                break
+            time.sleep(backoff_seconds * (2**attempt))
+    assert last_exc is not None
+    raise last_exc
 
 
 def _parse_ts(value: str | None) -> datetime | None:
@@ -42,6 +53,76 @@ def _age_seconds(ts: datetime | None, now: datetime) -> int | None:
     return max(0, int((now - ts).total_seconds()))
 
 
+def _int_or_zero(value: Any) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _path_tail(value: Any, parts: int = 2) -> str:
+    text = str(value or "").replace("\\", "/").strip()
+    if not text:
+        return ""
+    tokens = [item for item in text.split("/") if item]
+    return "/".join(tokens[-parts:]) if tokens else text
+
+
+def _text_excerpt(value: Any, limit: int = 120) -> str:
+    text = " ".join(str(value or "").split())
+    if len(text) <= limit:
+        return text
+    return text[: max(0, limit - 1)].rstrip() + "…"
+
+
+def _load_counter_state(path: Path) -> dict[str, Any]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        return {"counters": {}}
+    except Exception:
+        return {"counters": {}}
+    if not isinstance(payload, dict):
+        return {"counters": {}}
+    counters = payload.get("counters")
+    if not isinstance(counters, dict):
+        payload["counters"] = {}
+    return payload
+
+
+def _save_counter_state(path: Path, state: dict[str, Any]) -> str | None:
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp_path = path.parent / f".{path.name}.{os.getpid()}.tmp"
+        tmp_path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        os.replace(tmp_path, path)
+        return None
+    except Exception as exc:
+        try:
+            path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+            return None
+        except Exception:
+            return str(exc)
+
+
+def _counter_delta(counter_state: dict[str, Any] | None, key: str, current_value: int) -> tuple[int | None, int]:
+    if counter_state is None:
+        return None, current_value
+    counters = counter_state.setdefault("counters", {})
+    if not isinstance(counters, dict):
+        counters = {}
+        counter_state["counters"] = counters
+    previous: int | None = None
+    try:
+        previous = int(counters[key])
+    except (KeyError, TypeError, ValueError):
+        previous = None
+    counters[key] = current_value
+    if previous is None or current_value < previous:
+        return previous, 0
+    return previous, current_value - previous
+
+
 def _run_systemctl(*args: str) -> tuple[int, str]:
     proc = subprocess.run(
         ["systemctl", *args],
@@ -268,6 +349,134 @@ def check_incident_buckets(
     )
 
 
+def check_incident_runtime(
+    report: HealthReport,
+    api_base: str,
+    buckets: dict[str, Any],
+    sample_limit: int = 20,
+) -> None:
+    now = _now_utc()
+    prefix = "aw-dlp-incidents_"
+    matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix))
+    if not matched:
+        report.add("incident-runtime", "ok", "no incident buckets to sample", bucket_count=0)
+        return
+
+    if sample_limit <= 0:
+        metadata = []
+        for bucket_id in matched:
+            ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {}))
+            metadata.append(
+                {
+                    "bucket": bucket_id,
+                    "end": ts.isoformat().replace("+00:00", "Z") if ts else None,
+                    "age_seconds": _age_seconds(ts, now),
+                }
+            )
+        report.add(
+            "incident-runtime",
+            "ok",
+            "incident event sampling disabled",
+            bucket_count=len(matched),
+            sample_limit=sample_limit,
+            metadata=metadata,
+        )
+        return
+
+    sampled: list[dict[str, Any]] = []
+    latest_incidents: list[dict[str, Any]] = []
+    read_failed: list[dict[str, str]] = []
+    totals = {
+        "sampled_events": 0,
+        "real_incidents": 0,
+        "self_tests": 0,
+    }
+    severity_counts: dict[str, int] = {}
+    action_counts: dict[str, int] = {}
+    rule_counts: dict[str, int] = {}
+
+    for bucket_id in matched:
+        try:
+            events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}", timeout=5, attempts=1)
+        except Exception as exc:
+            read_failed.append({"bucket": bucket_id, "error": str(exc)})
+            continue
+        if not isinstance(events, list):
+            read_failed.append({"bucket": bucket_id, "error": "events response is not a list"})
+            continue
+
+        bucket_summary = {
+            "bucket": bucket_id,
+            "sampled_events": len(events),
+            "real_incidents": 0,
+            "self_tests": 0,
+        }
+        sampled.append(bucket_summary)
+        totals["sampled_events"] += len(events)
+
+        for event in events:
+            data = event.get("data") or {}
+            if not isinstance(data, dict):
+                continue
+
+            signal_type = str(data.get("signalType") or "").strip()
+            source = str(data.get("source") or "").strip()
+            rule_id = str(data.get("ruleId") or data.get("rule_id") or "").strip()
+            is_self_test = signal_type == "self_test" or source == "self-test" or rule_id.startswith("selftest-")
+            if is_self_test:
+                bucket_summary["self_tests"] += 1
+                totals["self_tests"] += 1
+                continue
+
+            ts = _parse_ts(event.get("timestamp"))
+            severity = str(data.get("severity") or "unknown").strip().lower() or "unknown"
+            action = str(data.get("action") or "unknown").strip().lower() or "unknown"
+            rule_key = rule_id or "unknown"
+            severity_counts[severity] = severity_counts.get(severity, 0) + 1
+            action_counts[action] = action_counts.get(action, 0) + 1
+            rule_counts[rule_key] = rule_counts.get(rule_key, 0) + 1
+            bucket_summary["real_incidents"] += 1
+            totals["real_incidents"] += 1
+            latest_incidents.append(
+                {
+                    "bucket": bucket_id,
+                    "timestamp": event.get("timestamp"),
+                    "age_seconds": _age_seconds(ts, now),
+                    "ruleId": rule_id,
+                    "severity": severity,
+                    "action": action,
+                    "username": str(data.get("username") or ""),
+                    "hostname": str(data.get("hostname") or ""),
+                    "source": source,
+                    "message_excerpt": _text_excerpt(data.get("message")),
+                }
+            )
+
+    latest_incidents.sort(key=lambda item: item.get("timestamp") or "", reverse=True)
+    status = "ok"
+    summary = f"{totals['real_incidents']} real incidents in sampled events"
+    if read_failed:
+        status = "warn"
+        summary = f"{len(read_failed)} incident buckets failed to sample"
+    elif totals["real_incidents"] == 0:
+        summary = "no real incidents in sampled events"
+
+    report.add(
+        "incident-runtime",
+        status,
+        summary,
+        bucket_count=len(matched),
+        sample_limit=sample_limit,
+        totals=totals,
+        sampled=sampled,
+        severity_counts=severity_counts,
+        action_counts=action_counts,
+        rule_counts=dict(sorted(rule_counts.items(), key=lambda item: (-item[1], item[0]))[:10]),
+        latest_incidents=latest_incidents[:5],
+        read_failed=read_failed,
+    )
+
+
 def _worktime_activity_map(api_base: str, buckets: dict[str, Any], max_age_seconds: int) -> dict[str, dict[str, Any]]:
     now = _now_utc()
     activity: dict[str, dict[str, Any]] = {}
@@ -377,8 +586,239 @@ def check_file_operations_buckets(
     )
 
 
-def check_endpoint_self_test_metrics(report: HealthReport, api_base: str, buckets: dict[str, Any]) -> None:
+def check_file_operations_runtime(
+    report: HealthReport,
+    api_base: str,
+    buckets: dict[str, Any],
+    sample_limit: int = 20,
+    queue_warn_depth: int = 100,
+    send_failure_warn_count: int = 1,
+    counter_state: dict[str, Any] | None = None,
+) -> None:
+    now = _now_utc()
+    prefix = "aw-file-operations_"
+    matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix))
+    if not matched:
+        report.add("file-operations-runtime", "warn", "no file-operations buckets to sample", bucket_count=0)
+        return
+
+    sampled: list[dict[str, Any]] = []
+    latest_operations: list[dict[str, Any]] = []
+    latest_health: list[dict[str, Any]] = []
+    warnings: list[dict[str, Any]] = []
+    read_failed: list[dict[str, str]] = []
+
+    for bucket_id in matched:
+        try:
+            events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}")
+        except Exception as exc:
+            read_failed.append({"bucket": bucket_id, "error": str(exc)})
+            continue
+        if not isinstance(events, list):
+            read_failed.append({"bucket": bucket_id, "error": "events response is not a list"})
+            continue
+
+        operation_counts: dict[str, int] = {}
+        latest_health_event: dict[str, Any] | None = None
+        latest_health_ts: datetime | None = None
+
+        for event in events:
+            data = event.get("data") or {}
+            if not isinstance(data, dict):
+                continue
+            ts = _parse_ts(event.get("timestamp"))
+            signal_type = str(data.get("signalType") or "")
+            operation = str(data.get("operation") or "")
+
+            if signal_type == "collector_health":
+                if latest_health_event is None or (ts is not None and (latest_health_ts is None or ts > latest_health_ts)):
+                    latest_health_event = event
+                    latest_health_ts = ts
+                continue
+
+            if operation:
+                operation_counts[operation] = operation_counts.get(operation, 0) + 1
+                latest_operations.append(
+                    {
+                        "bucket": bucket_id,
+                        "timestamp": event.get("timestamp"),
+                        "age_seconds": _age_seconds(ts, now),
+                        "operation": operation,
+                        "username": str(data.get("username") or ""),
+                        "hostname": str(data.get("hostname") or ""),
+                        "extension": str(data.get("extension") or ""),
+                        "archiveHint": bool(data.get("archiveHint")),
+                        "path_tail": _path_tail(data.get("path")),
+                        "size": _int_or_zero(data.get("size")),
+                    }
+                )
+
+        bucket_sample = {
+            "bucket": bucket_id,
+            "sampled_events": len(events),
+            "operation_counts": operation_counts,
+        }
+        sampled.append(bucket_sample)
+
+        if latest_health_event is None:
+            warnings.append({"bucket": bucket_id, "metric": "collector_health", "value": "missing_in_sample"})
+            continue
+
+        health_data = latest_health_event.get("data") or {}
+        send_failures = _int_or_zero(health_data.get("sendFailures"))
+        previous_send_failures, send_failures_delta = _counter_delta(
+            counter_state,
+            f"file-operations:{bucket_id}:sendFailures",
+            send_failures,
+        )
+        health_item = {
+            "bucket": bucket_id,
+            "timestamp": latest_health_event.get("timestamp"),
+            "age_seconds": _age_seconds(latest_health_ts, now),
+            "queueDepth": _int_or_zero(health_data.get("queueDepth")),
+            "eventsEnqueued": _int_or_zero(health_data.get("eventsEnqueued")),
+            "eventsFlushed": _int_or_zero(health_data.get("eventsFlushed")),
+            "sendFailures": send_failures,
+            "sendFailuresPrevious": previous_send_failures,
+            "sendFailuresDelta": send_failures_delta,
+            "username": str(health_data.get("username") or ""),
+            "hostname": str(health_data.get("hostname") or ""),
+            "sessionId": _int_or_zero(health_data.get("sessionId")),
+        }
+        latest_health.append(health_item)
+        if health_item["queueDepth"] > queue_warn_depth:
+            warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": health_item["queueDepth"], "threshold": queue_warn_depth})
+        if send_failure_warn_count > 0 and health_item["sendFailuresDelta"] >= send_failure_warn_count:
+            warnings.append(
+                {
+                    "bucket": bucket_id,
+                    "metric": "sendFailuresDelta",
+                    "value": health_item["sendFailuresDelta"],
+                    "current": health_item["sendFailures"],
+                    "previous": health_item["sendFailuresPrevious"],
+                    "threshold": send_failure_warn_count,
+                }
+            )
+
+    latest_operations.sort(key=lambda item: item.get("timestamp") or "", reverse=True)
+    status = "ok"
+    summary = f"{len(matched)} file-operations buckets sampled"
+    if read_failed:
+        status = "warn"
+        summary = f"{len(read_failed)} file-operations buckets failed to sample"
+    elif warnings:
+        status = "warn"
+        summary = "file-operations runtime counters outside expectations"
+
+    report.add(
+        "file-operations-runtime",
+        status,
+        summary,
+        bucket_count=len(matched),
+        sample_limit=sample_limit,
+        sampled=sampled,
+        latest_health=latest_health,
+        latest_operations=latest_operations[:5],
+        warnings=warnings,
+        read_failed=read_failed,
+        thresholds={
+            "queueDepth": queue_warn_depth,
+            "sendFailures": send_failure_warn_count,
+        },
+    )
+
+
+def check_endpoint_signal_buckets(
+    report: HealthReport,
+    api_base: str,
+    buckets: dict[str, Any],
+    max_age_seconds: int,
+) -> None:
+    now = _now_utc()
+    prefix = "aw-dlp-endpoint-signals_"
+    matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix))
+    worktime = _worktime_activity_map(api_base, buckets, max_age_seconds)
+    active_hosts = sorted(host for host, meta in worktime.items() if meta.get("active"))
+    matched_by_host = {_bucket_suffix(bucket_id, prefix): bucket_id for bucket_id in matched}
+
+    ignored_unmanaged: list[str] = []
+    ignored_inactive: list[str] = []
+    missing_active: list[str] = []
+    stale: list[dict[str, Any]] = []
+    unknown: list[str] = []
+    fresh: list[str] = []
+
+    for host, bucket_id in matched_by_host.items():
+        if host not in worktime:
+            ignored_unmanaged.append(bucket_id)
+            continue
+        if host not in active_hosts:
+            ignored_inactive.append(bucket_id)
+            continue
+        ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {}))
+        age = _age_seconds(ts, now)
+        if age is None:
+            unknown.append(bucket_id)
+            continue
+        if age > max_age_seconds:
+            stale.append({"bucket": bucket_id, "age_seconds": age})
+        else:
+            fresh.append(bucket_id)
+
+    for host in active_hosts:
+        if host not in matched_by_host:
+            missing_active.append(host)
+
+    if not active_hosts:
+        report.add(
+            "buckets:endpoint-signals",
+            "ok",
+            "no active managed hosts require endpoint-signals freshness",
+            active_hosts=[],
+            ignored_unmanaged=ignored_unmanaged,
+            ignored_inactive=ignored_inactive,
+            worktime_hosts=sorted(worktime),
+        )
+        return
+
+    status = "ok"
+    summary = f"{len(fresh)} active host buckets fresh"
+    if missing_active:
+        status = "fail"
+        summary = f"{len(missing_active)} active hosts missing endpoint-signals buckets"
+    elif stale:
+        status = "fail"
+        summary = f"{len(stale)} active host buckets stale"
+    elif unknown:
+        status = "warn"
+        summary = f"{len(unknown)} active host buckets without timestamp"
+
+    report.add(
+        "buckets:endpoint-signals",
+        status,
+        summary,
+        active_hosts=active_hosts,
+        fresh=fresh,
+        stale=stale,
+        missing_active=missing_active,
+        unknown=unknown,
+        ignored_unmanaged=ignored_unmanaged,
+        ignored_inactive=ignored_inactive,
+    )
+
+
+def check_endpoint_self_test_metrics(
+    report: HealthReport,
+    api_base: str,
+    buckets: dict[str, Any],
+    queue_warn_depth: int = 100,
+    send_failure_warn_count: int = 1,
+    counter_state: dict[str, Any] | None = None,
+) -> None:
+    now = _now_utc()
     missing: list[str] = []
+    latest_self_tests: list[dict[str, Any]] = []
+    warnings: list[dict[str, Any]] = []
     expected = ("queueDepth", "eventsEnqueued", "eventsFlushed", "sendFailures")
     for bucket_id in sorted(k for k in buckets if k.startswith("aw-dlp-endpoint-signals_")):
         try:
@@ -386,20 +826,89 @@ def check_endpoint_self_test_metrics(report: HealthReport, api_base: str, bucket
         except Exception as exc:
             report.add(f"endpoint-self-test:{bucket_id}", "warn", f"failed to read events: {exc}", bucket=bucket_id)
             continue
-        found = False
+        latest_event: dict[str, Any] | None = None
+        latest_ts: datetime | None = None
         if isinstance(events, list):
             for event in events:
                 data = event.get("data") or {}
-                if data.get("signalType") == "self_test" and all(key in data for key in expected):
-                    found = True
-                    break
-        if not found:
+                if data.get("signalType") != "self_test" or not all(key in data for key in expected):
+                    continue
+                ts = _parse_ts(event.get("timestamp"))
+                if latest_event is None or (ts is not None and (latest_ts is None or ts > latest_ts)):
+                    latest_event = event
+                    latest_ts = ts
+        if latest_event is None:
             missing.append(bucket_id)
+            continue
+
+        data = latest_event.get("data") or {}
+        send_failures = _int_or_zero(data.get("sendFailures"))
+        previous_send_failures, send_failures_delta = _counter_delta(
+            counter_state,
+            f"endpoint-self-test:{bucket_id}:sendFailures",
+            send_failures,
+        )
+        item = {
+            "bucket": bucket_id,
+            "timestamp": latest_event.get("timestamp"),
+            "age_seconds": _age_seconds(latest_ts, now),
+            "queueDepth": _int_or_zero(data.get("queueDepth")),
+            "eventsEnqueued": _int_or_zero(data.get("eventsEnqueued")),
+            "eventsFlushed": _int_or_zero(data.get("eventsFlushed")),
+            "sendFailures": send_failures,
+            "sendFailuresPrevious": previous_send_failures,
+            "sendFailuresDelta": send_failures_delta,
+        }
+        latest_self_tests.append(item)
+        if item["queueDepth"] > queue_warn_depth:
+            warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": item["queueDepth"], "threshold": queue_warn_depth})
+        if send_failure_warn_count > 0 and item["sendFailuresDelta"] >= send_failure_warn_count:
+            warnings.append(
+                {
+                    "bucket": bucket_id,
+                    "metric": "sendFailuresDelta",
+                    "value": item["sendFailuresDelta"],
+                    "current": item["sendFailures"],
+                    "previous": item["sendFailuresPrevious"],
+                    "threshold": send_failure_warn_count,
+                }
+            )
 
     if missing:
-        report.add("endpoint-self-test-metrics", "warn", "missing transport metrics in recent self_test events", buckets=missing)
+        report.add(
+            "endpoint-self-test-metrics",
+            "warn",
+            "missing transport metrics in sampled self_test events",
+            buckets=missing,
+            latest_self_tests=latest_self_tests,
+            thresholds={
+                "queueDepth": queue_warn_depth,
+                "sendFailures": send_failure_warn_count,
+            },
+        )
+    elif warnings:
+        report.add(
+            "endpoint-self-test-metrics",
+            "warn",
+            "endpoint transport counters outside thresholds",
+            latest_self_tests=latest_self_tests,
+            warnings=warnings,
+            thresholds={
+                "queueDepth": queue_warn_depth,
+                "sendFailures": send_failure_warn_count,
+            },
+        )
     else:
-        report.add("endpoint-self-test-metrics", "ok", "recent self_test metrics present")
+        report.add(
+            "endpoint-self-test-metrics",
+            "ok",
+            "self_test transport metrics present",
+            latest_self_tests=latest_self_tests,
+            thresholds={
+                "queueDepth": queue_warn_depth,
+                "sendFailures": send_failure_warn_count,
+            },
+        )
 
 
 def check_compliance_reports(report: HealthReport, report_dir: Path, profiles: list[str], month: str) -> None:
@@ -425,6 +934,13 @@ def main() -> int:
     parser.add_argument("--case-server", default=_env("AW_HEALTH_CASE_SERVER", "http://127.0.0.1:5602"))
     parser.add_argument("--max-age-seconds", type=int, default=int(_env("AW_HEALTH_MAX_AGE_SECONDS", "900")))
     parser.add_argument("--strict-fileops", action="store_true", default=_env("AW_HEALTH_STRICT_FILEOPS", "0").lower() in {"1", "true", "yes", "on"})
+    parser.add_argument("--endpoint-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_QUEUE_WARN_DEPTH", "100")))
+    parser.add_argument("--endpoint-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_SEND_FAILURE_WARN_COUNT", "1")))
+    parser.add_argument("--fileops-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SAMPLE_LIMIT", "20")))
+    parser.add_argument("--fileops-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_QUEUE_WARN_DEPTH", "100")))
+    parser.add_argument("--fileops-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SEND_FAILURE_WARN_COUNT", "1")))
+    parser.add_argument("--incident-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_INCIDENT_SAMPLE_LIMIT", "0")))
+    parser.add_argument("--state-dir", default=_env("AW_DLP_HEALTH_STATE_DIR", "/var/lib/activitywatch/health"))
     parser.add_argument("--report-dir", default=_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports"))
     parser.add_argument("--profiles", default=_env("AW_DLP_COMPLIANCE_PROFILES", "152-fz,pci-dss"))
     parser.add_argument("--json", action="store_true")
@@ -432,6 +948,8 @@ def main() -> int:
 
     report = HealthReport()
     aw_api_base = args.aw_server.rstrip("/") + "/api/0"
+    counter_state_path = Path(args.state_dir) / "dlp-health-check-counters.json"
+    counter_state = _load_counter_state(counter_state_path)
 
     check_http_endpoint(report, "http:aw", f"{aw_api_base}/info")
     check_http_endpoint(report, "http:policy", args.policy_server.rstrip("/") + "/healthz")
@@ -461,13 +979,34 @@ def main() -> int:
         if not isinstance(buckets, dict):
             raise RuntimeError("bucket list is not a dict")
         report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets))
-        check_bucket_group(report, aw_api_base, buckets, "endpoint-signals", "aw-dlp-endpoint-signals_", args.max_age_seconds)
+        check_endpoint_signal_buckets(report, aw_api_base, buckets, args.max_age_seconds)
         check_file_operations_buckets(report, aw_api_base, buckets, args.max_age_seconds, args.strict_fileops)
+        check_file_operations_runtime(
+            report,
+            aw_api_base,
+            buckets,
+            args.fileops_sample_limit,
+            args.fileops_queue_warn_depth,
+            args.fileops_send_failure_warn_count,
+            counter_state,
+        )
         check_incident_buckets(report, aw_api_base, buckets, args.max_age_seconds * 24)
-        check_endpoint_self_test_metrics(report, aw_api_base, buckets)
+        check_incident_runtime(report, aw_api_base, buckets, args.incident_sample_limit)
+        check_endpoint_self_test_metrics(
+            report,
+            aw_api_base,
+            buckets,
+            args.endpoint_queue_warn_depth,
+            args.endpoint_send_failure_warn_count,
+            counter_state,
+        )
     except Exception as exc:
         report.add("aw:buckets-index", "fail", f"failed to inspect bucket index: {exc}")
 
+    state_error = _save_counter_state(counter_state_path, counter_state)
+    if state_error:
+        report.add("state:counters", "warn", f"failed to save counter baseline: {state_error}", path=str(counter_state_path))
+
     month = _now_utc().strftime("%Y-%m")
     profiles = [x.strip() for x in args.profiles.split(",") if x.strip()]
     check_compliance_reports(report, Path(args.report_dir), profiles, month)