#!/usr/bin/env python3 from __future__ import annotations import argparse import json import os import subprocess import tempfile import time from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any from urllib import request ENV_FILE = Path("/etc/activitywatch/aw-server.env") DEFAULT_STATE_DIR = Path("/var/lib/activitywatch/slo") DEFAULT_HEALTHD_CMD = "/usr/local/bin/aw-rus-healthd.py --json" DEFAULT_HEALTHD_STATE_FILE = Path("/var/lib/activitywatch/health/aw-rus-health.json") def load_env_file(path: Path) -> None: if not path.exists(): return try: lines = path.read_text(encoding="utf-8").splitlines() except PermissionError: # systemd EnvironmentFile has already loaded the variables for the service. return for raw_line in lines: line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue key, value = line.split("=", 1) os.environ.setdefault(key.strip(), value.strip().strip("'").strip('"')) def env(name: str, default: str) -> str: value = os.environ.get(name) return value if value not in (None, "") else default def now_utc() -> datetime: return datetime.now(UTC) def parse_ts(value: str) -> datetime: return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) def iso(dt: datetime) -> str: return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") def write_atomic(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: handle.write(content) tmp_name = handle.name os.replace(tmp_name, path) def chmod_if_possible(path: Path, mode: int) -> None: try: path.chmod(mode) except OSError: pass def chown_like_parent_if_possible(path: Path) -> None: try: parent_stat = path.parent.stat() os.chown(path, parent_stat.st_uid, parent_stat.st_gid) except OSError: pass def fetch_url(url: str, timeout_seconds: float, *, accept: str | None = None, attempts: int = 1) -> dict[str, Any]: started = now_utc() last_error = "" for attempt in range(1, max(1, attempts) + 1): try: req = request.Request(url, headers={"Accept": accept}) if accept else url with request.urlopen(req, timeout=timeout_seconds) as resp: body = resp.read() status = int(resp.status) content_type = resp.headers.get("Content-Type", "") ok = 200 <= status < 300 return { "ok": ok, "status": status, "body_bytes": len(body), "content_type": content_type, "body": body, "attempts": attempt, "latency_ms": int((now_utc() - started).total_seconds() * 1000), "url": url, } except Exception as exc: last_error = str(exc) if attempt < max(1, attempts): time.sleep(1) return { "ok": False, "error": last_error, "body_bytes": 0, "attempts": max(1, attempts), "latency_ms": int((now_utc() - started).total_seconds() * 1000), "url": url, } def public_probe_result(result: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in result.items() if key != "body"} def http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: return public_probe_result(fetch_url(url, timeout_seconds, attempts=2)) def html_probe( url: str, timeout_seconds: float, *, min_bytes: int, required_markers: tuple[str, ...], ) -> dict[str, Any]: result = fetch_url(url, timeout_seconds, accept="text/html", attempts=2) if not result.get("ok"): return public_probe_result(result) body = result.get("body", b"") text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) missing = [marker for marker in required_markers if marker not in text] content_type = str(result.get("content_type") or "") if len(body) < min_bytes: result["ok"] = False result["error"] = f"body too small: {len(body)} < {min_bytes}" elif "text/html" not in content_type.lower(): result["ok"] = False result["error"] = f"unexpected content-type: {content_type or 'unknown'}" elif missing: result["ok"] = False result["error"] = "missing markers: " + ", ".join(missing) result["missing_markers"] = missing return public_probe_result(result) def json_probe( url: str, timeout_seconds: float, *, expected_values: dict[str, Any], required_keys: tuple[str, ...], ) -> dict[str, Any]: result = fetch_url(url, timeout_seconds, accept="application/json", attempts=2) if not result.get("ok"): return public_probe_result(result) body = result.get("body", b"") try: payload = json.loads(body.decode("utf-8") if isinstance(body, bytes) else str(body)) except Exception as exc: result["ok"] = False result["error"] = f"invalid json: {exc}" return public_probe_result(result) if not isinstance(payload, dict): result["ok"] = False result["error"] = "json root is not object" return public_probe_result(result) missing_keys = [key for key in required_keys if key not in payload] mismatched = { key: {"expected": expected, "actual": payload.get(key)} for key, expected in expected_values.items() if payload.get(key) != expected } if missing_keys: result["ok"] = False result["error"] = "missing json keys: " + ", ".join(missing_keys) result["missing_keys"] = missing_keys elif mismatched: result["ok"] = False result["error"] = "unexpected json values" result["mismatched_values"] = mismatched else: result["json_keys"] = sorted(payload.keys()) return public_probe_result(result) def run_healthd(command: str, timeout_seconds: int) -> dict[str, Any]: try: proc = subprocess.run( command, shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=timeout_seconds, ) except subprocess.TimeoutExpired as exc: return { "ok": False, "returncode": None, "error": f"timeout after {timeout_seconds}s", "output_tail": (exc.stdout or "")[-1000:] if isinstance(exc.stdout, str) else "", } payload: dict[str, Any] = {} try: payload = json.loads(proc.stdout or "{}") except json.JSONDecodeError: pass return { "ok": proc.returncode == 0 and bool(payload.get("ok")), "returncode": proc.returncode, "counts": payload.get("counts", {}), "payload": payload if isinstance(payload, dict) else {}, "output_tail": (proc.stdout or "")[-1000:], } def read_healthd_state(path: Path, max_age_seconds: int) -> dict[str, Any]: try: payload = json.loads(path.read_text(encoding="utf-8")) generated = parse_ts(str(payload["generated_at_utc"])) age = max(0, int((now_utc() - generated).total_seconds())) except Exception as exc: return {"ok": False, "error": str(exc), "path": str(path)} return { "ok": bool(payload.get("ok")) and age <= max_age_seconds, "counts": payload.get("counts", {}), "age_seconds": age, "path": str(path), "payload": payload if isinstance(payload, dict) else {}, } def load_samples(path: Path, cutoff: datetime) -> list[dict[str, Any]]: samples: list[dict[str, Any]] = [] if not path.exists(): return samples try: lines = path.read_text(encoding="utf-8").splitlines() except PermissionError: return samples for line in lines: if not line.strip(): continue try: item = json.loads(line) ts = parse_ts(str(item["ts"])) except Exception: continue if ts >= cutoff: samples.append(item) return samples def append_and_trim_sample(path: Path, sample: dict[str, Any], retention_seconds: int) -> list[dict[str, Any]]: cutoff = parse_ts(sample["ts"]) - timedelta(seconds=retention_seconds) samples = load_samples(path, cutoff) samples.append(sample) content = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n" for item in samples) write_atomic(path, content) chmod_if_possible(path, 0o644) chown_like_parent_if_possible(path) return samples def summarize_window( samples: list[dict[str, Any]], *, now: datetime, window_seconds: int, sample_interval_seconds: int, target_percent: float, ) -> dict[str, Any]: cutoff = now - timedelta(seconds=window_seconds) window = [] for item in samples: try: if parse_ts(str(item["ts"])) >= cutoff: window.append(item) except Exception: continue total = len(window) good = sum(1 for item in window if item.get("ok") is True) bad = total - good availability = round((good / total) * 100, 5) if total else None observed_bad_seconds = bad * sample_interval_seconds budget_seconds = int(window_seconds * ((100.0 - target_percent) / 100.0)) budget_remaining_seconds = budget_seconds - observed_bad_seconds status = "unknown" if total: status = "ok" if budget_remaining_seconds >= 0 else "burning" return { "window_seconds": window_seconds, "samples": total, "good_samples": good, "bad_samples": bad, "availability_percent": availability, "target_percent": target_percent, "observed_bad_seconds": observed_bad_seconds, "budget_seconds": budget_seconds, "budget_remaining_seconds": budget_remaining_seconds, "status": status, } def render_summary_text(summary: dict[str, Any]) -> str: lines = [ "=== AW-RUS SLO ===", f"Timestamp: {summary['generated_at_utc']}", f"Target: {summary['target_percent']}%", "", ] for name, data in summary["windows"].items(): availability = data["availability_percent"] availability_text = "n/a" if availability is None else f"{availability:.5f}%" remaining = data["budget_remaining_seconds"] lines.append( f"{name}: {data['status']} availability={availability_text} " f"samples={data['samples']} bad={data['bad_samples']} " f"bad_seconds={data['observed_bad_seconds']} budget_remaining_seconds={remaining}" ) lines.append("") lines.append(f"Current sample: {'OK' if summary['current_sample']['ok'] else 'FAIL'}") for name, probe in summary["current_sample"].get("probes", {}).items(): marker = "OK" if probe.get("ok") else "FAIL" detail = probe.get("status", probe.get("error", "")) lines.append(f"- {name}: {marker} {detail}") return "\n".join(lines) def build_sample(args: argparse.Namespace) -> dict[str, Any]: ts = iso(now_utc()) if args.healthd_mode == "run": health = run_healthd(args.healthd_cmd, args.health_timeout_seconds) else: health = read_healthd_state(Path(args.healthd_state_file), args.healthd_state_max_age_seconds) probes = { "aw_webui_index": html_probe( args.aw_webui_url, args.http_timeout_seconds, min_bytes=1000, required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), ), "worktime_today_html": html_probe( args.worktime_today_html_url, args.http_timeout_seconds, min_bytes=5000, required_markers=("AW-rus", ""), ), "worktime_management_html": html_probe( args.worktime_management_html_url, args.http_timeout_seconds, min_bytes=5000, required_markers=("AW-rus", ""), ), "worktime_today_csv": http_probe(args.worktime_today_csv_url, args.http_timeout_seconds), "worktime_management_json": json_probe( args.worktime_management_json_url, args.http_timeout_seconds, expected_values={"host": args.host}, required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), ), } ok = bool(health.get("ok")) and all(probe.get("ok") for probe in probes.values()) return { "ts": ts, "ok": ok, "healthd_ok": bool(health.get("ok")), "healthd_counts": health.get("counts", {}), "probes": probes, } def main() -> int: load_env_file(ENV_FILE) parser = argparse.ArgumentParser(description="AW-RUS rolling SLO sampler") parser.add_argument("--state-dir", default=env("AW_RUS_SLO_STATE_DIR", str(DEFAULT_STATE_DIR))) parser.add_argument("--healthd-cmd", default=env("AW_RUS_SLO_HEALTHD_CMD", DEFAULT_HEALTHD_CMD)) parser.add_argument("--healthd-mode", choices=["state", "run"], default=env("AW_RUS_SLO_HEALTHD_MODE", "state")) parser.add_argument("--healthd-state-file", default=env("AW_RUS_SLO_HEALTHD_STATE_FILE", str(DEFAULT_HEALTHD_STATE_FILE))) parser.add_argument("--healthd-state-max-age-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", "180"))) parser.add_argument("--aw-base", default=env("AW_RUS_SLO_AW_BASE", env("AW_SERVER_URL", "http://127.0.0.1:5600"))) parser.add_argument("--worktime-base", default=env("AW_RUS_SLO_WORKTIME_BASE", env("AW_RUS_HEALTH_WORKTIME_API", "http://127.0.0.1:5610"))) parser.add_argument("--host", default=env("AW_RUS_SLO_HOST", env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025"))) parser.add_argument("--target-percent", type=float, default=float(env("AW_RUS_SLO_TARGET_PERCENT", "99.97"))) parser.add_argument("--sample-interval-seconds", type=int, default=int(env("AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", "15"))) parser.add_argument("--retention-days", type=int, default=int(env("AW_RUS_SLO_RETENTION_DAYS", "35"))) parser.add_argument("--http-timeout-seconds", type=float, default=float(env("AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", "15"))) parser.add_argument("--health-timeout-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", "90"))) parser.add_argument("--json", action="store_true") args = parser.parse_args() aw_base = args.aw_base.rstrip("/") worktime_base = args.worktime_base.rstrip("/") args.aw_webui_url = f"{aw_base}/" args.worktime_today_html_url = f"{worktime_base}/reports/worktime/today?format=html&day=today&host={args.host}&allow_stale=1" args.worktime_management_html_url = f"{worktime_base}/reports/worktime/management?format=html&day=today&host={args.host}&allow_stale=1" args.worktime_today_csv_url = f"{worktime_base}/reports/worktime/today?format=csv&day=today&host={args.host}&allow_stale=1" args.worktime_management_json_url = f"{worktime_base}/reports/worktime/management?format=json&day=today&host={args.host}&allow_stale=1" state_dir = Path(args.state_dir) sample_path = state_dir / "aw-slo-samples.jsonl" sample = build_sample(args) samples = append_and_trim_sample( sample_path, sample, retention_seconds=max(1, args.retention_days) * 86400, ) generated_at = now_utc() summary = { "generated_at_utc": iso(generated_at), "target_percent": args.target_percent, "sample_interval_seconds": args.sample_interval_seconds, "current_sample": sample, "windows": { "24h": summarize_window( samples, now=generated_at, window_seconds=86400, sample_interval_seconds=args.sample_interval_seconds, target_percent=args.target_percent, ), "7d": summarize_window( samples, now=generated_at, window_seconds=7 * 86400, sample_interval_seconds=args.sample_interval_seconds, target_percent=args.target_percent, ), "30d": summarize_window( samples, now=generated_at, window_seconds=30 * 86400, sample_interval_seconds=args.sample_interval_seconds, target_percent=args.target_percent, ), }, } summary_json_path = state_dir / "aw-slo-summary.json" summary_txt_path = state_dir / "aw-slo-summary.txt" write_atomic(summary_json_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") text = render_summary_text(summary) write_atomic(summary_txt_path, text + "\n") chmod_if_possible(summary_json_path, 0o644) chmod_if_possible(summary_txt_path, 0o644) chown_like_parent_if_possible(summary_json_path) chown_like_parent_if_possible(summary_txt_path) if args.json: print(json.dumps(summary, ensure_ascii=False, indent=2)) else: print(text) return 0 if sample["ok"] else 1 if __name__ == "__main__": raise SystemExit(main())