diff --git a/.gitignore b/.gitignore index 919af6c..32b55b0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ secrets/runtime.env *.bak windows/*.report.json .rollout-logs/ +output/ +.ai/ +.autonomous/ # IDE .idea/ diff --git a/ansible/deploy_grafana_dashboards.yml b/ansible/deploy_grafana_dashboards.yml index f59ed49..1ff04fc 100644 --- a/ansible/deploy_grafana_dashboards.yml +++ b/ansible/deploy_grafana_dashboards.yml @@ -12,6 +12,10 @@ grafana_folder_title_eff: "{{ grafana_folder_title | default('AWatch-rus') }}" grafana_dashboards_import_overwrite_eff: "{{ grafana_dashboards_import_overwrite | default(true) }}" grafana_dashboards: + - file: "grafana/detmir-aw-main-dashboard.json" + uid: "detmir-aw-main" + title: "DetMir ActivityWatch" + folder_uid: "{{ grafana_detmir_folder_uid | default(grafana_folder_uid_eff) }}" - file: "grafana/detmir-rdp-user-activity-dashboard.json" uid: "detmir-rdp-user-activity" title: "DetMir: Работа пользователей в RDP" diff --git a/aw-server/aw-worktime-api.py b/aw-server/aw-worktime-api.py index 19247e4..f12324a 100644 --- a/aw-server/aw-worktime-api.py +++ b/aw-server/aw-worktime-api.py @@ -49,6 +49,8 @@ MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", " MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()} EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30"))) WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000"))) +TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180"))) +TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600"))) MODULE_PATH = Path(__file__).resolve() _ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}} _EVENTS_CACHE_LOCK = threading.Lock() @@ -85,6 +87,19 @@ def hhmm(total_seconds): return "%02d:%02d" % (total_seconds // 3600, (total_seconds % 3600) // 60) +def human_duration_ru(total_seconds): + total_seconds = max(0, int(total_seconds)) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + if hours and minutes: + return f"{hours} ч {minutes} мин" + if hours: + return f"{hours} ч" + if minutes: + return f"{minutes} мин" + return f"{total_seconds} сек" + + def now_utc(): return datetime.now(timezone.utc) @@ -397,6 +412,246 @@ def _merge_intervals(intervals): return merged +def _overlap_interval(left, right): + start = max(left[0], right[0]) + end = min(left[1], right[1]) + if end <= start: + return None + return start, end + + +def _interval_contains(interval, ts): + return interval[0] <= ts < interval[1] + + +def _event_timestamp(event): + try: + return pts(event.get("timestamp")) + except Exception: + return None + + +def _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS): + try: + duration = float(event.get("duration") or 0.0) + except Exception: + duration = 0.0 + if duration <= 0: + duration = default_seconds + return max(1.0, min(float(TRUE_ACTIVE_MAX_EVENT_SECONDS), duration)) + + +def _normalize_app_name(app, title=""): + app_raw = str(app or "").strip() + app_l = app_raw.lower() + title_raw = str(title or "").strip() + if app_l.startswith(("1cv8", "1cestart")): + return "1С" + if app_l in {"chrome.exe", "google chrome"} or "google chrome" in title_raw.lower(): + return "Chrome" + if app_l in {"msedge.exe", "microsoft edge"} or "microsoft edge" in title_raw.lower(): + return "Edge" + if app_l in {"browser.exe", "browser"} or "яндекс" in title_raw.lower(): + return "Яндекс Браузер" + if app_l in {"excel.exe"}: + return "Excel" + if app_l in {"winword.exe"}: + return "Word" + if app_l in {"powerpnt.exe"}: + return "PowerPoint" + if app_l in {"outlook.exe"}: + return "Outlook" + if app_l in {"explorer.exe"}: + return "Проводник" + if app_l in {"totalcmd.exe", "totalcmd64.exe"}: + return "Total Commander" + if app_l in {"cmd.exe"}: + return "Command Prompt" + if app_l in {"powershell.exe", "pwsh.exe"}: + return "PowerShell" + if app_l in {"acrord32.exe", "acrobat.exe"}: + return "Adobe Acrobat Reader" + if app_l in {"windowsterminal.exe", "windowsterminal"}: + return "Windows Terminal" + if app_raw: + return app_raw[:-4] if app_l.endswith(".exe") else app_raw + if title_raw: + return title_raw + return "Неизвестное приложение" + + +def _event_context(event): + data = event.get("data") or {} + for key in ("title", "url", "path", "filePath", "targetPath", "windowTitle", "foregroundTitle", "signalType"): + value = str(data.get(key) or "").strip() + if value: + return value + return "активность" + + +def _is_not_afk_event(event): + data = event.get("data") or {} + status = str(data.get("status") or data.get("state") or "").strip().lower() + return status in {"not-afk", "not_afk", "active", "активно"} + + +def _is_real_evidence_event(event): + data = event.get("data") or {} + signal_type = str(data.get("signalType") or data.get("type") or "").strip().lower() + if signal_type in {"collector_health", "self_test", "heartbeat", "health"}: + return False + if data.get("url") or data.get("title") or data.get("path") or data.get("filePath") or data.get("targetPath"): + return True + if signal_type: + return True + return False + + +def _events_for_bounds(events, start, end): + result = [] + for event in events: + ts = _event_timestamp(event) + if ts is None or ts < start or ts > end: + continue + result.append((ts, event)) + result.sort(key=lambda item: item[0]) + return result + + +def _build_window_intervals(window_events, start, end): + intervals = [] + previous_key = None + for ts, event in _events_for_bounds(window_events, start, end): + data = event.get("data") or {} + app = str(data.get("app") or data.get("process") or data.get("processName") or "").strip() + title = str(data.get("title") or data.get("windowTitle") or "").strip() + if not app and not title: + continue + duration = _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS) + interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) + if interval[1] <= interval[0]: + continue + app_name = _normalize_app_name(app, title) + current_key = (app_name, title) + title_changed = previous_key is not None and current_key != previous_key + previous_key = current_key + intervals.append( + { + "app": app_name, + "raw_app": app, + "title": title, + "start": interval[0], + "end": interval[1], + "title_changed": title_changed, + "timestamp": ts, + } + ) + return intervals + + +def _build_not_afk_intervals(afk_events, start, end): + intervals = [] + for ts, event in _events_for_bounds(afk_events, start, end): + if not _is_not_afk_event(event): + continue + duration = _event_duration_seconds(event, default_seconds=5) + interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) + if interval[1] > interval[0]: + intervals.append(interval) + return _merge_intervals(intervals) + + +def _find_window_at(window_intervals, ts): + for item in window_intervals: + if item["start"] <= ts < item["end"]: + return item + return None + + +def _add_app_evidence(evidence_by_app, app, ts, context): + evidence_by_app.setdefault(app, []).append((ts, str(context or "").strip() or "активность")) + + +def build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end): + window_intervals = _build_window_intervals(window_events, start, end) + not_afk_intervals = _build_not_afk_intervals(afk_events, start, end) + evidence_by_app = {} + + for item in window_intervals: + if item["title_changed"]: + _add_app_evidence(evidence_by_app, item["app"], item["timestamp"], item["title"] or item["raw_app"]) + + for events in evidence_events_by_bucket.values(): + for ts, event in _events_for_bounds(events, start, end): + if not _is_real_evidence_event(event): + continue + window = _find_window_at(window_intervals, ts) + if window is None: + continue + _add_app_evidence(evidence_by_app, window["app"], ts, _event_context(event)) + + rows = [] + evidence_delta = timedelta(seconds=TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS) + for app in sorted({item["app"] for item in window_intervals} | set(evidence_by_app)): + app_evidence = sorted(evidence_by_app.get(app, []), key=lambda item: item[0]) + if not app_evidence: + continue + evidence_windows = _merge_intervals([(ts - evidence_delta, ts + evidence_delta) for ts, _context in app_evidence]) + proved_intervals = [] + for window in [item for item in window_intervals if item["app"] == app]: + base = (window["start"], window["end"]) + for afk_interval in not_afk_intervals: + active_overlap = _overlap_interval(base, afk_interval) + if active_overlap is None: + continue + for evidence_interval in evidence_windows: + proved = _overlap_interval(active_overlap, evidence_interval) + if proved is not None: + proved_intervals.append(proved) + proved_intervals = _merge_intervals(proved_intervals) + proved_seconds = int(sum((right - left).total_seconds() for left, right in proved_intervals)) + if proved_seconds <= 0: + continue + last_ts, last_context = app_evidence[-1] + rows.append( + { + "application": app, + "proved_work_seconds": proved_seconds, + "proved_work_hhmm": hhmm(proved_seconds), + "proved_work_human": human_duration_ru(proved_seconds), + "last_action_utc": to_iso_utc(last_ts), + "last_action_local": last_ts.astimezone(REPORT_TZ).strftime("%H:%M"), + "last_action": last_context, + "evidence_events": len(app_evidence), + } + ) + rows.sort(key=lambda item: (-item["proved_work_seconds"], item["application"].lower())) + return rows + + +def build_true_active_apps(host, report_date): + bounds = get_report_bounds(report_date) + host = resolve_host(host) + window_events = fetch_bucket_events(f"aw-watcher-window_{host}", host) or fetch_bucket_events(f"aw-rdp-window_{host}", host) + afk_events = fetch_bucket_events(f"aw-watcher-afk_{host}", host) or fetch_bucket_events(f"aw-rdp-afk_{host}", host) + evidence_events_by_bucket = {} + for bucket_id in ( + f"aw-file-operations_{host}", + f"aw-dlp-endpoint-signals_{host}", + f"aw-watcher-web-chrome_{host}", + f"aw-watcher-web-edge_{host}", + f"aw-detmir-web-category_{host}", + ): + evidence_events_by_bucket[bucket_id] = fetch_bucket_events(bucket_id, host) + return build_true_active_apps_from_events( + window_events, + afk_events, + evidence_events_by_bucket, + bounds["start"], + bounds["end"], + ) + + def _collect_user_rows(events, start, end, host): end_exclusive = end + timedelta(seconds=1) by_user = {} @@ -1537,7 +1792,7 @@ def report_for_date_fresh(host, report_date): return module.report_for_date(host, report_date) -def render_html(rows, host, report_date, selected_day=None): +def render_html(rows, host, report_date, selected_day=None, true_active_apps=None): generated = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") date_local = report_date.strftime("%Y-%m-%d") day_query = f"&day={selected_day}" if selected_day in {"today", "yesterday"} else "" @@ -1556,6 +1811,20 @@ def render_html(rows, host, report_date, selected_day=None): ] trs = [] detail_cards = [] + true_active_apps = true_active_apps or [] + true_active_rows = [] + for app_row in true_active_apps: + last_action = app_row.get("last_action") or "-" + last_time = app_row.get("last_action_local") or "-" + true_active_rows.append( + "" + f"{html.escape(app_row.get('application') or '-')}" + f"{html.escape(app_row.get('proved_work_human') or app_row.get('proved_work_hhmm') or '0 сек')}" + f"{html.escape(last_time)} · {html.escape(last_action)}" + "" + ) + if not true_active_rows: + true_active_rows.append("Пока нет доказанной активной работы по приложениям за выбранную дату.") for row in rows: user_slug = safe_slug(row["user"]) active_seconds = int(row.get("active_seconds", 0) or 0) @@ -1811,6 +2080,21 @@ def render_html(rows, host, report_date, selected_day=None): {''.join(f"
{html.escape(label)}{html.escape(value)}
" for label, value in cards)} +
+

Доказанная работа по приложениям

+ + + + + + + + + + {''.join(true_active_rows)} + +
ПриложениеДоказанная работаПоследнее действие
+

Таблица по пользователям

@@ -2397,6 +2681,7 @@ class H(BaseHTTPRequestHandler): is_management = parsed.path == "/reports/worktime/management" management_payload = management_report_for_date(host, report_date, owner_filter=owner_filter, department_filter=department_filter) if is_management else None rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] + true_active_apps = [] if is_management else build_true_active_apps(host, report_date) if fmt == "csv": if is_management: @@ -2437,7 +2722,7 @@ class H(BaseHTTPRequestHandler): if is_management: data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") else: - data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") + data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") send_bytes(self, data, "text/html; charset=utf-8") return @@ -2454,6 +2739,7 @@ class H(BaseHTTPRequestHandler): "report_date": report_date.isoformat(), "bucket_id": get_sessions_bucket_id(host), "rows": rows, + "true_active_apps": true_active_apps, } data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") send_bytes(self, data, "application/json; charset=utf-8") diff --git a/aw-server/test_aw_worktime_api.py b/aw-server/test_aw_worktime_api.py index 3f34fc2..7e0f031 100644 --- a/aw-server/test_aw_worktime_api.py +++ b/aw-server/test_aw_worktime_api.py @@ -128,6 +128,69 @@ def test_aggregate_hourly_rows_splits_interval_by_local_hour(): assert [row["active_seconds"] for row in rows] == [300, 300] +def test_build_true_active_apps_requires_foreground_not_afk_and_evidence(): + start = datetime(2026, 5, 14, 6, 0, 0, tzinfo=timezone.utc) + end = datetime(2026, 5, 14, 7, 59, 59, tzinfo=timezone.utc) + window_events = [ + {"timestamp": "2026-05-14T06:00:00Z", "duration": 120, "data": {"app": "1cv8.exe", "title": "ИНФОВЕСТ"}}, + {"timestamp": "2026-05-14T06:02:00Z", "duration": 180, "data": {"app": "1cv8.exe", "title": "Счета учета: Материалы"}}, + {"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"app": "totalcmd.exe", "title": "Total Commander"}}, + ] + afk_events = [ + {"timestamp": "2026-05-14T06:00:00Z", "duration": 600, "data": {"status": "not-afk"}}, + {"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"status": "not-afk"}}, + ] + evidence_events_by_bucket = { + "aw-file-operations_SHARKON2025": [ + { + "timestamp": "2026-05-14T07:05:00Z", + "duration": 0, + "data": {"signalType": "file_write", "path": "C:\\data\\report.xlsx"}, + } + ], + "aw-dlp-endpoint-signals_SHARKON2025": [ + { + "timestamp": "2026-05-14T07:06:00Z", + "duration": 0, + "data": {"signalType": "collector_health", "eventsFlushed": 100}, + } + ], + } + + rows = MODULE.build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end) + + by_app = {row["application"]: row for row in rows} + assert "1С" in by_app + assert "Total Commander" in by_app + assert by_app["1С"]["proved_work_seconds"] == 300 + assert by_app["1С"]["last_action"] == "Счета учета: Материалы" + assert by_app["Total Commander"]["proved_work_seconds"] == 480 + assert by_app["Total Commander"]["last_action"] == "C:\\data\\report.xlsx" + + +def test_render_html_contains_true_active_apps_table(): + html = MODULE.render_html( + [], + "SHARKON2025", + datetime(2026, 5, 14, tzinfo=timezone.utc).date(), + selected_day="today", + true_active_apps=[ + { + "application": "1С", + "proved_work_human": "34 мин", + "proved_work_hhmm": "00:34", + "last_action_local": "15:31", + "last_action": "Счета учета: Материалы", + } + ], + ) + assert "Доказанная работа по приложениям" in html + assert "Приложение" in html + assert "Доказанная работа" in html + assert "Последнее действие" in html + assert "Счета учета: Материалы" in html + + def test_build_management_payload_creates_actions_for_missing_and_late_users(): rows = [ { diff --git a/grafana/detmir-aw-main-dashboard.json b/grafana/detmir-aw-main-dashboard.json new file mode 100644 index 0000000..3ed9480 --- /dev/null +++ b/grafana/detmir-aw-main-dashboard.json @@ -0,0 +1,230 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 27, + "links": [], + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "influxdb_aw" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "query": "from(bucket: \"aw_metrics\")\n |> range(start: -7d)\n |> filter(fn: (r) => (r._measurement == \"aw_window_event\" or r._measurement == \"aw_afk_event\") and r._field == \"duration_s\")\n |> last()\n |> group()\n |> sort(columns: [\"_time\"], desc: true)\n |> limit(n: 1)\n |> map(fn: (r) => ({ r with _value: float(v: uint(v: now()) - uint(v: r._time)) / 1000000000.0 }))\n |> keep(columns: [\"_value\"])\n |> yield(name: \"freshness_seconds\")", + "refId": "A" + } + ], + "title": "Свежесть данных AW", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb_aw" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> aggregateWindow(every: 5m, fn: sum, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"host\"])\n |> group(columns: [\"host\"])\n |> yield(name: \"active\")", + "refId": "A" + } + ], + "title": "Активность окон (сумма за 5 минут)", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb_aw" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_afk_event\" and r._field == \"duration_s\" and r.status == \"afk\")\n |> aggregateWindow(every: 5m, fn: sum, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"host\"])\n |> group(columns: [\"host\"])\n |> yield(name: \"afk\")", + "refId": "A" + } + ], + "title": "AFK (сумма за 5 минут)", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb_aw" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> map(fn: (r) => ({ r with category_view: if r.category == \"erp_1c\" then \"1С-Бухгалтерия\" else if r.category == \"file_manager\" then \"Файловый менеджер\" else if r.category == \"terminal_shell\" then \"Терминальная оболочка\" else if r.category == \"browser\" then \"Интернет-браузер\" else if r.category == \"office_docs\" then \"Офисные документы\" else if r.category == \"remote_admin\" then \"Удаленное администрирование\" else if r.category == \"development\" then \"Разработка и программирование\" else if r.category == \"communication\" then \"Средства связи\" else if r.category == \"database\" then \"Работа с базами данных\" else if r.category == \"monitoring\" then \"Мониторинг и диагностика\" else if r.category == \"virtualization\" then \"Виртуализация\" else if r.category == \"system\" or r.category == \"misc_user\" or r.category == \"other\" or r.category == \"\" then \"Системные приложения\" else r.category }))\n |> group(columns: [\"category_view\"])\n |> sum()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)\n |> yield(name: \"categories\")", + "refId": "A" + } + ], + "title": "Категории по активному времени", + "type": "barchart" + }, + { + "datasource": { + "type": "influxdb", + "uid": "influxdb_aw" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> group()\n |> sum()\n |> yield(name: \"total_active\")", + "refId": "A" + } + ], + "title": "Общее активное время", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "detmir", + "activitywatch" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DetMir ActivityWatch", + "uid": "detmir-aw-main", + "version": 3, + "weekStart": "" +} diff --git a/scripts/aw-contour-smoke-10.10.10.2.sh b/scripts/aw-contour-smoke-10.10.10.2.sh new file mode 100644 index 0000000..798c507 --- /dev/null +++ b/scripts/aw-contour-smoke-10.10.10.2.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Smoke checks for the Proxmox/gateway/1C host 10.10.10.2. + +set -uo pipefail + +OK_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 + +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + CYAN='\033[0;36m' + NC='\033[0m' +else + RED='' + GREEN='' + YELLOW='' + CYAN='' + NC='' +fi + +pass() { OK_COUNT=$((OK_COUNT + 1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; } +warn() { WARN_COUNT=$((WARN_COUNT + 1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; } +fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; } +skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; } + +section() { + printf "\n%b== %s ==%b\n" "$CYAN" "$*" "$NC" +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +check_command() { + local name="$1" + shift + local output + if output="$("$@" 2>&1)"; then + pass "$name" + [ -n "$output" ] && printf "%s\n" "$output" | sed 's/^/ /' + else + fail "$name" + [ -n "$output" ] && printf "%s\n" "$output" | sed 's/^/ /' + fi +} + +check_service() { + local unit="$1" + if ! systemctl list-unit-files "$unit" >/dev/null 2>&1; then + skip "$unit is not installed" + return + fi + if systemctl is-active --quiet "$unit"; then + pass "$unit active" + else + fail "$unit inactive or failed" + systemctl --no-pager --lines=8 status "$unit" 2>&1 | sed 's/^/ /' + fi +} + +check_timer() { + local unit="$1" + if ! systemctl list-unit-files "$unit" >/dev/null 2>&1; then + skip "$unit is not installed" + return + fi + if systemctl is-active --quiet "$unit"; then + pass "$unit active" + else + fail "$unit inactive or failed" + systemctl --no-pager --lines=8 status "$unit" 2>&1 | sed 's/^/ /' + fi +} + +check_tcp() { + local name="$1" + local host="$2" + local port="$3" + if timeout 4 bash -c ":/dev/null 2>&1; then + pass "$name TCP ${host}:${port}" + else + fail "$name TCP ${host}:${port}" + fi +} + +check_http_code() { + local name="$1" + local url="$2" + local expected="${3:-^2[0-9][0-9]$}" + local tmp code + tmp="$(mktemp)" + code="$(curl -k -sS --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + if printf "%s" "$code" | grep -Eq "$expected"; then + pass "$name HTTP $code $url" + else + fail "$name HTTP $code $url" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_redirect() { + local name="$1" + local url="$2" + local expected_code="${3:-^30[1278]$}" + local expected_location="${4:-}" + local tmp code location + tmp="$(mktemp)" + code="$(curl -k -sS -I --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/\r$/,""); print $0}' "$tmp" | tail -1)" + if printf "%s" "$code" | grep -Eq "$expected_code" && { [ -z "$expected_location" ] || printf "%s" "$location" | grep -Fq "$expected_location"; }; then + pass "$name HTTP $code ${location:-$url}" + else + fail "$name HTTP $code ${location:-$url}" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_docker_container() { + local name="$1" + if ! have docker; then + skip "docker command unavailable" + return + fi + if docker ps --format '{{.Names}}' 2>/dev/null | grep -Fxq "$name"; then + pass "docker container $name running" + docker ps --filter "name=^/${name}$" --format ' {{.Names}} {{.Status}} {{.Ports}}' + else + fail "docker container $name not running" + docker ps -a --filter "name=^/${name}$" --format ' {{.Names}} {{.Status}} {{.Ports}}' 2>/dev/null || true + fi +} + +section "Host" +hostnamectl 2>/dev/null | sed 's/^/ /' || hostname | sed 's/^/ /' +date -Is | sed 's/^/ /' +uptime | sed 's/^/ /' + +section "Core Services" +for unit in \ + nginx.service \ + pveproxy.service \ + pvedaemon.service \ + pvestatd.service \ + pve-cluster.service \ + docker.service \ + aw-1c-company-api.service \ + aw-pve-webadmin-logger.service +do + check_service "$unit" +done + +section "Timers" +for unit in \ + aw-1c-ingest.timer \ + aw-1c-proofcheck.timer \ + aw-1c-manager-brief.timer \ + aw-1c-recovery-brief.timer \ + aw-1c-weekly-digest.timer +do + check_timer "$unit" +done +systemctl list-timers --all --no-pager 2>/dev/null | grep -E 'aw-1c|NEXT|LEFT|PASSED' | sed 's/^/ /' || true + +section "Ports" +check_tcp "nginx http" 127.0.0.1 80 +check_tcp "nginx https" 127.0.0.1 443 +check_tcp "proxmox web" 127.0.0.1 8006 +check_tcp "1C company API" 10.10.10.2 8710 +check_tcp "clickhouse native" 127.0.0.1 9000 +check_tcp "clickhouse http" 127.0.0.1 8123 +ss -tulpn | grep -E ':(80|443|8006|8710|8123|9000)\b' | sed 's/^/ /' || true + +section "Gateway HTTP" +check_http_code "nginx healthz" "https://127.0.0.1/healthz" '^200$' +check_http_redirect "go proxmox gui" "https://127.0.0.1/go/proxmox-gui" '^30[1278]$' 'https://10.10.10.2:8006/' +check_http_redirect "go file1c brief" "https://127.0.0.1/go/file1c-brief" '^30[1278]$' 'http://10.10.10.2:8710/manager/brief' +check_http_redirect "go file1c actions" "https://127.0.0.1/go/file1c-actions" '^30[1278]$' 'http://10.10.10.2:8710/manager/actions' + +section "1C Company API" +check_http_code "1C root redirect" "http://10.10.10.2:8710/" '^307$' +check_http_code "1C /health" "http://10.10.10.2:8710/health" '^200$' +check_http_code "1C /api/health" "http://10.10.10.2:8710/api/health" '^200$' +check_http_code "1C manager brief" "http://10.10.10.2:8710/manager/brief" '^200$' +check_http_code "1C manager actions" "http://10.10.10.2:8710/manager/actions" '^200$' +check_http_code "1C manager recovery" "http://10.10.10.2:8710/manager/recovery" '^200$' +check_http_code "1C weekly digest" "http://10.10.10.2:8710/manager/digest/weekly" '^200$' + +section "ClickHouse" +check_docker_container "aw-rus-1c-clickhouse" +check_http_code "ClickHouse ping" "http://127.0.0.1:8123/ping" '^200$' +if have docker && docker ps --format '{{.Names}}' | grep -Fxq aw-rus-1c-clickhouse; then + check_command "ClickHouse SELECT 1" docker exec aw-rus-1c-clickhouse clickhouse-client --query "SELECT 1" +fi + +section "System Capacity" +df -h / /var /opt 2>/dev/null | sed 's/^/ /' +free -h 2>/dev/null | sed 's/^/ /' || true + +section "Summary" +printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" + +if [ "$FAIL_COUNT" -gt 0 ]; then + exit 2 +fi +exit 0 diff --git a/scripts/aw-contour-smoke-local.sh b/scripts/aw-contour-smoke-local.sh new file mode 100644 index 0000000..1226443 --- /dev/null +++ b/scripts/aw-contour-smoke-local.sh @@ -0,0 +1,597 @@ +#!/usr/bin/env bash +# End-to-end smoke checks from Igor's laptop for ActivityWatch-Russian. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SMOKE_ENV_FILE="${AW_SMOKE_ENV_FILE:-}" +for env_candidate in "$REPO_ROOT/secrets/runtime.env" "$HOME/.config/aw-contour-smoke.env" "$SMOKE_ENV_FILE"; do + if [ -n "$env_candidate" ] && [ -f "$env_candidate" ]; then + # Load local credentials and site-specific overrides without committing them. + # Later files override earlier defaults. + set -a + . "$env_candidate" + set +a + fi +done + +ANSIBLE_DIR="$REPO_ROOT/ansible" +INVENTORY="${AW_SMOKE_INVENTORY:-$ANSIBLE_DIR/inventory.ini}" +REMOTE_SCRIPT_SRC="$REPO_ROOT/scripts/aw-contour-smoke-10.10.10.2.sh" +REMOTE_SCRIPT_DST="${AW_SMOKE_REMOTE_SCRIPT:-/usr/local/sbin/aw-contour-smoke.sh}" +AW_SERVER="${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}" +WORKTIME_API="${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}" +GRAFANA_URL="${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}" +GRAFANA_USER="${GRAFANA_USER:-igor}" +GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-}" +PROXMOX_HOST="${AW_SMOKE_PROXMOX_HOST:-10.10.10.2}" +AW_HOST="${AW_SMOKE_AW_HOST:-10.10.10.13}" +GRAFANA_HOST="${AW_SMOKE_GRAFANA_HOST:-10.10.10.11}" +WINDOWS_HOST="${AW_SMOKE_WINDOWS_HOST:-192.168.100.18}" +AW_SOURCE_HOSTNAME="${AW_SMOKE_SOURCE_HOSTNAME:-SHARKON2025}" +LOG_DIR="${AW_SMOKE_LOG_DIR:-$REPO_ROOT/output/smoke}" +RUN_REMOTE="${AW_SMOKE_RUN_REMOTE:-1}" +RUN_WINRM="${AW_SMOKE_RUN_WINRM:-1}" +RUN_SERVER_SYSTEMD="${AW_SMOKE_RUN_SERVER_SYSTEMD:-1}" + +NO_PROXY_REQUIRED="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$WINDOWS_HOST,10.10.10.0/24,192.168.100.0/24" +if [ -n "${no_proxy:-}" ]; then + export no_proxy="$no_proxy,$NO_PROXY_REQUIRED" +else + export no_proxy="$NO_PROXY_REQUIRED" +fi +if [ -n "${NO_PROXY:-}" ]; then + export NO_PROXY="$NO_PROXY,$NO_PROXY_REQUIRED" +else + export NO_PROXY="$no_proxy" +fi + +OK_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 + +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + CYAN='\033[0;36m' + NC='\033[0m' +else + RED='' + GREEN='' + YELLOW='' + CYAN='' + NC='' +fi + +usage() { + cat <&2; usage >&2; exit 64 ;; + esac + shift +done + +pass() { OK_COUNT=$((OK_COUNT + 1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; } +warn() { WARN_COUNT=$((WARN_COUNT + 1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; } +fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; } +skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; } + +section() { + printf "\n%b== %s ==%b\n" "$CYAN" "$*" "$NC" +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +check_local_command() { + local cmd="$1" + if have "$cmd"; then + pass "command available: $cmd" + else + fail "command missing: $cmd" + fi +} + +check_tcp() { + local name="$1" + local host="$2" + local port="$3" + if timeout 4 bash -c ":/dev/null 2>&1; then + pass "$name TCP ${host}:${port}" + else + fail "$name TCP ${host}:${port}" + fi +} + +check_ssh_banner() { + local name="$1" + local host="$2" + local port="${3:-22}" + local banner + banner="$( + timeout 5 bash -c "exec 3<>/dev/tcp/${host}/${port}; IFS= read -r line <&3; printf '%s' \"\$line\"" 2>/dev/null || true + )" + if printf "%s" "$banner" | grep -Eq '^SSH-[0-9]+\.[0-9]+'; then + pass "$name SSH banner ${host}:${port} ${banner}" + else + fail "$name SSH banner ${host}:${port} unavailable" + fi +} + +check_http_code() { + local name="$1" + local url="$2" + local expected="${3:-^2[0-9][0-9]$}" + local tmp code + tmp="$(mktemp)" + code="$(curl -k -sS --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + if printf "%s" "$code" | grep -Eq "$expected"; then + pass "$name HTTP $code $url" + else + fail "$name HTTP $code $url" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_json_key() { + local name="$1" + local url="$2" + local jq_filter="$3" + local tmp + tmp="$(mktemp)" + if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then + pass "$name JSON $jq_filter" + jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/ /' | head -5 + else + fail "$name JSON $jq_filter" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_json_key_basic_auth() { + local name="$1" + local url="$2" + local jq_filter="$3" + local tmp + tmp="$(mktemp)" + if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then + pass "$name JSON $jq_filter" + jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/ /' | head -5 + else + fail "$name JSON $jq_filter" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_code_basic_auth() { + local name="$1" + local url="$2" + local expected="${3:-^2[0-9][0-9]$}" + local tmp code + tmp="$(mktemp)" + code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + if printf "%s" "$code" | grep -Eq "$expected"; then + pass "$name HTTP $code $url" + else + fail "$name HTTP $code $url" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_grafana_loki_proxy() { + local name="Grafana Loki datasource proxy" + local tmp uid code body + tmp="$(mktemp)" + if ! curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources" -o "$tmp" 2>"$tmp.err"; then + fail "$name cannot list datasources" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + rm -f "$tmp" "$tmp.err" + return + fi + uid="$(jq -r '.[] | select(.type == "loki") | .uid' "$tmp" 2>/dev/null | head -1)" + if [ -z "$uid" ]; then + fail "$name no loki datasource found" + jq -r '.[] | "\(.name) \(.type) \(.uid)"' "$tmp" 2>/dev/null | sed 's/^/ /' | head -20 + rm -f "$tmp" "$tmp.err" + return + fi + body="$(mktemp)" + code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$body" -w '%{http_code}' "$GRAFANA_URL/api/datasources/proxy/uid/$uid/loki/api/v1/labels" 2>"$tmp.err")" + if [ "$code" = "200" ]; then + pass "$name HTTP $code uid=$uid" + jq -r '.status // .data[0] // .' "$body" 2>/dev/null | sed 's/^/ /' | head -5 + else + fail "$name HTTP $code uid=$uid" + sed 's/^/ /' "$tmp.err" "$body" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" "$body" +} + +check_grafana_influx_health() { + local name="Grafana InfluxDB-AW datasource health" + local tmp + tmp="$(mktemp)" + if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources/uid/influxdb_aw/health" -o "$tmp" 2>"$tmp.err" && jq -e '.status == "OK"' "$tmp" >/dev/null 2>&1; then + pass "$name" + jq -r '.message // .status // .' "$tmp" 2>/dev/null | sed 's/^/ /' | head -5 + else + fail "$name" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +check_grafana_aw_main_dashboard_queries() { + local name="Grafana detmir-aw-main window/AFK panel queries" + local tmp + tmp="$(mktemp)" + if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/dashboards/uid/detmir-aw-main" -o "$tmp" 2>"$tmp.err" && \ + jq -e ' + def fixed: + (.targets[0].query // "") | contains("keep(columns: [\"_time\", \"_value\", \"host\"]"); + ([.dashboard.panels[] | select(.title == "Активность окон (сумма за 5 минут)" or .title == "AFK (сумма за 5 минут)") | fixed] | length == 2 and all(. == true)) + ' "$tmp" >/dev/null 2>&1; then + pass "$name" + else + fail "$name missing keep(_time,_value,host) after aggregateWindow" + jq -r '.dashboard.panels[]? | select(.title == "Активность окон (сумма за 5 минут)" or .title == "AFK (сумма за 5 минут)") | "\(.title): " + ((.targets[0].query // "") | gsub("\n"; " "))' "$tmp" 2>/dev/null | sed 's/^/ /' | head -20 + sed 's/^/ /' "$tmp.err" 2>/dev/null | head -20 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_redirect() { + local name="$1" + local url="$2" + local expected_code="${3:-^30[1278]$}" + local expected_location="${4:-}" + local tmp code location + tmp="$(mktemp)" + code="$(curl -k -sS -I --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/\r$/,""); print $0}' "$tmp" | tail -1)" + if printf "%s" "$code" | grep -Eq "$expected_code" && { [ -z "$expected_location" ] || printf "%s" "$location" | grep -Fq "$expected_location"; }; then + pass "$name HTTP $code ${location:-$url}" + else + fail "$name HTTP $code ${location:-$url}" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40 + fi + rm -f "$tmp" "$tmp.err" +} + +ansible_shell() { + local group="$1" + local command="$2" + ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command" +} + +ansible_win_shell() { + local group="$1" + local command="$2" + ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m win_shell -a "$command" +} + +check_ansible_shell() { + local name="$1" + local group="$2" + local command="$3" + local tmp + tmp="$(mktemp)" + if ansible_shell "$group" "$command" >"$tmp" 2>&1; then + pass "$name" + sed 's/^/ /' "$tmp" | head -80 + else + fail "$name" + sed 's/^/ /' "$tmp" | head -120 + fi + rm -f "$tmp" +} + +check_ansible_win_shell() { + local name="$1" + local group="$2" + local command="$3" + local tmp + tmp="$(mktemp)" + if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then + pass "$name" + sed 's/^/ /' "$tmp" | head -80 + else + fail "$name" + sed 's/^/ /' "$tmp" | head -120 + fi + rm -f "$tmp" +} + +check_ansible_win_shell_warn() { + local name="$1" + local group="$2" + local command="$3" + local tmp + tmp="$(mktemp)" + if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then + pass "$name" + sed 's/^/ /' "$tmp" | head -80 + else + warn "$name returned non-zero" + sed 's/^/ /' "$tmp" | head -120 + fi + rm -f "$tmp" +} + +check_ansible_module() { + local name="$1" + local group="$2" + local module="$3" + local args="${4:-}" + local tmp + tmp="$(mktemp)" + if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then + pass "$name" + sed 's/^/ /' "$tmp" | head -80 + else + fail "$name" + sed 's/^/ /' "$tmp" | head -120 + fi + rm -f "$tmp" +} + +classify_bucket_age() { + local bucket="$1" + local age_sec="$2" + case "$bucket" in + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + if [ "$age_sec" -lt 86400 ]; then + printf "fresh" + else + printf "event-driven" + fi + ;; + *) + if [ "$age_sec" -lt 3600 ]; then + printf "fresh" + elif [ "$age_sec" -lt 86400 ]; then + printf "stale" + else + printf "dead" + fi + ;; + esac +} + +check_bucket_freshness() { + local bucket="$1" + local bucket_id="${bucket}_${AW_SOURCE_HOSTNAME}" + local tmp last_ts last_id event_epoch now age_sec status + tmp="$(mktemp)" + if ! curl -fsS --connect-timeout 5 --max-time 20 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then + fail "bucket $bucket_id query failed" + sed 's/^/ /' "$tmp.err" | head -20 + rm -f "$tmp" "$tmp.err" + return + fi + + last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)" + last_id="$(jq -r '.[0].id // 0' "$tmp" 2>/dev/null)" + rm -f "$tmp" "$tmp.err" + + if [ -z "$last_ts" ]; then + case "$bucket" in + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + warn "bucket $bucket_id empty/event-driven" + ;; + *) + fail "bucket $bucket_id empty" + ;; + esac + return + fi + + event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || printf "0")" + now="$(date -u +%s)" + if [ "$event_epoch" -le 0 ]; then + warn "bucket $bucket_id has unparsable timestamp: $last_ts" + return + fi + age_sec=$((now - event_epoch)) + status="$(classify_bucket_age "$bucket" "$age_sec")" + case "$status" in + fresh|event-driven) + pass "bucket $bucket_id $status age=${age_sec}s id=$last_id" + ;; + stale) + warn "bucket $bucket_id stale age=${age_sec}s id=$last_id" + ;; + *) + fail "bucket $bucket_id dead age=${age_sec}s id=$last_id" + ;; + esac +} + +run_remote_proxmox_script() { + if [ "$RUN_REMOTE" != "1" ]; then + skip "remote 10.10.10.2 smoke skipped" + return + fi + if ! have ansible; then + fail "ansible unavailable; cannot deploy/run remote script" + return + fi + if [ ! -f "$REMOTE_SCRIPT_SRC" ]; then + fail "remote script source missing: $REMOTE_SCRIPT_SRC" + return + fi + + section "Deploy Remote Script To 10.10.10.2" + if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_SCRIPT_SRC dest=$REMOTE_SCRIPT_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy.$$ 2>&1; then + pass "remote script deployed to $REMOTE_SCRIPT_DST" + else + fail "remote script deploy failed" + sed 's/^/ /' /tmp/aw-smoke-copy.$$ | head -120 + rm -f /tmp/aw-smoke-copy.$$ + return + fi + rm -f /tmp/aw-smoke-copy.$$ + + section "Remote 10.10.10.2 Smoke" + local tmp + tmp="$(mktemp)" + if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m shell -a "$REMOTE_SCRIPT_DST" >"$tmp" 2>&1; then + pass "remote 10.10.10.2 smoke completed" + sed 's/^/ /' "$tmp" + else + fail "remote 10.10.10.2 smoke failed" + sed 's/^/ /' "$tmp" + fi + rm -f "$tmp" +} + +mkdir -p "$LOG_DIR" +LOG_FILE="$LOG_DIR/aw-contour-smoke-$(date +%Y%m%d-%H%M%S).log" +exec > >(tee "$LOG_FILE") 2>&1 + +section "Run Context" +printf "repo=%s\ninventory=%s\nlog=%s\n" "$REPO_ROOT" "$INVENTORY" "$LOG_FILE" | sed 's/^/ /' +date -Is | sed 's/^/ /' + +section "Local Prerequisites" +for cmd in bash curl jq timeout ansible ssh; do + check_local_command "$cmd" +done +printf " no_proxy=%s\n" "$no_proxy" + +section "Laptop Network" +ip -br addr 2>/dev/null | sed 's/^/ /' || true +ip route get "$PROXMOX_HOST" 2>/dev/null | sed 's/^/ /' || warn "no route detail for $PROXMOX_HOST" +ip route get "$AW_HOST" 2>/dev/null | sed 's/^/ /' || warn "no route detail for $AW_HOST" + +section "TCP Surface" +check_tcp "Proxmox SSH" "$PROXMOX_HOST" 22 +check_tcp "Proxmox HTTP" "$PROXMOX_HOST" 80 +check_tcp "Proxmox HTTPS" "$PROXMOX_HOST" 443 +check_tcp "Proxmox GUI" "$PROXMOX_HOST" 8006 +check_tcp "1C company API" "$PROXMOX_HOST" 8710 +check_tcp "AW server" "$AW_HOST" 5600 +check_tcp "AW worktime API" "$AW_HOST" 5610 +check_tcp "Grafana" "$GRAFANA_HOST" 3000 +check_tcp "Windows WinRM" "$WINDOWS_HOST" 5985 +check_tcp "Windows SSH" "$WINDOWS_HOST" 22 +check_ssh_banner "Windows SSH" "$WINDOWS_HOST" 22 + +section "ActivityWatch HTTP" +check_http_json_key "AW server info" "$AW_SERVER/api/0/info" '.version' +check_http_code "AW settings CORS" "$AW_SERVER/api/0/settings/" '^200$' +check_http_code "AW WebUI" "$AW_SERVER/" '^200$' +check_http_json_key "AW buckets list" "$AW_SERVER/api/0/buckets/" 'keys | length' + +section "ActivityWatch Buckets" +for bucket in \ + aw-watcher-afk \ + aw-watcher-window \ + aw-worktime-sessions \ + aw-session-events \ + aw-dlp-endpoint-signals \ + aw-dlp-incidents \ + aw-dlp-review \ + aw-dlp-rules +do + check_bucket_freshness "$bucket" +done + +section "Worktime API" +check_http_json_key "worktime health" "$WORKTIME_API/health" '.status // .ok // .' +check_http_code "worktime today html" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$' +check_http_json_key "worktime today json" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today" '.host // .report.host // .[0].host // .' +check_http_code "worktime management html" "$WORKTIME_API/reports/worktime/management?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$' + +section "Gateway And 1C HTTP" +check_http_code "gateway healthz" "https://$PROXMOX_HOST/healthz" '^200$' +check_http_redirect "gateway proxmox redirect" "https://$PROXMOX_HOST/go/proxmox-gui" '^30[1278]$' "https://$PROXMOX_HOST:8006/" +check_http_redirect "gateway file1c brief redirect" "https://$PROXMOX_HOST/go/file1c-brief" '^30[1278]$' "http://$PROXMOX_HOST:8710/manager/brief" +check_http_code "1C /health" "http://$PROXMOX_HOST:8710/health" '^200$' +check_http_code "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" '^200$' +check_http_code "1C manager brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$' +check_http_code "1C manager actions" "http://$PROXMOX_HOST:8710/manager/actions" '^200$' +check_http_code "1C manager recovery" "http://$PROXMOX_HOST:8710/manager/recovery" '^200$' +check_http_code "1C weekly digest" "http://$PROXMOX_HOST:8710/manager/digest/weekly" '^200$' + +section "Grafana HTTP" +check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" '.database // .version // .commit' +check_http_code "Grafana dashboards page" "$GRAFANA_URL/dashboards" '^200$|^302$' +check_http_code "Grafana pfSense dashboard" "$GRAFANA_URL/d/pfsense-loki-dashboard/pfsense-firewall-overview?orgId=1&from=now-1h&to=now&timezone=browser" '^200$|^302$' + +if [ -n "${GRAFANA_USER:-}" ] && [ -n "${GRAFANA_PASSWORD:-}" ]; then + section "Grafana Authenticated API" + check_http_json_key_basic_auth "Grafana datasources" "$GRAFANA_URL/api/datasources" 'length' + check_grafana_loki_proxy + check_grafana_influx_health + check_grafana_aw_main_dashboard_queries +else + skip "Grafana authenticated datasource checks need GRAFANA_USER and GRAFANA_PASSWORD env" +fi + +if [ "$RUN_SERVER_SYSTEMD" = "1" ] && have ansible; then + section "AW Server Systemd" + check_ansible_shell "AW server core units" aw_server 'systemctl is-active activitywatch-server aw-worktime-api aw-worktime-ui-bridge.timer aw-rus-healthd.timer aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer aw-worktime-autoheal.timer' + check_ansible_shell "AW server failed units" aw_server 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|influx|grafana|prometheus|loki" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }' + check_ansible_shell "AW server local health script" aw_server 'test -x /opt/activitywatch/health-check.sh && /opt/activitywatch/health-check.sh || test -x /usr/local/bin/health-check.sh && /usr/local/bin/health-check.sh || echo "health-check script not installed"' +else + skip "AW server systemd checks skipped" +fi + +if [ "$RUN_WINRM" = "1" ] && have ansible; then + section "Windows WinRM And Collectors" + check_ansible_module "Windows win_ping" aw_windows win_ping + check_ansible_win_shell "Windows sessions" aw_windows '$psi = [System.Diagnostics.ProcessStartInfo]::new(); $psi.FileName = "$env:SystemRoot\System32\query.exe"; $psi.Arguments = "user"; $psi.UseShellExecute = $false; $psi.RedirectStandardOutput = $true; $psi.RedirectStandardError = $true; $psi.StandardOutputEncoding = [System.Text.Encoding]::GetEncoding(866); $psi.StandardErrorEncoding = [System.Text.Encoding]::GetEncoding(866); $p = [System.Diagnostics.Process]::Start($psi); $out = $p.StandardOutput.ReadToEnd(); $err = $p.StandardError.ReadToEnd(); $p.WaitForExit(); [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $out; if ($err) { $err }; if ($out -match "USERNAME|ПОЛЬЗОВАТЕЛЬ|администратор|Администратор") { exit 0 } else { exit $p.ExitCode }' + check_ansible_win_shell_warn "Windows collector processes" aw_windows '$p = Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue; if ($p) { $p | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize } else { "no aw-watcher-afk/window process visible to this WinRM session" }' + check_ansible_win_shell "Windows ActivityWatch tasks" aw_windows 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V; schtasks /Query /TN "ActivityWatch Launch [SHARKON2025_Администратор]" /FO LIST /V' +else + skip "Windows WinRM checks skipped" +fi + +run_remote_proxmox_script + +section "Existing Repo Checks" +if [ -x "$REPO_ROOT/check-aw-data.sh" ]; then + if "$REPO_ROOT/check-aw-data.sh"; then pass "check-aw-data.sh completed"; else fail "check-aw-data.sh failed"; fi +else + skip "check-aw-data.sh missing" +fi +if [ -x "$REPO_ROOT/check-aw-full.sh" ]; then + if "$REPO_ROOT/check-aw-full.sh"; then pass "check-aw-full.sh completed"; else fail "check-aw-full.sh failed"; fi +else + skip "check-aw-full.sh missing" +fi + +section "Summary" +printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" +printf "Log: %s\n" "$LOG_FILE" + +if [ "$FAIL_COUNT" -gt 0 ]; then + exit 2 +fi +exit 0