feat(worktime): add true active app evidence and contour smoke
This commit is contained in:
igor04091968
@@ -8,6 +8,9 @@ secrets/runtime.env
|
||||
*.bak
|
||||
windows/*.report.json
|
||||
.rollout-logs/
|
||||
output/
|
||||
.ai/
|
||||
.autonomous/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
grafana_folder_title_eff: "{{ grafana_folder_title | default('AWatch-rus') }}"
|
||||
grafana_dashboards_import_overwrite_eff: "{{ grafana_dashboards_import_overwrite | default(true) }}"
|
||||
grafana_dashboards:
|
||||
- file: "grafana/detmir-aw-main-dashboard.json"
|
||||
uid: "detmir-aw-main"
|
||||
title: "DetMir ActivityWatch"
|
||||
folder_uid: "{{ grafana_detmir_folder_uid | default(grafana_folder_uid_eff) }}"
|
||||
- file: "grafana/detmir-rdp-user-activity-dashboard.json"
|
||||
uid: "detmir-rdp-user-activity"
|
||||
title: "DetMir: Работа пользователей в RDP"
|
||||
|
||||
@@ -49,6 +49,8 @@ MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", "
|
||||
MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()}
|
||||
EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30")))
|
||||
WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000")))
|
||||
TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180")))
|
||||
TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600")))
|
||||
MODULE_PATH = Path(__file__).resolve()
|
||||
_ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}}
|
||||
_EVENTS_CACHE_LOCK = threading.Lock()
|
||||
@@ -85,6 +87,19 @@ def hhmm(total_seconds):
|
||||
return "%02d:%02d" % (total_seconds // 3600, (total_seconds % 3600) // 60)
|
||||
|
||||
|
||||
def human_duration_ru(total_seconds):
|
||||
total_seconds = max(0, int(total_seconds))
|
||||
hours = total_seconds // 3600
|
||||
minutes = (total_seconds % 3600) // 60
|
||||
if hours and minutes:
|
||||
return f"{hours} ч {minutes} мин"
|
||||
if hours:
|
||||
return f"{hours} ч"
|
||||
if minutes:
|
||||
return f"{minutes} мин"
|
||||
return f"{total_seconds} сек"
|
||||
|
||||
|
||||
def now_utc():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
@@ -397,6 +412,246 @@ def _merge_intervals(intervals):
|
||||
return merged
|
||||
|
||||
|
||||
def _overlap_interval(left, right):
|
||||
start = max(left[0], right[0])
|
||||
end = min(left[1], right[1])
|
||||
if end <= start:
|
||||
return None
|
||||
return start, end
|
||||
|
||||
|
||||
def _interval_contains(interval, ts):
|
||||
return interval[0] <= ts < interval[1]
|
||||
|
||||
|
||||
def _event_timestamp(event):
|
||||
try:
|
||||
return pts(event.get("timestamp"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS):
|
||||
try:
|
||||
duration = float(event.get("duration") or 0.0)
|
||||
except Exception:
|
||||
duration = 0.0
|
||||
if duration <= 0:
|
||||
duration = default_seconds
|
||||
return max(1.0, min(float(TRUE_ACTIVE_MAX_EVENT_SECONDS), duration))
|
||||
|
||||
|
||||
def _normalize_app_name(app, title=""):
|
||||
app_raw = str(app or "").strip()
|
||||
app_l = app_raw.lower()
|
||||
title_raw = str(title or "").strip()
|
||||
if app_l.startswith(("1cv8", "1cestart")):
|
||||
return "1С"
|
||||
if app_l in {"chrome.exe", "google chrome"} or "google chrome" in title_raw.lower():
|
||||
return "Chrome"
|
||||
if app_l in {"msedge.exe", "microsoft edge"} or "microsoft edge" in title_raw.lower():
|
||||
return "Edge"
|
||||
if app_l in {"browser.exe", "browser"} or "яндекс" in title_raw.lower():
|
||||
return "Яндекс Браузер"
|
||||
if app_l in {"excel.exe"}:
|
||||
return "Excel"
|
||||
if app_l in {"winword.exe"}:
|
||||
return "Word"
|
||||
if app_l in {"powerpnt.exe"}:
|
||||
return "PowerPoint"
|
||||
if app_l in {"outlook.exe"}:
|
||||
return "Outlook"
|
||||
if app_l in {"explorer.exe"}:
|
||||
return "Проводник"
|
||||
if app_l in {"totalcmd.exe", "totalcmd64.exe"}:
|
||||
return "Total Commander"
|
||||
if app_l in {"cmd.exe"}:
|
||||
return "Command Prompt"
|
||||
if app_l in {"powershell.exe", "pwsh.exe"}:
|
||||
return "PowerShell"
|
||||
if app_l in {"acrord32.exe", "acrobat.exe"}:
|
||||
return "Adobe Acrobat Reader"
|
||||
if app_l in {"windowsterminal.exe", "windowsterminal"}:
|
||||
return "Windows Terminal"
|
||||
if app_raw:
|
||||
return app_raw[:-4] if app_l.endswith(".exe") else app_raw
|
||||
if title_raw:
|
||||
return title_raw
|
||||
return "Неизвестное приложение"
|
||||
|
||||
|
||||
def _event_context(event):
|
||||
data = event.get("data") or {}
|
||||
for key in ("title", "url", "path", "filePath", "targetPath", "windowTitle", "foregroundTitle", "signalType"):
|
||||
value = str(data.get(key) or "").strip()
|
||||
if value:
|
||||
return value
|
||||
return "активность"
|
||||
|
||||
|
||||
def _is_not_afk_event(event):
|
||||
data = event.get("data") or {}
|
||||
status = str(data.get("status") or data.get("state") or "").strip().lower()
|
||||
return status in {"not-afk", "not_afk", "active", "активно"}
|
||||
|
||||
|
||||
def _is_real_evidence_event(event):
|
||||
data = event.get("data") or {}
|
||||
signal_type = str(data.get("signalType") or data.get("type") or "").strip().lower()
|
||||
if signal_type in {"collector_health", "self_test", "heartbeat", "health"}:
|
||||
return False
|
||||
if data.get("url") or data.get("title") or data.get("path") or data.get("filePath") or data.get("targetPath"):
|
||||
return True
|
||||
if signal_type:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _events_for_bounds(events, start, end):
|
||||
result = []
|
||||
for event in events:
|
||||
ts = _event_timestamp(event)
|
||||
if ts is None or ts < start or ts > end:
|
||||
continue
|
||||
result.append((ts, event))
|
||||
result.sort(key=lambda item: item[0])
|
||||
return result
|
||||
|
||||
|
||||
def _build_window_intervals(window_events, start, end):
|
||||
intervals = []
|
||||
previous_key = None
|
||||
for ts, event in _events_for_bounds(window_events, start, end):
|
||||
data = event.get("data") or {}
|
||||
app = str(data.get("app") or data.get("process") or data.get("processName") or "").strip()
|
||||
title = str(data.get("title") or data.get("windowTitle") or "").strip()
|
||||
if not app and not title:
|
||||
continue
|
||||
duration = _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS)
|
||||
interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1)))
|
||||
if interval[1] <= interval[0]:
|
||||
continue
|
||||
app_name = _normalize_app_name(app, title)
|
||||
current_key = (app_name, title)
|
||||
title_changed = previous_key is not None and current_key != previous_key
|
||||
previous_key = current_key
|
||||
intervals.append(
|
||||
{
|
||||
"app": app_name,
|
||||
"raw_app": app,
|
||||
"title": title,
|
||||
"start": interval[0],
|
||||
"end": interval[1],
|
||||
"title_changed": title_changed,
|
||||
"timestamp": ts,
|
||||
}
|
||||
)
|
||||
return intervals
|
||||
|
||||
|
||||
def _build_not_afk_intervals(afk_events, start, end):
|
||||
intervals = []
|
||||
for ts, event in _events_for_bounds(afk_events, start, end):
|
||||
if not _is_not_afk_event(event):
|
||||
continue
|
||||
duration = _event_duration_seconds(event, default_seconds=5)
|
||||
interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1)))
|
||||
if interval[1] > interval[0]:
|
||||
intervals.append(interval)
|
||||
return _merge_intervals(intervals)
|
||||
|
||||
|
||||
def _find_window_at(window_intervals, ts):
|
||||
for item in window_intervals:
|
||||
if item["start"] <= ts < item["end"]:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _add_app_evidence(evidence_by_app, app, ts, context):
|
||||
evidence_by_app.setdefault(app, []).append((ts, str(context or "").strip() or "активность"))
|
||||
|
||||
|
||||
def build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end):
|
||||
window_intervals = _build_window_intervals(window_events, start, end)
|
||||
not_afk_intervals = _build_not_afk_intervals(afk_events, start, end)
|
||||
evidence_by_app = {}
|
||||
|
||||
for item in window_intervals:
|
||||
if item["title_changed"]:
|
||||
_add_app_evidence(evidence_by_app, item["app"], item["timestamp"], item["title"] or item["raw_app"])
|
||||
|
||||
for events in evidence_events_by_bucket.values():
|
||||
for ts, event in _events_for_bounds(events, start, end):
|
||||
if not _is_real_evidence_event(event):
|
||||
continue
|
||||
window = _find_window_at(window_intervals, ts)
|
||||
if window is None:
|
||||
continue
|
||||
_add_app_evidence(evidence_by_app, window["app"], ts, _event_context(event))
|
||||
|
||||
rows = []
|
||||
evidence_delta = timedelta(seconds=TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS)
|
||||
for app in sorted({item["app"] for item in window_intervals} | set(evidence_by_app)):
|
||||
app_evidence = sorted(evidence_by_app.get(app, []), key=lambda item: item[0])
|
||||
if not app_evidence:
|
||||
continue
|
||||
evidence_windows = _merge_intervals([(ts - evidence_delta, ts + evidence_delta) for ts, _context in app_evidence])
|
||||
proved_intervals = []
|
||||
for window in [item for item in window_intervals if item["app"] == app]:
|
||||
base = (window["start"], window["end"])
|
||||
for afk_interval in not_afk_intervals:
|
||||
active_overlap = _overlap_interval(base, afk_interval)
|
||||
if active_overlap is None:
|
||||
continue
|
||||
for evidence_interval in evidence_windows:
|
||||
proved = _overlap_interval(active_overlap, evidence_interval)
|
||||
if proved is not None:
|
||||
proved_intervals.append(proved)
|
||||
proved_intervals = _merge_intervals(proved_intervals)
|
||||
proved_seconds = int(sum((right - left).total_seconds() for left, right in proved_intervals))
|
||||
if proved_seconds <= 0:
|
||||
continue
|
||||
last_ts, last_context = app_evidence[-1]
|
||||
rows.append(
|
||||
{
|
||||
"application": app,
|
||||
"proved_work_seconds": proved_seconds,
|
||||
"proved_work_hhmm": hhmm(proved_seconds),
|
||||
"proved_work_human": human_duration_ru(proved_seconds),
|
||||
"last_action_utc": to_iso_utc(last_ts),
|
||||
"last_action_local": last_ts.astimezone(REPORT_TZ).strftime("%H:%M"),
|
||||
"last_action": last_context,
|
||||
"evidence_events": len(app_evidence),
|
||||
}
|
||||
)
|
||||
rows.sort(key=lambda item: (-item["proved_work_seconds"], item["application"].lower()))
|
||||
return rows
|
||||
|
||||
|
||||
def build_true_active_apps(host, report_date):
|
||||
bounds = get_report_bounds(report_date)
|
||||
host = resolve_host(host)
|
||||
window_events = fetch_bucket_events(f"aw-watcher-window_{host}", host) or fetch_bucket_events(f"aw-rdp-window_{host}", host)
|
||||
afk_events = fetch_bucket_events(f"aw-watcher-afk_{host}", host) or fetch_bucket_events(f"aw-rdp-afk_{host}", host)
|
||||
evidence_events_by_bucket = {}
|
||||
for bucket_id in (
|
||||
f"aw-file-operations_{host}",
|
||||
f"aw-dlp-endpoint-signals_{host}",
|
||||
f"aw-watcher-web-chrome_{host}",
|
||||
f"aw-watcher-web-edge_{host}",
|
||||
f"aw-detmir-web-category_{host}",
|
||||
):
|
||||
evidence_events_by_bucket[bucket_id] = fetch_bucket_events(bucket_id, host)
|
||||
return build_true_active_apps_from_events(
|
||||
window_events,
|
||||
afk_events,
|
||||
evidence_events_by_bucket,
|
||||
bounds["start"],
|
||||
bounds["end"],
|
||||
)
|
||||
|
||||
|
||||
def _collect_user_rows(events, start, end, host):
|
||||
end_exclusive = end + timedelta(seconds=1)
|
||||
by_user = {}
|
||||
@@ -1537,7 +1792,7 @@ def report_for_date_fresh(host, report_date):
|
||||
return module.report_for_date(host, report_date)
|
||||
|
||||
|
||||
def render_html(rows, host, report_date, selected_day=None):
|
||||
def render_html(rows, host, report_date, selected_day=None, true_active_apps=None):
|
||||
generated = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
date_local = report_date.strftime("%Y-%m-%d")
|
||||
day_query = f"&day={selected_day}" if selected_day in {"today", "yesterday"} else ""
|
||||
@@ -1556,6 +1811,20 @@ def render_html(rows, host, report_date, selected_day=None):
|
||||
]
|
||||
trs = []
|
||||
detail_cards = []
|
||||
true_active_apps = true_active_apps or []
|
||||
true_active_rows = []
|
||||
for app_row in true_active_apps:
|
||||
last_action = app_row.get("last_action") or "-"
|
||||
last_time = app_row.get("last_action_local") or "-"
|
||||
true_active_rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(app_row.get('application') or '-')}</td>"
|
||||
f"<td class='good'>{html.escape(app_row.get('proved_work_human') or app_row.get('proved_work_hhmm') or '0 сек')}</td>"
|
||||
f"<td>{html.escape(last_time)} · {html.escape(last_action)}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
if not true_active_rows:
|
||||
true_active_rows.append("<tr><td colspan='3'>Пока нет доказанной активной работы по приложениям за выбранную дату.</td></tr>")
|
||||
for row in rows:
|
||||
user_slug = safe_slug(row["user"])
|
||||
active_seconds = int(row.get("active_seconds", 0) or 0)
|
||||
@@ -1811,6 +2080,21 @@ def render_html(rows, host, report_date, selected_day=None):
|
||||
{''.join(f"<div class='summary-card'><span>{html.escape(label)}</span><strong>{html.escape(value)}</strong></div>" for label, value in cards)}
|
||||
</div>
|
||||
</section>
|
||||
<section class="card">
|
||||
<h2 class="section-title">Доказанная работа по приложениям</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Приложение</th>
|
||||
<th>Доказанная работа</th>
|
||||
<th>Последнее действие</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{''.join(true_active_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
<section class="card">
|
||||
<h2 class="section-title">Таблица по пользователям</h2>
|
||||
<table>
|
||||
@@ -2397,6 +2681,7 @@ class H(BaseHTTPRequestHandler):
|
||||
is_management = parsed.path == "/reports/worktime/management"
|
||||
management_payload = management_report_for_date(host, report_date, owner_filter=owner_filter, department_filter=department_filter) if is_management else None
|
||||
rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"]
|
||||
true_active_apps = [] if is_management else build_true_active_apps(host, report_date)
|
||||
|
||||
if fmt == "csv":
|
||||
if is_management:
|
||||
@@ -2437,7 +2722,7 @@ class H(BaseHTTPRequestHandler):
|
||||
if is_management:
|
||||
data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8")
|
||||
else:
|
||||
data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8")
|
||||
data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8")
|
||||
send_bytes(self, data, "text/html; charset=utf-8")
|
||||
return
|
||||
|
||||
@@ -2454,6 +2739,7 @@ class H(BaseHTTPRequestHandler):
|
||||
"report_date": report_date.isoformat(),
|
||||
"bucket_id": get_sessions_bucket_id(host),
|
||||
"rows": rows,
|
||||
"true_active_apps": true_active_apps,
|
||||
}
|
||||
data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")
|
||||
send_bytes(self, data, "application/json; charset=utf-8")
|
||||
|
||||
@@ -128,6 +128,69 @@ def test_aggregate_hourly_rows_splits_interval_by_local_hour():
|
||||
assert [row["active_seconds"] for row in rows] == [300, 300]
|
||||
|
||||
|
||||
def test_build_true_active_apps_requires_foreground_not_afk_and_evidence():
|
||||
start = datetime(2026, 5, 14, 6, 0, 0, tzinfo=timezone.utc)
|
||||
end = datetime(2026, 5, 14, 7, 59, 59, tzinfo=timezone.utc)
|
||||
window_events = [
|
||||
{"timestamp": "2026-05-14T06:00:00Z", "duration": 120, "data": {"app": "1cv8.exe", "title": "ИНФОВЕСТ"}},
|
||||
{"timestamp": "2026-05-14T06:02:00Z", "duration": 180, "data": {"app": "1cv8.exe", "title": "Счета учета: Материалы"}},
|
||||
{"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"app": "totalcmd.exe", "title": "Total Commander"}},
|
||||
]
|
||||
afk_events = [
|
||||
{"timestamp": "2026-05-14T06:00:00Z", "duration": 600, "data": {"status": "not-afk"}},
|
||||
{"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"status": "not-afk"}},
|
||||
]
|
||||
evidence_events_by_bucket = {
|
||||
"aw-file-operations_SHARKON2025": [
|
||||
{
|
||||
"timestamp": "2026-05-14T07:05:00Z",
|
||||
"duration": 0,
|
||||
"data": {"signalType": "file_write", "path": "C:\\data\\report.xlsx"},
|
||||
}
|
||||
],
|
||||
"aw-dlp-endpoint-signals_SHARKON2025": [
|
||||
{
|
||||
"timestamp": "2026-05-14T07:06:00Z",
|
||||
"duration": 0,
|
||||
"data": {"signalType": "collector_health", "eventsFlushed": 100},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
rows = MODULE.build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end)
|
||||
|
||||
by_app = {row["application"]: row for row in rows}
|
||||
assert "1С" in by_app
|
||||
assert "Total Commander" in by_app
|
||||
assert by_app["1С"]["proved_work_seconds"] == 300
|
||||
assert by_app["1С"]["last_action"] == "Счета учета: Материалы"
|
||||
assert by_app["Total Commander"]["proved_work_seconds"] == 480
|
||||
assert by_app["Total Commander"]["last_action"] == "C:\\data\\report.xlsx"
|
||||
|
||||
|
||||
def test_render_html_contains_true_active_apps_table():
|
||||
html = MODULE.render_html(
|
||||
[],
|
||||
"SHARKON2025",
|
||||
datetime(2026, 5, 14, tzinfo=timezone.utc).date(),
|
||||
selected_day="today",
|
||||
true_active_apps=[
|
||||
{
|
||||
"application": "1С",
|
||||
"proved_work_human": "34 мин",
|
||||
"proved_work_hhmm": "00:34",
|
||||
"last_action_local": "15:31",
|
||||
"last_action": "Счета учета: Материалы",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert "Доказанная работа по приложениям" in html
|
||||
assert "Приложение" in html
|
||||
assert "Доказанная работа" in html
|
||||
assert "Последнее действие" in html
|
||||
assert "Счета учета: Материалы" in html
|
||||
|
||||
|
||||
def test_build_management_payload_creates_actions_for_missing_and_late_users():
|
||||
rows = [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": 27,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "influxdb_aw"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"decimals": 0,
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "value_and_name"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"query": "from(bucket: \"aw_metrics\")\n |> range(start: -7d)\n |> filter(fn: (r) => (r._measurement == \"aw_window_event\" or r._measurement == \"aw_afk_event\") and r._field == \"duration_s\")\n |> last()\n |> group()\n |> sort(columns: [\"_time\"], desc: true)\n |> limit(n: 1)\n |> map(fn: (r) => ({ r with _value: float(v: uint(v: now()) - uint(v: r._time)) / 1000000000.0 }))\n |> keep(columns: [\"_value\"])\n |> yield(name: \"freshness_seconds\")",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Свежесть данных AW",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "influxdb_aw"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 18,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> aggregateWindow(every: 5m, fn: sum, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"host\"])\n |> group(columns: [\"host\"])\n |> yield(name: \"active\")",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Активность окон (сумма за 5 минут)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "influxdb_aw"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_afk_event\" and r._field == \"duration_s\" and r.status == \"afk\")\n |> aggregateWindow(every: 5m, fn: sum, createEmpty: false)\n |> keep(columns: [\"_time\", \"_value\", \"host\"])\n |> group(columns: [\"host\"])\n |> yield(name: \"afk\")",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "AFK (сумма за 5 минут)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "influxdb_aw"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> map(fn: (r) => ({ r with category_view: if r.category == \"erp_1c\" then \"1С-Бухгалтерия\" else if r.category == \"file_manager\" then \"Файловый менеджер\" else if r.category == \"terminal_shell\" then \"Терминальная оболочка\" else if r.category == \"browser\" then \"Интернет-браузер\" else if r.category == \"office_docs\" then \"Офисные документы\" else if r.category == \"remote_admin\" then \"Удаленное администрирование\" else if r.category == \"development\" then \"Разработка и программирование\" else if r.category == \"communication\" then \"Средства связи\" else if r.category == \"database\" then \"Работа с базами данных\" else if r.category == \"monitoring\" then \"Мониторинг и диагностика\" else if r.category == \"virtualization\" then \"Виртуализация\" else if r.category == \"system\" or r.category == \"misc_user\" or r.category == \"other\" or r.category == \"\" then \"Системные приложения\" else r.category }))\n |> group(columns: [\"category_view\"])\n |> sum()\n |> group()\n |> sort(columns: [\"_value\"], desc: true)\n |> yield(name: \"categories\")",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Категории по активному времени",
|
||||
"type": "barchart"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "influxdb",
|
||||
"uid": "influxdb_aw"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"decimals": 0,
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"sum"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"query": "import \"date\"\nstartBound = if v.timeRangeStart > date.sub(d: 2h, from: now()) then date.sub(d: 2h, from: now()) else v.timeRangeStart\nfrom(bucket: \"aw_metrics\")\n |> range(start: startBound, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"aw_window_event\" and r._field == \"duration_s\")\n |> group()\n |> sum()\n |> yield(name: \"total_active\")",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Общее активное время",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"detmir",
|
||||
"activitywatch"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-12h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "DetMir ActivityWatch",
|
||||
"uid": "detmir-aw-main",
|
||||
"version": 3,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
# Smoke checks for the Proxmox/gateway/1C host 10.10.10.2.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
OK_COUNT=0
|
||||
WARN_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
SKIP_COUNT=0
|
||||
|
||||
if [ -t 1 ]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED=''
|
||||
GREEN=''
|
||||
YELLOW=''
|
||||
CYAN=''
|
||||
NC=''
|
||||
fi
|
||||
|
||||
pass() { OK_COUNT=$((OK_COUNT + 1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; }
|
||||
warn() { WARN_COUNT=$((WARN_COUNT + 1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||||
fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; }
|
||||
skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||||
|
||||
section() {
|
||||
printf "\n%b== %s ==%b\n" "$CYAN" "$*" "$NC"
|
||||
}
|
||||
|
||||
have() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
check_command() {
|
||||
local name="$1"
|
||||
shift
|
||||
local output
|
||||
if output="$("$@" 2>&1)"; then
|
||||
pass "$name"
|
||||
[ -n "$output" ] && printf "%s\n" "$output" | sed 's/^/ /'
|
||||
else
|
||||
fail "$name"
|
||||
[ -n "$output" ] && printf "%s\n" "$output" | sed 's/^/ /'
|
||||
fi
|
||||
}
|
||||
|
||||
check_service() {
|
||||
local unit="$1"
|
||||
if ! systemctl list-unit-files "$unit" >/dev/null 2>&1; then
|
||||
skip "$unit is not installed"
|
||||
return
|
||||
fi
|
||||
if systemctl is-active --quiet "$unit"; then
|
||||
pass "$unit active"
|
||||
else
|
||||
fail "$unit inactive or failed"
|
||||
systemctl --no-pager --lines=8 status "$unit" 2>&1 | sed 's/^/ /'
|
||||
fi
|
||||
}
|
||||
|
||||
check_timer() {
|
||||
local unit="$1"
|
||||
if ! systemctl list-unit-files "$unit" >/dev/null 2>&1; then
|
||||
skip "$unit is not installed"
|
||||
return
|
||||
fi
|
||||
if systemctl is-active --quiet "$unit"; then
|
||||
pass "$unit active"
|
||||
else
|
||||
fail "$unit inactive or failed"
|
||||
systemctl --no-pager --lines=8 status "$unit" 2>&1 | sed 's/^/ /'
|
||||
fi
|
||||
}
|
||||
|
||||
check_tcp() {
|
||||
local name="$1"
|
||||
local host="$2"
|
||||
local port="$3"
|
||||
if timeout 4 bash -c ":</dev/tcp/${host}/${port}" >/dev/null 2>&1; then
|
||||
pass "$name TCP ${host}:${port}"
|
||||
else
|
||||
fail "$name TCP ${host}:${port}"
|
||||
fi
|
||||
}
|
||||
|
||||
check_http_code() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local expected="${3:-^2[0-9][0-9]$}"
|
||||
local tmp code
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -k -sS --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||||
if printf "%s" "$code" | grep -Eq "$expected"; then
|
||||
pass "$name HTTP $code $url"
|
||||
else
|
||||
fail "$name HTTP $code $url"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_http_redirect() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local expected_code="${3:-^30[1278]$}"
|
||||
local expected_location="${4:-}"
|
||||
local tmp code location
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -k -sS -I --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||||
location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/\r$/,""); print $0}' "$tmp" | tail -1)"
|
||||
if printf "%s" "$code" | grep -Eq "$expected_code" && { [ -z "$expected_location" ] || printf "%s" "$location" | grep -Fq "$expected_location"; }; then
|
||||
pass "$name HTTP $code ${location:-$url}"
|
||||
else
|
||||
fail "$name HTTP $code ${location:-$url}"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_docker_container() {
|
||||
local name="$1"
|
||||
if ! have docker; then
|
||||
skip "docker command unavailable"
|
||||
return
|
||||
fi
|
||||
if docker ps --format '{{.Names}}' 2>/dev/null | grep -Fxq "$name"; then
|
||||
pass "docker container $name running"
|
||||
docker ps --filter "name=^/${name}$" --format ' {{.Names}} {{.Status}} {{.Ports}}'
|
||||
else
|
||||
fail "docker container $name not running"
|
||||
docker ps -a --filter "name=^/${name}$" --format ' {{.Names}} {{.Status}} {{.Ports}}' 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
section "Host"
|
||||
hostnamectl 2>/dev/null | sed 's/^/ /' || hostname | sed 's/^/ /'
|
||||
date -Is | sed 's/^/ /'
|
||||
uptime | sed 's/^/ /'
|
||||
|
||||
section "Core Services"
|
||||
for unit in \
|
||||
nginx.service \
|
||||
pveproxy.service \
|
||||
pvedaemon.service \
|
||||
pvestatd.service \
|
||||
pve-cluster.service \
|
||||
docker.service \
|
||||
aw-1c-company-api.service \
|
||||
aw-pve-webadmin-logger.service
|
||||
do
|
||||
check_service "$unit"
|
||||
done
|
||||
|
||||
section "Timers"
|
||||
for unit in \
|
||||
aw-1c-ingest.timer \
|
||||
aw-1c-proofcheck.timer \
|
||||
aw-1c-manager-brief.timer \
|
||||
aw-1c-recovery-brief.timer \
|
||||
aw-1c-weekly-digest.timer
|
||||
do
|
||||
check_timer "$unit"
|
||||
done
|
||||
systemctl list-timers --all --no-pager 2>/dev/null | grep -E 'aw-1c|NEXT|LEFT|PASSED' | sed 's/^/ /' || true
|
||||
|
||||
section "Ports"
|
||||
check_tcp "nginx http" 127.0.0.1 80
|
||||
check_tcp "nginx https" 127.0.0.1 443
|
||||
check_tcp "proxmox web" 127.0.0.1 8006
|
||||
check_tcp "1C company API" 10.10.10.2 8710
|
||||
check_tcp "clickhouse native" 127.0.0.1 9000
|
||||
check_tcp "clickhouse http" 127.0.0.1 8123
|
||||
ss -tulpn | grep -E ':(80|443|8006|8710|8123|9000)\b' | sed 's/^/ /' || true
|
||||
|
||||
section "Gateway HTTP"
|
||||
check_http_code "nginx healthz" "https://127.0.0.1/healthz" '^200$'
|
||||
check_http_redirect "go proxmox gui" "https://127.0.0.1/go/proxmox-gui" '^30[1278]$' 'https://10.10.10.2:8006/'
|
||||
check_http_redirect "go file1c brief" "https://127.0.0.1/go/file1c-brief" '^30[1278]$' 'http://10.10.10.2:8710/manager/brief'
|
||||
check_http_redirect "go file1c actions" "https://127.0.0.1/go/file1c-actions" '^30[1278]$' 'http://10.10.10.2:8710/manager/actions'
|
||||
|
||||
section "1C Company API"
|
||||
check_http_code "1C root redirect" "http://10.10.10.2:8710/" '^307$'
|
||||
check_http_code "1C /health" "http://10.10.10.2:8710/health" '^200$'
|
||||
check_http_code "1C /api/health" "http://10.10.10.2:8710/api/health" '^200$'
|
||||
check_http_code "1C manager brief" "http://10.10.10.2:8710/manager/brief" '^200$'
|
||||
check_http_code "1C manager actions" "http://10.10.10.2:8710/manager/actions" '^200$'
|
||||
check_http_code "1C manager recovery" "http://10.10.10.2:8710/manager/recovery" '^200$'
|
||||
check_http_code "1C weekly digest" "http://10.10.10.2:8710/manager/digest/weekly" '^200$'
|
||||
|
||||
section "ClickHouse"
|
||||
check_docker_container "aw-rus-1c-clickhouse"
|
||||
check_http_code "ClickHouse ping" "http://127.0.0.1:8123/ping" '^200$'
|
||||
if have docker && docker ps --format '{{.Names}}' | grep -Fxq aw-rus-1c-clickhouse; then
|
||||
check_command "ClickHouse SELECT 1" docker exec aw-rus-1c-clickhouse clickhouse-client --query "SELECT 1"
|
||||
fi
|
||||
|
||||
section "System Capacity"
|
||||
df -h / /var /opt 2>/dev/null | sed 's/^/ /'
|
||||
free -h 2>/dev/null | sed 's/^/ /' || true
|
||||
|
||||
section "Summary"
|
||||
printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
|
||||
|
||||
if [ "$FAIL_COUNT" -gt 0 ]; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,597 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end smoke checks from Igor's laptop for ActivityWatch-Russian.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SMOKE_ENV_FILE="${AW_SMOKE_ENV_FILE:-}"
|
||||
for env_candidate in "$REPO_ROOT/secrets/runtime.env" "$HOME/.config/aw-contour-smoke.env" "$SMOKE_ENV_FILE"; do
|
||||
if [ -n "$env_candidate" ] && [ -f "$env_candidate" ]; then
|
||||
# Load local credentials and site-specific overrides without committing them.
|
||||
# Later files override earlier defaults.
|
||||
set -a
|
||||
. "$env_candidate"
|
||||
set +a
|
||||
fi
|
||||
done
|
||||
|
||||
ANSIBLE_DIR="$REPO_ROOT/ansible"
|
||||
INVENTORY="${AW_SMOKE_INVENTORY:-$ANSIBLE_DIR/inventory.ini}"
|
||||
REMOTE_SCRIPT_SRC="$REPO_ROOT/scripts/aw-contour-smoke-10.10.10.2.sh"
|
||||
REMOTE_SCRIPT_DST="${AW_SMOKE_REMOTE_SCRIPT:-/usr/local/sbin/aw-contour-smoke.sh}"
|
||||
AW_SERVER="${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}"
|
||||
WORKTIME_API="${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}"
|
||||
GRAFANA_URL="${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}"
|
||||
GRAFANA_USER="${GRAFANA_USER:-igor}"
|
||||
GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-}"
|
||||
PROXMOX_HOST="${AW_SMOKE_PROXMOX_HOST:-10.10.10.2}"
|
||||
AW_HOST="${AW_SMOKE_AW_HOST:-10.10.10.13}"
|
||||
GRAFANA_HOST="${AW_SMOKE_GRAFANA_HOST:-10.10.10.11}"
|
||||
WINDOWS_HOST="${AW_SMOKE_WINDOWS_HOST:-192.168.100.18}"
|
||||
AW_SOURCE_HOSTNAME="${AW_SMOKE_SOURCE_HOSTNAME:-SHARKON2025}"
|
||||
LOG_DIR="${AW_SMOKE_LOG_DIR:-$REPO_ROOT/output/smoke}"
|
||||
RUN_REMOTE="${AW_SMOKE_RUN_REMOTE:-1}"
|
||||
RUN_WINRM="${AW_SMOKE_RUN_WINRM:-1}"
|
||||
RUN_SERVER_SYSTEMD="${AW_SMOKE_RUN_SERVER_SYSTEMD:-1}"
|
||||
|
||||
NO_PROXY_REQUIRED="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$WINDOWS_HOST,10.10.10.0/24,192.168.100.0/24"
|
||||
if [ -n "${no_proxy:-}" ]; then
|
||||
export no_proxy="$no_proxy,$NO_PROXY_REQUIRED"
|
||||
else
|
||||
export no_proxy="$NO_PROXY_REQUIRED"
|
||||
fi
|
||||
if [ -n "${NO_PROXY:-}" ]; then
|
||||
export NO_PROXY="$NO_PROXY,$NO_PROXY_REQUIRED"
|
||||
else
|
||||
export NO_PROXY="$no_proxy"
|
||||
fi
|
||||
|
||||
OK_COUNT=0
|
||||
WARN_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
SKIP_COUNT=0
|
||||
|
||||
if [ -t 1 ]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED=''
|
||||
GREEN=''
|
||||
YELLOW=''
|
||||
CYAN=''
|
||||
NC=''
|
||||
fi
|
||||
|
||||
usage() {
|
||||
cat <<USAGE
|
||||
Usage: $(basename "$0") [--skip-remote] [--skip-winrm] [--skip-server-systemd]
|
||||
|
||||
Environment overrides:
|
||||
AW_SMOKE_AW_SERVER=http://10.10.10.13:5600
|
||||
AW_SMOKE_WORKTIME_API=http://10.10.10.13:5610
|
||||
AW_SMOKE_GRAFANA_URL=http://10.10.10.11:3000
|
||||
AW_SMOKE_SOURCE_HOSTNAME=SHARKON2025
|
||||
AW_SMOKE_ENV_FILE=$HOME/.config/aw-contour-smoke.env
|
||||
AW_SMOKE_LOG_DIR=$REPO_ROOT/output/smoke
|
||||
GRAFANA_USER/GRAFANA_PASSWORD via env or a local env file
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--skip-remote) RUN_REMOTE=0 ;;
|
||||
--skip-winrm) RUN_WINRM=0 ;;
|
||||
--skip-server-systemd) RUN_SERVER_SYSTEMD=0 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown argument: $1" >&2; usage >&2; exit 64 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
pass() { OK_COUNT=$((OK_COUNT + 1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; }
|
||||
warn() { WARN_COUNT=$((WARN_COUNT + 1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||||
fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; }
|
||||
skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||||
|
||||
section() {
|
||||
printf "\n%b== %s ==%b\n" "$CYAN" "$*" "$NC"
|
||||
}
|
||||
|
||||
have() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
check_local_command() {
|
||||
local cmd="$1"
|
||||
if have "$cmd"; then
|
||||
pass "command available: $cmd"
|
||||
else
|
||||
fail "command missing: $cmd"
|
||||
fi
|
||||
}
|
||||
|
||||
check_tcp() {
|
||||
local name="$1"
|
||||
local host="$2"
|
||||
local port="$3"
|
||||
if timeout 4 bash -c ":</dev/tcp/${host}/${port}" >/dev/null 2>&1; then
|
||||
pass "$name TCP ${host}:${port}"
|
||||
else
|
||||
fail "$name TCP ${host}:${port}"
|
||||
fi
|
||||
}
|
||||
|
||||
check_ssh_banner() {
|
||||
local name="$1"
|
||||
local host="$2"
|
||||
local port="${3:-22}"
|
||||
local banner
|
||||
banner="$(
|
||||
timeout 5 bash -c "exec 3<>/dev/tcp/${host}/${port}; IFS= read -r line <&3; printf '%s' \"\$line\"" 2>/dev/null || true
|
||||
)"
|
||||
if printf "%s" "$banner" | grep -Eq '^SSH-[0-9]+\.[0-9]+'; then
|
||||
pass "$name SSH banner ${host}:${port} ${banner}"
|
||||
else
|
||||
fail "$name SSH banner ${host}:${port} unavailable"
|
||||
fi
|
||||
}
|
||||
|
||||
check_http_code() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local expected="${3:-^2[0-9][0-9]$}"
|
||||
local tmp code
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -k -sS --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||||
if printf "%s" "$code" | grep -Eq "$expected"; then
|
||||
pass "$name HTTP $code $url"
|
||||
else
|
||||
fail "$name HTTP $code $url"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_http_json_key() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local jq_filter="$3"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
|
||||
pass "$name JSON $jq_filter"
|
||||
jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/ /' | head -5
|
||||
else
|
||||
fail "$name JSON $jq_filter"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_http_json_key_basic_auth() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local jq_filter="$3"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
|
||||
pass "$name JSON $jq_filter"
|
||||
jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/ /' | head -5
|
||||
else
|
||||
fail "$name JSON $jq_filter"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_http_code_basic_auth() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local expected="${3:-^2[0-9][0-9]$}"
|
||||
local tmp code
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||||
if printf "%s" "$code" | grep -Eq "$expected"; then
|
||||
pass "$name HTTP $code $url"
|
||||
else
|
||||
fail "$name HTTP $code $url"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_grafana_loki_proxy() {
|
||||
local name="Grafana Loki datasource proxy"
|
||||
local tmp uid code body
|
||||
tmp="$(mktemp)"
|
||||
if ! curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources" -o "$tmp" 2>"$tmp.err"; then
|
||||
fail "$name cannot list datasources"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
return
|
||||
fi
|
||||
uid="$(jq -r '.[] | select(.type == "loki") | .uid' "$tmp" 2>/dev/null | head -1)"
|
||||
if [ -z "$uid" ]; then
|
||||
fail "$name no loki datasource found"
|
||||
jq -r '.[] | "\(.name) \(.type) \(.uid)"' "$tmp" 2>/dev/null | sed 's/^/ /' | head -20
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
return
|
||||
fi
|
||||
body="$(mktemp)"
|
||||
code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$body" -w '%{http_code}' "$GRAFANA_URL/api/datasources/proxy/uid/$uid/loki/api/v1/labels" 2>"$tmp.err")"
|
||||
if [ "$code" = "200" ]; then
|
||||
pass "$name HTTP $code uid=$uid"
|
||||
jq -r '.status // .data[0] // .' "$body" 2>/dev/null | sed 's/^/ /' | head -5
|
||||
else
|
||||
fail "$name HTTP $code uid=$uid"
|
||||
sed 's/^/ /' "$tmp.err" "$body" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err" "$body"
|
||||
}
|
||||
|
||||
check_grafana_influx_health() {
|
||||
local name="Grafana InfluxDB-AW datasource health"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources/uid/influxdb_aw/health" -o "$tmp" 2>"$tmp.err" && jq -e '.status == "OK"' "$tmp" >/dev/null 2>&1; then
|
||||
pass "$name"
|
||||
jq -r '.message // .status // .' "$tmp" 2>/dev/null | sed 's/^/ /' | head -5
|
||||
else
|
||||
fail "$name"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_grafana_aw_main_dashboard_queries() {
|
||||
local name="Grafana detmir-aw-main window/AFK panel queries"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/dashboards/uid/detmir-aw-main" -o "$tmp" 2>"$tmp.err" && \
|
||||
jq -e '
|
||||
def fixed:
|
||||
(.targets[0].query // "") | contains("keep(columns: [\"_time\", \"_value\", \"host\"]");
|
||||
([.dashboard.panels[] | select(.title == "Активность окон (сумма за 5 минут)" or .title == "AFK (сумма за 5 минут)") | fixed] | length == 2 and all(. == true))
|
||||
' "$tmp" >/dev/null 2>&1; then
|
||||
pass "$name"
|
||||
else
|
||||
fail "$name missing keep(_time,_value,host) after aggregateWindow"
|
||||
jq -r '.dashboard.panels[]? | select(.title == "Активность окон (сумма за 5 минут)" or .title == "AFK (сумма за 5 минут)") | "\(.title): " + ((.targets[0].query // "") | gsub("\n"; " "))' "$tmp" 2>/dev/null | sed 's/^/ /' | head -20
|
||||
sed 's/^/ /' "$tmp.err" 2>/dev/null | head -20
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
check_http_redirect() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
local expected_code="${3:-^30[1278]$}"
|
||||
local expected_location="${4:-}"
|
||||
local tmp code location
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -k -sS -I --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||||
location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/\r$/,""); print $0}' "$tmp" | tail -1)"
|
||||
if printf "%s" "$code" | grep -Eq "$expected_code" && { [ -z "$expected_location" ] || printf "%s" "$location" | grep -Fq "$expected_location"; }; then
|
||||
pass "$name HTTP $code ${location:-$url}"
|
||||
else
|
||||
fail "$name HTTP $code ${location:-$url}"
|
||||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
|
||||
fi
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
}
|
||||
|
||||
ansible_shell() {
|
||||
local group="$1"
|
||||
local command="$2"
|
||||
ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command"
|
||||
}
|
||||
|
||||
ansible_win_shell() {
|
||||
local group="$1"
|
||||
local command="$2"
|
||||
ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m win_shell -a "$command"
|
||||
}
|
||||
|
||||
check_ansible_shell() {
|
||||
local name="$1"
|
||||
local group="$2"
|
||||
local command="$3"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if ansible_shell "$group" "$command" >"$tmp" 2>&1; then
|
||||
pass "$name"
|
||||
sed 's/^/ /' "$tmp" | head -80
|
||||
else
|
||||
fail "$name"
|
||||
sed 's/^/ /' "$tmp" | head -120
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
check_ansible_win_shell() {
|
||||
local name="$1"
|
||||
local group="$2"
|
||||
local command="$3"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
|
||||
pass "$name"
|
||||
sed 's/^/ /' "$tmp" | head -80
|
||||
else
|
||||
fail "$name"
|
||||
sed 's/^/ /' "$tmp" | head -120
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
check_ansible_win_shell_warn() {
|
||||
local name="$1"
|
||||
local group="$2"
|
||||
local command="$3"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
|
||||
pass "$name"
|
||||
sed 's/^/ /' "$tmp" | head -80
|
||||
else
|
||||
warn "$name returned non-zero"
|
||||
sed 's/^/ /' "$tmp" | head -120
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
check_ansible_module() {
|
||||
local name="$1"
|
||||
local group="$2"
|
||||
local module="$3"
|
||||
local args="${4:-}"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then
|
||||
pass "$name"
|
||||
sed 's/^/ /' "$tmp" | head -80
|
||||
else
|
||||
fail "$name"
|
||||
sed 's/^/ /' "$tmp" | head -120
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
classify_bucket_age() {
|
||||
local bucket="$1"
|
||||
local age_sec="$2"
|
||||
case "$bucket" in
|
||||
aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
|
||||
if [ "$age_sec" -lt 86400 ]; then
|
||||
printf "fresh"
|
||||
else
|
||||
printf "event-driven"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
if [ "$age_sec" -lt 3600 ]; then
|
||||
printf "fresh"
|
||||
elif [ "$age_sec" -lt 86400 ]; then
|
||||
printf "stale"
|
||||
else
|
||||
printf "dead"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_bucket_freshness() {
|
||||
local bucket="$1"
|
||||
local bucket_id="${bucket}_${AW_SOURCE_HOSTNAME}"
|
||||
local tmp last_ts last_id event_epoch now age_sec status
|
||||
tmp="$(mktemp)"
|
||||
if ! curl -fsS --connect-timeout 5 --max-time 20 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then
|
||||
fail "bucket $bucket_id query failed"
|
||||
sed 's/^/ /' "$tmp.err" | head -20
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
return
|
||||
fi
|
||||
|
||||
last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)"
|
||||
last_id="$(jq -r '.[0].id // 0' "$tmp" 2>/dev/null)"
|
||||
rm -f "$tmp" "$tmp.err"
|
||||
|
||||
if [ -z "$last_ts" ]; then
|
||||
case "$bucket" in
|
||||
aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
|
||||
warn "bucket $bucket_id empty/event-driven"
|
||||
;;
|
||||
*)
|
||||
fail "bucket $bucket_id empty"
|
||||
;;
|
||||
esac
|
||||
return
|
||||
fi
|
||||
|
||||
event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || printf "0")"
|
||||
now="$(date -u +%s)"
|
||||
if [ "$event_epoch" -le 0 ]; then
|
||||
warn "bucket $bucket_id has unparsable timestamp: $last_ts"
|
||||
return
|
||||
fi
|
||||
age_sec=$((now - event_epoch))
|
||||
status="$(classify_bucket_age "$bucket" "$age_sec")"
|
||||
case "$status" in
|
||||
fresh|event-driven)
|
||||
pass "bucket $bucket_id $status age=${age_sec}s id=$last_id"
|
||||
;;
|
||||
stale)
|
||||
warn "bucket $bucket_id stale age=${age_sec}s id=$last_id"
|
||||
;;
|
||||
*)
|
||||
fail "bucket $bucket_id dead age=${age_sec}s id=$last_id"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
run_remote_proxmox_script() {
|
||||
if [ "$RUN_REMOTE" != "1" ]; then
|
||||
skip "remote 10.10.10.2 smoke skipped"
|
||||
return
|
||||
fi
|
||||
if ! have ansible; then
|
||||
fail "ansible unavailable; cannot deploy/run remote script"
|
||||
return
|
||||
fi
|
||||
if [ ! -f "$REMOTE_SCRIPT_SRC" ]; then
|
||||
fail "remote script source missing: $REMOTE_SCRIPT_SRC"
|
||||
return
|
||||
fi
|
||||
|
||||
section "Deploy Remote Script To 10.10.10.2"
|
||||
if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_SCRIPT_SRC dest=$REMOTE_SCRIPT_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy.$$ 2>&1; then
|
||||
pass "remote script deployed to $REMOTE_SCRIPT_DST"
|
||||
else
|
||||
fail "remote script deploy failed"
|
||||
sed 's/^/ /' /tmp/aw-smoke-copy.$$ | head -120
|
||||
rm -f /tmp/aw-smoke-copy.$$
|
||||
return
|
||||
fi
|
||||
rm -f /tmp/aw-smoke-copy.$$
|
||||
|
||||
section "Remote 10.10.10.2 Smoke"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m shell -a "$REMOTE_SCRIPT_DST" >"$tmp" 2>&1; then
|
||||
pass "remote 10.10.10.2 smoke completed"
|
||||
sed 's/^/ /' "$tmp"
|
||||
else
|
||||
fail "remote 10.10.10.2 smoke failed"
|
||||
sed 's/^/ /' "$tmp"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
LOG_FILE="$LOG_DIR/aw-contour-smoke-$(date +%Y%m%d-%H%M%S).log"
|
||||
exec > >(tee "$LOG_FILE") 2>&1
|
||||
|
||||
section "Run Context"
|
||||
printf "repo=%s\ninventory=%s\nlog=%s\n" "$REPO_ROOT" "$INVENTORY" "$LOG_FILE" | sed 's/^/ /'
|
||||
date -Is | sed 's/^/ /'
|
||||
|
||||
section "Local Prerequisites"
|
||||
for cmd in bash curl jq timeout ansible ssh; do
|
||||
check_local_command "$cmd"
|
||||
done
|
||||
printf " no_proxy=%s\n" "$no_proxy"
|
||||
|
||||
section "Laptop Network"
|
||||
ip -br addr 2>/dev/null | sed 's/^/ /' || true
|
||||
ip route get "$PROXMOX_HOST" 2>/dev/null | sed 's/^/ /' || warn "no route detail for $PROXMOX_HOST"
|
||||
ip route get "$AW_HOST" 2>/dev/null | sed 's/^/ /' || warn "no route detail for $AW_HOST"
|
||||
|
||||
section "TCP Surface"
|
||||
check_tcp "Proxmox SSH" "$PROXMOX_HOST" 22
|
||||
check_tcp "Proxmox HTTP" "$PROXMOX_HOST" 80
|
||||
check_tcp "Proxmox HTTPS" "$PROXMOX_HOST" 443
|
||||
check_tcp "Proxmox GUI" "$PROXMOX_HOST" 8006
|
||||
check_tcp "1C company API" "$PROXMOX_HOST" 8710
|
||||
check_tcp "AW server" "$AW_HOST" 5600
|
||||
check_tcp "AW worktime API" "$AW_HOST" 5610
|
||||
check_tcp "Grafana" "$GRAFANA_HOST" 3000
|
||||
check_tcp "Windows WinRM" "$WINDOWS_HOST" 5985
|
||||
check_tcp "Windows SSH" "$WINDOWS_HOST" 22
|
||||
check_ssh_banner "Windows SSH" "$WINDOWS_HOST" 22
|
||||
|
||||
section "ActivityWatch HTTP"
|
||||
check_http_json_key "AW server info" "$AW_SERVER/api/0/info" '.version'
|
||||
check_http_code "AW settings CORS" "$AW_SERVER/api/0/settings/" '^200$'
|
||||
check_http_code "AW WebUI" "$AW_SERVER/" '^200$'
|
||||
check_http_json_key "AW buckets list" "$AW_SERVER/api/0/buckets/" 'keys | length'
|
||||
|
||||
section "ActivityWatch Buckets"
|
||||
for bucket in \
|
||||
aw-watcher-afk \
|
||||
aw-watcher-window \
|
||||
aw-worktime-sessions \
|
||||
aw-session-events \
|
||||
aw-dlp-endpoint-signals \
|
||||
aw-dlp-incidents \
|
||||
aw-dlp-review \
|
||||
aw-dlp-rules
|
||||
do
|
||||
check_bucket_freshness "$bucket"
|
||||
done
|
||||
|
||||
section "Worktime API"
|
||||
check_http_json_key "worktime health" "$WORKTIME_API/health" '.status // .ok // .'
|
||||
check_http_code "worktime today html" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$'
|
||||
check_http_json_key "worktime today json" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today" '.host // .report.host // .[0].host // .'
|
||||
check_http_code "worktime management html" "$WORKTIME_API/reports/worktime/management?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$'
|
||||
|
||||
section "Gateway And 1C HTTP"
|
||||
check_http_code "gateway healthz" "https://$PROXMOX_HOST/healthz" '^200$'
|
||||
check_http_redirect "gateway proxmox redirect" "https://$PROXMOX_HOST/go/proxmox-gui" '^30[1278]$' "https://$PROXMOX_HOST:8006/"
|
||||
check_http_redirect "gateway file1c brief redirect" "https://$PROXMOX_HOST/go/file1c-brief" '^30[1278]$' "http://$PROXMOX_HOST:8710/manager/brief"
|
||||
check_http_code "1C /health" "http://$PROXMOX_HOST:8710/health" '^200$'
|
||||
check_http_code "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" '^200$'
|
||||
check_http_code "1C manager brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$'
|
||||
check_http_code "1C manager actions" "http://$PROXMOX_HOST:8710/manager/actions" '^200$'
|
||||
check_http_code "1C manager recovery" "http://$PROXMOX_HOST:8710/manager/recovery" '^200$'
|
||||
check_http_code "1C weekly digest" "http://$PROXMOX_HOST:8710/manager/digest/weekly" '^200$'
|
||||
|
||||
section "Grafana HTTP"
|
||||
check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" '.database // .version // .commit'
|
||||
check_http_code "Grafana dashboards page" "$GRAFANA_URL/dashboards" '^200$|^302$'
|
||||
check_http_code "Grafana pfSense dashboard" "$GRAFANA_URL/d/pfsense-loki-dashboard/pfsense-firewall-overview?orgId=1&from=now-1h&to=now&timezone=browser" '^200$|^302$'
|
||||
|
||||
if [ -n "${GRAFANA_USER:-}" ] && [ -n "${GRAFANA_PASSWORD:-}" ]; then
|
||||
section "Grafana Authenticated API"
|
||||
check_http_json_key_basic_auth "Grafana datasources" "$GRAFANA_URL/api/datasources" 'length'
|
||||
check_grafana_loki_proxy
|
||||
check_grafana_influx_health
|
||||
check_grafana_aw_main_dashboard_queries
|
||||
else
|
||||
skip "Grafana authenticated datasource checks need GRAFANA_USER and GRAFANA_PASSWORD env"
|
||||
fi
|
||||
|
||||
if [ "$RUN_SERVER_SYSTEMD" = "1" ] && have ansible; then
|
||||
section "AW Server Systemd"
|
||||
check_ansible_shell "AW server core units" aw_server 'systemctl is-active activitywatch-server aw-worktime-api aw-worktime-ui-bridge.timer aw-rus-healthd.timer aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer aw-worktime-autoheal.timer'
|
||||
check_ansible_shell "AW server failed units" aw_server 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|influx|grafana|prometheus|loki" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
|
||||
check_ansible_shell "AW server local health script" aw_server 'test -x /opt/activitywatch/health-check.sh && /opt/activitywatch/health-check.sh || test -x /usr/local/bin/health-check.sh && /usr/local/bin/health-check.sh || echo "health-check script not installed"'
|
||||
else
|
||||
skip "AW server systemd checks skipped"
|
||||
fi
|
||||
|
||||
if [ "$RUN_WINRM" = "1" ] && have ansible; then
|
||||
section "Windows WinRM And Collectors"
|
||||
check_ansible_module "Windows win_ping" aw_windows win_ping
|
||||
check_ansible_win_shell "Windows sessions" aw_windows '$psi = [System.Diagnostics.ProcessStartInfo]::new(); $psi.FileName = "$env:SystemRoot\System32\query.exe"; $psi.Arguments = "user"; $psi.UseShellExecute = $false; $psi.RedirectStandardOutput = $true; $psi.RedirectStandardError = $true; $psi.StandardOutputEncoding = [System.Text.Encoding]::GetEncoding(866); $psi.StandardErrorEncoding = [System.Text.Encoding]::GetEncoding(866); $p = [System.Diagnostics.Process]::Start($psi); $out = $p.StandardOutput.ReadToEnd(); $err = $p.StandardError.ReadToEnd(); $p.WaitForExit(); [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $out; if ($err) { $err }; if ($out -match "USERNAME|ПОЛЬЗОВАТЕЛЬ|администратор|Администратор") { exit 0 } else { exit $p.ExitCode }'
|
||||
check_ansible_win_shell_warn "Windows collector processes" aw_windows '$p = Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue; if ($p) { $p | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize } else { "no aw-watcher-afk/window process visible to this WinRM session" }'
|
||||
check_ansible_win_shell "Windows ActivityWatch tasks" aw_windows 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V; schtasks /Query /TN "ActivityWatch Launch [SHARKON2025_Администратор]" /FO LIST /V'
|
||||
else
|
||||
skip "Windows WinRM checks skipped"
|
||||
fi
|
||||
|
||||
run_remote_proxmox_script
|
||||
|
||||
section "Existing Repo Checks"
|
||||
if [ -x "$REPO_ROOT/check-aw-data.sh" ]; then
|
||||
if "$REPO_ROOT/check-aw-data.sh"; then pass "check-aw-data.sh completed"; else fail "check-aw-data.sh failed"; fi
|
||||
else
|
||||
skip "check-aw-data.sh missing"
|
||||
fi
|
||||
if [ -x "$REPO_ROOT/check-aw-full.sh" ]; then
|
||||
if "$REPO_ROOT/check-aw-full.sh"; then pass "check-aw-full.sh completed"; else fail "check-aw-full.sh failed"; fi
|
||||
else
|
||||
skip "check-aw-full.sh missing"
|
||||
fi
|
||||
|
||||
section "Summary"
|
||||
printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
|
||||
printf "Log: %s\n" "$LOG_FILE"
|
||||
|
||||
if [ "$FAIL_COUNT" -gt 0 ]; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
Reference in New Issue
Block a user