feat(worktime): add automatic self-heal timer for zero-activity regressions

ansible/deploy_aw_server.yml

@@ -375,6 +375,14 @@
             group: root
             mode: "0755"
 
+        - name: Установить скрипт AW worktime autoheal
+          ansible.builtin.copy:
+            src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.sh"
+            dest: /usr/local/bin/aw-worktime-autoheal.sh
+            owner: root
+            group: root
+            mode: "0755"
+
         - name: Установить скрипт aw-health-check
           ansible.builtin.copy:
             src: "{{ aw_repo_root }}/aw-server/health-check.sh"
@@ -399,6 +407,22 @@
             group: root
             mode: "0644"
 
+        - name: Установить systemd unit AW worktime autoheal
+          ansible.builtin.copy:
+            src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.service"
+            dest: /etc/systemd/system/aw-worktime-autoheal.service
+            owner: root
+            group: root
+            mode: "0644"
+
+        - name: Установить systemd timer AW worktime autoheal
+          ansible.builtin.copy:
+            src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.timer"
+            dest: /etc/systemd/system/aw-worktime-autoheal.timer
+            owner: root
+            group: root
+            mode: "0644"
+
         - name: Перезагрузить systemd после установки AW worktime API
           ansible.builtin.systemd:
             daemon_reload: true
@@ -435,6 +459,18 @@
             state: started
           failed_when: false
 
+        - name: Включить и перезапустить AW worktime autoheal timer
+          ansible.builtin.systemd:
+            name: aw-worktime-autoheal.timer
+            enabled: true
+            state: restarted
+
+        - name: Выполнить разовый прогон AW worktime autoheal
+          ansible.builtin.systemd:
+            name: aw-worktime-autoheal.service
+            state: started
+          failed_when: false
+
         - name: Применить хотфиксы compiled JS чанков (Trends, Timespiral, Category helper)
           ansible.builtin.command:
             cmd: "/opt/activitywatch/aw-server/apply_webui_ru_patch.sh"

aw-server/aw-worktime-autoheal.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=AW Worktime Autoheal (SHARKON2025)
+After=network-online.target activitywatch-server.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=AW_URL=http://127.0.0.1:5600
+Environment=AW_WORKTIME_HOST=SHARKON2025
+ExecStart=/usr/local/bin/aw-worktime-autoheal.sh
+User=root
+Group=root
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=aw-worktime-autoheal
+

aw-server/aw-worktime-autoheal.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+AW_URL="${AW_URL:-http://127.0.0.1:5600}"
+HOST="${AW_WORKTIME_HOST:-SHARKON2025}"
+PYTHON_BIN="${PYTHON_BIN:-/usr/bin/python3}"
+LOG_TAG="aw-worktime-autoheal"
+
+log() {
+  logger -t "$LOG_TAG" "$*"
+  printf '%s %s\n' "$(date '+%F %T')" "$*"
+}
+
+need_heal="$("$PYTHON_BIN" - <<'PY'
+import json, urllib.request, datetime, os, sys
+AW=os.environ.get("AW_URL","http://127.0.0.1:5600")
+host=os.environ.get("HOST","SHARKON2025")
+bucket=f"aw-watcher-window_{host}"
+msk=datetime.timezone(datetime.timedelta(hours=3))
+start=datetime.datetime.now(msk).replace(hour=0,minute=0,second=0,microsecond=0).astimezone(datetime.timezone.utc)
+
+def p(ts):
+    if ts.endswith("Z"): ts=ts[:-1] + "+00:00"
+    return datetime.datetime.fromisoformat(ts).astimezone(datetime.timezone.utc)
+
+try:
+    with urllib.request.urlopen(f"{AW}/api/0/buckets/{bucket}/events?limit=12000", timeout=25) as r:
+        ev=json.loads(r.read().decode("utf-8"))
+except Exception:
+    print("1")
+    sys.exit(0)
+
+active=0.0
+for e in ev:
+    ts=e.get("timestamp")
+    if not ts:
+        continue
+    try:
+        if p(ts) < start:
+            continue
+    except Exception:
+        continue
+    d=float(e.get("duration",0) or 0)
+    title=((e.get("data") or {}).get("title") or "").strip().lower()
+    if d > 0 and "rdp active" in title:
+        active += d
+
+print("1" if active <= 0 else "0")
+PY
+)"
+
+if [[ "$need_heal" != "1" ]]; then
+  log "health ok: activity present for ${HOST}, no action"
+  exit 0
+fi
+
+log "detected zero activity for ${HOST}, running heal"
+systemctl restart aw-worktime-ui-bridge.timer
+systemctl start aw-worktime-ui-bridge.service || true
+
+"$PYTHON_BIN" - <<'PY'
+import json, urllib.request, datetime, os
+AW=os.environ.get("AW_URL","http://127.0.0.1:5600")
+host=os.environ.get("HOST","SHARKON2025")
+sb=f"aw-worktime-sessions_{host}"
+afk=f"aw-watcher-afk_{host}"
+win=f"aw-watcher-window_{host}"
+msk=datetime.timezone(datetime.timedelta(hours=3))
+start=datetime.datetime.now(msk).replace(hour=0,minute=0,second=0,microsecond=0).astimezone(datetime.timezone.utc)
+
+def req(method,path,payload=None):
+    data=None; headers={}
+    if payload is not None:
+        data=json.dumps(payload,ensure_ascii=False).encode("utf-8"); headers["Content-Type"]="application/json"
+    r=urllib.request.Request(AW+path,data=data,headers=headers,method=method)
+    with urllib.request.urlopen(r,timeout=30) as resp:
+        body=resp.read()
+        return json.loads(body.decode("utf-8")) if body else None
+
+def parse(ts):
+    if ts.endswith("Z"): ts=ts[:-1]+"+00:00"
+    return datetime.datetime.fromisoformat(ts).astimezone(datetime.timezone.utc)
+
+def is_active(d):
+    if isinstance(d.get("active"), bool) and d.get("active"): return True
+    st=str(d.get("state","")).strip().lower()
+    if st in ("active","активно"): return True
+    if st=="unknown":
+        try: sid=int(d.get("sessionId"))
+        except: sid=-1
+        u=str(d.get("username","")).strip().lower()
+        if sid>0 and u and (not u.endswith("$")): return True
+    return False
+
+rows=req("GET",f"/api/0/buckets/{sb}/events?limit=12000") or []
+rows=[e for e in rows if e.get("timestamp") and parse(e["timestamp"])>=start]
+if not rows:
+    raise SystemExit(0)
+
+by={}
+for e in rows:
+    by.setdefault(e["timestamp"],[]).append(e)
+keys=sorted(by.keys())
+out_afk=[]; out_win=[]
+for i,ts in enumerate(keys):
+    cur=parse(ts)
+    nxt=parse(keys[i+1]) if i+1<len(keys) else None
+    dur=max(0.0,(nxt-cur).total_seconds()) if nxt else 10.0
+    dur=min(3600.0, dur if dur>0 else 10.0)
+    act=[]
+    for r in by[ts]:
+        d=r.get("data") or {}
+        u=str(d.get("username","")).strip()
+        if u and is_active(d): act.append(u)
+    act=sorted(set(act))
+    active=bool(act)
+    out_afk.append({"timestamp":ts,"duration":dur,"data":{"status":"not-afk" if active else "afk","source":"aw-worktime-autoheal"}})
+    out_win.append({"timestamp":ts,"duration":dur,"data":{"app":"RDP","title":("RDP active (%d): %s"%(len(act),", ".join(act))) if active else "RDP idle","source":"aw-worktime-autoheal"}})
+
+req("POST",f"/api/0/buckets/{afk}/events",out_afk)
+req("POST",f"/api/0/buckets/{win}/events",out_win)
+print(f"autoheal backfill posted afk={len(out_afk)} win={len(out_win)}")
+PY
+
+log "heal completed for ${HOST}"

aw-server/aw-worktime-autoheal.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Run AW Worktime Autoheal every 3 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=3min
+Unit=aw-worktime-autoheal.service
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+