feat(detmir): add rust-first operations tooling

2026-06-02 17:57:58 +03:00
parent 60670d30a8
commit 19e3682bc8
263 changed files with 51678 additions and 718 deletions
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cmd="$(basename "$0")"
+polli_chat="${POLLI_CHAT:-/usr/local/bin/polli-chat}"
+
+need_text() {
+  if [ "$#" -lt 1 ]; then
+    echo "Нужен текст запроса. Пример: $cmd \"объясни ошибку\"" >&2
+    exit 1
+  fi
+}
+
+key() {
+  python3 - <<'PY'
+import json
+from pathlib import Path
+print(json.loads(Path.home().joinpath(".pollinations/credentials.json").read_text())["apiKey"])
+PY
+}
+
+urlencode() {
+  python3 - "$*" <<'PY'
+import sys, urllib.parse
+print(urllib.parse.quote(sys.argv[1]))
+PY
+}
+
+case "$cmd" in
+  ai)
+    need_text "$@"
+    exec "$polli_chat" --model text.daily "$*"
+    ;;
+  ai-fast)
+    need_text "$@"
+    exec "$polli_chat" --model text.cheap "$*"
+    ;;
+  ai-smart)
+    need_text "$@"
+    exec "$polli_chat" --model text.heavy "$*"
+    ;;
+  ai-code)
+    need_text "$@"
+    exec "$polli_chat" --model text.coding_primary "$*"
+    ;;
+  ai-search)
+    need_text "$@"
+    exec "$polli_chat" --model text.search_reasoning "$*"
+    ;;
+  ai-report)
+    if [ "$#" -lt 1 ]; then
+      echo "Пример: ai-report /tmp/check-output.txt" >&2
+      exit 1
+    fi
+    file="$1"
+    [ -f "$file" ] || { echo "Файл не найден: $file" >&2; exit 1; }
+    {
+      echo "Сделай короткий рабочий отчет: состояние, что важно, следующие действия."
+      echo
+      sed -n '1,1400p' "$file"
+    } | "$polli_chat" --model text.daily
+    ;;
+  ai-models)
+    k="$(key)"
+    curl -fsSL 'https://gen.pollinations.ai/v1/models' \
+      -H 'User-Agent: curl/8.5 detmir-proxmox' \
+      -H "Authorization: Bearer ${k}" |
+      python3 -c 'import json,sys; data=json.load(sys.stdin); [print(item.get("id") or item.get("name") or item.get("model")) for item in data.get("data", []) if isinstance(item, dict)]'
+    ;;
+  ai-test)
+    echo "1) text:"
+    "$polli_chat" --model text.cheap --max-tokens 20 'Return exactly: text-ok'
+    echo
+    echo "2) embedding:"
+    k="$(key)"
+    curl -fsSL 'https://gen.pollinations.ai/v1/embeddings' \
+      -H "Authorization: Bearer ${k}" \
+      -H 'User-Agent: curl/8.5 detmir-proxmox' \
+      -H 'Content-Type: application/json' \
+      -d '{"model":"openai-3-small","input":"embedding test","dimensions":128}' |
+      python3 -c 'import json,sys; data=json.load(sys.stdin); emb=data["data"][0]["embedding"]; print("embedding-ok len=" + str(len(emb)))'
+    ;;
+  ai-image)
+    need_text "$@"
+    prompt="$1"
+    out="${2:-$HOME/ai-image-$(date +%Y%m%d-%H%M%S).jpg}"
+    model="${3:-zimage}"
+    encoded="$(urlencode "$prompt")"
+    curl -fsSL "https://gen.pollinations.ai/image/${encoded}?model=${model}&width=1024&height=1024" \
+      -H 'User-Agent: curl/8.5 detmir-proxmox' \
+      -H "Authorization: Bearer $(key)" \
+      -o "$out"
+    echo "$out"
+    ;;
+  ai-voice)
+    need_text "$@"
+    text="$1"
+    out="${2:-$HOME/ai-voice-$(date +%Y%m%d-%H%M%S).mp3}"
+    voice="${3:-nova}"
+    encoded="$(urlencode "$text")"
+    curl -fsSL "https://gen.pollinations.ai/audio/${encoded}?voice=${voice}" \
+      -H 'User-Agent: curl/8.5 detmir-proxmox' \
+      -H "Authorization: Bearer $(key)" \
+      -o "$out"
+    echo "$out"
+    ;;
+  *)
+    echo "Unknown AI tool name: $cmd" >&2
+    exit 2
+    ;;
+esac
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+out_dir="${DETMIR_AI_OUT_DIR:-/var/tmp/detmir-ai-${USER:-$(id -un)}}"
+mkdir -p "$out_dir"
+stamp="$(date -u +%Y%m%d-%H%M%S)"
+check_file="$out_dir/detmir-check-$stamp.json"
+dlp_file="$out_dir/detmir-dlp-$stamp.json"
+bundle_file="$out_dir/detmir-ai-bundle-$stamp.txt"
+
+check_rc=0
+dlp_rc=0
+
+detmir-check --json >"$check_file" || check_rc=$?
+detmir-dlp >"$dlp_file" || dlp_rc=$?
+
+{
+  echo "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет."
+  echo "Структура ответа:"
+  echo "1. Состояние: OK/WARN/FAIL"
+  echo "2. Что важно"
+  echo "3. Что сделать дальше"
+  echo
+  echo "Правила:"
+  echo "- Не предлагай рестарты, если факты чистые."
+  echo "- Отличай event-driven bucket от dead/stale."
+  echo "- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail."
+  echo
+  echo "=== detmir-check exit=$check_rc ==="
+  sed -n '1,1600p' "$check_file"
+  echo
+  echo "=== detmir-dlp exit=$dlp_rc ==="
+  sed -n '1,1600p' "$dlp_file"
+} >"$bundle_file"
+
+echo "Files:"
+echo "  $check_file"
+echo "  $dlp_file"
+echo "  $bundle_file"
+echo
+
+polli-chat --model text.daily --max-tokens 700 <"$bundle_file"
+
+if [ "$check_rc" -ne 0 ] || [ "$dlp_rc" -ne 0 ]; then
+  exit 2
+fi
@@ -0,0 +1,8 @@
+# DetMir autonomous AI helpers. Installed on Proxmox host.
+# Run `ai-test`, `detmir-check`, `detmir-dlp`, or `detmir-ai`.
+
+export POLLI_CHAT=/usr/local/bin/polli-chat
+
+alias detmir-report='detmir-ai'
+alias detmir-auto-report='detmir-status'
+alias detmir-models='ai-models'
@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+state_dir="${DETMIR_AI_STATE_DIR:-/var/lib/detmir-ai}"
+lock_dir="${DETMIR_AI_RUN_DIR:-${XDG_RUNTIME_DIR:-/tmp}}"
+auto_heal="${DETMIR_AUTO_HEAL:-1}"
+retain_days="${DETMIR_AI_RETAIN_DAYS:-14}"
+
+mkdir -p "$state_dir"/runs "$state_dir"/reports "$state_dir"/logs
+
+lock_file="$lock_dir/detmir-auto.lock"
+exec 9>"$lock_file"
+if ! flock -n 9; then
+  echo "detmir-auto: another run is active"
+  exit 0
+fi
+
+stamp="$(date -u +%Y%m%d-%H%M%S)"
+run_dir_full="$state_dir/runs/$stamp"
+mkdir -p "$run_dir_full"
+
+check_file="$run_dir_full/detmir-check.json"
+dlp_file="$run_dir_full/detmir-dlp.json"
+heal_log="$run_dir_full/heal.log"
+bundle_file="$run_dir_full/bundle.txt"
+report_file="$state_dir/reports/detmir-report-$stamp.md"
+state_file="$state_dir/state-$stamp.json"
+
+run_check() {
+  local rc=0
+  detmir-check --json >"$check_file" || rc=$?
+  echo "$rc" >"$run_dir_full/check.rc"
+}
+
+run_dlp() {
+  local rc=0
+  detmir-dlp >"$dlp_file" || rc=$?
+  echo "$rc" >"$run_dir_full/dlp.rc"
+}
+
+summarize() {
+  python3 - "$check_file" "$dlp_file" "$run_dir_full/check.rc" "$run_dir_full/dlp.rc" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+check_path, dlp_path, check_rc_path, dlp_rc_path = map(Path, sys.argv[1:])
+check_rc = int(check_rc_path.read_text().strip())
+dlp_rc = int(dlp_rc_path.read_text().strip())
+summary = {
+    "check_rc": check_rc,
+    "dlp_rc": dlp_rc,
+    "check_ok": False,
+    "dlp_ok": False,
+    "severity": "FAIL" if check_rc or dlp_rc else "OK",
+    "needs_heal": bool(check_rc or dlp_rc),
+    "reasons": [],
+}
+try:
+    check = json.loads(check_path.read_text())
+    summary["check_ok"] = bool(check.get("ok"))
+    cs = check.get("summary") or {}
+    summary["detmir_summary"] = cs
+    if cs.get("bucket_dead", 0) or cs.get("bucket_stale", 0) or cs.get("service_failures", 0):
+        summary["reasons"].append("detmir-check has stale/dead bucket or required service failure")
+except Exception as exc:
+    summary["reasons"].append(f"detmir-check parse failed: {exc}")
+
+try:
+    dlp = json.loads(dlp_path.read_text())
+    summary["dlp_ok"] = bool(dlp.get("ok"))
+    summary["dlp_counts"] = dlp.get("counts")
+    counts = dlp.get("counts") or {}
+    if counts.get("fail", 0) or counts.get("warn", 0):
+        summary["reasons"].append("dlp-health-check has warn/fail")
+except Exception as exc:
+    summary["reasons"].append(f"detmir-dlp parse failed: {exc}")
+
+if summary["check_ok"] and summary["dlp_ok"]:
+    summary["severity"] = "OK"
+    summary["needs_heal"] = False
+elif not summary["reasons"]:
+    summary["severity"] = "WARN"
+else:
+    summary["severity"] = "FAIL"
+
+print(json.dumps(summary, ensure_ascii=False, indent=2))
+PY
+}
+
+run_check
+run_dlp
+summary_before="$(summarize)"
+printf '%s\n' "$summary_before" >"$run_dir_full/summary-before.json"
+
+if [ "$auto_heal" = "1" ] && python3 -c 'import json,sys; print("yes" if json.load(sys.stdin).get("needs_heal") else "no")' <<<"$summary_before" | grep -qx yes; then
+  {
+    echo "detmir-heal-safe started at $(date -u --iso-8601=seconds)"
+    detmir-heal-safe
+    echo "detmir-heal-safe finished at $(date -u --iso-8601=seconds)"
+  } >"$heal_log" 2>&1 || true
+  sleep 10
+  run_check
+  run_dlp
+else
+  echo "auto-heal skipped" >"$heal_log"
+fi
+
+summary_after="$(summarize)"
+printf '%s\n' "$summary_after" >"$state_file"
+
+{
+  echo "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет."
+  echo "Структура ответа:"
+  echo "1. Состояние: OK/WARN/FAIL"
+  echo "2. Что важно"
+  echo "3. Что уже сделал автомат"
+  echo "4. Что сделать человеку, если нужно"
+  echo
+  echo "Правила:"
+  echo "- Не предлагай рестарты, если факты чистые."
+  echo "- Отличай event-driven bucket от dead/stale."
+  echo "- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail."
+  echo "- Auto-heal умеет только серверные systemd-сервисы AW/DLP; Windows/RDP не трогает."
+  echo
+  echo "=== summary-before ==="
+  cat "$run_dir_full/summary-before.json"
+  echo
+  echo "=== summary-after ==="
+  cat "$state_file"
+  echo
+  echo "=== heal-log ==="
+  sed -n '1,300p' "$heal_log"
+  echo
+  echo "=== detmir-check ==="
+  sed -n '1,1600p' "$check_file"
+  echo
+  echo "=== detmir-dlp ==="
+  sed -n '1,1600p' "$dlp_file"
+} >"$bundle_file"
+
+{
+  echo "# DetMir Autonomous Report"
+  echo
+  echo "- generated_at_utc: $(date -u --iso-8601=seconds)"
+  echo "- run_dir: $run_dir_full"
+  echo
+  polli-chat --model text.daily --max-tokens 900 <"$bundle_file" || {
+    echo "Pollinations report failed; raw summary follows."
+    cat "$state_file"
+  }
+} >"$report_file"
+
+ln -sfn "$run_dir_full" "$state_dir/latest-run"
+ln -sfn "$report_file" "$state_dir/latest-report.md"
+ln -sfn "$state_file" "$state_dir/latest-state.json"
+
+find "$state_dir/runs" -mindepth 1 -maxdepth 1 -type d -mtime +"$retain_days" -exec rm -rf {} +
+find "$state_dir/reports" -type f -name 'detmir-report-*.md' -mtime +"$retain_days" -delete
+find "$state_dir" -maxdepth 1 -type f -name 'state-*.json' -mtime +"$retain_days" -delete
+
+cat "$report_file"
+
+python3 - "$state_file" <<'PY'
+import json
+import sys
+state = json.load(open(sys.argv[1]))
+raise SystemExit(0 if state.get("severity") == "OK" else 2)
+PY
@@ -0,0 +1,20 @@
+[Unit]
+Description=DetMir Rust autonomous shadow health loop
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=igor
+Group=igor
+ExecCondition=/bin/sh -c '! systemctl -q is-active detmir-auto.service'
+Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust
+Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust/locks
+Environment=no_proxy=localhost,127.0.0.1,192.168.100.18,10.10.10.13,10.10.10.2,10.10.10.0/24,192.168.100.0/24
+Environment=NO_PROXY=localhost,127.0.0.1,192.168.100.18,10.10.10.13,10.10.10.2,10.10.10.0/24,192.168.100.0/24
+ExecStart=/usr/local/bin/detmir-auto-rust --no-heal --no-report --command-timeout-seconds 180
+SuccessExitStatus=2
+TimeoutStartSec=240
+Nice=5
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
@@ -0,0 +1,13 @@
+[Unit]
+Description=Run DetMir Rust autonomous shadow loop every 15 minutes
+
+[Timer]
+OnBootSec=7min
+OnUnitActiveSec=30min
+AccuracySec=1min
+RandomizedDelaySec=5min
+Persistent=true
+Unit=detmir-auto-rust-shadow.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,17 @@
+[Unit]
+Description=DetMir autonomous health, AI report, and safe server-side recovery
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=igor
+Group=igor
+Environment=DETMIR_AUTO_HEAL=1
+Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai
+Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/locks
+ExecStart=/usr/local/bin/detmir-auto
+TimeoutStartSec=300
+Nice=5
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
@@ -0,0 +1,12 @@
+[Unit]
+Description=Run DetMir autonomous health loop every 15 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=15min
+AccuracySec=1min
+Persistent=true
+Unit=detmir-auto.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exec "${DETMIR_CHECK_BIN:-/usr/local/bin/detmir-check}" "$@"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exec "${DETMIR_DLP_BIN:-/usr/local/bin/detmir-dlp}" "$@"
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+aw_host="${DETMIR_AW_SSH_HOST:-igor@10.10.10.13}"
+
+ssh -o BatchMode=yes \
+  -o ConnectTimeout=10 \
+  -o StrictHostKeyChecking=accept-new \
+  "$aw_host" \
+  'bash -s' <<'REMOTE'
+set -euo pipefail
+
+services=(
+  activitywatch-server.service
+  aw-worktime-api.service
+  aw-worktime-ui-bridge.service
+  activitywatch-dlp-aggregator.service
+)
+
+sudo -n systemctl reset-failed "${services[@]}" >/dev/null 2>&1 || true
+
+for service in activitywatch-server.service aw-worktime-api.service aw-worktime-ui-bridge.service; do
+  if ! systemctl is-active --quiet "$service"; then
+    echo "restart $service"
+    sudo -n systemctl restart "$service"
+  else
+    echo "active $service"
+  fi
+done
+
+if systemctl list-unit-files activitywatch-dlp-aggregator.service >/dev/null 2>&1; then
+  if ! systemctl is-active --quiet activitywatch-dlp-aggregator.service; then
+    echo "start activitywatch-dlp-aggregator.service"
+    sudo -n systemctl start activitywatch-dlp-aggregator.service || true
+  else
+    echo "active activitywatch-dlp-aggregator.service"
+  fi
+fi
+
+sudo -n /usr/local/bin/dlp-health-check --json >/tmp/detmir-heal-dlp-health.json || true
+REMOTE
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exec "${DETMIR_STATUS_BIN:-/usr/local/bin/detmir-status}" "$@"
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from urllib import error, request
+
+
+DEFAULT_BASE_URL = "https://gen.pollinations.ai"
+DEFAULT_KEY_PATH = Path.home() / ".pollinations" / "credentials.json"
+
+ALIASES = {
+    "daily": "gpt-5.4-mini",
+    "cheap": "gemini-flash-lite-3.1",
+    "heavy": "gpt-5.5",
+    "coding_primary": "qwen-coder",
+    "coding_backup": "qwen-coder-large",
+    "search": "perplexity-fast",
+    "search_reasoning": "perplexity-reasoning",
+    "vision_primary": "qwen-vision",
+    "vision_heavy": "qwen-vision-pro",
+    "guardrail": "qwen-safety",
+    "text.daily": "gpt-5.4-mini",
+    "text.cheap": "gemini-flash-lite-3.1",
+    "text.heavy": "gpt-5.5",
+    "text.coding_primary": "qwen-coder",
+    "text.coding_backup": "qwen-coder-large",
+    "text.search": "perplexity-fast",
+    "text.search_reasoning": "perplexity-reasoning",
+    "text.vision_primary": "qwen-vision",
+    "text.vision_heavy": "qwen-vision-pro",
+    "text.guardrail": "qwen-safety",
+    "embeddings.primary": "openai-3-small",
+    "embeddings.quality": "openai-3-large",
+    "image.fast_preview": "zimage",
+    "image.background_change": "kontext",
+    "image.final_quality": "gptimage-large",
+    "video.primary": "ltx-2",
+    "video.backup": "wan",
+    "audio.tts_primary": "qwen-tts-instruct",
+    "audio.tts_fast": "qwen-tts",
+    "audio.tts_premium": "elevenlabs",
+}
+
+
+def load_api_key() -> str:
+    inline = os.getenv("POLLINATIONS_API_KEY", "").strip()
+    if inline:
+        return inline
+    key_path = Path(os.getenv("POLLINATIONS_KEY_PATH", str(DEFAULT_KEY_PATH))).expanduser()
+    data = json.loads(key_path.read_text(encoding="utf-8"))
+    key = str(data.get("apiKey", "")).strip()
+    if not key:
+        raise RuntimeError(f"apiKey not found in {key_path}")
+    return key
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Pollinations OpenAI-compatible chat helper.")
+    parser.add_argument("prompt", nargs="?", help="Prompt. If omitted, stdin is used.")
+    parser.add_argument("--model", default="text.daily", help="Model alias or exact model id.")
+    parser.add_argument("--system", default="", help="Optional system prompt.")
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--max-tokens", type=int, default=1200)
+    parser.add_argument("--json", action="store_true", help="Print raw JSON.")
+    parser.add_argument("--list-aliases", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if args.list_aliases:
+        print(json.dumps(ALIASES, ensure_ascii=False, indent=2, sort_keys=True))
+        return 0
+
+    prompt = args.prompt if args.prompt is not None else sys.stdin.read()
+    prompt = prompt.strip()
+    if not prompt:
+        raise RuntimeError("prompt is empty")
+
+    model = ALIASES.get(args.model.strip(), args.model.strip())
+    messages = []
+    if args.system.strip():
+        messages.append({"role": "system", "content": args.system.strip()})
+    messages.append({"role": "user", "content": prompt})
+
+    base_url = os.getenv("POLLINATIONS_BASE_URL", DEFAULT_BASE_URL).rstrip("/")
+    payload = {
+        "model": model,
+        "messages": messages,
+        "temperature": args.temperature,
+        "max_tokens": args.max_tokens,
+    }
+    req = request.Request(
+        f"{base_url}/v1/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={
+            "Authorization": f"Bearer {load_api_key()}",
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "User-Agent": "curl/8.5 detmir-proxmox",
+        },
+        method="POST",
+    )
+    try:
+        with request.urlopen(req, timeout=180) as resp:
+            body = resp.read().decode("utf-8", errors="replace")
+    except error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"pollinations http {exc.code}: {body[:2000]}") from exc
+
+    data = json.loads(body)
+    if args.json:
+        print(json.dumps(data, ensure_ascii=False, indent=2))
+        return 0
+
+    choices = data.get("choices") or []
+    if not choices:
+        raise RuntimeError(f"no choices in response: {body[:2000]}")
+    message = choices[0].get("message") or {}
+    content = message.get("content")
+    if isinstance(content, list):
+        text = "\n".join(str(part.get("text", "")) for part in content if isinstance(part, dict))
+    else:
+        text = str(content or "")
+    print(text.strip())
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"polli-chat error: {exc}", file=sys.stderr)
+        raise SystemExit(1)