chore(scripts): add full diagnostics package for detmir

scripts/detmir-full-diagnostics/DETMIR_FULL_DIAGNOSTICS_RU.md

@@ -0,0 +1,41 @@
+# Полная диагностика развернутого контура DetMir
+
+Пакет предназначен для расширенной проверки уже развернутого контура
+DetMir, не заменяет и не дублирует скрипты TЗ поддержки.
+
+Состав:
+- `detmir-full-diagnostics.sh` — оркестратор:
+  - `aw-contour-diag.sh` (по-умолчанию полный прогон),
+  - `detmir-support-run.sh` (daily/weekly/monthly),
+  - `check_production_inventory_placeholders.sh`.
+- `detmir-full-diagnostics-daily.sh` — быстрый старт с `--scope daily`.
+- `detmir-full-diagnostics-weekly.sh` — запуск с `--scope weekly`.
+- `detmir-full-diagnostics-monthly.sh` — запуск с `--scope monthly`.
+- `detmir-full-diagnostics.env.example` — шаблон параметров.
+
+## Быстрый старт
+
+1. Скопировать файл конфигурации:
+
+```bash
+cp scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example \
+   scripts/detmir-full-diagnostics/detmir-full-diagnostics.env
+```
+
+2. Заполнить пути/хосты под текущий контур.
+
+3. Запуск:
+
+```bash
+./scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh
+./scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope weekly --quick
+./scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh --output-dir /var/log/detmir-full-diagnostics
+```
+
+## Примечание
+
+- Для запуска в `~/root/scripts/support` пакет может использовать скопированные
+  `aw-contour-diag.sh` и `check_production_inventory_placeholders.sh`.
+  Если локально эти файлы не требуются — удалите их или отключите шаги флагами.
+- Для запуска в окружении с root-доступом через `sudo` выполняйте скрипты как
+  пользователь с правом `sudo` без `NOPASSWD`.

scripts/detmir-full-diagnostics/aw-contour-diag.sh

@@ -0,0 +1,507 @@
+#!/usr/bin/env bash
+# aw-contour-diag.sh - Диагностика всего контура ActivityWatch-Russian
+# Запускать с машины администратора (где есть доступ по SSH/curl ко всем узлам).
+#
+# Использование:
+#   ./scripts/aw-contour-diag.sh                  # полная диагностика
+#   ./scripts/aw-contour-diag.sh --quick           # быстрая (только AW server + buckets)
+#   ./scripts/aw-contour-diag.sh --skip-windows    # без RDP/WinRM проверок
+#
+# При красных проверках в скрипте указаны разделы 'REMEDIATION: ...'
+# с конкретными командами для восстановления.
+
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ANSIBLE_DIR="$REPO_ROOT/ansible"
+INVENTORY="$ANSIBLE_DIR/inventory.ini"
+AW_SERVER="http://10.10.10.13:5600"
+AW_WORKTIME_API="http://10.10.10.13:5610"
+INFLUXDB_URL="http://10.10.10.10:8086"
+GRAFANA_URL="http://10.10.10.11:3000"
+PROXMOX_HOST="10.10.10.2"
+AW_HOST="10.10.10.13"
+GRAFANA_HOST="10.10.10.11"
+INFLUXDB_HOST="10.10.10.10"
+WINDOWS_HOST="192.168.100.18"
+CLICKHOUSE_HOST="10.10.10.2"
+SOURCE_HOSTNAME="SHARKON2025"
+
+QUICK_MODE=0
+SKIP_WINDOWS=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --quick) QUICK_MODE=1; shift ;;
+    --skip-windows) SKIP_WINDOWS=1; shift ;;
+    -h|--help)
+      echo "Usage: $(basename "$0") [--quick] [--skip-windows]"
+      exit 0 ;;
+    *) echo "Unknown: $1"; exit 2 ;;
+  esac
+done
+
+export no_proxy="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$INFLUXDB_HOST,$WINDOWS_HOST,$CLICKHOUSE_HOST,10.10.10.0/24,192.168.100.0/24"
+export NO_PROXY="$no_proxy"
+
+OK_COUNT=0; WARN_COUNT=0; FAIL_COUNT=0; SKIP_COUNT=0
+
+if [ -t 1 ]; then
+  RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; CYAN=''; NC=''
+fi
+
+pass()  { OK_COUNT=$((OK_COUNT+1)); printf "%b[OK]%b   %s\n"      "$GREEN" "$NC" "$*"; }
+warn()  { WARN_COUNT=$((WARN_COUNT+1)); printf "%b[WARN]%b %s\n"  "$YELLOW" "$NC" "$*"; }
+fail()  { FAIL_COUNT=$((FAIL_COUNT+1)); printf "%b[FAIL]%b %s\n"  "$RED" "$NC" "$*"; }
+skip()  { SKIP_COUNT=$((SKIP_COUNT+1)); printf "%b[SKIP]%b %s\n"  "$YELLOW" "$NC" "$*"; }
+section() { printf "\n%b=== %s ===%b\n" "$CYAN" "$*" "$NC"; }
+have()  { command -v "$1" >/dev/null 2>&1; }
+
+check_tcp() {
+  local name="$1" host="$2" port="$3"
+  if timeout 4 bash -c ":</dev/tcp/${host}/${port}" >/dev/null 2>&1; then
+    pass "TCP $host:$port ($name)"
+  else
+    fail "TCP $host:$port ($name)"
+    echo "       REMEDIATION: Проверьте, запущен ли сервис на $host:$port."
+    echo "       Для systemd: ssh igor@$host 'systemctl status <unit>'"
+    echo "       Для Docker:  ssh igor@$PROXMOX_HOST 'sudo docker ps | grep <container>'"
+  fi
+}
+
+check_http_code() {
+  local name="$1" url="$2" expected="${3:-^2[0-9][0-9]$}"
+  local tmp code
+  tmp="$(mktemp)"
+  code="$(curl -k -sS --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
+  if printf "%s" "$code" | grep -Eq "$expected"; then
+    pass "HTTP $code $url ($name)"
+  else
+    fail "HTTP $code $url ($name)"
+    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -20
+  fi
+  rm -f "$tmp" "$tmp.err"
+}
+
+check_http_json_key() {
+  local name="$1" url="$2" jq_filter="$3" remediation="$4"
+  local tmp
+  tmp="$(mktemp)"
+  if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
+    pass "$name"
+  else
+    fail "$name"
+    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -10
+    echo "       REMEDIATION: $remediation"
+  fi
+  rm -f "$tmp" "$tmp.err"
+}
+
+check_bucket_freshness() {
+  local bucket="$1" label="$2" remediation="$3"
+  local bucket_id="${bucket}_${SOURCE_HOSTNAME}"
+  local tmp last_ts event_epoch now age_sec
+  tmp="$(mktemp)"
+  if ! curl -fsS --connect-timeout 5 --max-time 15 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then
+    fail "bucket $label ($bucket_id) — запрос не удался"
+    sed 's/^/       /' "$tmp.err" | head -5
+    echo "       REMEDIATION: $remediation"
+    rm -f "$tmp" "$tmp.err"
+    return
+  fi
+  last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)"
+  rm -f "$tmp"
+  if [ -z "$last_ts" ]; then
+    warn "bucket $label ($bucket_id) — нет событий"
+    echo "       REMEDIATION: $remediation"
+    return
+  fi
+  event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || echo 0)"
+  now="$(date -u +%s)"
+  age_sec=$((now - event_epoch))
+
+  case "$bucket" in
+    aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
+      if [ "$age_sec" -lt 86400 ]; then
+        pass "bucket $label — ${age_sec}s назад"
+      else
+        warn "bucket $label — ${age_sec}s назад (event-driven)"
+      fi ;;
+    aw-watcher-window|aw-dlp-endpoint-signals)
+      if [ "$age_sec" -lt 7200 ]; then
+        pass "bucket $label — ${age_sec}s назад"
+      else
+        warn "bucket $label — ${age_sec}s назад (INACTIVE)"
+      fi ;;
+    *)
+      if [ "$age_sec" -lt 3600 ]; then
+        pass "bucket $label — ${age_sec}s назад"
+      elif [ "$age_sec" -lt 86400 ]; then
+        warn "bucket $label — ${age_sec}s назад (STALE)"
+        echo "       REMEDIATION: $remediation"
+      else
+        fail "bucket $label — ${age_sec}s назад (DEAD)"
+        echo "       REMEDIATION: $remediation"
+      fi ;;
+  esac
+}
+
+check_ansible_shell() {
+  local name="$1" group="$2" command="$3"
+  if ! have ansible; then
+    skip "$name (ansible not available)"
+    return
+  fi
+  if [ ! -f "$INVENTORY" ]; then
+    skip "$name (inventory not found: $INVENTORY)"
+    return
+  fi
+  local tmp
+  tmp="$(mktemp)"
+  if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command" >"$tmp" 2>&1; then
+    pass "$name"
+  else
+    fail "$name"
+    sed 's/^/       /' "$tmp" | head -20
+  fi
+  rm -f "$tmp"
+}
+
+check_ansible_win_shell() {
+  local name="$1" command="$2"
+  if ! have ansible; then skip "$name (ansible not available)"; return; fi
+  if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi
+  local tmp
+  tmp="$(mktemp)"
+  if ANSIBLE_NOCOLOR=1 ansible aw_windows -i "$INVENTORY" -m win_shell -a "$command" >"$tmp" 2>&1; then
+    pass "$name"
+  else
+    fail "$name"
+    sed 's/^/       /' "$tmp" | head -20
+  fi
+  rm -f "$tmp"
+}
+
+check_ansible_module() {
+  local name="$1" group="$2" module="$3" args="${4:-}"
+  if ! have ansible; then skip "$name (ansible not available)"; return; fi
+  if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi
+  local tmp
+  tmp="$(mktemp)"
+  if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then
+    pass "$name"
+  else
+    fail "$name"
+    sed 's/^/       /' "$tmp" | head -20
+  fi
+  rm -f "$tmp"
+}
+
+ssh_with_diag_password() {
+  local host="$1"
+  shift
+  if [[ -z "${AW_DIAG_SSH_PASSWORD:-}" ]]; then
+    return 125
+  fi
+  sshpass -p "$AW_DIAG_SSH_PASSWORD" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "igor@$host" "$@"
+}
+
+ssh_aw()  { ssh_with_diag_password 10.10.10.13 "$@"; }
+ssh_pve() { ssh_with_diag_password 10.10.10.2 "$@"; }
+
+check_service_remote() {
+  local name="$1" host="$2" unit="$3" remediation="$4"
+  local result
+  result=$(ssh_with_diag_password "$host" "systemctl is-active $unit 2>/dev/null || echo not_found" 2>/dev/null)
+  local rc=$?
+  if [ "$rc" -eq 125 ]; then
+    skip "$name ($unit on $host — set AW_DIAG_SSH_PASSWORD for SSH checks)"
+    return
+  fi
+  if [ "$rc" -ne 0 ] || [ "$result" = "not_found" ]; then
+    skip "$name ($unit on $host — не удалось проверить)"
+    return
+  fi
+  if [ "$result" = "active" ]; then
+    pass "$name ($unit active on $host)"
+  else
+    fail "$name ($unit $result on $host)"
+    echo "       REMEDIATION: $remediation"
+  fi
+}
+
+printf "%b=== ActivityWatch-Russian: Диагностика контура ===%b\n" "$CYAN" "$NC"
+echo "       $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
+echo ""
+
+# ============================================================
+section "1. Локальные предусловия"
+# ============================================================
+for cmd in bash curl jq timeout ssh sshpass; do
+  if have "$cmd"; then pass "утилита $cmd найдена"; else fail "утилита $cmd не найдена (установите: apt install $cmd)"; fi
+done
+echo ""
+
+# ============================================================
+section "2. TCP доступность узлов"
+# ============================================================
+check_tcp "AW Server"              "$AW_HOST"        5600
+check_tcp "Worktime API"           "$AW_HOST"        5610
+check_tcp "RDP WinRM"              "$WINDOWS_HOST"   5985
+check_tcp "Proxmox SSH"            "$PROXMOX_HOST"   22
+check_tcp "Proxmox HTTPS"          "$PROXMOX_HOST"   443
+check_tcp "1C Company API"         "$PROXMOX_HOST"   8710
+check_tcp "ClickHouse HTTP"        "$CLICKHOUSE_HOST" 8123
+check_tcp "ClickHouse Native"      "$CLICKHOUSE_HOST" 9000
+check_tcp "InfluxDB"               "$INFLUXDB_HOST"  8086
+check_tcp "Grafana"                "$GRAFANA_HOST"   3000
+
+if [ "$QUICK_MODE" = "1" ]; then
+  # В быстром режиме проверяем только AW Server и buckets
+  echo ""
+  section "3. AW Server (быстрый режим)"
+  check_http_code "AW Server info" "$AW_SERVER/api/0/info" '^200$'
+  check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$'
+  check_http_code "Worktime API health" "$AW_WORKTIME_API/health" '^200$'
+
+  section "4. Buckets (быстрый режим)"
+  for entry in \
+    "aw-watcher-afk|AFK watcher|Запустите на RDP: schtasks /Run /TN \"ActivityWatch Recovery\" или schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"" \
+    "aw-watcher-window|Window watcher|Запустите через ansible: ansible aw_windows -i $INVENTORY -m win_shell -a 'Start-Process -FilePath \"C:\\Program Files\\AWatch-rus\\bin\\aw-watcher-window\\aw-watcher-window.exe\" -ArgumentList @(\"--host\", \"10.10.10.13\", \"--port\", \"5600\") -WindowStyle Hidden'" \
+    "aw-worktime-sessions|Worktime sessions|Проверьте работу worktime-api: systemctl status aw-worktime-api на AW сервере" \
+    "aw-session-events|Session events|Проверьте collector-guard и aw-session-events-collector на RDP" \
+    "aw-dlp-endpoint-signals|DLP signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Sleep 30'" \
+    "aw-dlp-incidents|DLP incidents|Проверьте aw-detmir-dlp-collector.ps1 на RDP (логи: C:\\ProgramData\\AWatch-rus\\logs\\)" \
+  ; do
+    bucket="${entry%%|*}"; rest="${entry#*|}"
+    label="${rest%%|*}"; remediation="${rest#*|}"
+    check_bucket_freshness "$bucket" "$label" "$remediation"
+  done
+  echo ""
+  echo "=== Быстрая диагностика завершена ==="
+  printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
+  [ "$FAIL_COUNT" -gt 0 ] && exit 2 || exit 0
+fi
+
+# ============================================================
+section "3. AW Server (10.10.10.13)"
+# ============================================================
+check_http_json_key "AW Server info" "$AW_SERVER/api/0/info" \
+  '.version' \
+  "Проверьте: ssh igor@$AW_HOST 'systemctl status activitywatch-server'"
+check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$'
+check_http_code "AW WebUI" "$AW_SERVER/" '^200$'
+check_http_json_key "Worktime API health" "$AW_WORKTIME_API/health" \
+  '.status // .ok' \
+  "Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'"
+
+# ============================================================
+section "4. Buckets (свежесть данных)"
+# ============================================================
+for entry in \
+  "aw-watcher-afk|AFK watcher|Запустите на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"'" \
+  "aw-watcher-window|Window watcher|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Process -FilePath \"C:\\Program Files\\AWatch-rus\\bin\\aw-watcher-window\\aw-watcher-window.exe\" -ArgumentList @(\"--host\", \"10.10.10.13\", \"--port\", \"5600\") -WindowStyle Hidden'" \
+  "aw-worktime-sessions|Worktime sessions|Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'" \
+  "aw-session-events|Session events|Проверьте collector-guard на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'Get-Process -Name aw-session-events-* -ErrorAction SilentlyContinue'" \
+  "aw-dlp-endpoint-signals|DLP endpoint signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Sleep 60'" \
+  "aw-dlp-incidents|DLP incidents|Проверьте: ansible aw_windows -i $INVENTORY -m win_shell -a \"Get-Content 'C:\\ProgramData\\AWatch-rus\\logs\\dlp-*.log' -Tail 20\"" \
+  "aw-dlp-review|DLP review|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-policy-engine.service -n 20 --no-pager'" \
+  "aw-dlp-rules|DLP rules|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-ioc-refresh.service -n 20 --no-pager'" \
+; do
+  bucket="${entry%%|*}"; rest="${entry#*|}"
+  label="${rest%%|*}"; remediation="${rest#*|}"
+  check_bucket_freshness "$bucket" "$label" "$remediation"
+done
+
+# ============================================================
+section "5. InfluxDB (10.10.10.10:8086)"
+# ============================================================
+check_http_json_key "InfluxDB health" "$INFLUXDB_URL/health" \
+  '.status == "pass"' \
+  "Проверьте InfluxDB на LXC 200: ssh igor@$PROXMOX_HOST 'sudo pct exec 200 -- systemctl status influxdb'"
+
+# ============================================================
+section "6. Grafana (10.10.10.11:3000)"
+# ============================================================
+check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" \
+  '.database == "ok"' \
+  "Проверьте: ssh igor@$PROXMOX_HOST 'sudo pct exec 201 -- systemctl status grafana-server'"
+check_http_code "Grafana datasources API" "$GRAFANA_URL/api/datasources" '^200$|^302$|^401$'
+
+# ============================================================
+section "7. ClickHouse (10.10.10.2:8123)"
+# ============================================================
+# Проверяем через прямой HTTP — AUTHENTICATION_FAILED = сервер жив
+local_ch_ok=0
+ch_code=$(curl -sS --max-time 5 "http://$CLICKHOUSE_HOST:8123/?query=SELECT%201" 2>/dev/null | head -1)
+if echo "$ch_code" | grep -q "AUTHENTICATION_FAILED"; then
+  pass "ClickHouse HTTP — отвечает (требуется аутентификация, это нормально)"
+  local_ch_ok=1
+elif echo "$ch_code" | grep -q "1"; then
+  pass "ClickHouse HTTP — SELECT 1 OK"
+  local_ch_ok=1
+else
+  fail "ClickHouse HTTP — не отвечает: $ch_code"
+  echo "       REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose ps; sudo docker compose logs --tail=20'"
+fi
+
+# Проверка Docker контейнера через SSH
+container_status=$(ssh_pve 'sudo docker ps --filter name=aw-rus-1c-clickhouse --format "{{.Status}}" 2>/dev/null' 2>/dev/null)
+if [ -n "$container_status" ]; then
+  pass "ClickHouse Docker контейнер: $container_status"
+else
+  fail "ClickHouse Docker контейнер не запущен"
+  echo "       REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose up -d'"
+fi
+
+# ClickHouse health timer
+check_service_remote "ClickHouse health timer" "$PROXMOX_HOST" "aw-1c-clickhouse-health.timer" \
+  "Проверьте: ssh igor@$PROXMOX_HOST 'sudo journalctl -u aw-1c-clickhouse-health.service -n 30 --no-pager'"
+
+# ClickHouse network health timer (с AW сервера)
+check_service_remote "ClickHouse network health timer" "$AW_HOST" "aw-clickhouse-network-health.timer" \
+  "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-clickhouse-network-health.service -n 30 --no-pager'"
+
+# 1C-ingest timer
+check_service_remote "1C ingest timer" "$PROXMOX_HOST" "aw-1c-ingest.timer" \
+  "Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status aw-1c-ingest.timer; sudo journalctl -u aw-1c-ingest.service -n 20'"
+
+# ============================================================
+section "8. 1C Manager API (10.10.10.2:8710)"
+# ============================================================
+check_http_json_key "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" \
+  '.status == "ok"' \
+  "Проверьте Python процесс: ssh igor@$PROXMOX_HOST 'ps aux | grep 8710 | grep -v grep'"
+check_http_code "1C /manager/brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$'
+
+# ============================================================
+section "9. Nginx Gateway (10.10.10.2)"
+# ============================================================
+check_http_code "Gateway /healthz" "https://$PROXMOX_HOST/healthz" '^200$'
+check_http_code "Gateway /go/proxmox-gui (401=protected, OK)" "https://$PROXMOX_HOST/go/proxmox-gui" '^30[1278]$|^401$'
+check_http_code "Gateway /go/file1c-brief (401=protected, OK)" "https://$PROXMOX_HOST/go/file1c-brief" '^30[1278]$|^401$'
+
+check_service_remote "Nginx service" "$PROXMOX_HOST" "nginx.service" \
+  "Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status nginx; sudo nginx -t'"
+
+# ============================================================
+section "10. DLP Pipeline (10.10.10.13)"
+# ============================================================
+# DLP Policy Engine — должен быть active (running)
+check_service_remote "DLP Policy Engine" "$AW_HOST" "aw-dlp-policy-engine.service" \
+  "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-policy-engine.service -n 30 --no-pager'"
+
+# DLP Case Management
+check_service_remote "DLP Case Management" "$AW_HOST" "aw-dlp-case-management.service" \
+  "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-case-management.service -n 30 --no-pager'"
+
+# DLP Aggregator
+aggr_status=$(ssh_aw 'systemctl is-active activitywatch-dlp-aggregator.timer 2>/dev/null || echo not_found' 2>/dev/null)
+if [ "$aggr_status" = "active" ]; then
+  pass "DLP Aggregator timer (active)"
+else
+  fail "DLP Aggregator timer ($aggr_status)"
+  echo "       REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now activitywatch-dlp-aggregator.timer; sudo journalctl -u activitywatch-dlp-aggregator.service -n 30'"
+fi
+
+# DLP Influx Exporter
+influx_exp_status=$(ssh_aw 'systemctl is-active aw-dlp-influx-exporter.timer 2>/dev/null || echo not_found' 2>/dev/null)
+if [ "$influx_exp_status" = "active" ]; then
+  pass "DLP Influx Exporter timer (active)"
+else
+  fail "DLP Influx Exporter timer ($influx_exp_status)"
+  echo "       REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-influx-exporter.timer; sudo journalctl -u aw-dlp-influx-exporter.service -n 30'"
+fi
+
+# DLP CEF Exporter
+check_service_remote "DLP CEF Exporter timer" "$AW_HOST" "aw-dlp-cef-exporter.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-cef-exporter.timer; journalctl -u aw-dlp-cef-exporter.service -n 20'"
+
+# DLP IOC Refresh
+check_service_remote "DLP IOC Refresh timer" "$AW_HOST" "aw-dlp-ioc-refresh.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-ioc-refresh.timer'"
+
+# DLP Syslog Forwarder
+check_service_remote "DLP Syslog Forwarder timer" "$AW_HOST" "aw-dlp-syslog-forwarder.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-syslog-forwarder.timer'"
+
+# DLP Webhook Sender
+check_service_remote "DLP Webhook Sender timer" "$AW_HOST" "aw-dlp-webhook-sender.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-webhook-sender.timer'"
+
+# DLP Report Scheduler
+check_service_remote "DLP Report Scheduler timer" "$AW_HOST" "aw-dlp-report-scheduler.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-report-scheduler.timer'"
+
+# Worktime Influx Exporter
+check_service_remote "Worktime Influx Exporter timer" "$AW_HOST" "aw-worktime-influx-exporter.timer" \
+  "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-worktime-influx-exporter.timer; journalctl -u aw-worktime-influx-exporter.service -n 20'"
+
+# ============================================================
+if [ "$SKIP_WINDOWS" = "1" ]; then
+  skip "Проверки RDP хоста пропущены (--skip-windows)"
+else
+  section "11. RDP хост (192.168.100.18)"
+  if have ansible && [ -f "$INVENTORY" ]; then
+    check_ansible_module "WinRM ping" aw_windows win_ping
+
+    check_ansible_win_shell "Сессии RDP" 'query user 2>&1'
+
+    check_ansible_win_shell "Процессы watcher" \
+      'Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize'
+
+    check_ansible_win_shell "Количество процессов powershell" \
+      '(Get-Process powershell -ErrorAction SilentlyContinue | Measure-Object).Count'
+
+    check_ansible_win_shell "Scheduled tasks (Recovery)" \
+      'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V | Select-String "Status|Run|Next"'
+
+    check_ansible_win_shell "Scheduled tasks (Launch Admin)" \
+      'schtasks /Query /TN "ActivityWatch Launch [SHARKON2025_Администратор]" /FO LIST /V | Select-String "Status|Run|Next"'
+  else
+    skip "Ansible или inventory не найдены"
+  fi
+fi
+
+# ============================================================
+section "12. Systemd health (AW server)"
+# ============================================================
+check_ansible_shell "AW server core units" aw_server \
+  'systemctl is-active activitywatch-server aw-worktime-api aw-dlp-policy-engine aw-dlp-case-management aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer activitywatch-dlp-aggregator.timer aw-clickhouse-network-health.timer | paste -sd,'
+
+check_ansible_shell "AW server — нет failed units" aw_server \
+  'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|dlp" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
+
+# ============================================================
+section "13. Systemd health (Proxmox)"
+# ============================================================
+check_ansible_shell "Proxmox core units" proxmox \
+  'systemctl is-active nginx docker aw-1c-clickhouse-health.timer aw-1c-ingest.timer 2>/dev/null | paste -sd,'
+
+# ============================================================
+section "14. Диск и память"
+# ============================================================
+check_ansible_shell "Диски AW server" aw_server 'df -h / /var /opt 2>/dev/null | tail -5'
+check_ansible_shell "Память AW server" aw_server 'free -h | tail -5'
+
+# ============================================================
+section "Итог диагностики"
+# ============================================================
+printf "  OK=%s  WARN=%s  FAIL=%s  SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
+
+if [ "$FAIL_COUNT" -gt 0 ]; then
+  echo ""
+  echo "  Есть проблемы! Смотрите REMEDIATION выше для каждого FAIL."
+  echo "  После исправления запустите повторно: $0"
+  exit 2
+elif [ "$WARN_COUNT" -gt 0 ]; then
+  echo ""
+  echo "  Есть предупреждения (WARN) — стоит проверить, но не критично."
+  exit 1
+else
+  echo ""
+  echo "  Все проверки пройдены. Контур в рабочем состоянии."
+  exit 0
+fi

scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  scripts/check_production_inventory_placeholders.sh [--allow-missing] [--strict] FILE...
+  DETMIR_PRODUCTION_CONFIG_PATHS="file1:file2" scripts/check_production_inventory_placeholders.sh
+  scripts/check_production_inventory_placeholders.sh --self-test
+
+Fails when production inventory/env files contain public placeholder values:
+TEST-NET addresses, HOST-EXAMPLE, WINDOWS_USER_EXAMPLE, CHANGE_ME, YOUR_*,
+replace-me, or angle-bracket placeholders.
+
+Use --strict for private production override files. Strict mode requires at
+least one existing path and rejects public/example/default files.
+EOF
+}
+
+pattern='(192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE|CHANGE_ME|CHANGEME|REPLACE_ME|replace-me|YOUR_[A-Z0-9_]*|<[A-Z0-9_ -]+>)'
+allow_missing=0
+strict=0
+self_test=0
+paths=()
+
+while (($#)); do
+  case "$1" in
+    --allow-missing)
+      allow_missing=1
+      ;;
+    --strict)
+      strict=1
+      ;;
+    --self-test)
+      self_test=1
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      paths+=("$1")
+      ;;
+  esac
+  shift
+done
+
+run_scan() {
+  local file
+  local found=0
+  for file in "$@"; do
+    if (( strict == 1 )) && [[ "$file" == ansible/group_vars* || "$file" == ansible/inventory.ini || "$file" == *".example."* || "$file" == *"example."* ]]; then
+      printf 'strict mode refuses public/default config path: %s\n' "$file" >&2
+      found=1
+      continue
+    fi
+    if [[ ! -e "$file" ]]; then
+      if (( allow_missing == 0 )); then
+        printf 'missing production config path: %s\n' "$file" >&2
+        found=1
+      fi
+      continue
+    fi
+    if [[ -d "$file" ]]; then
+      while IFS= read -r -d '' child; do
+        if grep -nE "$pattern" "$child" >/dev/null; then
+          printf 'placeholder found in %s\n' "$child" >&2
+          grep -nE "$pattern" "$child" | sed -E 's/^([0-9]+):.*/  line \1: placeholder marker/' >&2
+          found=1
+        fi
+      done < <(find "$file" -type f \( -name '*.env' -o -name '*.ini' -o -name '*.yml' -o -name '*.yaml' \) -print0)
+    elif grep -nE "$pattern" "$file" >/dev/null; then
+      printf 'placeholder found in %s\n' "$file" >&2
+      grep -nE "$pattern" "$file" | sed -E 's/^([0-9]+):.*/  line \1: placeholder marker/' >&2
+      found=1
+    fi
+  done
+  return "$found"
+}
+
+if (( self_test == 1 )); then
+  tmp_dir="$(mktemp -d)"
+  trap 'rm -rf "$tmp_dir"' EXIT
+  good="$tmp_dir/good.env"
+  bad="$tmp_dir/bad.env"
+  cat >"$good" <<'EOF'
+AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086
+AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST
+AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value
+EOF
+  cat >"$bad" <<'EOF'
+AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086
+AW_WORKTIME_INFLUX_HOSTS=HOST-EXAMPLE
+AW_WORKTIME_INFLUX_TOKEN=CHANGE_ME
+EOF
+  run_scan "$good"
+  if run_scan "$bad" >/dev/null 2>&1; then
+    echo "self-test failed: bad fixture was accepted" >&2
+    exit 1
+  fi
+  saved_strict="$strict"
+  strict=1
+  if run_scan "ansible/group_vars/all.yml" >/dev/null 2>&1; then
+    echo "self-test failed: strict mode accepted public/default file name" >&2
+    exit 1
+  fi
+  strict="$saved_strict"
+  echo "production inventory placeholder guard self-test: OK"
+  exit 0
+fi
+
+if ((${#paths[@]} == 0)) && [[ -n "${DETMIR_PRODUCTION_CONFIG_PATHS:-}" ]]; then
+  IFS=':' read -r -a paths <<<"$DETMIR_PRODUCTION_CONFIG_PATHS"
+fi
+
+if ((${#paths[@]} == 0)); then
+  if (( allow_missing == 1 && strict == 0 )); then
+    echo "production inventory placeholder guard: skipped (no production paths)"
+    exit 0
+  fi
+  usage >&2
+  exit 2
+fi
+
+run_scan "${paths[@]}"
+echo "production inventory placeholder guard: OK"

scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope daily "$@"

scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope monthly "$@"

scripts/detmir-full-diagnostics/detmir-full-diagnostics-weekly.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope weekly "$@"

scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example

@@ -0,0 +1,20 @@
+## Базовые параметры для расширенной диагностики контуров DetMir
+DETMIR_FULL_DIAGNOSTICS_SCOPE=daily
+DETMIR_FULL_DIAGNOSTICS_OUTPUT_DIR=/var/log/detmir-full-diagnostics
+
+# Опционально указывайте явные пути.
+# Если переменная не задана, используются пути относительно этой папки:
+# - aw-contour-diag.sh: ./aw-contour-diag.sh
+# - detmir-support-run.sh: ../detmir-support-run.sh
+# - detmir-support.env: ../detmir-support.env
+DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SCRIPT=
+DETMIR_FULL_DIAGNOSTICS_AW_DIAG_QUICK=0
+DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SKIP_WINDOWS=0
+DETMIR_FULL_DIAGNOSTICS_SUPPORT_RUN=
+DETMIR_FULL_DIAGNOSTICS_SUPPORT_ENV=
+
+# Проверка placeholder-значений в production-конфиге (по желанию)
+DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_GUARD=1
+DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_SCRIPT=
+# Если путь пустой — проверка placeholder-guard будет пропущена.
+DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS=

scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+ENV_FILE="${DETMIR_FULL_DIAGNOSTICS_ENV_FILE:-$SCRIPT_DIR/detmir-full-diagnostics.env}"
+if [[ -f "$ENV_FILE" ]]; then
+  set -a
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+  set +a
+fi
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh [options]
+
+Options:
+  --scope daily|weekly|monthly   Scope of operational checks (default: daily)
+  --output-dir PATH              Root output directory for all checks
+  --quick                        Pass --quick to aw-contour-diag
+  --skip-windows                 Skip WinRM checks in aw-contour-diag
+  --no-support                   Skip detmir-support-run checks
+  --no-aw-diagnostic             Skip aw-contour-diag
+  --no-placeholder-check          Skip production placeholder guard check
+  --help                         Show this help
+
+Examples:
+  scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope daily
+  scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope weekly --quick
+  scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh
+USAGE
+}
+
+SCOPE="${DETMIR_FULL_DIAGNOSTICS_SCOPE:-daily}"
+OUTPUT_DIR="${DETMIR_FULL_DIAGNOSTICS_OUTPUT_DIR:-$REPO_ROOT/output/detmir-full-diagnostics}"
+RUN_AW_DIAG=1
+RUN_SUPPORT=1
+RUN_PLACEHOLDER_GUARD=1
+AW_QUICK="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_QUICK:-0}"
+AW_SKIP_WINDOWS="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SKIP_WINDOWS:-0}"
+
+AW_DIAG_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SCRIPT:-$SCRIPT_DIR/aw-contour-diag.sh}"
+SUPPORT_RUN_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_SUPPORT_RUN:-$SCRIPT_DIR/../detmir-support-run.sh}"
+SUPPORT_ENV_FILE="${DETMIR_FULL_DIAGNOSTICS_SUPPORT_ENV:-$SCRIPT_DIR/../detmir-support.env}"
+PLACEHOLDER_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_SCRIPT:-$SCRIPT_DIR/../check_production_inventory_placeholders.sh}"
+PLACEHOLDER_GUARD="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_GUARD:-1}"
+PLACEHOLDER_PATHS="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS:-$REPO_ROOT/ansible:$REPO_ROOT/private-config}"
+
+while (( $# > 0 )); do
+  case "$1" in
+    --scope)
+      if (( $# < 2 )); then
+        usage
+        echo "error: --scope requires daily|weekly|monthly" >&2
+        exit 2
+      fi
+      SCOPE="$2"
+      shift 2
+      ;;
+    --output-dir)
+      if (( $# < 2 )); then
+        usage
+        echo "error: --output-dir requires path" >&2
+        exit 2
+      fi
+      OUTPUT_DIR="$2"
+      shift 2
+      ;;
+    --quick)
+      AW_QUICK=1
+      shift
+      ;;
+    --skip-windows)
+      AW_SKIP_WINDOWS=1
+      shift
+      ;;
+    --no-support)
+      RUN_SUPPORT=0
+      shift
+      ;;
+    --no-aw-diagnostic)
+      RUN_AW_DIAG=0
+      shift
+      ;;
+    --no-placeholder-check)
+      RUN_PLACEHOLDER_GUARD=0
+      shift
+      ;;
+    --help)
+      usage
+      exit 0
+      ;;
+    *)
+      usage
+      echo "error: unknown arg '$1'" >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [[ "$SCOPE" != "daily" && "$SCOPE" != "weekly" && "$SCOPE" != "monthly" ]]; then
+  echo "error: invalid scope '$SCOPE', expected daily|weekly|monthly" >&2
+  exit 2
+fi
+
+if [[ "$RUN_PLACEHOLDER_GUARD" == "1" ]]; then
+  RUN_PLACEHOLDER_GUARD="$PLACEHOLDER_GUARD"
+fi
+
+RUN_DIR="$OUTPUT_DIR/$(date -u +%Y%m%dT%H%M%SZ)"
+mkdir -p "$RUN_DIR"
+LOG_FILE="$RUN_DIR/full-diagnostics.log"
+
+exec > >(tee -a "$LOG_FILE") 2>&1
+
+TOTAL=0
+OK_COUNT=0
+FAIL_COUNT=0
+
+log() {
+  printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"
+}
+
+run_step() {
+  local step="$1"
+  shift
+
+  TOTAL=$((TOTAL + 1))
+  log "----"
+  log "STEP [$TOTAL]: $step"
+
+  if "$@"; then
+    OK_COUNT=$((OK_COUNT + 1))
+    log "RESULT: OK"
+    return 0
+  fi
+
+  FAIL_COUNT=$((FAIL_COUNT + 1))
+  log "RESULT: FAIL"
+  return 1
+}
+
+log "DetMir full diagnostics started"
+log "Repository: $REPO_ROOT"
+log "Scope: $SCOPE"
+log "Output: $RUN_DIR"
+log "Log: $LOG_FILE"
+
+if [[ ! -x "$AW_DIAG_SCRIPT" ]] && (( RUN_AW_DIAG == 1 )); then
+  log "WARN: aw-contour-diag script not found or not executable: $AW_DIAG_SCRIPT"
+  log "      Диагностика AW-контуром будет пропущена."
+  RUN_AW_DIAG=0
+fi
+
+if [[ ! -x "$SUPPORT_RUN_SCRIPT" ]] && (( RUN_SUPPORT == 1 )); then
+  log "ERROR: support script not found or not executable: $SUPPORT_RUN_SCRIPT"
+  exit 2
+fi
+if [[ ! -f "$SUPPORT_ENV_FILE" ]] && (( RUN_SUPPORT == 1 )); then
+  log "WARN: support env file not found, defaults will be used from detmir-support-run defaults: $SUPPORT_ENV_FILE"
+fi
+
+if [[ ! -f "$PLACEHOLDER_SCRIPT" ]] && (( RUN_PLACEHOLDER_GUARD == 1 )); then
+  log "WARN: placeholder guard script not found: $PLACEHOLDER_SCRIPT"
+  RUN_PLACEHOLDER_GUARD=0
+fi
+
+if (( RUN_AW_DIAG == 1 )); then
+  if [[ -x "$AW_DIAG_SCRIPT" ]]; then
+    AW_ARGS=()
+    if [[ "$AW_QUICK" == "1" ]]; then
+      AW_ARGS+=(--quick)
+    fi
+    if [[ "$AW_SKIP_WINDOWS" == "1" ]]; then
+      AW_ARGS+=(--skip-windows)
+    fi
+    run_step "aw-contour-diag (scope=$SCOPE)" "$AW_DIAG_SCRIPT" "${AW_ARGS[@]}" || true
+  fi
+fi
+
+if (( RUN_SUPPORT == 1 )); then
+  run_step "detmir-support-run --scope $SCOPE" \
+    env \
+      DETMIR_SUPPORT_ENV_FILE="$SUPPORT_ENV_FILE" \
+      DETMIR_SUPPORT_OUTPUT_DIR="$RUN_DIR/support" \
+    "$SUPPORT_RUN_SCRIPT" --scope "$SCOPE" --output-dir "$RUN_DIR/support" || true
+fi
+
+if [[ "$RUN_PLACEHOLDER_GUARD" == "1" ]]; then
+  if [[ -z "${PLACEHOLDER_PATHS// }" ]]; then
+    log "WARN: DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS is empty. Placeholder guard skipped."
+  else
+    run_step "check production placeholders" \
+      env DETMIR_PRODUCTION_CONFIG_PATHS="$PLACEHOLDER_PATHS" \
+      "$PLACEHOLDER_SCRIPT" --allow-missing || true
+  fi
+fi
+
+SUMMARY_FILE="$RUN_DIR/summary.txt"
+{
+  printf "Scope: %s\n" "$SCOPE"
+  printf "Total steps: %s\n" "$TOTAL"
+  printf "OK: %s\n" "$OK_COUNT"
+  printf "FAIL: %s\n" "$FAIL_COUNT"
+  if (( FAIL_COUNT == 0 )); then
+    printf "Result: OK\n"
+  else
+    printf "Result: FAIL\n"
+  fi
+} > "$SUMMARY_FILE"
+
+log "Summary: $SUMMARY_FILE"
+log "DetMir full diagnostics completed"
+
+if (( FAIL_COUNT > 0 )); then
+  exit 1
+fi
+
+exit 0