523 lines
24 KiB
Bash
523 lines
24 KiB
Bash
#!/usr/bin/env bash
|
||
# aw-contour-diag.sh - Диагностика всего контура ActivityWatch-Russian
|
||
# Запускать с машины администратора (где есть доступ по SSH/curl ко всем узлам).
|
||
#
|
||
# Использование:
|
||
# ./scripts/aw-contour-diag.sh # полная диагностика
|
||
# ./scripts/aw-contour-diag.sh --quick # быстрая (только AW server + buckets)
|
||
# ./scripts/aw-contour-diag.sh --skip-windows # без RDP/WinRM проверок
|
||
#
|
||
# При красных проверках в скрипте указаны разделы 'REMEDIATION: ...'
|
||
# с конкретными командами для восстановления.
|
||
|
||
set -uo pipefail
|
||
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
ANSIBLE_DIR="$REPO_ROOT/ansible"
|
||
INVENTORY="$ANSIBLE_DIR/inventory.ini"
|
||
AW_SERVER="http://10.10.10.13:5600"
|
||
AW_WORKTIME_API="http://10.10.10.13:5610"
|
||
INFLUXDB_URL="http://10.10.10.10:8086"
|
||
GRAFANA_URL="http://10.10.10.11:3000"
|
||
PROXMOX_HOST="10.10.10.2"
|
||
AW_HOST="10.10.10.13"
|
||
GRAFANA_HOST="10.10.10.11"
|
||
INFLUXDB_HOST="10.10.10.10"
|
||
WINDOWS_HOST="${AW_WINDOWS_HOST:-${AW_SMOKE_WINDOWS_HOST:-192.168.100.19}}"
|
||
CLICKHOUSE_HOST="10.10.10.2"
|
||
SOURCE_HOSTNAME="${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-${AW_SMOKE_SOURCE_HOSTNAME:-SHARKON2025}}}"
|
||
DLP_ENABLED="${AW_DLP_ENABLED:-${DETMIR_DLP_ENABLED:-true}}"
|
||
case "${DLP_ENABLED,,}" in
|
||
0|false|no|off) DLP_ENABLED=false ;;
|
||
*) DLP_ENABLED=true ;;
|
||
esac
|
||
|
||
QUICK_MODE=0
|
||
SKIP_WINDOWS=0
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--quick) QUICK_MODE=1; shift ;;
|
||
--skip-windows) SKIP_WINDOWS=1; shift ;;
|
||
-h|--help)
|
||
echo "Usage: $(basename "$0") [--quick] [--skip-windows]"
|
||
exit 0 ;;
|
||
*) echo "Unknown: $1"; exit 2 ;;
|
||
esac
|
||
done
|
||
|
||
export no_proxy="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$INFLUXDB_HOST,$WINDOWS_HOST,$CLICKHOUSE_HOST,10.10.10.0/24,192.168.100.0/24"
|
||
export NO_PROXY="$no_proxy"
|
||
|
||
OK_COUNT=0; WARN_COUNT=0; FAIL_COUNT=0; SKIP_COUNT=0
|
||
|
||
if [ -t 1 ]; then
|
||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
|
||
else
|
||
RED=''; GREEN=''; YELLOW=''; CYAN=''; NC=''
|
||
fi
|
||
|
||
pass() { OK_COUNT=$((OK_COUNT+1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; }
|
||
warn() { WARN_COUNT=$((WARN_COUNT+1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||
fail() { FAIL_COUNT=$((FAIL_COUNT+1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; }
|
||
skip() { SKIP_COUNT=$((SKIP_COUNT+1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; }
|
||
section() { printf "\n%b=== %s ===%b\n" "$CYAN" "$*" "$NC"; }
|
||
have() { command -v "$1" >/dev/null 2>&1; }
|
||
|
||
check_tcp() {
|
||
local name="$1" host="$2" port="$3"
|
||
if timeout 4 bash -c ":</dev/tcp/${host}/${port}" >/dev/null 2>&1; then
|
||
pass "TCP $host:$port ($name)"
|
||
else
|
||
fail "TCP $host:$port ($name)"
|
||
echo " REMEDIATION: Проверьте, запущен ли сервис на $host:$port."
|
||
echo " Для systemd: ssh igor@$host 'systemctl status <unit>'"
|
||
echo " Для Docker: ssh igor@$PROXMOX_HOST 'sudo docker ps | grep <container>'"
|
||
fi
|
||
}
|
||
|
||
check_http_code() {
|
||
local name="$1" url="$2" expected="${3:-^2[0-9][0-9]$}"
|
||
local tmp code
|
||
tmp="$(mktemp)"
|
||
code="$(curl -k -sS --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
|
||
if printf "%s" "$code" | grep -Eq "$expected"; then
|
||
pass "HTTP $code $url ($name)"
|
||
else
|
||
fail "HTTP $code $url ($name)"
|
||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -20
|
||
fi
|
||
rm -f "$tmp" "$tmp.err"
|
||
}
|
||
|
||
check_http_json_key() {
|
||
local name="$1" url="$2" jq_filter="$3" remediation="$4"
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
|
||
pass "$name"
|
||
else
|
||
fail "$name"
|
||
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -10
|
||
echo " REMEDIATION: $remediation"
|
||
fi
|
||
rm -f "$tmp" "$tmp.err"
|
||
}
|
||
|
||
check_bucket_freshness() {
|
||
local bucket="$1" label="$2" remediation="$3"
|
||
local bucket_id="${bucket}_${SOURCE_HOSTNAME}"
|
||
local tmp last_ts event_epoch now age_sec meta_ts
|
||
tmp="$(mktemp)"
|
||
if ! curl -fsS --connect-timeout 5 --max-time 15 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then
|
||
fail "bucket $label ($bucket_id) — запрос не удался"
|
||
sed 's/^/ /' "$tmp.err" | head -5
|
||
echo " REMEDIATION: $remediation"
|
||
rm -f "$tmp" "$tmp.err"
|
||
return
|
||
fi
|
||
last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)"
|
||
rm -f "$tmp"
|
||
if [ "$bucket" = "aw-watcher-afk" ]; then
|
||
meta_ts="$(curl -fsS --connect-timeout 5 --max-time 15 "$AW_SERVER/api/0/buckets/$bucket_id" 2>/dev/null | jq -r '.metadata.end // empty' 2>/dev/null || true)"
|
||
if [ -n "$meta_ts" ]; then
|
||
last_ts="$meta_ts"
|
||
fi
|
||
fi
|
||
if [ -z "$last_ts" ]; then
|
||
warn "bucket $label ($bucket_id) — нет событий"
|
||
echo " REMEDIATION: $remediation"
|
||
return
|
||
fi
|
||
event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || echo 0)"
|
||
now="$(date -u +%s)"
|
||
age_sec=$((now - event_epoch))
|
||
|
||
case "$bucket" in
|
||
aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
|
||
if [ "$age_sec" -lt 86400 ]; then
|
||
pass "bucket $label — ${age_sec}s назад"
|
||
else
|
||
warn "bucket $label — ${age_sec}s назад (event-driven)"
|
||
fi ;;
|
||
aw-watcher-window|aw-dlp-endpoint-signals)
|
||
if [ "$age_sec" -lt 7200 ]; then
|
||
pass "bucket $label — ${age_sec}s назад"
|
||
else
|
||
warn "bucket $label — ${age_sec}s назад (INACTIVE)"
|
||
fi ;;
|
||
*)
|
||
if [ "$age_sec" -lt 3600 ]; then
|
||
pass "bucket $label — ${age_sec}s назад"
|
||
elif [ "$age_sec" -lt 86400 ]; then
|
||
warn "bucket $label — ${age_sec}s назад (STALE)"
|
||
echo " REMEDIATION: $remediation"
|
||
else
|
||
fail "bucket $label — ${age_sec}s назад (DEAD)"
|
||
echo " REMEDIATION: $remediation"
|
||
fi ;;
|
||
esac
|
||
}
|
||
|
||
check_ansible_shell() {
|
||
local name="$1" group="$2" command="$3"
|
||
if ! have ansible; then
|
||
skip "$name (ansible not available)"
|
||
return
|
||
fi
|
||
if [ ! -f "$INVENTORY" ]; then
|
||
skip "$name (inventory not found: $INVENTORY)"
|
||
return
|
||
fi
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command" >"$tmp" 2>&1; then
|
||
pass "$name"
|
||
else
|
||
fail "$name"
|
||
sed 's/^/ /' "$tmp" | head -20
|
||
fi
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
check_ansible_win_shell() {
|
||
local name="$1" command="$2"
|
||
if ! have ansible; then skip "$name (ansible not available)"; return; fi
|
||
if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
if ANSIBLE_NOCOLOR=1 ansible aw_windows -i "$INVENTORY" -m win_shell -a "$command" >"$tmp" 2>&1; then
|
||
pass "$name"
|
||
else
|
||
fail "$name"
|
||
sed 's/^/ /' "$tmp" | head -20
|
||
fi
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
check_ansible_module() {
|
||
local name="$1" group="$2" module="$3" args="${4:-}"
|
||
if ! have ansible; then skip "$name (ansible not available)"; return; fi
|
||
if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then
|
||
pass "$name"
|
||
else
|
||
fail "$name"
|
||
sed 's/^/ /' "$tmp" | head -20
|
||
fi
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
ssh_with_diag_password() {
|
||
local host="$1"
|
||
shift
|
||
if [[ -z "${AW_DIAG_SSH_PASSWORD:-}" ]]; then
|
||
return 125
|
||
fi
|
||
sshpass -p "$AW_DIAG_SSH_PASSWORD" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "igor@$host" "$@"
|
||
}
|
||
|
||
ssh_aw() { ssh_with_diag_password 10.10.10.13 "$@"; }
|
||
ssh_pve() { ssh_with_diag_password 10.10.10.2 "$@"; }
|
||
|
||
check_service_remote() {
|
||
local name="$1" host="$2" unit="$3" remediation="$4"
|
||
local result
|
||
result=$(ssh_with_diag_password "$host" "systemctl is-active $unit 2>/dev/null || echo not_found" 2>/dev/null)
|
||
local rc=$?
|
||
if [ "$rc" -eq 125 ]; then
|
||
skip "$name ($unit on $host — set AW_DIAG_SSH_PASSWORD for SSH checks)"
|
||
return
|
||
fi
|
||
if [ "$rc" -ne 0 ] || [ "$result" = "not_found" ]; then
|
||
skip "$name ($unit on $host — не удалось проверить)"
|
||
return
|
||
fi
|
||
if [ "$result" = "active" ]; then
|
||
pass "$name ($unit active on $host)"
|
||
else
|
||
fail "$name ($unit $result on $host)"
|
||
echo " REMEDIATION: $remediation"
|
||
fi
|
||
}
|
||
|
||
printf "%b=== ActivityWatch-Russian: Диагностика контура ===%b\n" "$CYAN" "$NC"
|
||
echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
||
echo ""
|
||
|
||
# ============================================================
|
||
section "1. Локальные предусловия"
|
||
# ============================================================
|
||
for cmd in bash curl jq timeout ssh sshpass; do
|
||
if have "$cmd"; then pass "утилита $cmd найдена"; else fail "утилита $cmd не найдена (установите: apt install $cmd)"; fi
|
||
done
|
||
echo ""
|
||
|
||
# ============================================================
|
||
section "2. TCP доступность узлов"
|
||
# ============================================================
|
||
check_tcp "AW Server" "$AW_HOST" 5600
|
||
check_tcp "Worktime API" "$AW_HOST" 5610
|
||
check_tcp "RDP WinRM" "$WINDOWS_HOST" 5985
|
||
check_tcp "Proxmox SSH" "$PROXMOX_HOST" 22
|
||
check_tcp "Proxmox HTTPS" "$PROXMOX_HOST" 443
|
||
check_tcp "1C Company API" "$PROXMOX_HOST" 8710
|
||
check_tcp "ClickHouse HTTP" "$CLICKHOUSE_HOST" 8123
|
||
check_tcp "ClickHouse Native" "$CLICKHOUSE_HOST" 9000
|
||
check_tcp "InfluxDB" "$INFLUXDB_HOST" 8086
|
||
check_tcp "Grafana" "$GRAFANA_HOST" 3000
|
||
|
||
if [ "$QUICK_MODE" = "1" ]; then
|
||
# В быстром режиме проверяем только AW Server и buckets
|
||
echo ""
|
||
section "3. AW Server (быстрый режим)"
|
||
check_http_code "AW Server info" "$AW_SERVER/api/0/info" '^200$'
|
||
check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$'
|
||
check_http_code "Worktime API health" "$AW_WORKTIME_API/health" '^200$'
|
||
|
||
section "4. Buckets (быстрый режим)"
|
||
for entry in \
|
||
"aw-watcher-afk|AFK watcher|Запустите на RDP: schtasks /Run /TN \"ActivityWatch Recovery\" или launch task для logical host $SOURCE_HOSTNAME" \
|
||
"aw-watcher-window|Window watcher|Запустите через ansible: ansible aw_windows -i $INVENTORY -m win_shell -a 'Start-Process -FilePath \"C:\\Program Files\\AWatch-rus\\bin\\aw-watcher-window\\aw-watcher-window.exe\" -ArgumentList @(\"--host\", \"10.10.10.13\", \"--port\", \"5600\") -WindowStyle Hidden'" \
|
||
"aw-worktime-sessions|Worktime sessions|Проверьте работу worktime-api: systemctl status aw-worktime-api на AW сервере" \
|
||
"aw-session-events|Session events|Проверьте collector-guard и aw-session-events-collector на RDP" \
|
||
"aw-dlp-endpoint-signals|DLP signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"; Start-Sleep 30'" \
|
||
"aw-dlp-incidents|DLP incidents|Проверьте aw-detmir-dlp-collector.ps1 на RDP (логи: C:\\ProgramData\\AWatch-rus\\logs\\)" \
|
||
; do
|
||
bucket="${entry%%|*}"; rest="${entry#*|}"
|
||
label="${rest%%|*}"; remediation="${rest#*|}"
|
||
check_bucket_freshness "$bucket" "$label" "$remediation"
|
||
done
|
||
echo ""
|
||
echo "=== Быстрая диагностика завершена ==="
|
||
printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
|
||
[ "$FAIL_COUNT" -gt 0 ] && exit 2 || exit 0
|
||
fi
|
||
|
||
# ============================================================
|
||
section "3. AW Server (10.10.10.13)"
|
||
# ============================================================
|
||
check_http_json_key "AW Server info" "$AW_SERVER/api/0/info" \
|
||
'.version' \
|
||
"Проверьте: ssh igor@$AW_HOST 'systemctl status activitywatch-server'"
|
||
check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$'
|
||
check_http_code "AW WebUI" "$AW_SERVER/" '^200$'
|
||
check_http_json_key "Worktime API health" "$AW_WORKTIME_API/health" \
|
||
'.status // .ok' \
|
||
"Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'"
|
||
|
||
# ============================================================
|
||
section "4. Buckets (свежесть данных)"
|
||
# ============================================================
|
||
for entry in \
|
||
"aw-watcher-afk|AFK watcher|Запустите на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"'" \
|
||
"aw-watcher-window|Window watcher|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"; Start-Sleep 60'" \
|
||
"aw-worktime-sessions|Worktime sessions|Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'" \
|
||
"aw-session-events|Session events|Проверьте collector-guard на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'Get-Process -Name aw-session-events-* -ErrorAction SilentlyContinue'" \
|
||
"aw-dlp-endpoint-signals|DLP endpoint signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"; Start-Sleep 60'" \
|
||
"aw-dlp-incidents|DLP incidents|Проверьте: ansible aw_windows -i $INVENTORY -m win_shell -a \"Get-Content 'C:\\ProgramData\\AWatch-rus\\logs\\dlp-*.log' -Tail 20\"" \
|
||
"aw-dlp-review|DLP review|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-policy-engine.service -n 20 --no-pager'" \
|
||
"aw-dlp-rules|DLP rules|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-ioc-refresh.service -n 20 --no-pager'" \
|
||
; do
|
||
bucket="${entry%%|*}"; rest="${entry#*|}"
|
||
label="${rest%%|*}"; remediation="${rest#*|}"
|
||
check_bucket_freshness "$bucket" "$label" "$remediation"
|
||
done
|
||
|
||
# ============================================================
|
||
section "5. InfluxDB (10.10.10.10:8086)"
|
||
# ============================================================
|
||
check_http_json_key "InfluxDB health" "$INFLUXDB_URL/health" \
|
||
'.status == "pass"' \
|
||
"Проверьте InfluxDB на LXC 200: ssh igor@$PROXMOX_HOST 'sudo pct exec 200 -- systemctl status influxdb'"
|
||
|
||
# ============================================================
|
||
section "6. Grafana (10.10.10.11:3000)"
|
||
# ============================================================
|
||
check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" \
|
||
'.database == "ok"' \
|
||
"Проверьте: ssh igor@$PROXMOX_HOST 'sudo pct exec 201 -- systemctl status grafana-server'"
|
||
check_http_code "Grafana datasources API" "$GRAFANA_URL/api/datasources" '^200$|^302$|^401$'
|
||
|
||
# ============================================================
|
||
section "7. ClickHouse (10.10.10.2:8123)"
|
||
# ============================================================
|
||
# Проверяем через прямой HTTP — AUTHENTICATION_FAILED = сервер жив
|
||
local_ch_ok=0
|
||
ch_code=$(curl -sS --max-time 5 "http://$CLICKHOUSE_HOST:8123/?query=SELECT%201" 2>/dev/null | head -1)
|
||
if echo "$ch_code" | grep -q "AUTHENTICATION_FAILED"; then
|
||
pass "ClickHouse HTTP — отвечает (требуется аутентификация, это нормально)"
|
||
local_ch_ok=1
|
||
elif echo "$ch_code" | grep -q "1"; then
|
||
pass "ClickHouse HTTP — SELECT 1 OK"
|
||
local_ch_ok=1
|
||
else
|
||
fail "ClickHouse HTTP — не отвечает: $ch_code"
|
||
echo " REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose ps; sudo docker compose logs --tail=20'"
|
||
fi
|
||
|
||
# Проверка Docker контейнера через SSH
|
||
container_status=$(ssh_pve 'sudo docker ps --filter name=aw-rus-1c-clickhouse --format "{{.Status}}" 2>/dev/null' 2>/dev/null)
|
||
if [ -n "$container_status" ]; then
|
||
pass "ClickHouse Docker контейнер: $container_status"
|
||
else
|
||
fail "ClickHouse Docker контейнер не запущен"
|
||
echo " REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose up -d'"
|
||
fi
|
||
|
||
# ClickHouse health timer
|
||
check_service_remote "ClickHouse health timer" "$PROXMOX_HOST" "aw-1c-clickhouse-health.timer" \
|
||
"Проверьте: ssh igor@$PROXMOX_HOST 'sudo journalctl -u aw-1c-clickhouse-health.service -n 30 --no-pager'"
|
||
|
||
# ClickHouse network health timer (с AW сервера)
|
||
check_service_remote "ClickHouse network health timer" "$AW_HOST" "aw-clickhouse-network-health.timer" \
|
||
"Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-clickhouse-network-health.service -n 30 --no-pager'"
|
||
|
||
# 1C-ingest timer
|
||
check_service_remote "1C ingest timer" "$PROXMOX_HOST" "aw-1c-ingest.timer" \
|
||
"Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status aw-1c-ingest.timer; sudo journalctl -u aw-1c-ingest.service -n 20'"
|
||
|
||
# ============================================================
|
||
section "8. 1C Manager API (10.10.10.2:8710)"
|
||
# ============================================================
|
||
check_http_json_key "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" \
|
||
'.status == "ok"' \
|
||
"Проверьте Python процесс: ssh igor@$PROXMOX_HOST 'ps aux | grep 8710 | grep -v grep'"
|
||
check_http_code "1C /manager/brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$'
|
||
|
||
# ============================================================
|
||
section "9. Nginx Gateway (10.10.10.2)"
|
||
# ============================================================
|
||
check_http_code "Gateway /healthz" "https://$PROXMOX_HOST/healthz" '^200$'
|
||
check_http_code "Gateway /go/proxmox-gui (401=protected, OK)" "https://$PROXMOX_HOST/go/proxmox-gui" '^30[1278]$|^401$'
|
||
check_http_code "Gateway /go/file1c-brief (401=protected, OK)" "https://$PROXMOX_HOST/go/file1c-brief" '^30[1278]$|^401$'
|
||
|
||
check_service_remote "Nginx service" "$PROXMOX_HOST" "nginx.service" \
|
||
"Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status nginx; sudo nginx -t'"
|
||
|
||
# ============================================================
|
||
section "10. DLP Pipeline (10.10.10.13)"
|
||
# ============================================================
|
||
if [ "$DLP_ENABLED" = "false" ]; then
|
||
skip "DLP Pipeline disabled by AW_DLP_ENABLED/DETMIR_DLP_ENABLED=false"
|
||
else
|
||
# DLP Policy Engine — должен быть active (running)
|
||
check_service_remote "DLP Policy Engine" "$AW_HOST" "aw-dlp-policy-engine.service" \
|
||
"Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-policy-engine.service -n 30 --no-pager'"
|
||
|
||
# DLP Case Management
|
||
check_service_remote "DLP Case Management" "$AW_HOST" "aw-dlp-case-management.service" \
|
||
"Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-case-management.service -n 30 --no-pager'"
|
||
|
||
# DLP Aggregator
|
||
aggr_status=$(ssh_aw 'systemctl is-active activitywatch-dlp-aggregator.timer 2>/dev/null || echo not_found' 2>/dev/null)
|
||
if [ "$aggr_status" = "active" ]; then
|
||
pass "DLP Aggregator timer (active)"
|
||
else
|
||
fail "DLP Aggregator timer ($aggr_status)"
|
||
echo " REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now activitywatch-dlp-aggregator.timer; sudo journalctl -u activitywatch-dlp-aggregator.service -n 30'"
|
||
fi
|
||
|
||
# DLP Influx Exporter
|
||
influx_exp_status=$(ssh_aw 'systemctl is-active aw-dlp-influx-exporter.timer 2>/dev/null || echo not_found' 2>/dev/null)
|
||
if [ "$influx_exp_status" = "active" ]; then
|
||
pass "DLP Influx Exporter timer (active)"
|
||
else
|
||
fail "DLP Influx Exporter timer ($influx_exp_status)"
|
||
echo " REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-influx-exporter.timer; sudo journalctl -u aw-dlp-influx-exporter.service -n 30'"
|
||
fi
|
||
|
||
# DLP CEF Exporter
|
||
check_service_remote "DLP CEF Exporter timer" "$AW_HOST" "aw-dlp-cef-exporter.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-cef-exporter.timer; journalctl -u aw-dlp-cef-exporter.service -n 20'"
|
||
|
||
# DLP IOC Refresh
|
||
check_service_remote "DLP IOC Refresh timer" "$AW_HOST" "aw-dlp-ioc-refresh.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-ioc-refresh.timer'"
|
||
|
||
# DLP Syslog Forwarder
|
||
check_service_remote "DLP Syslog Forwarder timer" "$AW_HOST" "aw-dlp-syslog-forwarder.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-syslog-forwarder.timer'"
|
||
|
||
# DLP Webhook Sender
|
||
check_service_remote "DLP Webhook Sender timer" "$AW_HOST" "aw-dlp-webhook-sender.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-webhook-sender.timer'"
|
||
|
||
# DLP Report Scheduler
|
||
check_service_remote "DLP Report Scheduler timer" "$AW_HOST" "aw-dlp-report-scheduler.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-report-scheduler.timer'"
|
||
fi
|
||
|
||
# Worktime Influx Exporter
|
||
check_service_remote "Worktime Influx Exporter timer" "$AW_HOST" "aw-worktime-influx-exporter.timer" \
|
||
"ssh igor@$AW_HOST 'sudo systemctl enable --now aw-worktime-influx-exporter.timer; journalctl -u aw-worktime-influx-exporter.service -n 20'"
|
||
|
||
# ============================================================
|
||
if [ "$SKIP_WINDOWS" = "1" ]; then
|
||
skip "Проверки RDP хоста пропущены (--skip-windows)"
|
||
else
|
||
section "11. RDP хост ($WINDOWS_HOST)"
|
||
if have ansible && [ -f "$INVENTORY" ]; then
|
||
check_ansible_module "WinRM ping" aw_windows win_ping
|
||
|
||
check_ansible_win_shell "Сессии RDP" 'query user 2>&1'
|
||
|
||
check_ansible_win_shell "Процессы watcher" \
|
||
'Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize'
|
||
|
||
check_ansible_win_shell "Количество процессов powershell" \
|
||
'(Get-Process powershell -ErrorAction SilentlyContinue | Measure-Object).Count'
|
||
|
||
check_ansible_win_shell "Scheduled tasks (Recovery)" \
|
||
'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V | Select-String "Status|Run|Next"'
|
||
|
||
check_ansible_win_shell "Scheduled tasks (Launch Admin)" \
|
||
'Get-Content -Raw "C:\ProgramData\AWatch-rus\deployment-config.json" | ConvertFrom-Json | Select-Object -ExpandProperty userTasks | Select-Object LaunchTaskName,UserId | Format-Table -AutoSize'
|
||
else
|
||
skip "Ansible или inventory не найдены"
|
||
fi
|
||
fi
|
||
|
||
# ============================================================
|
||
section "12. Systemd health (AW server)"
|
||
# ============================================================
|
||
check_ansible_shell "AW server core units" aw_server \
|
||
'if [ "${AW_DLP_ENABLED:-true}" = "false" ] || [ "${DETMIR_DLP_ENABLED:-true}" = "false" ]; then systemctl is-active activitywatch-server aw-worktime-api aw-worktime-influx-exporter.timer aw-clickhouse-network-health.timer | paste -sd,; else systemctl is-active activitywatch-server aw-worktime-api aw-dlp-policy-engine aw-dlp-case-management aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer activitywatch-dlp-aggregator.timer aw-clickhouse-network-health.timer | paste -sd,; fi'
|
||
|
||
check_ansible_shell "AW server — нет failed units" aw_server \
|
||
'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|dlp" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
|
||
|
||
# ============================================================
|
||
section "13. Systemd health (Proxmox)"
|
||
# ============================================================
|
||
check_ansible_shell "Proxmox core units" proxmox \
|
||
'systemctl is-active nginx docker aw-1c-clickhouse-health.timer aw-1c-ingest.timer 2>/dev/null | paste -sd,'
|
||
|
||
# ============================================================
|
||
section "14. Диск и память"
|
||
# ============================================================
|
||
check_ansible_shell "Диски AW server" aw_server 'df -h / /var /opt 2>/dev/null | tail -5'
|
||
check_ansible_shell "Память AW server" aw_server 'free -h | tail -5'
|
||
|
||
# ============================================================
|
||
section "Итог диагностики"
|
||
# ============================================================
|
||
printf " OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
|
||
|
||
if [ "$FAIL_COUNT" -gt 0 ]; then
|
||
echo ""
|
||
echo " Есть проблемы! Смотрите REMEDIATION выше для каждого FAIL."
|
||
echo " После исправления запустите повторно: $0"
|
||
exit 2
|
||
elif [ "$WARN_COUNT" -gt 0 ]; then
|
||
echo ""
|
||
echo " Есть предупреждения (WARN) — стоит проверить, но не критично."
|
||
exit 1
|
||
else
|
||
echo ""
|
||
echo " Все проверки пройдены. Контур в рабочем состоянии."
|
||
exit 0
|
||
fi
|