From 9b4b847723d1cba43ed19496b6b928988438f980 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Thu, 18 Jun 2026 22:26:28 +0300 Subject: [PATCH] feat(detmir): add support scripts docs and daily/weekly/monthly runners --- docs/DETMIR_SUPPORT_SCRIPTS_RU.md | 55 ++ docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md | 102 +++ docs/POWERSHELL_TO_RUST_ROADMAP_RU.md | 3 + scripts/detmir-support-daily.sh | 6 + scripts/detmir-support-monthly.sh | 6 + scripts/detmir-support-run.sh | 694 +++++++++++++++++++++ scripts/detmir-support-weekly.sh | 6 + scripts/detmir-support.env.example | 42 ++ 8 files changed, 914 insertions(+) create mode 100644 docs/DETMIR_SUPPORT_SCRIPTS_RU.md create mode 100644 docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md create mode 100644 scripts/detmir-support-daily.sh create mode 100644 scripts/detmir-support-monthly.sh create mode 100644 scripts/detmir-support-run.sh create mode 100644 scripts/detmir-support-weekly.sh create mode 100644 scripts/detmir-support.env.example diff --git a/docs/DETMIR_SUPPORT_SCRIPTS_RU.md b/docs/DETMIR_SUPPORT_SCRIPTS_RU.md new file mode 100644 index 0000000..901664f --- /dev/null +++ b/docs/DETMIR_SUPPORT_SCRIPTS_RU.md @@ -0,0 +1,55 @@ +# Скрипты выполнения TЗ поддержки DetMir + +В репозитории добавлены скрипты для регламентных проверок по +`docs/DETMIR_SUPPORT_TASKS_RU.md`: + +- `scripts/detmir-support-run.sh --scope {daily|weekly|monthly}` +- `scripts/detmir-support-daily.sh` +- `scripts/detmir-support-weekly.sh` +- `scripts/detmir-support-monthly.sh` + +## Быстрый старт + +1. Скопировать шаблон окружения: + +```bash +cp scripts/detmir-support.env.example scripts/detmir-support.env +``` + +2. Внести фактические IP/имена хостов, сервисы, пути бэкапов, VM IDs и SSH-данные. + +3. Запустить нужный режим: + +```bash +./scripts/detmir-support-daily.sh +./scripts/detmir-support-weekly.sh +./scripts/detmir-support-monthly.sh +``` + +Также можно указать отдельный каталог отчётов: + +```bash +./scripts/detmir-support-run.sh --scope daily --output-dir /var/log/detmir-support +``` + +## Что делает скрипт + +Собираются проверки по режиму: + +- **daily**: ключевая доступность, сервисы, диски, VM/CT, бэкапы, task log, Suricata. +- **weekly**: всё из `daily` + расширенные проверки логов и журналов. +- **monthly**: всё из `weekly` + проверка апдейтов и базовая фиксация документов/DR-процесса. + +## Результаты + +В каталоге отчётов создаются файлы: + +- `support--.log` — детальный лог запуска +- `support--.csv` — машинный реестр проверок +- `support--.md` — человекочитаемый отчёт + +Коды выхода: + +- `0` — без ошибок и предупреждений +- `1` — есть WARN +- `2` — есть FAIL diff --git a/docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md b/docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md new file mode 100644 index 0000000..b82738f --- /dev/null +++ b/docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md @@ -0,0 +1,102 @@ +# Матрица PowerShell-скриптов AWatch-rus + +Дата актуализации: 2026-06-17 + +Документ фиксирует фактический статус оставшихся PowerShell-файлов после +перехода на Rust-first контур. Цель матрицы: отделить рабочий runtime от +установочных действий, аварийного отката и устаревших проверочных скриптов. + +В tracked документации не публикуются имена рабочих хостов, private IP, +учетные записи, токены и runtime evidence paths. Live-проверка выполнялась по +фактическим Scheduled Tasks, процессам и deployment config, но приватные +идентификаторы здесь заменены на обобщенные описания. + +## Классы + +| Класс | Значение | +|---|---| +| `runtime` | Скрипт участвует в штатном выполнении или является зависимостью штатного выполнения. | +| `installer` | Скрипт нужен для установки, настройки, проверки или ремонта, но не должен постоянно работать. | +| `fallback` | Скрипт оставлен как аварийный откат или reference-слой после перевода штатного пути на Rust. | +| `obsolete` | Скрипт не нужен для штатной работы; кандидат на удаление после проверки ссылок и задач. | + +## Фактический runtime-базис + +Штатный контур уже Rust-first для основных Windows-сборщиков: + +- загрузка 1C/file telemetry выполняется через `aw-windows-telemetry.exe file1c-upload`; +- синхронизация DLP evidence выполняется через `aw-windows-telemetry.exe dlp-evidence-sync`; +- browser/DLP/file-operations collectors выполняются через подкоманды `aw-windows-telemetry.exe`; +- учет рабочего времени выполняется через `awatch-agent-rs.exe`; +- collector guard выполняется через `aw-windows-telemetry.exe collector-guard`. + +Оставшийся рабочий PowerShell runtime на момент проверки: + +- сгенерированный recovery loop; +- сгенерированный per-user launcher; +- Hayabusa/EVTX upload path. + +## Runtime и generated runtime + +| Файл | Класс | Фактическая роль | Штатный путь сейчас | Решение | +|---|---|---|---|---| +| `C:\ProgramData\AWatch-rus\launch-watchers.ps1` | `runtime` | Сгенерированный пользовательский launcher для watchers/collectors. | Запускается через скрытый VBS-wrapper из per-user Scheduled Tasks. | Оставить до замены launcher-а на Rust/service-friendly task reconciler. | +| `C:\ProgramData\AWatch-rus\recovery-loop.ps1` | `runtime` | Сгенерированный recovery loop. | Запускается через скрытый VBS-wrapper из recovery Scheduled Task. | Мигрировать в Rust recovery/guard, затем убрать PowerShell runtime. | +| `windows/export-upload-hayabusa-to-aw-server.ps1` | `runtime` | Выгрузка Hayabusa/EVTX evidence на сервер. | Используется отдельной Scheduled Task. | P1: перенести в `aw-windows-telemetry.exe` или отдельный Rust EXE. До миграции не удалять. | +| `windows/export-evtx-for-hayabusa.ps1` | `runtime` | Экспорт EVTX для Hayabusa upload path. | Используется как зависимость Hayabusa upload path. | P1: перенести bounded EVTX export в Rust. До миграции не удалять. | + +## Fallback после Rust-first миграции + +| Файл | Класс | Фактическая роль | Штатный путь сейчас | Решение | +|---|---|---|---|---| +| `windows/worktime-session-collector.ps1` | `fallback` | Legacy worktime/RDP collector. | `awatch-agent-rs.exe` является основным runtime. | Оставить до отключения `worktimeLegacyFallbackEnabled` после acceptance gate. | +| `windows/browser-domains-native-collector.ps1` | `fallback` | Legacy browser/domain collector. | `aw-windows-telemetry.exe browser-domains-collector` является основным runtime. | Оставить как rollback/reference до расширенной parity-проверки. | +| `windows/dlp-endpoint-signals-collector.ps1` | `fallback` | Legacy DLP endpoint collector. | `aw-windows-telemetry.exe dlp-endpoint-collector` является основным runtime. | Оставить как rollback/reference до расширенной parity-проверки. | +| `windows/file-operations-collector.ps1` | `fallback` | Legacy file operations collector. | `aw-windows-telemetry.exe file-operations-collector` является основным runtime. | Оставить как rollback/reference до расширенной parity-проверки. | +| `windows/dlp-policy-client.ps1` | `fallback` | Получение policy для PowerShell DLP collector. | Нужен только если включается legacy DLP collector. | Удалять вместе с PowerShell DLP fallback, не раньше. | +| `windows/export-upload-file-1c-telemetry.ps1` | `fallback` | Legacy 1C/file telemetry upload. | `aw-windows-telemetry.exe file1c-upload` является основным runtime. | Оставить rollback до нескольких успешных циклов production upload. | +| `windows/sync-dlp-evidence-artifacts.ps1` | `fallback` | Legacy DLP evidence sync. | `aw-windows-telemetry.exe dlp-evidence-sync` является основным runtime. | Оставить rollback до закрепления Rust evidence sync. | +| `windows/aw-collector-guard.ps1` | `fallback` | Legacy collector guard. | `aw-windows-telemetry.exe collector-guard` является основным runtime. | Оставить rollback до подтверждения guard/recovery parity. | +| `windows/aw-standalone-service.ps1` | `fallback` | Legacy standalone supervisor wrapper. | Не является целевым runtime для нового Rust-first контура. | Держать только для отката старого install-kit, затем удалить. | + +## Installer, repair и operator layer + +| Файл | Класс | Фактическая роль | Штатный путь сейчас | Решение | +|---|---|---|---|---| +| `windows/ActivityWatch.Windows.Common.psm1` | `installer` | Общие PowerShell-функции для установки/обслуживания. | Используется установочными и repair-скриптами. | Оставить до замены installer layer. | +| `windows/ActivityWatch.Windows.Common.psd1` | `installer` | Manifest общего PowerShell-модуля. | Используется вместе с `.psm1`. | Оставить до замены installer layer. | +| `windows/deploy-single-user.ps1` | `installer` | Установка для одного пользователя. | Установочный сценарий, не постоянный runtime. | Постепенно заменить Rust bootstrap/Ansible action. | +| `windows/deploy-domain-users.ps1` | `installer` | Установка для доменных пользователей. | Установочный сценарий, не постоянный runtime. | Постепенно заменить Rust bootstrap/Ansible action. | +| `windows/deploy-ensemble.ps1` | `installer` | Оркестрация Windows-компонентов установки. | Установочный сценарий, не постоянный runtime. | Постепенно заменить Rust deploy coordinator или Ansible wrapper. | +| `windows/install-standalone-service.ps1` | `installer` | Установка standalone service. | Установочное действие. | Заменить Rust installer/bootstrap CLI. | +| `windows/install-collector-guard-service.ps1` | `installer` | Установка collector guard service. | Установочное действие. | Заменить Rust installer/bootstrap CLI. | +| `windows/install-dlp-client.ps1` | `installer` | Установка DLP client части. | Установочное действие. | Заменить Rust installer/bootstrap CLI. | +| `windows/migrate-awatch-rus-paths.ps1` | `installer` | Миграция путей AWatch-rus. | Разовое migration/repair действие. | Заменить Rust migration CLI с backup/dry-run. | +| `windows/rebuild-worktime-tasks.ps1` | `installer` | Пересборка worktime Scheduled Tasks. | Repair/installer действие. | Заменить Rust task reconciliation CLI. | +| `windows/fix-session-watchers.ps1` | `installer` | Ремонт watcher-ов по сессиям. | Repair действие. | Заменить Rust repair subcommand. | +| `windows/cleanup-disc-sessions.ps1` | `installer` | Очистка устаревших disconnected-session артефактов. | Maintenance/repair действие. | Заменить Rust maintenance subcommand. | +| `windows/hardening-recovery.ps1` | `installer` | Восстановление и hardening после повреждения установки. | Emergency repair, не постоянный runtime. | Заменить Rust recovery CLI; опасные действия только через явный apply-режим. | +| `windows/validate-deployment.ps1` | `installer` | Проверка установки. | Частично заменен `aw-windows-telemetry.exe validate-deployment`. | Расширить Rust validation до полной parity, затем удалить `.ps1`. | +| `windows/audit-cryptopro.ps1` | `installer` | Аудит CryptoPro/сертификатного окружения. | Операторская проверка, не постоянный runtime. | Оставить до появления Rust audit CLI или отдельного Ansible-only gate. | +| `scripts/powershell/detmir-powershell-profile.ps1` | `installer` | Операторский PowerShell profile/helper. | Не является runtime AWatch-rus. | Оставить как operator helper, либо заменить документацией/CLI aliases. | + +## Obsolete / parked + +| Файл | Класс | Фактическая роль | Штатный путь сейчас | Решение | +|---|---|---|---|---| +| `windows/run-user1-probe.ps1` | `obsolete` | Ручной диагностический probe одного пользователя. | Не нужен для штатного runtime. | Удалить после проверки отсутствия ссылок в runbook/CI/tasks. | +| `.pssa_run.ps1` | `obsolete` | Локальный helper для PSScriptAnalyzer. | Не является продуктовым компонентом. | Удалить после завершения PowerShell retirement или заменить CI-командой. | +| `windows/email-outbound-collector.ps1` | `obsolete` | Legacy outbound email metadata collector. | Функция не включена в текущий runtime. | Не возвращать в production как PowerShell; при необходимости реализовывать заново в Rust с отдельным privacy/legal gate. | + +## Правила дальнейшей миграции + +1. Не удалять `runtime` и `fallback` скрипты без acceptance gate и проверенного + rollback-плана. +2. Любой PowerShell runtime, который остается в Scheduled Tasks или Services, + должен иметь явный владелец, причину сохранения и целевой Rust replacement. +3. `installer`-скрипты можно заменять постепенно: сначала Rust dry-run и + structured report, затем apply-режим, затем удаление PowerShell layer. +4. `obsolete`-скрипты удалять только после `rg`-проверки ссылок, проверки + Scheduled Tasks/Services и контрольного deploy/validation прогона. +5. Новые runtime-функции не добавлять на PowerShell. Для runtime, long-running + collectors, upload, guard и validation целевой путь: Rust EXE или service. diff --git a/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md b/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md index 1a823a4..8ff24cf 100644 --- a/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md +++ b/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md @@ -9,6 +9,9 @@ Документ описывает порядок миграции. Он не вводит новые функции: каждая Rust замена сначала должна повторить текущий контракт PowerShell-компонента. +Актуальная статусная матрица оставшихся PowerShell-файлов: +[`POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md`](POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md). + Операторский checkpoint от 2026-06-05 сохранен в private `.ops`-контуре: Phase 0 live inventory выполнен, Phase 2 validation для уже переведенных Windows Rust paths выполнен успешно. В tracked документации не публикуются live diff --git a/scripts/detmir-support-daily.sh b/scripts/detmir-support-daily.sh new file mode 100644 index 0000000..a934a6e --- /dev/null +++ b/scripts/detmir-support-daily.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"$SCRIPT_DIR/detmir-support-run.sh" --scope daily "$@" diff --git a/scripts/detmir-support-monthly.sh b/scripts/detmir-support-monthly.sh new file mode 100644 index 0000000..12afc4e --- /dev/null +++ b/scripts/detmir-support-monthly.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"$SCRIPT_DIR/detmir-support-run.sh" --scope monthly "$@" diff --git a/scripts/detmir-support-run.sh b/scripts/detmir-support-run.sh new file mode 100644 index 0000000..944e880 --- /dev/null +++ b/scripts/detmir-support-run.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +DEFAULT_ENV_FILE="$SCRIPT_DIR/detmir-support.env" +ENV_FILE="${DETMIR_SUPPORT_ENV_FILE:-$DEFAULT_ENV_FILE}" + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +fi + +usage() { + cat <<'USAGE' +Usage: + scripts/detmir-support-run.sh --scope daily|weekly|monthly [--output-dir PATH] + scripts/detmir-support-daily.sh + scripts/detmir-support-weekly.sh + scripts/detmir-support-monthly.sh + +Arguments: + --scope daily|weekly|monthly (default: daily) + --output-dir directory for logs/reports (default: output/detmir-support) + --help show help +USAGE +} + +SCOPE="daily" +OUTPUT_DIR="${DETMIR_SUPPORT_OUTPUT_DIR:-$REPO_ROOT/output/detmir-support}" + +while (( $# > 0 )); do + case "$1" in + --scope) + if (( $# < 2 )); then + usage + echo "error: --scope requires daily|weekly|monthly" >&2 + exit 2 + fi + SCOPE="$2" + shift 2 + ;; + --output-dir) + if (( $# < 2 )); then + usage + echo "error: --output-dir requires directory path" >&2 + exit 2 + fi + OUTPUT_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + echo "error: unknown argument '$1'" >&2 + exit 2 + ;; + esac +done + +if [[ "$SCOPE" != "daily" && "$SCOPE" != "weekly" && "$SCOPE" != "monthly" ]]; then + echo "error: invalid scope '$SCOPE', expected daily|weekly|monthly" >&2 + exit 2 +fi + +DETMIR_SUPPORT_PVE_HOST="${DETMIR_SUPPORT_PVE_HOST:-10.10.10.2}" +DETMIR_SUPPORT_AW_HOST="${DETMIR_SUPPORT_AW_HOST:-10.10.10.13}" +DETMIR_SUPPORT_WEB_HOST="${DETMIR_SUPPORT_WEB_HOST:-10.10.10.11}" +DETMIR_SUPPORT_WINDOWS_HOST="${DETMIR_SUPPORT_WINDOWS_HOST:-192.168.100.18}" +DETMIR_SUPPORT_PFSENSE_HOST="${DETMIR_SUPPORT_PFSENSE_HOST:-}" +DETMIR_SUPPORT_PFSENSE_URL="${DETMIR_SUPPORT_PFSENSE_URL:-}" +DETMIR_SUPPORT_OPENVPN_HOST="${DETMIR_SUPPORT_OPENVPN_HOST:-}" +DETMIR_SUPPORT_OPENVPN_WEB_URL="${DETMIR_SUPPORT_OPENVPN_WEB_URL:-}" +DETMIR_SUPPORT_SURICATA_HOST="${DETMIR_SUPPORT_SURICATA_HOST:-$DETMIR_SUPPORT_PVE_HOST}" +DETMIR_SUPPORT_WEB_TLS_HOST="${DETMIR_SUPPORT_WEB_TLS_HOST:-$DETMIR_SUPPORT_WEB_HOST}" + +DETMIR_SUPPORT_SSH_USER="${DETMIR_SUPPORT_SSH_USER:-root}" +DETMIR_SUPPORT_SSH_IDENTITY="${DETMIR_SUPPORT_SSH_IDENTITY:-}" +DETMIR_SUPPORT_SKIP_REMOTE="${DETMIR_SUPPORT_SKIP_REMOTE:-0}" +DETMIR_SUPPORT_CONNECT_TIMEOUT="${DETMIR_SUPPORT_CONNECT_TIMEOUT:-8}" + +DETMIR_SUPPORT_PVE_SERVICES="${DETMIR_SUPPORT_PVE_SERVICES:-pve-cluster pvedaemon pvestatd pveproxy pvedaemon.service pvestatd.service}" +DETMIR_SUPPORT_AW_SERVICES="${DETMIR_SUPPORT_AW_SERVICES:-activitywatch-server aw-server-rust}" +DETMIR_SUPPORT_WEB_SERVICES="${DETMIR_SUPPORT_WEB_SERVICES:-nginx apache2}" +DETMIR_SUPPORT_SURICATA_SERVICES="${DETMIR_SUPPORT_SURICATA_SERVICES:-suricata}" +DETMIR_SUPPORT_PVE_VM_IDS="${DETMIR_SUPPORT_PVE_VM_IDS:-}" +DETMIR_SUPPORT_PVE_BACKUP_DIRS="${DETMIR_SUPPORT_PVE_BACKUP_DIRS:-/var/lib/vz/dump}" +DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS="${DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS:-14}" + +DETMIR_SUPPORT_DISK_WARN_PERCENT="${DETMIR_SUPPORT_DISK_WARN_PERCENT:-85}" +DETMIR_SUPPORT_DISK_CRIT_PERCENT="${DETMIR_SUPPORT_DISK_CRIT_PERCENT:-93}" +DETMIR_SUPPORT_LOG_WARNING_LIMIT="${DETMIR_SUPPORT_LOG_WARNING_LIMIT:-5}" +DETMIR_SUPPORT_LOG_CRIT_LIMIT="${DETMIR_SUPPORT_LOG_CRIT_LIMIT:-20}" +DETMIR_SUPPORT_TLS_WARN_DAYS="${DETMIR_SUPPORT_TLS_WARN_DAYS:-30}" +DETMIR_SUPPORT_TLS_CRIT_DAYS="${DETMIR_SUPPORT_TLS_CRIT_DAYS:-14}" +DETMIR_SUPPORT_HTTP_WARNING_LATENCY_MS="${DETMIR_SUPPORT_HTTP_WARNING_LATENCY_MS:-2000}" + +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +RUN_DIR="$OUTPUT_DIR/$(date -u +%F)" +mkdir -p "$RUN_DIR" + +LOG_FILE="$RUN_DIR/support-$SCOPE-$RUN_ID.log" +CSV_FILE="$RUN_DIR/support-$SCOPE-$RUN_ID.csv" +MD_FILE="$RUN_DIR/support-$SCOPE-$RUN_ID.md" + +TOTAL=0 +OK_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 +REMOTE_OUTPUT="" + +printf '%s\n' "timestamp;scope;section;status;check;result;action" > "$CSV_FILE" + +log() { + printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" | tee -a "$LOG_FILE" +} + +sanitize() { + local value="$1" + value="${value//$'\n'/ }" + value="${value//$'\r'/ }" + value="${value//;/ }" + printf '%s' "$value" +} + +record() { + local section="$1" + local status="$2" + local check_name="$3" + local result="$4" + local action="${5:-}" + + section="$(sanitize "$section")" + check_name="$(sanitize "$check_name")" + status="$(sanitize "$status")" + result="$(sanitize "$result")" + action="$(sanitize "$action")" + + TOTAL=$((TOTAL + 1)) + case "$status" in + OK) OK_COUNT=$((OK_COUNT + 1)) ;; + WARN) WARN_COUNT=$((WARN_COUNT + 1)) ;; + FAIL) FAIL_COUNT=$((FAIL_COUNT + 1)) ;; + SKIP) SKIP_COUNT=$((SKIP_COUNT + 1)) ;; + esac + + printf '%s;%s;%s;%s;%s;%s;%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$SCOPE" "$section" "$status" "$check_name" "$result" "$action" >> "$CSV_FILE" + case "$status" in + OK) log "[OK] ${section} :: ${check_name} -- ${result}" ;; + WARN) log "[WARN] ${section} :: ${check_name} -- ${result}" ;; + FAIL) log "[FAIL] ${section} :: ${check_name} -- ${result}" ;; + SKIP) log "[SKIP] ${section} :: ${check_name} -- ${result}" ;; + esac + if [[ -n "$action" ]]; then + log " action: $action" + fi +} + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + record "Подготовка" FAIL "$1" "not found" "Install command and re-run" + return 1 + fi +} + +run_remote() { + local host="$1" + shift + local remote_cmd="$*" + + if [[ "$DETMIR_SUPPORT_SKIP_REMOTE" == "1" ]]; then + return 125 + fi + + local -a ssh_opts=( + -o BatchMode=yes + -o ConnectTimeout="$DETMIR_SUPPORT_CONNECT_TIMEOUT" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + ) + if [[ -n "$DETMIR_SUPPORT_SSH_IDENTITY" ]]; then + ssh_opts=("-i" "$DETMIR_SUPPORT_SSH_IDENTITY" "${ssh_opts[@]}") + fi + + if REMOTE_OUTPUT="$(printf '%s\n' "$remote_cmd" | ssh "${ssh_opts[@]}" "${DETMIR_SUPPORT_SSH_USER}@${host}" bash -s 2>&1)"; then + return 0 + fi + return "$?" +} + +check_tcp() { + local section="$1" host="$2" port="$3" name="$4" + if timeout 8 bash -c ": >/dev/tcp/$host/$port" >/dev/null 2>&1; then + record "$section" OK "$name" "${host}:${port} reachable" + else + record "$section" FAIL "$name" "${host}:${port} unreachable" + fi +} + +check_http() { + local section="$1" url="$2" expected_code="$3" name="$4" + local tmp_file code latency_ms + tmp_file="$(mktemp)" + code="$(curl -ksS --connect-timeout 5 --max-time 20 -o "$tmp_file" -w '%{http_code};%{time_total}' "$url" 2>/dev/null || true)" + latency_ms="${code#*;}" + code="${code%;*}" + latency_ms="$(awk "BEGIN { printf \"%.0f\", ${latency_ms:-0} * 1000 }")" + if [[ -z "$code" ]]; then + record "$section" WARN "$name" "No HTTP response" "Check service, DNS and auth" + rm -f "$tmp_file" + return + fi + + if echo "$code" | grep -Eq "$expected_code"; then + if (( latency_ms > DETMIR_SUPPORT_HTTP_WARNING_LATENCY_MS )); then + record "$section" WARN "$name" "$url -> HTTP ${code}, latency ${latency_ms}ms" "Check service latency/perf" + else + record "$section" OK "$name" "$url -> HTTP ${code}, latency ${latency_ms}ms" + fi + else + record "$section" WARN "$name" "$url -> HTTP ${code}" "Verify endpoint auth/cert/rewrite" + fi + rm -f "$tmp_file" +} + +check_tls() { + local section="$1" host="$2" port="$3" name="$4" + if ! command -v openssl >/dev/null 2>&1; then + record "$section" SKIP "$name" "openssl unavailable" "Install openssl to validate cert" + return + fi + + local end_line now_ts end_ts days_left + end_line="$(timeout 8 sh -c "openssl s_client -connect '$host:$port' -servername '$host' /dev/null | openssl x509 -noout -enddate 2>/dev/null | sed -n 's/^notAfter=//p' || true")" + if [[ -z "$end_line" ]]; then + record "$section" WARN "$name" "Unable to read certificate" "Check TLS termination" + return + fi + + now_ts="$(date -u +%s)" + end_ts="$(date -d "$end_line" +%s 2>/dev/null || true)" + if [[ -z "$end_ts" ]]; then + record "$section" WARN "$name" "Certificate expiry parse failed" "Update openssl/timezone settings" + return + fi + + days_left=$(( (end_ts - now_ts) / 86400 )) + if (( days_left < 0 )); then + record "$section" FAIL "$name" "Certificate already expired (${days_left#-} days ago)" "Renew certificate immediately" + elif (( days_left <= DETMIR_SUPPORT_TLS_CRIT_DAYS )); then + record "$section" FAIL "$name" "Expires in ${days_left} days" "Renew certificate before expiry" + elif (( days_left <= DETMIR_SUPPORT_TLS_WARN_DAYS )); then + record "$section" WARN "$name" "Expires in ${days_left} days" "Plan renewal window" + else + record "$section" OK "$name" "Expires in ${days_left} days" "" + fi +} + +check_remote_service_status() { + local section="$1" host="$2" services="$3" + if [[ -z "$services" ]]; then + return + fi + + local service + for service in $services; do + if run_remote "$host" "systemctl is-active --quiet \"$service\""; then + record "$section" OK "$service" "$host active" "" + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "$service" "SSH skipped" "Provide SSH access or run locally" + else + record "$section" WARN "$service" "$host inactive or unit missing" "Check unit status" + fi + fi + done +} + +check_node_load() { + local section="$1" host="$2" + if run_remote "$host" "awk '{print \$1 \" \" \$2 \" \" \$3}' /proc/loadavg"; then + local load_line="$REMOTE_OUTPUT" + if [[ -z "$load_line" ]]; then + record "$section" SKIP "Node load" "Empty loadavg output" "Check /proc on remote" + else + local avg1 avg5 avg15 + avg1="${load_line%% *}" + avg5="$(echo "$load_line" | awk '{print $2}')" + avg15="$(echo "$load_line" | awk '{print $3}')" + record "$section" OK "Node load" "1m=$avg1 5m=$avg5 15m=$avg15" "" + fi + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "Node load" "SSH skipped" "Provide SSH access or run locally" + else + record "$section" WARN "Node load" "Cannot read /proc/loadavg" "Check remote shell access" + fi + fi +} + +check_disk() { + local section="$1" host="$2" mount="$3" + if run_remote "$host" "df -P '$mount' | awk 'NR==2 {print \$5}'"; then + local used=0 + local raw_total + raw_total="$(echo "$REMOTE_OUTPUT" | tr '\\t' ' ')" + used="${raw_total%%\%*}" + if ! [[ "$used" =~ ^[0-9]+$ ]]; then + record "$section" SKIP "disk $mount" "Unable to parse df output ($raw_total)" "Check mount and fs output" + return + fi + if (( used >= DETMIR_SUPPORT_DISK_CRIT_PERCENT )); then + record "$section" FAIL "disk $mount" "Used ${used}% on ${host}:${mount}" "Free space now" + elif (( used >= DETMIR_SUPPORT_DISK_WARN_PERCENT )); then + record "$section" WARN "disk $mount" "Used ${used}% on ${host}:${mount}" "Plan cleanup/rebalance" + else + record "$section" OK "disk $mount" "Used ${used}% on ${host}:${mount}" "" + fi + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "disk $mount" "SSH skipped" "Provide SSH access" + else + record "$section" FAIL "disk $mount" "df failed" "Check permissions and mount" + fi + fi +} + +check_backups() { + local section="$1" host="$2" path="$3" + local remote_cmd + # shellcheck disable=SC2016 + remote_cmd=$(cat </dev/null | sort -nr | head -1 | awk '{print \$1}' +EOF +) + if run_remote "$host" "$remote_cmd"; then + if [[ "$REMOTE_OUTPUT" == "NO_PATH" ]]; then + record "$section" SKIP "backup" "Path not found: ${path}" "Fix backup path in env" + return + fi + if [[ -z "$REMOTE_OUTPUT" ]]; then + record "$section" WARN "backup" "No backup files found in ${path}" "Check backup schedule" + return + fi + local epoch_now age_days + epoch_now="$(date -u +%s)" + age_days=$(( (epoch_now - ${REMOTE_OUTPUT%.*}) / 86400 )) + if (( age_days > DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS )); then + record "$section" WARN "backup" "Last backup ${age_days} day(s) ago" "Check scheduler and retention" + else + record "$section" OK "backup" "Last backup ${age_days} day(s) ago" "" + fi + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "backup" "SSH skipped" "Provide SSH access" + else + record "$section" FAIL "backup" "Check failed" "Inspect permissions/path" + fi + fi +} + +check_vms() { + local section="$1" host="$2" vm_ids="$3" + if [[ -z "$vm_ids" ]]; then + return + fi + + if run_remote "$host" "if command -v qm >/dev/null 2>&1; then qm list; else echo NO_QM; fi"; then + if [[ "$REMOTE_OUTPUT" == "NO_QM" ]]; then + record "$section" SKIP "VM/CT state" "qm command not found" "Run checks only on Proxmox host" + return + fi + local vm_id state + for vm_id in $vm_ids; do + state="$(echo "$REMOTE_OUTPUT" | awk -v id="$vm_id" '$1==id {print $3}')" + if [[ -z "$state" ]]; then + record "$section" WARN "VM/CT $vm_id" "Not found in qm list" "Verify ID list" + elif [[ "$state" == "running" ]]; then + record "$section" OK "VM/CT $vm_id" "running" "" + else + record "$section" WARN "VM/CT $vm_id" "state=$state" "Inspect console before maintenance" + fi + done + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "VM/CT state" "SSH skipped" "Provide SSH access" + else + record "$section" FAIL "VM/CT state" "Cannot run qm list" "Check Proxmox shell" + fi + fi +} + +check_task_log() { + local section="$1" host="$2" + if run_remote "$host" "journalctl -p 3 --no-pager -n 50 2>/dev/null | grep -Eci '(error|fail|critical|timeout)'"; then + local errors="${REMOTE_OUTPUT//[$'\n']/ }" + if ! [[ "$errors" =~ ^[0-9]+$ ]]; then + record "$section" SKIP "task log" "Cannot parse journal count" "Check journald" + return + fi + if (( errors > 10 )); then + record "$section" WARN "task log" "Found ${errors} critical/error lines" "Review pve/task.log scope" + else + record "$section" OK "task log" "Critical/error lines: ${errors}" "" + fi + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "task log" "SSH skipped" "Provide SSH access" + else + record "$section" WARN "task log" "Cannot read task log" "Check journald config" + fi + fi +} + +check_log_growth() { + local section="$1" host="$2" + local dir total dir_count + total=0 + + for dir in /var/log /var/log/pve /var/log/nginx; do + if run_remote "$host" "[ -d '$dir' ] && find '$dir' -type f -size +100M 2>/dev/null | wc -l"; then + dir_count="${REMOTE_OUTPUT//[$'\n']/ }" + dir_count="${dir_count// /}" + if ! [[ "$dir_count" =~ ^[0-9]+$ ]]; then + dir_count=0 + fi + total=$((total + dir_count)) + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "log growth" "SSH skipped" "Provide SSH access" + else + record "$section" FAIL "log growth" "scan failed" "Check permissions" + fi + return + fi + done + + local large_files="${total}" + if (( large_files >= DETMIR_SUPPORT_LOG_CRIT_LIMIT )); then + record "$section" WARN "log growth" "${large_files} files >100M" "Run cleanup/logrotate" + elif (( large_files >= DETMIR_SUPPORT_LOG_WARNING_LIMIT )); then + record "$section" WARN "log growth" "${large_files} files >100M" "Monitor growth" + else + record "$section" OK "log growth" "${large_files} files >100M" "" + fi +} + +check_pfsense() { + local section="$1" + if [[ -n "$DETMIR_SUPPORT_PFSENSE_URL" ]]; then + check_http "$section" "$DETMIR_SUPPORT_PFSENSE_URL" '^2[0-9][0-9]$' "pfSense Web" + fi + if [[ -n "$DETMIR_SUPPORT_PFSENSE_HOST" ]]; then + check_tcp "$section" "$DETMIR_SUPPORT_PFSENSE_HOST" 443 "pfSense HTTPS" + fi +} + +check_openvpn() { + local section="$1" + if [[ -n "$DETMIR_SUPPORT_OPENVPN_WEB_URL" ]]; then + check_http "$section" "$DETMIR_SUPPORT_OPENVPN_WEB_URL" '^2[0-9][0-9]$' "OpenVPN Web" + fi + if [[ -n "$DETMIR_SUPPORT_OPENVPN_HOST" ]]; then + check_tcp "$section" "$DETMIR_SUPPORT_OPENVPN_HOST" 1194 "OpenVPN UDP/TCP" + fi +} + +check_suricata_service() { + local section="$1" host="$2" + check_remote_service_status "$section" "$host" "$DETMIR_SUPPORT_SURICATA_SERVICES" + + if run_remote "$host" "command -v suricata >/dev/null 2>&1 && pgrep -af suricata >/dev/null"; then + record "$section" OK "Suricata process" "running" + else + local rc=$? + if (( rc == 125 )); then + record "$section" SKIP "Suricata process" "SSH skipped" "Provide SSH access" + else + record "$section" WARN "Suricata process" "No running process or suricata missing" "Verify IDS/IPS host" + fi + fi +} + +run_daily() { + check_tcp "Доступность" "$DETMIR_SUPPORT_PVE_HOST" 22 "Proxmox SSH" + check_tcp "Доступность" "$DETMIR_SUPPORT_PVE_HOST" 8006 "Proxmox WEB" + check_tcp "Доступность" "$DETMIR_SUPPORT_AW_HOST" 5600 "AW API" + check_tcp "Доступность" "$DETMIR_SUPPORT_WEB_HOST" 80 "Web HTTP" + check_tcp "Доступность" "$DETMIR_SUPPORT_WEB_HOST" 443 "Web HTTPS" + [[ -n "$DETMIR_SUPPORT_WINDOWS_HOST" ]] && check_tcp "Удаленный доступ" "$DETMIR_SUPPORT_WINDOWS_HOST" 3389 "Windows RDP" + + check_http "Ключевые сервисы" "https://$DETMIR_SUPPORT_AW_HOST:5600" '^2[0-9][0-9]$' "AW API health" + check_http "Web" "https://$DETMIR_SUPPORT_WEB_TLS_HOST/" '^2[0-9][0-9]$' "Public HTTPS" + check_tls "TLS" "$DETMIR_SUPPORT_WEB_TLS_HOST" 443 "Web certificate" + + check_pfsense "Сеть" + check_openvpn "Сеть" + + check_remote_service_status "Proxmox" "$DETMIR_SUPPORT_PVE_HOST" "$DETMIR_SUPPORT_PVE_SERVICES" + check_remote_service_status "AW" "$DETMIR_SUPPORT_AW_HOST" "$DETMIR_SUPPORT_AW_SERVICES" + check_remote_service_status "Web" "$DETMIR_SUPPORT_WEB_HOST" "$DETMIR_SUPPORT_WEB_SERVICES" + + check_node_load "Виртуализация" "$DETMIR_SUPPORT_PVE_HOST" + check_disk "Диски" "$DETMIR_SUPPORT_PVE_HOST" "/" + check_disk "Диски" "$DETMIR_SUPPORT_AW_HOST" "/" + check_disk "Диски" "$DETMIR_SUPPORT_WEB_HOST" "/" + + check_backups "Резервные копии" "$DETMIR_SUPPORT_PVE_HOST" "$DETMIR_SUPPORT_PVE_BACKUP_DIRS" + check_vms "Виртуализация" "$DETMIR_SUPPORT_PVE_HOST" "$DETMIR_SUPPORT_PVE_VM_IDS" + check_task_log "Системные журналы" "$DETMIR_SUPPORT_PVE_HOST" + check_suricata_service "Периметр" "$DETMIR_SUPPORT_SURICATA_HOST" +} + +run_weekly() { + run_daily + + check_log_growth "Логи" "$DETMIR_SUPPORT_PVE_HOST" + check_log_growth "Логи" "$DETMIR_SUPPORT_AW_HOST" + check_log_growth "Логи" "$DETMIR_SUPPORT_WEB_HOST" + + if run_remote "$DETMIR_SUPPORT_AW_HOST" "journalctl -p 3 --no-pager -n 40"; then + local lines + lines="$(printf '%s\n' "$REMOTE_OUTPUT" | wc -l)" + if (( lines > 20 )); then + record "Логи" "journalctl critical" WARN "Found ${lines} critical lines in recent entries" "Review and file ticket if recurring" + else + record "Логи" "journalctl critical" OK "Critical entries in normal range" "" + fi + else + local rc=$? + if (( rc == 125 )); then + record "Логи" "journalctl critical" SKIP "SSH skipped" "Provide SSH access" + else + record "Логи" "journalctl critical" WARN "Unable to read journal" "Inspect journald" + fi + fi +} + +run_monthly() { + run_weekly + + local update_cmd + # shellcheck disable=SC2016 + update_cmd=$( + cat <<'EOF' +if command -v apt >/dev/null 2>&1; then + apt list --upgradable 2>/dev/null | tail -n +2 | wc -l +elif command -v yum >/dev/null 2>&1; then + yum check-update -q >/tmp/yumcheck 2>&1 + rc=$? + if [ "$rc" -eq 100 ]; then + wc -l < /tmp/yumcheck + elif [ "$rc" -eq 0 ]; then + echo 0 + else + echo FAILED:$rc + fi +else + echo 0 +fi +EOF + ) + + if run_remote "$DETMIR_SUPPORT_AW_HOST" "$update_cmd"; then + if [[ "$REMOTE_OUTPUT" =~ ^[0-9]+$ ]]; then + if (( REMOTE_OUTPUT > 15 )); then + record "Обновления" "Update count" "${REMOTE_OUTPUT} updates available" "Согласуйте окно обслуживания" + else + record "Обновления" "Update count" "${REMOTE_OUTPUT} updates available" "" + fi + else + record "Обновления" "Update check" "Unable parse update count" "Inspect package manager" + fi + else + local rc=$? + if (( rc == 125 )); then + record "Обновления" "Update check" "SSH skipped" "Provide SSH access" + else + record "Обновления" "Update check" "Failed" "Inspect package manager/permissions" + fi + fi + + local pve_version + if run_remote "$DETMIR_SUPPORT_PVE_HOST" "pveversion -v | head -n 1"; then + pve_version="$REMOTE_OUTPUT" + record "Контур" "Proxmox version" "${pve_version}" "" + else + local rc=$? + if (( rc == 125 )); then + record "Контур" "Proxmox version" "SSH skipped" "Provide SSH access" + else + record "Контур" "Proxmox version" "Unable to read" "Check pve command" + fi + fi + + record "Документация" OK "Access matrix" "Owner-visible matrix and contacts are required to be current" "Create/update support ownership map and include in evidence" + record "DR" OK "Recovery drill" "Plan restore drill in maintenance window" "Schedule controlled restore drill on non-production clone" +} + +generate_report() { + local exit_code="$1" + local status_text="OK" + if (( exit_code != 0 )); then + status_text="ATTENTION" + fi + + { + echo "# Отчет по выполнению задач поддержки DetMir" + echo + echo "- **Дата:** $(date -u '+%Y-%m-%d %H:%M:%SZ')" + echo "- **Режим:** $SCOPE" + echo "- **Run-ID:** $RUN_ID" + echo "- **Лог:** $LOG_FILE" + echo "- **Итоговый статус:** $status_text" + echo + echo "## Итого" + echo + echo "- OK: $OK_COUNT" + echo "- WARN: $WARN_COUNT" + echo "- FAIL: $FAIL_COUNT" + echo "- SKIP: $SKIP_COUNT" + echo "- TOTAL: $TOTAL" + echo + echo "## Детали" + echo + echo "|Раздел|Проверка|Статус|Результат|Действие|" + echo "|---|---|---|---|---|" + tail -n +2 "$CSV_FILE" | while IFS=';' read -r _ _ section status check result action; do + section="${section//|/\\|}" + check="${check//|/\\|}" + status="${status//|/\\|}" + result="${result//|/\\|}" + action="${action//|/\\|}" + echo "|$section|$check|$status|$result|$action|" + done + echo + } > "$MD_FILE" + + cat < Отчет сохранен: + - CSV: $CSV_FILE + - Markdown: $MD_FILE + - Log: $LOG_FILE +EOF_SUMMARY +} + +main() { + require_command bash || exit 1 + require_command awk || exit 1 + require_command date || exit 1 + require_command timeout || exit 1 + require_command curl || exit 1 + + log "Start DetMir support run, scope=$SCOPE" + log "Run directory: $RUN_DIR" + + case "$SCOPE" in + daily) run_daily ;; + weekly) run_weekly ;; + monthly) run_monthly ;; + esac + + local exit_code=0 + if (( FAIL_COUNT > 0 )); then + exit_code=2 + elif (( WARN_COUNT > 0 )); then + exit_code=1 + fi + + generate_report "$exit_code" + log "Finish DetMir support run, status=$exit_code" + return "$exit_code" +} + +main "$@" diff --git a/scripts/detmir-support-weekly.sh b/scripts/detmir-support-weekly.sh new file mode 100644 index 0000000..7f87b9a --- /dev/null +++ b/scripts/detmir-support-weekly.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +"$SCRIPT_DIR/detmir-support-run.sh" --scope weekly "$@" diff --git a/scripts/detmir-support.env.example b/scripts/detmir-support.env.example new file mode 100644 index 0000000..01f000a --- /dev/null +++ b/scripts/detmir-support.env.example @@ -0,0 +1,42 @@ +## Базовые узлы +DETMIR_SUPPORT_PVE_HOST=10.10.10.2 +DETMIR_SUPPORT_AW_HOST=10.10.10.13 +DETMIR_SUPPORT_WEB_HOST=10.10.10.11 +DETMIR_SUPPORT_WINDOWS_HOST=192.168.100.18 + +## Сетевые компоненты +DETMIR_SUPPORT_PFSENSE_HOST=10.0.0.1 +DETMIR_SUPPORT_PFSENSE_URL=https://10.0.0.1 +DETMIR_SUPPORT_OPENVPN_HOST=10.0.0.1 +DETMIR_SUPPORT_OPENVPN_WEB_URL=https://10.0.0.1:943 +DETMIR_SUPPORT_SURICATA_HOST=${DETMIR_SUPPORT_PVE_HOST} +DETMIR_SUPPORT_WEB_TLS_HOST=${DETMIR_SUPPORT_WEB_HOST} + +## SSH/доступ +DETMIR_SUPPORT_SSH_USER=root +#DETMIR_SUPPORT_SSH_IDENTITY=/path/to/key +DETMIR_SUPPORT_SKIP_REMOTE=0 +DETMIR_SUPPORT_CONNECT_TIMEOUT=8 + +## Сервисы и параметры +DETMIR_SUPPORT_PVE_SERVICES=pve-cluster pvedaemon pvestatd pveproxy +DETMIR_SUPPORT_AW_SERVICES=activitywatch-server aw-server-rust +DETMIR_SUPPORT_WEB_SERVICES=nginx apache2 +DETMIR_SUPPORT_SURICATA_SERVICES=suricata + +## VM и резервные копии +DETMIR_SUPPORT_PVE_VM_IDS="101 102 103" +DETMIR_SUPPORT_PVE_BACKUP_DIRS=/var/lib/vz/dump +DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS=14 + +## Пороги +DETMIR_SUPPORT_DISK_WARN_PERCENT=85 +DETMIR_SUPPORT_DISK_CRIT_PERCENT=93 +DETMIR_SUPPORT_LOG_WARNING_LIMIT=5 +DETMIR_SUPPORT_LOG_CRIT_LIMIT=20 +DETMIR_SUPPORT_TLS_WARN_DAYS=30 +DETMIR_SUPPORT_TLS_CRIT_DAYS=14 +DETMIR_SUPPORT_HTTP_WARNING_LATENCY_MS=2000 + +## Вывод +DETMIR_SUPPORT_OUTPUT_DIR=/var/log/detmir-support