diff --git a/docs/CONTOUR_CHECK_MATRIX_RU.md b/docs/CONTOUR_CHECK_MATRIX_RU.md new file mode 100644 index 0000000..cbe8c86 --- /dev/null +++ b/docs/CONTOUR_CHECK_MATRIX_RU.md @@ -0,0 +1,127 @@ +# Матрица проверки контура AWatch-rus + +Дата фиксации: 2026-06-21. + +Документ описывает ежедневную и еженедельную проверку эксплуатационного контура +AWatch-rus. Он не является заявлением о сертификации, не подменяет юридическую +экспертизу и не описывает GitHub Actions как основной контур выпуска. + +## Цель + +Ежедневная проверка должна отвечать на один вопрос: можно ли оператору считать +контур AWatch-rus пригодным для работы сегодня без ручного обхода всех +компонентов. + +Еженедельная проверка расширяет ежедневную: добавляет portal smoke, registry +docs check и доказательные артефакты для последующего release/readiness пакета. + +## Канонический запуск + +Основной запускной скрипт: + +```bash +scripts/run_awatch_contour_check.sh +``` + +## Локальный запуск с ноутбука + +Для ручной проверки на ноутбуке удобно явно указать env-файл и корень артефактов: + +```bash +export CONTOUR_CHECK_ENV_FILE=~/path/to/your/contour-check.env +export CONTOUR_CHECK_OUTPUT_ROOT=~/tmp/contour-check-runs +export CONTOUR_CHECK_STREAM=1 +scripts/run_awatch_contour_check.sh +``` + +Во время выполнения можно наблюдать прогресс отдельным окном: + +```bash +ls -dt ~/tmp/contour-check-runs/* | head -n 1 | xargs -r -I{} tail -f {}/logs/detmir-check-json.log +``` + +`CONTOUR_CHECK_STREAM=1` полезен для локального запуска, если кажется, что скрипт «завис» на `== detmir-check-json ==`; фактически это длительный шаг в `detmir-check` с таймаутами по каждому endpoint. + +Рекомендуемые systemd units для российского/internal контура: + +```text +ops/systemd/awatch-contour-daily-check.service +ops/systemd/awatch-contour-daily-check.timer +ops/systemd/awatch-contour-weekly-check.service +ops/systemd/awatch-contour-weekly-check.timer +``` + +Live endpoints, hostnames, tokens and passwords must be supplied through +`/etc/detmir/detmir-check.env` (systemd units) or another private environment +file outside the public repository. + +## Текущий планировщик Proxmox, проверено 2026-06-21 + +На Proxmox уже присутствуют следующие регулярные проверки: + +| Timer | Частота | Роль | +|---|---:|---| +| `detmir-auto.timer` | каждые 30 минут | общий read-only check, AI report и safe recovery | +| `detmir-readiness-sync.timer` | каждые 10 минут | синхронизация readiness bundle | +| `detmir-portal-prewarm.timer` | каждые 30 минут | прогрев portal report cache | +| `aw-1c-clickhouse-health.timer` | каждые 5 минут | здоровье ClickHouse/1C layer | +| `aw-1c-ingest.timer` | каждые 15 минут | ingest cycle 1C/file layer | +| `aw-1c-proofcheck.timer` | каждые 6 часов | freshness proof check | +| `aw-1c-manager-brief.timer` | каждые 6 часов | manager brief | +| `aw-1c-recovery-brief.timer` | каждые 6 часов | recovery brief | +| `aw-1c-weekly-digest.timer` | понедельник 08:20 | weekly executive digest | +| `proxmox-lxc-critical-updates-check.timer` | ежедневно 03:00 | critical/important updates check | + +Наблюдение: отдельный ежедневный полный gate по всей матрице AWatch-rus +отсутствует. Его роль должен закрыть `awatch-contour-daily-check.timer`. + +Наблюдение: последняя проверка `detmir-portal-prewarm.service` на момент осмотра +имела `Result=exit-code` и `ExecMainStatus=28`. Это не надо маскировать: +канонический check должен показывать такой сбой как fail/warn в зависимости от +политики эксплуатации. + +## Матрица требований и проверок + +| Область | Что проверяется | Исполнитель | Ежедневно | Еженедельно | +|---|---|---|---:|---:| +| ActivityWatch API | `/api/0/info`, доступность API | `detmir-check` | да | да | +| Worktime API | `/reports/worktime/today` | `detmir-check` | да | да | +| 1C API | `/api/health` | `detmir-check` | да | да | +| Gateway | `/healthz` | `detmir-check` | да | да | +| Portal hardening | `/healthz`, `/readyz`, `/version`, `/metrics` | `detmir-check` | да | да | +| Windows/RDP | TCP 5985 и 22 | `detmir-check` | да | да | +| ActivityWatch buckets | AFK/window/worktime/session events | `detmir-check` | да | да | +| AWatch DLP buckets | endpoint signals/incidents/review/rules | `detmir-check` | да | да | +| AWatch DLP health | remote `dlp-health-check --json` через `detmir-dlp` | `detmir-check` | да | да | +| Grafana evidence | свежий JSON артефакт Grafana check | `detmir-check` | да | да | +| Security events backend | ClickHouse events, если включено | `detmir-check` | да | да | +| Portal contract | role/API smoke | `scripts/awatch-production-hardening-smoke.mjs` | нет | да | +| Pilot contract | demo/API smoke | `scripts/detmir-pilot-demo-smoke.mjs` | нет | да | +| Registry/readiness docs | registry readiness check | `scripts/registry_readiness_check.sh` | опционально | да | + +## Fail-closed политика + +Ежедневный check должен завершаться non-zero, если падает обязательная область: + +- ActivityWatch API; +- Worktime API; +- Gateway/Portal health; +- RDP/Windows reachability; +- свежесть обязательных bucket streams; +- AWatch DLP health; +- Grafana evidence freshness. + +Event-driven buckets не должны считаться stale только из-за отсутствия новых +инцидентов. Для них фиксируется статус `EVENT-DRIVEN`. + +## Что ещё требуется довести + +- Развернуть `awatch-contour-daily-check.timer` на Proxmox после установки + обновленного `detmir-check`. +- Развернуть `awatch-contour-weekly-check.timer` для расширенного smoke/docs + контроля. +- Заполнить `/etc/awatch-rus/contour-check.env` live значениями без записи + секретов в репозиторий. +- Настроить retention для `.ops/contour-check-runs` или серверного output path. +- После первого успешного недельного запуска приложить summary к + registry/readiness evidence пакету. diff --git a/ops/systemd/awatch-contour-daily-check.service b/ops/systemd/awatch-contour-daily-check.service new file mode 100644 index 0000000..a7fbfb2 --- /dev/null +++ b/ops/systemd/awatch-contour-daily-check.service @@ -0,0 +1,21 @@ +[Unit] +Description=AWatch-rus daily contour check +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/var/lib/detmir-ai +Environment=CONTOUR_CHECK_SCOPE=daily +Environment=RUN_PORTAL_SMOKE=0 +Environment=DETMIR_DLP_COMMAND=detmir-dlp +Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs +Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env +Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 +EnvironmentFile=-/etc/detmir/detmir-check.env +EnvironmentFile=-/etc/awatch-rus/contour-check.env +ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check +TimeoutStartSec=10min +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/ops/systemd/awatch-contour-daily-check.timer b/ops/systemd/awatch-contour-daily-check.timer new file mode 100644 index 0000000..13b9dc1 --- /dev/null +++ b/ops/systemd/awatch-contour-daily-check.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AWatch-rus daily contour check + +[Timer] +OnCalendar=*-*-* 07:10:00 +RandomizedDelaySec=10m +Persistent=true +Unit=awatch-contour-daily-check.service + +[Install] +WantedBy=timers.target diff --git a/ops/systemd/awatch-contour-weekly-check.service b/ops/systemd/awatch-contour-weekly-check.service new file mode 100644 index 0000000..06b75d4 --- /dev/null +++ b/ops/systemd/awatch-contour-weekly-check.service @@ -0,0 +1,22 @@ +[Unit] +Description=AWatch-rus weekly extended contour check +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/var/lib/detmir-ai +Environment=CONTOUR_CHECK_SCOPE=weekly +Environment=RUN_PORTAL_SMOKE=1 +Environment=RUN_REGISTRY_CHECK=1 +Environment=DETMIR_DLP_COMMAND=detmir-dlp +Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs +Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env +Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 +EnvironmentFile=-/etc/detmir/detmir-check.env +EnvironmentFile=-/etc/awatch-rus/contour-check.env +ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check +TimeoutStartSec=20min +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/ops/systemd/awatch-contour-weekly-check.timer b/ops/systemd/awatch-contour-weekly-check.timer new file mode 100644 index 0000000..afea1b6 --- /dev/null +++ b/ops/systemd/awatch-contour-weekly-check.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AWatch-rus weekly extended contour check + +[Timer] +OnCalendar=Mon *-*-* 09:10:00 +RandomizedDelaySec=20m +Persistent=true +Unit=awatch-contour-weekly-check.service + +[Install] +WantedBy=timers.target diff --git a/scripts/run_awatch_contour_check.sh b/scripts/run_awatch_contour_check.sh new file mode 100644 index 0000000..6560e89 --- /dev/null +++ b/scripts/run_awatch_contour_check.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Canonical daily/weekly AWatch-rus contour check launcher. +# Read-only: does not push, tag, heal, restart services, or change Git history. +# Live endpoints must be supplied through environment files outside the public +# repository. + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${AWATCH_REPO_ROOT:-$(cd -- "${SCRIPT_DIR}/.." && pwd)}" +SCOPE="${CONTOUR_CHECK_SCOPE:-daily}" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +OUTPUT_ROOT="${CONTOUR_CHECK_OUTPUT_ROOT:-${REPO_ROOT}/.ops/contour-check-runs}" +OUTPUT_DIR="${OUTPUT_DIR:-${OUTPUT_ROOT}/${SCOPE}-${RUN_ID}}" +ENV_FILE="${CONTOUR_CHECK_ENV_FILE:-${REPO_ROOT}/private-config/awatch-contour-check.env}" +LIB_DIR="${CONTOUR_CHECK_LIB_DIR:-/usr/local/lib/awatch-contour}" +SMOKE_TIMEOUT_SECONDS="${CONTOUR_CHECK_SMOKE_TIMEOUT_SECONDS:-120}" + +mkdir -p "${OUTPUT_DIR}/logs" + +if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${ENV_FILE}" +fi + +configure_detmir_env() { + if [[ -z "${DETMIR_DLP_COMMAND:-}" ]]; then + export DETMIR_DLP_COMMAND="detmir-dlp" + fi + + if [[ -z "${DETMIR_PORTAL_URL:-}" ]]; then + export DETMIR_PORTAL_URL="http://127.0.0.1:8720" + fi + + if [[ -z "${DETMIR_GATEWAY_HOST:-}" ]]; then + local derived_host="${DETMIR_PORTAL_URL#*://}" + derived_host="${derived_host%%/*}" + derived_host="${derived_host%%:*}" + if [[ -n "${derived_host}" ]]; then + export DETMIR_GATEWAY_HOST="${derived_host}" + else + export DETMIR_GATEWAY_HOST="127.0.0.1" + fi + fi + + export DETMIR_AW_API="${DETMIR_AW_API:-http://192.0.2.13:5600/api/0}" + export DETMIR_WORKTIME_URL="${DETMIR_WORKTIME_URL:-http://192.0.2.13:5610}" + export DETMIR_ONE_C_URL="${DETMIR_ONE_C_URL:-http://192.0.2.2:8710}" + export DETMIR_RDP_HOST="${DETMIR_RDP_HOST:-198.51.100.18}" + export DETMIR_HOSTNAME="${DETMIR_HOSTNAME:-HOST-EXAMPLE}" +} + +write_summary() { + { + printf '# AWatch-rus contour check\n\n' + printf -- '- scope: %s\n' "${SCOPE}" + printf -- '- generated_at_utc: %s\n' "${RUN_ID}" + printf -- '- output_dir: %s\n\n' "${OUTPUT_DIR}" + printf 'GitHub/public CI is not the primary registry release contour. This run is intended for the Russian/internal operational contour.\n\n' + printf 'DETMIR_AW_API=%s\n' "${DETMIR_AW_API}" + printf 'DETMIR_WORKTIME_URL=%s\n' "${DETMIR_WORKTIME_URL}" + printf 'DETMIR_ONE_C_URL=%s\n' "${DETMIR_ONE_C_URL}" + printf 'DETMIR_RDP_HOST=%s\n' "${DETMIR_RDP_HOST}" + printf 'DETMIR_HOSTNAME=%s\n' "${DETMIR_HOSTNAME}" + printf 'DETMIR_GATEWAY_HOST=%s\n' "${DETMIR_GATEWAY_HOST}" + printf 'DETMIR_PORTAL_URL=%s\n' "${DETMIR_PORTAL_URL}" + printf 'DETMIR_DLP_COMMAND=%s\n' "${DETMIR_DLP_COMMAND}" + printf 'DETMIR_DISABLE_PORTAL_CHECK=%s\n' "${DETMIR_DISABLE_PORTAL_CHECK:-0}" + printf 'DETMIR_DISABLE_DLP_HEALTH_CHECK=%s\n' "${DETMIR_DISABLE_DLP_HEALTH_CHECK:-0}" + printf '\n' + } >"${OUTPUT_DIR}/SUMMARY.md" +} + +configure_detmir_env + +find_detmir_check() { + if [[ -n "${DETMIR_CHECK_BIN:-}" ]]; then + printf '%s\n' "${DETMIR_CHECK_BIN}" + elif command -v detmir-check >/dev/null 2>&1; then + printf '%s\n' "detmir-check" + elif [[ -x "${REPO_ROOT}/adk-rust/target/release/detmir-check" ]]; then + printf '%s\n' "${REPO_ROOT}/adk-rust/target/release/detmir-check" + else + printf '%s\n' "cargo run --manifest-path ${REPO_ROOT}/adk-rust/Cargo.toml -p detmir-check --" + fi +} + +run_and_log() { + local name="$1" + shift + local log="${OUTPUT_DIR}/logs/${name}.log" + printf '== %s ==\n' "${name}" | tee -a "${OUTPUT_DIR}/SUMMARY.md" + local started_at + started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'started_at_utc: %s\n' "${started_at}" | tee -a "${OUTPUT_DIR}/SUMMARY.md" + local rc=0 + local started_at_unix + started_at_unix="$(date -u +%s)" + + local stream_logs="${CONTOUR_CHECK_STREAM:-0}" + if [[ "${stream_logs}" == "1" && -t 1 ]]; then + "$@" >"${log}" 2>&1 & + local _run_pid=$! + while kill -0 "${_run_pid}" 2>/dev/null; do + sleep 10 + local now + now="$(date -u +%s)" + local running_elapsed=$(( now - started_at_unix )) + printf 'running %s for %ss\n' "${name}" "${running_elapsed}" + done + wait "${_run_pid}"; rc=$? + else + "$@" >"${log}" 2>&1 || rc=$? + fi + + local finished_at elapsed + finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + elapsed="$(( $(date -u -d "${finished_at}" +%s) - $(date -u -d "${started_at}" +%s) ))" + if [[ "${rc}" -eq 0 ]]; then + printf 'status: ok elapsed=%ss\n\n' "${elapsed}" | tee -a "${OUTPUT_DIR}/SUMMARY.md" + return 0 + fi + printf 'status: fail rc=%s elapsed=%ss log=%s\n\n' "${rc}" "${elapsed}" "${log}" | tee -a "${OUTPUT_DIR}/SUMMARY.md" + return "${rc}" +} + +run_node_smoke() { + local name="$1" + local script="$2" + if command -v timeout >/dev/null 2>&1; then + run_and_log "${name}" timeout "${SMOKE_TIMEOUT_SECONDS}" node "${script}" + else + run_and_log "${name}" node "${script}" + fi +} + +write_summary + +status=0 +detmir_check_cmd="$(find_detmir_check)" + +if [[ "${detmir_check_cmd}" == cargo\ run* ]]; then + # shellcheck disable=SC2086 + if ! run_and_log "detmir-check-json" bash -lc "${detmir_check_cmd} --json"; then + status=1 + fi +else + if ! run_and_log "detmir-check-json" "${detmir_check_cmd}" --json; then + status=1 + fi +fi + +if [[ "${RUN_PORTAL_SMOKE:-0}" == "1" ]]; then + hardening_smoke="${REPO_ROOT}/scripts/awatch-production-hardening-smoke.mjs" + if [[ ! -f "${hardening_smoke}" && -f "${LIB_DIR}/awatch-production-hardening-smoke.mjs" ]]; then + hardening_smoke="${LIB_DIR}/awatch-production-hardening-smoke.mjs" + fi + if [[ -f "${hardening_smoke}" ]] && command -v node >/dev/null 2>&1; then + if ! run_node_smoke "portal-hardening-smoke" "${hardening_smoke}"; then + status=1 + fi + else + printf '== portal-hardening-smoke ==\nstatus: skipped reason=node_or_script_missing\n\n' | tee -a "${OUTPUT_DIR}/SUMMARY.md" + fi + + pilot_smoke="${REPO_ROOT}/scripts/detmir-pilot-demo-smoke.mjs" + if [[ -f "${pilot_smoke}" ]] && command -v node >/dev/null 2>&1; then + if ! run_node_smoke "pilot-demo-smoke" "${pilot_smoke}"; then + status=1 + fi + else + printf '== pilot-demo-smoke ==\nstatus: skipped reason=requires_repo_docs_and_fixtures\n\n' | tee -a "${OUTPUT_DIR}/SUMMARY.md" + fi +else + printf '== portal-smoke ==\nstatus: skipped reason=RUN_PORTAL_SMOKE_not_enabled\n\n' | tee -a "${OUTPUT_DIR}/SUMMARY.md" +fi + +if [[ "${RUN_REGISTRY_CHECK:-0}" == "1" ]] && [[ -x "${REPO_ROOT}/scripts/registry_readiness_check.sh" ]]; then + if ! run_and_log "registry-readiness-check" bash "${REPO_ROOT}/scripts/registry_readiness_check.sh"; then + status=1 + fi +fi + +printf 'final_status: %s\n' "$([[ "${status}" -eq 0 ]] && printf ok || printf fail)" | tee -a "${OUTPUT_DIR}/SUMMARY.md" +exit "${status}"