From c9ad5ecca7024f8031677c3dbfb797be7d98fdbb Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Thu, 18 Jun 2026 22:11:38 +0300 Subject: [PATCH] chore(scripts): add full diagnostics package for detmir --- .../DETMIR_FULL_DIAGNOSTICS_RU.md | 41 ++ .../aw-contour-diag.sh | 507 ++++++++++++++++++ ...check_production_inventory_placeholders.sh | 126 +++++ .../detmir-full-diagnostics-daily.sh | 5 + .../detmir-full-diagnostics-monthly.sh | 5 + .../detmir-full-diagnostics-weekly.sh | 5 + .../detmir-full-diagnostics.env.example | 20 + .../detmir-full-diagnostics.sh | 222 ++++++++ 8 files changed, 931 insertions(+) create mode 100644 scripts/detmir-full-diagnostics/DETMIR_FULL_DIAGNOSTICS_RU.md create mode 100644 scripts/detmir-full-diagnostics/aw-contour-diag.sh create mode 100644 scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh create mode 100644 scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh create mode 100644 scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh create mode 100644 scripts/detmir-full-diagnostics/detmir-full-diagnostics-weekly.sh create mode 100644 scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example create mode 100644 scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh diff --git a/scripts/detmir-full-diagnostics/DETMIR_FULL_DIAGNOSTICS_RU.md b/scripts/detmir-full-diagnostics/DETMIR_FULL_DIAGNOSTICS_RU.md new file mode 100644 index 0000000..cb94d80 --- /dev/null +++ b/scripts/detmir-full-diagnostics/DETMIR_FULL_DIAGNOSTICS_RU.md @@ -0,0 +1,41 @@ +# Полная диагностика развернутого контура DetMir + +Пакет предназначен для расширенной проверки уже развернутого контура +DetMir, не заменяет и не дублирует скрипты TЗ поддержки. + +Состав: +- `detmir-full-diagnostics.sh` — оркестратор: + - `aw-contour-diag.sh` (по-умолчанию полный прогон), + - `detmir-support-run.sh` (daily/weekly/monthly), + - `check_production_inventory_placeholders.sh`. +- `detmir-full-diagnostics-daily.sh` — быстрый старт с `--scope daily`. +- `detmir-full-diagnostics-weekly.sh` — запуск с `--scope weekly`. +- `detmir-full-diagnostics-monthly.sh` — запуск с `--scope monthly`. +- `detmir-full-diagnostics.env.example` — шаблон параметров. + +## Быстрый старт + +1. Скопировать файл конфигурации: + +```bash +cp scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example \ + scripts/detmir-full-diagnostics/detmir-full-diagnostics.env +``` + +2. Заполнить пути/хосты под текущий контур. + +3. Запуск: + +```bash +./scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh +./scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope weekly --quick +./scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh --output-dir /var/log/detmir-full-diagnostics +``` + +## Примечание + +- Для запуска в `~/root/scripts/support` пакет может использовать скопированные + `aw-contour-diag.sh` и `check_production_inventory_placeholders.sh`. + Если локально эти файлы не требуются — удалите их или отключите шаги флагами. +- Для запуска в окружении с root-доступом через `sudo` выполняйте скрипты как + пользователь с правом `sudo` без `NOPASSWD`. diff --git a/scripts/detmir-full-diagnostics/aw-contour-diag.sh b/scripts/detmir-full-diagnostics/aw-contour-diag.sh new file mode 100644 index 0000000..0373e6d --- /dev/null +++ b/scripts/detmir-full-diagnostics/aw-contour-diag.sh @@ -0,0 +1,507 @@ +#!/usr/bin/env bash +# aw-contour-diag.sh - Диагностика всего контура ActivityWatch-Russian +# Запускать с машины администратора (где есть доступ по SSH/curl ко всем узлам). +# +# Использование: +# ./scripts/aw-contour-diag.sh # полная диагностика +# ./scripts/aw-contour-diag.sh --quick # быстрая (только AW server + buckets) +# ./scripts/aw-contour-diag.sh --skip-windows # без RDP/WinRM проверок +# +# При красных проверках в скрипте указаны разделы 'REMEDIATION: ...' +# с конкретными командами для восстановления. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ANSIBLE_DIR="$REPO_ROOT/ansible" +INVENTORY="$ANSIBLE_DIR/inventory.ini" +AW_SERVER="http://10.10.10.13:5600" +AW_WORKTIME_API="http://10.10.10.13:5610" +INFLUXDB_URL="http://10.10.10.10:8086" +GRAFANA_URL="http://10.10.10.11:3000" +PROXMOX_HOST="10.10.10.2" +AW_HOST="10.10.10.13" +GRAFANA_HOST="10.10.10.11" +INFLUXDB_HOST="10.10.10.10" +WINDOWS_HOST="192.168.100.18" +CLICKHOUSE_HOST="10.10.10.2" +SOURCE_HOSTNAME="SHARKON2025" + +QUICK_MODE=0 +SKIP_WINDOWS=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --quick) QUICK_MODE=1; shift ;; + --skip-windows) SKIP_WINDOWS=1; shift ;; + -h|--help) + echo "Usage: $(basename "$0") [--quick] [--skip-windows]" + exit 0 ;; + *) echo "Unknown: $1"; exit 2 ;; + esac +done + +export no_proxy="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$INFLUXDB_HOST,$WINDOWS_HOST,$CLICKHOUSE_HOST,10.10.10.0/24,192.168.100.0/24" +export NO_PROXY="$no_proxy" + +OK_COUNT=0; WARN_COUNT=0; FAIL_COUNT=0; SKIP_COUNT=0 + +if [ -t 1 ]; then + RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +else + RED=''; GREEN=''; YELLOW=''; CYAN=''; NC='' +fi + +pass() { OK_COUNT=$((OK_COUNT+1)); printf "%b[OK]%b %s\n" "$GREEN" "$NC" "$*"; } +warn() { WARN_COUNT=$((WARN_COUNT+1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; } +fail() { FAIL_COUNT=$((FAIL_COUNT+1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; } +skip() { SKIP_COUNT=$((SKIP_COUNT+1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; } +section() { printf "\n%b=== %s ===%b\n" "$CYAN" "$*" "$NC"; } +have() { command -v "$1" >/dev/null 2>&1; } + +check_tcp() { + local name="$1" host="$2" port="$3" + if timeout 4 bash -c ":/dev/null 2>&1; then + pass "TCP $host:$port ($name)" + else + fail "TCP $host:$port ($name)" + echo " REMEDIATION: Проверьте, запущен ли сервис на $host:$port." + echo " Для systemd: ssh igor@$host 'systemctl status '" + echo " Для Docker: ssh igor@$PROXMOX_HOST 'sudo docker ps | grep '" + fi +} + +check_http_code() { + local name="$1" url="$2" expected="${3:-^2[0-9][0-9]$}" + local tmp code + tmp="$(mktemp)" + code="$(curl -k -sS --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")" + if printf "%s" "$code" | grep -Eq "$expected"; then + pass "HTTP $code $url ($name)" + else + fail "HTTP $code $url ($name)" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -20 + fi + rm -f "$tmp" "$tmp.err" +} + +check_http_json_key() { + local name="$1" url="$2" jq_filter="$3" remediation="$4" + local tmp + tmp="$(mktemp)" + if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then + pass "$name" + else + fail "$name" + sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -10 + echo " REMEDIATION: $remediation" + fi + rm -f "$tmp" "$tmp.err" +} + +check_bucket_freshness() { + local bucket="$1" label="$2" remediation="$3" + local bucket_id="${bucket}_${SOURCE_HOSTNAME}" + local tmp last_ts event_epoch now age_sec + tmp="$(mktemp)" + if ! curl -fsS --connect-timeout 5 --max-time 15 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then + fail "bucket $label ($bucket_id) — запрос не удался" + sed 's/^/ /' "$tmp.err" | head -5 + echo " REMEDIATION: $remediation" + rm -f "$tmp" "$tmp.err" + return + fi + last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)" + rm -f "$tmp" + if [ -z "$last_ts" ]; then + warn "bucket $label ($bucket_id) — нет событий" + echo " REMEDIATION: $remediation" + return + fi + event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || echo 0)" + now="$(date -u +%s)" + age_sec=$((now - event_epoch)) + + case "$bucket" in + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + if [ "$age_sec" -lt 86400 ]; then + pass "bucket $label — ${age_sec}s назад" + else + warn "bucket $label — ${age_sec}s назад (event-driven)" + fi ;; + aw-watcher-window|aw-dlp-endpoint-signals) + if [ "$age_sec" -lt 7200 ]; then + pass "bucket $label — ${age_sec}s назад" + else + warn "bucket $label — ${age_sec}s назад (INACTIVE)" + fi ;; + *) + if [ "$age_sec" -lt 3600 ]; then + pass "bucket $label — ${age_sec}s назад" + elif [ "$age_sec" -lt 86400 ]; then + warn "bucket $label — ${age_sec}s назад (STALE)" + echo " REMEDIATION: $remediation" + else + fail "bucket $label — ${age_sec}s назад (DEAD)" + echo " REMEDIATION: $remediation" + fi ;; + esac +} + +check_ansible_shell() { + local name="$1" group="$2" command="$3" + if ! have ansible; then + skip "$name (ansible not available)" + return + fi + if [ ! -f "$INVENTORY" ]; then + skip "$name (inventory not found: $INVENTORY)" + return + fi + local tmp + tmp="$(mktemp)" + if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command" >"$tmp" 2>&1; then + pass "$name" + else + fail "$name" + sed 's/^/ /' "$tmp" | head -20 + fi + rm -f "$tmp" +} + +check_ansible_win_shell() { + local name="$1" command="$2" + if ! have ansible; then skip "$name (ansible not available)"; return; fi + if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi + local tmp + tmp="$(mktemp)" + if ANSIBLE_NOCOLOR=1 ansible aw_windows -i "$INVENTORY" -m win_shell -a "$command" >"$tmp" 2>&1; then + pass "$name" + else + fail "$name" + sed 's/^/ /' "$tmp" | head -20 + fi + rm -f "$tmp" +} + +check_ansible_module() { + local name="$1" group="$2" module="$3" args="${4:-}" + if ! have ansible; then skip "$name (ansible not available)"; return; fi + if [ ! -f "$INVENTORY" ]; then skip "$name (inventory not found)"; return; fi + local tmp + tmp="$(mktemp)" + if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then + pass "$name" + else + fail "$name" + sed 's/^/ /' "$tmp" | head -20 + fi + rm -f "$tmp" +} + +ssh_with_diag_password() { + local host="$1" + shift + if [[ -z "${AW_DIAG_SSH_PASSWORD:-}" ]]; then + return 125 + fi + sshpass -p "$AW_DIAG_SSH_PASSWORD" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "igor@$host" "$@" +} + +ssh_aw() { ssh_with_diag_password 10.10.10.13 "$@"; } +ssh_pve() { ssh_with_diag_password 10.10.10.2 "$@"; } + +check_service_remote() { + local name="$1" host="$2" unit="$3" remediation="$4" + local result + result=$(ssh_with_diag_password "$host" "systemctl is-active $unit 2>/dev/null || echo not_found" 2>/dev/null) + local rc=$? + if [ "$rc" -eq 125 ]; then + skip "$name ($unit on $host — set AW_DIAG_SSH_PASSWORD for SSH checks)" + return + fi + if [ "$rc" -ne 0 ] || [ "$result" = "not_found" ]; then + skip "$name ($unit on $host — не удалось проверить)" + return + fi + if [ "$result" = "active" ]; then + pass "$name ($unit active on $host)" + else + fail "$name ($unit $result on $host)" + echo " REMEDIATION: $remediation" + fi +} + +printf "%b=== ActivityWatch-Russian: Диагностика контура ===%b\n" "$CYAN" "$NC" +echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +echo "" + +# ============================================================ +section "1. Локальные предусловия" +# ============================================================ +for cmd in bash curl jq timeout ssh sshpass; do + if have "$cmd"; then pass "утилита $cmd найдена"; else fail "утилита $cmd не найдена (установите: apt install $cmd)"; fi +done +echo "" + +# ============================================================ +section "2. TCP доступность узлов" +# ============================================================ +check_tcp "AW Server" "$AW_HOST" 5600 +check_tcp "Worktime API" "$AW_HOST" 5610 +check_tcp "RDP WinRM" "$WINDOWS_HOST" 5985 +check_tcp "Proxmox SSH" "$PROXMOX_HOST" 22 +check_tcp "Proxmox HTTPS" "$PROXMOX_HOST" 443 +check_tcp "1C Company API" "$PROXMOX_HOST" 8710 +check_tcp "ClickHouse HTTP" "$CLICKHOUSE_HOST" 8123 +check_tcp "ClickHouse Native" "$CLICKHOUSE_HOST" 9000 +check_tcp "InfluxDB" "$INFLUXDB_HOST" 8086 +check_tcp "Grafana" "$GRAFANA_HOST" 3000 + +if [ "$QUICK_MODE" = "1" ]; then + # В быстром режиме проверяем только AW Server и buckets + echo "" + section "3. AW Server (быстрый режим)" + check_http_code "AW Server info" "$AW_SERVER/api/0/info" '^200$' + check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$' + check_http_code "Worktime API health" "$AW_WORKTIME_API/health" '^200$' + + section "4. Buckets (быстрый режим)" + for entry in \ + "aw-watcher-afk|AFK watcher|Запустите на RDP: schtasks /Run /TN \"ActivityWatch Recovery\" или schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"" \ + "aw-watcher-window|Window watcher|Запустите через ansible: ansible aw_windows -i $INVENTORY -m win_shell -a 'Start-Process -FilePath \"C:\\Program Files\\AWatch-rus\\bin\\aw-watcher-window\\aw-watcher-window.exe\" -ArgumentList @(\"--host\", \"10.10.10.13\", \"--port\", \"5600\") -WindowStyle Hidden'" \ + "aw-worktime-sessions|Worktime sessions|Проверьте работу worktime-api: systemctl status aw-worktime-api на AW сервере" \ + "aw-session-events|Session events|Проверьте collector-guard и aw-session-events-collector на RDP" \ + "aw-dlp-endpoint-signals|DLP signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Sleep 30'" \ + "aw-dlp-incidents|DLP incidents|Проверьте aw-detmir-dlp-collector.ps1 на RDP (логи: C:\\ProgramData\\AWatch-rus\\logs\\)" \ + ; do + bucket="${entry%%|*}"; rest="${entry#*|}" + label="${rest%%|*}"; remediation="${rest#*|}" + check_bucket_freshness "$bucket" "$label" "$remediation" + done + echo "" + echo "=== Быстрая диагностика завершена ===" + printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" + [ "$FAIL_COUNT" -gt 0 ] && exit 2 || exit 0 +fi + +# ============================================================ +section "3. AW Server (10.10.10.13)" +# ============================================================ +check_http_json_key "AW Server info" "$AW_SERVER/api/0/info" \ + '.version' \ + "Проверьте: ssh igor@$AW_HOST 'systemctl status activitywatch-server'" +check_http_code "AW Server CORS" "$AW_SERVER/api/0/settings/" '^200$' +check_http_code "AW WebUI" "$AW_SERVER/" '^200$' +check_http_json_key "Worktime API health" "$AW_WORKTIME_API/health" \ + '.status // .ok' \ + "Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'" + +# ============================================================ +section "4. Buckets (свежесть данных)" +# ============================================================ +for entry in \ + "aw-watcher-afk|AFK watcher|Запустите на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"'" \ + "aw-watcher-window|Window watcher|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Process -FilePath \"C:\\Program Files\\AWatch-rus\\bin\\aw-watcher-window\\aw-watcher-window.exe\" -ArgumentList @(\"--host\", \"10.10.10.13\", \"--port\", \"5600\") -WindowStyle Hidden'" \ + "aw-worktime-sessions|Worktime sessions|Проверьте: ssh igor@$AW_HOST 'systemctl status aw-worktime-api && journalctl -u aw-worktime-api -n 20'" \ + "aw-session-events|Session events|Проверьте collector-guard на RDP: ansible aw_windows -i $INVENTORY -m win_shell -a 'Get-Process -Name aw-session-events-* -ErrorAction SilentlyContinue'" \ + "aw-dlp-endpoint-signals|DLP endpoint signals|Запустите: ansible aw_windows -i $INVENTORY -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Launch [SHARKON2025_Администратор]\"; Start-Sleep 60'" \ + "aw-dlp-incidents|DLP incidents|Проверьте: ansible aw_windows -i $INVENTORY -m win_shell -a \"Get-Content 'C:\\ProgramData\\AWatch-rus\\logs\\dlp-*.log' -Tail 20\"" \ + "aw-dlp-review|DLP review|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-policy-engine.service -n 20 --no-pager'" \ + "aw-dlp-rules|DLP rules|Проверьте: ssh igor@$AW_HOST 'journalctl -u aw-dlp-ioc-refresh.service -n 20 --no-pager'" \ +; do + bucket="${entry%%|*}"; rest="${entry#*|}" + label="${rest%%|*}"; remediation="${rest#*|}" + check_bucket_freshness "$bucket" "$label" "$remediation" +done + +# ============================================================ +section "5. InfluxDB (10.10.10.10:8086)" +# ============================================================ +check_http_json_key "InfluxDB health" "$INFLUXDB_URL/health" \ + '.status == "pass"' \ + "Проверьте InfluxDB на LXC 200: ssh igor@$PROXMOX_HOST 'sudo pct exec 200 -- systemctl status influxdb'" + +# ============================================================ +section "6. Grafana (10.10.10.11:3000)" +# ============================================================ +check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" \ + '.database == "ok"' \ + "Проверьте: ssh igor@$PROXMOX_HOST 'sudo pct exec 201 -- systemctl status grafana-server'" +check_http_code "Grafana datasources API" "$GRAFANA_URL/api/datasources" '^200$|^302$|^401$' + +# ============================================================ +section "7. ClickHouse (10.10.10.2:8123)" +# ============================================================ +# Проверяем через прямой HTTP — AUTHENTICATION_FAILED = сервер жив +local_ch_ok=0 +ch_code=$(curl -sS --max-time 5 "http://$CLICKHOUSE_HOST:8123/?query=SELECT%201" 2>/dev/null | head -1) +if echo "$ch_code" | grep -q "AUTHENTICATION_FAILED"; then + pass "ClickHouse HTTP — отвечает (требуется аутентификация, это нормально)" + local_ch_ok=1 +elif echo "$ch_code" | grep -q "1"; then + pass "ClickHouse HTTP — SELECT 1 OK" + local_ch_ok=1 +else + fail "ClickHouse HTTP — не отвечает: $ch_code" + echo " REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose ps; sudo docker compose logs --tail=20'" +fi + +# Проверка Docker контейнера через SSH +container_status=$(ssh_pve 'sudo docker ps --filter name=aw-rus-1c-clickhouse --format "{{.Status}}" 2>/dev/null' 2>/dev/null) +if [ -n "$container_status" ]; then + pass "ClickHouse Docker контейнер: $container_status" +else + fail "ClickHouse Docker контейнер не запущен" + echo " REMEDIATION: ssh igor@$PROXMOX_HOST 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose up -d'" +fi + +# ClickHouse health timer +check_service_remote "ClickHouse health timer" "$PROXMOX_HOST" "aw-1c-clickhouse-health.timer" \ + "Проверьте: ssh igor@$PROXMOX_HOST 'sudo journalctl -u aw-1c-clickhouse-health.service -n 30 --no-pager'" + +# ClickHouse network health timer (с AW сервера) +check_service_remote "ClickHouse network health timer" "$AW_HOST" "aw-clickhouse-network-health.timer" \ + "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-clickhouse-network-health.service -n 30 --no-pager'" + +# 1C-ingest timer +check_service_remote "1C ingest timer" "$PROXMOX_HOST" "aw-1c-ingest.timer" \ + "Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status aw-1c-ingest.timer; sudo journalctl -u aw-1c-ingest.service -n 20'" + +# ============================================================ +section "8. 1C Manager API (10.10.10.2:8710)" +# ============================================================ +check_http_json_key "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" \ + '.status == "ok"' \ + "Проверьте Python процесс: ssh igor@$PROXMOX_HOST 'ps aux | grep 8710 | grep -v grep'" +check_http_code "1C /manager/brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$' + +# ============================================================ +section "9. Nginx Gateway (10.10.10.2)" +# ============================================================ +check_http_code "Gateway /healthz" "https://$PROXMOX_HOST/healthz" '^200$' +check_http_code "Gateway /go/proxmox-gui (401=protected, OK)" "https://$PROXMOX_HOST/go/proxmox-gui" '^30[1278]$|^401$' +check_http_code "Gateway /go/file1c-brief (401=protected, OK)" "https://$PROXMOX_HOST/go/file1c-brief" '^30[1278]$|^401$' + +check_service_remote "Nginx service" "$PROXMOX_HOST" "nginx.service" \ + "Проверьте: ssh igor@$PROXMOX_HOST 'sudo systemctl status nginx; sudo nginx -t'" + +# ============================================================ +section "10. DLP Pipeline (10.10.10.13)" +# ============================================================ +# DLP Policy Engine — должен быть active (running) +check_service_remote "DLP Policy Engine" "$AW_HOST" "aw-dlp-policy-engine.service" \ + "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-policy-engine.service -n 30 --no-pager'" + +# DLP Case Management +check_service_remote "DLP Case Management" "$AW_HOST" "aw-dlp-case-management.service" \ + "Проверьте: ssh igor@$AW_HOST 'sudo journalctl -u aw-dlp-case-management.service -n 30 --no-pager'" + +# DLP Aggregator +aggr_status=$(ssh_aw 'systemctl is-active activitywatch-dlp-aggregator.timer 2>/dev/null || echo not_found' 2>/dev/null) +if [ "$aggr_status" = "active" ]; then + pass "DLP Aggregator timer (active)" +else + fail "DLP Aggregator timer ($aggr_status)" + echo " REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now activitywatch-dlp-aggregator.timer; sudo journalctl -u activitywatch-dlp-aggregator.service -n 30'" +fi + +# DLP Influx Exporter +influx_exp_status=$(ssh_aw 'systemctl is-active aw-dlp-influx-exporter.timer 2>/dev/null || echo not_found' 2>/dev/null) +if [ "$influx_exp_status" = "active" ]; then + pass "DLP Influx Exporter timer (active)" +else + fail "DLP Influx Exporter timer ($influx_exp_status)" + echo " REMEDIATION: ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-influx-exporter.timer; sudo journalctl -u aw-dlp-influx-exporter.service -n 30'" +fi + +# DLP CEF Exporter +check_service_remote "DLP CEF Exporter timer" "$AW_HOST" "aw-dlp-cef-exporter.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-cef-exporter.timer; journalctl -u aw-dlp-cef-exporter.service -n 20'" + +# DLP IOC Refresh +check_service_remote "DLP IOC Refresh timer" "$AW_HOST" "aw-dlp-ioc-refresh.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-ioc-refresh.timer'" + +# DLP Syslog Forwarder +check_service_remote "DLP Syslog Forwarder timer" "$AW_HOST" "aw-dlp-syslog-forwarder.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-syslog-forwarder.timer'" + +# DLP Webhook Sender +check_service_remote "DLP Webhook Sender timer" "$AW_HOST" "aw-dlp-webhook-sender.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-webhook-sender.timer'" + +# DLP Report Scheduler +check_service_remote "DLP Report Scheduler timer" "$AW_HOST" "aw-dlp-report-scheduler.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-dlp-report-scheduler.timer'" + +# Worktime Influx Exporter +check_service_remote "Worktime Influx Exporter timer" "$AW_HOST" "aw-worktime-influx-exporter.timer" \ + "ssh igor@$AW_HOST 'sudo systemctl enable --now aw-worktime-influx-exporter.timer; journalctl -u aw-worktime-influx-exporter.service -n 20'" + +# ============================================================ +if [ "$SKIP_WINDOWS" = "1" ]; then + skip "Проверки RDP хоста пропущены (--skip-windows)" +else + section "11. RDP хост (192.168.100.18)" + if have ansible && [ -f "$INVENTORY" ]; then + check_ansible_module "WinRM ping" aw_windows win_ping + + check_ansible_win_shell "Сессии RDP" 'query user 2>&1' + + check_ansible_win_shell "Процессы watcher" \ + 'Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize' + + check_ansible_win_shell "Количество процессов powershell" \ + '(Get-Process powershell -ErrorAction SilentlyContinue | Measure-Object).Count' + + check_ansible_win_shell "Scheduled tasks (Recovery)" \ + 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V | Select-String "Status|Run|Next"' + + check_ansible_win_shell "Scheduled tasks (Launch Admin)" \ + 'schtasks /Query /TN "ActivityWatch Launch [SHARKON2025_Администратор]" /FO LIST /V | Select-String "Status|Run|Next"' + else + skip "Ansible или inventory не найдены" + fi +fi + +# ============================================================ +section "12. Systemd health (AW server)" +# ============================================================ +check_ansible_shell "AW server core units" aw_server \ + 'systemctl is-active activitywatch-server aw-worktime-api aw-dlp-policy-engine aw-dlp-case-management aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer activitywatch-dlp-aggregator.timer aw-clickhouse-network-health.timer | paste -sd,' + +check_ansible_shell "AW server — нет failed units" aw_server \ + 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|dlp" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }' + +# ============================================================ +section "13. Systemd health (Proxmox)" +# ============================================================ +check_ansible_shell "Proxmox core units" proxmox \ + 'systemctl is-active nginx docker aw-1c-clickhouse-health.timer aw-1c-ingest.timer 2>/dev/null | paste -sd,' + +# ============================================================ +section "14. Диск и память" +# ============================================================ +check_ansible_shell "Диски AW server" aw_server 'df -h / /var /opt 2>/dev/null | tail -5' +check_ansible_shell "Память AW server" aw_server 'free -h | tail -5' + +# ============================================================ +section "Итог диагностики" +# ============================================================ +printf " OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT" + +if [ "$FAIL_COUNT" -gt 0 ]; then + echo "" + echo " Есть проблемы! Смотрите REMEDIATION выше для каждого FAIL." + echo " После исправления запустите повторно: $0" + exit 2 +elif [ "$WARN_COUNT" -gt 0 ]; then + echo "" + echo " Есть предупреждения (WARN) — стоит проверить, но не критично." + exit 1 +else + echo "" + echo " Все проверки пройдены. Контур в рабочем состоянии." + exit 0 +fi diff --git a/scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh b/scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh new file mode 100644 index 0000000..355f45b --- /dev/null +++ b/scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/check_production_inventory_placeholders.sh [--allow-missing] [--strict] FILE... + DETMIR_PRODUCTION_CONFIG_PATHS="file1:file2" scripts/check_production_inventory_placeholders.sh + scripts/check_production_inventory_placeholders.sh --self-test + +Fails when production inventory/env files contain public placeholder values: +TEST-NET addresses, HOST-EXAMPLE, WINDOWS_USER_EXAMPLE, CHANGE_ME, YOUR_*, +replace-me, or angle-bracket placeholders. + +Use --strict for private production override files. Strict mode requires at +least one existing path and rejects public/example/default files. +EOF +} + +pattern='(192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE|CHANGE_ME|CHANGEME|REPLACE_ME|replace-me|YOUR_[A-Z0-9_]*|<[A-Z0-9_ -]+>)' +allow_missing=0 +strict=0 +self_test=0 +paths=() + +while (($#)); do + case "$1" in + --allow-missing) + allow_missing=1 + ;; + --strict) + strict=1 + ;; + --self-test) + self_test=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + paths+=("$1") + ;; + esac + shift +done + +run_scan() { + local file + local found=0 + for file in "$@"; do + if (( strict == 1 )) && [[ "$file" == ansible/group_vars* || "$file" == ansible/inventory.ini || "$file" == *".example."* || "$file" == *"example."* ]]; then + printf 'strict mode refuses public/default config path: %s\n' "$file" >&2 + found=1 + continue + fi + if [[ ! -e "$file" ]]; then + if (( allow_missing == 0 )); then + printf 'missing production config path: %s\n' "$file" >&2 + found=1 + fi + continue + fi + if [[ -d "$file" ]]; then + while IFS= read -r -d '' child; do + if grep -nE "$pattern" "$child" >/dev/null; then + printf 'placeholder found in %s\n' "$child" >&2 + grep -nE "$pattern" "$child" | sed -E 's/^([0-9]+):.*/ line \1: placeholder marker/' >&2 + found=1 + fi + done < <(find "$file" -type f \( -name '*.env' -o -name '*.ini' -o -name '*.yml' -o -name '*.yaml' \) -print0) + elif grep -nE "$pattern" "$file" >/dev/null; then + printf 'placeholder found in %s\n' "$file" >&2 + grep -nE "$pattern" "$file" | sed -E 's/^([0-9]+):.*/ line \1: placeholder marker/' >&2 + found=1 + fi + done + return "$found" +} + +if (( self_test == 1 )); then + tmp_dir="$(mktemp -d)" + trap 'rm -rf "$tmp_dir"' EXIT + good="$tmp_dir/good.env" + bad="$tmp_dir/bad.env" + cat >"$good" <<'EOF' +AW_WORKTIME_INFLUX_URL=http://influxdb.internal:8086 +AW_WORKTIME_INFLUX_HOSTS=WINDOWS-HOST +AW_WORKTIME_INFLUX_TOKEN=prod-write-token-value +EOF + cat >"$bad" <<'EOF' +AW_WORKTIME_INFLUX_URL=http://192.0.2.10:8086 +AW_WORKTIME_INFLUX_HOSTS=HOST-EXAMPLE +AW_WORKTIME_INFLUX_TOKEN=CHANGE_ME +EOF + run_scan "$good" + if run_scan "$bad" >/dev/null 2>&1; then + echo "self-test failed: bad fixture was accepted" >&2 + exit 1 + fi + saved_strict="$strict" + strict=1 + if run_scan "ansible/group_vars/all.yml" >/dev/null 2>&1; then + echo "self-test failed: strict mode accepted public/default file name" >&2 + exit 1 + fi + strict="$saved_strict" + echo "production inventory placeholder guard self-test: OK" + exit 0 +fi + +if ((${#paths[@]} == 0)) && [[ -n "${DETMIR_PRODUCTION_CONFIG_PATHS:-}" ]]; then + IFS=':' read -r -a paths <<<"$DETMIR_PRODUCTION_CONFIG_PATHS" +fi + +if ((${#paths[@]} == 0)); then + if (( allow_missing == 1 && strict == 0 )); then + echo "production inventory placeholder guard: skipped (no production paths)" + exit 0 + fi + usage >&2 + exit 2 +fi + +run_scan "${paths[@]}" +echo "production inventory placeholder guard: OK" diff --git a/scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh new file mode 100644 index 0000000..46d2bca --- /dev/null +++ b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope daily "$@" diff --git a/scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh new file mode 100644 index 0000000..6d146c6 --- /dev/null +++ b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-monthly.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope monthly "$@" diff --git a/scripts/detmir-full-diagnostics/detmir-full-diagnostics-weekly.sh b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-weekly.sh new file mode 100644 index 0000000..d5fa83a --- /dev/null +++ b/scripts/detmir-full-diagnostics/detmir-full-diagnostics-weekly.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$SCRIPT_DIR/detmir-full-diagnostics.sh" --scope weekly "$@" diff --git a/scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example b/scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example new file mode 100644 index 0000000..999754c --- /dev/null +++ b/scripts/detmir-full-diagnostics/detmir-full-diagnostics.env.example @@ -0,0 +1,20 @@ +## Базовые параметры для расширенной диагностики контуров DetMir +DETMIR_FULL_DIAGNOSTICS_SCOPE=daily +DETMIR_FULL_DIAGNOSTICS_OUTPUT_DIR=/var/log/detmir-full-diagnostics + +# Опционально указывайте явные пути. +# Если переменная не задана, используются пути относительно этой папки: +# - aw-contour-diag.sh: ./aw-contour-diag.sh +# - detmir-support-run.sh: ../detmir-support-run.sh +# - detmir-support.env: ../detmir-support.env +DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SCRIPT= +DETMIR_FULL_DIAGNOSTICS_AW_DIAG_QUICK=0 +DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SKIP_WINDOWS=0 +DETMIR_FULL_DIAGNOSTICS_SUPPORT_RUN= +DETMIR_FULL_DIAGNOSTICS_SUPPORT_ENV= + +# Проверка placeholder-значений в production-конфиге (по желанию) +DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_GUARD=1 +DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_SCRIPT= +# Если путь пустой — проверка placeholder-guard будет пропущена. +DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS= diff --git a/scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh b/scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh new file mode 100644 index 0000000..40d5a79 --- /dev/null +++ b/scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +ENV_FILE="${DETMIR_FULL_DIAGNOSTICS_ENV_FILE:-$SCRIPT_DIR/detmir-full-diagnostics.env}" +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +fi + +usage() { + cat <<'USAGE' +Usage: + scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh [options] + +Options: + --scope daily|weekly|monthly Scope of operational checks (default: daily) + --output-dir PATH Root output directory for all checks + --quick Pass --quick to aw-contour-diag + --skip-windows Skip WinRM checks in aw-contour-diag + --no-support Skip detmir-support-run checks + --no-aw-diagnostic Skip aw-contour-diag + --no-placeholder-check Skip production placeholder guard check + --help Show this help + +Examples: + scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope daily + scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh --scope weekly --quick + scripts/detmir-full-diagnostics/detmir-full-diagnostics-daily.sh +USAGE +} + +SCOPE="${DETMIR_FULL_DIAGNOSTICS_SCOPE:-daily}" +OUTPUT_DIR="${DETMIR_FULL_DIAGNOSTICS_OUTPUT_DIR:-$REPO_ROOT/output/detmir-full-diagnostics}" +RUN_AW_DIAG=1 +RUN_SUPPORT=1 +RUN_PLACEHOLDER_GUARD=1 +AW_QUICK="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_QUICK:-0}" +AW_SKIP_WINDOWS="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SKIP_WINDOWS:-0}" + +AW_DIAG_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_AW_DIAG_SCRIPT:-$SCRIPT_DIR/aw-contour-diag.sh}" +SUPPORT_RUN_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_SUPPORT_RUN:-$SCRIPT_DIR/../detmir-support-run.sh}" +SUPPORT_ENV_FILE="${DETMIR_FULL_DIAGNOSTICS_SUPPORT_ENV:-$SCRIPT_DIR/../detmir-support.env}" +PLACEHOLDER_SCRIPT="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_SCRIPT:-$SCRIPT_DIR/../check_production_inventory_placeholders.sh}" +PLACEHOLDER_GUARD="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_GUARD:-1}" +PLACEHOLDER_PATHS="${DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS:-$REPO_ROOT/ansible:$REPO_ROOT/private-config}" + +while (( $# > 0 )); do + case "$1" in + --scope) + if (( $# < 2 )); then + usage + echo "error: --scope requires daily|weekly|monthly" >&2 + exit 2 + fi + SCOPE="$2" + shift 2 + ;; + --output-dir) + if (( $# < 2 )); then + usage + echo "error: --output-dir requires path" >&2 + exit 2 + fi + OUTPUT_DIR="$2" + shift 2 + ;; + --quick) + AW_QUICK=1 + shift + ;; + --skip-windows) + AW_SKIP_WINDOWS=1 + shift + ;; + --no-support) + RUN_SUPPORT=0 + shift + ;; + --no-aw-diagnostic) + RUN_AW_DIAG=0 + shift + ;; + --no-placeholder-check) + RUN_PLACEHOLDER_GUARD=0 + shift + ;; + --help) + usage + exit 0 + ;; + *) + usage + echo "error: unknown arg '$1'" >&2 + exit 2 + ;; + esac +done + +if [[ "$SCOPE" != "daily" && "$SCOPE" != "weekly" && "$SCOPE" != "monthly" ]]; then + echo "error: invalid scope '$SCOPE', expected daily|weekly|monthly" >&2 + exit 2 +fi + +if [[ "$RUN_PLACEHOLDER_GUARD" == "1" ]]; then + RUN_PLACEHOLDER_GUARD="$PLACEHOLDER_GUARD" +fi + +RUN_DIR="$OUTPUT_DIR/$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$RUN_DIR" +LOG_FILE="$RUN_DIR/full-diagnostics.log" + +exec > >(tee -a "$LOG_FILE") 2>&1 + +TOTAL=0 +OK_COUNT=0 +FAIL_COUNT=0 + +log() { + printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +run_step() { + local step="$1" + shift + + TOTAL=$((TOTAL + 1)) + log "----" + log "STEP [$TOTAL]: $step" + + if "$@"; then + OK_COUNT=$((OK_COUNT + 1)) + log "RESULT: OK" + return 0 + fi + + FAIL_COUNT=$((FAIL_COUNT + 1)) + log "RESULT: FAIL" + return 1 +} + +log "DetMir full diagnostics started" +log "Repository: $REPO_ROOT" +log "Scope: $SCOPE" +log "Output: $RUN_DIR" +log "Log: $LOG_FILE" + +if [[ ! -x "$AW_DIAG_SCRIPT" ]] && (( RUN_AW_DIAG == 1 )); then + log "WARN: aw-contour-diag script not found or not executable: $AW_DIAG_SCRIPT" + log " Диагностика AW-контуром будет пропущена." + RUN_AW_DIAG=0 +fi + +if [[ ! -x "$SUPPORT_RUN_SCRIPT" ]] && (( RUN_SUPPORT == 1 )); then + log "ERROR: support script not found or not executable: $SUPPORT_RUN_SCRIPT" + exit 2 +fi +if [[ ! -f "$SUPPORT_ENV_FILE" ]] && (( RUN_SUPPORT == 1 )); then + log "WARN: support env file not found, defaults will be used from detmir-support-run defaults: $SUPPORT_ENV_FILE" +fi + +if [[ ! -f "$PLACEHOLDER_SCRIPT" ]] && (( RUN_PLACEHOLDER_GUARD == 1 )); then + log "WARN: placeholder guard script not found: $PLACEHOLDER_SCRIPT" + RUN_PLACEHOLDER_GUARD=0 +fi + +if (( RUN_AW_DIAG == 1 )); then + if [[ -x "$AW_DIAG_SCRIPT" ]]; then + AW_ARGS=() + if [[ "$AW_QUICK" == "1" ]]; then + AW_ARGS+=(--quick) + fi + if [[ "$AW_SKIP_WINDOWS" == "1" ]]; then + AW_ARGS+=(--skip-windows) + fi + run_step "aw-contour-diag (scope=$SCOPE)" "$AW_DIAG_SCRIPT" "${AW_ARGS[@]}" || true + fi +fi + +if (( RUN_SUPPORT == 1 )); then + run_step "detmir-support-run --scope $SCOPE" \ + env \ + DETMIR_SUPPORT_ENV_FILE="$SUPPORT_ENV_FILE" \ + DETMIR_SUPPORT_OUTPUT_DIR="$RUN_DIR/support" \ + "$SUPPORT_RUN_SCRIPT" --scope "$SCOPE" --output-dir "$RUN_DIR/support" || true +fi + +if [[ "$RUN_PLACEHOLDER_GUARD" == "1" ]]; then + if [[ -z "${PLACEHOLDER_PATHS// }" ]]; then + log "WARN: DETMIR_FULL_DIAGNOSTICS_PLACEHOLDER_PATHS is empty. Placeholder guard skipped." + else + run_step "check production placeholders" \ + env DETMIR_PRODUCTION_CONFIG_PATHS="$PLACEHOLDER_PATHS" \ + "$PLACEHOLDER_SCRIPT" --allow-missing || true + fi +fi + +SUMMARY_FILE="$RUN_DIR/summary.txt" +{ + printf "Scope: %s\n" "$SCOPE" + printf "Total steps: %s\n" "$TOTAL" + printf "OK: %s\n" "$OK_COUNT" + printf "FAIL: %s\n" "$FAIL_COUNT" + if (( FAIL_COUNT == 0 )); then + printf "Result: OK\n" + else + printf "Result: FAIL\n" + fi +} > "$SUMMARY_FILE" + +log "Summary: $SUMMARY_FILE" +log "DetMir full diagnostics completed" + +if (( FAIL_COUNT > 0 )); then + exit 1 +fi + +exit 0