#!/usr/bin/env bash
# End-to-end smoke checks from Igor's laptop for ActivityWatch-Russian.

set -uo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SMOKE_ENV_FILE="${AW_SMOKE_ENV_FILE:-}"
for env_candidate in "$REPO_ROOT/private-config/runtime.env" "$HOME/.config/aw-contour-smoke.env" "$SMOKE_ENV_FILE"; do
  if [ -n "$env_candidate" ] && [ -f "$env_candidate" ]; then
    # Load local credentials and site-specific overrides without committing them.
    # Later files override earlier defaults.
    set -a
    . "$env_candidate"
    set +a
  fi
done

ANSIBLE_DIR="$REPO_ROOT/ansible"
INVENTORY="${AW_SMOKE_INVENTORY:-$ANSIBLE_DIR/inventory.ini}"
REMOTE_SCRIPT_SRC="$REPO_ROOT/scripts/aw-contour-smoke-gateway.sh"
REMOTE_SCRIPT_DST="${AW_SMOKE_REMOTE_SCRIPT:-/usr/local/sbin/aw-contour-smoke.sh}"
DEFAULT_REMOTE_RUST_SRC=""
for rust_candidate in \
  "${CARGO_TARGET_DIR:-}/release/aw-contour-smoke" \
  "$HOME/.cache/detmir-adk-rust-target/release/aw-contour-smoke" \
  "$REPO_ROOT/adk-rust/target/release/aw-contour-smoke"
do
  if [ -n "$rust_candidate" ] && [ -x "$rust_candidate" ]; then
    DEFAULT_REMOTE_RUST_SRC="$rust_candidate"
    break
  fi
done
REMOTE_RUST_SRC="${AW_SMOKE_REMOTE_RUST_SRC:-$DEFAULT_REMOTE_RUST_SRC}"
REMOTE_RUST_DST="${AW_SMOKE_REMOTE_RUST_BIN:-/usr/local/sbin/aw-contour-smoke}"
AW_SERVER="${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}"
WORKTIME_API="${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}"
GRAFANA_URL="${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}"
GRAFANA_USER="${GRAFANA_USER:-igor}"
GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-}"
PROXMOX_HOST="${AW_SMOKE_PROXMOX_HOST:-10.10.10.2}"
AW_HOST="${AW_SMOKE_AW_HOST:-10.10.10.13}"
GRAFANA_HOST="${AW_SMOKE_GRAFANA_HOST:-10.10.10.11}"
WINDOWS_HOST="${AW_SMOKE_WINDOWS_HOST:-192.168.100.18}"
AW_SOURCE_HOSTNAME="${AW_SMOKE_SOURCE_HOSTNAME:-SHARKON2025}"
LOG_DIR="${AW_SMOKE_LOG_DIR:-$REPO_ROOT/output/smoke}"
RUN_REMOTE="${AW_SMOKE_RUN_REMOTE:-1}"
RUN_WINRM="${AW_SMOKE_RUN_WINRM:-1}"
RUN_SERVER_SYSTEMD="${AW_SMOKE_RUN_SERVER_SYSTEMD:-1}"

NO_PROXY_REQUIRED="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$WINDOWS_HOST,10.10.10.0/24,192.168.100.0/24"
if [ -n "${no_proxy:-}" ]; then
  export no_proxy="$no_proxy,$NO_PROXY_REQUIRED"
else
  export no_proxy="$NO_PROXY_REQUIRED"
fi
if [ -n "${NO_PROXY:-}" ]; then
  export NO_PROXY="$NO_PROXY,$NO_PROXY_REQUIRED"
else
  export NO_PROXY="$no_proxy"
fi

OK_COUNT=0
WARN_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0

if [ -t 1 ]; then
  RED='\033[0;31m'
  GREEN='\033[0;32m'
  YELLOW='\033[1;33m'
  CYAN='\033[0;36m'
  NC='\033[0m'
else
  RED=''
  GREEN=''
  YELLOW=''
  CYAN=''
  NC=''
fi

usage() {
  cat <<USAGE
Usage: $(basename "$0") [--skip-remote] [--skip-winrm] [--skip-server-systemd]

Environment overrides:
  AW_SMOKE_AW_SERVER=http://10.10.10.13:5600
  AW_SMOKE_WORKTIME_API=http://10.10.10.13:5610
  AW_SMOKE_GRAFANA_URL=http://10.10.10.11:3000
  AW_SMOKE_SOURCE_HOSTNAME=SHARKON2025
  AW_SMOKE_ENV_FILE=$HOME/.config/aw-contour-smoke.env
  AW_SMOKE_LOG_DIR=$REPO_ROOT/output/smoke
  GRAFANA_USER/GRAFANA_PASSWORD via env or a local env file
USAGE
}

while [ "$#" -gt 0 ]; do
  case "$1" in
    --skip-remote) RUN_REMOTE=0 ;;
    --skip-winrm) RUN_WINRM=0 ;;
    --skip-server-systemd) RUN_SERVER_SYSTEMD=0 ;;
    -h|--help) usage; exit 0 ;;
    *) echo "Unknown argument: $1" >&2; usage >&2; exit 64 ;;
  esac
  shift
done

pass() { OK_COUNT=$((OK_COUNT + 1)); printf "%b[OK]%b   %s\n" "$GREEN" "$NC" "$*"; }
warn() { WARN_COUNT=$((WARN_COUNT + 1)); printf "%b[WARN]%b %s\n" "$YELLOW" "$NC" "$*"; }
fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf "%b[FAIL]%b %s\n" "$RED" "$NC" "$*"; }
skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); printf "%b[SKIP]%b %s\n" "$YELLOW" "$NC" "$*"; }

section() {
  printf "\n%b== %s ==%b\n" "$CYAN" "$*" "$NC"
}

have() {
  command -v "$1" >/dev/null 2>&1
}

check_local_command() {
  local cmd="$1"
  if have "$cmd"; then
    pass "command available: $cmd"
  else
    fail "command missing: $cmd"
  fi
}

check_tcp() {
  local name="$1"
  local host="$2"
  local port="$3"
  if timeout 4 bash -c ":</dev/tcp/${host}/${port}" >/dev/null 2>&1; then
    pass "$name TCP ${host}:${port}"
  else
    fail "$name TCP ${host}:${port}"
  fi
}

check_ssh_banner() {
  local name="$1"
  local host="$2"
  local port="${3:-22}"
  local banner
  banner="$(
    timeout 5 bash -c "exec 3<>/dev/tcp/${host}/${port}; IFS= read -r line <&3; printf '%s' \"\$line\"" 2>/dev/null || true
  )"
  if printf "%s" "$banner" | grep -Eq '^SSH-[0-9]+\.[0-9]+'; then
    pass "$name SSH banner ${host}:${port} ${banner}"
  else
    fail "$name SSH banner ${host}:${port} unavailable"
  fi
}

check_http_code() {
  local name="$1"
  local url="$2"
  local expected="${3:-^2[0-9][0-9]$}"
  local tmp code
  tmp="$(mktemp)"
  code="$(curl -k -sS --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
  if printf "%s" "$code" | grep -Eq "$expected"; then
    pass "$name HTTP $code $url"
  else
    fail "$name HTTP $code $url"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

check_http_json_key() {
  local name="$1"
  local url="$2"
  local jq_filter="$3"
  local tmp
  tmp="$(mktemp)"
  if curl -k -fsS --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
    pass "$name JSON $jq_filter"
    jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/       /' | head -5
  else
    fail "$name JSON $jq_filter"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

check_http_json_key_basic_auth() {
  local name="$1"
  local url="$2"
  local jq_filter="$3"
  local tmp
  tmp="$(mktemp)"
  if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$url" -o "$tmp" 2>"$tmp.err" && jq -e "$jq_filter" "$tmp" >/dev/null 2>&1; then
    pass "$name JSON $jq_filter"
    jq -r "$jq_filter" "$tmp" 2>/dev/null | sed 's/^/       /' | head -5
  else
    fail "$name JSON $jq_filter"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

check_http_code_basic_auth() {
  local name="$1"
  local url="$2"
  local expected="${3:-^2[0-9][0-9]$}"
  local tmp code
  tmp="$(mktemp)"
  code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
  if printf "%s" "$code" | grep -Eq "$expected"; then
    pass "$name HTTP $code $url"
  else
    fail "$name HTTP $code $url"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

check_grafana_loki_proxy() {
  local name="Grafana Loki datasource proxy"
  local tmp uid code body
  tmp="$(mktemp)"
  if ! curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources" -o "$tmp" 2>"$tmp.err"; then
    fail "$name cannot list datasources"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
    rm -f "$tmp" "$tmp.err"
    return
  fi
  uid="$(jq -r '.[] | select(.type == "loki") | .uid' "$tmp" 2>/dev/null | head -1)"
  if [ -z "$uid" ]; then
    fail "$name no loki datasource found"
    jq -r '.[] | "\(.name) \(.type) \(.uid)"' "$tmp" 2>/dev/null | sed 's/^/       /' | head -20
    rm -f "$tmp" "$tmp.err"
    return
  fi
  body="$(mktemp)"
  code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$body" -w '%{http_code}' "$GRAFANA_URL/api/datasources/proxy/uid/$uid/loki/api/v1/labels" 2>"$tmp.err")"
  if [ "$code" = "200" ]; then
    pass "$name HTTP $code uid=$uid"
    jq -r '.status // .data[0] // .' "$body" 2>/dev/null | sed 's/^/       /' | head -5
  else
    fail "$name HTTP $code uid=$uid"
    sed 's/^/       /' "$tmp.err" "$body" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err" "$body"
}

check_grafana_influx_health() {
  local name="Grafana InfluxDB-AW datasource health"
  local tmp
  tmp="$(mktemp)"
  if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources/uid/influxdb_aw/health" -o "$tmp" 2>"$tmp.err" && jq -e '.status == "OK"' "$tmp" >/dev/null 2>&1; then
    pass "$name"
    jq -r '.message // .status // .' "$tmp" 2>/dev/null | sed 's/^/       /' | head -5
  else
    fail "$name"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

check_grafana_aw_main_dashboard_queries() {
  local name="Grafana detmir-aw-main worktime panel queries"
  local tmp
  tmp="$(mktemp)"
  if curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/dashboards/uid/detmir-aw-main" -o "$tmp" 2>"$tmp.err" && \
    jq -e '
      def query: (.targets[0].query // "");
      def has_panel($title; $checks):
        any(.dashboard.panels[]?; .title == $title and (query as $q | all($checks[]; $q | contains(.))));
      has_panel("Активность RDP по часам"; [
        "aw_rdp_worktime_hourly",
        "r.user_id !~ /\\$$/",
        "r.user_id !~ /�/",
        "group(columns: [\"_time\", \"user\"])",
        "max(column: \"_value\")"
      ]) and
      has_panel("Сегодня: активность по сотрудникам"; [
        "aw_rdp_worktime_daily",
        "group(columns: [\"report_date\", \"user\"])",
        "max(column: \"_value\")",
        "last()"
      ]) and
      has_panel("Все сотрудники: активное время по дням"; [
        "aw_rdp_worktime_summary_daily",
        "Все сотрудники, ч"
      ])
    ' "$tmp" >/dev/null 2>&1; then
    pass "$name"
  else
    fail "$name missing expected DetMir worktime/dedupe query contract"
    jq -r '.dashboard.panels[]? | select(.title == "Активность RDP по часам" or .title == "Сегодня: активность по сотрудникам" or .title == "Все сотрудники: активное время по дням") | "\(.title): " + ((.targets[0].query // "") | gsub("\n"; " "))' "$tmp" 2>/dev/null | sed 's/^/       /' | head -20
    sed 's/^/       /' "$tmp.err" 2>/dev/null | head -20
  fi
  rm -f "$tmp" "$tmp.err"
}

check_http_redirect() {
  local name="$1"
  local url="$2"
  local expected_code="${3:-^30[1278]$}"
  local expected_location="${4:-}"
  local tmp code location
  tmp="$(mktemp)"
  code="$(curl -k -sS -I --connect-timeout 5 --max-time 15 -o "$tmp" -w '%{http_code}' "$url" 2>"$tmp.err")"
  location="$(awk 'BEGIN{IGNORECASE=1} /^location:/ {sub(/\r$/,""); print $0}' "$tmp" | tail -1)"
  if printf "%s" "$code" | grep -Eq "$expected_code" && { [ -z "$expected_location" ] || printf "%s" "$location" | grep -Fq "$expected_location"; }; then
    pass "$name HTTP $code ${location:-$url}"
  else
    fail "$name HTTP $code ${location:-$url}"
    sed 's/^/       /' "$tmp.err" "$tmp" 2>/dev/null | head -40
  fi
  rm -f "$tmp" "$tmp.err"
}

ansible_shell() {
  local group="$1"
  local command="$2"
  ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m shell -a "$command"
}

ansible_win_shell() {
  local group="$1"
  local command="$2"
  ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m win_shell -a "$command"
}

check_ansible_shell() {
  local name="$1"
  local group="$2"
  local command="$3"
  local tmp
  tmp="$(mktemp)"
  if ansible_shell "$group" "$command" >"$tmp" 2>&1; then
    pass "$name"
    sed 's/^/       /' "$tmp" | head -80
  else
    fail "$name"
    sed 's/^/       /' "$tmp" | head -120
  fi
  rm -f "$tmp"
}

check_ansible_win_shell() {
  local name="$1"
  local group="$2"
  local command="$3"
  local tmp
  tmp="$(mktemp)"
  if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
    pass "$name"
    sed 's/^/       /' "$tmp" | head -80
  else
    fail "$name"
    sed 's/^/       /' "$tmp" | head -120
  fi
  rm -f "$tmp"
}

check_ansible_win_shell_warn() {
  local name="$1"
  local group="$2"
  local command="$3"
  local tmp
  tmp="$(mktemp)"
  if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
    pass "$name"
    sed 's/^/       /' "$tmp" | head -80
  else
    warn "$name returned non-zero"
    sed 's/^/       /' "$tmp" | head -120
  fi
  rm -f "$tmp"
}

check_ansible_module() {
  local name="$1"
  local group="$2"
  local module="$3"
  local args="${4:-}"
  local tmp
  tmp="$(mktemp)"
  if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then
    pass "$name"
    sed 's/^/       /' "$tmp" | head -80
  else
    fail "$name"
    sed 's/^/       /' "$tmp" | head -120
  fi
  rm -f "$tmp"
}

classify_bucket_age() {
  local bucket="$1"
  local age_sec="$2"
  case "$bucket" in
    aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
      if [ "$age_sec" -lt 86400 ]; then
        printf "fresh"
      else
        printf "event-driven"
      fi
      ;;
    *)
      if [ "$age_sec" -lt 3600 ]; then
        printf "fresh"
      elif [ "$age_sec" -lt 86400 ]; then
        printf "stale"
      else
        printf "dead"
      fi
      ;;
  esac
}

check_bucket_freshness() {
  local bucket="$1"
  local bucket_id="${bucket}_${AW_SOURCE_HOSTNAME}"
  local tmp last_ts last_id event_epoch now age_sec status
  tmp="$(mktemp)"
  if ! curl -fsS --connect-timeout 5 --max-time 20 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then
    fail "bucket $bucket_id query failed"
    sed 's/^/       /' "$tmp.err" | head -20
    rm -f "$tmp" "$tmp.err"
    return
  fi

  last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)"
  last_id="$(jq -r '.[0].id // 0' "$tmp" 2>/dev/null)"
  rm -f "$tmp" "$tmp.err"

  if [ -z "$last_ts" ]; then
    case "$bucket" in
      aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
        warn "bucket $bucket_id empty/event-driven"
        ;;
      *)
        fail "bucket $bucket_id empty"
        ;;
    esac
    return
  fi

  event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || printf "0")"
  now="$(date -u +%s)"
  if [ "$event_epoch" -le 0 ]; then
    warn "bucket $bucket_id has unparsable timestamp: $last_ts"
    return
  fi
  age_sec=$((now - event_epoch))
  status="$(classify_bucket_age "$bucket" "$age_sec")"
  case "$status" in
    fresh|event-driven)
      pass "bucket $bucket_id $status age=${age_sec}s id=$last_id"
      ;;
    stale)
      warn "bucket $bucket_id stale age=${age_sec}s id=$last_id"
      ;;
    *)
      fail "bucket $bucket_id dead age=${age_sec}s id=$last_id"
      ;;
  esac
}

run_remote_proxmox_script() {
  if [ "$RUN_REMOTE" != "1" ]; then
    skip "remote 10.10.10.2 smoke skipped"
    return
  fi
  if ! have ansible; then
    fail "ansible unavailable; cannot deploy/run remote script"
    return
  fi
  if [ ! -f "$REMOTE_SCRIPT_SRC" ]; then
    fail "remote script source missing: $REMOTE_SCRIPT_SRC"
    return
  fi

  section "Deploy Remote Script To 10.10.10.2"
  if [ -x "$REMOTE_RUST_SRC" ]; then
    if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_RUST_SRC dest=$REMOTE_RUST_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy-rust.$$ 2>&1; then
      pass "remote Rust smoke deployed to $REMOTE_RUST_DST"
    else
      warn "remote Rust smoke deploy failed; shell fallback remains available"
      sed 's/^/       /' /tmp/aw-smoke-copy-rust.$$ | head -120
    fi
    rm -f /tmp/aw-smoke-copy-rust.$$
  else
    skip "remote Rust smoke binary not found: $REMOTE_RUST_SRC"
  fi

  if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_SCRIPT_SRC dest=$REMOTE_SCRIPT_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy.$$ 2>&1; then
    pass "remote script deployed to $REMOTE_SCRIPT_DST"
  else
    fail "remote script deploy failed"
    sed 's/^/       /' /tmp/aw-smoke-copy.$$ | head -120
    rm -f /tmp/aw-smoke-copy.$$
    return
  fi
  rm -f /tmp/aw-smoke-copy.$$

  section "Remote 10.10.10.2 Smoke"
  local tmp
  tmp="$(mktemp)"
  if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m shell -a "$REMOTE_SCRIPT_DST" >"$tmp" 2>&1; then
    pass "remote 10.10.10.2 smoke completed"
    sed 's/^/       /' "$tmp"
  else
    fail "remote 10.10.10.2 smoke failed"
    sed 's/^/       /' "$tmp"
  fi
  rm -f "$tmp"
}

mkdir -p "$LOG_DIR"
LOG_FILE="$LOG_DIR/aw-contour-smoke-$(date +%Y%m%d-%H%M%S).log"
exec > >(tee "$LOG_FILE") 2>&1

section "Run Context"
printf "repo=%s\ninventory=%s\nlog=%s\n" "$REPO_ROOT" "$INVENTORY" "$LOG_FILE" | sed 's/^/       /'
date -Is | sed 's/^/       /'

section "Local Prerequisites"
for cmd in bash curl jq timeout ansible ssh; do
  check_local_command "$cmd"
done
printf "       no_proxy=%s\n" "$no_proxy"

section "Laptop Network"
ip -br addr 2>/dev/null | sed 's/^/       /' || true
ip route get "$PROXMOX_HOST" 2>/dev/null | sed 's/^/       /' || warn "no route detail for $PROXMOX_HOST"
ip route get "$AW_HOST" 2>/dev/null | sed 's/^/       /' || warn "no route detail for $AW_HOST"

section "TCP Surface"
check_tcp "Proxmox SSH" "$PROXMOX_HOST" 22
check_tcp "Proxmox HTTP" "$PROXMOX_HOST" 80
check_tcp "Proxmox HTTPS" "$PROXMOX_HOST" 443
check_tcp "Proxmox GUI" "$PROXMOX_HOST" 8006
check_tcp "1C company API" "$PROXMOX_HOST" 8710
check_tcp "AW server" "$AW_HOST" 5600
check_tcp "AW worktime API" "$AW_HOST" 5610
check_tcp "Grafana" "$GRAFANA_HOST" 3000
check_tcp "Windows WinRM" "$WINDOWS_HOST" 5985
check_tcp "Windows SSH" "$WINDOWS_HOST" 22
check_ssh_banner "Windows SSH" "$WINDOWS_HOST" 22

section "ActivityWatch HTTP"
check_http_json_key "AW server info" "$AW_SERVER/api/0/info" '.version'
check_http_code "AW settings CORS" "$AW_SERVER/api/0/settings/" '^200$'
check_http_code "AW WebUI" "$AW_SERVER/" '^200$'
check_http_json_key "AW buckets list" "$AW_SERVER/api/0/buckets/" 'keys | length'

section "ActivityWatch Buckets"
for bucket in \
  aw-watcher-afk \
  aw-watcher-window \
  aw-worktime-sessions \
  aw-session-events \
  aw-dlp-endpoint-signals \
  aw-dlp-incidents \
  aw-dlp-review \
  aw-dlp-rules
do
  check_bucket_freshness "$bucket"
done

section "Worktime API"
check_http_json_key "worktime health" "$WORKTIME_API/health" '.status // .ok // .'
check_http_code "worktime today html" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$'
check_http_json_key "worktime today json" "$WORKTIME_API/reports/worktime/today?host=$AW_SOURCE_HOSTNAME&day=today" '.host // .report.host // .[0].host // .'
check_http_code "worktime management html" "$WORKTIME_API/reports/worktime/management?host=$AW_SOURCE_HOSTNAME&day=today&format=html" '^200$'

section "Gateway And 1C HTTP"
check_http_code "gateway healthz" "https://$PROXMOX_HOST/healthz" '^200$'
check_http_code "gateway proxmox protected" "https://$PROXMOX_HOST/go/proxmox-gui" '^401$'
check_http_code "gateway file1c brief protected" "https://$PROXMOX_HOST/go/file1c-brief" '^401$'
check_http_code "gateway file1c actions protected" "https://$PROXMOX_HOST/go/file1c-actions" '^401$'
check_http_code "1C /health" "http://$PROXMOX_HOST:8710/health" '^200$'
check_http_code "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" '^200$'
check_http_code "1C manager brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$'
check_http_code "1C manager actions" "http://$PROXMOX_HOST:8710/manager/actions" '^200$'
check_http_code "1C manager recovery" "http://$PROXMOX_HOST:8710/manager/recovery" '^200$'
check_http_code "1C weekly digest" "http://$PROXMOX_HOST:8710/manager/digest/weekly" '^200$'

section "Grafana HTTP"
check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" '.database // .version // .commit'
check_http_code "Grafana dashboards page" "$GRAFANA_URL/dashboards" '^200$|^302$'
check_http_code "Grafana pfSense dashboard" "$GRAFANA_URL/d/pfsense-loki-dashboard/pfsense-firewall-overview?orgId=1&from=now-1h&to=now&timezone=browser" '^200$|^302$'

if [ -n "${GRAFANA_USER:-}" ] && [ -n "${GRAFANA_PASSWORD:-}" ]; then
  section "Grafana Authenticated API"
  check_http_json_key_basic_auth "Grafana datasources" "$GRAFANA_URL/api/datasources" 'length'
  check_grafana_loki_proxy
  check_grafana_influx_health
  check_grafana_aw_main_dashboard_queries
else
  skip "Grafana authenticated datasource checks need GRAFANA_USER and GRAFANA_PASSWORD env"
fi

if [ "$RUN_SERVER_SYSTEMD" = "1" ] && have ansible; then
  section "AW Server Systemd"
  check_ansible_shell "AW server core units" aw_server 'systemctl is-active activitywatch-server aw-worktime-api aw-worktime-ui-bridge.timer aw-rus-healthd.timer aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer aw-worktime-autoheal.timer'
  check_ansible_shell "AW server failed units" aw_server 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|influx|grafana|prometheus|loki" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
  check_ansible_shell "AW server local health script" aw_server 'test -x /opt/activitywatch/health-check.sh && /opt/activitywatch/health-check.sh || test -x /usr/local/bin/health-check.sh && /usr/local/bin/health-check.sh || echo "health-check script not installed"'
else
  skip "AW server systemd checks skipped"
fi

if [ "$RUN_WINRM" = "1" ] && have ansible; then
  section "Windows WinRM And Collectors"
  check_ansible_module "Windows win_ping" aw_windows win_ping
  check_ansible_win_shell "Windows sessions" aw_windows '$psi = [System.Diagnostics.ProcessStartInfo]::new(); $psi.FileName = "$env:SystemRoot\System32\query.exe"; $psi.Arguments = "user"; $psi.UseShellExecute = $false; $psi.RedirectStandardOutput = $true; $psi.RedirectStandardError = $true; $psi.StandardOutputEncoding = [System.Text.Encoding]::GetEncoding(866); $psi.StandardErrorEncoding = [System.Text.Encoding]::GetEncoding(866); $p = [System.Diagnostics.Process]::Start($psi); $out = $p.StandardOutput.ReadToEnd(); $err = $p.StandardError.ReadToEnd(); $p.WaitForExit(); [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $out; if ($err) { $err }; if ($out -match "USERNAME|ПОЛЬЗОВАТЕЛЬ|администратор|Администратор") { exit 0 } else { exit $p.ExitCode }'
  check_ansible_win_shell_warn "Windows collector processes" aw_windows '$p = Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue; if ($p) { $p | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize } else { "no aw-watcher-afk/window process visible to this WinRM session" }'
  check_ansible_win_shell "Windows ActivityWatch tasks" aw_windows 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V; schtasks /Query /TN "ActivityWatch Launch [SHARKON2025_Администратор]" /FO LIST /V'
else
  skip "Windows WinRM checks skipped"
fi

run_remote_proxmox_script

section "Existing Repo Checks"
if [ -x "$REPO_ROOT/check-aw-data.sh" ]; then
  if "$REPO_ROOT/check-aw-data.sh"; then pass "check-aw-data.sh completed"; else fail "check-aw-data.sh failed"; fi
else
  skip "check-aw-data.sh missing"
fi
if [ -x "$REPO_ROOT/check-aw-full.sh" ]; then
  if "$REPO_ROOT/check-aw-full.sh"; then pass "check-aw-full.sh completed"; else fail "check-aw-full.sh failed"; fi
else
  skip "check-aw-full.sh missing"
fi

section "Summary"
printf "OK=%s WARN=%s FAIL=%s SKIP=%s\n" "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" "$SKIP_COUNT"
printf "Log: %s\n" "$LOG_FILE"

if [ "$FAIL_COUNT" -gt 0 ]; then
  exit 2
fi
exit 0