fix(1c): harden post-ingest manager brief refresh

2026-05-22 16:34:53 +03:00
parent 5670ecd84b
commit 4f1b935bdc
6 changed files with 51 additions and 7 deletions
@@ -6,6 +6,8 @@ Requires=docker.service
 [Service]
 Type=oneshot
 WorkingDirectory=/opt/activitywatch/clickhouse-1c
+Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh
+TimeoutStartSec=45min
 User=root
 Group=root
@@ -7,6 +7,7 @@ Wants=network-online.target
 Type=oneshot
 Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_manager_brief.sh
+TimeoutStartSec=30min

 [Install]
 WantedBy=multi-user.target
@@ -7,6 +7,7 @@ Wants=network-online.target
 Type=oneshot
 Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_weekly_digest.sh
+TimeoutStartSec=45min

 [Install]
 WantedBy=multi-user.target
@@ -7,6 +7,8 @@ VENV="${ROOT}/.venv"
 CONFIG="${ROOT}/etl/config.yml"
 CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
 LOCK_FILE="${ROOT}/.ingest.lock"
+RUN_MANAGER_BRIEF_AFTER_INGEST="${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-1}"
+RUN_PROOFCHECK_AFTER_INGEST="${AW_1C_PROOFCHECK_RUN_AFTER_INGEST:-1}"

 if [[ ! -f "${ENV_FILE}" ]]; then
  echo "missing env file: ${ENV_FILE}" >&2
@@ -66,8 +68,14 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
  --database "${CLICKHOUSE_DB}" \
  < "${ROOT}/detections/open_cases_from_detections.sql"

-if [[ "${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-0}" == "1" ]]; then
+if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then
  if ! "${ROOT}/ops/run_manager_brief.sh"; then
    echo "warning: manager brief refresh failed after ingest" >&2
  fi
 fi
+
+if [[ "${RUN_PROOFCHECK_AFTER_INGEST}" == "1" ]]; then
+  if ! "${ROOT}/ops/check_ingest_freshness.sh"; then
+    echo "warning: freshness proof check failed after ingest" >&2
+  fi
+fi
@@ -4,6 +4,10 @@ set -euo pipefail
 ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
 ENV_FILE="${ROOT}/.env"
 VENV="${ROOT}/.venv"
+LOCK_FILE="${ROOT}/.manager-brief.lock"
+RETRIES="${AW_1C_MANAGER_BRIEF_RETRIES:-3}"
+RETRY_DELAY_SEC="${AW_1C_MANAGER_BRIEF_RETRY_DELAY_SEC:-20}"
+LOCK_WAIT_SEC="${AW_1C_MANAGER_BRIEF_LOCK_WAIT_SEC:-1200}"

 if [[ ! -f "${ENV_FILE}" ]]; then
  echo "missing env file: ${ENV_FILE}" >&2
@@ -27,9 +31,29 @@ fi

 export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}"

-exec "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \
-  --host "${CH_RUNTIME_HOST}" \
-  --port "${CLICKHOUSE_PORT}" \
-  --user "${CLICKHOUSE_USER}" \
-  --password "${CLICKHOUSE_PASSWORD}" \
-  --database "${CLICKHOUSE_DB}"
+exec 9>"${LOCK_FILE}"
+if ! flock -w "${LOCK_WAIT_SEC}" 9; then
+  echo "manager brief lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2
+  exit 1
+fi
+
+attempt=1
+while (( attempt <= RETRIES )); do
+  if "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \
+    --host "${CH_RUNTIME_HOST}" \
+    --port "${CLICKHOUSE_PORT}" \
+    --user "${CLICKHOUSE_USER}" \
+    --password "${CLICKHOUSE_PASSWORD}" \
+    --database "${CLICKHOUSE_DB}"; then
+    exit 0
+  fi
+  if (( attempt == RETRIES )); then
+    break
+  fi
+  echo "manager brief attempt ${attempt}/${RETRIES} failed, retrying in ${RETRY_DELAY_SEC}s" >&2
+  sleep "${RETRY_DELAY_SEC}"
+  attempt=$((attempt + 1))
+done
+
+echo "manager brief failed after ${RETRIES} attempts" >&2
+exit 1
@@ -4,6 +4,8 @@ set -euo pipefail
 ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
 ENV_FILE="${ROOT}/.env"
 VENV="${ROOT}/.venv"
+LOCK_FILE="${ROOT}/.weekly-digest.lock"
+LOCK_WAIT_SEC="${AW_1C_WEEKLY_DIGEST_LOCK_WAIT_SEC:-1800}"

 if [[ ! -f "${ENV_FILE}" ]]; then
  echo "missing env file: ${ENV_FILE}" >&2
@@ -20,4 +22,10 @@ set -a
 . "${ENV_FILE}"
 set +a

+exec 9>"${LOCK_FILE}"
+if ! flock -w "${LOCK_WAIT_SEC}" 9; then
+  echo "weekly digest lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2
+  exit 1
+fi
+
 exec "${VENV}/bin/python" "${ROOT}/ai/generate_weekly_digest.py"