fix(1c): harden post-ingest manager brief refresh

clickhouse-1c/ops/aw-1c-ingest.service

@@ -6,6 +6,8 @@ Requires=docker.service
 [Service]
 Type=oneshot
 WorkingDirectory=/opt/activitywatch/clickhouse-1c
+Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh
+TimeoutStartSec=45min
 User=root
 Group=root

clickhouse-1c/ops/aw-1c-manager-brief.service

@@ -7,6 +7,7 @@ Wants=network-online.target
 Type=oneshot
 Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_manager_brief.sh
+TimeoutStartSec=30min
 
 [Install]
 WantedBy=multi-user.target

clickhouse-1c/ops/aw-1c-weekly-digest.service

@@ -7,6 +7,7 @@ Wants=network-online.target
 Type=oneshot
 Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
 ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_weekly_digest.sh
+TimeoutStartSec=45min
 
 [Install]
 WantedBy=multi-user.target

clickhouse-1c/ops/run_ingest_cycle.sh

@@ -7,6 +7,8 @@ VENV="${ROOT}/.venv"
 CONFIG="${ROOT}/etl/config.yml"
 CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
 LOCK_FILE="${ROOT}/.ingest.lock"
+RUN_MANAGER_BRIEF_AFTER_INGEST="${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-1}"
+RUN_PROOFCHECK_AFTER_INGEST="${AW_1C_PROOFCHECK_RUN_AFTER_INGEST:-1}"
 
 if [[ ! -f "${ENV_FILE}" ]]; then
   echo "missing env file: ${ENV_FILE}" >&2
@@ -66,8 +68,14 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
   --database "${CLICKHOUSE_DB}" \
   < "${ROOT}/detections/open_cases_from_detections.sql"
 
-if [[ "${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-0}" == "1" ]]; then
+if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then
   if ! "${ROOT}/ops/run_manager_brief.sh"; then
     echo "warning: manager brief refresh failed after ingest" >&2
   fi
 fi
+
+if [[ "${RUN_PROOFCHECK_AFTER_INGEST}" == "1" ]]; then
+  if ! "${ROOT}/ops/check_ingest_freshness.sh"; then
+    echo "warning: freshness proof check failed after ingest" >&2
+  fi
+fi

clickhouse-1c/ops/run_manager_brief.sh

@@ -4,6 +4,10 @@ set -euo pipefail
 ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
 ENV_FILE="${ROOT}/.env"
 VENV="${ROOT}/.venv"
+LOCK_FILE="${ROOT}/.manager-brief.lock"
+RETRIES="${AW_1C_MANAGER_BRIEF_RETRIES:-3}"
+RETRY_DELAY_SEC="${AW_1C_MANAGER_BRIEF_RETRY_DELAY_SEC:-20}"
+LOCK_WAIT_SEC="${AW_1C_MANAGER_BRIEF_LOCK_WAIT_SEC:-1200}"
 
 if [[ ! -f "${ENV_FILE}" ]]; then
   echo "missing env file: ${ENV_FILE}" >&2
@@ -27,9 +31,29 @@ fi
 
 export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}"
 
-exec "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \
+exec 9>"${LOCK_FILE}"
-  --host "${CH_RUNTIME_HOST}" \
+if ! flock -w "${LOCK_WAIT_SEC}" 9; then
-  --port "${CLICKHOUSE_PORT}" \
+  echo "manager brief lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2
-  --user "${CLICKHOUSE_USER}" \
+  exit 1
-  --password "${CLICKHOUSE_PASSWORD}" \
+fi
-  --database "${CLICKHOUSE_DB}"
+
+attempt=1
+while (( attempt <= RETRIES )); do
+  if "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \
+    --host "${CH_RUNTIME_HOST}" \
+    --port "${CLICKHOUSE_PORT}" \
+    --user "${CLICKHOUSE_USER}" \
+    --password "${CLICKHOUSE_PASSWORD}" \
+    --database "${CLICKHOUSE_DB}"; then
+    exit 0
+  fi
+  if (( attempt == RETRIES )); then
+    break
+  fi
+  echo "manager brief attempt ${attempt}/${RETRIES} failed, retrying in ${RETRY_DELAY_SEC}s" >&2
+  sleep "${RETRY_DELAY_SEC}"
+  attempt=$((attempt + 1))
+done
+
+echo "manager brief failed after ${RETRIES} attempts" >&2
+exit 1

clickhouse-1c/ops/run_weekly_digest.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
 ENV_FILE="${ROOT}/.env"
 VENV="${ROOT}/.venv"
+LOCK_FILE="${ROOT}/.weekly-digest.lock"
+LOCK_WAIT_SEC="${AW_1C_WEEKLY_DIGEST_LOCK_WAIT_SEC:-1800}"
 
 if [[ ! -f "${ENV_FILE}" ]]; then
   echo "missing env file: ${ENV_FILE}" >&2
@@ -20,4 +22,10 @@ set -a
 . "${ENV_FILE}"
 set +a
 
+exec 9>"${LOCK_FILE}"
+if ! flock -w "${LOCK_WAIT_SEC}" 9; then
+  echo "weekly digest lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2
+  exit 1
+fi
+
 exec "${VENV}/bin/python" "${ROOT}/ai/generate_weekly_digest.py"