feat(1c): add codex recovery brief layer
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=AW-rus 1C Recovery Brief
|
||||
After=network-online.target aw-1c-manager-brief.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
|
||||
ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_recovery_brief.sh
|
||||
TimeoutStartSec=45min
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Run AW-rus 1C recovery brief every 6 hours
|
||||
|
||||
[Timer]
|
||||
OnBootSec=45min
|
||||
OnUnitActiveSec=6h
|
||||
Unit=aw-1c-recovery-brief.service
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -8,6 +8,7 @@ CONFIG="${ROOT}/etl/config.yml"
|
||||
CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
|
||||
LOCK_FILE="${ROOT}/.ingest.lock"
|
||||
RUN_MANAGER_BRIEF_AFTER_INGEST="${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-1}"
|
||||
RUN_RECOVERY_BRIEF_AFTER_INGEST="${AW_1C_RECOVERY_BRIEF_RUN_AFTER_INGEST:-1}"
|
||||
RUN_PROOFCHECK_AFTER_INGEST="${AW_1C_PROOFCHECK_RUN_AFTER_INGEST:-1}"
|
||||
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
@@ -74,6 +75,12 @@ if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${RUN_RECOVERY_BRIEF_AFTER_INGEST}" == "1" ]]; then
|
||||
if ! "${ROOT}/ops/run_recovery_brief.sh"; then
|
||||
echo "warning: recovery brief refresh failed after ingest" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${RUN_PROOFCHECK_AFTER_INGEST}" == "1" ]]; then
|
||||
if ! "${ROOT}/ops/check_ingest_freshness.sh"; then
|
||||
echo "warning: freshness proof check failed after ingest" >&2
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
|
||||
ENV_FILE="${ROOT}/.env"
|
||||
VENV="${ROOT}/.venv"
|
||||
LOCK_FILE="${ROOT}/.recovery-brief.lock"
|
||||
LOCK_WAIT_SEC="${AW_1C_RECOVERY_BRIEF_LOCK_WAIT_SEC:-1800}"
|
||||
RETRIES="${AW_1C_RECOVERY_BRIEF_RETRIES:-2}"
|
||||
RETRY_DELAY_SEC="${AW_1C_RECOVERY_BRIEF_RETRY_DELAY_SEC:-20}"
|
||||
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
echo "missing env file: ${ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -x "${VENV}/bin/python" ]]; then
|
||||
echo "missing venv python: ${VENV}/bin/python" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
. "${ENV_FILE}"
|
||||
set +a
|
||||
|
||||
CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
|
||||
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
|
||||
CH_RUNTIME_HOST="127.0.0.1"
|
||||
fi
|
||||
|
||||
exec 9>"${LOCK_FILE}"
|
||||
if ! flock -w "${LOCK_WAIT_SEC}" 9; then
|
||||
echo "recovery brief lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
attempt=1
|
||||
while (( attempt <= RETRIES )); do
|
||||
if "${VENV}/bin/python" "${ROOT}/ai/generate_recovery_brief.py" \
|
||||
--host "${CH_RUNTIME_HOST}" \
|
||||
--port "${CLICKHOUSE_PORT}" \
|
||||
--user "${CLICKHOUSE_USER}" \
|
||||
--password "${CLICKHOUSE_PASSWORD}" \
|
||||
--database "${CLICKHOUSE_DB}"; then
|
||||
exit 0
|
||||
fi
|
||||
if (( attempt == RETRIES )); then
|
||||
break
|
||||
fi
|
||||
echo "recovery brief attempt ${attempt}/${RETRIES} failed, retrying in ${RETRY_DELAY_SEC}s" >&2
|
||||
sleep "${RETRY_DELAY_SEC}"
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
|
||||
echo "recovery brief failed after ${RETRIES} attempts" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user