diff --git a/clickhouse-1c/ops/aw-1c-ingest.service b/clickhouse-1c/ops/aw-1c-ingest.service index f4e1993..70ac3d6 100644 --- a/clickhouse-1c/ops/aw-1c-ingest.service +++ b/clickhouse-1c/ops/aw-1c-ingest.service @@ -6,6 +6,8 @@ Requires=docker.service [Service] Type=oneshot WorkingDirectory=/opt/activitywatch/clickhouse-1c +Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh +TimeoutStartSec=45min User=root Group=root diff --git a/clickhouse-1c/ops/aw-1c-manager-brief.service b/clickhouse-1c/ops/aw-1c-manager-brief.service index cf2d35e..57f28ea 100644 --- a/clickhouse-1c/ops/aw-1c-manager-brief.service +++ b/clickhouse-1c/ops/aw-1c-manager-brief.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=oneshot Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_manager_brief.sh +TimeoutStartSec=30min [Install] WantedBy=multi-user.target diff --git a/clickhouse-1c/ops/aw-1c-weekly-digest.service b/clickhouse-1c/ops/aw-1c-weekly-digest.service index e50ca17..b20f729 100644 --- a/clickhouse-1c/ops/aw-1c-weekly-digest.service +++ b/clickhouse-1c/ops/aw-1c-weekly-digest.service @@ -7,6 +7,7 @@ Wants=network-online.target Type=oneshot Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_weekly_digest.sh +TimeoutStartSec=45min [Install] WantedBy=multi-user.target diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh index a376845..8bed3ca 100644 --- a/clickhouse-1c/ops/run_ingest_cycle.sh +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -7,6 +7,8 @@ VENV="${ROOT}/.venv" CONFIG="${ROOT}/etl/config.yml" CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}" LOCK_FILE="${ROOT}/.ingest.lock" +RUN_MANAGER_BRIEF_AFTER_INGEST="${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-1}" +RUN_PROOFCHECK_AFTER_INGEST="${AW_1C_PROOFCHECK_RUN_AFTER_INGEST:-1}" if [[ ! -f "${ENV_FILE}" ]]; then echo "missing env file: ${ENV_FILE}" >&2 @@ -66,8 +68,14 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \ --database "${CLICKHOUSE_DB}" \ < "${ROOT}/detections/open_cases_from_detections.sql" -if [[ "${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-0}" == "1" ]]; then +if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then if ! "${ROOT}/ops/run_manager_brief.sh"; then echo "warning: manager brief refresh failed after ingest" >&2 fi fi + +if [[ "${RUN_PROOFCHECK_AFTER_INGEST}" == "1" ]]; then + if ! "${ROOT}/ops/check_ingest_freshness.sh"; then + echo "warning: freshness proof check failed after ingest" >&2 + fi +fi diff --git a/clickhouse-1c/ops/run_manager_brief.sh b/clickhouse-1c/ops/run_manager_brief.sh index 34ebc9a..b6feedb 100755 --- a/clickhouse-1c/ops/run_manager_brief.sh +++ b/clickhouse-1c/ops/run_manager_brief.sh @@ -4,6 +4,10 @@ set -euo pipefail ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" ENV_FILE="${ROOT}/.env" VENV="${ROOT}/.venv" +LOCK_FILE="${ROOT}/.manager-brief.lock" +RETRIES="${AW_1C_MANAGER_BRIEF_RETRIES:-3}" +RETRY_DELAY_SEC="${AW_1C_MANAGER_BRIEF_RETRY_DELAY_SEC:-20}" +LOCK_WAIT_SEC="${AW_1C_MANAGER_BRIEF_LOCK_WAIT_SEC:-1200}" if [[ ! -f "${ENV_FILE}" ]]; then echo "missing env file: ${ENV_FILE}" >&2 @@ -27,9 +31,29 @@ fi export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}" -exec "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \ - --host "${CH_RUNTIME_HOST}" \ - --port "${CLICKHOUSE_PORT}" \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" +exec 9>"${LOCK_FILE}" +if ! flock -w "${LOCK_WAIT_SEC}" 9; then + echo "manager brief lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2 + exit 1 +fi + +attempt=1 +while (( attempt <= RETRIES )); do + if "${VENV}/bin/python" "${ROOT}/ai/generate_manager_brief.py" \ + --host "${CH_RUNTIME_HOST}" \ + --port "${CLICKHOUSE_PORT}" \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}"; then + exit 0 + fi + if (( attempt == RETRIES )); then + break + fi + echo "manager brief attempt ${attempt}/${RETRIES} failed, retrying in ${RETRY_DELAY_SEC}s" >&2 + sleep "${RETRY_DELAY_SEC}" + attempt=$((attempt + 1)) +done + +echo "manager brief failed after ${RETRIES} attempts" >&2 +exit 1 diff --git a/clickhouse-1c/ops/run_weekly_digest.sh b/clickhouse-1c/ops/run_weekly_digest.sh index b2f02c0..1c5d481 100755 --- a/clickhouse-1c/ops/run_weekly_digest.sh +++ b/clickhouse-1c/ops/run_weekly_digest.sh @@ -4,6 +4,8 @@ set -euo pipefail ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" ENV_FILE="${ROOT}/.env" VENV="${ROOT}/.venv" +LOCK_FILE="${ROOT}/.weekly-digest.lock" +LOCK_WAIT_SEC="${AW_1C_WEEKLY_DIGEST_LOCK_WAIT_SEC:-1800}" if [[ ! -f "${ENV_FILE}" ]]; then echo "missing env file: ${ENV_FILE}" >&2 @@ -20,4 +22,10 @@ set -a . "${ENV_FILE}" set +a +exec 9>"${LOCK_FILE}" +if ! flock -w "${LOCK_WAIT_SEC}" 9; then + echo "weekly digest lock wait exceeded: ${LOCK_WAIT_SEC}s" >&2 + exit 1 +fi + exec "${VENV}/bin/python" "${ROOT}/ai/generate_weekly_digest.py"