#!/usr/bin/env bash # shellcheck disable=SC2119 set -euo pipefail ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" ENV_FILE="${ROOT}/.env" VENV="${ROOT}/.venv" CONFIG="${ROOT}/etl/config.yml" CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}" LOCK_FILE="${ROOT}/.ingest.lock" RUN_MANAGER_BRIEF_AFTER_INGEST="${AW_1C_MANAGER_BRIEF_RUN_AFTER_INGEST:-1}" RUN_RECOVERY_BRIEF_AFTER_INGEST="${AW_1C_RECOVERY_BRIEF_RUN_AFTER_INGEST:-1}" RUN_PROOFCHECK_AFTER_INGEST="${AW_1C_PROOFCHECK_RUN_AFTER_INGEST:-1}" RUN_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST="${AW_1C_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST:-0}" if [[ ! -f "${ENV_FILE}" ]]; then echo "missing env file: ${ENV_FILE}" >&2 exit 1 fi if [[ ! -f "${CONFIG}" ]]; then echo "missing etl config: ${CONFIG}" >&2 exit 1 fi if [[ ! -x "${VENV}/bin/python" ]]; then echo "missing venv python: ${VENV}/bin/python" >&2 exit 1 fi if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then echo "clickhouse container not running: ${CH_CONTAINER}" >&2 exit 1 fi exec 9>"${LOCK_FILE}" if ! flock -n 9; then echo "ingest cycle already running" >&2 exit 0 fi # shellcheck disable=SC1090 . "${ENV_FILE}" # shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh . "${ROOT}/ops/clickhouse-client-safe.sh" if [[ "${RUN_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST}" == "1" ]]; then "${VENV}/bin/python" "${ROOT}/etl/extract_1c_mcp_toolkit.py" --config "${CONFIG}" fi "${VENV}/bin/python" "${ROOT}/etl/build_business_event_exports.py" --config "${CONFIG}" "${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}" "${VENV}/bin/python" "${ROOT}/etl/load_company_registry_xlsx.py" --config "${CONFIG}" --landing "${ROOT}/landing/registry" aw_1c_clickhouse_client \ < "${ROOT}/detections/build_entity_timeline.sql" aw_1c_clickhouse_client \ < "${ROOT}/clickhouse/init/04_company_intelligence.sql" "${ROOT}/ops/run_company_registry_bindings_refresh.sh" "${ROOT}/ops/run_company_intelligence_refresh.sh" aw_1c_clickhouse_client \ < "${ROOT}/detections/insert_detections.sql" aw_1c_clickhouse_client \ < "${ROOT}/detections/open_cases_from_detections.sql" if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then if ! "${ROOT}/ops/run_manager_brief.sh"; then echo "warning: manager brief refresh failed after ingest" >&2 fi fi if [[ "${RUN_RECOVERY_BRIEF_AFTER_INGEST}" == "1" ]]; then if ! "${ROOT}/ops/run_recovery_brief.sh"; then echo "warning: recovery brief refresh failed after ingest" >&2 fi fi if [[ "${RUN_PROOFCHECK_AFTER_INGEST}" == "1" ]]; then if ! "${ROOT}/ops/check_ingest_freshness.sh"; then echo "warning: freshness proof check failed after ingest" >&2 fi fi