feat(1c): automate live file telemetry ingestion

2026-05-22 08:09:00 +03:00
parent 41f7a869e5
commit 5c6c23ba0f
19 changed files with 1070 additions and 100 deletions
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
+ENV_FILE="${ROOT}/.env"
+VENV="${ROOT}/.venv"
+CONFIG="${ROOT}/etl/config.yml"
+CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
+
+if [[ ! -f "${ENV_FILE}" ]]; then
+  echo "missing env file: ${ENV_FILE}" >&2
+  exit 1
+fi
+
+if [[ ! -f "${CONFIG}" ]]; then
+  echo "missing etl config: ${CONFIG}" >&2
+  exit 1
+fi
+
+if [[ ! -x "${VENV}/bin/python" ]]; then
+  echo "missing venv python: ${VENV}/bin/python" >&2
+  exit 1
+fi
+
+if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then
+  echo "clickhouse container not running: ${CH_CONTAINER}" >&2
+  exit 1
+fi
+
+# shellcheck disable=SC1090
+. "${ENV_FILE}"
+
+"${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/build_entity_timeline.sql"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/insert_detections.sql"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/open_cases_from_detections.sql"