feat(1c): automate live file telemetry ingestion

2026-05-22 08:09:00 +03:00
parent 41f7a869e5
commit 5c6c23ba0f
19 changed files with 1070 additions and 100 deletions
@@ -0,0 +1,11 @@
+[Unit]
+Description=AW-rus file-1C ingest cycle
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/activitywatch/clickhouse-1c
+ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh
+User=root
+Group=root
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run AW-rus file-1C ingest cycle every 15 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=15min
+Unit=aw-1c-ingest.service
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,9 @@
+[Unit]
+Description=AW-rus file-1C ingest freshness proof check
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c
+ExecStart=/opt/activitywatch/clickhouse-1c/ops/check_ingest_freshness.sh
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run AW-rus file-1C freshness proof check every 6 hours
+
+[Timer]
+OnBootSec=20min
+OnUnitActiveSec=6h
+Unit=aw-1c-proofcheck.service
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
+
+mkdir -p \
+  "${ROOT}/landing/documents" \
+  "${ROOT}/landing/postings" \
+  "${ROOT}/landing/reglog" \
+  "${ROOT}/landing/audit" \
+  "${ROOT}/landing/host" \
+  "${ROOT}/archive/documents" \
+  "${ROOT}/archive/postings" \
+  "${ROOT}/archive/reglog" \
+  "${ROOT}/archive/audit" \
+  "${ROOT}/archive/host"
+
+if [[ ! -f "${ROOT}/etl/config.yml" ]]; then
+  cp "${ROOT}/etl/config.example.yml" "${ROOT}/etl/config.yml"
+fi
+
+python3 -m venv "${ROOT}/.venv"
+"${ROOT}/.venv/bin/pip" install --upgrade pip
+"${ROOT}/.venv/bin/pip" install -r "${ROOT}/etl/requirements.txt"
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
+ENV_FILE="${ROOT}/.env"
+CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
+MAX_AGE_HOURS="${AW_1C_MAX_AGE_HOURS:-8}"
+
+if [[ ! -f "${ENV_FILE}" ]]; then
+  echo "missing env file: ${ENV_FILE}" >&2
+  exit 1
+fi
+
+if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then
+  echo "clickhouse container not running: ${CH_CONTAINER}" >&2
+  exit 1
+fi
+
+# shellcheck disable=SC1090
+. "${ENV_FILE}"
+
+query_max_age() {
+  local table="$1"
+  docker exec "${CH_CONTAINER}" clickhouse-client \
+    --user "${CLICKHOUSE_USER}" \
+    --password "${CLICKHOUSE_PASSWORD}" \
+    --database "${CLICKHOUSE_DB}" \
+    -q "SELECT if(count()=0, -1, dateDiff('hour', max(ts), now())) FROM ${table}"
+}
+
+documents_age="$(query_max_age documents)"
+reglog_age="$(query_max_age reglog_events)"
+audit_age="$(query_max_age audit_events)"
+host_age="$(query_max_age host_events)"
+
+printf 'freshness documents=%sh reglog=%sh audit=%sh host=%sh threshold=%sh\n' \
+  "${documents_age}" "${reglog_age}" "${audit_age}" "${host_age}" "${MAX_AGE_HOURS}"
+
+for age in "${documents_age}" "${reglog_age}" "${audit_age}" "${host_age}"; do
+  if [[ "${age}" == "-1" ]]; then
+    echo "one or more datasets are empty" >&2
+    exit 1
+  fi
+  if (( age > MAX_AGE_HOURS )); then
+    echo "freshness threshold exceeded" >&2
+    exit 1
+  fi
+done
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
+ENV_FILE="${ROOT}/.env"
+VENV="${ROOT}/.venv"
+CONFIG="${ROOT}/etl/config.yml"
+CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}"
+
+if [[ ! -f "${ENV_FILE}" ]]; then
+  echo "missing env file: ${ENV_FILE}" >&2
+  exit 1
+fi
+
+if [[ ! -f "${CONFIG}" ]]; then
+  echo "missing etl config: ${CONFIG}" >&2
+  exit 1
+fi
+
+if [[ ! -x "${VENV}/bin/python" ]]; then
+  echo "missing venv python: ${VENV}/bin/python" >&2
+  exit 1
+fi
+
+if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then
+  echo "clickhouse container not running: ${CH_CONTAINER}" >&2
+  exit 1
+fi
+
+# shellcheck disable=SC1090
+. "${ENV_FILE}"
+
+"${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/build_entity_timeline.sql"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/insert_detections.sql"
+
+docker exec -i "${CH_CONTAINER}" clickhouse-client \
+  --user "${CLICKHOUSE_USER}" \
+  --password "${CLICKHOUSE_PASSWORD}" \
+  --database "${CLICKHOUSE_DB}" \
+  < "${ROOT}/detections/open_cases_from_detections.sql"