feat(workforce): add ClickHouse workforce analytics contour

clickhouse-workforce/ops/apply_catalogs.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+CATALOG_DIR="${CATALOG_DIR:-$ROOT_DIR/catalog}"
+
+CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}"
+CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}"
+CLICKHOUSE_USER="${CLICKHOUSE_USER:-}"
+CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
+CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}"
+CLICKHOUSE_READY_TIMEOUT_SEC="${CLICKHOUSE_READY_TIMEOUT_SEC:-60}"
+REBUILD_AGGREGATES="${REBUILD_AGGREGATES:-0}"
+
+client_auth_args=()
+if [[ -n "$CLICKHOUSE_USER" ]]; then
+  client_auth_args+=(--user "$CLICKHOUSE_USER")
+fi
+if [[ -n "$CLICKHOUSE_PASSWORD" ]]; then
+  client_auth_args+=(--password "$CLICKHOUSE_PASSWORD")
+fi
+
+run_query() {
+  local query="$1"
+
+  if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then
+    docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \
+      --database "$CLICKHOUSE_DATABASE" --query "$query"
+    return
+  fi
+
+  if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then
+    "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" \
+      --database "$CLICKHOUSE_DATABASE" --query "$query"
+    return
+  fi
+
+  printf 'No running ClickHouse container "%s" and no %s in PATH\n' \
+    "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2
+  return 127
+}
+
+run_query_file() {
+  local query_file="$1"
+
+  if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then
+    docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \
+      --multiquery <"$query_file"
+    return
+  fi
+
+  if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then
+    "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --multiquery <"$query_file"
+    return
+  fi
+
+  printf 'No running ClickHouse container "%s" and no %s in PATH\n' \
+    "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2
+  return 127
+}
+
+run_insert_file() {
+  local query="$1"
+  local data_file="$2"
+
+  if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then
+    docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \
+      --database "$CLICKHOUSE_DATABASE" --query "$query" <"$data_file"
+    return
+  fi
+
+  if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then
+    "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" \
+      --database "$CLICKHOUSE_DATABASE" --query "$query" <"$data_file"
+    return
+  fi
+
+  printf 'No running ClickHouse container "%s" and no %s in PATH\n' \
+    "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2
+  return 127
+}
+
+wait_for_clickhouse() {
+  local deadline
+  deadline=$((SECONDS + CLICKHOUSE_READY_TIMEOUT_SEC))
+  while (( SECONDS < deadline )); do
+    if run_query "SELECT 1" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+
+  printf 'ClickHouse is not ready after %s seconds\n' "$CLICKHOUSE_READY_TIMEOUT_SEC" >&2
+  return 1
+}
+
+require_file() {
+  local path="$1"
+  if [[ ! -f "$path" ]]; then
+    printf 'Required catalog file is missing: %s\n' "$path" >&2
+    return 1
+  fi
+}
+
+wait_for_clickhouse
+
+require_file "$CATALOG_DIR/workstation_users.tsv"
+require_file "$CATALOG_DIR/application_categories.tsv"
+require_file "$CATALOG_DIR/domain_categories.tsv"
+
+printf '[catalog] truncate dimension tables\n'
+run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_workstation_user"
+run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_application_category"
+run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_domain_category"
+
+printf '[catalog] load workstation users\n'
+run_insert_file "
+INSERT INTO $CLICKHOUSE_DATABASE.dim_workstation_user
+    (host_name, user_login, user_domain, employee_id, employee_name, department, branch, position, source, is_active)
+FORMAT TabSeparatedWithNames
+" "$CATALOG_DIR/workstation_users.tsv"
+
+printf '[catalog] load application categories\n'
+run_insert_file "
+INSERT INTO $CLICKHOUSE_DATABASE.dim_application_category
+    (process_name, application_name, vendor, category, productivity_class, risk_level, is_system, source, comment, is_active)
+FORMAT TabSeparatedWithNames
+" "$CATALOG_DIR/application_categories.tsv"
+
+printf '[catalog] load domain categories\n'
+run_insert_file "
+INSERT INTO $CLICKHOUSE_DATABASE.dim_domain_category
+    (domain, site_name, category, productivity_class, risk_level, business_allowed, source, comment, is_active)
+FORMAT TabSeparatedWithNames
+" "$CATALOG_DIR/domain_categories.tsv"
+
+printf '[catalog] reload dictionaries\n'
+run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_workstation_user"
+run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_application_category"
+run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_domain_category"
+
+if [[ "$REBUILD_AGGREGATES" == "1" ]]; then
+  printf '[catalog] rebuild aggregates\n'
+  run_query_file "$ROOT_DIR/admin/rebuild_aggregates.sql"
+fi
+
+printf '[catalog] dictionary status\n'
+run_query "
+SELECT name, status, last_exception
+FROM system.dictionaries
+WHERE database = '$CLICKHOUSE_DATABASE'
+  AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category')
+ORDER BY name
+FORMAT PrettyCompact
+"
+
+printf '[catalog] raw unknown summary\n'
+run_query "
+SELECT 'subjects' AS area, count() AS rows
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_subjects
+UNION ALL
+SELECT 'processes' AS area, count() AS rows
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_processes
+UNION ALL
+SELECT 'domains' AS area, count() AS rows
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_domains
+FORMAT PrettyCompact
+"

clickhouse-workforce/ops/aw-workforce-ingest.env.example

@@ -0,0 +1,14 @@
+AW_WORKFORCE_AW_URL=http://10.10.10.13:5600/api/0
+AW_WORKFORCE_CLICKHOUSE_URL=http://10.10.10.2:8123
+AW_WORKFORCE_CLICKHOUSE_DATABASE=aw_workforce
+AW_WORKFORCE_CLICKHOUSE_USER=
+AW_WORKFORCE_CLICKHOUSE_PASSWORD=
+AW_WORKFORCE_HOST=SHARKON2025
+AW_WORKFORCE_STATE_PATH=/var/lib/aw-workforce-ingest/state.json
+AW_WORKFORCE_OVERLAP_SECONDS=300
+AW_WORKFORCE_RETRY_ATTEMPTS=3
+AW_WORKFORCE_RETRY_BACKOFF_MS=1000
+AW_WORKFORCE_FAIL_ON_EMPTY=false
+AW_WORKFORCE_JSON=true
+no_proxy=localhost,127.0.0.1,10.10.10.13,10.10.10.2,10.10.10.0/24
+NO_PROXY=localhost,127.0.0.1,10.10.10.13,10.10.10.2,10.10.10.0/24

clickhouse-workforce/ops/aw-workforce-ingest.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=AW-rus workforce ClickHouse ingest
+After=network-online.target docker.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/activitywatch/clickhouse-workforce
+EnvironmentFile=-/etc/activitywatch/aw-workforce-ingest.env
+ExecStart=/usr/local/bin/aw-workforce-ingest
+TimeoutStartSec=20min
+User=root
+Group=root
+Nice=5
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=aw-workforce-ingest

clickhouse-workforce/ops/aw-workforce-ingest.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Run AW-rus workforce ClickHouse ingest every 5 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+AccuracySec=30s
+RandomizedDelaySec=30s
+Persistent=true
+Unit=aw-workforce-ingest.service
+
+[Install]
+WantedBy=timers.target

clickhouse-workforce/ops/bootstrap_runtime.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="${AW_WORKFORCE_ROOT:-/opt/activitywatch/clickhouse-workforce}"
+ENV_DIR="${AW_WORKFORCE_ENV_DIR:-/etc/activitywatch}"
+STATE_DIR="${AW_WORKFORCE_STATE_DIR:-/var/lib/aw-workforce-ingest}"
+SYSTEMD_DIR="${AW_WORKFORCE_SYSTEMD_DIR:-/etc/systemd/system}"
+
+install -d -m 0755 "$ROOT" "$ENV_DIR" "$STATE_DIR" "$SYSTEMD_DIR"
+
+if [[ ! -f "$ENV_DIR/aw-workforce-ingest.env" ]]; then
+  install -m 0640 "$SCRIPT_DIR/aw-workforce-ingest.env.example" \
+    "$ENV_DIR/aw-workforce-ingest.env"
+fi
+
+install -m 0644 "$SCRIPT_DIR/aw-workforce-ingest.service" \
+  "$SYSTEMD_DIR/aw-workforce-ingest.service"
+install -m 0644 "$SCRIPT_DIR/aw-workforce-ingest.timer" \
+  "$SYSTEMD_DIR/aw-workforce-ingest.timer"
+
+if command -v systemctl >/dev/null 2>&1; then
+  systemctl daemon-reload
+fi
+
+cat <<EOF
+Installed aw-workforce-ingest runtime files.
+
+Next manual deployment steps:
+  install -m 0755 <built aw-workforce-ingest binary> /usr/local/bin/aw-workforce-ingest
+  edit $ENV_DIR/aw-workforce-ingest.env
+  systemctl enable --now aw-workforce-ingest.timer
+  systemctl start aw-workforce-ingest.service
+EOF

clickhouse-workforce/ops/report_unknowns.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}"
+CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}"
+CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}"
+LIMIT="${LIMIT:-50}"
+
+run_query() {
+  local query="$1"
+
+  if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then
+    docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client \
+      --database "$CLICKHOUSE_DATABASE" --query "$query"
+    return
+  fi
+
+  "$CLICKHOUSE_CLIENT_BIN" --database "$CLICKHOUSE_DATABASE" --query "$query"
+}
+
+cd "$ROOT_DIR"
+
+printf '\n[unknown subjects]\n'
+run_query "
+SELECT *
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_subjects
+LIMIT $LIMIT
+FORMAT PrettyCompact
+"
+
+printf '\n[unknown processes]\n'
+run_query "
+SELECT *
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_processes
+LIMIT $LIMIT
+FORMAT PrettyCompact
+"
+
+printf '\n[unknown domains]\n'
+run_query "
+SELECT *
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_domains
+LIMIT $LIMIT
+FORMAT PrettyCompact
+"

clickhouse-workforce/ops/run_smoke.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}"
+CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}"
+CLICKHOUSE_USER="${CLICKHOUSE_USER:-}"
+CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
+CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}"
+LOAD_DEMO_SEED="${LOAD_DEMO_SEED:-1}"
+CLICKHOUSE_READY_TIMEOUT_SEC="${CLICKHOUSE_READY_TIMEOUT_SEC:-60}"
+
+client_auth_args=()
+if [[ -n "$CLICKHOUSE_USER" ]]; then
+  client_auth_args+=(--user "$CLICKHOUSE_USER")
+fi
+if [[ -n "$CLICKHOUSE_PASSWORD" ]]; then
+  client_auth_args+=(--password "$CLICKHOUSE_PASSWORD")
+fi
+
+run_client() {
+  local query_file="${1:-}"
+  local query="${2:-}"
+
+  if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then
+    if [[ -n "$query_file" ]]; then
+      docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" --multiquery <"$query_file"
+    else
+      docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" --database "$CLICKHOUSE_DATABASE" --query "$query"
+    fi
+    return
+  fi
+
+  if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then
+    if [[ -n "$query_file" ]]; then
+      "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --multiquery <"$query_file"
+    else
+      "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --database "$CLICKHOUSE_DATABASE" --query "$query"
+    fi
+    return
+  fi
+
+  printf 'No running ClickHouse container "%s" and no %s in PATH\n' \
+    "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2
+  return 127
+}
+
+wait_for_clickhouse() {
+  local deadline
+  deadline=$((SECONDS + CLICKHOUSE_READY_TIMEOUT_SEC))
+  while (( SECONDS < deadline )); do
+    if run_client "" "SELECT 1" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+
+  printf 'ClickHouse is not ready after %s seconds\n' "$CLICKHOUSE_READY_TIMEOUT_SEC" >&2
+  return 1
+}
+
+apply_sql_dir() {
+  local sql_file
+  for sql_file in "$ROOT_DIR"/clickhouse/init/*.sql; do
+    printf '[sql] %s\n' "${sql_file#$ROOT_DIR/}"
+    run_client "$sql_file" ""
+  done
+}
+
+assert_scalar_nonzero() {
+  local name="$1"
+  local query="$2"
+  local value
+  value="$(run_client "" "$query" | tr -d '[:space:]')"
+  if [[ ! "$value" =~ ^[0-9]+$ ]] || (( value < 1 )); then
+    printf '[FAIL] %s: expected positive integer, got "%s"\n' "$name" "$value" >&2
+    return 1
+  fi
+  printf '[OK]   %s: %s\n' "$name" "$value"
+}
+
+assert_no_dictionary_errors() {
+  local errors
+  errors="$(run_client "" "
+SELECT count()
+FROM system.dictionaries
+WHERE database = '$CLICKHOUSE_DATABASE'
+  AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category')
+  AND (status != 'LOADED' OR last_exception != '')
+")"
+  errors="$(printf '%s' "$errors" | tr -d '[:space:]')"
+  if [[ "$errors" != "0" ]]; then
+    printf '[FAIL] dictionaries have load errors\n' >&2
+    run_client "" "
+SELECT database, name, status, last_exception
+FROM system.dictionaries
+WHERE database = '$CLICKHOUSE_DATABASE'
+  AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category')
+FORMAT Vertical
+"
+    return 1
+  fi
+  printf '[OK]   dictionaries loaded\n'
+}
+
+wait_for_clickhouse
+apply_sql_dir
+
+if [[ "$LOAD_DEMO_SEED" == "1" ]]; then
+  printf '[sql] sample/seed_demo.sql\n'
+  run_client "$ROOT_DIR/sample/seed_demo.sql" ""
+fi
+
+assert_no_dictionary_errors
+assert_scalar_nonzero "dictionary count" "
+SELECT count()
+FROM system.dictionaries
+WHERE database = '$CLICKHOUSE_DATABASE'
+  AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category')
+"
+assert_scalar_nonzero "hourly aggregate rows" "
+SELECT count()
+FROM $CLICKHOUSE_DATABASE.agg_workforce_productivity_hourly
+"
+assert_scalar_nonzero "daily productivity rows" "
+SELECT count()
+FROM $CLICKHOUSE_DATABASE.v_workforce_productivity_daily
+"
+assert_scalar_nonzero "unknown quality rows" "
+SELECT count()
+FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_quality_daily
+"
+
+printf '[OK] ClickHouse workforce smoke completed\n'