From f7114bde0301b81cdcf1a8fe9487d5019e9d6adf Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Fri, 22 May 2026 10:11:53 +0300 Subject: [PATCH] feat(1c): add company intelligence forecasting layer --- README.md | 3 +- clickhouse-1c/.env.example | 6 + clickhouse-1c/README.md | 24 +- clickhouse-1c/ai/INVESTIGATOR_API.md | 50 ++ clickhouse-1c/ai/company_intelligence_api.py | 206 ++++++++ .../ai/refresh_company_intelligence.py | 301 +++++++++++ clickhouse-1c/ai/requirements.txt | 3 + .../init/04_company_intelligence.sql | 201 ++++++++ .../files/1c-company-intelligence.json | 480 ++++++++++++++++++ clickhouse-1c/ops/aw-1c-company-api.service | 14 + clickhouse-1c/ops/bootstrap_runtime.sh | 1 + .../ops/run_company_intelligence_api.sh | 29 ++ .../ops/run_company_intelligence_refresh.sh | 43 ++ clickhouse-1c/ops/run_ingest_cycle.sh | 8 + docs/1C_COMPANY_INTELLIGENCE_RU.md | 155 ++++++ docs/wiki/1C-Company-Intelligence.md | 17 + docs/wiki/Home.md | 1 + 17 files changed, 1539 insertions(+), 3 deletions(-) create mode 100644 clickhouse-1c/ai/company_intelligence_api.py create mode 100644 clickhouse-1c/ai/refresh_company_intelligence.py create mode 100644 clickhouse-1c/ai/requirements.txt create mode 100644 clickhouse-1c/clickhouse/init/04_company_intelligence.sql create mode 100644 clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json create mode 100644 clickhouse-1c/ops/aw-1c-company-api.service create mode 100644 clickhouse-1c/ops/run_company_intelligence_api.sh create mode 100644 clickhouse-1c/ops/run_company_intelligence_refresh.sh create mode 100644 docs/1C_COMPANY_INTELLIGENCE_RU.md create mode 100644 docs/wiki/1C-Company-Intelligence.md diff --git a/README.md b/README.md index e472760..b1441aa 100755 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - `docs/GRAFANA_DASHBOARDS_RU.md` — импорт и сопровождение Grafana dashboard'ов через Ansible API playbook. - `docs/PRESENTATION_RU.md` — презентационные экраны Grafana и AW-rus со скриншотами. - `docs/1C_FILE_ANALYTICS_STACK_RU.md` — production guide по файловой 1С Detmir: topology, rollout, verification, recovery, task principal и hardening. +- `docs/1C_COMPANY_INTELLIGENCE_RU.md` — слой анализа и прогноза по компаниям поверх `clickhouse-1c`: marts, forecasting, API и Grafana. - `docs/windows/ensemble.md` — orchestration-пакет для Windows-деплоя и проверки. - `docs/linux-client.md` — user-space rollout Linux-клиента ActivityWatch на удалённый `AW server`. - `docs/linux-remote-worker.md` — полный Linux remote-worker stack: GUI, SSH/console и browser admin UI вроде Proxmox `:8006`. @@ -25,7 +26,7 @@ - `aw-server/` — установочные скрипты, env-шаблон, systemd unit и RU patch для Web UI. - `ansible/` — Ansible-ensemble для автоматизированного сервера (Debian/CT). - `grafana/` — version-controlled Grafana dashboard JSON для RDP/worktime, DLP/ИБ и overview-экранов. -- `clickhouse-1c/` — отдельный analytics stack для **файловой 1С**: ETL, ClickHouse schema, detections, Grafana catalog и AI Investigator contract. +- `clickhouse-1c/` — отдельный analytics stack для **файловой 1С**: ETL, ClickHouse schema, detections, company intelligence marts/forecasting, Grafana catalog и AI Investigator contract. - `pfsense/` — внешний poller для pfSense API и systemd unit под Debian/Ubuntu utility VM. - `windows/` — PowerShell toolkit: single-user, domain-users, ensemble orchestration, hardening/recovery, validation, Windows/RDP DLP telemetry (`aw-dlp-incidents_*`, `aw-dlp-endpoint-signals_*`) и session-level presence для удалённых Windows/RDP пользователей (`aw-worktime-sessions_*`). - `scripts/quality-gate.sh` — локальный preflight-пайплайн проверок. diff --git a/clickhouse-1c/.env.example b/clickhouse-1c/.env.example index e2a7a2a..597a133 100644 --- a/clickhouse-1c/.env.example +++ b/clickhouse-1c/.env.example @@ -7,3 +7,9 @@ GRAFANA_ADMIN_USER=admin GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_PORT=3300 CLICKHOUSE_HOST=clickhouse +AW_1C_CLICKHOUSE_RUNTIME_HOST=127.0.0.1 +AW_1C_COMPANY_API_HOST=127.0.0.1 +AW_1C_COMPANY_API_PORT=8710 +AW_1C_COMPANY_LOOKBACK_DAYS=30 +AW_1C_COMPANY_MIN_DAYS=3 +AW_1C_COMPANY_HORIZONS=7,30 diff --git a/clickhouse-1c/README.md b/clickhouse-1c/README.md index 908913e..5c9b7db 100644 --- a/clickhouse-1c/README.md +++ b/clickhouse-1c/README.md @@ -31,7 +31,9 @@ File 1C + reglog + host telemetry ├─ host_events ├─ entity_timeline ├─ detections - └─ cases + ├─ cases + ├─ company_forecasts + └─ company_health_signals ↓ Grafana + Alerting + AI Investigator ``` @@ -48,11 +50,14 @@ File 1C + reglog + host telemetry - `grafana/dashboard-catalog.md` — целевая структура дашбордов. - `grafana/query-pack.sql` — базовые SQL-запросы для панелей. - `grafana/provisioning/datasources/clickhouse.yml` — provisioned datasource для Grafana. +- `grafana/provisioning/dashboards/files/1c-company-intelligence.json` — source dashboard для анализа и прогноза по компаниям. - `detections/build_entity_timeline.sql` — сборка единого timeline слоя. - `detections/open_cases_from_detections.sql` — шаблон открытия cases из detections. - `ops/etl-cron.example` — пример расписания каждые 6 часов. - `ops/retention-policy.md` — минимальная retention policy. - `ai/INVESTIGATOR_API.md` — контракт AI Investigator поверх ClickHouse/cases. +- `ai/refresh_company_intelligence.py` — materialization forecast/signals по `counterparty`. +- `ai/company_intelligence_api.py` — read-only API для AI/аналитики по компаниям. ## Когда использовать именно этот контур @@ -106,7 +111,20 @@ python etl/load_1c_exports.py --config etl/config.yml clickhouse-client --queries-file detections/insert_detections.sql ``` -6. В Grafana строить dashboards из `grafana/dashboard-catalog.md` и +6. Включить company intelligence слой: + +```bash +clickhouse-client --queries-file clickhouse/init/04_company_intelligence.sql +python ai/refresh_company_intelligence.py --host localhost --port 8123 --user default --password change-me --database analytics_1c +``` + +7. Запустить read-only API: + +```bash +python ai/company_intelligence_api.py --host 127.0.0.1 --port 8710 +``` + +8. В Grafana строить dashboards из `grafana/dashboard-catalog.md` и `grafana/query-pack.sql`. ## Ожидаемые источники данных @@ -116,6 +134,7 @@ clickhouse-client --queries-file detections/insert_detections.sql - журнал регистрации 1С; - audit/export критичных изменений; - host telemetry с Windows/RDP host. +- для company intelligence нужны документы с непустым `counterparty`. ## Границы @@ -123,3 +142,4 @@ clickhouse-client --queries-file detections/insert_detections.sql - LLM не ходит прямо в production 1С; - в ClickHouse кладутся нормализованные выгрузки и enrichment; - case/timeline слой считается вне 1С. +- если `counterparty` в live-выгрузках пустой, company-forecast слой останется корректно пустым. diff --git a/clickhouse-1c/ai/INVESTIGATOR_API.md b/clickhouse-1c/ai/INVESTIGATOR_API.md index 56f821d..70e7897 100644 --- a/clickhouse-1c/ai/INVESTIGATOR_API.md +++ b/clickhouse-1c/ai/INVESTIGATOR_API.md @@ -15,6 +15,9 @@ AI Investigator не должен ходить напрямую в файлов - что произошло по case `X`; - собрать summary по entity timeline; - предложить next steps без write-действий. +- какие компании выпали из активности; +- где ожидается спад или рост объёма по компаниям; +- какие компании требуют проверки из-за резкого падения документооборота. ## Рекомендуемые API endpoints @@ -61,6 +64,52 @@ AI Investigator не должен ходить напрямую в файлов - related detections - related timeline rows +### `GET /api/1/analytics-1c/companies/overview` + +Фильтры: + +- `infobase` +- `min_signal_score` +- `limit` + +Возвращает: + +- компании с активностью за 30 дней +- последние сигналы риска +- прогноз `amount/docs` на `7/30` дней + +### `GET /api/1/analytics-1c/companies/{counterparty}/summary` + +Возвращает: + +- текущую карточку компании +- AI-ready short summary +- последние документы +- forecasts +- signals + +### `GET /api/1/analytics-1c/companies/{counterparty}/forecast` + +Возвращает: + +- `metric` +- `horizon_days` +- `baseline_daily` +- `trend_slope` +- `predicted_daily` +- `predicted_total` +- `confidence` + +### `GET /api/1/analytics-1c/companies/{counterparty}/timeline` + +Возвращает: + +- последние документы по компании +- базу +- автора +- тип операции +- статус + ## Guardrails - read-only SQL; @@ -68,6 +117,7 @@ AI Investigator не должен ходить напрямую в файлов - no direct write-back into 1С; - no direct execution of arbitrary SQL from prompt; - all investigator requests are logged. +- если live-данные не содержат `counterparty`, company endpoints должны честно возвращать пустой результат, а не симулировать прогноз. ## Output style diff --git a/clickhouse-1c/ai/company_intelligence_api.py b/clickhouse-1c/ai/company_intelligence_api.py new file mode 100644 index 0000000..08d1185 --- /dev/null +++ b/clickhouse-1c/ai/company_intelligence_api.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any + +import clickhouse_connect +import uvicorn +from fastapi import FastAPI, HTTPException, Query + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Read-only company intelligence API for analytics_1c") + p.add_argument("--host", default=os.getenv("AW_1C_COMPANY_API_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("AW_1C_COMPANY_API_PORT", "8710"))) + return p.parse_args() + + +def q(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def to_plain(value: Any) -> Any: + if isinstance(value, Decimal): + return float(value) + if isinstance(value, datetime): + return value.isoformat() + return value + + +def rows_to_dict(result) -> list[dict[str, Any]]: + return [ + {name: to_plain(value) for name, value in zip(result.column_names, row)} + for row in result.result_rows + ] + + +def ch_client(): + return clickhouse_connect.get_client( + host=os.getenv("CLICKHOUSE_HOST", "localhost"), + port=int(os.getenv("CLICKHOUSE_PORT", "8123")), + username=os.getenv("CLICKHOUSE_USER", "default"), + password=os.getenv("CLICKHOUSE_PASSWORD", ""), + database=os.getenv("CLICKHOUSE_DB", "analytics_1c"), + ) + + +app = FastAPI(title="AW-rus 1C Company Intelligence API", version="1.0.0") + + +@app.get("/health") +def health() -> dict[str, Any]: + client = ch_client() + summary = rows_to_dict( + client.query( + """ + SELECT + countIf(counterparty != '') AS documents_with_counterparty, + (SELECT count() FROM analytics_1c.company_forecasts) AS forecasts_total, + (SELECT count() FROM analytics_1c.company_health_signals) AS health_signals_total + FROM analytics_1c.documents + """ + ) + )[0] + return {"status": "ok", "generated_at": datetime.now(UTC).isoformat(), **summary} + + +@app.get("/api/1/analytics-1c/companies/overview") +def companies_overview( + infobase: str | None = None, + min_signal_score: int = Query(default=0, ge=0, le=100), + limit: int = Query(default=50, ge=1, le=500), +) -> dict[str, Any]: + client = ch_client() + where = [f"signal_score >= {int(min_signal_score)}"] + if infobase: + where.append(f"infobase = {q(infobase)}") + sql = f""" + SELECT + infobase, + organization, + counterparty, + last_seen_at, + days_since_last_activity, + docs_7d, + amount_7d, + docs_30d, + amount_30d, + amount_forecast_30d, + docs_forecast_30d, + signal_severity, + signal_score, + top_signal + FROM analytics_1c.v_company_portfolio_overview + WHERE {' AND '.join(where)} + ORDER BY signal_score DESC, amount_30d DESC, counterparty + LIMIT {int(limit)} + """ + rows = rows_to_dict(client.query(sql)) + return {"items": rows, "count": len(rows)} + + +@app.get("/api/1/analytics-1c/companies/{counterparty}/summary") +def company_summary(counterparty: str, infobase: str | None = None) -> dict[str, Any]: + client = ch_client() + filters = [f"counterparty = {q(counterparty)}"] + if infobase: + filters.append(f"infobase = {q(infobase)}") + sql = f""" + SELECT * + FROM analytics_1c.v_company_portfolio_overview + WHERE {' AND '.join(filters)} + ORDER BY amount_30d DESC + LIMIT 1 + """ + rows = rows_to_dict(client.query(sql)) + if not rows: + raise HTTPException(status_code=404, detail="counterparty not found in analytics_1c.v_company_portfolio_overview") + card = rows[0] + forecast_sql = f""" + SELECT metric, horizon_days, baseline_daily, trend_slope, predicted_daily, predicted_total, confidence, note + FROM analytics_1c.v_company_forecasts_current + WHERE counterparty = {q(counterparty)} + {"AND infobase = " + q(infobase) if infobase else ""} + ORDER BY metric, horizon_days + """ + signals_sql = f""" + SELECT generated_at, severity, score, signal_type, summary + FROM analytics_1c.v_company_health_current + WHERE counterparty = {q(counterparty)} + {"AND infobase = " + q(infobase) if infobase else ""} + ORDER BY score DESC, generated_at DESC + """ + timeline_sql = f""" + SELECT ts, infobase, doc_type, operation_type, amount, status, author + FROM analytics_1c.documents + WHERE counterparty = {q(counterparty)} + {"AND infobase = " + q(infobase) if infobase else ""} + ORDER BY ts DESC + LIMIT 20 + """ + forecasts = rows_to_dict(client.query(forecast_sql)) + signals = rows_to_dict(client.query(signals_sql)) + timeline = rows_to_dict(client.query(timeline_sql)) + essence = ( + f"Компания {counterparty}: за 30 дней документов {card['docs_30d']}, объём {card['amount_30d']}, " + f"прогноз на 30 дней {card['amount_forecast_30d']}, риск {card['signal_severity']}." + ) + return { + "essence": essence, + "card": card, + "forecasts": forecasts, + "signals": signals, + "recent_documents": timeline, + } + + +@app.get("/api/1/analytics-1c/companies/{counterparty}/forecast") +def company_forecast( + counterparty: str, + infobase: str | None = None, + horizon_days: int | None = Query(default=None, ge=1, le=365), +) -> dict[str, Any]: + client = ch_client() + filters = [f"counterparty = {q(counterparty)}"] + if infobase: + filters.append(f"infobase = {q(infobase)}") + if horizon_days is not None: + filters.append(f"horizon_days = {int(horizon_days)}") + sql = f""" + SELECT generated_at, infobase, counterparty, metric, horizon_days, baseline_daily, trend_slope, predicted_daily, predicted_total, confidence, note + FROM analytics_1c.v_company_forecasts_current + WHERE {' AND '.join(filters)} + ORDER BY metric, horizon_days + """ + rows = rows_to_dict(client.query(sql)) + return {"items": rows, "count": len(rows)} + + +@app.get("/api/1/analytics-1c/companies/{counterparty}/timeline") +def company_timeline( + counterparty: str, + infobase: str | None = None, + limit: int = Query(default=100, ge=1, le=500), +) -> dict[str, Any]: + client = ch_client() + filters = [f"counterparty = {q(counterparty)}"] + if infobase: + filters.append(f"infobase = {q(infobase)}") + sql = f""" + SELECT ts, infobase, organization, doc_type, doc_number, author, operation_type, amount, status, posted + FROM analytics_1c.documents + WHERE {' AND '.join(filters)} + ORDER BY ts DESC + LIMIT {int(limit)} + """ + rows = rows_to_dict(client.query(sql)) + return {"items": rows, "count": len(rows)} + + +if __name__ == "__main__": + args = parse_args() + uvicorn.run(app, host=args.host, port=args.port) diff --git a/clickhouse-1c/ai/refresh_company_intelligence.py b/clickhouse-1c/ai/refresh_company_intelligence.py new file mode 100644 index 0000000..abade6c --- /dev/null +++ b/clickhouse-1c/ai/refresh_company_intelligence.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import math +import os +from collections import defaultdict +from dataclasses import dataclass +from datetime import UTC, date, datetime, timedelta +from statistics import fmean, pstdev +from typing import Any + +import clickhouse_connect + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Refresh company forecasts and health signals for analytics_1c") + p.add_argument("--host", default=os.getenv("CLICKHOUSE_HOST", "localhost")) + p.add_argument("--port", type=int, default=int(os.getenv("CLICKHOUSE_PORT", "8123"))) + p.add_argument("--user", default=os.getenv("CLICKHOUSE_USER", "default")) + p.add_argument("--password", default=os.getenv("CLICKHOUSE_PASSWORD", "")) + p.add_argument("--database", default=os.getenv("CLICKHOUSE_DB", "analytics_1c")) + p.add_argument("--lookback-days", type=int, default=int(os.getenv("AW_1C_COMPANY_LOOKBACK_DAYS", "30"))) + p.add_argument("--min-days", type=int, default=int(os.getenv("AW_1C_COMPANY_MIN_DAYS", "3"))) + p.add_argument("--horizons", default=os.getenv("AW_1C_COMPANY_HORIZONS", "7,30")) + return p.parse_args() + + +@dataclass +class DailyPoint: + d: date + docs_total: float + amount_total: float + + +def ch_client(args: argparse.Namespace): + return clickhouse_connect.get_client( + host=args.host, + port=args.port, + username=args.user, + password=args.password, + database=args.database, + ) + + +def query_rows(client, sql: str) -> list[dict[str, Any]]: + result = client.query(sql) + return [dict(zip(result.column_names, row)) for row in result.result_rows] + + +def fill_daily_series(points: list[DailyPoint]) -> list[DailyPoint]: + if not points: + return [] + by_day = {p.d: p for p in points} + current = points[0].d + end = points[-1].d + filled: list[DailyPoint] = [] + while current <= end: + filled.append(by_day.get(current, DailyPoint(current, 0.0, 0.0))) + current += timedelta(days=1) + return filled + + +def linear_slope(values: list[float]) -> float: + n = len(values) + if n < 2: + return 0.0 + x_mean = (n - 1) / 2 + y_mean = fmean(values) + num = sum((i - x_mean) * (v - y_mean) for i, v in enumerate(values)) + den = sum((i - x_mean) ** 2 for i in range(n)) + if den == 0: + return 0.0 + return num / den + + +def build_forecast(values: list[float], horizon: int, min_days: int, lookback_days: int) -> tuple[float, float, float, float, int, str]: + if len(values) < min_days: + raise ValueError("not enough data") + window = values[-min(len(values), lookback_days):] + baseline = fmean(window) + slope = linear_slope(window) + projected = [max(0.0, baseline + slope * step) for step in range(1, horizon + 1)] + predicted_total = sum(projected) + predicted_daily = projected[-1] if projected else baseline + if len(window) > 1 and baseline > 0: + volatility = pstdev(window) / baseline + elif len(window) > 1: + volatility = pstdev(window) + else: + volatility = 0.0 + coverage = min(1.0, len(window) / max(lookback_days, 1)) + stability = max(0.15, 1.0 - min(volatility, 1.0)) + confidence = max(0.1, min(0.95, coverage * stability)) + note_parts: list[str] = [] + if len(values) < lookback_days: + note_parts.append("sparse_history") + if abs(slope) < 0.01: + note_parts.append("flat_trend") + note = ",".join(note_parts) if note_parts else "ok" + return baseline, slope, predicted_daily, predicted_total, len(window), note + + +def severity_score_to_label(score: int) -> str: + if score >= 80: + return "critical" + if score >= 60: + return "high" + if score >= 35: + return "medium" + return "low" + + +def main() -> int: + args = parse_args() + client = ch_client(args) + horizons = [int(x.strip()) for x in args.horizons.split(",") if x.strip()] + generated_at = datetime.now(UTC).replace(tzinfo=None, microsecond=0) + + daily_rows = query_rows( + client, + """ + SELECT infobase, organization, counterparty, d, docs_total, amount_total + FROM analytics_1c.v_counterparty_daily + ORDER BY infobase, counterparty, d + """, + ) + if not daily_rows: + print("no counterparty rows in analytics_1c.v_counterparty_daily; nothing to refresh") + return 0 + + grouped: dict[tuple[str, str, str], list[DailyPoint]] = defaultdict(list) + for row in daily_rows: + key = (row["infobase"], row["organization"], row["counterparty"]) + grouped[key].append( + DailyPoint( + d=row["d"], + docs_total=float(row["docs_total"] or 0), + amount_total=float(row["amount_total"] or 0), + ) + ) + + cases_map = { + (row["infobase"], row["counterparty"]): int(row["open_cases_total"] or 0) + for row in query_rows( + client, + """ + SELECT infobase, entity_id AS counterparty, countIf(status != 'closed') AS open_cases_total + FROM analytics_1c.cases + WHERE entity_type = 'counterparty' + GROUP BY infobase, counterparty + """, + ) + } + detections_map = { + (row["infobase"], row["counterparty"]): int(row["detections_total"] or 0) + for row in query_rows( + client, + """ + SELECT infobase, entity_id AS counterparty, count() AS detections_total + FROM analytics_1c.detections + WHERE entity_type = 'counterparty' AND status != 'closed' + GROUP BY infobase, counterparty + """, + ) + } + + forecast_rows: list[list[Any]] = [] + signal_rows: list[list[Any]] = [] + + for (infobase, _organization, counterparty), points in grouped.items(): + points.sort(key=lambda p: p.d) + filled = fill_daily_series(points) + docs_series = [p.docs_total for p in filled] + amount_series = [p.amount_total for p in filled] + if len(filled) < args.min_days: + continue + + latest_day = filled[-1].d + last_7 = filled[-7:] + prev_7 = filled[-14:-7] + docs_7d = int(sum(p.docs_total for p in last_7)) + docs_prev_7d = int(sum(p.docs_total for p in prev_7)) + amount_7d = float(sum(p.amount_total for p in last_7)) + amount_prev_7d = float(sum(p.amount_total for p in prev_7)) + days_since_last_activity = (date.today() - latest_day).days + open_cases_total = cases_map.get((infobase, counterparty), 0) + detections_total = detections_map.get((infobase, counterparty), 0) + + for metric, values in (("docs_total", docs_series), ("amount_total", amount_series)): + for horizon in horizons: + baseline, slope, predicted_daily, predicted_total, source_days, note = build_forecast( + values=values, + horizon=horizon, + min_days=args.min_days, + lookback_days=args.lookback_days, + ) + forecast_rows.append( + [ + generated_at, + latest_day, + infobase, + counterparty, + int(horizon), + metric, + float(baseline), + float(slope), + float(predicted_daily), + float(predicted_total), + round(float(max(0.1, min(0.95, 1.0 - abs(slope) / (abs(baseline) + 1.0)))), 4), + "linear_baseline", + int(source_days), + note, + ] + ) + + signals: list[tuple[str, int, str, str]] = [] + if days_since_last_activity >= 14 and (docs_prev_7d > 0 or amount_prev_7d > 0): + signals.append(("inactive_company", 85, "high", f"Нет активности по компании {counterparty} уже {days_since_last_activity} дн.")) + if amount_prev_7d > 0 and amount_7d < amount_prev_7d * 0.5: + signals.append(("amount_drop", 70, "high", f"Объём по компании {counterparty} упал более чем на 50% неделя к неделе.")) + if docs_prev_7d > 0 and docs_7d == 0: + signals.append(("docs_stopped", 55, "medium", f"По компании {counterparty} прекратился поток документов за последние 7 дней.")) + if open_cases_total > 0: + signals.append(("open_cases", min(95, 40 + open_cases_total * 10), severity_score_to_label(min(95, 40 + open_cases_total * 10)), f"По компании {counterparty} есть открытые кейсы: {open_cases_total}.")) + if detections_total > 0: + signals.append(("open_detections", min(90, 35 + detections_total * 5), severity_score_to_label(min(90, 35 + detections_total * 5)), f"По компании {counterparty} есть активные detections: {detections_total}.")) + + for signal_type, score, severity, summary in signals: + signal_rows.append( + [ + generated_at, + infobase, + counterparty, + f"{signal_type}:{infobase}:{counterparty}", + severity, + int(score), + signal_type, + summary, + float(amount_7d), + float(amount_prev_7d), + int(docs_7d), + int(docs_prev_7d), + int(max(days_since_last_activity, 0)), + int(open_cases_total), + int(detections_total), + ] + ) + + if forecast_rows: + client.insert( + "analytics_1c.company_forecasts", + forecast_rows, + column_names=[ + "generated_at", + "as_of_date", + "infobase", + "counterparty", + "horizon_days", + "metric", + "baseline_daily", + "trend_slope", + "predicted_daily", + "predicted_total", + "confidence", + "model", + "source_days", + "note", + ], + ) + if signal_rows: + client.insert( + "analytics_1c.company_health_signals", + signal_rows, + column_names=[ + "generated_at", + "infobase", + "counterparty", + "signal_id", + "severity", + "score", + "signal_type", + "summary", + "amount_7d", + "amount_prev_7d", + "docs_7d", + "docs_prev_7d", + "days_since_last_activity", + "open_cases_total", + "detections_total", + ], + ) + + print( + f"company intelligence refreshed: forecasts={len(forecast_rows)} signals={len(signal_rows)} generated_at={generated_at.isoformat()}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/clickhouse-1c/ai/requirements.txt b/clickhouse-1c/ai/requirements.txt new file mode 100644 index 0000000..b0f50bf --- /dev/null +++ b/clickhouse-1c/ai/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.2 +uvicorn[standard]==0.32.0 +clickhouse-connect==0.7.16 diff --git a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql new file mode 100644 index 0000000..8b6d848 --- /dev/null +++ b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql @@ -0,0 +1,201 @@ +CREATE TABLE IF NOT EXISTS analytics_1c.company_forecasts +( + generated_at DateTime, + as_of_date Date, + infobase LowCardinality(String), + counterparty String, + horizon_days UInt16, + metric LowCardinality(String), + baseline_daily Float64, + trend_slope Float64, + predicted_daily Float64, + predicted_total Float64, + confidence Float32, + model LowCardinality(String), + source_days UInt16, + note String +) +ENGINE = MergeTree +ORDER BY (generated_at, infobase, counterparty, metric, horizon_days); + +CREATE TABLE IF NOT EXISTS analytics_1c.company_health_signals +( + generated_at DateTime, + infobase LowCardinality(String), + counterparty String, + signal_id String, + severity LowCardinality(String), + score UInt32, + signal_type LowCardinality(String), + summary String, + amount_7d Float64, + amount_prev_7d Float64, + docs_7d UInt32, + docs_prev_7d UInt32, + days_since_last_activity UInt16, + open_cases_total UInt32, + detections_total UInt32 +) +ENGINE = MergeTree +ORDER BY (generated_at, severity, infobase, counterparty, signal_id); + +CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_daily AS +SELECT + toDate(ts) AS d, + infobase, + organization, + counterparty, + count() AS docs_total, + sum(amount) AS amount_total, + countIf(posted = 1) AS posted_docs_total, + countIf(posted = 0) AS unposted_docs_total, + countIf(status = 'busy') AS busy_docs_total, + countIf(status = 'online') AS online_docs_total, + uniqExact(doc_type) AS doc_types_total +FROM analytics_1c.documents +WHERE counterparty != '' +GROUP BY d, infobase, organization, counterparty; + +CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_latest_activity AS +SELECT + infobase, + organization, + counterparty, + max(ts) AS last_seen_at, + argMax(doc_type, ts) AS last_doc_type, + argMax(operation_type, ts) AS last_operation_type, + argMax(status, ts) AS last_status, + argMax(amount, ts) AS last_amount, + count() AS docs_lifetime, + sum(amount) AS amount_lifetime +FROM analytics_1c.documents +WHERE counterparty != '' +GROUP BY infobase, organization, counterparty; + +CREATE VIEW IF NOT EXISTS analytics_1c.v_company_forecasts_current AS +SELECT * +FROM analytics_1c.company_forecasts +WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_forecasts); + +CREATE VIEW IF NOT EXISTS analytics_1c.v_company_health_current AS +SELECT * +FROM analytics_1c.company_health_signals +WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_health_signals); + +CREATE VIEW IF NOT EXISTS analytics_1c.v_company_portfolio_overview AS +WITH +base AS +( + SELECT * + FROM analytics_1c.v_counterparty_latest_activity +), +d7 AS +( + SELECT + infobase, + counterparty, + sum(docs_total) AS docs_7d, + sum(amount_total) AS amount_7d + FROM analytics_1c.v_counterparty_daily + WHERE d >= today() - 7 + GROUP BY infobase, counterparty +), +d30 AS +( + SELECT + infobase, + counterparty, + countDistinct(d) AS active_days_30d, + sum(docs_total) AS docs_30d, + sum(amount_total) AS amount_30d, + sum(busy_docs_total) AS busy_docs_30d + FROM analytics_1c.v_counterparty_daily + WHERE d >= today() - 30 + GROUP BY infobase, counterparty +), +signals AS +( + SELECT + infobase, + counterparty, + max(score) AS signal_score, + argMax(severity, score) AS signal_severity, + argMax(summary, score) AS top_signal + FROM analytics_1c.v_company_health_current + GROUP BY infobase, counterparty +), +amount_forecast AS +( + SELECT + infobase, + counterparty, + predicted_total AS amount_forecast_30d, + confidence AS amount_forecast_confidence + FROM analytics_1c.v_company_forecasts_current + WHERE metric = 'amount_total' + AND horizon_days = 30 +), +docs_forecast AS +( + SELECT + infobase, + counterparty, + predicted_total AS docs_forecast_30d, + confidence AS docs_forecast_confidence + FROM analytics_1c.v_company_forecasts_current + WHERE metric = 'docs_total' + AND horizon_days = 30 +), +cases_current AS +( + SELECT + infobase, + entity_id AS counterparty, + countIf(status != 'closed') AS open_cases_total + FROM analytics_1c.cases + WHERE entity_type = 'counterparty' + GROUP BY infobase, counterparty +), +detections_current AS +( + SELECT + infobase, + entity_id AS counterparty, + count() AS detections_total + FROM analytics_1c.detections + WHERE entity_type = 'counterparty' + AND status != 'closed' + GROUP BY infobase, counterparty +) +SELECT + base.infobase AS infobase, + base.organization, + base.counterparty AS counterparty, + base.last_seen_at, + base.last_doc_type, + base.last_operation_type, + base.last_status, + dateDiff('day', toDate(base.last_seen_at), today()) AS days_since_last_activity, + ifNull(d7.docs_7d, 0) AS docs_7d, + ifNull(d7.amount_7d, 0) AS amount_7d, + ifNull(d30.active_days_30d, 0) AS active_days_30d, + ifNull(d30.docs_30d, 0) AS docs_30d, + ifNull(d30.amount_30d, 0) AS amount_30d, + ifNull(d30.busy_docs_30d, 0) AS busy_docs_30d, + ifNull(amount_forecast.amount_forecast_30d, 0) AS amount_forecast_30d, + ifNull(amount_forecast.amount_forecast_confidence, 0) AS amount_forecast_confidence, + ifNull(docs_forecast.docs_forecast_30d, 0) AS docs_forecast_30d, + ifNull(docs_forecast.docs_forecast_confidence, 0) AS docs_forecast_confidence, + ifNull(cases_current.open_cases_total, 0) AS open_cases_total, + ifNull(detections_current.detections_total, 0) AS detections_total, + ifNull(signals.signal_severity, 'none') AS signal_severity, + ifNull(signals.signal_score, 0) AS signal_score, + ifNull(signals.top_signal, '') AS top_signal +FROM base +LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty +LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty +LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty +LEFT JOIN amount_forecast ON amount_forecast.infobase = base.infobase AND amount_forecast.counterparty = base.counterparty +LEFT JOIN docs_forecast ON docs_forecast.infobase = base.infobase AND docs_forecast.counterparty = base.counterparty +LEFT JOIN cases_current ON cases_current.infobase = base.infobase AND cases_current.counterparty = base.counterparty +LEFT JOIN detections_current ON detections_current.infobase = base.infobase AND detections_current.counterparty = base.counterparty; diff --git a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json new file mode 100644 index 0000000..f8d3049 --- /dev/null +++ b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json @@ -0,0 +1,480 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "id": 1, + "title": "Активные компании 30д", + "type": "stat", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT countDistinct(counterparty) AS value FROM analytics_1c.v_counterparty_daily WHERE d >= today() - 30", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 2, + "title": "Прогноз объёма 30д", + "type": "stat", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT round(sum(predicted_total), 2) AS value FROM analytics_1c.v_company_forecasts_current WHERE metric = 'amount_total' AND horizon_days = 30", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + }, + { + "id": 3, + "title": "Компании с high/critical сигналами", + "type": "stat", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT countDistinct(counterparty) AS value FROM analytics_1c.v_company_health_current WHERE severity IN ('high', 'critical')", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + }, + { + "id": 4, + "title": "Counterparty cases", + "type": "stat", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT count() AS value FROM analytics_1c.cases WHERE entity_type = 'counterparty' AND status != 'closed'", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + }, + { + "id": 5, + "title": "Объём по компаниям", + "type": "timeseries", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 0, + "y": 6, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "timeSeries", + "rawSql": "SELECT toDateTime(d) AS time, sum(amount_total) AS value FROM analytics_1c.v_counterparty_daily WHERE d >= today() - 30 GROUP BY time ORDER BY time", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "id": 6, + "title": "Количество документов по компаниям", + "type": "timeseries", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 12, + "y": 6, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "timeSeries", + "rawSql": "SELECT toDateTime(d) AS time, sum(docs_total) AS value FROM analytics_1c.v_counterparty_daily WHERE d >= today() - 30 GROUP BY time ORDER BY time", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "id": 7, + "title": "Портфель компаний", + "type": "table", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 0, + "y": 14, + "w": 16, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT infobase, counterparty, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", + "refId": "A" + } + ], + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "signal_score", + "desc": true + } + ] + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 8, + "title": "Последние AI-сигналы", + "type": "table", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "pluginVersion": "11.2.2", + "gridPos": { + "x": 16, + "y": 14, + "w": 8, + "h": 10 + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "editorType": "sql", + "format": 1, + "pluginVersion": "11.2.2", + "queryType": "table", + "rawSql": "SELECT generated_at, infobase, counterparty, severity, score, signal_type, summary FROM analytics_1c.v_company_health_current ORDER BY score DESC, generated_at DESC LIMIT 20", + "refId": "A" + } + ], + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "score", + "desc": true + } + ] + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "1c", + "companies", + "forecast", + "clickhouse" + ], + "templating": { + "list": [ + { + "name": "infobase", + "label": "Infobase", + "type": "query", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "refresh": 1, + "definition": "SELECT DISTINCT infobase FROM analytics_1c.v_counterparty_latest_activity ORDER BY infobase", + "query": "SELECT DISTINCT infobase FROM analytics_1c.v_counterparty_latest_activity ORDER BY infobase", + "multi": true, + "includeAll": true, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + }, + { + "name": "counterparty", + "label": "Counterparty", + "type": "query", + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-1c" + }, + "refresh": 1, + "definition": "SELECT DISTINCT counterparty FROM analytics_1c.v_counterparty_latest_activity ORDER BY counterparty", + "query": "SELECT DISTINCT counterparty FROM analytics_1c.v_counterparty_latest_activity ORDER BY counterparty", + "multi": true, + "includeAll": true, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "1C File - Company Intelligence", + "uid": "1c-file-companies", + "version": 1, + "weekStart": "" +} diff --git a/clickhouse-1c/ops/aw-1c-company-api.service b/clickhouse-1c/ops/aw-1c-company-api.service new file mode 100644 index 0000000..6898db7 --- /dev/null +++ b/clickhouse-1c/ops/aw-1c-company-api.service @@ -0,0 +1,14 @@ +[Unit] +Description=AW-rus 1C Company Intelligence API +After=network.target docker.service +Requires=docker.service + +[Service] +Type=simple +Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c +ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_company_intelligence_api.sh +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/clickhouse-1c/ops/bootstrap_runtime.sh b/clickhouse-1c/ops/bootstrap_runtime.sh index 8079379..71b803c 100644 --- a/clickhouse-1c/ops/bootstrap_runtime.sh +++ b/clickhouse-1c/ops/bootstrap_runtime.sh @@ -22,3 +22,4 @@ fi python3 -m venv "${ROOT}/.venv" "${ROOT}/.venv/bin/pip" install --upgrade pip "${ROOT}/.venv/bin/pip" install -r "${ROOT}/etl/requirements.txt" +"${ROOT}/.venv/bin/pip" install -r "${ROOT}/ai/requirements.txt" diff --git a/clickhouse-1c/ops/run_company_intelligence_api.sh b/clickhouse-1c/ops/run_company_intelligence_api.sh new file mode 100644 index 0000000..12cfcf5 --- /dev/null +++ b/clickhouse-1c/ops/run_company_intelligence_api.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" +ENV_FILE="${ROOT}/.env" +VENV="${ROOT}/.venv" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "missing env file: ${ENV_FILE}" >&2 + exit 1 +fi + +if [[ ! -x "${VENV}/bin/python" ]]; then + echo "missing venv python: ${VENV}/bin/python" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +set -a +. "${ENV_FILE}" +set +a + +CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" +if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then + CH_RUNTIME_HOST="127.0.0.1" +fi +export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}" + +exec "${VENV}/bin/python" "${ROOT}/ai/company_intelligence_api.py" diff --git a/clickhouse-1c/ops/run_company_intelligence_refresh.sh b/clickhouse-1c/ops/run_company_intelligence_refresh.sh new file mode 100644 index 0000000..d3b9651 --- /dev/null +++ b/clickhouse-1c/ops/run_company_intelligence_refresh.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" +ENV_FILE="${ROOT}/.env" +VENV="${ROOT}/.venv" +CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "missing env file: ${ENV_FILE}" >&2 + exit 1 +fi + +if [[ ! -x "${VENV}/bin/python" ]]; then + echo "missing venv python: ${VENV}/bin/python" >&2 + exit 1 +fi + +if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then + echo "clickhouse container not running: ${CH_CONTAINER}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +. "${ENV_FILE}" + +CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}" +if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then + CH_RUNTIME_HOST="127.0.0.1" +fi + +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/clickhouse/init/04_company_intelligence.sql" + +"${VENV}/bin/python" "${ROOT}/ai/refresh_company_intelligence.py" \ + --host "${CH_RUNTIME_HOST}" \ + --port "${CLICKHOUSE_PORT}" \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh index 45bc2fe..5834fff 100644 --- a/clickhouse-1c/ops/run_ingest_cycle.sh +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -56,3 +56,11 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \ --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}" \ < "${ROOT}/detections/open_cases_from_detections.sql" + +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/clickhouse/init/04_company_intelligence.sql" + +"${ROOT}/ops/run_company_intelligence_refresh.sh" diff --git a/docs/1C_COMPANY_INTELLIGENCE_RU.md b/docs/1C_COMPANY_INTELLIGENCE_RU.md new file mode 100644 index 0000000..5843cdf --- /dev/null +++ b/docs/1C_COMPANY_INTELLIGENCE_RU.md @@ -0,0 +1,155 @@ +# 1C Company Intelligence для AW-rus + +Этот слой строится **поверх** `clickhouse-1c/` и не трогает саму 1С. + +Его задача: + +- анализировать работу с компаниями (`counterparty`); +- показывать, где компании выпали из активности; +- считать простой, объяснимый прогноз по документам и объёму; +- давать read-only API для AI Investigator и внешних аналитических сервисов. + +## Что считается компанией + +В этом контуре компания = `documents.counterparty`. + +Если в live-выгрузках поле `counterparty` пустое, слой остаётся корректно пустым. +Он не выдумывает данные и не пытается прогнозировать то, чего нет. + +## Что добавлено + +### ClickHouse + +Файл: + +- `clickhouse-1c/clickhouse/init/04_company_intelligence.sql` + +Создаёт: + +- `company_forecasts` +- `company_health_signals` +- `v_counterparty_daily` +- `v_counterparty_latest_activity` +- `v_company_forecasts_current` +- `v_company_health_current` +- `v_company_portfolio_overview` + +### Forecast refresh + +Файл: + +- `clickhouse-1c/ai/refresh_company_intelligence.py` + +Что делает: + +- строит daily series по `counterparty`; +- считает базовую линию и линейный тренд; +- материализует прогнозы на `7` и `30` дней; +- создаёт health-signals: + - `inactive_company` + - `amount_drop` + - `docs_stopped` + - `open_cases` + - `open_detections` + +### Read-only API + +Файл: + +- `clickhouse-1c/ai/company_intelligence_api.py` + +Endpoints: + +- `GET /health` +- `GET /api/1/analytics-1c/companies/overview` +- `GET /api/1/analytics-1c/companies/{counterparty}/summary` +- `GET /api/1/analytics-1c/companies/{counterparty}/forecast` +- `GET /api/1/analytics-1c/companies/{counterparty}/timeline` + +### Ops + +Файлы: + +- `clickhouse-1c/ops/run_company_intelligence_refresh.sh` +- `clickhouse-1c/ops/run_company_intelligence_api.sh` +- `clickhouse-1c/ops/aw-1c-company-api.service` + +И `run_ingest_cycle.sh` теперь: + +1. грузит новые выгрузки; +2. обновляет timeline/detections/cases; +3. применяет `04_company_intelligence.sql`; +4. пересчитывает company forecasts/signals. + +## Развёртывание + +### 1. Установить зависимости + +```bash +cd clickhouse-1c +python3 -m venv .venv +. .venv/bin/activate +pip install -r etl/requirements.txt +pip install -r ai/requirements.txt +``` + +### 2. Применить schema + +```bash +clickhouse-client --queries-file clickhouse/init/04_company_intelligence.sql +``` + +### 3. Пересчитать company intelligence + +```bash +./ops/run_company_intelligence_refresh.sh +``` + +### 4. Запустить API + +```bash +./ops/run_company_intelligence_api.sh +``` + +По умолчанию: + +- host: `127.0.0.1` +- port: `8710` + +## Переменные окружения + +См.: + +- `clickhouse-1c/.env.example` + +Ключевые: + +- `AW_1C_COMPANY_API_HOST` +- `AW_1C_COMPANY_API_PORT` +- `AW_1C_COMPANY_LOOKBACK_DAYS` +- `AW_1C_COMPANY_MIN_DAYS` +- `AW_1C_COMPANY_HORIZONS` + +## Что прогноз реально означает + +Это не black-box ML и не «магический AI». + +Сейчас используется объяснимый MVP: + +- daily baseline; +- linear trend; +- confidence; +- health signals на простых правилах. + +Этого достаточно для: + +- раннего обнаружения выпадения компаний из потока; +- ранжирования портфеля; +- AI summary поверх уже объяснимых чисел. + +## Ограничения + +- без `counterparty` в выгрузках слой пустой; +- это прогноз тенденции, а не финансовое обещание; +- API строго read-only; +- никакой записи обратно в 1С нет. diff --git a/docs/wiki/1C-Company-Intelligence.md b/docs/wiki/1C-Company-Intelligence.md new file mode 100644 index 0000000..cc53c79 --- /dev/null +++ b/docs/wiki/1C-Company-Intelligence.md @@ -0,0 +1,17 @@ +# 1C Company Intelligence + +Слой `1C Company Intelligence` расширяет `File 1C analytics` и добавляет: + +- mart по `counterparty`; +- прогноз по документам и объёму на `7/30` дней; +- health-signals по компаниям; +- read-only API для AI Investigator; +- source dashboard `1c-file-companies`. + +Основной документ: + +- [1C_COMPANY_INTELLIGENCE_RU.md](../1C_COMPANY_INTELLIGENCE_RU.md) + +Ключевая граница: + +- если `counterparty` в live-выгрузках пустой, слой остаётся пустым честно и ничего не симулирует. diff --git a/docs/wiki/Home.md b/docs/wiki/Home.md index 56f82ec..1fc807d 100644 --- a/docs/wiki/Home.md +++ b/docs/wiki/Home.md @@ -16,6 +16,7 @@ - [Hayabusa Security Analytics](Hayabusa-Security-Analytics) - текущий production-контур: auto-upload, auto-case, severity scoring и Telegram alerts - [Security analytics stack v1](../security-analytics-stack-v1.md) - целевая v1-модель без претензии на Splunk-class SIEM - [File 1C analytics](File-1C-Analytics) - ClickHouse/Grafana/AI Investigator контур для файловой 1С +- [1C Company Intelligence](1C-Company-Intelligence) - AI-ready слой анализа и прогноза по компаниям поверх файловой 1С ### Компоненты - [DLP Endpoint Monitoring](DLP-Endpoint-Monitoring) - мониторинг clipboard, печати, USB