From 8993fbf0beedd9fdcaf6f72f97a1f09b646dd7f8 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Fri, 22 May 2026 11:29:40 +0300 Subject: [PATCH] feat(1c): normalize company registry aliases --- clickhouse-1c/ai/company_intelligence_api.py | 2 + .../ai/refresh_company_intelligence.py | 23 ++++++ .../init/04_company_intelligence.sql | 71 +++++++++++++++++-- .../files/1c-company-intelligence.json | 4 +- 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/clickhouse-1c/ai/company_intelligence_api.py b/clickhouse-1c/ai/company_intelligence_api.py index ea15844..6c58804 100644 --- a/clickhouse-1c/ai/company_intelligence_api.py +++ b/clickhouse-1c/ai/company_intelligence_api.py @@ -86,6 +86,8 @@ def companies_overview( organization, counterparty, company_name, + normalized_counterparty, + registry_match_mode, registry_assignee_name, registry_status, registry_share_text, diff --git a/clickhouse-1c/ai/refresh_company_intelligence.py b/clickhouse-1c/ai/refresh_company_intelligence.py index 96e7247..14d2de3 100644 --- a/clickhouse-1c/ai/refresh_company_intelligence.py +++ b/clickhouse-1c/ai/refresh_company_intelligence.py @@ -74,6 +74,16 @@ def linear_slope(values: list[float]) -> float: return num / den +def normalize_company_key(value: str) -> str: + import re + + text = (value or "").upper().replace("Ё", "Е") + text = re.sub(r"(^|\s)20\d{2}($|\s)", " ", text) + text = re.sub(r"[^0-9A-ZА-Я]+", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + def build_forecast(values: list[float], horizon: int, min_days: int, lookback_days: int) -> tuple[float, float, float, float, int, str]: if len(values) < min_days: raise ValueError("not enough data") @@ -180,11 +190,24 @@ def main() -> int: """, ) } + excluded_company_keys = { + str(row["source_company_key"]) + for row in query_rows( + client, + """ + SELECT source_company_key + FROM analytics_1c.v_company_registry_alias_map + WHERE exclude_from_portfolio = 1 + """, + ) + } forecast_rows: list[list[Any]] = [] signal_rows: list[list[Any]] = [] for (infobase, _organization, counterparty), points in grouped.items(): + if normalize_company_key(counterparty) in excluded_company_keys: + continue points.sort(key=lambda p: p.d) filled = fill_daily_series(points) docs_series = [p.docs_total for p in filled] diff --git a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql index 25ab962..8c338ba 100644 --- a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql +++ b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql @@ -115,11 +115,65 @@ SELECT FROM analytics_1c.company_registry GROUP BY company_key; +CREATE OR REPLACE VIEW analytics_1c.v_company_registry_alias_map AS +SELECT + source_company_key, + target_company_key, + target_company_name, + exclude_from_portfolio, + note +FROM +( + SELECT + 'ИНФОРМАЦИОННАЯ БАЗА' AS source_company_key, + '' AS target_company_key, + 'Информационная база' AS target_company_name, + toUInt8(1) AS exclude_from_portfolio, + 'system_filebase' AS note + UNION ALL + SELECT + 'МОСКОТЕЛЬИНКОВ АЛЕНСАНДР', + 'МОСКОТЕЛЬНИКОВ АЛ В', + 'МОСКОТЕЛЬНИКОВ АЛ.В', + toUInt8(0), + 'manual_alias_typo' + UNION ALL + SELECT + 'МОСКОТЕЛЬНИКОВ АЛЕКСАНДР', + 'МОСКОТЕЛЬНИКОВ АЛ В', + 'МОСКОТЕЛЬНИКОВ АЛ.В', + toUInt8(0), + 'manual_alias_expansion' + UNION ALL + SELECT + 'СЕРДИТОВ АНДРЕЙ', + 'СЕРДИТОВ АВ', + 'СЕРДИТОВ АВ', + toUInt8(0), + 'manual_alias_initials' + UNION ALL + SELECT + 'СЕРДИТОВ СЕРГЕЙ', + 'СЕРДИТОВ СВ', + 'СЕРДИТОВ СВ', + toUInt8(0), + 'manual_alias_initials' + UNION ALL + SELECT + 'СОРФИН', + 'СОЛФИН', + 'СОЛФИН', + toUInt8(0), + 'manual_alias_typo' +); + CREATE OR REPLACE VIEW analytics_1c.v_company_portfolio_overview AS WITH base AS ( - SELECT * + SELECT + *, + trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(counterparty), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')) AS counterparty_key FROM analytics_1c.v_counterparty_latest_activity ), d7 AS @@ -156,6 +210,11 @@ registry_state AS SELECT * FROM analytics_1c.v_company_registry_current ), +alias_state AS +( + SELECT * + FROM analytics_1c.v_company_registry_alias_map +), signals AS ( SELECT @@ -214,7 +273,9 @@ SELECT base.infobase AS infobase, if(company_state.organization != '', company_state.organization, base.organization) AS organization, base.counterparty AS counterparty, - if(company_state.company_name != '', company_state.company_name, base.counterparty) AS company_name, + if(alias_state.target_company_name != '', alias_state.target_company_name, if(company_state.company_name != '', company_state.company_name, base.counterparty)) AS company_name, + if(alias_state.target_company_name != '', alias_state.target_company_name, base.counterparty) AS normalized_counterparty, + multiIf(ifNull(alias_state.exclude_from_portfolio, 0) = 1, 'excluded', alias_state.target_company_key != '', 'alias', registry_state.company_key != '', 'direct', 'none') AS registry_match_mode, ifNull(registry_state.assignee_name, '') AS registry_assignee_name, ifNull(registry_state.registry_status, '') AS registry_status, ifNull(registry_state.share_text, '') AS registry_share_text, @@ -254,11 +315,13 @@ SELECT ifNull(signals.top_signal, '') AS top_signal FROM base LEFT JOIN company_state ON company_state.infobase = base.infobase -LEFT JOIN registry_state ON registry_state.company_key = trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(base.counterparty), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')) +LEFT JOIN alias_state ON alias_state.source_company_key = base.counterparty_key +LEFT JOIN registry_state ON registry_state.company_key = if(alias_state.target_company_key != '', alias_state.target_company_key, base.counterparty_key) LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty LEFT JOIN amount_forecast ON amount_forecast.infobase = base.infobase AND amount_forecast.counterparty = base.counterparty LEFT JOIN docs_forecast ON docs_forecast.infobase = base.infobase AND docs_forecast.counterparty = base.counterparty LEFT JOIN cases_current ON cases_current.infobase = base.infobase AND cases_current.counterparty = base.counterparty -LEFT JOIN detections_current ON detections_current.infobase = base.infobase AND detections_current.counterparty = base.counterparty; +LEFT JOIN detections_current ON detections_current.infobase = base.infobase AND detections_current.counterparty = base.counterparty +WHERE ifNull(alias_state.exclude_from_portfolio, 0) = 0; diff --git a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json index acc8b1f..0dad01b 100644 --- a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json +++ b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json @@ -352,7 +352,7 @@ "format": 1, "pluginVersion": "11.2.2", "queryType": "table", - "rawSql": "SELECT infobase, company_name, registry_assignee_name, registry_inn, registry_kpp, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", + "rawSql": "SELECT infobase, counterparty, normalized_counterparty, registry_match_mode, company_name, registry_assignee_name, registry_inn, registry_kpp, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", "refId": "A" } ], @@ -475,6 +475,6 @@ "timezone": "browser", "title": "1C File - Company Intelligence", "uid": "1c-file-companies", - "version": 4, + "version": 5, "weekStart": "" }