feat(1c): decouple company intelligence from 1c names
This commit is contained in:
igor04091968
@@ -39,38 +39,101 @@ CREATE TABLE IF NOT EXISTS analytics_1c.company_health_signals
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (generated_at, severity, infobase, counterparty, signal_id);
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_daily AS
|
||||
SELECT
|
||||
toDate(ts) AS d,
|
||||
infobase,
|
||||
organization,
|
||||
counterparty,
|
||||
count() AS docs_total,
|
||||
sum(amount) AS amount_total,
|
||||
countIf(posted = 1) AS posted_docs_total,
|
||||
countIf(posted = 0) AS unposted_docs_total,
|
||||
countIf(status = 'busy') AS busy_docs_total,
|
||||
countIf(status = 'online') AS online_docs_total,
|
||||
uniqExact(doc_type) AS doc_types_total
|
||||
FROM analytics_1c.documents
|
||||
WHERE counterparty != ''
|
||||
GROUP BY d, infobase, organization, counterparty;
|
||||
CREATE TABLE IF NOT EXISTS analytics_1c.company_registry_bindings
|
||||
(
|
||||
ts DateTime,
|
||||
infobase LowCardinality(String),
|
||||
company_entity_key String,
|
||||
base_id String,
|
||||
base_path String,
|
||||
base_path_key String,
|
||||
registry_company_key String,
|
||||
registry_company_name String,
|
||||
binding_source LowCardinality(String),
|
||||
note String
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (company_entity_key, ts);
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_latest_activity AS
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_companies_current AS
|
||||
SELECT
|
||||
infobase,
|
||||
company_name,
|
||||
organization,
|
||||
counterparty,
|
||||
max(ts) AS last_seen_at,
|
||||
argMax(doc_type, ts) AS last_doc_type,
|
||||
argMax(operation_type, ts) AS last_operation_type,
|
||||
argMax(status, ts) AS last_status,
|
||||
argMax(amount, ts) AS last_amount,
|
||||
owner_user,
|
||||
base_id,
|
||||
base_path,
|
||||
trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(base_path), '[\\\\/]+', '/'), '[^0-9A-ZА-ЯЁ:/._ -]+', ' '), '\\s+', ' ')) AS base_path_key,
|
||||
multiIf(
|
||||
base_id != '', concat('baseid:', base_id),
|
||||
base_path != '', concat('basepath:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(base_path), '[\\\\/]+', '/'), '[^0-9A-ZА-ЯЁ:/._ -]+', ' '), '\\s+', ' '))),
|
||||
concat('infobase:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(infobase), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')))
|
||||
) AS company_entity_key,
|
||||
current_status,
|
||||
db_size_bytes,
|
||||
reglog_size_bytes,
|
||||
active_locks,
|
||||
temp_db_present,
|
||||
scheduler_touched,
|
||||
current_activity_score,
|
||||
last_company_snapshot_at
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
argMax(company_name, ts) AS company_name,
|
||||
argMax(organization, ts) AS organization,
|
||||
argMax(owner_user, ts) AS owner_user,
|
||||
argMax(base_id, ts) AS base_id,
|
||||
argMax(base_path, ts) AS base_path,
|
||||
argMax(status, ts) AS current_status,
|
||||
argMax(db_size_bytes, ts) AS db_size_bytes,
|
||||
argMax(reglog_size_bytes, ts) AS reglog_size_bytes,
|
||||
argMax(active_locks, ts) AS active_locks,
|
||||
argMax(temp_db_present, ts) AS temp_db_present,
|
||||
argMax(scheduler_touched, ts) AS scheduler_touched,
|
||||
argMax(activity_score, ts) AS current_activity_score,
|
||||
max(ts) AS last_company_snapshot_at
|
||||
FROM analytics_1c.companies
|
||||
GROUP BY infobase
|
||||
);
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_activity_daily AS
|
||||
SELECT
|
||||
toDate(documents.ts) AS d,
|
||||
documents.infobase AS infobase,
|
||||
ifNull(companies.organization, documents.organization) AS organization,
|
||||
ifNull(companies.company_entity_key, concat('infobase:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(documents.infobase), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')))) AS company_entity_key,
|
||||
argMax(documents.counterparty, documents.ts) AS source_counterparty,
|
||||
count() AS docs_total,
|
||||
sum(documents.amount) AS amount_total,
|
||||
countIf(documents.posted = 1) AS posted_docs_total,
|
||||
countIf(documents.posted = 0) AS unposted_docs_total,
|
||||
countIf(documents.status = 'busy') AS busy_docs_total,
|
||||
countIf(documents.status = 'online') AS online_docs_total,
|
||||
uniqExact(documents.doc_type) AS doc_types_total
|
||||
FROM analytics_1c.documents AS documents
|
||||
LEFT JOIN analytics_1c.v_companies_current AS companies ON companies.infobase = documents.infobase
|
||||
WHERE documents.counterparty != ''
|
||||
GROUP BY d, documents.infobase, ifNull(companies.organization, documents.organization), ifNull(companies.company_entity_key, concat('infobase:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(documents.infobase), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' '))));
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_activity_latest AS
|
||||
SELECT
|
||||
documents.infobase AS infobase,
|
||||
ifNull(companies.organization, documents.organization) AS organization,
|
||||
ifNull(companies.company_entity_key, concat('infobase:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(documents.infobase), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')))) AS company_entity_key,
|
||||
argMax(documents.counterparty, documents.ts) AS source_counterparty,
|
||||
max(documents.ts) AS last_seen_at,
|
||||
argMax(documents.doc_type, documents.ts) AS last_doc_type,
|
||||
argMax(documents.operation_type, documents.ts) AS last_operation_type,
|
||||
argMax(documents.status, documents.ts) AS last_status,
|
||||
argMax(documents.amount, documents.ts) AS last_amount,
|
||||
count() AS docs_lifetime,
|
||||
sum(amount) AS amount_lifetime
|
||||
FROM analytics_1c.documents
|
||||
WHERE counterparty != ''
|
||||
GROUP BY infobase, organization, counterparty;
|
||||
sum(documents.amount) AS amount_lifetime
|
||||
FROM analytics_1c.documents AS documents
|
||||
LEFT JOIN analytics_1c.v_companies_current AS companies ON companies.infobase = documents.infobase
|
||||
WHERE documents.counterparty != ''
|
||||
GROUP BY documents.infobase, ifNull(companies.organization, documents.organization), ifNull(companies.company_entity_key, concat('infobase:', trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(documents.infobase), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' '))));
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_forecasts_current AS
|
||||
SELECT *
|
||||
@@ -82,24 +145,35 @@ SELECT *
|
||||
FROM analytics_1c.company_health_signals
|
||||
WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_health_signals);
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_companies_current AS
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_daily AS
|
||||
SELECT
|
||||
d,
|
||||
infobase,
|
||||
organization,
|
||||
company_entity_key AS counterparty,
|
||||
docs_total,
|
||||
amount_total,
|
||||
posted_docs_total,
|
||||
unposted_docs_total,
|
||||
busy_docs_total,
|
||||
online_docs_total,
|
||||
doc_types_total
|
||||
FROM analytics_1c.v_company_activity_daily;
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_latest_activity AS
|
||||
SELECT
|
||||
infobase,
|
||||
argMax(company_name, ts) AS company_name,
|
||||
argMax(organization, ts) AS organization,
|
||||
argMax(owner_user, ts) AS owner_user,
|
||||
argMax(base_id, ts) AS base_id,
|
||||
argMax(base_path, ts) AS base_path,
|
||||
argMax(status, ts) AS current_status,
|
||||
argMax(db_size_bytes, ts) AS db_size_bytes,
|
||||
argMax(reglog_size_bytes, ts) AS reglog_size_bytes,
|
||||
argMax(active_locks, ts) AS active_locks,
|
||||
argMax(temp_db_present, ts) AS temp_db_present,
|
||||
argMax(scheduler_touched, ts) AS scheduler_touched,
|
||||
argMax(activity_score, ts) AS current_activity_score,
|
||||
max(ts) AS last_company_snapshot_at
|
||||
FROM analytics_1c.companies
|
||||
GROUP BY infobase;
|
||||
organization,
|
||||
company_entity_key AS counterparty,
|
||||
source_counterparty,
|
||||
last_seen_at,
|
||||
last_doc_type,
|
||||
last_operation_type,
|
||||
last_status,
|
||||
last_amount,
|
||||
docs_lifetime,
|
||||
amount_lifetime
|
||||
FROM analytics_1c.v_company_activity_latest;
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_registry_current AS
|
||||
SELECT
|
||||
@@ -115,6 +189,21 @@ SELECT
|
||||
FROM analytics_1c.company_registry
|
||||
GROUP BY company_key;
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_registry_bindings_current AS
|
||||
SELECT
|
||||
company_entity_key,
|
||||
argMax(infobase, ts) AS infobase,
|
||||
argMax(base_id, ts) AS base_id,
|
||||
argMax(base_path, ts) AS base_path,
|
||||
argMax(base_path_key, ts) AS base_path_key,
|
||||
argMax(registry_company_key, ts) AS registry_company_key,
|
||||
argMax(registry_company_name, ts) AS registry_company_name,
|
||||
argMax(binding_source, ts) AS binding_source,
|
||||
argMax(note, ts) AS note,
|
||||
max(ts) AS last_binding_at
|
||||
FROM analytics_1c.company_registry_bindings
|
||||
GROUP BY company_entity_key;
|
||||
|
||||
CREATE OR REPLACE VIEW analytics_1c.v_company_registry_alias_map AS
|
||||
SELECT
|
||||
source_company_key,
|
||||
@@ -220,7 +309,7 @@ base AS
|
||||
(
|
||||
SELECT
|
||||
*,
|
||||
trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(counterparty), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')) AS counterparty_key
|
||||
trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(source_counterparty), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')) AS source_counterparty_key
|
||||
FROM analytics_1c.v_counterparty_latest_activity
|
||||
),
|
||||
d7 AS
|
||||
@@ -252,6 +341,11 @@ company_state AS
|
||||
SELECT *
|
||||
FROM analytics_1c.v_companies_current
|
||||
),
|
||||
binding_state AS
|
||||
(
|
||||
SELECT *
|
||||
FROM analytics_1c.v_company_registry_bindings_current
|
||||
),
|
||||
registry_state AS
|
||||
(
|
||||
SELECT *
|
||||
@@ -271,18 +365,18 @@ signals AS
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
counterparty,
|
||||
counterparty AS company_entity_key,
|
||||
max(score) AS signal_score,
|
||||
argMax(severity, score) AS signal_severity,
|
||||
argMax(summary, score) AS top_signal
|
||||
FROM analytics_1c.v_company_health_current
|
||||
GROUP BY infobase, counterparty
|
||||
GROUP BY infobase, company_entity_key
|
||||
),
|
||||
amount_forecast AS
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
counterparty,
|
||||
counterparty AS company_entity_key,
|
||||
predicted_total AS amount_forecast_30d,
|
||||
confidence AS amount_forecast_confidence
|
||||
FROM analytics_1c.v_company_forecasts_current
|
||||
@@ -293,7 +387,7 @@ docs_forecast AS
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
counterparty,
|
||||
counterparty AS company_entity_key,
|
||||
predicted_total AS docs_forecast_30d,
|
||||
confidence AS docs_forecast_confidence
|
||||
FROM analytics_1c.v_company_forecasts_current
|
||||
@@ -304,30 +398,35 @@ cases_current AS
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
entity_id AS counterparty,
|
||||
entity_id AS company_entity_key,
|
||||
countIf(status != 'closed') AS open_cases_total
|
||||
FROM analytics_1c.cases
|
||||
WHERE entity_type = 'counterparty'
|
||||
GROUP BY infobase, counterparty
|
||||
GROUP BY infobase, company_entity_key
|
||||
),
|
||||
detections_current AS
|
||||
(
|
||||
SELECT
|
||||
infobase,
|
||||
entity_id AS counterparty,
|
||||
entity_id AS company_entity_key,
|
||||
count() AS detections_total
|
||||
FROM analytics_1c.detections
|
||||
WHERE entity_type = 'counterparty'
|
||||
AND status != 'closed'
|
||||
GROUP BY infobase, counterparty
|
||||
GROUP BY infobase, company_entity_key
|
||||
)
|
||||
SELECT
|
||||
base.infobase AS infobase,
|
||||
base.counterparty AS company_entity_key,
|
||||
if(company_state.organization != '', company_state.organization, base.organization) AS organization,
|
||||
base.counterparty AS counterparty,
|
||||
if(alias_state.target_company_name != '', alias_state.target_company_name, if(manual_state.company_name != '', manual_state.company_name, if(company_state.company_name != '', company_state.company_name, base.counterparty))) AS company_name,
|
||||
if(alias_state.target_company_name != '', alias_state.target_company_name, if(manual_state.company_name != '', manual_state.company_name, base.counterparty)) AS normalized_counterparty,
|
||||
multiIf(ifNull(alias_state.exclude_from_portfolio, 0) = 1, 'excluded', alias_state.target_company_key != '' AND registry_state.company_key != '', 'alias', registry_state.company_key != '', 'direct', manual_state.company_key != '', 'manual', 'none') AS registry_match_mode,
|
||||
base.source_counterparty AS source_counterparty,
|
||||
if(binding_state.registry_company_name != '', binding_state.registry_company_name, if(alias_state.target_company_name != '', alias_state.target_company_name, if(manual_state.company_name != '', manual_state.company_name, if(company_state.company_name != '', company_state.company_name, base.source_counterparty)))) AS counterparty,
|
||||
if(binding_state.registry_company_name != '', binding_state.registry_company_name, if(alias_state.target_company_name != '', alias_state.target_company_name, if(manual_state.company_name != '', manual_state.company_name, if(company_state.company_name != '', company_state.company_name, base.source_counterparty)))) AS company_name,
|
||||
if(binding_state.registry_company_name != '', binding_state.registry_company_name, if(alias_state.target_company_name != '', alias_state.target_company_name, if(manual_state.company_name != '', manual_state.company_name, base.source_counterparty))) AS normalized_counterparty,
|
||||
multiIf(binding_state.registry_company_key != '', 'technical', ifNull(alias_state.exclude_from_portfolio, 0) = 1, 'excluded', alias_state.target_company_key != '' AND registry_state.company_key != '', 'alias', registry_state.company_key != '', 'direct', manual_state.company_key != '', 'manual', 'none') AS registry_match_mode,
|
||||
if(binding_state.registry_company_key != '', binding_state.registry_company_key, if(registry_state.company_key != '', registry_state.company_key, ifNull(manual_state.company_key, ''))) AS registry_company_key,
|
||||
ifNull(binding_state.binding_source, '') AS registry_binding_source,
|
||||
ifNull(binding_state.note, '') AS registry_binding_note,
|
||||
if(registry_state.assignee_name != '', registry_state.assignee_name, ifNull(manual_state.assignee_name, '')) AS registry_assignee_name,
|
||||
if(registry_state.registry_status != '', registry_state.registry_status, ifNull(manual_state.registry_status, '')) AS registry_status,
|
||||
if(registry_state.share_text != '', registry_state.share_text, ifNull(manual_state.share_text, '')) AS registry_share_text,
|
||||
@@ -337,6 +436,7 @@ SELECT
|
||||
ifNull(company_state.owner_user, '') AS owner_user,
|
||||
ifNull(company_state.base_id, '') AS base_id,
|
||||
ifNull(company_state.base_path, '') AS base_path,
|
||||
ifNull(company_state.base_path_key, '') AS base_path_key,
|
||||
base.last_seen_at,
|
||||
company_state.last_company_snapshot_at,
|
||||
base.last_doc_type,
|
||||
@@ -367,14 +467,15 @@ SELECT
|
||||
ifNull(signals.top_signal, '') AS top_signal
|
||||
FROM base
|
||||
LEFT JOIN company_state ON company_state.infobase = base.infobase
|
||||
LEFT JOIN alias_state ON alias_state.source_company_key = base.counterparty_key
|
||||
LEFT JOIN registry_state ON registry_state.company_key = if(alias_state.target_company_key != '', alias_state.target_company_key, base.counterparty_key)
|
||||
LEFT JOIN manual_state ON manual_state.company_key = if(alias_state.target_company_key != '', alias_state.target_company_key, base.counterparty_key)
|
||||
LEFT JOIN binding_state ON binding_state.company_entity_key = base.counterparty
|
||||
LEFT JOIN alias_state ON alias_state.source_company_key = base.source_counterparty_key
|
||||
LEFT JOIN registry_state ON registry_state.company_key = if(binding_state.registry_company_key != '', binding_state.registry_company_key, if(alias_state.target_company_key != '', alias_state.target_company_key, base.source_counterparty_key))
|
||||
LEFT JOIN manual_state ON manual_state.company_key = if(binding_state.registry_company_key != '', binding_state.registry_company_key, if(alias_state.target_company_key != '', alias_state.target_company_key, base.source_counterparty_key))
|
||||
LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty
|
||||
LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty
|
||||
LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty
|
||||
LEFT JOIN amount_forecast ON amount_forecast.infobase = base.infobase AND amount_forecast.counterparty = base.counterparty
|
||||
LEFT JOIN docs_forecast ON docs_forecast.infobase = base.infobase AND docs_forecast.counterparty = base.counterparty
|
||||
LEFT JOIN cases_current ON cases_current.infobase = base.infobase AND cases_current.counterparty = base.counterparty
|
||||
LEFT JOIN detections_current ON detections_current.infobase = base.infobase AND detections_current.counterparty = base.counterparty
|
||||
LEFT JOIN signals ON signals.infobase = base.infobase AND signals.company_entity_key = base.counterparty
|
||||
LEFT JOIN amount_forecast ON amount_forecast.infobase = base.infobase AND amount_forecast.company_entity_key = base.counterparty
|
||||
LEFT JOIN docs_forecast ON docs_forecast.infobase = base.infobase AND docs_forecast.company_entity_key = base.counterparty
|
||||
LEFT JOIN cases_current ON cases_current.infobase = base.infobase AND cases_current.company_entity_key = base.counterparty
|
||||
LEFT JOIN detections_current ON detections_current.infobase = base.infobase AND detections_current.company_entity_key = base.counterparty
|
||||
WHERE ifNull(alias_state.exclude_from_portfolio, 0) = 0;
|
||||
|
||||
Reference in New Issue
Block a user