awatch-rus/AWatch-rus
1
0
Fork 0
Code Issues Pull Requests Actions 100 Packages Projects Releases Wiki Activity

feat(1c): add read-only companies export layer

Browse Source
This commit is contained in:
igor04091968
2026-05-22 10:46:06 +03:00
parent 31fe43f4bc
commit 02e896b05a
13 changed files with 214 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/deploy_file_1c_analytics.yml
+18
View File
@@ -134,6 +134,7 @@
landing:
documents: {{ aw_file_1c_root }}/landing/documents
postings: {{ aw_file_1c_root }}/landing/postings
companies: {{ aw_file_1c_root }}/landing/companies
reglog: {{ aw_file_1c_root }}/landing/reglog
audit: {{ aw_file_1c_root }}/landing/audit
host: {{ aw_file_1c_root }}/landing/host
@@ -142,6 +143,7 @@
default: jsonl
documents: jsonl
postings: jsonl
companies: jsonl
reglog: jsonl
audit: jsonl
host: jsonl
@@ -181,6 +183,22 @@
args:
chdir: "{{ aw_file_1c_root }}"
- name: Переапплиить ClickHouse schema и views для file-1C analytics
ansible.builtin.shell: |
set -eu
docker exec -i aw-rus-1c-clickhouse clickhouse-client \
--user {{ aw_file_1c_clickhouse_user }} \
--password {{ aw_file_1c_clickhouse_password }} \
--multiquery < "{{ aw_file_1c_root }}/clickhouse/init/{{ item }}"
args:
executable: /bin/bash
loop:
- 00_database.sql
- 01_raw_tables.sql
- 02_core_tables.sql
- 03_views.sql
- 04_company_intelligence.sql
- name: Установить systemd unit aw-1c-ingest.service
ansible.builtin.copy:
src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.service"
clickhouse-1c/ai/company_intelligence_api.py
+20 -2
View File
@@ -59,6 +59,7 @@ def health() -> dict[str, Any]:
"""
SELECT
countIf(counterparty != '') AS documents_with_counterparty,
(SELECT count() FROM analytics_1c.companies) AS companies_total,
(SELECT count() FROM analytics_1c.company_forecasts) AS forecasts_total,
(SELECT count() FROM analytics_1c.company_health_signals) AS health_signals_total
FROM analytics_1c.documents
@@ -83,6 +84,14 @@ def companies_overview(
infobase,
organization,
counterparty,
company_name,
owner_user,
base_path,
current_status,
db_size_bytes,
reglog_size_bytes,
active_locks,
current_activity_score,
last_seen_at,
days_since_last_activity,
docs_7d,
@@ -135,6 +144,16 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str,
ORDER BY score DESC, generated_at DESC
"""
timeline_sql = f"""
SELECT ts, infobase, company_name, owner_user, current_status, db_size_bytes, reglog_size_bytes, active_locks, current_activity_score
FROM analytics_1c.v_company_portfolio_overview
WHERE counterparty = {q(counterparty)}
{"AND infobase = " + q(infobase) if infobase else ""}
LIMIT 1
"""
forecasts = rows_to_dict(client.query(forecast_sql))
signals = rows_to_dict(client.query(signals_sql))
company_state = rows_to_dict(client.query(timeline_sql))
timeline_sql = f"""
SELECT ts, infobase, doc_type, operation_type, amount, status, author
FROM analytics_1c.documents
WHERE counterparty = {q(counterparty)}
@@ -142,8 +161,6 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str,
ORDER BY ts DESC
LIMIT 20
"""
forecasts = rows_to_dict(client.query(forecast_sql))
signals = rows_to_dict(client.query(signals_sql))
timeline = rows_to_dict(client.query(timeline_sql))
essence = (
f"Компания {counterparty}: за 30 дней событий {card['docs_30d']}, суммарная активность {card['amount_30d']}, "
@@ -152,6 +169,7 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str,
return {
"essence": essence,
"card": card,
"company_state": company_state[0] if company_state else None,
"forecasts": forecasts,
"signals": signals,
"recent_documents": timeline,
clickhouse-1c/ai/refresh_company_intelligence.py
+27
View File
@@ -164,6 +164,22 @@ def main() -> int:
""",
)
}
company_state_map = {
row["infobase"]: row
for row in query_rows(
client,
"""
SELECT
infobase,
current_status,
active_locks,
temp_db_present,
scheduler_touched,
current_activity_score
FROM analytics_1c.v_companies_current
""",
)
}
forecast_rows: list[list[Any]] = []
signal_rows: list[list[Any]] = []
@@ -186,6 +202,12 @@ def main() -> int:
days_since_last_activity = (date.today() - latest_day).days
open_cases_total = cases_map.get((infobase, counterparty), 0)
detections_total = detections_map.get((infobase, counterparty), 0)
company_state = company_state_map.get(infobase, {})
current_status = str(company_state.get("current_status") or "")
active_locks = int(company_state.get("active_locks") or 0)
temp_db_present = int(company_state.get("temp_db_present") or 0)
scheduler_touched = int(company_state.get("scheduler_touched") or 0)
current_activity_score = float(company_state.get("current_activity_score") or 0)
for metric, values in (("docs_total", docs_series), ("amount_total", amount_series)):
for horizon in horizons:
@@ -221,6 +243,11 @@ def main() -> int:
signals.append(("amount_drop", 70, "high", f"Активность по компании {counterparty} упала более чем на 50% неделя к неделе."))
if docs_prev_7d > 0 and docs_7d == 0:
signals.append(("docs_stopped", 55, "medium", f"По компании {counterparty} прекратился поток документов за последние 7 дней."))
if current_status == "busy" or active_locks > 0 or temp_db_present > 0:
score = min(85, 45 + active_locks * 5 + temp_db_present * 10)
signals.append(("base_busy", score, severity_score_to_label(score), f"Файловая база компании {counterparty} занята: status={current_status}, locks={active_locks}, tempDb={temp_db_present}."))
if scheduler_touched > 0 and current_activity_score >= 15:
signals.append(("scheduler_activity", 35, "medium", f"По компании {counterparty} есть активность scheduler и повышенный activity score {current_activity_score}."))
if open_cases_total > 0:
signals.append(("open_cases", min(95, 40 + open_cases_total * 10), severity_score_to_label(min(95, 40 + open_cases_total * 10)), f"По компании {counterparty} есть открытые кейсы: {open_cases_total}."))
if detections_total > 0:
clickhouse-1c/clickhouse/init/01_raw_tables.sql
+9
View File
@@ -16,6 +16,15 @@ CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_postings
ENGINE = MergeTree
ORDER BY (ingested_at, source_file);
CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_companies
(
ingested_at DateTime DEFAULT now(),
source_file String,
payload String
)
ENGINE = MergeTree
ORDER BY (ingested_at, source_file);
CREATE TABLE IF NOT EXISTS analytics_1c.raw_reglog
(
ingested_at DateTime DEFAULT now(),
clickhouse-1c/clickhouse/init/02_core_tables.sql
+21
View File
@@ -32,6 +32,27 @@ CREATE TABLE IF NOT EXISTS analytics_1c.postings
ENGINE = MergeTree
ORDER BY (infobase, ts, registrar);
CREATE TABLE IF NOT EXISTS analytics_1c.companies
(
ts DateTime,
infobase LowCardinality(String),
company_name String,
organization String,
owner_user String,
base_id String,
base_path String,
status LowCardinality(String),
db_size_bytes UInt64,
reglog_size_bytes UInt64,
active_locks UInt32,
temp_db_present UInt8,
scheduler_touched UInt8,
activity_score Float32,
source_file String
)
ENGINE = MergeTree
ORDER BY (infobase, ts);
CREATE TABLE IF NOT EXISTS analytics_1c.reglog_events
(
ts DateTime,
clickhouse-1c/clickhouse/init/03_views.sql
+3 -3
View File
@@ -1,4 +1,4 @@
CREATE VIEW IF NOT EXISTS analytics_1c.v_documents_daily AS
CREATE OR REPLACE VIEW analytics_1c.v_documents_daily AS
SELECT
toDate(ts) AS d,
infobase,
@@ -10,7 +10,7 @@ SELECT
FROM analytics_1c.documents
GROUP BY d, infobase, organization, doc_type;
CREATE VIEW IF NOT EXISTS analytics_1c.v_detections_daily AS
CREATE OR REPLACE VIEW analytics_1c.v_detections_daily AS
SELECT
toDate(ts) AS d,
infobase,
@@ -20,7 +20,7 @@ SELECT
FROM analytics_1c.detections
GROUP BY d, infobase, severity;
CREATE VIEW IF NOT EXISTS analytics_1c.v_open_cases AS
CREATE OR REPLACE VIEW analytics_1c.v_open_cases AS
SELECT *
FROM analytics_1c.cases
WHERE status != 'closed';
clickhouse-1c/clickhouse/init/04_company_intelligence.sql
+43 -6
View File
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS analytics_1c.company_health_signals
ENGINE = MergeTree
ORDER BY (generated_at, severity, infobase, counterparty, signal_id);
CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_daily AS
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_daily AS
SELECT
toDate(ts) AS d,
infobase,
@@ -56,7 +56,7 @@ FROM analytics_1c.documents
WHERE counterparty != ''
GROUP BY d, infobase, organization, counterparty;
CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_latest_activity AS
CREATE OR REPLACE VIEW analytics_1c.v_counterparty_latest_activity AS
SELECT
infobase,
organization,
@@ -72,17 +72,36 @@ FROM analytics_1c.documents
WHERE counterparty != ''
GROUP BY infobase, organization, counterparty;
CREATE VIEW IF NOT EXISTS analytics_1c.v_company_forecasts_current AS
CREATE OR REPLACE VIEW analytics_1c.v_company_forecasts_current AS
SELECT *
FROM analytics_1c.company_forecasts
WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_forecasts);
CREATE VIEW IF NOT EXISTS analytics_1c.v_company_health_current AS
CREATE OR REPLACE VIEW analytics_1c.v_company_health_current AS
SELECT *
FROM analytics_1c.company_health_signals
WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_health_signals);
CREATE VIEW IF NOT EXISTS analytics_1c.v_company_portfolio_overview AS
CREATE OR REPLACE VIEW analytics_1c.v_companies_current AS
SELECT
infobase,
argMax(company_name, ts) AS company_name,
argMax(organization, ts) AS organization,
argMax(owner_user, ts) AS owner_user,
argMax(base_id, ts) AS base_id,
argMax(base_path, ts) AS base_path,
argMax(status, ts) AS current_status,
argMax(db_size_bytes, ts) AS db_size_bytes,
argMax(reglog_size_bytes, ts) AS reglog_size_bytes,
argMax(active_locks, ts) AS active_locks,
argMax(temp_db_present, ts) AS temp_db_present,
argMax(scheduler_touched, ts) AS scheduler_touched,
argMax(activity_score, ts) AS current_activity_score,
max(ts) AS last_company_snapshot_at
FROM analytics_1c.companies
GROUP BY infobase;
CREATE OR REPLACE VIEW analytics_1c.v_company_portfolio_overview AS
WITH
base AS
(
@@ -113,6 +132,11 @@ d30 AS
WHERE d >= today() - 30
GROUP BY infobase, counterparty
),
company_state AS
(
SELECT *
FROM analytics_1c.v_companies_current
),
signals AS
(
SELECT
@@ -169,12 +193,24 @@ detections_current AS
)
SELECT
base.infobase AS infobase,
base.organization,
if(company_state.organization != '', company_state.organization, base.organization) AS organization,
base.counterparty AS counterparty,
if(company_state.company_name != '', company_state.company_name, base.counterparty) AS company_name,
ifNull(company_state.owner_user, '') AS owner_user,
ifNull(company_state.base_id, '') AS base_id,
ifNull(company_state.base_path, '') AS base_path,
base.last_seen_at,
company_state.last_company_snapshot_at,
base.last_doc_type,
base.last_operation_type,
base.last_status,
ifNull(company_state.current_status, base.last_status) AS current_status,
ifNull(company_state.db_size_bytes, 0) AS db_size_bytes,
ifNull(company_state.reglog_size_bytes, 0) AS reglog_size_bytes,
ifNull(company_state.active_locks, 0) AS active_locks,
ifNull(company_state.temp_db_present, 0) AS temp_db_present,
ifNull(company_state.scheduler_touched, 0) AS scheduler_touched,
ifNull(company_state.current_activity_score, 0) AS current_activity_score,
dateDiff('day', toDate(base.last_seen_at), today()) AS days_since_last_activity,
ifNull(d7.docs_7d, 0) AS docs_7d,
ifNull(d7.amount_7d, 0) AS amount_7d,
@@ -192,6 +228,7 @@ SELECT
ifNull(signals.signal_score, 0) AS signal_score,
ifNull(signals.top_signal, '') AS top_signal
FROM base
LEFT JOIN company_state ON company_state.infobase = base.infobase
LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty
LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty
LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty
clickhouse-1c/detections/build_entity_timeline.sql
+19
View File
@@ -37,6 +37,25 @@ FROM (
) AS src
WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline);
INSERT INTO analytics_1c.entity_timeline
SELECT *
FROM (
SELECT
ts,
'counterparty' AS entity_type,
company_name AS entity_id,
infobase,
owner_user AS actor,
'companies' AS source,
'company_snapshot' AS event_type,
if(status = 'busy' OR active_locks > 0 OR temp_db_present = 1, 'medium', 'low') AS severity,
greatest(10, toUInt32(round(activity_score))) AS score,
concat('company:', infobase, ':', toString(toUnixTimestamp(ts))) AS ref_id,
concat('Company snapshot ', company_name, ': status=', status, ' locks=', toString(active_locks), ' score=', toString(activity_score)) AS summary
FROM analytics_1c.companies
) AS src
WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline);
INSERT INTO analytics_1c.entity_timeline
SELECT *
FROM (
clickhouse-1c/etl/config.example.yml
+2
View File
@@ -8,6 +8,7 @@ clickhouse:
landing:
documents: ./landing/documents
postings: ./landing/postings
companies: ./landing/companies
reglog: ./landing/reglog
audit: ./landing/audit
host: ./landing/host
@@ -16,6 +17,7 @@ formats:
default: jsonl
documents: jsonl
postings: jsonl
companies: jsonl
reglog: jsonl
audit: jsonl
host: jsonl
clickhouse-1c/etl/load_1c_exports.py
+23 -1
View File
@@ -17,6 +17,7 @@ from dateutil import parser as date_parser
RAW_TABLES = {
"documents": "raw_1c_documents",
"postings": "raw_1c_postings",
"companies": "raw_1c_companies",
"reglog": "raw_reglog",
"audit": "raw_audit",
"host": "raw_host_metrics",
@@ -25,6 +26,7 @@ RAW_TABLES = {
CORE_TABLES = {
"documents": "documents",
"postings": "postings",
"companies": "companies",
"reglog": "reglog_events",
"audit": "audit_events",
"host": "host_events",
@@ -44,7 +46,7 @@ class Config:
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Load file-based 1C exports into ClickHouse")
p.add_argument("--config", required=True, help="Path to YAML config")
p.add_argument("--dataset", choices=["documents", "postings", "reglog", "audit", "host"], help="Load only one dataset")
p.add_argument("--dataset", choices=["documents", "postings", "companies", "reglog", "audit", "host"], help="Load only one dataset")
return p.parse_args()
@@ -127,6 +129,24 @@ def map_core_row(dataset: str, source_file: str, row: dict[str, Any]) -> list[An
float(row.get("amount", 0) or 0),
source_file,
]
if dataset == "companies":
return [
normalize_ts(row.get("ts")),
row.get("infobase", ""),
row.get("company_name", row.get("counterparty", row.get("infobase", ""))),
row.get("organization", ""),
row.get("owner_user", row.get("author", "")),
row.get("base_id", row.get("doc_id", "")),
row.get("base_path", ""),
row.get("status", ""),
int(row.get("db_size_bytes", 0) or 0),
int(row.get("reglog_size_bytes", 0) or 0),
int(row.get("active_locks", 0) or 0),
int(row.get("temp_db_present", 0) or 0),
int(row.get("scheduler_touched", 0) or 0),
float(row.get("activity_score", row.get("amount", 0)) or 0),
source_file,
]
if dataset == "reglog":
return [
normalize_ts(row.get("ts")),
@@ -174,6 +194,8 @@ def core_columns(dataset: str) -> list[str]:
return ["ts", "infobase", "organization", "department", "doc_type", "doc_id", "doc_number", "author", "counterparty", "operation_type", "amount", "status", "posted", "source_file"]
if dataset == "postings":
return ["ts", "infobase", "registrar", "operation_type", "account_dt", "account_ct", "amount", "source_file"]
if dataset == "companies":
return ["ts", "infobase", "company_name", "organization", "owner_user", "base_id", "base_path", "status", "db_size_bytes", "reglog_size_bytes", "active_locks", "temp_db_present", "scheduler_touched", "activity_score", "source_file"]
if dataset == "reglog":
return ["ts", "infobase", "user", "host", "app", "event_name", "level", "duration_ms", "message", "source_file"]
if dataset == "audit":
clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json
+3 -3
View File
@@ -33,7 +33,7 @@
"format": 1,
"pluginVersion": "11.2.2",
"queryType": "table",
"rawSql": "SELECT countDistinct(counterparty) AS value FROM analytics_1c.v_counterparty_daily WHERE d >= today() - 30",
"rawSql": "SELECT count() AS value FROM analytics_1c.v_companies_current",
"refId": "A"
}
],
@@ -352,7 +352,7 @@
"format": 1,
"pluginVersion": "11.2.2",
"queryType": "table",
"rawSql": "SELECT infobase, counterparty, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20",
"rawSql": "SELECT infobase, company_name, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20",
"refId": "A"
}
],
@@ -475,6 +475,6 @@
"timezone": "browser",
"title": "1C File - Company Intelligence",
"uid": "1c-file-companies",
"version": 1,
"version": 3,
"weekStart": ""
}
clickhouse-1c/ops/bootstrap_runtime.sh
+2
View File
@@ -6,11 +6,13 @@ ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
mkdir -p \
"${ROOT}/landing/documents" \
"${ROOT}/landing/postings" \
"${ROOT}/landing/companies" \
"${ROOT}/landing/reglog" \
"${ROOT}/landing/audit" \
"${ROOT}/landing/host" \
"${ROOT}/archive/documents" \
"${ROOT}/archive/postings" \
"${ROOT}/archive/companies" \
"${ROOT}/archive/reglog" \
"${ROOT}/archive/audit" \
"${ROOT}/archive/host"
windows/export-upload-file-1c-telemetry.ps1
+24 -2
View File
@@ -268,6 +268,7 @@ $nextExporterState = @{}
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$documents = New-Object System.Collections.Generic.List[object]
$companies = New-Object System.Collections.Generic.List[object]
$reglog = New-Object System.Collections.Generic.List[object]
$audit = New-Object System.Collections.Generic.List[object]
@@ -333,6 +334,23 @@ foreach ($base in $infobases) {
posted = 1
})
$companies.Add([ordered]@{
ts = $nowUtc
infobase = [string]$base.infobase
company_name = [string]$base.infobase
organization = $organization
owner_user = $owner
base_id = $docId
base_path = [string]$base.path
status = $status
db_size_bytes = $dbSizeBytes
reglog_size_bytes = $mainLogBytes
active_locks = $activeLocks.Count
temp_db_present = if ($tempDb) { 1 } else { 0 }
scheduler_touched = if ($schedulerTouched) { 1 } else { 0 }
activity_score = $activityScore
})
if ($activityScore -gt 0) {
$documents.Add([ordered]@{
ts = $nowUtc
@@ -434,17 +452,20 @@ New-Item -ItemType Directory -Path $outRoot -Force | Out-Null
$files = @{
documents = Join-Path $outRoot "documents-$stamp.jsonl"
companies = Join-Path $outRoot "companies-$stamp.jsonl"
reglog = Join-Path $outRoot "reglog-$stamp.jsonl"
audit = Join-Path $outRoot "audit-$stamp.jsonl"
host = Join-Path $outRoot "host-$stamp.jsonl"
}
$documentRows = @($documents | ForEach-Object { $_ })
$companyRows = @($companies | ForEach-Object { $_ })
$reglogRows = @($reglog | ForEach-Object { $_ })
$auditRows = @($audit | ForEach-Object { $_ })
$hostRowsNormalized = @($hostRows | ForEach-Object { $_ })
Write-JsonLines -Path ([string]$files['documents']) -Rows $documentRows
Write-JsonLines -Path ([string]$files['companies']) -Rows $companyRows
Write-JsonLines -Path ([string]$files['reglog']) -Rows $reglogRows
Write-JsonLines -Path ([string]$files['audit']) -Rows $auditRows
Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized
@@ -452,8 +473,8 @@ Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized
$effectiveKeyPath = New-TemporarySshKeyCopy -SourceKeyPath $RemoteKeyPath
try {
Write-RunLog "prepared datasets documents=$($documentRows.Count) reglog=$($reglogRows.Count) audit=$($auditRows.Count) host=$($hostRowsNormalized.Count)"
foreach ($dataset in 'documents', 'reglog', 'audit', 'host') {
Write-RunLog "prepared datasets documents=$($documentRows.Count) companies=$($companyRows.Count) reglog=$($reglogRows.Count) audit=$($auditRows.Count) host=$($hostRowsNormalized.Count)"
foreach ($dataset in 'documents', 'companies', 'reglog', 'audit', 'host') {
Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files[$dataset]) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/"
}
Save-ExporterState -Path $ExporterStatePath -State $nextExporterState
@@ -473,6 +494,7 @@ Write-RunLog 'file1c exporter done'
infobases = @($infobases | ForEach-Object { $_.infobase })
datasets = [ordered]@{
documents = $documents.Count
companies = $companies.Count
reglog = $reglog.Count
audit = $audit.Count
host = $hostRows.Count