From 02e896b05ad8d363a4823b39833c363055539619 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Fri, 22 May 2026 10:46:06 +0300 Subject: [PATCH] feat(1c): add read-only companies export layer --- ansible/deploy_file_1c_analytics.yml | 18 +++++++ clickhouse-1c/ai/company_intelligence_api.py | 22 ++++++++- .../ai/refresh_company_intelligence.py | 27 ++++++++++ .../clickhouse/init/01_raw_tables.sql | 9 ++++ .../clickhouse/init/02_core_tables.sql | 21 ++++++++ clickhouse-1c/clickhouse/init/03_views.sql | 6 +-- .../init/04_company_intelligence.sql | 49 ++++++++++++++++--- .../detections/build_entity_timeline.sql | 19 +++++++ clickhouse-1c/etl/config.example.yml | 2 + clickhouse-1c/etl/load_1c_exports.py | 24 ++++++++- .../files/1c-company-intelligence.json | 6 +-- clickhouse-1c/ops/bootstrap_runtime.sh | 2 + windows/export-upload-file-1c-telemetry.ps1 | 26 +++++++++- 13 files changed, 214 insertions(+), 17 deletions(-) diff --git a/ansible/deploy_file_1c_analytics.yml b/ansible/deploy_file_1c_analytics.yml index ace6d29..230112f 100644 --- a/ansible/deploy_file_1c_analytics.yml +++ b/ansible/deploy_file_1c_analytics.yml @@ -134,6 +134,7 @@ landing: documents: {{ aw_file_1c_root }}/landing/documents postings: {{ aw_file_1c_root }}/landing/postings + companies: {{ aw_file_1c_root }}/landing/companies reglog: {{ aw_file_1c_root }}/landing/reglog audit: {{ aw_file_1c_root }}/landing/audit host: {{ aw_file_1c_root }}/landing/host @@ -142,6 +143,7 @@ default: jsonl documents: jsonl postings: jsonl + companies: jsonl reglog: jsonl audit: jsonl host: jsonl @@ -181,6 +183,22 @@ args: chdir: "{{ aw_file_1c_root }}" + - name: Переапплиить ClickHouse schema и views для file-1C analytics + ansible.builtin.shell: | + set -eu + docker exec -i aw-rus-1c-clickhouse clickhouse-client \ + --user {{ aw_file_1c_clickhouse_user }} \ + --password {{ aw_file_1c_clickhouse_password }} \ + --multiquery < "{{ aw_file_1c_root }}/clickhouse/init/{{ item }}" + args: + executable: /bin/bash + loop: + - 00_database.sql + - 01_raw_tables.sql + - 02_core_tables.sql + - 03_views.sql + - 04_company_intelligence.sql + - name: Установить systemd unit aw-1c-ingest.service ansible.builtin.copy: src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.service" diff --git a/clickhouse-1c/ai/company_intelligence_api.py b/clickhouse-1c/ai/company_intelligence_api.py index e42e049..c6d78a9 100644 --- a/clickhouse-1c/ai/company_intelligence_api.py +++ b/clickhouse-1c/ai/company_intelligence_api.py @@ -59,6 +59,7 @@ def health() -> dict[str, Any]: """ SELECT countIf(counterparty != '') AS documents_with_counterparty, + (SELECT count() FROM analytics_1c.companies) AS companies_total, (SELECT count() FROM analytics_1c.company_forecasts) AS forecasts_total, (SELECT count() FROM analytics_1c.company_health_signals) AS health_signals_total FROM analytics_1c.documents @@ -83,6 +84,14 @@ def companies_overview( infobase, organization, counterparty, + company_name, + owner_user, + base_path, + current_status, + db_size_bytes, + reglog_size_bytes, + active_locks, + current_activity_score, last_seen_at, days_since_last_activity, docs_7d, @@ -135,6 +144,16 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str, ORDER BY score DESC, generated_at DESC """ timeline_sql = f""" + SELECT ts, infobase, company_name, owner_user, current_status, db_size_bytes, reglog_size_bytes, active_locks, current_activity_score + FROM analytics_1c.v_company_portfolio_overview + WHERE counterparty = {q(counterparty)} + {"AND infobase = " + q(infobase) if infobase else ""} + LIMIT 1 + """ + forecasts = rows_to_dict(client.query(forecast_sql)) + signals = rows_to_dict(client.query(signals_sql)) + company_state = rows_to_dict(client.query(timeline_sql)) + timeline_sql = f""" SELECT ts, infobase, doc_type, operation_type, amount, status, author FROM analytics_1c.documents WHERE counterparty = {q(counterparty)} @@ -142,8 +161,6 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str, ORDER BY ts DESC LIMIT 20 """ - forecasts = rows_to_dict(client.query(forecast_sql)) - signals = rows_to_dict(client.query(signals_sql)) timeline = rows_to_dict(client.query(timeline_sql)) essence = ( f"Компания {counterparty}: за 30 дней событий {card['docs_30d']}, суммарная активность {card['amount_30d']}, " @@ -152,6 +169,7 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str, return { "essence": essence, "card": card, + "company_state": company_state[0] if company_state else None, "forecasts": forecasts, "signals": signals, "recent_documents": timeline, diff --git a/clickhouse-1c/ai/refresh_company_intelligence.py b/clickhouse-1c/ai/refresh_company_intelligence.py index cf9cc5b..96e7247 100644 --- a/clickhouse-1c/ai/refresh_company_intelligence.py +++ b/clickhouse-1c/ai/refresh_company_intelligence.py @@ -164,6 +164,22 @@ def main() -> int: """, ) } + company_state_map = { + row["infobase"]: row + for row in query_rows( + client, + """ + SELECT + infobase, + current_status, + active_locks, + temp_db_present, + scheduler_touched, + current_activity_score + FROM analytics_1c.v_companies_current + """, + ) + } forecast_rows: list[list[Any]] = [] signal_rows: list[list[Any]] = [] @@ -186,6 +202,12 @@ def main() -> int: days_since_last_activity = (date.today() - latest_day).days open_cases_total = cases_map.get((infobase, counterparty), 0) detections_total = detections_map.get((infobase, counterparty), 0) + company_state = company_state_map.get(infobase, {}) + current_status = str(company_state.get("current_status") or "") + active_locks = int(company_state.get("active_locks") or 0) + temp_db_present = int(company_state.get("temp_db_present") or 0) + scheduler_touched = int(company_state.get("scheduler_touched") or 0) + current_activity_score = float(company_state.get("current_activity_score") or 0) for metric, values in (("docs_total", docs_series), ("amount_total", amount_series)): for horizon in horizons: @@ -221,6 +243,11 @@ def main() -> int: signals.append(("amount_drop", 70, "high", f"Активность по компании {counterparty} упала более чем на 50% неделя к неделе.")) if docs_prev_7d > 0 and docs_7d == 0: signals.append(("docs_stopped", 55, "medium", f"По компании {counterparty} прекратился поток документов за последние 7 дней.")) + if current_status == "busy" or active_locks > 0 or temp_db_present > 0: + score = min(85, 45 + active_locks * 5 + temp_db_present * 10) + signals.append(("base_busy", score, severity_score_to_label(score), f"Файловая база компании {counterparty} занята: status={current_status}, locks={active_locks}, tempDb={temp_db_present}.")) + if scheduler_touched > 0 and current_activity_score >= 15: + signals.append(("scheduler_activity", 35, "medium", f"По компании {counterparty} есть активность scheduler и повышенный activity score {current_activity_score}.")) if open_cases_total > 0: signals.append(("open_cases", min(95, 40 + open_cases_total * 10), severity_score_to_label(min(95, 40 + open_cases_total * 10)), f"По компании {counterparty} есть открытые кейсы: {open_cases_total}.")) if detections_total > 0: diff --git a/clickhouse-1c/clickhouse/init/01_raw_tables.sql b/clickhouse-1c/clickhouse/init/01_raw_tables.sql index 1093fe2..aeefe3b 100644 --- a/clickhouse-1c/clickhouse/init/01_raw_tables.sql +++ b/clickhouse-1c/clickhouse/init/01_raw_tables.sql @@ -16,6 +16,15 @@ CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_postings ENGINE = MergeTree ORDER BY (ingested_at, source_file); +CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_companies +( + ingested_at DateTime DEFAULT now(), + source_file String, + payload String +) +ENGINE = MergeTree +ORDER BY (ingested_at, source_file); + CREATE TABLE IF NOT EXISTS analytics_1c.raw_reglog ( ingested_at DateTime DEFAULT now(), diff --git a/clickhouse-1c/clickhouse/init/02_core_tables.sql b/clickhouse-1c/clickhouse/init/02_core_tables.sql index 7666259..f21f7cc 100644 --- a/clickhouse-1c/clickhouse/init/02_core_tables.sql +++ b/clickhouse-1c/clickhouse/init/02_core_tables.sql @@ -32,6 +32,27 @@ CREATE TABLE IF NOT EXISTS analytics_1c.postings ENGINE = MergeTree ORDER BY (infobase, ts, registrar); +CREATE TABLE IF NOT EXISTS analytics_1c.companies +( + ts DateTime, + infobase LowCardinality(String), + company_name String, + organization String, + owner_user String, + base_id String, + base_path String, + status LowCardinality(String), + db_size_bytes UInt64, + reglog_size_bytes UInt64, + active_locks UInt32, + temp_db_present UInt8, + scheduler_touched UInt8, + activity_score Float32, + source_file String +) +ENGINE = MergeTree +ORDER BY (infobase, ts); + CREATE TABLE IF NOT EXISTS analytics_1c.reglog_events ( ts DateTime, diff --git a/clickhouse-1c/clickhouse/init/03_views.sql b/clickhouse-1c/clickhouse/init/03_views.sql index 61fc03d..899fcf1 100644 --- a/clickhouse-1c/clickhouse/init/03_views.sql +++ b/clickhouse-1c/clickhouse/init/03_views.sql @@ -1,4 +1,4 @@ -CREATE VIEW IF NOT EXISTS analytics_1c.v_documents_daily AS +CREATE OR REPLACE VIEW analytics_1c.v_documents_daily AS SELECT toDate(ts) AS d, infobase, @@ -10,7 +10,7 @@ SELECT FROM analytics_1c.documents GROUP BY d, infobase, organization, doc_type; -CREATE VIEW IF NOT EXISTS analytics_1c.v_detections_daily AS +CREATE OR REPLACE VIEW analytics_1c.v_detections_daily AS SELECT toDate(ts) AS d, infobase, @@ -20,7 +20,7 @@ SELECT FROM analytics_1c.detections GROUP BY d, infobase, severity; -CREATE VIEW IF NOT EXISTS analytics_1c.v_open_cases AS +CREATE OR REPLACE VIEW analytics_1c.v_open_cases AS SELECT * FROM analytics_1c.cases WHERE status != 'closed'; diff --git a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql index 8b6d848..ef35830 100644 --- a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql +++ b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS analytics_1c.company_health_signals ENGINE = MergeTree ORDER BY (generated_at, severity, infobase, counterparty, signal_id); -CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_daily AS +CREATE OR REPLACE VIEW analytics_1c.v_counterparty_daily AS SELECT toDate(ts) AS d, infobase, @@ -56,7 +56,7 @@ FROM analytics_1c.documents WHERE counterparty != '' GROUP BY d, infobase, organization, counterparty; -CREATE VIEW IF NOT EXISTS analytics_1c.v_counterparty_latest_activity AS +CREATE OR REPLACE VIEW analytics_1c.v_counterparty_latest_activity AS SELECT infobase, organization, @@ -72,17 +72,36 @@ FROM analytics_1c.documents WHERE counterparty != '' GROUP BY infobase, organization, counterparty; -CREATE VIEW IF NOT EXISTS analytics_1c.v_company_forecasts_current AS +CREATE OR REPLACE VIEW analytics_1c.v_company_forecasts_current AS SELECT * FROM analytics_1c.company_forecasts WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_forecasts); -CREATE VIEW IF NOT EXISTS analytics_1c.v_company_health_current AS +CREATE OR REPLACE VIEW analytics_1c.v_company_health_current AS SELECT * FROM analytics_1c.company_health_signals WHERE generated_at = (SELECT max(generated_at) FROM analytics_1c.company_health_signals); -CREATE VIEW IF NOT EXISTS analytics_1c.v_company_portfolio_overview AS +CREATE OR REPLACE VIEW analytics_1c.v_companies_current AS +SELECT + infobase, + argMax(company_name, ts) AS company_name, + argMax(organization, ts) AS organization, + argMax(owner_user, ts) AS owner_user, + argMax(base_id, ts) AS base_id, + argMax(base_path, ts) AS base_path, + argMax(status, ts) AS current_status, + argMax(db_size_bytes, ts) AS db_size_bytes, + argMax(reglog_size_bytes, ts) AS reglog_size_bytes, + argMax(active_locks, ts) AS active_locks, + argMax(temp_db_present, ts) AS temp_db_present, + argMax(scheduler_touched, ts) AS scheduler_touched, + argMax(activity_score, ts) AS current_activity_score, + max(ts) AS last_company_snapshot_at +FROM analytics_1c.companies +GROUP BY infobase; + +CREATE OR REPLACE VIEW analytics_1c.v_company_portfolio_overview AS WITH base AS ( @@ -113,6 +132,11 @@ d30 AS WHERE d >= today() - 30 GROUP BY infobase, counterparty ), +company_state AS +( + SELECT * + FROM analytics_1c.v_companies_current +), signals AS ( SELECT @@ -169,12 +193,24 @@ detections_current AS ) SELECT base.infobase AS infobase, - base.organization, + if(company_state.organization != '', company_state.organization, base.organization) AS organization, base.counterparty AS counterparty, + if(company_state.company_name != '', company_state.company_name, base.counterparty) AS company_name, + ifNull(company_state.owner_user, '') AS owner_user, + ifNull(company_state.base_id, '') AS base_id, + ifNull(company_state.base_path, '') AS base_path, base.last_seen_at, + company_state.last_company_snapshot_at, base.last_doc_type, base.last_operation_type, base.last_status, + ifNull(company_state.current_status, base.last_status) AS current_status, + ifNull(company_state.db_size_bytes, 0) AS db_size_bytes, + ifNull(company_state.reglog_size_bytes, 0) AS reglog_size_bytes, + ifNull(company_state.active_locks, 0) AS active_locks, + ifNull(company_state.temp_db_present, 0) AS temp_db_present, + ifNull(company_state.scheduler_touched, 0) AS scheduler_touched, + ifNull(company_state.current_activity_score, 0) AS current_activity_score, dateDiff('day', toDate(base.last_seen_at), today()) AS days_since_last_activity, ifNull(d7.docs_7d, 0) AS docs_7d, ifNull(d7.amount_7d, 0) AS amount_7d, @@ -192,6 +228,7 @@ SELECT ifNull(signals.signal_score, 0) AS signal_score, ifNull(signals.top_signal, '') AS top_signal FROM base +LEFT JOIN company_state ON company_state.infobase = base.infobase LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty diff --git a/clickhouse-1c/detections/build_entity_timeline.sql b/clickhouse-1c/detections/build_entity_timeline.sql index 4f886ce..7ddc597 100644 --- a/clickhouse-1c/detections/build_entity_timeline.sql +++ b/clickhouse-1c/detections/build_entity_timeline.sql @@ -37,6 +37,25 @@ FROM ( ) AS src WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); +INSERT INTO analytics_1c.entity_timeline +SELECT * +FROM ( + SELECT + ts, + 'counterparty' AS entity_type, + company_name AS entity_id, + infobase, + owner_user AS actor, + 'companies' AS source, + 'company_snapshot' AS event_type, + if(status = 'busy' OR active_locks > 0 OR temp_db_present = 1, 'medium', 'low') AS severity, + greatest(10, toUInt32(round(activity_score))) AS score, + concat('company:', infobase, ':', toString(toUnixTimestamp(ts))) AS ref_id, + concat('Company snapshot ', company_name, ': status=', status, ' locks=', toString(active_locks), ' score=', toString(activity_score)) AS summary + FROM analytics_1c.companies +) AS src +WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); + INSERT INTO analytics_1c.entity_timeline SELECT * FROM ( diff --git a/clickhouse-1c/etl/config.example.yml b/clickhouse-1c/etl/config.example.yml index 861e451..b5f459d 100644 --- a/clickhouse-1c/etl/config.example.yml +++ b/clickhouse-1c/etl/config.example.yml @@ -8,6 +8,7 @@ clickhouse: landing: documents: ./landing/documents postings: ./landing/postings + companies: ./landing/companies reglog: ./landing/reglog audit: ./landing/audit host: ./landing/host @@ -16,6 +17,7 @@ formats: default: jsonl documents: jsonl postings: jsonl + companies: jsonl reglog: jsonl audit: jsonl host: jsonl diff --git a/clickhouse-1c/etl/load_1c_exports.py b/clickhouse-1c/etl/load_1c_exports.py index 55077d5..ce19062 100644 --- a/clickhouse-1c/etl/load_1c_exports.py +++ b/clickhouse-1c/etl/load_1c_exports.py @@ -17,6 +17,7 @@ from dateutil import parser as date_parser RAW_TABLES = { "documents": "raw_1c_documents", "postings": "raw_1c_postings", + "companies": "raw_1c_companies", "reglog": "raw_reglog", "audit": "raw_audit", "host": "raw_host_metrics", @@ -25,6 +26,7 @@ RAW_TABLES = { CORE_TABLES = { "documents": "documents", "postings": "postings", + "companies": "companies", "reglog": "reglog_events", "audit": "audit_events", "host": "host_events", @@ -44,7 +46,7 @@ class Config: def parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description="Load file-based 1C exports into ClickHouse") p.add_argument("--config", required=True, help="Path to YAML config") - p.add_argument("--dataset", choices=["documents", "postings", "reglog", "audit", "host"], help="Load only one dataset") + p.add_argument("--dataset", choices=["documents", "postings", "companies", "reglog", "audit", "host"], help="Load only one dataset") return p.parse_args() @@ -127,6 +129,24 @@ def map_core_row(dataset: str, source_file: str, row: dict[str, Any]) -> list[An float(row.get("amount", 0) or 0), source_file, ] + if dataset == "companies": + return [ + normalize_ts(row.get("ts")), + row.get("infobase", ""), + row.get("company_name", row.get("counterparty", row.get("infobase", ""))), + row.get("organization", ""), + row.get("owner_user", row.get("author", "")), + row.get("base_id", row.get("doc_id", "")), + row.get("base_path", ""), + row.get("status", ""), + int(row.get("db_size_bytes", 0) or 0), + int(row.get("reglog_size_bytes", 0) or 0), + int(row.get("active_locks", 0) or 0), + int(row.get("temp_db_present", 0) or 0), + int(row.get("scheduler_touched", 0) or 0), + float(row.get("activity_score", row.get("amount", 0)) or 0), + source_file, + ] if dataset == "reglog": return [ normalize_ts(row.get("ts")), @@ -174,6 +194,8 @@ def core_columns(dataset: str) -> list[str]: return ["ts", "infobase", "organization", "department", "doc_type", "doc_id", "doc_number", "author", "counterparty", "operation_type", "amount", "status", "posted", "source_file"] if dataset == "postings": return ["ts", "infobase", "registrar", "operation_type", "account_dt", "account_ct", "amount", "source_file"] + if dataset == "companies": + return ["ts", "infobase", "company_name", "organization", "owner_user", "base_id", "base_path", "status", "db_size_bytes", "reglog_size_bytes", "active_locks", "temp_db_present", "scheduler_touched", "activity_score", "source_file"] if dataset == "reglog": return ["ts", "infobase", "user", "host", "app", "event_name", "level", "duration_ms", "message", "source_file"] if dataset == "audit": diff --git a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json index ea0e604..c529999 100644 --- a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json +++ b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json @@ -33,7 +33,7 @@ "format": 1, "pluginVersion": "11.2.2", "queryType": "table", - "rawSql": "SELECT countDistinct(counterparty) AS value FROM analytics_1c.v_counterparty_daily WHERE d >= today() - 30", + "rawSql": "SELECT count() AS value FROM analytics_1c.v_companies_current", "refId": "A" } ], @@ -352,7 +352,7 @@ "format": 1, "pluginVersion": "11.2.2", "queryType": "table", - "rawSql": "SELECT infobase, counterparty, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", + "rawSql": "SELECT infobase, company_name, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", "refId": "A" } ], @@ -475,6 +475,6 @@ "timezone": "browser", "title": "1C File - Company Intelligence", "uid": "1c-file-companies", - "version": 1, + "version": 3, "weekStart": "" } diff --git a/clickhouse-1c/ops/bootstrap_runtime.sh b/clickhouse-1c/ops/bootstrap_runtime.sh index 71b803c..7cdb2ad 100644 --- a/clickhouse-1c/ops/bootstrap_runtime.sh +++ b/clickhouse-1c/ops/bootstrap_runtime.sh @@ -6,11 +6,13 @@ ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" mkdir -p \ "${ROOT}/landing/documents" \ "${ROOT}/landing/postings" \ + "${ROOT}/landing/companies" \ "${ROOT}/landing/reglog" \ "${ROOT}/landing/audit" \ "${ROOT}/landing/host" \ "${ROOT}/archive/documents" \ "${ROOT}/archive/postings" \ + "${ROOT}/archive/companies" \ "${ROOT}/archive/reglog" \ "${ROOT}/archive/audit" \ "${ROOT}/archive/host" diff --git a/windows/export-upload-file-1c-telemetry.ps1 b/windows/export-upload-file-1c-telemetry.ps1 index 4f94021..980068d 100644 --- a/windows/export-upload-file-1c-telemetry.ps1 +++ b/windows/export-upload-file-1c-telemetry.ps1 @@ -268,6 +268,7 @@ $nextExporterState = @{} $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $documents = New-Object System.Collections.Generic.List[object] +$companies = New-Object System.Collections.Generic.List[object] $reglog = New-Object System.Collections.Generic.List[object] $audit = New-Object System.Collections.Generic.List[object] @@ -333,6 +334,23 @@ foreach ($base in $infobases) { posted = 1 }) + $companies.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + company_name = [string]$base.infobase + organization = $organization + owner_user = $owner + base_id = $docId + base_path = [string]$base.path + status = $status + db_size_bytes = $dbSizeBytes + reglog_size_bytes = $mainLogBytes + active_locks = $activeLocks.Count + temp_db_present = if ($tempDb) { 1 } else { 0 } + scheduler_touched = if ($schedulerTouched) { 1 } else { 0 } + activity_score = $activityScore + }) + if ($activityScore -gt 0) { $documents.Add([ordered]@{ ts = $nowUtc @@ -434,17 +452,20 @@ New-Item -ItemType Directory -Path $outRoot -Force | Out-Null $files = @{ documents = Join-Path $outRoot "documents-$stamp.jsonl" + companies = Join-Path $outRoot "companies-$stamp.jsonl" reglog = Join-Path $outRoot "reglog-$stamp.jsonl" audit = Join-Path $outRoot "audit-$stamp.jsonl" host = Join-Path $outRoot "host-$stamp.jsonl" } $documentRows = @($documents | ForEach-Object { $_ }) +$companyRows = @($companies | ForEach-Object { $_ }) $reglogRows = @($reglog | ForEach-Object { $_ }) $auditRows = @($audit | ForEach-Object { $_ }) $hostRowsNormalized = @($hostRows | ForEach-Object { $_ }) Write-JsonLines -Path ([string]$files['documents']) -Rows $documentRows +Write-JsonLines -Path ([string]$files['companies']) -Rows $companyRows Write-JsonLines -Path ([string]$files['reglog']) -Rows $reglogRows Write-JsonLines -Path ([string]$files['audit']) -Rows $auditRows Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized @@ -452,8 +473,8 @@ Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized $effectiveKeyPath = New-TemporarySshKeyCopy -SourceKeyPath $RemoteKeyPath try { - Write-RunLog "prepared datasets documents=$($documentRows.Count) reglog=$($reglogRows.Count) audit=$($auditRows.Count) host=$($hostRowsNormalized.Count)" - foreach ($dataset in 'documents', 'reglog', 'audit', 'host') { + Write-RunLog "prepared datasets documents=$($documentRows.Count) companies=$($companyRows.Count) reglog=$($reglogRows.Count) audit=$($auditRows.Count) host=$($hostRowsNormalized.Count)" + foreach ($dataset in 'documents', 'companies', 'reglog', 'audit', 'host') { Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files[$dataset]) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/" } Save-ExporterState -Path $ExporterStatePath -State $nextExporterState @@ -473,6 +494,7 @@ Write-RunLog 'file1c exporter done' infobases = @($infobases | ForEach-Object { $_.infobase }) datasets = [ordered]@{ documents = $documents.Count + companies = $companies.Count reglog = $reglog.Count audit = $audit.Count host = $hostRows.Count