diff --git a/ansible/deploy_aw_windows.yml b/ansible/deploy_aw_windows.yml index 7f44e56..83a40b8 100644 --- a/ansible/deploy_aw_windows.yml +++ b/ansible/deploy_aw_windows.yml @@ -39,6 +39,7 @@ aw_windows_file_1c_auto_upload_interval_hours: 6 aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" aw_windows_file_1c_target_user: "igor" + aw_windows_file_1c_registry_workbook_path: "E:\\USER1\\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx" aw_windows_afk_enabled_default: true aw_windows_window_enabled_default: true aw_windows_file_ops_enabled: true @@ -250,6 +251,7 @@ File1CAutoUploadTaskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" File1CTargetHost = "{{ aw_windows_file_1c_target_host_effective }}" File1CTargetUser = "{{ aw_windows_file_1c_target_user }}" + File1CRegistryWorkbookPath = "{{ aw_windows_file_1c_registry_workbook_path }}" CustomRulesPath = "{{ aw_windows_rules_path }}" CustomPolicyPath = "{{ aw_windows_policy_path }}" } diff --git a/ansible/deploy_file_1c_windows_telemetry.yml b/ansible/deploy_file_1c_windows_telemetry.yml index 9c3a601..a7cbc17 100644 --- a/ansible/deploy_file_1c_windows_telemetry.yml +++ b/ansible/deploy_file_1c_windows_telemetry.yml @@ -11,6 +11,7 @@ aw_windows_file_1c_auto_upload_interval_hours: 6 aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" aw_windows_file_1c_remote_root: "/opt/activitywatch/clickhouse-1c/landing" + aw_windows_file_1c_registry_workbook_path: "E:\\USER1\\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx" aw_windows_upload_key_private_path: /tmp/awops_ed25519 aw_windows_upload_key_public_path: /tmp/awops_ed25519.pub @@ -87,6 +88,7 @@ targetHost = "{{ aw_windows_file_1c_target_host_effective }}" targetUser = "{{ aw_windows_file_1c_target_user }}" remoteRoot = "{{ aw_windows_file_1c_remote_root }}" + registryWorkbookPath = "{{ aw_windows_file_1c_registry_workbook_path }}" } if ($config.analytics.PSObject.Properties.Name -contains 'file1cAutomation') { diff --git a/clickhouse-1c/ai/company_intelligence_api.py b/clickhouse-1c/ai/company_intelligence_api.py index c3b1d09..ea15844 100644 --- a/clickhouse-1c/ai/company_intelligence_api.py +++ b/clickhouse-1c/ai/company_intelligence_api.py @@ -60,6 +60,7 @@ def health() -> dict[str, Any]: SELECT countIf(counterparty != '') AS documents_with_counterparty, (SELECT count() FROM analytics_1c.companies) AS companies_total, + (SELECT count() FROM analytics_1c.company_registry) AS registry_rows_total, (SELECT count() FROM analytics_1c.company_forecasts) AS forecasts_total, (SELECT count() FROM analytics_1c.company_health_signals) AS health_signals_total FROM analytics_1c.documents @@ -85,6 +86,12 @@ def companies_overview( organization, counterparty, company_name, + registry_assignee_name, + registry_status, + registry_share_text, + registry_key_contour, + registry_inn, + registry_kpp, owner_user, base_path, current_status, diff --git a/clickhouse-1c/clickhouse/init/01_raw_tables.sql b/clickhouse-1c/clickhouse/init/01_raw_tables.sql index aeefe3b..81ac911 100644 --- a/clickhouse-1c/clickhouse/init/01_raw_tables.sql +++ b/clickhouse-1c/clickhouse/init/01_raw_tables.sql @@ -25,6 +25,16 @@ CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_companies ENGINE = MergeTree ORDER BY (ingested_at, source_file); +CREATE TABLE IF NOT EXISTS analytics_1c.raw_1c_company_registry +( + ingested_at DateTime DEFAULT now(), + source_file String, + source_sheet String, + payload String +) +ENGINE = MergeTree +ORDER BY (ingested_at, source_file, source_sheet); + CREATE TABLE IF NOT EXISTS analytics_1c.raw_reglog ( ingested_at DateTime DEFAULT now(), diff --git a/clickhouse-1c/clickhouse/init/02_core_tables.sql b/clickhouse-1c/clickhouse/init/02_core_tables.sql index f21f7cc..670b5f9 100644 --- a/clickhouse-1c/clickhouse/init/02_core_tables.sql +++ b/clickhouse-1c/clickhouse/init/02_core_tables.sql @@ -53,6 +53,23 @@ CREATE TABLE IF NOT EXISTS analytics_1c.companies ENGINE = MergeTree ORDER BY (infobase, ts); +CREATE TABLE IF NOT EXISTS analytics_1c.company_registry +( + ts DateTime, + source_file String, + source_sheet LowCardinality(String), + company_name String, + company_key String, + assignee_name String, + registry_status LowCardinality(String), + share_text String, + key_contour UInt8, + inn String, + kpp String +) +ENGINE = MergeTree +ORDER BY (company_key, ts, source_sheet); + CREATE TABLE IF NOT EXISTS analytics_1c.reglog_events ( ts DateTime, diff --git a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql index ef35830..25ab962 100644 --- a/clickhouse-1c/clickhouse/init/04_company_intelligence.sql +++ b/clickhouse-1c/clickhouse/init/04_company_intelligence.sql @@ -101,6 +101,20 @@ SELECT FROM analytics_1c.companies GROUP BY infobase; +CREATE OR REPLACE VIEW analytics_1c.v_company_registry_current AS +SELECT + company_key, + argMax(company_name, ts) AS company_name, + argMax(assignee_name, ts) AS assignee_name, + argMax(registry_status, ts) AS registry_status, + argMax(share_text, ts) AS share_text, + argMax(key_contour, ts) AS key_contour, + argMax(inn, ts) AS inn, + argMax(kpp, ts) AS kpp, + max(ts) AS last_registry_snapshot_at +FROM analytics_1c.company_registry +GROUP BY company_key; + CREATE OR REPLACE VIEW analytics_1c.v_company_portfolio_overview AS WITH base AS @@ -137,6 +151,11 @@ company_state AS SELECT * FROM analytics_1c.v_companies_current ), +registry_state AS +( + SELECT * + FROM analytics_1c.v_company_registry_current +), signals AS ( SELECT @@ -196,6 +215,12 @@ SELECT if(company_state.organization != '', company_state.organization, base.organization) AS organization, base.counterparty AS counterparty, if(company_state.company_name != '', company_state.company_name, base.counterparty) AS company_name, + ifNull(registry_state.assignee_name, '') AS registry_assignee_name, + ifNull(registry_state.registry_status, '') AS registry_status, + ifNull(registry_state.share_text, '') AS registry_share_text, + ifNull(registry_state.key_contour, 0) AS registry_key_contour, + ifNull(registry_state.inn, '') AS registry_inn, + ifNull(registry_state.kpp, '') AS registry_kpp, ifNull(company_state.owner_user, '') AS owner_user, ifNull(company_state.base_id, '') AS base_id, ifNull(company_state.base_path, '') AS base_path, @@ -229,6 +254,7 @@ SELECT ifNull(signals.top_signal, '') AS top_signal FROM base LEFT JOIN company_state ON company_state.infobase = base.infobase +LEFT JOIN registry_state ON registry_state.company_key = trimBoth(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(upperUTF8(base.counterparty), '(^|\\s)20[0-9]{2}($|\\s)', ' '), '[^0-9A-ZА-ЯЁ]+', ' '), '\\s+', ' ')) LEFT JOIN d7 ON d7.infobase = base.infobase AND d7.counterparty = base.counterparty LEFT JOIN d30 ON d30.infobase = base.infobase AND d30.counterparty = base.counterparty LEFT JOIN signals ON signals.infobase = base.infobase AND signals.counterparty = base.counterparty diff --git a/clickhouse-1c/etl/load_company_registry_xlsx.py b/clickhouse-1c/etl/load_company_registry_xlsx.py new file mode 100644 index 0000000..7294a86 --- /dev/null +++ b/clickhouse-1c/etl/load_company_registry_xlsx.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import shutil +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import clickhouse_connect +import yaml +from openpyxl import load_workbook + + +@dataclass +class Config: + clickhouse: dict[str, Any] + archive_dir: str | None + delete_after_load: bool + min_file_age_seconds: int + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Load company registry xlsx into ClickHouse") + p.add_argument("--config", required=True) + p.add_argument("--landing", required=True) + return p.parse_args() + + +def load_config(path: str) -> Config: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + return Config( + clickhouse=raw["clickhouse"], + archive_dir=raw.get("archive_dir"), + delete_after_load=bool(raw.get("delete_after_load", False)), + min_file_age_seconds=int(raw.get("min_file_age_seconds", 180)), + ) + + +def ch_client(conf: Config): + return clickhouse_connect.get_client( + host=conf.clickhouse["host"], + port=conf.clickhouse.get("port", 8123), + username=conf.clickhouse.get("username", "default"), + password=conf.clickhouse.get("password", ""), + database=conf.clickhouse.get("database", "analytics_1c"), + ) + + +def normalize_company_key(value: str) -> str: + text = (value or "").upper().replace("Ё", "Е") + text = re.sub(r"(^|\s)20\d{2}($|\s)", " ", text) + text = re.sub(r"[^0-9A-ZА-Я]+", " ", text) + text = re.sub(r"\s+", " ", text) + return text.strip() + + +def as_text(value: Any) -> str: + if value is None: + return "" + return str(value).strip() + + +def archive_or_delete(conf: Config, path: Path) -> None: + if conf.archive_dir: + archive_root = Path(conf.archive_dir) / "registry" + archive_root.mkdir(parents=True, exist_ok=True) + shutil.move(str(path), archive_root / path.name) + return + if conf.delete_after_load: + path.unlink(missing_ok=True) + + +def parse_registry(path: Path) -> list[dict[str, Any]]: + wb = load_workbook(path, read_only=True, data_only=True) + now = datetime.now(UTC).replace(tzinfo=None, microsecond=0) + + tax_map: dict[str, tuple[str, str]] = {} + if "Лист2" in wb.sheetnames: + ws = wb["Лист2"] + for row in ws.iter_rows(min_row=3, values_only=True): + company_name = as_text(row[1] if len(row) > 1 else "") + if not company_name: + continue + tax_map[normalize_company_key(company_name)] = ( + as_text(row[2] if len(row) > 2 else ""), + as_text(row[3] if len(row) > 3 else ""), + ) + + rows: list[dict[str, Any]] = [] + if "ОСНОВНОЙ" not in wb.sheetnames: + return rows + + ws = wb["ОСНОВНОЙ"] + top_headers = [as_text(v) for v in next(ws.iter_rows(min_row=1, max_row=1, values_only=True))] + manager_headers = [as_text(v) for v in next(ws.iter_rows(min_row=2, max_row=2, values_only=True))] + + col_specs: list[dict[str, Any]] = [] + current_manager: dict[str, Any] | None = None + for idx in range(1, len(manager_headers)): + manager = manager_headers[idx] + top = top_headers[idx] if idx < len(top_headers) else "" + if manager: + current_manager = { + "col": idx, + "assignee_name": manager, + "meta_col": None, + "meta_label": "", + "registry_status": "active", + } + col_specs.append(current_manager) + continue + if top and "исключ" in top.lower(): + current_manager = { + "col": idx, + "assignee_name": "", + "meta_col": None, + "meta_label": "", + "registry_status": "excluded", + } + col_specs.append(current_manager) + continue + if current_manager is not None: + current_manager["meta_col"] = idx + current_manager["meta_label"] = top or "meta" + current_manager = None + + for row in ws.iter_rows(min_row=3, values_only=True): + for spec in col_specs: + company_name = as_text(row[spec["col"]] if spec["col"] < len(row) else "") + if not company_name: + continue + meta_value = "" + if spec.get("meta_col") is not None: + meta_value = as_text(row[spec["meta_col"]] if spec["meta_col"] < len(row) else "") + company_key = normalize_company_key(company_name) + inn, kpp = tax_map.get(company_key, ("", "")) + rows.append( + { + "ts": now, + "source_file": path.name, + "source_sheet": "ОСНОВНОЙ", + "company_name": company_name, + "company_key": company_key, + "assignee_name": spec["assignee_name"], + "registry_status": spec["registry_status"], + "share_text": meta_value if "ключ" not in spec.get("meta_label", "").lower() else "", + "key_contour": 1 if meta_value.upper() == "ЕСТЬ" and "ключ" in spec.get("meta_label", "").lower() else 0, + "inn": inn, + "kpp": kpp, + } + ) + + return rows + + +def main() -> int: + args = parse_args() + conf = load_config(args.config) + client = ch_client(conf) + landing = Path(args.landing) + if not landing.exists(): + return 0 + + for path in sorted(p for p in landing.iterdir() if p.is_file() and p.suffix.lower() == ".xlsx"): + age_seconds = max(0, int((datetime.now(UTC) - datetime.fromtimestamp(path.stat().st_mtime, UTC)).total_seconds())) + if age_seconds < conf.min_file_age_seconds: + print(f"skip registry: {path.name} age={age_seconds}s < min_file_age_seconds={conf.min_file_age_seconds}") + continue + rows = parse_registry(path) + if rows: + client.insert( + "analytics_1c.raw_1c_company_registry", + [[row["source_file"], row["source_sheet"], json.dumps({k: (v.isoformat() if isinstance(v, datetime) else v) for k, v in row.items()}, ensure_ascii=False)] for row in rows], + column_names=["source_file", "source_sheet", "payload"], + ) + client.insert( + "analytics_1c.company_registry", + [[row["ts"], row["source_file"], row["source_sheet"], row["company_name"], row["company_key"], row["assignee_name"], row["registry_status"], row["share_text"], row["key_contour"], row["inn"], row["kpp"]] for row in rows], + column_names=["ts", "source_file", "source_sheet", "company_name", "company_key", "assignee_name", "registry_status", "share_text", "key_contour", "inn", "kpp"], + ) + print(f"loaded registry: {path.name} rows={len(rows)}") + archive_or_delete(conf, path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/clickhouse-1c/etl/requirements.txt b/clickhouse-1c/etl/requirements.txt index f824425..5a30164 100644 --- a/clickhouse-1c/etl/requirements.txt +++ b/clickhouse-1c/etl/requirements.txt @@ -1,3 +1,4 @@ clickhouse-connect>=0.7.16 PyYAML>=6.0.2 python-dateutil>=2.9.0 +openpyxl>=3.1.5 diff --git a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json index c529999..acc8b1f 100644 --- a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json +++ b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json @@ -352,7 +352,7 @@ "format": 1, "pluginVersion": "11.2.2", "queryType": "table", - "rawSql": "SELECT infobase, company_name, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", + "rawSql": "SELECT infobase, company_name, registry_assignee_name, registry_inn, registry_kpp, owner_user, current_status, round(db_size_bytes / 1048576, 2) AS db_size_mb, amount_30d, docs_30d, amount_forecast_30d, signal_severity, signal_score, top_signal, last_seen_at FROM analytics_1c.v_company_portfolio_overview ORDER BY signal_score DESC, amount_30d DESC LIMIT 20", "refId": "A" } ], @@ -475,6 +475,6 @@ "timezone": "browser", "title": "1C File - Company Intelligence", "uid": "1c-file-companies", - "version": 3, + "version": 4, "weekStart": "" } diff --git a/clickhouse-1c/ops/bootstrap_runtime.sh b/clickhouse-1c/ops/bootstrap_runtime.sh index 7cdb2ad..9ce7568 100644 --- a/clickhouse-1c/ops/bootstrap_runtime.sh +++ b/clickhouse-1c/ops/bootstrap_runtime.sh @@ -7,12 +7,14 @@ mkdir -p \ "${ROOT}/landing/documents" \ "${ROOT}/landing/postings" \ "${ROOT}/landing/companies" \ + "${ROOT}/landing/registry" \ "${ROOT}/landing/reglog" \ "${ROOT}/landing/audit" \ "${ROOT}/landing/host" \ "${ROOT}/archive/documents" \ "${ROOT}/archive/postings" \ "${ROOT}/archive/companies" \ + "${ROOT}/archive/registry" \ "${ROOT}/archive/reglog" \ "${ROOT}/archive/audit" \ "${ROOT}/archive/host" diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh index e38b3e1..655d80b 100644 --- a/clickhouse-1c/ops/run_ingest_cycle.sh +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -38,6 +38,7 @@ fi . "${ENV_FILE}" "${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}" +"${VENV}/bin/python" "${ROOT}/etl/load_company_registry_xlsx.py" --config "${CONFIG}" --landing "${ROOT}/landing/registry" docker exec -i "${CH_CONTAINER}" clickhouse-client \ --user "${CLICKHOUSE_USER}" \ diff --git a/windows/ActivityWatch.Windows.Common.psm1 b/windows/ActivityWatch.Windows.Common.psm1 index a12a1db..2294465 100755 --- a/windows/ActivityWatch.Windows.Common.psm1 +++ b/windows/ActivityWatch.Windows.Common.psm1 @@ -560,6 +560,7 @@ function New-ActivityWatchDeploymentConfig { [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', [string]$File1CRemoteRoot = '/opt/activitywatch/clickhouse-1c/landing', + [string]$File1CRegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx', [switch]$IntegrationTestEnabled ) @@ -645,6 +646,7 @@ function New-ActivityWatchDeploymentConfig { targetHost = $File1CTargetHost targetUser = $File1CTargetUser remoteRoot = $File1CRemoteRoot + registryWorkbookPath = $File1CRegistryWorkbookPath } } sessionEvents = [pscustomobject]@{ diff --git a/windows/deploy-domain-users.ps1 b/windows/deploy-domain-users.ps1 index 71a3943..38bd8e3 100755 --- a/windows/deploy-domain-users.ps1 +++ b/windows/deploy-domain-users.ps1 @@ -49,6 +49,7 @@ param( [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', + [string]$File1CRegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx', [switch]$IntegrationTestEnabled ) @@ -157,6 +158,7 @@ $config = New-ActivityWatchDeploymentConfig ` -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` -File1CTargetHost $File1CTargetHost ` -File1CTargetUser $File1CTargetUser ` + -File1CRegistryWorkbookPath $File1CRegistryWorkbookPath ` -LaunchScriptPath $launchScriptPath ` -RecoveryScriptPath $recoveryScriptPath ` -UserTasks $taskDefinitions ` diff --git a/windows/deploy-ensemble.ps1 b/windows/deploy-ensemble.ps1 index 59a8a5a..a58eb2f 100644 --- a/windows/deploy-ensemble.ps1 +++ b/windows/deploy-ensemble.ps1 @@ -50,6 +50,7 @@ param( [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', + [string]$File1CRegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx', [switch]$SkipHardening, [switch]$ValidateAfterDeploy, [switch]$IntegrationTestEnabled @@ -118,6 +119,7 @@ if (-not (Test-Path -LiteralPath $deployScript)) { -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` -File1CTargetHost $File1CTargetHost ` -File1CTargetUser $File1CTargetUser ` + -File1CRegistryWorkbookPath $File1CRegistryWorkbookPath ` -IntegrationTestEnabled:$IntegrationTestEnabled if (-not $SkipHardening) { diff --git a/windows/export-upload-file-1c-telemetry.ps1 b/windows/export-upload-file-1c-telemetry.ps1 index f2cc5a5..3073543 100644 --- a/windows/export-upload-file-1c-telemetry.ps1 +++ b/windows/export-upload-file-1c-telemetry.ps1 @@ -4,7 +4,8 @@ param( [string]$AnalyticsHost = '', [string]$AnalyticsUser = 'igor', [string]$RemoteRoot = '/opt/activitywatch/clickhouse-1c/landing', - [string]$RemoteKeyPath = 'C:\ProgramData\AWatch-rus\ssh\awops_ed25519' + [string]$RemoteKeyPath = 'C:\ProgramData\AWatch-rus\ssh\awops_ed25519', + [string]$RegistryWorkbookPath = '' ) Set-StrictMode -Version Latest @@ -263,6 +264,16 @@ if ($config.PSObject.Properties.Name -contains 'analytics' -and -not [string]::IsNullOrWhiteSpace([string]$config.analytics.file1cAutomation.remoteRoot)) { $RemoteRoot = [string]$config.analytics.file1cAutomation.remoteRoot } +if ([string]::IsNullOrWhiteSpace($RegistryWorkbookPath) -and + $config.PSObject.Properties.Name -contains 'analytics' -and + $config.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and + $config.analytics.file1cAutomation.PSObject.Properties.Name -contains 'registryWorkbookPath' -and + -not [string]::IsNullOrWhiteSpace([string]$config.analytics.file1cAutomation.registryWorkbookPath)) { + $RegistryWorkbookPath = [string]$config.analytics.file1cAutomation.registryWorkbookPath +} +if ([string]::IsNullOrWhiteSpace($RegistryWorkbookPath)) { + $RegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx' +} $infobases = @(Get-1CFileInfobases) $nowUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') @@ -459,6 +470,7 @@ $files = @{ reglog = Join-Path $outRoot "reglog-$stamp.jsonl" audit = Join-Path $outRoot "audit-$stamp.jsonl" host = Join-Path $outRoot "host-$stamp.jsonl" + registry = Join-Path $outRoot "company-registry-$stamp.xlsx" } $documentRows = @($documents | ForEach-Object { $_ }) @@ -473,6 +485,11 @@ Write-JsonLines -Path ([string]$files['reglog']) -Rows $reglogRows Write-JsonLines -Path ([string]$files['audit']) -Rows $auditRows Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized +$registryUploaded = $false +if (Test-Path -LiteralPath $RegistryWorkbookPath) { + Copy-Item -LiteralPath $RegistryWorkbookPath -Destination ([string]$files['registry']) -Force +} + $effectiveKeyPath = New-TemporarySshKeyCopy -SourceKeyPath $RemoteKeyPath try { @@ -480,6 +497,10 @@ try { foreach ($dataset in 'documents', 'companies', 'reglog', 'audit', 'host') { Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files[$dataset]) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/" } + if (Test-Path -LiteralPath ([string]$files['registry'])) { + Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files['registry']) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/registry/" + $registryUploaded = $true + } Save-ExporterState -Path $ExporterStatePath -State $nextExporterState Write-RunLog "upload complete analyticsHost=$AnalyticsHost remoteRoot=$RemoteRoot" } @@ -501,5 +522,6 @@ Write-RunLog 'file1c exporter done' reglog = $reglog.Count audit = $audit.Count host = $hostRows.Count + registry = if ($registryUploaded) { 1 } else { 0 } } } | ConvertTo-Json -Depth 8