diff --git a/clickhouse-1c/ai/company_intelligence_api.py b/clickhouse-1c/ai/company_intelligence_api.py index 08d1185..e42e049 100644 --- a/clickhouse-1c/ai/company_intelligence_api.py +++ b/clickhouse-1c/ai/company_intelligence_api.py @@ -146,8 +146,8 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str, signals = rows_to_dict(client.query(signals_sql)) timeline = rows_to_dict(client.query(timeline_sql)) essence = ( - f"Компания {counterparty}: за 30 дней документов {card['docs_30d']}, объём {card['amount_30d']}, " - f"прогноз на 30 дней {card['amount_forecast_30d']}, риск {card['signal_severity']}." + f"Компания {counterparty}: за 30 дней событий {card['docs_30d']}, суммарная активность {card['amount_30d']}, " + f"прогноз активности на 30 дней {card['amount_forecast_30d']}, риск {card['signal_severity']}." ) return { "essence": essence, diff --git a/clickhouse-1c/ai/refresh_company_intelligence.py b/clickhouse-1c/ai/refresh_company_intelligence.py index abade6c..94bff90 100644 --- a/clickhouse-1c/ai/refresh_company_intelligence.py +++ b/clickhouse-1c/ai/refresh_company_intelligence.py @@ -218,7 +218,7 @@ def main() -> int: if days_since_last_activity >= 14 and (docs_prev_7d > 0 or amount_prev_7d > 0): signals.append(("inactive_company", 85, "high", f"Нет активности по компании {counterparty} уже {days_since_last_activity} дн.")) if amount_prev_7d > 0 and amount_7d < amount_prev_7d * 0.5: - signals.append(("amount_drop", 70, "high", f"Объём по компании {counterparty} упал более чем на 50% неделя к неделе.")) + signals.append(("amount_drop", 70, "high", f"Активность по компании {counterparty} упала более чем на 50% неделя к неделе.")) if docs_prev_7d > 0 and docs_7d == 0: signals.append(("docs_stopped", 55, "medium", f"По компании {counterparty} прекратился поток документов за последние 7 дней.")) if open_cases_total > 0: diff --git a/clickhouse-1c/detections/build_entity_timeline.sql b/clickhouse-1c/detections/build_entity_timeline.sql index 578751d..4f886ce 100644 --- a/clickhouse-1c/detections/build_entity_timeline.sql +++ b/clickhouse-1c/detections/build_entity_timeline.sql @@ -11,12 +11,32 @@ FROM ( concat('document:', doc_type) AS event_type, if(posted = 1, 'low', 'medium') AS severity, if(posted = 1, 5, 20) AS score, - doc_id AS ref_id, + concat(doc_id, ':', toString(toUnixTimestamp(ts)), ':', doc_type) AS ref_id, concat('Документ ', doc_type, ' №', doc_number, ' статус=', status) AS summary FROM analytics_1c.documents ) AS src WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); +INSERT INTO analytics_1c.entity_timeline +SELECT * +FROM ( + SELECT + ts, + 'counterparty' AS entity_type, + counterparty AS entity_id, + infobase, + author AS actor, + 'documents' AS source, + concat('counterparty:', operation_type) AS event_type, + if(status = 'busy', 'medium', 'low') AS severity, + greatest(10, toUInt32(round(amount))) AS score, + concat('counterparty:', counterparty, ':', doc_id, ':', toString(toUnixTimestamp(ts))) AS ref_id, + concat('Активность компании ', counterparty, ': ', doc_type, ' score=', toString(amount), ' status=', status) AS summary + FROM analytics_1c.documents + WHERE counterparty != '' +) AS src +WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); + INSERT INTO analytics_1c.entity_timeline SELECT * FROM ( diff --git a/clickhouse-1c/detections/insert_detections.sql b/clickhouse-1c/detections/insert_detections.sql index b6eb3ac..6883944 100644 --- a/clickhouse-1c/detections/insert_detections.sql +++ b/clickhouse-1c/detections/insert_detections.sql @@ -19,6 +19,26 @@ FROM ( ) AS src WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections); +INSERT INTO analytics_1c.detections +SELECT * +FROM ( + SELECT + generated_at AS ts, + concat('company_signal:', signal_id, ':', toString(toUnixTimestamp(generated_at))) AS detection_id, + infobase, + concat('company_', signal_type) AS rule_id, + 'Сигнал активности компании' AS rule_title, + 'counterparty' AS entity_type, + counterparty AS entity_id, + severity, + score, + summary, + 'open' AS status + FROM analytics_1c.v_company_health_current + WHERE severity IN ('medium', 'high', 'critical') +) AS src +WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections); + INSERT INTO analytics_1c.detections SELECT * FROM ( diff --git a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json index f8d3049..ea0e604 100644 --- a/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json +++ b/clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json @@ -79,7 +79,7 @@ }, { "id": 2, - "title": "Прогноз объёма 30д", + "title": "Прогноз активности 30д", "type": "stat", "datasource": { "type": "grafana-clickhouse-datasource", @@ -235,7 +235,7 @@ }, { "id": 5, - "title": "Объём по компаниям", + "title": "Активность по компаниям", "type": "timeseries", "datasource": { "type": "grafana-clickhouse-datasource", @@ -282,7 +282,7 @@ }, { "id": 6, - "title": "Количество документов по компаниям", + "title": "Количество событий по компаниям", "type": "timeseries", "datasource": { "type": "grafana-clickhouse-datasource", diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh index 5834fff..e38b3e1 100644 --- a/clickhouse-1c/ops/run_ingest_cycle.sh +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -45,6 +45,14 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \ --database "${CLICKHOUSE_DB}" \ < "${ROOT}/detections/build_entity_timeline.sql" +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/clickhouse/init/04_company_intelligence.sql" + +"${ROOT}/ops/run_company_intelligence_refresh.sh" + docker exec -i "${CH_CONTAINER}" clickhouse-client \ --user "${CLICKHOUSE_USER}" \ --password "${CLICKHOUSE_PASSWORD}" \ @@ -56,11 +64,3 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \ --password "${CLICKHOUSE_PASSWORD}" \ --database "${CLICKHOUSE_DB}" \ < "${ROOT}/detections/open_cases_from_detections.sql" - -docker exec -i "${CH_CONTAINER}" clickhouse-client \ - --user "${CLICKHOUSE_USER}" \ - --password "${CLICKHOUSE_PASSWORD}" \ - --database "${CLICKHOUSE_DB}" \ - < "${ROOT}/clickhouse/init/04_company_intelligence.sql" - -"${ROOT}/ops/run_company_intelligence_refresh.sh" diff --git a/windows/export-upload-file-1c-telemetry.ps1 b/windows/export-upload-file-1c-telemetry.ps1 index 6107985..4f94021 100644 --- a/windows/export-upload-file-1c-telemetry.ps1 +++ b/windows/export-upload-file-1c-telemetry.ps1 @@ -13,6 +13,7 @@ $ErrorActionPreference = 'Stop' $LogDir = 'C:\ProgramData\AWatch-rus\logs' $LogPath = Join-Path $LogDir 'file1c-telemetry.log' $ScpExe = Join-Path $env:WINDIR 'System32\OpenSSH\scp.exe' +$ExporterStatePath = Join-Path (Split-Path -Parent $ConfigPath) 'file1c-telemetry-state.json' New-Item -ItemType Directory -Path $LogDir -Force | Out-Null @@ -156,6 +157,76 @@ function Invoke-SshUploadWithRetry { } } +function Get-ExporterState { + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + if (-not (Test-Path -LiteralPath $Path)) { + return @{} + } + + $raw = Get-Content -Raw -LiteralPath $Path -Encoding UTF8 + if ([string]::IsNullOrWhiteSpace($raw)) { + return @{} + } + + $payload = $raw | ConvertFrom-Json + $state = @{} + foreach ($prop in $payload.PSObject.Properties) { + $state[[string]$prop.Name] = $prop.Value + } + return $state +} + +function Save-ExporterState { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [Parameter(Mandatory = $true)] + [hashtable]$State + ) + + $directory = Split-Path -Parent $Path + if ($directory) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + $State | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $Path -Encoding UTF8 +} + +function Get-CompanyActivityScore { + param( + [double]$DbDeltaMb, + [double]$ReglogDeltaMb, + [int]$ActiveLocks, + [bool]$HasTempDb, + [bool]$SchedulerTouched, + [string]$Status, + [bool]$IsBootstrap + ) + + $score = 0.0 + $score += [math]::Abs($DbDeltaMb) + $score += [math]::Abs($ReglogDeltaMb) + if ($ActiveLocks -gt 0) { + $score += ($ActiveLocks * 5) + } + if ($HasTempDb) { + $score += 10 + } + if ($SchedulerTouched) { + $score += 3 + } + if ($Status -eq 'busy') { + $score += 5 + } + if ($IsBootstrap -and $score -le 0) { + $score = 1 + } + return [math]::Round($score, 2) +} + Write-RunLog 'file1c exporter start' if (-not (Test-Path -LiteralPath $ScpExe)) { @@ -192,6 +263,9 @@ if ($config.PSObject.Properties.Name -contains 'analytics' -and $infobases = @(Get-1CFileInfobases) $nowUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') +$exporterState = Get-ExporterState -Path $ExporterStatePath +$nextExporterState = @{} +$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $documents = New-Object System.Collections.Generic.List[object] $reglog = New-Object System.Collections.Generic.List[object] @@ -207,13 +281,46 @@ foreach ($base in $infobases) { $tempDb = Get-Item -LiteralPath (Join-Path $base.path '1Cv8tmp.1CD') -ErrorAction SilentlyContinue $schedulerDir = Get-Item -LiteralPath (Join-Path $base.path '1Cv8JobScheduler') -ErrorAction SilentlyContinue $owner = if ($base.userName) { [string]$base.userName } else { 'unknown' } + $organization = [string](Split-Path -Leaf (Split-Path -Parent $base.path)) $status = if ($activeLocks.Count -gt 0 -or $tempDb) { 'busy' } else { 'online' } $docId = if ($base.baseId) { [string]$base.baseId } else { ([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$base.path)).TrimEnd('=').Replace('/','_').Replace('+','-')) } + $stateKey = [string]$docId + $previous = if ($exporterState.ContainsKey($stateKey)) { $exporterState[$stateKey] } else { $null } + $dbSizeBytes = if ($dbItem) { [int64]$dbItem.Length } else { 0 } + $mainLogBytes = if ($mainLog) { [int64]$mainLog.Length } else { 0 } + $schedulerWriteUtc = if ($schedulerDir) { [datetime]$schedulerDir.LastWriteTimeUtc } else { $null } + $dbDeltaMb = 0.0 + $reglogDeltaMb = 0.0 + $schedulerTouched = $false + $isBootstrap = ($null -eq $previous) + + if (-not $isBootstrap) { + $prevDbSize = if ($previous.PSObject.Properties.Name -contains 'dbSizeBytes') { [double]$previous.dbSizeBytes } else { 0 } + $prevMainLogSize = if ($previous.PSObject.Properties.Name -contains 'mainLogBytes') { [double]$previous.mainLogBytes } else { 0 } + $dbDeltaMb = [math]::Round(($dbSizeBytes - $prevDbSize) / 1MB, 2) + $reglogDeltaMb = [math]::Round(($mainLogBytes - $prevMainLogSize) / 1MB, 2) + if ($schedulerWriteUtc -and $previous.PSObject.Properties.Name -contains 'schedulerWriteUtc' -and -not [string]::IsNullOrWhiteSpace([string]$previous.schedulerWriteUtc)) { + $prevSchedulerWriteUtc = [datetime]::Parse([string]$previous.schedulerWriteUtc) + $schedulerTouched = ($schedulerWriteUtc -gt $prevSchedulerWriteUtc) + } + elseif ($schedulerWriteUtc) { + $schedulerTouched = $true + } + } + + $activityScore = Get-CompanyActivityScore ` + -DbDeltaMb $dbDeltaMb ` + -ReglogDeltaMb $reglogDeltaMb ` + -ActiveLocks $activeLocks.Count ` + -HasTempDb ([bool]$tempDb) ` + -SchedulerTouched $schedulerTouched ` + -Status $status ` + -IsBootstrap $isBootstrap $documents.Add([ordered]@{ ts = $nowUtc infobase = [string]$base.infobase - organization = '' + organization = $organization department = 'FileBase' doc_type = 'InfobaseSnapshot' doc_id = $docId @@ -226,6 +333,24 @@ foreach ($base in $infobases) { posted = 1 }) + if ($activityScore -gt 0) { + $documents.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + organization = $organization + department = 'FileBaseActivity' + doc_type = 'CompanyActivitySnapshot' + doc_id = "$docId-$stamp" + doc_number = $stamp + author = $owner + counterparty = [string]$base.infobase + operation_type = 'activity_snapshot' + amount = $activityScore + status = $status + posted = 1 + }) + } + $audit.Add([ordered]@{ ts = $nowUtc infobase = [string]$base.infobase @@ -279,11 +404,31 @@ foreach ($base in $infobases) { message = "1Cv8JobScheduler touched at $([datetime]$schedulerDir.LastWriteTimeUtc)" }) } + + $reglog.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + user = $owner + host = $env:COMPUTERNAME + app = '1cv8-file' + event_name = 'CompanyActivitySnapshot' + level = if ($activityScore -gt 20) { 'warn' } else { 'info' } + duration_ms = 0 + message = "activityScore=$activityScore dbDeltaMb=$dbDeltaMb reglogDeltaMb=$reglogDeltaMb locks=$($activeLocks.Count) tempDb=$([bool]$tempDb) schedulerTouched=$schedulerTouched" + }) + + $nextExporterState[$stateKey] = [ordered]@{ + infobase = [string]$base.infobase + path = [string]$base.path + dbSizeBytes = $dbSizeBytes + mainLogBytes = $mainLogBytes + schedulerWriteUtc = if ($schedulerWriteUtc) { $schedulerWriteUtc.ToString('o') } else { '' } + updatedAtUtc = $nowUtc + } } $hostRows = @((Get-HostSample)) -$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $outRoot = Join-Path $env:TEMP "aw-rus-1c-outbox-$stamp" New-Item -ItemType Directory -Path $outRoot -Force | Out-Null @@ -311,6 +456,7 @@ try { foreach ($dataset in 'documents', 'reglog', 'audit', 'host') { Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files[$dataset]) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/" } + Save-ExporterState -Path $ExporterStatePath -State $nextExporterState Write-RunLog "upload complete analyticsHost=$AnalyticsHost remoteRoot=$RemoteRoot" } finally {