awatch-rus/AWatch-rus
1
0
Fork 0
Code Issues Pull Requests Actions 100 Packages Projects Releases Wiki Activity

feat(1c): derive company activity from file-base telemetry

Browse Source
This commit is contained in:
igor04091968
2026-05-22 10:25:09 +03:00
parent f7114bde03
commit 1ca43398a4
7 changed files with 203 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clickhouse-1c/ai/company_intelligence_api.py
+2 -2
View File
@@ -146,8 +146,8 @@ def company_summary(counterparty: str, infobase: str | None = None) -> dict[str,
signals = rows_to_dict(client.query(signals_sql))
timeline = rows_to_dict(client.query(timeline_sql))
essence = (
f"Компания {counterparty}: за 30 дней документов {card['docs_30d']}, объём {card['amount_30d']}, "
f"прогноз на 30 дней {card['amount_forecast_30d']}, риск {card['signal_severity']}."
f"Компания {counterparty}: за 30 дней событий {card['docs_30d']}, суммарная активность {card['amount_30d']}, "
f"прогноз активности на 30 дней {card['amount_forecast_30d']}, риск {card['signal_severity']}."
)
return {
"essence": essence,
clickhouse-1c/ai/refresh_company_intelligence.py
+1 -1
View File
@@ -218,7 +218,7 @@ def main() -> int:
if days_since_last_activity >= 14 and (docs_prev_7d > 0 or amount_prev_7d > 0):
signals.append(("inactive_company", 85, "high", f"Нет активности по компании {counterparty} уже {days_since_last_activity} дн."))
if amount_prev_7d > 0 and amount_7d < amount_prev_7d * 0.5:
signals.append(("amount_drop", 70, "high", f"Объём по компании {counterparty} упал более чем на 50% неделя к неделе."))
signals.append(("amount_drop", 70, "high", f"Активность по компании {counterparty} упала более чем на 50% неделя к неделе."))
if docs_prev_7d > 0 and docs_7d == 0:
signals.append(("docs_stopped", 55, "medium", f"По компании {counterparty} прекратился поток документов за последние 7 дней."))
if open_cases_total > 0:
clickhouse-1c/detections/build_entity_timeline.sql
+21 -1
View File
@@ -11,12 +11,32 @@ FROM (
concat('document:', doc_type) AS event_type,
if(posted = 1, 'low', 'medium') AS severity,
if(posted = 1, 5, 20) AS score,
doc_id AS ref_id,
concat(doc_id, ':', toString(toUnixTimestamp(ts)), ':', doc_type) AS ref_id,
concat('Документ ', doc_type, ' №', doc_number, ' статус=', status) AS summary
FROM analytics_1c.documents
) AS src
WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline);
INSERT INTO analytics_1c.entity_timeline
SELECT *
FROM (
SELECT
ts,
'counterparty' AS entity_type,
counterparty AS entity_id,
infobase,
author AS actor,
'documents' AS source,
concat('counterparty:', operation_type) AS event_type,
if(status = 'busy', 'medium', 'low') AS severity,
greatest(10, toUInt32(round(amount))) AS score,
concat('counterparty:', counterparty, ':', doc_id, ':', toString(toUnixTimestamp(ts))) AS ref_id,
concat('Активность компании ', counterparty, ': ', doc_type, ' score=', toString(amount), ' status=', status) AS summary
FROM analytics_1c.documents
WHERE counterparty != ''
) AS src
WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline);
INSERT INTO analytics_1c.entity_timeline
SELECT *
FROM (
clickhouse-1c/detections/insert_detections.sql
+20
View File
@@ -19,6 +19,26 @@ FROM (
) AS src
WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections);
INSERT INTO analytics_1c.detections
SELECT *
FROM (
SELECT
generated_at AS ts,
concat('company_signal:', signal_id, ':', toString(toUnixTimestamp(generated_at))) AS detection_id,
infobase,
concat('company_', signal_type) AS rule_id,
'Сигнал активности компании' AS rule_title,
'counterparty' AS entity_type,
counterparty AS entity_id,
severity,
score,
summary,
'open' AS status
FROM analytics_1c.v_company_health_current
WHERE severity IN ('medium', 'high', 'critical')
) AS src
WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections);
INSERT INTO analytics_1c.detections
SELECT *
FROM (
clickhouse-1c/grafana/provisioning/dashboards/files/1c-company-intelligence.json
+3 -3
View File
@@ -79,7 +79,7 @@
},
{
"id": 2,
"title": "Прогноз объёма 30д",
"title": "Прогноз активности 30д",
"type": "stat",
"datasource": {
"type": "grafana-clickhouse-datasource",
@@ -235,7 +235,7 @@
},
{
"id": 5,
"title": "Объём по компаниям",
"title": "Активность по компаниям",
"type": "timeseries",
"datasource": {
"type": "grafana-clickhouse-datasource",
@@ -282,7 +282,7 @@
},
{
"id": 6,
"title": "Количество документов по компаниям",
"title": "Количество событий по компаниям",
"type": "timeseries",
"datasource": {
"type": "grafana-clickhouse-datasource",
clickhouse-1c/ops/run_ingest_cycle.sh
+8 -8
View File
@@ -45,6 +45,14 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
--database "${CLICKHOUSE_DB}" \
< "${ROOT}/detections/build_entity_timeline.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_intelligence_refresh.sh"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
@@ -56,11 +64,3 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
< "${ROOT}/detections/open_cases_from_detections.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_intelligence_refresh.sh"
windows/export-upload-file-1c-telemetry.ps1
+148 -2
View File
@@ -13,6 +13,7 @@ $ErrorActionPreference = 'Stop'
$LogDir = 'C:\ProgramData\AWatch-rus\logs'
$LogPath = Join-Path $LogDir 'file1c-telemetry.log'
$ScpExe = Join-Path $env:WINDIR 'System32\OpenSSH\scp.exe'
$ExporterStatePath = Join-Path (Split-Path -Parent $ConfigPath) 'file1c-telemetry-state.json'
New-Item -ItemType Directory -Path $LogDir -Force | Out-Null
@@ -156,6 +157,76 @@ function Invoke-SshUploadWithRetry {
}
}
function Get-ExporterState {
param(
[Parameter(Mandatory = $true)]
[string]$Path
)
if (-not (Test-Path -LiteralPath $Path)) {
return @{}
}
$raw = Get-Content -Raw -LiteralPath $Path -Encoding UTF8
if ([string]::IsNullOrWhiteSpace($raw)) {
return @{}
}
$payload = $raw | ConvertFrom-Json
$state = @{}
foreach ($prop in $payload.PSObject.Properties) {
$state[[string]$prop.Name] = $prop.Value
}
return $state
}
function Save-ExporterState {
param(
[Parameter(Mandatory = $true)]
[string]$Path,
[Parameter(Mandatory = $true)]
[hashtable]$State
)
$directory = Split-Path -Parent $Path
if ($directory) {
New-Item -ItemType Directory -Path $directory -Force | Out-Null
}
$State | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $Path -Encoding UTF8
}
function Get-CompanyActivityScore {
param(
[double]$DbDeltaMb,
[double]$ReglogDeltaMb,
[int]$ActiveLocks,
[bool]$HasTempDb,
[bool]$SchedulerTouched,
[string]$Status,
[bool]$IsBootstrap
)
$score = 0.0
$score += [math]::Abs($DbDeltaMb)
$score += [math]::Abs($ReglogDeltaMb)
if ($ActiveLocks -gt 0) {
$score += ($ActiveLocks * 5)
}
if ($HasTempDb) {
$score += 10
}
if ($SchedulerTouched) {
$score += 3
}
if ($Status -eq 'busy') {
$score += 5
}
if ($IsBootstrap -and $score -le 0) {
$score = 1
}
return [math]::Round($score, 2)
}
Write-RunLog 'file1c exporter start'
if (-not (Test-Path -LiteralPath $ScpExe)) {
@@ -192,6 +263,9 @@ if ($config.PSObject.Properties.Name -contains 'analytics' -and
$infobases = @(Get-1CFileInfobases)
$nowUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
$exporterState = Get-ExporterState -Path $ExporterStatePath
$nextExporterState = @{}
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$documents = New-Object System.Collections.Generic.List[object]
$reglog = New-Object System.Collections.Generic.List[object]
@@ -207,13 +281,46 @@ foreach ($base in $infobases) {
$tempDb = Get-Item -LiteralPath (Join-Path $base.path '1Cv8tmp.1CD') -ErrorAction SilentlyContinue
$schedulerDir = Get-Item -LiteralPath (Join-Path $base.path '1Cv8JobScheduler') -ErrorAction SilentlyContinue
$owner = if ($base.userName) { [string]$base.userName } else { 'unknown' }
$organization = [string](Split-Path -Leaf (Split-Path -Parent $base.path))
$status = if ($activeLocks.Count -gt 0 -or $tempDb) { 'busy' } else { 'online' }
$docId = if ($base.baseId) { [string]$base.baseId } else { ([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$base.path)).TrimEnd('=').Replace('/','_').Replace('+','-')) }
$stateKey = [string]$docId
$previous = if ($exporterState.ContainsKey($stateKey)) { $exporterState[$stateKey] } else { $null }
$dbSizeBytes = if ($dbItem) { [int64]$dbItem.Length } else { 0 }
$mainLogBytes = if ($mainLog) { [int64]$mainLog.Length } else { 0 }
$schedulerWriteUtc = if ($schedulerDir) { [datetime]$schedulerDir.LastWriteTimeUtc } else { $null }
$dbDeltaMb = 0.0
$reglogDeltaMb = 0.0
$schedulerTouched = $false
$isBootstrap = ($null -eq $previous)
if (-not $isBootstrap) {
$prevDbSize = if ($previous.PSObject.Properties.Name -contains 'dbSizeBytes') { [double]$previous.dbSizeBytes } else { 0 }
$prevMainLogSize = if ($previous.PSObject.Properties.Name -contains 'mainLogBytes') { [double]$previous.mainLogBytes } else { 0 }
$dbDeltaMb = [math]::Round(($dbSizeBytes - $prevDbSize) / 1MB, 2)
$reglogDeltaMb = [math]::Round(($mainLogBytes - $prevMainLogSize) / 1MB, 2)
if ($schedulerWriteUtc -and $previous.PSObject.Properties.Name -contains 'schedulerWriteUtc' -and -not [string]::IsNullOrWhiteSpace([string]$previous.schedulerWriteUtc)) {
$prevSchedulerWriteUtc = [datetime]::Parse([string]$previous.schedulerWriteUtc)
$schedulerTouched = ($schedulerWriteUtc -gt $prevSchedulerWriteUtc)
}
elseif ($schedulerWriteUtc) {
$schedulerTouched = $true
}
}
$activityScore = Get-CompanyActivityScore `
-DbDeltaMb $dbDeltaMb `
-ReglogDeltaMb $reglogDeltaMb `
-ActiveLocks $activeLocks.Count `
-HasTempDb ([bool]$tempDb) `
-SchedulerTouched $schedulerTouched `
-Status $status `
-IsBootstrap $isBootstrap
$documents.Add([ordered]@{
ts = $nowUtc
infobase = [string]$base.infobase
organization = ''
organization = $organization
department = 'FileBase'
doc_type = 'InfobaseSnapshot'
doc_id = $docId
@@ -226,6 +333,24 @@ foreach ($base in $infobases) {
posted = 1
})
if ($activityScore -gt 0) {
$documents.Add([ordered]@{
ts = $nowUtc
infobase = [string]$base.infobase
organization = $organization
department = 'FileBaseActivity'
doc_type = 'CompanyActivitySnapshot'
doc_id = "$docId-$stamp"
doc_number = $stamp
author = $owner
counterparty = [string]$base.infobase
operation_type = 'activity_snapshot'
amount = $activityScore
status = $status
posted = 1
})
}
$audit.Add([ordered]@{
ts = $nowUtc
infobase = [string]$base.infobase
@@ -279,11 +404,31 @@ foreach ($base in $infobases) {
message = "1Cv8JobScheduler touched at $([datetime]$schedulerDir.LastWriteTimeUtc)"
})
}
$reglog.Add([ordered]@{
ts = $nowUtc
infobase = [string]$base.infobase
user = $owner
host = $env:COMPUTERNAME
app = '1cv8-file'
event_name = 'CompanyActivitySnapshot'
level = if ($activityScore -gt 20) { 'warn' } else { 'info' }
duration_ms = 0
message = "activityScore=$activityScore dbDeltaMb=$dbDeltaMb reglogDeltaMb=$reglogDeltaMb locks=$($activeLocks.Count) tempDb=$([bool]$tempDb) schedulerTouched=$schedulerTouched"
})
$nextExporterState[$stateKey] = [ordered]@{
infobase = [string]$base.infobase
path = [string]$base.path
dbSizeBytes = $dbSizeBytes
mainLogBytes = $mainLogBytes
schedulerWriteUtc = if ($schedulerWriteUtc) { $schedulerWriteUtc.ToString('o') } else { '' }
updatedAtUtc = $nowUtc
}
}
$hostRows = @((Get-HostSample))
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$outRoot = Join-Path $env:TEMP "aw-rus-1c-outbox-$stamp"
New-Item -ItemType Directory -Path $outRoot -Force | Out-Null
@@ -311,6 +456,7 @@ try {
foreach ($dataset in 'documents', 'reglog', 'audit', 'host') {
Invoke-SshUploadWithRetry -KeyPath $effectiveKeyPath -SourcePath ([string]$files[$dataset]) -Destination "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/"
}
Save-ExporterState -Path $ExporterStatePath -State $nextExporterState
Write-RunLog "upload complete analyticsHost=$AnalyticsHost remoteRoot=$RemoteRoot"
}
finally {