diff --git a/.planning/phases/01-collectors-and-api-flow-hardening/PLAN.md b/.planning/phases/01-collectors-and-api-flow-hardening/PLAN.md index 69bfe67..0e565c7 100644 --- a/.planning/phases/01-collectors-and-api-flow-hardening/PLAN.md +++ b/.planning/phases/01-collectors-and-api-flow-hardening/PLAN.md @@ -20,3 +20,23 @@ Deliver stable collector-to-server data flow so activity/worktime pages have con ## Status Planned. + +## 2. Варианты доработки DLP + +### Вариант A: “Hardening” — Стабилизация текущего + +Цель: довести текущие коллекторы до production-grade уровня надёжности. + +| # | Задача | Усилие | Влияние | +|---|---|---|---| +| A1 | HTTP retry + exponential backoff во всех коллекторах | 3-5 дней | Высокое — перестанут теряться события | +| A2 | Локальный WAL (Write-Ahead Log) — буферизация событий при недоступности сервера | 1-2 нед | Критическое — гарантия доставки | +| A3 | Healthcheck endpoint и self-diagnostics в каждом коллекторе | 3-5 дней | Среднее — видимость состояния агентов | +| A4 | Расширить aggregator: добавить `aw-email-monitor_` и `aw-dlp-endpoint-signals_` в сбор | 1 день | Среднее | +| A5 | Systemd timer / Windows Task для aggregator (автоматический запуск) | 1 день | Среднее | +| A6 | Убрать пароль из `inventory.ini` → использовать Ansible Vault или env var | 1 час | Критическое (безопасность) | +| A7 | Graceful shutdown и cleanup event subscriptions во всех коллекторах | 2-3 дня | Среднее | + +Общее усилие: ~3-4 недели. + +Рекомендация: обязательно сделать перед любым масштабированием. Без этого DLP — “best effort” мониторинг, а не надёжная система. diff --git a/docs/dlp-aggregator.md b/docs/dlp-aggregator.md index c54edbb..7f63473 100644 --- a/docs/dlp-aggregator.md +++ b/docs/dlp-aggregator.md @@ -8,6 +8,8 @@ The prototype reads: - `aw-file-operations_*` (`aw.file.operation`) — file create/delete/rename telemetry, including `archiveHint`. - `aw-dlp-incidents_*` (`aw.dlp.incident`) — browser/endpoint DLP incidents and screenshot metadata when available. +- `aw-dlp-endpoint-signals_*` (`aw.dlp.endpoint.signal`) — endpoint signal heartbeats/events. +- `aw-email-monitor_*` (`aw.email.signal`) — outbound email signal stream. ## SQLite smoke test diff --git a/scripts/aggregate_dlp_events.py b/scripts/aggregate_dlp_events.py index 7be7760..dcc58cd 100755 --- a/scripts/aggregate_dlp_events.py +++ b/scripts/aggregate_dlp_events.py @@ -18,7 +18,12 @@ JsonScalar: TypeAlias = str | int | float | bool | None JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -DEFAULT_BUCKET_PREFIXES = ("aw-file-operations_", "aw-dlp-incidents_") +DEFAULT_BUCKET_PREFIXES = ( + "aw-file-operations_", + "aw-dlp-incidents_", + "aw-dlp-endpoint-signals_", + "aw-email-monitor_", +) DEFAULT_SQLITE_PATH = "data/dlp-events.sqlite3" EVENT_COLUMNS = ( "bucket_id", @@ -134,6 +139,10 @@ def bucket_stream_type(bucket: Bucket) -> str | None: return "file_operation" if bucket.id.startswith("aw-dlp-incidents_") or bucket.type == "aw.dlp.incident": return "dlp_incident" + if bucket.id.startswith("aw-dlp-endpoint-signals_") or bucket.type == "aw.dlp.endpoint.signal": + return "dlp_endpoint_signal" + if bucket.id.startswith("aw-email-monitor_") or bucket.type == "aw.email.signal": + return "email_monitor" return None diff --git a/windows/file-operations-collector.ps1 b/windows/file-operations-collector.ps1 index bc63d55..3160319 100644 --- a/windows/file-operations-collector.ps1 +++ b/windows/file-operations-collector.ps1 @@ -46,24 +46,47 @@ function Write-FileCollectorLog { function Invoke-AwJsonPost { param( [Parameter(Mandatory = $true)][string]$Uri, - [Parameter(Mandatory = $true)][string]$Json + [Parameter(Mandatory = $true)][string]$Json, + [int]$MaxAttempts = 5, + [int]$InitialBackoffMs = 500 ) - $httpClient = $null - try { - $httpClient = New-Object System.Net.Http.HttpClient - $content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json") - $response = $httpClient.PostAsync($Uri, $content).Result - if (-not $response.IsSuccessStatusCode) { + $attempt = 1 + $backoff = [Math]::Max(100, $InitialBackoffMs) + + while ($attempt -le $MaxAttempts) { + $httpClient = $null + try { + $httpClient = New-Object System.Net.Http.HttpClient + $content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json") + $response = $httpClient.PostAsync($Uri, $content).Result + if ($response.IsSuccessStatusCode) { + return + } + $status = [int]$response.StatusCode $reason = [string]$response.ReasonPhrase $body = $response.Content.ReadAsStringAsync().Result - Write-FileCollectorLog ("POST failed: uri={0} status={1} reason={2} body={3}" -f $Uri, $status, $reason, $body) - } - } catch { - Write-FileCollectorLog "POST Error: $($_.Exception.Message)" - } finally { - if ($null -ne $httpClient) { - $httpClient.Dispose() + Write-FileCollectorLog ("POST failed: attempt={0}/{1} uri={2} status={3} reason={4} body={5}" -f $attempt, $MaxAttempts, $Uri, $status, $reason, $body) + if ($attempt -ge $MaxAttempts) { + return + } + Start-Sleep -Milliseconds $backoff + $backoff = [Math]::Min($backoff * 2, 10000) + $attempt++ + continue + } catch { + Write-FileCollectorLog ("POST error: attempt={0}/{1} uri={2} error={3}" -f $attempt, $MaxAttempts, $Uri, $_.Exception.Message) + if ($attempt -ge $MaxAttempts) { + return + } + Start-Sleep -Milliseconds $backoff + $backoff = [Math]::Min($backoff * 2, 10000) + $attempt++ + continue + } finally { + if ($null -ne $httpClient) { + $httpClient.Dispose() + } } } }