feat(dlp-hardening): secure inventory and extend aggregator sources

.planning/phases/01-collectors-and-api-flow-hardening/PLAN.md

@@ -20,3 +20,23 @@ Deliver stable collector-to-server data flow so activity/worktime pages have con
 ## Status
 
 Planned.
+
+## 2. Варианты доработки DLP
+
+### Вариант A: “Hardening” — Стабилизация текущего
+
+Цель: довести текущие коллекторы до production-grade уровня надёжности.
+
+| # | Задача | Усилие | Влияние |
+|---|---|---|---|
+| A1 | HTTP retry + exponential backoff во всех коллекторах | 3-5 дней | Высокое — перестанут теряться события |
+| A2 | Локальный WAL (Write-Ahead Log) — буферизация событий при недоступности сервера | 1-2 нед | Критическое — гарантия доставки |
+| A3 | Healthcheck endpoint и self-diagnostics в каждом коллекторе | 3-5 дней | Среднее — видимость состояния агентов |
+| A4 | Расширить aggregator: добавить `aw-email-monitor_` и `aw-dlp-endpoint-signals_` в сбор | 1 день | Среднее |
+| A5 | Systemd timer / Windows Task для aggregator (автоматический запуск) | 1 день | Среднее |
+| A6 | Убрать пароль из `inventory.ini` → использовать Ansible Vault или env var | 1 час | Критическое (безопасность) |
+| A7 | Graceful shutdown и cleanup event subscriptions во всех коллекторах | 2-3 дня | Среднее |
+
+Общее усилие: ~3-4 недели.
+
+Рекомендация: обязательно сделать перед любым масштабированием. Без этого DLP — “best effort” мониторинг, а не надёжная система.

docs/dlp-aggregator.md

@@ -8,6 +8,8 @@ The prototype reads:
 
 - `aw-file-operations_*` (`aw.file.operation`) — file create/delete/rename telemetry, including `archiveHint`.
 - `aw-dlp-incidents_*` (`aw.dlp.incident`) — browser/endpoint DLP incidents and screenshot metadata when available.
+- `aw-dlp-endpoint-signals_*` (`aw.dlp.endpoint.signal`) — endpoint signal heartbeats/events.
+- `aw-email-monitor_*` (`aw.email.signal`) — outbound email signal stream.
 
 ## SQLite smoke test
 

scripts/aggregate_dlp_events.py

@@ -18,7 +18,12 @@ JsonScalar: TypeAlias = str | int | float | bool | None
 JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"]
 
 
-DEFAULT_BUCKET_PREFIXES = ("aw-file-operations_", "aw-dlp-incidents_")
+DEFAULT_BUCKET_PREFIXES = (
+    "aw-file-operations_",
+    "aw-dlp-incidents_",
+    "aw-dlp-endpoint-signals_",
+    "aw-email-monitor_",
+)
 DEFAULT_SQLITE_PATH = "data/dlp-events.sqlite3"
 EVENT_COLUMNS = (
     "bucket_id",
@@ -134,6 +139,10 @@ def bucket_stream_type(bucket: Bucket) -> str | None:
         return "file_operation"
     if bucket.id.startswith("aw-dlp-incidents_") or bucket.type == "aw.dlp.incident":
         return "dlp_incident"
+    if bucket.id.startswith("aw-dlp-endpoint-signals_") or bucket.type == "aw.dlp.endpoint.signal":
+        return "dlp_endpoint_signal"
+    if bucket.id.startswith("aw-email-monitor_") or bucket.type == "aw.email.signal":
+        return "email_monitor"
     return None
 
 

windows/file-operations-collector.ps1

@@ -46,24 +46,47 @@ function Write-FileCollectorLog {
 function Invoke-AwJsonPost {
     param(
         [Parameter(Mandatory = $true)][string]$Uri,
-        [Parameter(Mandatory = $true)][string]$Json
+        [Parameter(Mandatory = $true)][string]$Json,
+        [int]$MaxAttempts = 5,
+        [int]$InitialBackoffMs = 500
     )
-    $httpClient = $null
+    $attempt = 1
-    try {
+    $backoff = [Math]::Max(100, $InitialBackoffMs)
-        $httpClient = New-Object System.Net.Http.HttpClient
+
-        $content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json")
+    while ($attempt -le $MaxAttempts) {
-        $response = $httpClient.PostAsync($Uri, $content).Result
+        $httpClient = $null
-        if (-not $response.IsSuccessStatusCode) {
+        try {
+            $httpClient = New-Object System.Net.Http.HttpClient
+            $content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json")
+            $response = $httpClient.PostAsync($Uri, $content).Result
+            if ($response.IsSuccessStatusCode) {
+                return
+            }
+
             $status = [int]$response.StatusCode
             $reason = [string]$response.ReasonPhrase
             $body = $response.Content.ReadAsStringAsync().Result
-            Write-FileCollectorLog ("POST failed: uri={0} status={1} reason={2} body={3}" -f $Uri, $status, $reason, $body)
+            Write-FileCollectorLog ("POST failed: attempt={0}/{1} uri={2} status={3} reason={4} body={5}" -f $attempt, $MaxAttempts, $Uri, $status, $reason, $body)
-        }
+            if ($attempt -ge $MaxAttempts) {
-    } catch {
+                return
-        Write-FileCollectorLog "POST Error: $($_.Exception.Message)"
+            }
-    } finally {
+            Start-Sleep -Milliseconds $backoff
-        if ($null -ne $httpClient) {
+            $backoff = [Math]::Min($backoff * 2, 10000)
-            $httpClient.Dispose()
+            $attempt++
+            continue
+        } catch {
+            Write-FileCollectorLog ("POST error: attempt={0}/{1} uri={2} error={3}" -f $attempt, $MaxAttempts, $Uri, $_.Exception.Message)
+            if ($attempt -ge $MaxAttempts) {
+                return
+            }
+            Start-Sleep -Milliseconds $backoff
+            $backoff = [Math]::Min($backoff * 2, 10000)
+            $attempt++
+            continue
+        } finally {
+            if ($null -ne $httpClient) {
+                $httpClient.Dispose()
+            }
         }
     }
 }