chore: merge main into PR #17

dlp-reliability-roadmap.md

@@ -0,0 +1,151 @@
+# DLP Reliability Roadmap
+
+## Scope
+Roadmap for improving runtime reliability of:
+- `windows/dlp-endpoint-signals-collector.ps1`
+- `windows/file-operations-collector.ps1`
+
+Date: 2026-05-04
+
+---
+
+## Stage 1 (1-2 days): Quick wins
+
+### 1) Disk queue + sender loop + retry/backoff/jitter
+
+**Goal:** no data loss on temporary network/server outages.
+
+**Tasks**
+- Add local append-only queue file per collector (`*.jsonl`) under ProgramData logs/artifacts root.
+- Write events to queue first, then send asynchronously.
+- Implement sender loop:
+  - reads oldest unsent records,
+  - sends in small batches,
+  - marks sent records,
+  - compacts queue periodically.
+- Implement retry policy with exponential backoff + jitter.
+
+**Acceptance criteria**
+- When API is unavailable, queue grows and collector keeps running.
+- When API recovers, queued events are flushed automatically.
+- No collector crash during repeated network failures.
+
+### 2) `eventId` + dedupe contract
+
+**Goal:** at-least-once delivery without logical duplicates.
+
+**Tasks**
+- Add `eventId` (UUID), `eventCreatedAt`, `collectorType`, `hostname` to every payload.
+- Define server dedupe contract:
+  - dedupe key = `eventId`,
+  - TTL for dedupe cache,
+  - idempotent processing semantics.
+
+**Acceptance criteria**
+- Retried sends do not create duplicate incidents/events in downstream storage.
+- Payload schema documentation updated.
+
+### 3) Basic metrics/logging
+
+**Goal:** visibility into health and data delivery.
+
+**Tasks**
+- Emit counters/gauges to log and heartbeat:
+  - `queueDepth`,
+  - `oldestUnsentAgeSec`,
+  - `eventsEnqueued`,
+  - `eventsSent`,
+  - `sendFailures`,
+  - `lastSendStatus`.
+
+**Acceptance criteria**
+- Operators can identify stuck queue and send failures from logs only.
+
+---
+
+## Stage 2: Hardening
+
+### 1) Circuit breaker + health probes
+
+**Tasks**
+- Add transport circuit breaker (Closed/Open/HalfOpen).
+- Open breaker after N consecutive failures.
+- In Open state perform probe every M seconds.
+- Close breaker on successful probe.
+
+**Acceptance criteria**
+- Reduced request storm during outage.
+- Deterministic recovery behavior after outage.
+
+### 2) Watcher auto-recreate
+
+**Tasks**
+- Handle `FileSystemWatcher` error/overflow events.
+- Recreate watcher and subscriptions automatically.
+- Keep watchdog timer to ensure watcher health.
+
+**Acceptance criteria**
+- Watcher resumes after overflow without manual restart.
+
+### 3) Last-known-good policy
+
+**Tasks**
+- Validate new policy before apply.
+- Cache last valid policy with checksum/version.
+- Rollback to cached policy on parse/validation errors.
+
+**Acceptance criteria**
+- Broken policy cannot stop detection loop.
+
+---
+
+## Stage 3: Reliability operations
+
+### 1) Chaos tests
+
+Scenarios:
+- network disconnect,
+- API 5xx bursts,
+- slow disk / queue write delay,
+- headless UI context,
+- forced collector restart.
+
+**Acceptance criteria**
+- For each scenario, documented expected behavior and observed result.
+- No silent data loss in tested outage windows.
+
+### 2) SLO + error budget process
+
+**Initial SLO proposals**
+- Event delivery latency P95 < 120s under normal conditions.
+- Data loss = 0 for outages shorter than 30 minutes (with available disk).
+- Collector liveness heartbeat every `pollSeconds * 3` max.
+
+**Process**
+- Define SLI dashboards.
+- Define release gates tied to error budget burn.
+- Freeze risky changes when budget exhausted.
+
+---
+
+## Suggested implementation order inside repository
+
+1. `file-operations-collector.ps1`: queue + sender + metrics (simpler flow).
+2. `dlp-endpoint-signals-collector.ps1`: queue + sender + metrics.
+3. Shared helper module extraction (`windows/lib/aw-transport.psm1`) for queue, retry, breaker.
+4. Policy cache and validation.
+5. Chaos test scripts and runbook.
+
+---
+
+## Deliverables checklist
+
+- [ ] Transport queue implementation in both collectors.
+- [ ] Payload schema update with `eventId`.
+- [ ] Dedupe contract documented for server side.
+- [ ] Metrics fields added to heartbeat/logs.
+- [ ] Circuit breaker implemented.
+- [ ] Watcher auto-recreate implemented.
+- [ ] Last-known-good policy implemented.
+- [ ] Chaos test runbook and results.
+- [ ] SLO/error budget document adopted.

windows/dlp-endpoint-signals-collector.ps1

@@ -33,6 +33,66 @@ function Write-EndpointLog {
     }
 }
 
+
+function Get-NewEventId { return ([guid]::NewGuid().ToString()) }
+
+function Initialize-TransportQueue {
+    param([Parameter(Mandatory = $true)][string]$QueuePath)
+    $script:QueuePath = $QueuePath
+    try {
+        $dir = Split-Path -Path $QueuePath -Parent
+        if ($dir -and -not (Test-Path -LiteralPath $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }
+        if (-not (Test-Path -LiteralPath $QueuePath)) { New-Item -ItemType File -Path $QueuePath -Force | Out-Null }
+    }
+    catch {
+        Write-EndpointLog ("Queue init error: {0}" -f $_.Exception.Message)
+    }
+}
+
+function Add-TransportQueueRecord {
+    param([string]$Uri,[string]$Json)
+    if (-not $script:QueuePath) { return }
+    $record = @{ id = (Get-NewEventId); createdAt = (Get-Date).ToUniversalTime().ToString('o'); uri = $Uri; payload = $Json }
+    Add-Content -LiteralPath $script:QueuePath -Value ($record | ConvertTo-Json -Compress)
+    $script:TransportStats.eventsEnqueued++
+}
+
+function Get-QueueDepth {
+    if (-not $script:QueuePath -or -not (Test-Path -LiteralPath $script:QueuePath)) { return 0 }
+    return @((Get-Content -LiteralPath $script:QueuePath)).Count
+}
+
+function Send-WithQueue {
+    param([string]$Uri,[string]$Json)
+    Add-TransportQueueRecord -Uri $Uri -Json $Json
+    Try-FlushTransportQueue -MaxItems 20
+}
+
+function Try-FlushTransportQueue {
+    param([int]$MaxItems = 20)
+    if (-not $script:QueuePath -or -not (Test-Path -LiteralPath $script:QueuePath)) { return }
+    $lines = @(Get-Content -LiteralPath $script:QueuePath)
+    if ($lines.Count -eq 0) { return }
+    $remaining = New-Object System.Collections.Generic.List[string]
+    $sent = 0
+    foreach ($line in $lines) {
+        if ($sent -ge $MaxItems) { $remaining.Add($line); continue }
+        try { $rec = $line | ConvertFrom-Json } catch { $remaining.Add($line); continue }
+        if (Invoke-AwJsonPost -Uri ([string]$rec.uri) -Json ([string]$rec.payload)) {
+            $script:TransportStats.eventsSent++
+            $script:TransportStats.lastSendStatus = 'ok'
+            $sent++
+        }
+        else {
+            $script:TransportStats.sendFailures++
+            $script:TransportStats.lastSendStatus = 'failed'
+            $remaining.Add($line)
+            break
+        }
+    }
+    Set-Content -LiteralPath $script:QueuePath -Value $remaining
+}
+
 function Invoke-AwJsonPost {
     param(
         [Parameter(Mandatory = $true)][string]$Uri,
@@ -40,7 +100,14 @@ function Invoke-AwJsonPost {
     )
 
     $bytes = [Text.Encoding]::UTF8.GetBytes($Json)
-    Invoke-RestMethod -Method Post -Uri $Uri -ContentType 'application/json; charset=utf-8' -Body $bytes -TimeoutSec 15 -DisableKeepAlive | Out-Null
+    try {
+        Invoke-RestMethod -Method Post -Uri $Uri -ContentType 'application/json; charset=utf-8' -Body $bytes | Out-Null
+        return $true
+    }
+    catch {
+        Write-EndpointLog ("POST Error: {0}" -f $_.Exception.Message)
+        return $false
+    }
 }
 
 function Ensure-Bucket {
@@ -68,18 +135,7 @@ function Ensure-Bucket {
         hostname = $script:Hostname
     } | ConvertTo-Json -Compress
 
-    try {
-        Invoke-AwJsonPost -Uri "$($script:ApiBase)/buckets/$BucketId" -Json $body
-    }
-    catch {
-        try {
-            Invoke-RestMethod -Method Get -Uri "$($script:ApiBase)/buckets/$BucketId" | Out-Null
-        }
-        catch {
-            Write-EndpointLog "Bucket create/check failed for ${BucketId}: $($_.Exception.Message)"
-            throw
-        }
-    }
+    Send-WithQueue -Uri "$($script:ApiBase)/buckets/$BucketId" -Json $body
     $script:KnownBuckets[$BucketId] = $true
 }
 
@@ -96,6 +152,8 @@ function Send-EndpointSignalHeartbeat {
         timestamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
         duration  = 0
         data      = @{
+            eventId    = (Get-NewEventId)
+            eventCreatedAt = (Get-Date).ToUniversalTime().ToString('o')
             signalType = $SignalType
             username   = $env:USERNAME
             sessionId  = $script:SessionId
@@ -104,7 +162,7 @@ function Send-EndpointSignalHeartbeat {
         } + $Data
     } | ConvertTo-Json -Depth 6 -Compress
 
-    Invoke-AwJsonPost -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=$script:PulseSeconds" -Json $payload
+    Send-WithQueue -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=$script:PulseSeconds" -Json $payload
 }
 
 function Send-DlpIncidentHeartbeat {
@@ -133,6 +191,8 @@ function Send-DlpIncidentHeartbeat {
         timestamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')
         duration  = 0
         data      = @{
+            eventId   = (Get-NewEventId)
+            eventCreatedAt = (Get-Date).ToUniversalTime().ToString('o')
             ruleId    = $RuleId
             action    = $Action
             severity  = $Severity
@@ -145,7 +205,7 @@ function Send-DlpIncidentHeartbeat {
         } + $Data + $captureData
     } | ConvertTo-Json -Depth 7 -Compress
 
-    Invoke-AwJsonPost -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=$script:PulseSeconds" -Json $payload
+    Send-WithQueue -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=$script:PulseSeconds" -Json $payload
 }
 
 function Get-FileSha256Hex {
@@ -867,7 +927,11 @@ $script:LogPath = $resolvedLogPath
 $script:IncidentArtifactsRoot = $resolvedIncidentArtifactsRoot
 $script:IncidentScreenshotEnabled = $resolvedIncidentScreenshotEnabled
 $script:ScreenshotTypesLoaded = $false
-$script:HeadlessMode = ($env:SESSIONNAME -eq 'Service') -or (-not [Environment]::UserInteractive)
+$script:TransportStats = @{ eventsEnqueued = 0; eventsSent = 0; sendFailures = 0; lastSendStatus = 'init' }
+$script:QueuePath = $null
+
+$queueFile = Join-Path $resolvedLogsRoot ("endpoint-queue-{0}.jsonl" -f $env:USERNAME)
+Initialize-TransportQueue -QueuePath $queueFile
 
 Load-DlpPolicy -Path $resolvedPolicyPath
 Write-EndpointLog ("endpoint collector started against {0}" -f $script:ApiBase)
@@ -1042,5 +1106,7 @@ while ($true) {
         Write-EndpointLog ("collector error: {0}" -f $_.Exception.Message)
     }
 
+    Try-FlushTransportQueue -MaxItems 50
+    Write-EndpointLog ("transport metrics queueDepth={0} enqueued={1} sent={2} failures={3} lastStatus={4}" -f (Get-QueueDepth), $script:TransportStats.eventsEnqueued, $script:TransportStats.eventsSent, $script:TransportStats.sendFailures, $script:TransportStats.lastSendStatus)
     Start-Sleep -Seconds $resolvedPollSeconds
 }

windows/file-operations-collector.ps1

@@ -26,6 +26,8 @@ $script:SessionId = [System.Diagnostics.Process]::GetCurrentProcess().SessionId
 # Настройка логирования
 $script:LogPath = $LogPath
 $script:LocalAgentLogsEnabled = [bool]$LogPath
+$script:TransportStats = @{ eventsEnqueued = 0; eventsSent = 0; sendFailures = 0; lastSendStatus = 'init' }
+$script:QueuePath = $null
 
 function Get-DeploymentConfig {
     param([string]$Path)
@@ -43,29 +45,89 @@ function Write-FileCollectorLog {
     } catch {}
 }
 
+
+function Get-NewEventId { return ([guid]::NewGuid().ToString()) }
+
+function Initialize-TransportQueue {
+    param([Parameter(Mandatory = $true)][string]$QueuePath)
+    $script:QueuePath = $QueuePath
+    try {
+        $dir = Split-Path -Path $QueuePath -Parent
+        if ($dir -and -not (Test-Path -LiteralPath $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }
+        if (-not (Test-Path -LiteralPath $QueuePath)) { New-Item -ItemType File -Path $QueuePath -Force | Out-Null }
+    } catch {
+        Write-FileCollectorLog "Queue init error: $($_.Exception.Message)"
+    }
+}
+
+function Add-TransportQueueRecord {
+    param([string]$Uri,[string]$Json)
+    if (-not $script:QueuePath) { return }
+    $record = @{ id = (Get-NewEventId); createdAt = (Get-Date).ToUniversalTime().ToString('o'); uri = $Uri; payload = $Json }
+    Add-Content -LiteralPath $script:QueuePath -Value ($record | ConvertTo-Json -Compress)
+    $script:TransportStats.eventsEnqueued++
+}
+
+function Get-QueueDepth {
+    if (-not $script:QueuePath -or -not (Test-Path -LiteralPath $script:QueuePath)) { return 0 }
+    return @((Get-Content -LiteralPath $script:QueuePath)).Count
+}
+
+function Try-FlushTransportQueue {
+    param([int]$MaxItems = 20)
+    if (-not $script:QueuePath -or -not (Test-Path -LiteralPath $script:QueuePath)) { return }
+    $lines = @(Get-Content -LiteralPath $script:QueuePath)
+    if ($lines.Count -eq 0) { return }
+    $remaining = New-Object System.Collections.Generic.List[string]
+    $sentInRun = 0
+    foreach ($line in $lines) {
+        if ($sentInRun -ge $MaxItems) { $remaining.Add($line); continue }
+        try { $rec = $line | ConvertFrom-Json } catch { $remaining.Add($line); continue }
+        if (Invoke-AwJsonPost -Uri ([string]$rec.uri) -Json ([string]$rec.payload)) {
+            $script:TransportStats.eventsSent++
+            $script:TransportStats.lastSendStatus = 'ok'
+            $sentInRun++
+        } else {
+            $script:TransportStats.sendFailures++
+            $script:TransportStats.lastSendStatus = 'failed'
+            $remaining.Add($line)
+            break
+        }
+    }
+    if ($sentInRun -lt $lines.Count) {
+        for ($i=$sentInRun+($lines.Count-$remaining.Count); $i -lt $lines.Count; $i++) { }
+    }
+    Set-Content -LiteralPath $script:QueuePath -Value $remaining
+}
+
+function Send-WithQueue {
+    param([string]$Uri,[string]$Json)
+    Add-TransportQueueRecord -Uri $Uri -Json $Json
+    Try-FlushTransportQueue -MaxItems 10
+}
+
 function Invoke-AwJsonPost {
     param(
         [Parameter(Mandatory = $true)][string]$Uri,
         [Parameter(Mandatory = $true)][string]$Json
     )
-    $httpClient = $null
+    $httpClient = New-Object System.Net.Http.HttpClient
     try {
-        $httpClient = New-Object System.Net.Http.HttpClient
         $content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json")
         $response = $httpClient.PostAsync($Uri, $content).Result
         if (-not $response.IsSuccessStatusCode) {
-            $status = [int]$response.StatusCode
+            $statusCode = [int]$response.StatusCode
             $reason = [string]$response.ReasonPhrase
-            $body = $response.Content.ReadAsStringAsync().Result
-            Write-FileCollectorLog ("POST failed: uri={0} status={1} reason={2} body={3}" -f $Uri, $status, $reason, $body)
+            $responseBody = $response.Content.ReadAsStringAsync().Result
+            Write-FileCollectorLog ("POST failed uri={0} status={1} reason={2} body={3}" -f $Uri, $statusCode, $reason, $responseBody)
         }
     } catch {
         Write-FileCollectorLog "POST Error: $($_.Exception.Message)"
+        return $false
     } finally {
-        if ($null -ne $httpClient) {
-            $httpClient.Dispose()
-        }
+        $httpClient.Dispose()
     }
+    return $response.IsSuccessStatusCode
 }
 
 function Ensure-Bucket {
@@ -92,19 +154,7 @@ function Ensure-Bucket {
         type     = $BucketType
         hostname = $script:Hostname
     } | ConvertTo-Json -Compress
-
-    try {
-        Invoke-AwJsonPost -Uri "$($script:ApiBase)/buckets/$BucketId" -Json $body
-    }
-    catch {
-        try {
-            Invoke-RestMethod -Method Get -Uri "$($script:ApiBase)/buckets/$BucketId" | Out-Null
-        }
-        catch {
-            Write-FileCollectorLog "Bucket create/check failed for ${BucketId}: $($_.Exception.Message)"
-            throw
-        }
-    }
+    Send-WithQueue -Uri "$($script:ApiBase)/buckets/$BucketId" -Json $body
     $script:KnownBuckets[$BucketId] = $true
 }
 
@@ -120,6 +170,9 @@ function Send-FileOperationEvent {
     Ensure-Bucket -BucketId $bucketId -ClientName 'aw-file-operations' -BucketType 'aw.file.operation'
 
     $data = @{
+        eventId   = (Get-NewEventId)
+        eventCreatedAt = (Get-Date).ToUniversalTime().ToString('o')
+
         operation = $Operation
         path      = $FilePath
         extension = [System.IO.Path]::GetExtension($FilePath)
@@ -140,7 +193,7 @@ function Send-FileOperationEvent {
         data      = $data
     } | ConvertTo-Json -Depth 5 -Compress
 
-    Invoke-AwJsonPost -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=15" -Json $payload
+    Send-WithQueue -Uri "$($script:ApiBase)/buckets/$bucketId/heartbeat?pulsetime=15" -Json $payload
 }
 
 $config = Get-DeploymentConfig -Path $ConfigPath
@@ -151,6 +204,9 @@ $hostName = if ($ServerHost) { $ServerHost } elseif ($config.server.host) { $con
 $port = if ($ServerPort) { $ServerPort } elseif ($config.server.port) { $config.server.port } else { 5600 }
 $script:ApiBase = "{0}://{1}:{2}/api/0" -f $scheme, $hostName, $port
 
+$queueFile = Join-Path ([System.IO.Path]::GetDirectoryName($script:LogPath)) ("file-collector-queue-{0}.jsonl" -f $env:USERNAME)
+Initialize-TransportQueue -QueuePath $queueFile
+
 $bucketId = 'aw-file-operations_' + $script:Hostname
 Ensure-Bucket -BucketId $bucketId -ClientName 'aw-file-operations' -BucketType 'aw.file.operation'
 
@@ -197,7 +253,7 @@ foreach ($path in $resolvedPaths) {
     $onRenamed = Register-ObjectEvent $watcher "Renamed" -Action {
         Send-FileOperationEvent -Operation 'Renamed' -FilePath $Event.SourceEventArgs.FullPath -OldFilePath $Event.SourceEventArgs.OldFullPath
     }
-    
+    $subscriptions += @($onChanged, $onDeleted, $onRenamed)
     $watchers += $watcher
     $subscriptions += @($onChanged, $onDeleted, $onRenamed)
 }
@@ -206,18 +262,18 @@ Write-FileCollectorLog "Collector started. Waiting for events..."
 
 try {
     while ($true) {
+        Try-FlushTransportQueue -MaxItems 50
+        Write-FileCollectorLog ("transport metrics queueDepth={0} enqueued={1} sent={2} failures={3} lastStatus={4}" -f (Get-QueueDepth), $script:TransportStats.eventsEnqueued, $script:TransportStats.eventsSent, $script:TransportStats.sendFailures, $script:TransportStats.lastSendStatus)
         Start-Sleep -Seconds $PollSeconds
     }
 }
 finally {
     Write-FileCollectorLog "Stopping collector..."
     foreach ($sub in @($subscriptions)) {
-        try {
-            if ($sub -and $sub.Id) {
-                Unregister-Event -SubscriptionId $sub.Id -ErrorAction SilentlyContinue
-                Remove-Job -Id $sub.Id -Force -ErrorAction SilentlyContinue
-            }
-        } catch {}
+        if ($null -ne $sub) {
+            try { Unregister-Event -SubscriptionId $sub.Id -ErrorAction SilentlyContinue } catch {}
+            try { Remove-Job -Id $sub.Id -Force -ErrorAction SilentlyContinue } catch {}
+        }
     }
     foreach ($w in $watchers) {
         $w.EnableRaisingEvents = $false