awatch-rus/AWatch-rus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dlp): add WAL buffering and health snapshots for collectors

Browse Source
This commit is contained in:
igor04091968
2026-05-07 23:47:03 +03:00
parent f3c5e9ea53
commit ac59d44719
2 changed files with 141 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
windows/email-outbound-collector.ps1
+74 -2
View File
@@ -53,13 +53,79 @@ function Write-CollectorLog {
catch { }
}
function Invoke-AwJsonPost {
function Add-WalEntry {
param(
[Parameter(Mandatory = $true)][string]$Uri,
[Parameter(Mandatory = $true)][string]$Json
)
if ([string]::IsNullOrWhiteSpace($script:WalPath)) { return }
try {
$entry = @{ ts = (Get-Date).ToUniversalTime().ToString('o'); uri = $Uri; json = $Json } | ConvertTo-Json -Compress
Add-Content -LiteralPath $script:WalPath -Value $entry -Encoding UTF8
} catch {}
}
function Flush-Wal {
if ([string]::IsNullOrWhiteSpace($script:WalPath) -or -not (Test-Path -LiteralPath $script:WalPath)) { return }
$remaining = New-Object System.Collections.Generic.List[string]
try {
$script:WalFlushing = $true
foreach ($line in (Get-Content -LiteralPath $script:WalPath -ErrorAction SilentlyContinue)) {
if ([string]::IsNullOrWhiteSpace($line)) { continue }
try {
$entry = $line | ConvertFrom-Json
if ($null -eq $entry -or -not $entry.uri -or -not $entry.json) { continue }
if (-not (Invoke-AwJsonPost -Uri ([string]$entry.uri) -Json ([string]$entry.json))) { $remaining.Add($line) }
} catch { $remaining.Add($line) }
}
if ($remaining.Count -eq 0) {
Remove-Item -LiteralPath $script:WalPath -Force -ErrorAction SilentlyContinue
} else {
Set-Content -LiteralPath $script:WalPath -Value ($remaining -join [Environment]::NewLine) -Encoding UTF8
}
} finally {
$script:WalFlushing = $false
}
}
function Write-CollectorHealth {
param([string]$Status = 'running')
if ([string]::IsNullOrWhiteSpace($script:HealthPath)) { return }
try {
$walDepth = 0
if ($script:WalPath -and (Test-Path -LiteralPath $script:WalPath)) { $walDepth = @((Get-Content -LiteralPath $script:WalPath)).Count }
$health = @{
collector = 'email-outbound'; hostname = $script:Hostname; sessionId = $script:SessionId;
status = $Status; apiBase = $script:ApiBase; walDepth = $walDepth; ts = (Get-Date).ToUniversalTime().ToString('o')
} | ConvertTo-Json -Depth 5
Set-Content -LiteralPath $script:HealthPath -Value $health -Encoding UTF8
} catch {}
}
function Invoke-AwJsonPost {
param(
[Parameter(Mandatory = $true)][string]$Uri,
[Parameter(Mandatory = $true)][string]$Json,
[int]$MaxAttempts = 5,
[int]$InitialBackoffMs = 500
)
$attempt = 1
$backoff = [Math]::Max(100, $InitialBackoffMs)
$bytes = [Text.Encoding]::UTF8.GetBytes($Json)
Invoke-RestMethod -Method Post -Uri $Uri -ContentType 'application/json; charset=utf-8' -Body $bytes | Out-Null
while ($attempt -le $MaxAttempts) {
try {
Invoke-RestMethod -Method Post -Uri $Uri -ContentType 'application/json; charset=utf-8' -Body $bytes | Out-Null
return $true
} catch {
if ($attempt -ge $MaxAttempts) {
if (-not $script:WalFlushing) { Add-WalEntry -Uri $Uri -Json $Json }
return $false
}
Start-Sleep -Milliseconds $backoff
$backoff = [Math]::Min($backoff * 2, 10000)
$attempt++
}
}
}
function Ensure-Bucket {
@@ -517,6 +583,10 @@ $script:SeenSmtpConnections = @{}
$script:PulseSeconds = [Math]::Max($resolvedPollSeconds * 3, 30)
$script:LocalAgentLogsEnabled = $resolvedLocalAgentLogsEnabled
$script:LogPath = $resolvedLogPath
$stateRoot = if ($deploymentConfig -and $deploymentConfig.paths -and $deploymentConfig.paths.stateRoot) { [string]$deploymentConfig.paths.stateRoot } else { 'C:\ProgramData\AWatch-rus' }
$script:WalPath = Join-Path $stateRoot 'wal-email-outbound.ndjson'
$script:HealthPath = Join-Path $stateRoot 'health-email-outbound.json'
$script:WalFlushing = $false
$script:OutlookApp = $null
$script:OutlookNamespace = $null
$script:SentFolder = $null
@@ -542,6 +612,8 @@ if ($useOutlook) {
while ($true) {
try {
Flush-Wal
Write-CollectorHealth -Status 'running'
if (-not $script:Policy.defaults.enabled) {
Start-Sleep -Seconds $resolvedPollSeconds
continue
windows/file-operations-collector.ps1
+67 -21
View File
@@ -1,4 +1,4 @@
[CmdletBinding()]
[CmdletBinding()]
param(
[string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json',
[string]$ServerHost,
@@ -22,6 +22,9 @@ Add-Type -AssemblyName System.Net.Http
$script:KnownBuckets = @{}
$script:Hostname = $env:COMPUTERNAME
$script:SessionId = [System.Diagnostics.Process]::GetCurrentProcess().SessionId
$script:WalPath = $null
$script:HealthPath = $null
$script:WalFlushing = $false
# Настройка логирования
$script:LogPath = $LogPath
@@ -43,6 +46,55 @@ function Write-FileCollectorLog {
} catch {}
}
function Add-WalEntry {
param(
[Parameter(Mandatory = $true)][string]$Uri,
[Parameter(Mandatory = $true)][string]$Json
)
if ([string]::IsNullOrWhiteSpace($script:WalPath)) { return }
try {
$entry = @{ ts = (Get-Date).ToUniversalTime().ToString('o'); uri = $Uri; json = $Json } | ConvertTo-Json -Compress
Add-Content -LiteralPath $script:WalPath -Value $entry -Encoding UTF8
} catch {}
}
function Flush-Wal {
if ([string]::IsNullOrWhiteSpace($script:WalPath) -or -not (Test-Path -LiteralPath $script:WalPath)) { return }
$remaining = New-Object System.Collections.Generic.List[string]
try {
$script:WalFlushing = $true
foreach ($line in (Get-Content -LiteralPath $script:WalPath -ErrorAction SilentlyContinue)) {
if ([string]::IsNullOrWhiteSpace($line)) { continue }
try {
$entry = $line | ConvertFrom-Json
if ($null -eq $entry -or -not $entry.uri -or -not $entry.json) { continue }
if (-not (Invoke-AwJsonPost -Uri ([string]$entry.uri) -Json ([string]$entry.json))) { $remaining.Add($line) }
} catch { $remaining.Add($line) }
}
if ($remaining.Count -eq 0) {
Remove-Item -LiteralPath $script:WalPath -Force -ErrorAction SilentlyContinue
} else {
Set-Content -LiteralPath $script:WalPath -Value ($remaining -join [Environment]::NewLine) -Encoding UTF8
}
} finally {
$script:WalFlushing = $false
}
}
function Write-CollectorHealth {
param([string]$Status = 'running')
if ([string]::IsNullOrWhiteSpace($script:HealthPath)) { return }
try {
$walDepth = 0
if ($script:WalPath -and (Test-Path -LiteralPath $script:WalPath)) { $walDepth = @((Get-Content -LiteralPath $script:WalPath)).Count }
$health = @{
collector = 'file-operations'; hostname = $script:Hostname; sessionId = $script:SessionId;
status = $Status; apiBase = $script:ApiBase; walDepth = $walDepth; ts = (Get-Date).ToUniversalTime().ToString('o')
} | ConvertTo-Json -Depth 5
Set-Content -LiteralPath $script:HealthPath -Value $health -Encoding UTF8
} catch {}
}
function Invoke-AwJsonPost {
param(
[Parameter(Mandatory = $true)][string]$Uri,
@@ -52,42 +104,30 @@ function Invoke-AwJsonPost {
)
$attempt = 1
$backoff = [Math]::Max(100, $InitialBackoffMs)
while ($attempt -le $MaxAttempts) {
$httpClient = $null
try {
$httpClient = New-Object System.Net.Http.HttpClient
$content = New-Object System.Net.Http.StringContent($Json, [System.Text.Encoding]::UTF8, "application/json")
$response = $httpClient.PostAsync($Uri, $content).Result
if ($response.IsSuccessStatusCode) {
return
}
$status = [int]$response.StatusCode
$reason = [string]$response.ReasonPhrase
$body = $response.Content.ReadAsStringAsync().Result
Write-FileCollectorLog ("POST failed: attempt={0}/{1} uri={2} status={3} reason={4} body={5}" -f $attempt, $MaxAttempts, $Uri, $status, $reason, $body)
if ($response.IsSuccessStatusCode) { return $true }
if ($attempt -ge $MaxAttempts) {
return
if (-not $script:WalFlushing) { Add-WalEntry -Uri $Uri -Json $Json }
return $false
}
Start-Sleep -Milliseconds $backoff
$backoff = [Math]::Min($backoff * 2, 10000)
$attempt++
continue
} catch {
Write-FileCollectorLog ("POST error: attempt={0}/{1} uri={2} error={3}" -f $attempt, $MaxAttempts, $Uri, $_.Exception.Message)
if ($attempt -ge $MaxAttempts) {
return
if (-not $script:WalFlushing) { Add-WalEntry -Uri $Uri -Json $Json }
return $false
}
Start-Sleep -Milliseconds $backoff
$backoff = [Math]::Min($backoff * 2, 10000)
$attempt++
continue
} finally {
if ($null -ne $httpClient) {
$httpClient.Dispose()
}
}
Start-Sleep -Milliseconds $backoff
$backoff = [Math]::Min($backoff * 2, 10000)
$attempt++
}
}
@@ -173,6 +213,9 @@ $scheme = if ($ServerScheme) { $ServerScheme } elseif ($config.server.scheme) {
$hostName = if ($ServerHost) { $ServerHost } elseif ($config.server.host) { $config.server.host } else { 'localhost' }
$port = if ($ServerPort) { $ServerPort } elseif ($config.server.port) { $config.server.port } else { 5600 }
$script:ApiBase = "{0}://{1}:{2}/api/0" -f $scheme, $hostName, $port
$stateRoot = if ($config.paths -and $config.paths.stateRoot) { [string]$config.paths.stateRoot } else { 'C:\ProgramData\AWatch-rus' }
$script:WalPath = Join-Path $stateRoot 'wal-file-operations.ndjson'
$script:HealthPath = Join-Path $stateRoot 'health-file-operations.json'
$bucketId = 'aw-file-operations_' + $script:Hostname
Ensure-Bucket -BucketId $bucketId -ClientName 'aw-file-operations' -BucketType 'aw.file.operation'
@@ -229,11 +272,14 @@ Write-FileCollectorLog "Collector started. Waiting for events..."
try {
while ($true) {
Flush-Wal
Write-CollectorHealth -Status 'running'
Start-Sleep -Seconds $PollSeconds
}
}
finally {
Write-FileCollectorLog "Stopping collector..."
Write-CollectorHealth -Status 'stopped'
foreach ($sub in @($subscriptions)) {
try {
if ($sub -and $sub.Id) {