feat(dlp): add enterprise phase scaffolds (policy role, content analysis, siem, case, compliance, cli)

2026-05-11 23:18:20 +03:00
parent 8be0de1c85
commit b213552d9d
19 changed files with 454 additions and 0 deletions
@@ -0,0 +1,10 @@
 ---
 - import_playbook: deploy_aw_server.yml
 - import_playbook: deploy_aw_windows.yml
 - name: Deploy DLP policy engine
  hosts: aw_server
  become: true
  gather_facts: false
  roles:
    - role: dlp-policy-engine
@@ -0,0 +1,41 @@
 ---
 - name: Ensure policy engine directory exists
  become: true
  ansible.builtin.file:
    path: /opt/activitywatch/dlp-policy-engine
    state: directory
    owner: activitywatch
    group: activitywatch
    mode: "0755"
 - name: Deploy policy engine files
  become: true
  ansible.builtin.copy:
    src: "{{ item.src }}"
    dest: "{{ item.dest }}"
    owner: activitywatch
    group: activitywatch
    mode: "{{ item.mode | default('0644') }}"
  loop:
    - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_service.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_service.py" }
    - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_schema.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_schema.py" }
    - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_storage.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_storage.py" }
    - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_distributor.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_distributor.py" }
    - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/dlp-policy-engine.service", dest: "/etc/systemd/system/dlp-policy-engine.service" }
 - name: Install python deps for policy engine
  become: true
  ansible.builtin.pip:
    name:
      - fastapi
      - uvicorn
      - pydantic
    executable: pip3
 - name: Enable and restart policy engine
  become: true
  ansible.builtin.systemd:
    daemon_reload: true
    name: dlp-policy-engine.service
    enabled: true
    state: restarted
@@ -0,0 +1,53 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import sqlite3
 from pathlib import Path
 from typing import Any
 from fastapi import FastAPI
 from pydantic import BaseModel
 DB = Path("/opt/activitywatch/dlp-case-management/cases.db")
 APP = FastAPI(title="AWatch DLP Case Management")
 class CaseCreate(BaseModel):
    incident_id: str
    title: str
    severity: str = "medium"
    assignee: str | None = None
 def _conn() -> sqlite3.Connection:
    DB.parent.mkdir(parents=True, exist_ok=True)
    c = sqlite3.connect(DB)
    c.execute(
        "CREATE TABLE IF NOT EXISTS cases (id INTEGER PRIMARY KEY, incident_id TEXT, title TEXT, severity TEXT, assignee TEXT, status TEXT DEFAULT 'open')"
    )
    return c
@APP.post("/api/0/dlp/cases")
 def create_case(payload: CaseCreate) -> dict[str, Any]:
    c = _conn()
    cur = c.cursor()
    cur.execute(
        "INSERT INTO cases (incident_id,title,severity,assignee,status) VALUES (?,?,?,?,?)",
        (payload.incident_id, payload.title, payload.severity, payload.assignee, "open"),
    )
    c.commit()
    case_id = cur.lastrowid
    c.close()
    return {"id": case_id}
@APP.get("/api/0/dlp/cases")
 def list_cases() -> list[dict[str, Any]]:
    c = _conn()
    rows = c.execute("SELECT id,incident_id,title,severity,assignee,status FROM cases ORDER BY id DESC").fetchall()
    c.close()
    return [
        {"id": r[0], "incident_id": r[1], "title": r[2], "severity": r[3], "assignee": r[4], "status": r[5]}
        for r in rows
    ]
@@ -0,0 +1,21 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 from datetime import datetime
 from pathlib import Path
 def render_html(period: str) -> str:
    return f"""<html><body><h1>Отчет 152-ФЗ</h1><p>Период: {period}</p><p>Сгенерирован: {datetime.now().isoformat()}</p></body></html>"""
 def main() -> None:
    period = datetime.now().strftime("%Y-%m")
    out = Path("/opt/activitywatch/dlp-compliance/reports")
    out.mkdir(parents=True, exist_ok=True)
    html = out / f"152-fz-{period}.html"
    html.write_text(render_html(period), encoding="utf-8")
 if __name__ == "__main__":
    main()
@@ -0,0 +1,37 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import re
 def validate_inn(value: str) -> bool:
    digits = re.sub(r"\D", "", value)
    if len(digits) == 10:
        coef = [2, 4, 10, 3, 5, 9, 4, 6, 8]
        chk = sum(int(digits[i]) * coef[i] for i in range(9)) % 11 % 10
        return chk == int(digits[9])
    if len(digits) == 12:
        c11 = [7, 2, 4, 10, 3, 5, 9, 4, 6, 8]
        c12 = [3, 7, 2, 4, 10, 3, 5, 9, 4, 6, 8]
        chk11 = sum(int(digits[i]) * c11[i] for i in range(10)) % 11 % 10
        chk12 = sum(int(digits[i]) * c12[i] for i in range(11)) % 11 % 10
        return chk11 == int(digits[10]) and chk12 == int(digits[11])
    return False
 def validate_snils(value: str) -> bool:
    digits = re.sub(r"\D", "", value)
    if len(digits) != 11:
        return False
    number = digits[:9]
    checksum = int(digits[9:])
    s = sum(int(number[i]) * (9 - i) for i in range(9))
    if s < 100:
        expected = s
    elif s in (100, 101):
        expected = 0
    else:
        expected = s % 101
        if expected == 100:
            expected = 0
    return checksum == expected
@@ -0,0 +1,17 @@
 {
  "inn": {
    "regex": "\\b\\d{10}\\b|\\b\\d{12}\\b",
    "checksum": "inn",
    "description": "ИНН"
  },
  "snils": {
    "regex": "\\b\\d{3}-\\d{3}-\\d{3}\\s?\\d{2}\\b",
    "checksum": "snils",
    "description": "СНИЛС"
  },
  "passport": {
    "regex": "\\b\\d{4}\\s?\\d{6}\\b",
    "checksum": "none",
    "description": "Паспорт РФ"
  }
 }
@@ -0,0 +1,38 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import json
 import pathlib
 import re
 from typing import Any
 from checksum_validator import validate_inn, validate_snils
 def _validate(kind: str, value: str) -> bool:
    if kind == "inn":
        return validate_inn(value)
    if kind == "snils":
        return validate_snils(value)
    return True
 def match_text(text: str, dictionary_path: str) -> list[dict[str, Any]]:
    rules = json.loads(pathlib.Path(dictionary_path).read_text(encoding="utf-8"))
    results: list[dict[str, Any]] = []
    for name, rule in rules.items():
        regex = re.compile(rule["regex"])
        checksum_kind = rule.get("checksum", "none")
        for m in regex.finditer(text):
            token = m.group(0)
            if _validate(checksum_kind, token):
                results.append(
                    {
                        "name": name,
                        "description": rule.get("description", name),
                        "value": token,
                        "start": m.start(),
                        "end": m.end(),
                    }
                )
    return results
@@ -0,0 +1,15 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 from pathlib import Path
 from PIL import Image
 import pytesseract
 def extract_text(image_path: str) -> str:
    path = Path(image_path)
    if not path.exists():
        return ""
    img = Image.open(path)
    return pytesseract.image_to_string(img, lang="rus+eng")
@@ -0,0 +1,6 @@
 {
  "rules": [
    { "id": "email", "regex": "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}", "severity": "low" },
    { "id": "phone-ru", "regex": "(?:\\+7|8)\\s*\\(?\\d{3}\\)?\\s*\\d{3}[- ]?\\d{2}[- ]?\\d{2}", "severity": "low" }
  ]
 }
@@ -0,0 +1,6 @@
 {
  "rules": [
    { "id": "card-pan", "regex": "\\b(?:\\d[ -]*?){13,19}\\b", "severity": "high" },
    { "id": "iban", "regex": "\\b[A-Z]{2}\\d{2}[A-Z0-9]{11,30}\\b", "severity": "medium" }
  ]
 }
@@ -0,0 +1,6 @@
 {
  "rules": [
    { "id": "aws-access-key", "regex": "AKIA[0-9A-Z]{16}", "severity": "high" },
    { "id": "generic-password", "regex": "(?i)(password|пароль)\\s*[:=]\\s*\\S{6,}", "severity": "medium" }
  ]
 }
@@ -0,0 +1,12 @@
 [Unit]
 Description=AWatch DLP CEF Exporter
 After=network-online.target
 [Service]
 Type=oneshot
 User=activitywatch
 Group=activitywatch
 ExecStart=/usr/bin/python3 /opt/activitywatch/dlp-integrations/cef_exporter.py
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,11 @@
 [Unit]
 Description=Run AWatch DLP CEF Exporter every 5 minutes
 [Timer]
 OnBootSec=2min
 OnUnitActiveSec=5min
 Unit=cef-exporter.service
 Persistent=true
 [Install]
 WantedBy=timers.target
@@ -0,0 +1,41 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import json
 import logging
 import os
 import socket
 from datetime import datetime, timezone
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 def build_cef(event: dict) -> str:
    sev_map = {"low": 3, "medium": 6, "high": 10}
    sev = sev_map.get(event.get("severity", "low"), 3)
    ts = datetime.now(timezone.utc).isoformat()
    msg = event.get("message", "")
    host = event.get("hostname", "unknown")
    return f"CEF:0|AWatch-rus|DLP|1.0|{event.get('id','dlp')}|{msg}|{sev}|rt={ts} shost={host}"
 def send_syslog(line: str, host: str, port: int) -> None:
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        sock.sendto(line.encode("utf-8", errors="ignore"), (host, port))
    finally:
        sock.close()
 def main() -> None:
    sample = os.environ.get("AW_DLP_CEF_SAMPLE", "")
    event = json.loads(sample) if sample else {"id": "startup", "message": "cef exporter heartbeat", "severity": "low"}
    line = build_cef(event)
    host = os.environ.get("AW_DLP_SYSLOG_HOST", "127.0.0.1")
    port = int(os.environ.get("AW_DLP_SYSLOG_PORT", "514"))
    send_syslog(line, host, port)
    logging.info("sent CEF event to %s:%d", host, port)
 if __name__ == "__main__":
    main()
@@ -0,0 +1,30 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import json
 import os
 import time
 from urllib import request
 def post(url: str, payload: dict, retries: int = 3) -> bool:
    body = json.dumps(payload).encode("utf-8")
    for i in range(retries):
        try:
            req = request.Request(url, data=body, headers={"Content-Type": "application/json"}, method="POST")
            with request.urlopen(req, timeout=10):
                return True
        except Exception:
            time.sleep(2 ** i)
    return False
 def main() -> None:
    hooks = [h.strip() for h in os.environ.get("AW_DLP_CRITICAL_WEBHOOKS", "").split(",") if h.strip()]
    payload = {"text": "AWatch DLP critical incident", "severity": "high"}
    for hook in hooks:
        post(hook, payload)
 if __name__ == "__main__":
    main()
@@ -0,0 +1,19 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 from fastapi import FastAPI
 from fastapi.responses import PlainTextResponse
 app = FastAPI(title="AWatch DLP Metrics")
@app.get("/metrics", response_class=PlainTextResponse)
 def metrics() -> str:
    # Minimal exporter baseline for Prometheus scraping.
    return "\n".join(
        [
            "# HELP aw_dlp_exporter_up Exporter availability.",
            "# TYPE aw_dlp_exporter_up gauge",
            "aw_dlp_exporter_up 1",
        ]
    )
@@ -0,0 +1,15 @@
 {
  "title": "AWatch DLP Overview",
  "schemaVersion": 39,
  "version": 1,
  "panels": [
    {
      "type": "stat",
      "title": "DLP Exporter Up",
      "gridPos": { "x": 0, "y": 0, "w": 8, "h": 4 },
      "targets": [
        { "expr": "aw_dlp_exporter_up" }
      ]
    }
  ]
 }
@@ -0,0 +1,32 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 import argparse
 import json
 from urllib import request
 def get_json(url: str):
    with request.urlopen(url, timeout=10) as r:
        return json.loads(r.read().decode("utf-8"))
 def main() -> None:
    p = argparse.ArgumentParser(description="AWatch DLP admin CLI")
    p.add_argument("--server", default="http://127.0.0.1:5601")
    sub = p.add_subparsers(dest="cmd", required=True)
    sub.add_parser("policies-list")
    sub.add_parser("health-check")
    args = p.parse_args()
    if args.cmd == "policies-list":
        data = get_json(f"{args.server}/api/0/dlp/policies")
        print(json.dumps(data, ensure_ascii=False, indent=2))
    elif args.cmd == "health-check":
        data = get_json(f"{args.server}/health")
        print(json.dumps(data, ensure_ascii=False, indent=2))
 if __name__ == "__main__":
    main()
@@ -0,0 +1,44 @@
 param(
  [Parameter(Mandatory = $true)][string]$ServerHost,
  [int]$ServerPort = 5600,
  [string]$InstallRoot = "C:\ProgramData\AWatch-rus"
 )
 $ErrorActionPreference = "Stop"
 New-Item -ItemType Directory -Path $InstallRoot -Force | Out-Null
 New-Item -ItemType Directory -Path (Join-Path $InstallRoot "logs") -Force | Out-Null
 $configPath = Join-Path $InstallRoot "deployment-config.json"
 $policyPath = Join-Path $InstallRoot "dlp-policy.json"
 $cfg = @{
  server = @{
    host = $ServerHost
    port = $ServerPort
    apiBase = "http://$ServerHost`:$ServerPort/api/0"
  }
  paths = @{
    logsRoot = (Join-Path $InstallRoot "logs")
  }
  dlp = @{
    policyMode = "server"
  }
  localAgentLogsEnabled = $true
 }
 $cfg | ConvertTo-Json -Depth 8 | Set-Content -Path $configPath -Encoding UTF8
 if (-not (Test-Path $policyPath)) {
  @{
    version = 1
    defaults = @{
      enabled = $true
      action = "log"
      severity = "low"
      cooldownSeconds = 300
    }
    rules = @()
  } | ConvertTo-Json -Depth 8 | Set-Content -Path $policyPath -Encoding UTF8
 }
 Write-Host "DLP client config written: $configPath"