From e0561bf8658f5f8a14bcf18175b2185d3caf53db Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Mon, 11 May 2026 20:46:39 +0300 Subject: [PATCH] fix(windows): disable outlook popup and enforce smtp-only email monitoring --- ansible/deploy_aw_server.yml | 57 +++ ansible/deploy_aw_windows.yml | 18 +- ansible/group_vars/all.example.yml | 5 + ansible/group_vars/all.yml | 4 + ansible/group_vars/aw_server.yml | 9 + ansible/group_vars/aw_windows.yml | 9 + aw-server/aw-server.env.example | 24 + aw-server/aw-worktime-api.service | 9 +- aw-server/aw-worktime-ui-bridge.service | 13 +- .../dlp-policy-engine.service | 22 + .../dlp-policy-engine/policy_distributor.py | 26 + aw-server/dlp-policy-engine/policy_schema.py | 67 +++ aw-server/dlp-policy-engine/policy_service.py | 117 +++++ aw-server/dlp-policy-engine/policy_storage.py | 272 +++++++++++ aw-server/dlp-policy-engine/requirements.txt | 3 + aw-server/ensure-reliability.sh | 146 ++++++ aw-server/health-check.sh | 55 +++ aw-server/logrotate.conf | 28 ++ docs/dlp-policy-engine.md | 89 ++++ docs/dlp-production-execution-roadmap.md | 205 ++++++++ docs/dlp-production-plan-windows-10-19.md | 458 ++++++++++++++++++ windows/ActivityWatch.Windows.Common.psm1 | 62 ++- windows/deploy-domain-users.ps1 | 25 +- windows/deploy-ensemble.ps1 | 22 +- windows/dlp-endpoint-signals-collector.ps1 | 165 ++++++- windows/dlp-policy-client.ps1 | 85 ++++ windows/email-outbound-collector.ps1 | 46 +- windows/validate-deployment.ps1 | 6 +- 28 files changed, 2025 insertions(+), 22 deletions(-) create mode 100644 aw-server/dlp-policy-engine/dlp-policy-engine.service create mode 100644 aw-server/dlp-policy-engine/policy_distributor.py create mode 100644 aw-server/dlp-policy-engine/policy_schema.py create mode 100644 aw-server/dlp-policy-engine/policy_service.py create mode 100644 aw-server/dlp-policy-engine/policy_storage.py create mode 100644 aw-server/dlp-policy-engine/requirements.txt create mode 100644 aw-server/ensure-reliability.sh create mode 100644 aw-server/health-check.sh create mode 100644 aw-server/logrotate.conf create mode 100644 docs/dlp-policy-engine.md create mode 100644 docs/dlp-production-execution-roadmap.md create mode 100644 docs/dlp-production-plan-windows-10-19.md create mode 100644 windows/dlp-policy-client.ps1 diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index 783b522..c7e7700 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -20,6 +20,7 @@ ansible.builtin.apt: name: - curl + - python3-venv - rsync - unzip state: present @@ -298,9 +299,58 @@ AW_WORKTIME_REPORT_BASE={{ aw_worktime_report_base }} AW_WORKTIME_TZ={{ aw_worktime_timezone }} AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output + AW_DLP_POLICY_ENGINE_BIND_HOST={{ aw_dlp_policy_engine_bind_host }} + AW_DLP_POLICY_ENGINE_PORT={{ aw_dlp_policy_engine_port }} + AW_DLP_POLICY_ENGINE_DB_PATH={{ aw_dlp_policy_engine_db_path }} XDG_DATA_HOME={{ aw_server_data_dir }}/.local/share XDG_CONFIG_HOME={{ aw_server_data_dir }}/.config + - name: Создать каталог DLP policy engine + ansible.builtin.file: + path: /opt/activitywatch/dlp-policy-engine + state: directory + owner: "{{ aw_server_user }}" + group: "{{ aw_server_group }}" + mode: "0755" + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Скопировать файлы DLP policy engine + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/{{ item }}" + dest: "/opt/activitywatch/dlp-policy-engine/{{ item }}" + owner: "{{ aw_server_user }}" + group: "{{ aw_server_group }}" + mode: "0644" + loop: + - policy_service.py + - policy_schema.py + - policy_storage.py + - policy_distributor.py + - requirements.txt + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Создать virtualenv DLP policy engine + ansible.builtin.command: + cmd: python3 -m venv /opt/activitywatch/dlp-policy-engine/.venv + args: + creates: /opt/activitywatch/dlp-policy-engine/.venv/bin/python + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Установить зависимости DLP policy engine + ansible.builtin.pip: + requirements: /opt/activitywatch/dlp-policy-engine/requirements.txt + virtualenv: /opt/activitywatch/dlp-policy-engine/.venv + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Установить systemd unit DLP policy engine + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/dlp-policy-engine.service" + dest: /etc/systemd/system/aw-dlp-policy-engine.service + owner: root + group: root + mode: "0644" + when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Установить скрипт AW worktime API ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" @@ -345,6 +395,13 @@ ansible.builtin.systemd: daemon_reload: true + - name: Включить и перезапустить DLP policy engine + ansible.builtin.systemd: + name: aw-dlp-policy-engine.service + enabled: true + state: restarted + when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Включить и перезапустить AW worktime API ansible.builtin.systemd: name: aw-worktime-api.service diff --git a/ansible/deploy_aw_windows.yml b/ansible/deploy_aw_windows.yml index 5f8f348..931a663 100644 --- a/ansible/deploy_aw_windows.yml +++ b/ansible/deploy_aw_windows.yml @@ -25,6 +25,12 @@ aw_windows_users_effective: "{{ (aw_windows_users + aw_windows_extra_users) | unique }}" aw_windows_install_root: "C:\\Program Files\\AWatch-rus\\bin" aw_windows_state_root: "C:\\ProgramData\\AWatch-rus" + aw_windows_policy_mode: "local" + aw_windows_policy_refresh_seconds: 300 + aw_windows_policy_engine_enabled: false + aw_windows_policy_engine_host: "{{ aw_windows_server_host }}" + aw_windows_policy_engine_port: 5601 + aw_windows_policy_engine_scheme: "http" aw_windows_afk_enabled: true aw_windows_window_enabled: true aw_windows_file_ops_enabled: true @@ -83,6 +89,7 @@ - ActivityWatch.Windows.Common.psm1 - browser-domains-native-collector.ps1 - dlp-endpoint-signals-collector.ps1 + - dlp-policy-client.ps1 - email-outbound-collector.ps1 - file-operations-collector.ps1 - worktime-session-collector.ps1 @@ -156,6 +163,12 @@ IncidentScreenshotEnabled = {{ '$true' if (aw_windows_incident_screenshot_enabled | bool) else '$false' }} IncidentArtifactsRoot = "{{ aw_windows_incident_artifacts_root }}" LogonMarkerEnabled = {{ '$true' if (aw_windows_logon_marker_enabled | bool) else '$false' }} + PolicyMode = "{{ aw_windows_policy_mode }}" + PolicyEngineEnabled = {{ '$true' if (aw_windows_policy_engine_enabled | bool) else '$false' }} + PolicyEngineHost = "{{ aw_windows_policy_engine_host }}" + PolicyEnginePort = {{ aw_windows_policy_engine_port }} + PolicyEngineScheme = "{{ aw_windows_policy_engine_scheme }}" + PolicyRefreshSeconds = {{ aw_windows_policy_refresh_seconds }} CustomRulesPath = "{{ aw_windows_rules_path }}" CustomPolicyPath = "{{ aw_windows_policy_path }}" } @@ -171,6 +184,9 @@ {% if aw_windows_skip_hardening | bool %} $params.SkipHardening = $true {% endif %} + {% if aw_windows_integration_test_enabled | bool %} + $params.IntegrationTestEnabled = $true + {% endif %} & "{{ aw_windows_deploy_root }}\windows\deploy-ensemble.ps1" @params - name: Удалить лишние ActivityWatch Launch tasks вне текущего deployment-config @@ -346,7 +362,7 @@ ansible.builtin.shell: | python3 - <<'PY' import json, sys - with open('{{ aw_windows_validation_local_dir }}/{{ inventory_hostname }}-aw_validate_ansible.json', 'r') as f: + with open('{{ aw_windows_validation_local_dir }}/{{ inventory_hostname }}-aw_validate_ansible.json', 'r', encoding='utf-8-sig') as f: data = json.load(f) if not data.get('overallOk', False): print(f"Validation failed for {{ inventory_hostname }}: {data.get('summary', 'Unknown error')}") diff --git a/ansible/group_vars/all.example.yml b/ansible/group_vars/all.example.yml index 171b218..ffb0c05 100644 --- a/ansible/group_vars/all.example.yml +++ b/ansible/group_vars/all.example.yml @@ -35,3 +35,8 @@ aw_worktime_to: "17:00" aw_worktime_start_of_day: "{{ aw_worktime_from }}" aw_server_always_active_pattern: "aw-watcher-window" aw_server_landingpage: "/activity/SHARKON2025/view/" + +aw_dlp_policy_engine_enabled: true +aw_dlp_policy_engine_bind_host: "0.0.0.0" +aw_dlp_policy_engine_port: 5601 +aw_dlp_policy_engine_db_path: "{{ aw_server_data_dir }}/dlp-policy-engine.sqlite" diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 266c5d7..d1310e5 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -27,6 +27,10 @@ aw_dlp_ioc_workdir: "/opt/activitywatch/dlp-ioc" aw_dlp_ioc_rules_zip_url: "https://github.com/Yamato-Security/hayabusa-rules/archive/refs/heads/main.zip" aw_dlp_ioc_refresh_on_boot_sec: "5min" aw_dlp_ioc_refresh_interval: "6h" +aw_dlp_policy_engine_enabled: true +aw_dlp_policy_engine_bind_host: "0.0.0.0" +aw_dlp_policy_engine_port: 5601 +aw_dlp_policy_engine_db_path: "{{ aw_server_data_dir }}/dlp-policy-engine.sqlite" aw_worktime_from: "08:00" aw_worktime_to: "17:00" diff --git a/ansible/group_vars/aw_server.yml b/ansible/group_vars/aw_server.yml index 73b9934..d4013db 100644 --- a/ansible/group_vars/aw_server.yml +++ b/ansible/group_vars/aw_server.yml @@ -7,3 +7,12 @@ ansible_become: true ansible_become_method: sudo # If sudo password differs, set AW_SUDO_PASSWORD. Otherwise it will reuse AW_SSH_PASSWORD. ansible_become_password: "{{ lookup('env', 'AW_SUDO_PASSWORD') | default(lookup('env', 'AW_SSH_PASSWORD'), true) }}" + +# Hayabusa IOC refresh settings (backward compatible - defaults to disabled) +aw_hayabusa_ioc_refresh_enabled: false +aw_hayabusa_rules_root: "/mnt/usb_hdd1/Projects/hayabusa/rules" +aw_hayabusa_ioc_output_dir: "{{ aw_server_data_dir }}/dlp-ioc" +aw_dlp_policy_engine_enabled: true +aw_dlp_policy_engine_bind_host: "0.0.0.0" +aw_dlp_policy_engine_port: 5601 +aw_dlp_policy_engine_db_path: "{{ aw_server_data_dir }}/dlp-policy-engine.sqlite" diff --git a/ansible/group_vars/aw_windows.yml b/ansible/group_vars/aw_windows.yml index 41cbbe8..bc95cf9 100644 --- a/ansible/group_vars/aw_windows.yml +++ b/ansible/group_vars/aw_windows.yml @@ -25,6 +25,12 @@ aw_windows_extra_users: [] aw_windows_install_root: "C:\\Program Files\\AWatch-rus\\bin" aw_windows_state_root: "C:\\ProgramData\\AWatch-rus" aw_windows_hostname_override: "" +aw_windows_policy_mode: "local" +aw_windows_policy_refresh_seconds: 300 +aw_windows_policy_engine_enabled: false +aw_windows_policy_engine_host: "{{ aw_windows_server_host }}" +aw_windows_policy_engine_port: 5601 +aw_windows_policy_engine_scheme: "http" aw_windows_afk_enabled: true aw_windows_window_enabled: true @@ -53,3 +59,6 @@ aw_windows_migration_report_remote_path: "{{ aw_windows_state_root }}\\aw_migrat aw_windows_api_smoke_check_enabled: true aw_windows_api_smoke_check_bucket: "" aw_windows_api_smoke_check_limit: 10 + +# Integration test settings (backward compatible - defaults to disabled) +aw_windows_integration_test_enabled: false diff --git a/aw-server/aw-server.env.example b/aw-server/aw-server.env.example index 03693fb..5c02cd4 100755 --- a/aw-server/aw-server.env.example +++ b/aw-server/aw-server.env.example @@ -1,13 +1,37 @@ # Copy to /etc/activitywatch/aw-server.env and fill with real values. +# Core AW Server Configuration AW_SERVER_VERSION=0.13.2 AW_SERVER_DOWNLOAD_URL=https://github.com/ActivityWatch/aw-server-rust/releases/download/v0.13.2/aw-server-rust-linux-x86_64.zip AW_SERVER_BIND_HOST=0.0.0.0 AW_SERVER_PORT=5600 AW_SERVER_WEBUI_DIR=/opt/activitywatch/webui-ru AW_SERVER_DATA_DIR=/var/lib/activitywatch +AW_SERVER_DB_PATH=/var/lib/activitywatch/pebble.db AW_SERVER_LOG_DIR=/var/log/activitywatch AW_SERVER_USER=activitywatch AW_SERVER_GROUP=activitywatch + +# Worktime API Configuration AW_WORKTIME_REPORT_BASE=http://10.10.10.13:5610 AW_WORKTIME_TZ=Europe/Moscow + +# DLP IOC Configuration +AW_DLP_IOC_DIR=/opt/activitywatch/dlp-ioc/output + +# DLP Policy Engine Configuration +AW_DLP_POLICY_ENGINE_BIND_HOST=0.0.0.0 +AW_DLP_POLICY_ENGINE_PORT=5601 +AW_DLP_POLICY_ENGINE_DB_PATH=/var/lib/activitywatch/dlp-policy-engine.sqlite + +# Logging Configuration +AW_LOG_LEVEL=info +AW_LOG_TO_JOURNAL=true +AW_LOG_TO_FILE=true + +# Health Check Configuration +AW_HEALTH_CHECK_ENABLED=true +AW_HEALTH_CHECK_INTERVAL=60 + +# Integration Test Configuration +AW_INTEGRATION_TEST_ENABLED=false diff --git a/aw-server/aw-worktime-api.service b/aw-server/aw-worktime-api.service index 4225fb5..dedc128 100644 --- a/aw-server/aw-worktime-api.service +++ b/aw-server/aw-worktime-api.service @@ -7,10 +7,15 @@ Wants=activitywatch-server.service Type=simple EnvironmentFile=/etc/activitywatch/aw-server.env ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-api.py -Restart=always -RestartSec=2 +Restart=on-failure +RestartSec=5 +StartLimitBurst=3 +StartLimitIntervalSec=60 User=activitywatch Group=activitywatch +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-worktime-api [Install] WantedBy=multi-user.target diff --git a/aw-server/aw-worktime-ui-bridge.service b/aw-server/aw-worktime-ui-bridge.service index 0a48ba5..475e4c7 100644 --- a/aw-server/aw-worktime-ui-bridge.service +++ b/aw-server/aw-worktime-ui-bridge.service @@ -4,12 +4,19 @@ After=network-online.target activitywatch-server.service Wants=network-online.target [Service] -Type=oneshot +Type=simple Environment=AW_SERVER_URL=http://127.0.0.1:5600 Environment=AW_WORKTIME_HOST=SHARKON2025 ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-ui-bridge.py -User=root -Group=root +Restart=on-failure +RestartSec=10 +StartLimitBurst=3 +StartLimitIntervalSec=120 +User=activitywatch +Group=activitywatch +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-worktime-ui-bridge [Install] WantedBy=multi-user.target diff --git a/aw-server/dlp-policy-engine/dlp-policy-engine.service b/aw-server/dlp-policy-engine/dlp-policy-engine.service new file mode 100644 index 0000000..a183f87 --- /dev/null +++ b/aw-server/dlp-policy-engine/dlp-policy-engine.service @@ -0,0 +1,22 @@ +[Unit] +Description=AW DLP Policy Engine +After=network.target activitywatch-server.service +Wants=activitywatch-server.service + +[Service] +Type=simple +EnvironmentFile=/etc/activitywatch/aw-server.env +WorkingDirectory=/opt/activitywatch/dlp-policy-engine +ExecStart=/opt/activitywatch/dlp-policy-engine/.venv/bin/uvicorn policy_service:app --host ${AW_DLP_POLICY_ENGINE_BIND_HOST} --port ${AW_DLP_POLICY_ENGINE_PORT} +Restart=on-failure +RestartSec=5 +StartLimitBurst=3 +StartLimitIntervalSec=60 +User=activitywatch +Group=activitywatch +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-dlp-policy-engine + +[Install] +WantedBy=multi-user.target diff --git a/aw-server/dlp-policy-engine/policy_distributor.py b/aw-server/dlp-policy-engine/policy_distributor.py new file mode 100644 index 0000000..915bb13 --- /dev/null +++ b/aw-server/dlp-policy-engine/policy_distributor.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Any + + +def build_policy_bundle(record: dict[str, Any] | None) -> dict[str, Any]: + if not record: + return { + "active": False, + "policyId": None, + "name": None, + "version": None, + "checksum": None, + "updatedAtUtc": None, + "policy": None, + } + + return { + "active": True, + "policyId": record["id"], + "name": record["name"], + "version": record["current_version"], + "checksum": record["checksum"], + "updatedAtUtc": record["updated_at"], + "policy": record["policy"], + } diff --git a/aw-server/dlp-policy-engine/policy_schema.py b/aw-server/dlp-policy-engine/policy_schema.py new file mode 100644 index 0000000..56c80d4 --- /dev/null +++ b/aw-server/dlp-policy-engine/policy_schema.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field, ConfigDict + + +class PolicyDocument(BaseModel): + model_config = ConfigDict(extra="allow") + + version: int = 1 + defaults: dict[str, Any] = Field( + default_factory=lambda: { + "enabled": True, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium", + } + ) + endpoint: dict[str, list[dict[str, Any]]] = Field( + default_factory=lambda: { + "clipboard": [], + "usb": [], + "print": [], + } + ) + + +class PolicyCreateRequest(BaseModel): + name: str = Field(min_length=1, max_length=128) + description: str | None = Field(default=None, max_length=2048) + policy: PolicyDocument + activate: bool = False + actor: str | None = Field(default="api") + + +class PolicyUpdateRequest(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=128) + description: str | None = Field(default=None, max_length=2048) + policy: PolicyDocument | None = None + activate: bool = False + actor: str | None = Field(default="api") + + +class PolicyActivateRequest(BaseModel): + actor: str | None = Field(default="api") + + +class PolicyRecord(BaseModel): + id: int + name: str + description: str | None + is_active: bool + current_version: int + checksum: str + created_at: datetime + updated_at: datetime + + +class PolicyVersionRecord(BaseModel): + policy_id: int + version: int + checksum: str + created_at: datetime + created_by: str | None + diff --git a/aw-server/dlp-policy-engine/policy_service.py b/aw-server/dlp-policy-engine/policy_service.py new file mode 100644 index 0000000..33840fc --- /dev/null +++ b/aw-server/dlp-policy-engine/policy_service.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import os +from pathlib import Path + +from fastapi import FastAPI, HTTPException + +from policy_distributor import build_policy_bundle +from policy_schema import PolicyActivateRequest, PolicyCreateRequest, PolicyUpdateRequest +from policy_storage import PolicyStorage + + +def _env(name: str, default: str) -> str: + value = os.environ.get(name) + return value if value not in (None, "") else default + + +APP_NAME = "aw-dlp-policy-engine" +APP_VERSION = "0.1.0" +DB_PATH = _env("AW_DLP_POLICY_ENGINE_DB_PATH", "/var/lib/activitywatch/dlp-policy-engine.sqlite") +storage = PolicyStorage(DB_PATH) + +app = FastAPI(title=APP_NAME, version=APP_VERSION) + + +@app.get("/healthz") +def healthz() -> dict[str, str]: + return { + "status": "ok", + "service": APP_NAME, + "db_path": DB_PATH, + "db_exists": str(Path(DB_PATH).exists()).lower(), + } + + +@app.get("/api/0/dlp/policies") +def list_policies() -> dict[str, object]: + return {"items": storage.list_policies()} + + +@app.post("/api/0/dlp/policies", status_code=201) +def create_policy(payload: PolicyCreateRequest) -> dict[str, object]: + try: + item = storage.create_policy( + name=payload.name, + description=payload.description, + policy=payload.policy.model_dump(mode="json"), + activate=payload.activate, + actor=payload.actor, + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"item": item} + + +@app.get("/api/0/dlp/policies/active") +def get_active_policy() -> dict[str, object]: + item = storage.get_active_policy() + if not item: + raise HTTPException(status_code=404, detail="no active policy configured") + return build_policy_bundle(item) + + +@app.post("/api/0/dlp/policies/rollback") +def rollback_active_policy(payload: PolicyActivateRequest) -> dict[str, object]: + item = storage.rollback_active_policy(actor=payload.actor) + if not item: + raise HTTPException(status_code=404, detail="no active policy configured") + return {"item": item} + + +@app.get("/api/0/dlp/policies/{policy_id}") +def get_policy(policy_id: int) -> dict[str, object]: + item = storage.get_policy(policy_id) + if not item: + raise HTTPException(status_code=404, detail="policy not found") + return {"item": item} + + +@app.put("/api/0/dlp/policies/{policy_id}") +def update_policy(policy_id: int, payload: PolicyUpdateRequest) -> dict[str, object]: + try: + item = storage.update_policy( + policy_id=policy_id, + name=payload.name, + description=payload.description, + policy=payload.policy.model_dump(mode="json") if payload.policy is not None else None, + activate=payload.activate, + actor=payload.actor, + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + if not item: + raise HTTPException(status_code=404, detail="policy not found") + return {"item": item} + + +@app.post("/api/0/dlp/policies/{policy_id}/activate") +def activate_policy(policy_id: int, payload: PolicyActivateRequest) -> dict[str, object]: + item = storage.activate_policy(policy_id=policy_id, actor=payload.actor) + if not item: + raise HTTPException(status_code=404, detail="policy not found") + return {"item": item} + + +@app.delete("/api/0/dlp/policies/{policy_id}") +def delete_policy(policy_id: int) -> dict[str, bool]: + try: + deleted = storage.delete_policy(policy_id) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + if not deleted: + raise HTTPException(status_code=404, detail="policy not found") + return {"deleted": True} diff --git a/aw-server/dlp-policy-engine/policy_storage.py b/aw-server/dlp-policy-engine/policy_storage.py new file mode 100644 index 0000000..6acb6de --- /dev/null +++ b/aw-server/dlp-policy-engine/policy_storage.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterator + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def canonical_policy_json(policy: dict[str, Any]) -> str: + return json.dumps(policy, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def checksum_policy(policy: dict[str, Any]) -> str: + return hashlib.sha256(canonical_policy_json(policy).encode("utf-8")).hexdigest() + + +class PolicyStorage: + def __init__(self, db_path: str) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init_schema() + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def _init_schema(self) -> None: + with self.connect() as conn: + conn.executescript( + """ + PRAGMA journal_mode=WAL; + + CREATE TABLE IF NOT EXISTS policies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT, + is_active INTEGER NOT NULL DEFAULT 0, + current_version INTEGER NOT NULL DEFAULT 1, + checksum TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS policy_versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + policy_id INTEGER NOT NULL, + version INTEGER NOT NULL, + policy_json TEXT NOT NULL, + checksum TEXT NOT NULL, + created_at TEXT NOT NULL, + created_by TEXT, + rollback_of_version INTEGER, + FOREIGN KEY(policy_id) REFERENCES policies(id), + UNIQUE(policy_id, version) + ); + + CREATE INDEX IF NOT EXISTS idx_policies_active ON policies(is_active); + CREATE INDEX IF NOT EXISTS idx_policy_versions_policy ON policy_versions(policy_id, version DESC); + """ + ) + + def list_policies(self) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT id, name, description, is_active, current_version, checksum, created_at, updated_at + FROM policies + ORDER BY is_active DESC, updated_at DESC, id DESC + """ + ).fetchall() + return [dict(row) for row in rows] + + def get_policy(self, policy_id: int) -> dict[str, Any] | None: + with self.connect() as conn: + policy_row = conn.execute( + """ + SELECT id, name, description, is_active, current_version, checksum, created_at, updated_at + FROM policies + WHERE id = ? + """, + (policy_id,), + ).fetchone() + if not policy_row: + return None + + version_row = conn.execute( + """ + SELECT version, policy_json, checksum, created_at, created_by + FROM policy_versions + WHERE policy_id = ? AND version = ? + """, + (policy_id, policy_row["current_version"]), + ).fetchone() + if not version_row: + return None + + result = dict(policy_row) + result["policy"] = json.loads(version_row["policy_json"]) + result["version_created_at"] = version_row["created_at"] + result["version_created_by"] = version_row["created_by"] + return result + + def get_active_policy(self) -> dict[str, Any] | None: + with self.connect() as conn: + row = conn.execute("SELECT id FROM policies WHERE is_active = 1 ORDER BY updated_at DESC LIMIT 1").fetchone() + if not row: + return None + return self.get_policy(int(row["id"])) + + def create_policy(self, name: str, description: str | None, policy: dict[str, Any], activate: bool, actor: str | None) -> dict[str, Any]: + checksum = checksum_policy(policy) + now = utc_now() + policy_json = canonical_policy_json(policy) + with self.connect() as conn: + if activate: + conn.execute("UPDATE policies SET is_active = 0") + cursor = conn.execute( + """ + INSERT INTO policies(name, description, is_active, current_version, checksum, created_at, updated_at) + VALUES(?, ?, ?, 1, ?, ?, ?) + """, + (name, description, 1 if activate else 0, checksum, now, now), + ) + policy_id = int(cursor.lastrowid) + conn.execute( + """ + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, 1, ?, ?, ?, ?, NULL) + """, + (policy_id, policy_json, checksum, now, actor), + ) + return self.get_policy(policy_id) # type: ignore[return-value] + + def update_policy( + self, + policy_id: int, + name: str | None, + description: str | None, + policy: dict[str, Any] | None, + activate: bool, + actor: str | None, + ) -> dict[str, Any] | None: + current = self.get_policy(policy_id) + if not current: + return None + + with self.connect() as conn: + new_name = name if name is not None else current["name"] + new_description = description if description is not None else current["description"] + new_version = int(current["current_version"]) + new_checksum = current["checksum"] + + if policy is not None: + new_version += 1 + new_checksum = checksum_policy(policy) + policy_json = canonical_policy_json(policy) + conn.execute( + """ + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, ?, ?, ?, ?, ?, NULL) + """, + (policy_id, new_version, policy_json, new_checksum, utc_now(), actor), + ) + + if activate: + conn.execute("UPDATE policies SET is_active = 0") + + conn.execute( + """ + UPDATE policies + SET name = ?, description = ?, is_active = ?, current_version = ?, checksum = ?, updated_at = ? + WHERE id = ? + """, + ( + new_name, + new_description, + 1 if activate else current["is_active"], + new_version, + new_checksum, + utc_now(), + policy_id, + ), + ) + return self.get_policy(policy_id) + + def activate_policy(self, policy_id: int, actor: str | None) -> dict[str, Any] | None: + current = self.get_policy(policy_id) + if not current: + return None + + with self.connect() as conn: + conn.execute("UPDATE policies SET is_active = 0") + conn.execute( + "UPDATE policies SET is_active = 1, updated_at = ? WHERE id = ?", + (utc_now(), policy_id), + ) + return self.get_policy(policy_id) + + def rollback_active_policy(self, actor: str | None) -> dict[str, Any] | None: + active = self.get_active_policy() + if not active: + return None + + with self.connect() as conn: + rows = conn.execute( + """ + SELECT version, policy_json + FROM policy_versions + WHERE policy_id = ? + ORDER BY version DESC + LIMIT 2 + """, + (active["id"],), + ).fetchall() + if len(rows) < 2: + return active + + previous_version = int(rows[1]["version"]) + previous_policy = json.loads(rows[1]["policy_json"]) + rollback_version = int(active["current_version"]) + 1 + rollback_checksum = checksum_policy(previous_policy) + now = utc_now() + + conn.execute( + """ + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, ?, ?, ?, ?, ?, ?) + """, + ( + active["id"], + rollback_version, + canonical_policy_json(previous_policy), + rollback_checksum, + now, + actor, + previous_version, + ), + ) + conn.execute( + """ + UPDATE policies + SET current_version = ?, checksum = ?, updated_at = ? + WHERE id = ? + """, + (rollback_version, rollback_checksum, now, active["id"]), + ) + return self.get_policy(int(active["id"])) + + def delete_policy(self, policy_id: int) -> bool: + current = self.get_policy(policy_id) + if not current: + return False + if current["is_active"]: + raise ValueError("cannot delete active policy") + + with self.connect() as conn: + conn.execute("DELETE FROM policy_versions WHERE policy_id = ?", (policy_id,)) + conn.execute("DELETE FROM policies WHERE id = ?", (policy_id,)) + return True diff --git a/aw-server/dlp-policy-engine/requirements.txt b/aw-server/dlp-policy-engine/requirements.txt new file mode 100644 index 0000000..dcda2cd --- /dev/null +++ b/aw-server/dlp-policy-engine/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +pydantic==2.11.4 diff --git a/aw-server/ensure-reliability.sh b/aw-server/ensure-reliability.sh new file mode 100644 index 0000000..093fcd1 --- /dev/null +++ b/aw-server/ensure-reliability.sh @@ -0,0 +1,146 @@ +#!/bin/bash +set -euo pipefail + +# Service reliability script for AW services +# Fixes common issues and ensures proper configuration + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="/etc/activitywatch/aw-server.env" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" +} + +check_env_file() { + if [[ ! -f "$ENV_FILE" ]]; then + log "ERROR: Environment file not found: $ENV_FILE" + return 1 + fi + log "✓ Environment file exists" +} + +fix_permissions() { + log "Fixing permissions..." + + # Ensure proper ownership + chown -R activitywatch:activitywatch /var/lib/activitywatch + chown -R activitywatch:activitywatch /var/log/activitywatch + chown -R activitywatch:activitywatch /opt/activitywatch + + # Ensure proper permissions + chmod 755 /var/lib/activitywatch + chmod 755 /var/log/activitywatch + chmod 755 /opt/activitywatch + + log "✓ Permissions fixed" +} + +restart_services() { + log "Restarting services..." + + systemctl daemon-reload + + # Stop all services + systemctl stop aw-worktime-api aw-worktime-ui-bridge activitywatch-server || true + + # Wait for stop + sleep 2 + + # Start in dependency order + systemctl start activitywatch-server + sleep 3 + systemctl start aw-worktime-api + sleep 2 + systemctl start aw-worktime-ui-bridge + + log "✓ Services restarted" +} + +enable_services() { + log "Enabling services..." + + systemctl enable activitywatch-server + systemctl enable aw-worktime-api + systemctl enable aw-worktime-ui-bridge + + log "✓ Services enabled" +} + +setup_logrotate() { + local logrotate_file="/etc/logrotate.d/activitywatch" + + if [[ ! -f "$logrotate_file" ]]; then + log "Setting up log rotation..." + cp "$SCRIPT_DIR/logrotate.conf" "$logrotate_file" + log "✓ Log rotation configured" + else + log "✓ Log rotation already configured" + fi +} + +setup_health_check() { + local health_script="/usr/local/bin/aw-health-check" + local health_timer="/etc/systemd/system/aw-health-check.timer" + local health_service="/etc/systemd/system/aw-health-check.service" + + if [[ ! -f "$health_script" ]]; then + log "Setting up health check..." + cp "$SCRIPT_DIR/health-check.sh" "$health_script" + chmod +x "$health_script" + + # Create systemd timer for health checks + cat > "$health_timer" << 'EOF' +[Unit] +Description=AW Health Check Timer +Requires=aw-health-check.service + +[Timer] +OnCalendar=*:0/5:00 +Persistent=true + +[Install] +WantedBy=timers.target +EOF + + cat > "$health_service" << 'EOF' +[Unit] +Description=AW Health Check +After=network.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/aw-health-check +User=root +Group=root +EOF + + systemctl daemon-reload + systemctl enable aw-health-check.timer + systemctl start aw-health-check.timer + + log "✓ Health check configured" + else + log "✓ Health check already configured" + fi +} + +main() { + log "=== AW Service Reliability Fix ===" + + check_env_file || exit 1 + fix_permissions + setup_logrotate + setup_health_check + restart_services + enable_services + + log + log "=== Reliability Fix Complete ===" + log "Check status with: systemctl status activitywatch-server aw-worktime-api aw-worktime-ui-bridge" + log "Check health with: /usr/local/bin/aw-health-check" + log "View logs with: journalctl -u activitywatch-server -u aw-worktime-api -u aw-worktime-ui-bridge -f" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi \ No newline at end of file diff --git a/aw-server/health-check.sh b/aw-server/health-check.sh new file mode 100644 index 0000000..c7a9736 --- /dev/null +++ b/aw-server/health-check.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -euo pipefail + +# Health check script for AW services +# Returns 0 if all services are healthy, 1 otherwise + +SERVICES=("activitywatch-server" "aw-worktime-api" "aw-worktime-ui-bridge") +UNHEALTHY_SERVICES=() + +check_service() { + local service=$1 + if systemctl is-active --quiet "$service"; then + echo "✓ $service is running" + else + echo "✗ $service is not running" + UNHEALTHY_SERVICES+=("$service") + fi +} + +check_api_endpoint() { + local url=$1 + local service_name=$2 + + if curl -s --max-time 5 "$url" >/dev/null 2>&1; then + echo "✓ $service_name API endpoint is responding" + else + echo "✗ $service_name API endpoint is not responding" + UNHEALTHY_SERVICES+=("$service_name-api") + fi +} + +echo "=== AW Services Health Check ===" +echo "Timestamp: $(date)" +echo + +# Check systemd services +for service in "${SERVICES[@]}"; do + check_service "$service" +done + +echo + +# Check API endpoints +check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" +check_api_endpoint "http://127.0.0.1:5610/reports/worktime/today" "aw-worktime-api" + +echo + +if [ ${#UNHEALTHY_SERVICES[@]} -eq 0 ]; then + echo "✓ All services are healthy" + exit 0 +else + echo "✗ Unhealthy services: ${UNHEALTHY_SERVICES[*]}" + exit 1 +fi \ No newline at end of file diff --git a/aw-server/logrotate.conf b/aw-server/logrotate.conf new file mode 100644 index 0000000..35ac6ed --- /dev/null +++ b/aw-server/logrotate.conf @@ -0,0 +1,28 @@ +# Log rotation for ActivityWatch services +# Place in /etc/logrotate.d/ + +/var/log/activitywatch/*.log { + daily + missingok + rotate 30 + compress + delaycompress + notifempty + create 644 activitywatch activitywatch + postrotate + systemctl reload activitywatch-server >/dev/null 2>&1 || true + systemctl reload aw-worktime-api >/dev/null 2>&1 || true + endscript +} + +/var/log/journal/*activitywatch*.journal { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + postrotate + systemctl restart systemd-journald >/dev/null 2>&1 || true + endscript +} \ No newline at end of file diff --git a/docs/dlp-policy-engine.md b/docs/dlp-policy-engine.md new file mode 100644 index 0000000..d6513e7 --- /dev/null +++ b/docs/dlp-policy-engine.md @@ -0,0 +1,89 @@ +# DLP Policy Engine + +## Purpose + +`aw-server/dlp-policy-engine` centralizes DLP policy lifecycle for `AWatch-rus` Windows endpoints. + +It does not replace endpoint-local safety. Endpoints can run in: +- `local` +- `server` +- `cached` fallback after server outage + +## API + +Base URL: + +```text +http://:5601 +``` + +Routes: + +- `GET /healthz` +- `GET /api/0/dlp/policies` +- `POST /api/0/dlp/policies` +- `GET /api/0/dlp/policies/active` +- `POST /api/0/dlp/policies/rollback` +- `GET /api/0/dlp/policies/{id}` +- `PUT /api/0/dlp/policies/{id}` +- `POST /api/0/dlp/policies/{id}/activate` +- `DELETE /api/0/dlp/policies/{id}` + +## Policy create example + +```json +{ + "name": "base-windows-policy", + "description": "Primary DLP policy for pilot endpoints", + "activate": true, + "actor": "ansible", + "policy": { + "version": 1, + "defaults": { + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium" + }, + "endpoint": { + "clipboard": [], + "usb": [], + "print": [] + } + } +} +``` + +## Active policy response + +```json +{ + "active": true, + "policyId": 1, + "name": "base-windows-policy", + "version": 3, + "checksum": "sha256...", + "updatedAtUtc": "2026-05-11T12:00:00Z", + "policy": { + "version": 1, + "defaults": { + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium" + }, + "endpoint": { + "clipboard": [], + "usb": [], + "print": [] + } + } +} +``` + +## Deployment Notes + +- Service runs as `aw-dlp-policy-engine.service`. +- SQLite path is controlled by `AW_DLP_POLICY_ENGINE_DB_PATH`. +- Default port is `5601`. +- Endpoints should use `server` mode only after `GET /healthz` and `GET /api/0/dlp/policies/active` are confirmed. diff --git a/docs/dlp-production-execution-roadmap.md b/docs/dlp-production-execution-roadmap.md new file mode 100644 index 0000000..5f8ef16 --- /dev/null +++ b/docs/dlp-production-execution-roadmap.md @@ -0,0 +1,205 @@ +# DLP Production Execution Roadmap + +## Purpose + +This document converts the high-level production DLP plan into an execution sequence with explicit phase boundaries, dependencies, deliverables, and acceptance gates. + +Source plan: +- [dlp-production-plan-windows-10-19.md](/mnt/usb_hdd2/Projects/ActivityWatch-Russian/docs/dlp-production-plan-windows-10-19.md:1) + +## Execution Rule + +Execution order is based on dependency and operational value, not on the original stage numbering from the idea draft. + +## Phase Table + +| Phase | Name | Depends on | Primary outcome | +|-------|------|------------|-----------------| +| 01 | Policy Engine | None | Central policy control plane with endpoint fallback | +| 02 | Content Analysis | 01 | Dictionary, regex, and OCR enrichment | +| 03 | Admin Tooling | 01 | Health checks and operator CLI | +| 04 | SIEM/SOAR Integrations | 01, 03 | External incident delivery and alerting | +| 05 | Case Management | 01, 03 | Investigation workflow and audit trail | +| 06 | Compliance Reporting | 01, 02, 03, 05 | Periodic reports for 152-FZ operations | + +## Phase 01: Policy Engine + +**Goal** +Build the server-side policy control plane without breaking endpoint autonomy. + +**Deliverables** +- `aw-server/dlp-policy-engine/` service package +- SQLite-backed policy/version storage +- Active policy API +- rollback and backup flow +- endpoint `server/local/cached` policy modes +- Ansible deployment role and server integration + +**Acceptance** +- Policies can be created, activated, versioned, and rolled back. +- Endpoints keep detecting while the policy service is unavailable. +- Invalid policies cannot become active. +- Operators can tell which policy source is active on each endpoint. + +**First tasks** +- Create service skeleton and schema model. +- Define SQLite schema for `policies` and `policy_versions`. +- Add active policy endpoint and local cache contract. +- Update endpoint collector with `-PolicyMode` and cache fallback. +- Add Ansible variables and service deployment. + +## Phase 02: Content Analysis + +**Goal** +Increase detection quality with practical Russian PII and OCR enrichment. + +**Depends on** +- Phase 01 active policy delivery + +**Deliverables** +- `152-fz-pdn.json` +- checksum validation module +- dictionary matcher +- regex packs for finance, contacts, and secrets +- OCR processor and server-side artifact enrichment +- endpoint policy fields for dictionary/regex/OCR + +**Acceptance** +- Incidents can be enriched by dictionary and regex matches. +- SNILS/INN validation reduces false positives. +- OCR can be turned on and off via policy. +- Screenshot processing path is explicit and auditable. + +**First tasks** +- Create checksum validator for INN and SNILS. +- Define server-side pack loading contract. +- Add policy fields for `dictionaryPack`, `regexPack`, and `ocrEnabled`. +- Implement OCR wrapper and artifact processing pipeline. +- Extend endpoint incident payload for OCR-bound artifacts. + +## Phase 03: Admin Tooling + +**Goal** +Replace ad hoc operational steps with one supported CLI and one health check path. + +**Depends on** +- Phase 01 policy engine API + +**Deliverables** +- `scripts/dlp-admin-cli.py` +- `scripts/dlp-health-check.py` +- health check coverage for API, service state, endpoint sync, and disk +- documented operational commands + +**Acceptance** +- Operator can inspect policy, incident, case, and service state from CLI. +- Health checks have machine-readable exit codes. +- Critical failures are visible without manual DB access. + +**First tasks** +- Define CLI command surface and argument model. +- Implement policy list/push and health check commands first. +- Add endpoint sync status probe. +- Add service/systemd state checks. +- Document standard operator usage. + +## Phase 04: SIEM/SOAR Integrations + +**Goal** +Export actionable incidents outside the AW UI. + +**Depends on** +- Phase 01 policy engine +- Phase 03 admin tooling and health checks + +**Deliverables** +- CEF exporter +- webhook sender +- systemd service/timer units +- Ansible deployment for integrations + +**Acceptance** +- High-severity incidents reach syslog/webhook targets. +- Retry/backoff protects against transient delivery failure. +- Failed exports are visible through logs and health checks. + +**First tasks** +- Define normalized export schema. +- Build CEF severity mapping. +- Implement webhook retry/backoff. +- Add integration service configs and timers. +- Extend health checks to include delivery status. + +## Phase 05: Case Management + +**Goal** +Introduce a practical incident-to-case workflow with immutable audit history. + +**Depends on** +- Phase 01 policy engine +- Phase 03 admin tooling + +**Deliverables** +- case API and schema +- SQLite case store +- `case_audit` append-only log +- UI hooks in `aw-ru-patch.js` and `aw-case-management-ui.js` + +**Acceptance** +- Operator can create a case from a DLP incident. +- Case status changes preserve history. +- Evidence links remain attached through case lifecycle. + +**First tasks** +- Define case model, status model, and audit table. +- Implement create/list/update case endpoints. +- Add incident-to-case action in UI. +- Expose related cases on incident view. +- Add CLI support for case creation and listing. + +## Phase 06: Compliance Reporting + +**Goal** +Generate scheduled DLP compliance reporting usable for 152-FZ operations. + +**Depends on** +- Phase 01 policy engine +- Phase 02 content analysis +- Phase 03 admin tooling +- Phase 05 case management + +**Deliverables** +- report generator +- HTML template +- PDF export via `weasyprint` +- monthly scheduler service/timer +- email delivery path + +**Acceptance** +- Monthly report can be generated without manual data prep. +- Report includes incidents, channels, users, and case linkage where available. +- Scheduler health and last-success state are visible operationally. + +**First tasks** +- Define report input model and time-period filters. +- Create HTML template and PDF renderer wrapper. +- Implement monthly scheduler. +- Add email delivery configuration. +- Extend health check for report freshness. + +## Release Discipline + +- Deploy server components behind feature flags first. +- Keep endpoint fallback local until server path is proven. +- Pilot on a subset of Windows `10-19` before wide rollout. +- Do not enable OCR or external exports by default on first deployment wave. + +## Completion Standard + +This roadmap is complete only when each phase has: +- deployed code +- Ansible coverage +- health checks +- operator documentation +- rollback notes +- a passed acceptance gate diff --git a/docs/dlp-production-plan-windows-10-19.md b/docs/dlp-production-plan-windows-10-19.md new file mode 100644 index 0000000..b050837 --- /dev/null +++ b/docs/dlp-production-plan-windows-10-19.md @@ -0,0 +1,458 @@ +# Production DLP Plan for Windows 10-19 + +## Goal + +Build a reliable, easy-to-deploy, and maintainable production DLP system on top of the existing `AWatch-rus` platform, with Windows hosts `10-19` as the primary target scope. + +This plan assumes the current baseline already exists: +- Windows collectors for endpoint/browser/email/file telemetry +- ActivityWatch-based ingestion and Web UI overlays +- Ansible deployment for AW server and Windows endpoints +- InnoSetup-based Windows install kit + +## Scope + +In scope: +- Centralized DLP policy lifecycle +- Advanced practical content analysis for Russian personal data +- SIEM/SOAR exports and notifications +- Case management for investigations +- Compliance reporting +- Administrative tooling and health checks + +Out of scope for this phase: +- Full enterprise RBAC/SoD model +- Multi-tenant administration +- Heavy ML/UEBA +- Approval workflows more complex than basic policy rollback/case review + +## Delivery Principles + +- Keep deployment simple: Python + SQLite + systemd on server, PowerShell on endpoints. +- Default to last-known-good behavior on every critical component. +- Do not break current Phase-1/2 DLP behavior while adding server-side controls. +- Prefer additive rollout behind feature flags and config toggles. +- Every new service must have Ansible deployment, health checks, logs, and rollback notes. + +## Current Baseline and Gap + +Current AWatch-rus DLP already provides: +- Rule-based endpoint detection +- Incident buckets and review UI +- Basic enforcement for clipboard/USB/print +- Email/file/browser collectors +- Initial reliability hardening work + +Main gap to production DLP: +- Policies are still too endpoint-local +- Content analysis is not centralized or rich enough +- Incident export/investigation/reporting chain is incomplete +- Health/operations model is not yet unified + +## Stage 1: Policy Engine + +### Objective + +Centralize policy management and distribution without breaking endpoint autonomy. + +### Files + +- `aw-server/dlp-policy-engine/policy_service.py` +- `aw-server/dlp-policy-engine/policy_schema.py` +- `aw-server/dlp-policy-engine/policy_storage.py` +- `aw-server/dlp-policy-engine/policy_distributor.py` +- `aw-server/dlp-policy-engine/requirements.txt` +- `aw-server/dlp-policy-engine/dlp-policy-engine.service` +- `ansible/roles/dlp-policy-engine/tasks/main.yml` +- `docs/dlp-policy-engine.md` +- `windows/dlp-policy-client.ps1` + +### Server responsibilities + +- REST API: + - `GET /api/0/dlp/policies` + - `POST /api/0/dlp/policies` + - `PUT /api/0/dlp/policies/{id}` + - `DELETE /api/0/dlp/policies/{id}` + - `GET /api/0/dlp/policies/active` +- SQLite-backed storage with version history in `policy_versions` +- Validation through Pydantic and JSON schema before activation +- Backup and rollback of active policy versions +- Heartbeat-aware policy distribution model + +### Endpoint changes + +Update `windows/dlp-endpoint-signals-collector.ps1`: +- add `-PolicyMode` with values `local` and `server` +- in `server` mode pull active policy every 5 minutes +- cache last valid server policy locally +- fallback to local cached policy if server is unavailable + +### Acceptance criteria + +- Policy can be created, versioned, activated, and rolled back through API. +- Endpoints continue working during policy engine outage. +- Invalid policy cannot become active. +- Endpoint logs clearly show source of active policy: `local`, `server`, or `cached`. + +### Main risks + +- Breaking current local-policy-only flow +- Partial rollout where server mode is enabled before engine is healthy +- Policy drift between server and endpoints + +### Risk controls + +- Feature flag: `aw_dlp_policy_engine_enabled` +- Default endpoint mode remains `local` until validation is complete +- Store active policy checksum/version on both server and endpoint + +## Stage 2: Advanced Content Analysis + +### Objective + +Add practical, legally relevant detection quality without overengineering. + +### 2.1 Dictionary packs for 152-FZ personal data + +#### Files + +- `aw-server/dlp-content-analysis/dictionaries/152-fz-pdn.json` +- `aw-server/dlp-content-analysis/checksum_validator.py` +- `aw-server/dlp-content-analysis/dictionary_matcher.py` + +#### Required capabilities + +- Detect: + - INN + - SNILS + - Russian passport patterns +- Validate checksums where applicable to reduce false positives + +### 2.2 Regex packs + +#### Files + +- `aw-server/dlp-content-analysis/regex-packs/financial.json` +- `aw-server/dlp-content-analysis/regex-packs/contacts.json` +- `aw-server/dlp-content-analysis/regex-packs/secrets.json` + +#### Required capabilities + +- Reusable grouped pattern packs +- Server-defined matching rules distributed via policy +- Match metadata attached to incidents + +### 2.3 OCR for screenshots + +#### Files + +- `aw-server/dlp-content-analysis/ocr_processor.py` +- `aw-server/dlp-content-analysis/requirements.txt` +- `ansible/roles/dlp-content-analysis/tasks/main.yml` + +#### Required capabilities + +- Tesseract wrapper for screenshots in `incident_artifacts` +- OCR text sent through regex and dictionary pipeline +- OCR enrichment attached to the original incident + +### 2.4 Endpoint integration + +Update: +- `windows/dlp-policy.example.json` +- `windows/dlp-endpoint-signals-collector.ps1` + +Add policy fields: +- `dictionaryPack` +- `regexPack` +- `ocrEnabled` + +Endpoint behavior: +- load server-delivered dictionary/regex references +- perform checksum-aware validation for supported PII +- upload screenshots for OCR when enabled by policy + +### Acceptance criteria + +- Server can enrich incidents with dictionary, regex, and OCR findings. +- False positives are reduced through checksum validation. +- OCR can be disabled per policy without code changes. +- Screenshot upload path is explicit and logged. + +### Main risks + +- OCR cost and latency +- Privacy overreach from over-collecting screenshots +- Regex pack sprawl and poor maintainability + +### Risk controls + +- OCR disabled by default +- Artifact retention policy documented +- Pack ownership and naming convention enforced + +## Stage 3: SIEM and SOAR Integrations + +### Objective + +Make DLP incidents operational outside the AW UI. + +### 3.1 CEF exporter + +#### Files + +- `aw-server/dlp-integrations/cef_exporter.py` +- `aw-server/dlp-integrations/cef-config.yaml` +- `aw-server/dlp-integrations/cef-exporter.service` +- `aw-server/dlp-integrations/cef-exporter.timer` + +#### Required capabilities + +- Read normalized incidents from SQLite/PostgreSQL +- Convert incidents to CEF +- Send via syslog +- Map DLP severities to CEF severities + +### 3.2 Webhook notifications + +#### Files + +- `aw-server/dlp-integrations/webhook_sender.py` +- `aw-server/dlp-integrations/webhook-config.yaml` + +#### Required capabilities + +- Notify on `severity=high` +- Retry with backoff +- Include incident details, source host, user, rule, and evidence link + +### 3.3 Ansible integration + +Update `ansible/deploy_aw_server.yml`: +- install Python dependencies +- deploy configs/services/timers +- manage enable/start state + +### Acceptance criteria + +- High-severity incidents can be exported to SIEM and webhook endpoints. +- Export failures are visible and retry safely. +- Timers/services are idempotently managed by Ansible. + +### Main risks + +- Duplicate exports +- Alert fatigue +- Silent delivery failure to external systems + +### Risk controls + +- Event ID based dedupe +- Severity thresholding +- Delivery logs and health checks + +## Stage 4: Case Management + +### Objective + +Provide a practical investigation workflow without introducing a heavy IR platform. + +### Files + +- `aw-server/dlp-case-management/case_service.py` +- `aw-server/dlp-case-management/case_schema.py` +- `aw-server/dlp-case-management/case_storage.py` +- `aw-server/dlp-case-management/case-service.service` +- `install-kit-awindows-20260427-211240/aw-server/aw-case-management-ui.js` + +### Required capabilities + +- Create a case from an incident +- Attach evidence links +- Support statuses: + - `open` + - `investigating` + - `resolved` + - `closed` +- Support comments +- Maintain immutable audit records in `case_audit` + +### UI integration + +Update `install-kit-awindows-20260427-211240/aw-server/aw-ru-patch.js`: +- add `Create case` action in DLP review table +- add `Case Management` section in UI +- show linked cases on incident views + +### Acceptance criteria + +- An operator can create and track a case directly from a DLP incident. +- Evidence remains linked after status transitions. +- Case audit trail is append-only. + +### Main risks + +- UI debt in current patch overlay +- Weak evidence chain semantics +- Mixing incident review and case workflow logic + +### Risk controls + +- Keep case service isolated from core AW server +- Use immutable audit table +- Treat evidence as links/references first, not copied blobs + +## Stage 5: Compliance Reporting + +### Objective + +Generate regular compliance-grade reporting for Russian personal data handling. + +### Files + +- `aw-server/dlp-compliance/report_generator.py` +- `aw-server/dlp-compliance/templates/152-fz-report.html` +- `aw-server/dlp-compliance/report-scheduler.service` +- `aw-server/dlp-compliance/report-scheduler.timer` + +### Required capabilities + +- Period incident report +- Leak-channel statistics +- User statistics +- PDF export via `weasyprint` +- Scheduled email delivery + +### Acceptance criteria + +- Monthly report can be generated unattended. +- Report includes traceable source metrics. +- Output is usable by operations/compliance without manual cleanup. + +### Main risks + +- Weak data quality in upstream incidents +- PDF rendering dependency issues +- Email delivery failures + +### Risk controls + +- Validate report inputs before generation +- Keep HTML template under version control +- Add health check for scheduler and last successful report + +## Stage 6: Administrative Tooling + +### Objective + +Make the whole stack operable without manual database edits or ad hoc scripts. + +### Files + +- `scripts/dlp-admin-cli.py` +- `scripts/dlp-health-check.py` + +### Required CLI functions + +- `python3 dlp-admin-cli.py policies list` +- `python3 dlp-admin-cli.py policies push --host HOSTNAME` +- `python3 dlp-admin-cli.py incidents list --severity high` +- `python3 dlp-admin-cli.py cases create --incident-id ID` +- `python3 dlp-admin-cli.py health check` + +### Health checks + +- API endpoint availability +- endpoint reachability and policy sync state +- queue health if implemented +- disk space +- systemd service state + +### Acceptance criteria + +- Operator can inspect policy, incident, case, and service health from CLI. +- Health check has machine-readable exit status. +- All critical services are covered by a single operational runbook. + +## Cross-Cutting Ansible Work + +Update: +- `ansible/deploy_aw_server.yml` +- `ansible/group_vars/all.example.yml` +- `ansible/roles/dlp-policy-engine/tasks/main.yml` +- `ansible/roles/dlp-content-analysis/tasks/main.yml` + +Required variables: +- `aw_dlp_policy_engine_enabled: true` +- `aw_dlp_policy_engine_port: 5601` + +Ansible quality bar: +- idempotent +- rollback-aware +- systemd-managed +- config templated, not hand-edited in prod + +## Execution Order + +1. Stage 1: Policy Engine +2. Stage 2: Advanced Content Analysis +3. Stage 6: Administrative Tooling +4. Stage 3: SIEM and SOAR Integrations +5. Stage 4: Case Management +6. Stage 5: Compliance Reporting + +Rationale: +- centralized policies are the control plane +- content analysis increases signal quality +- admin tooling is needed before broadening operations +- integrations, cases, and reports depend on stable normalized incidents + +## Release Strategy + +### Wave 1 + +- Deploy policy engine on AW server +- Keep endpoints in `local` mode +- Validate API, versioning, rollback + +### Wave 2 + +- Enable `server` policy mode for a pilot subset of Windows `10-19` +- Validate cache/fallback behavior +- Measure heartbeat and policy freshness + +### Wave 3 + +- Roll out dictionary/regex/OCR selectively +- Enable SIEM/webhook export +- Stabilize case management and reporting + +## Definition of Done + +The plan is considered implemented only when: +- all new services are deployed by Ansible +- endpoint fallback works under server outage +- policy activation/rollback is proven +- content analysis is documented and testable +- SIEM/webhook integrations are observable +- case workflow is usable from UI +- monthly compliance report is generated automatically +- admin CLI and health checks replace ad hoc operational steps + +## Deliverables Checklist + +- [ ] Policy engine service and API +- [ ] Endpoint policy client and cache/fallback +- [ ] SQLite policy versioning and rollback +- [ ] Dictionary packs for 152-FZ +- [ ] Regex packs for financial/contact/secret data +- [ ] OCR processing pipeline +- [ ] CEF exporter +- [ ] Webhook sender +- [ ] Case management service and UI integration +- [ ] 152-FZ report generator and scheduler +- [ ] Administrative CLI +- [ ] Unified health check +- [ ] Ansible deployment coverage +- [ ] Operational documentation diff --git a/windows/ActivityWatch.Windows.Common.psm1 b/windows/ActivityWatch.Windows.Common.psm1 index af4e0c6..e9c854f 100755 --- a/windows/ActivityWatch.Windows.Common.psm1 +++ b/windows/ActivityWatch.Windows.Common.psm1 @@ -265,6 +265,7 @@ function Copy-ActivityWatchCollectorAssets { [string]$CollectorScriptSource, [Parameter(Mandatory = $true)] [string]$EndpointCollectorScriptSource, + [string]$PolicyClientScriptSource, [Parameter(Mandatory = $true)] [string]$FileCollectorScriptSource, [Parameter(Mandatory = $true)] @@ -284,6 +285,7 @@ function Copy-ActivityWatchCollectorAssets { $collectorTarget = Join-Path $StateRoot 'browser-domains-native-collector.ps1' $endpointCollectorTarget = Join-Path $StateRoot 'dlp-endpoint-signals-collector.ps1' + $policyClientTarget = Join-Path $StateRoot 'dlp-policy-client.ps1' $fileCollectorTarget = Join-Path $StateRoot 'file-operations-collector.ps1' $sessionCollectorTarget = Join-Path $StateRoot 'worktime-session-collector.ps1' $emailCollectorTarget = Join-Path $StateRoot 'email-outbound-collector.ps1' @@ -294,6 +296,9 @@ function Copy-ActivityWatchCollectorAssets { Copy-Item -LiteralPath $CollectorScriptSource -Destination $collectorTarget -Force Copy-Item -LiteralPath $EndpointCollectorScriptSource -Destination $endpointCollectorTarget -Force + if ($PolicyClientScriptSource -and (Test-Path -LiteralPath $PolicyClientScriptSource)) { + Copy-Item -LiteralPath $PolicyClientScriptSource -Destination $policyClientTarget -Force + } Copy-Item -LiteralPath $FileCollectorScriptSource -Destination $fileCollectorTarget -Force Copy-Item -LiteralPath $SessionCollectorScriptSource -Destination $sessionCollectorTarget -Force if ($EmailCollectorScriptSource -and (Test-Path -LiteralPath $EmailCollectorScriptSource)) { @@ -321,6 +326,7 @@ function Copy-ActivityWatchCollectorAssets { return [pscustomobject]@{ CollectorScript = $collectorTarget EndpointCollectorScript = $endpointCollectorTarget + PolicyClientScript = $policyClientTarget FileCollectorScript = $fileCollectorTarget SessionCollectorScript = $sessionCollectorTarget EmailCollectorScript = $emailCollectorTarget @@ -349,6 +355,7 @@ function New-ActivityWatchDeploymentConfig { [string]$CollectorScript, [Parameter(Mandatory = $true)] [string]$EndpointCollectorScript, + [string]$PolicyClientScript, [Parameter(Mandatory = $true)] [string]$FileCollectorScript, [Parameter(Mandatory = $true)] @@ -377,12 +384,24 @@ function New-ActivityWatchDeploymentConfig { [Parameter(Mandatory = $true)] [string]$RecoveryScriptPath, [string]$AwHostname, + [ValidateSet('local', 'server')] + [string]$PolicyMode = 'local', + [bool]$PolicyEngineEnabled = $false, + [string]$PolicyEngineHost, + [int]$PolicyEnginePort = 5601, + [ValidateSet('http', 'https')] + [string]$PolicyEngineScheme = 'http', + [int]$PolicyRefreshSeconds = 300, + [string]$PolicyCachePath, [Parameter(Mandatory = $true)] [pscustomobject[]]$UserTasks, - [string]$PackageVersion = 'v0.13.2' + [string]$PackageVersion = 'v0.13.2', + [switch]$IntegrationTestEnabled ) $effectiveIncidentArtifactsRoot = if ($IncidentArtifactsRoot) { $IncidentArtifactsRoot } else { Join-Path $StateRoot 'incident-artifacts' } + $effectivePolicyEngineHost = if ([string]::IsNullOrWhiteSpace($PolicyEngineHost)) { $ServerHost } else { $PolicyEngineHost } + $effectivePolicyCachePath = if ([string]::IsNullOrWhiteSpace($PolicyCachePath)) { Join-Path $StateRoot 'dlp-policy-cache.json' } else { $PolicyCachePath } return [pscustomobject]@{ version = 1 @@ -399,6 +418,7 @@ function New-ActivityWatchDeploymentConfig { logsRoot = $LogsRoot collectorScript = $CollectorScript endpointCollectorScript = $EndpointCollectorScript + policyClientScript = $PolicyClientScript emailCollectorScript = $EmailCollectorScript fileCollectorScript = $FileCollectorScript sessionCollectorScript = $SessionCollectorScript @@ -415,7 +435,7 @@ function New-ActivityWatchDeploymentConfig { afkEnabled = $AfkEnabled windowEnabled = $WindowEnabled fileOpsEnabled = $FileOpsEnabled - emailEnabled = ($null -ne $EmailCollectorScript -and $EmailCollectorScript -ne '') + emailEnabled = $false } logging = [pscustomobject]@{ localAgentLogsEnabled = $LocalAgentLogsEnabled @@ -437,10 +457,20 @@ function New-ActivityWatchDeploymentConfig { incidentBucketPrefix = 'aw-dlp-incidents' enabled = $true } + policyEngine = [pscustomobject]@{ + enabled = $PolicyEngineEnabled + mode = $PolicyMode + host = $effectivePolicyEngineHost + port = $PolicyEnginePort + scheme = $PolicyEngineScheme + refreshSeconds = $PolicyRefreshSeconds + cachePath = $effectivePolicyCachePath + } package = [pscustomobject]@{ version = $PackageVersion } userTasks = @($UserTasks) + integrationTestEnabled = [bool]$IntegrationTestEnabled } } @@ -1048,10 +1078,24 @@ function Set-ActivityWatchScheduledTaskAction { [string]$Arguments ) - $taskCommand = ('"{0}" {1}' -f $Execute, $Arguments) - & schtasks.exe /Change /TN $TaskName /TR $taskCommand | Out-Null - if ($LASTEXITCODE -ne 0) { - throw "schtasks.exe /Change завершился с ошибкой для $TaskName" + $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue + if (-not $task) { + return $false + } + + $newAction = New-ScheduledTaskAction -Execute $Execute -Argument $Arguments + try { + # Non-interactive update path. Avoids schtasks.exe /Change password prompt for user-bound tasks. + Set-ScheduledTask -TaskName $TaskName -Action $newAction -ErrorAction Stop | Out-Null + return $true + } + catch { + $taskCommand = ('"{0}" {1}' -f $Execute, $Arguments) + & schtasks.exe /Change /TN $TaskName /TR $taskCommand | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Не удалось обновить action задачи ${TaskName}: $($_.Exception.Message)" + } + return $true } } @@ -1136,8 +1180,10 @@ function Register-ActivityWatchUserTasks { $existingTask = Get-ActivityWatchScheduledTaskByCommand -TaskName $definition.LaunchTaskName -CommandMatch $ConfigPath if ($existingTask) { - Set-ActivityWatchScheduledTaskAction -TaskName $existingTask.TaskName -Execute $wscriptExe -Arguments $action.Arguments - continue + $updated = Set-ActivityWatchScheduledTaskAction -TaskName $existingTask.TaskName -Execute $wscriptExe -Arguments $action.Arguments + if ($updated) { + continue + } } Remove-ActivityWatchScheduledTask -TaskName $definition.LaunchTaskName diff --git a/windows/deploy-domain-users.ps1 b/windows/deploy-domain-users.ps1 index e2dd12b..d27bfef 100755 --- a/windows/deploy-domain-users.ps1 +++ b/windows/deploy-domain-users.ps1 @@ -26,7 +26,17 @@ param( [bool]$LogonMarkerEnabled = $true, [string]$AwHostname, [string]$CustomRulesPath, - [string]$CustomPolicyPath + [string]$CustomPolicyPath, + [ValidateSet('local', 'server')] + [string]$PolicyMode = 'local', + [bool]$PolicyEngineEnabled = $false, + [string]$PolicyEngineHost, + [int]$PolicyEnginePort = 5601, + [ValidateSet('http', 'https')] + [string]$PolicyEngineScheme = 'http', + [int]$PolicyRefreshSeconds = 300, + [string]$PolicyCachePath, + [switch]$IntegrationTestEnabled ) Set-StrictMode -Version Latest @@ -46,6 +56,7 @@ $launchScriptPath = Join-Path $StateRoot 'launch-watchers.ps1' $recoveryScriptPath = Join-Path $StateRoot 'recovery-loop.ps1' $collectorSource = Join-Path $PSScriptRoot 'browser-domains-native-collector.ps1' $endpointCollectorSource = Join-Path $PSScriptRoot 'dlp-endpoint-signals-collector.ps1' +$policyClientSource = Join-Path $PSScriptRoot 'dlp-policy-client.ps1' $emailCollectorSource = Join-Path $PSScriptRoot 'email-outbound-collector.ps1' $fileCollectorSource = Join-Path $PSScriptRoot 'file-operations-collector.ps1' $sessionCollectorSource = Join-Path $PSScriptRoot 'worktime-session-collector.ps1' @@ -63,6 +74,7 @@ Get-ActivityWatchExecutableMap -InstallRoot $InstallRoot | Out-Null $assetResult = Copy-ActivityWatchCollectorAssets ` -CollectorScriptSource $collectorSource ` -EndpointCollectorScriptSource $endpointCollectorSource ` + -PolicyClientScriptSource $policyClientSource ` -EmailCollectorScriptSource $emailCollectorSource ` -FileCollectorScriptSource $fileCollectorSource ` -SessionCollectorScriptSource $sessionCollectorSource ` @@ -85,6 +97,7 @@ $config = New-ActivityWatchDeploymentConfig ` -LogsRoot $logsRoot ` -CollectorScript $assetResult.CollectorScript ` -EndpointCollectorScript $assetResult.EndpointCollectorScript ` + -PolicyClientScript $assetResult.PolicyClientScript ` -EmailCollectorScript $assetResult.EmailCollectorScript ` -FileCollectorScript $assetResult.FileCollectorScript ` -SessionCollectorScript $assetResult.SessionCollectorScript ` @@ -102,10 +115,18 @@ $config = New-ActivityWatchDeploymentConfig ` -IncidentArtifactsRoot $IncidentArtifactsRoot ` -LogonMarkerEnabled $LogonMarkerEnabled ` -AwHostname $AwHostname ` + -PolicyMode $PolicyMode ` + -PolicyEngineEnabled $PolicyEngineEnabled ` + -PolicyEngineHost $PolicyEngineHost ` + -PolicyEnginePort $PolicyEnginePort ` + -PolicyEngineScheme $PolicyEngineScheme ` + -PolicyRefreshSeconds $PolicyRefreshSeconds ` + -PolicyCachePath $PolicyCachePath ` -LaunchScriptPath $launchScriptPath ` -RecoveryScriptPath $recoveryScriptPath ` -UserTasks $taskDefinitions ` - -PackageVersion $Version + -PackageVersion $Version ` + -IntegrationTestEnabled:$IntegrationTestEnabled Write-ActivityWatchDeploymentConfig -Config $config -Path $configPath Remove-LegacyActivityWatchEntries diff --git a/windows/deploy-ensemble.ps1 b/windows/deploy-ensemble.ps1 index b915666..bc700fa 100644 --- a/windows/deploy-ensemble.ps1 +++ b/windows/deploy-ensemble.ps1 @@ -27,9 +27,19 @@ param( [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath, + [ValidateSet('local', 'server')] + [string]$PolicyMode = 'local', + [bool]$PolicyEngineEnabled = $false, + [string]$PolicyEngineHost, + [int]$PolicyEnginePort = 5601, + [ValidateSet('http', 'https')] + [string]$PolicyEngineScheme = 'http', + [int]$PolicyRefreshSeconds = 300, + [string]$PolicyCachePath, [string]$ReportPath, [switch]$SkipHardening, - [switch]$ValidateAfterDeploy + [switch]$ValidateAfterDeploy, + [switch]$IntegrationTestEnabled ) Set-StrictMode -Version Latest @@ -74,7 +84,15 @@ if (-not (Test-Path -LiteralPath $deployScript)) { -LogonMarkerEnabled $LogonMarkerEnabled ` -AwHostname $AwHostname ` -CustomRulesPath $CustomRulesPath ` - -CustomPolicyPath $CustomPolicyPath + -CustomPolicyPath $CustomPolicyPath ` + -PolicyMode $PolicyMode ` + -PolicyEngineEnabled $PolicyEngineEnabled ` + -PolicyEngineHost $PolicyEngineHost ` + -PolicyEnginePort $PolicyEnginePort ` + -PolicyEngineScheme $PolicyEngineScheme ` + -PolicyRefreshSeconds $PolicyRefreshSeconds ` + -PolicyCachePath $PolicyCachePath ` + -IntegrationTestEnabled:$IntegrationTestEnabled if (-not $SkipHardening) { & $hardeningScript ` diff --git a/windows/dlp-endpoint-signals-collector.ps1 b/windows/dlp-endpoint-signals-collector.ps1 index 2743fc1..61c4960 100644 --- a/windows/dlp-endpoint-signals-collector.ps1 +++ b/windows/dlp-endpoint-signals-collector.ps1 @@ -5,7 +5,15 @@ param( [int]$ServerPort, [ValidateSet('http', 'https')] [string]$ServerScheme, + [string]$PolicyEngineHost, + [int]$PolicyEnginePort, + [ValidateSet('http', 'https')] + [string]$PolicyEngineScheme, [string]$PolicyPath, + [ValidateSet('local', 'server')] + [string]$PolicyMode, + [int]$PolicyRefreshSeconds, + [string]$PolicyCachePath, [string]$LogPath, [int]$PollSeconds ) @@ -20,6 +28,20 @@ try { catch { } +$policyClientModulePath = Join-Path $PSScriptRoot 'dlp-policy-client.ps1' +if (Test-Path -LiteralPath $policyClientModulePath) { + try { + Import-Module $policyClientModulePath -Force -DisableNameChecking + $script:PolicyClientAvailable = $true + } + catch { + $script:PolicyClientAvailable = $false + } +} +else { + $script:PolicyClientAvailable = $false +} + function Get-DeploymentConfig { param([string]$Path) if ($Path -and (Test-Path -LiteralPath $Path)) { @@ -464,6 +486,10 @@ function Load-DlpPolicy { } } + $script:PolicySource = 'defaults' + $script:PolicyVersion = $null + $script:PolicyChecksum = $null + if (-not $Path -or -not (Test-Path -LiteralPath $Path)) { Write-EndpointLog ("policy not found, using defaults: {0}" -f $Path) return @@ -485,12 +511,87 @@ function Load-DlpPolicy { if ($props -contains 'usb' -and $raw.endpoint.usb) { $script:Policy.endpoint.usb = @($raw.endpoint.usb) } if ($props -contains 'print' -and $raw.endpoint.print) { $script:Policy.endpoint.print = @($raw.endpoint.print) } } + $script:PolicySource = 'local' } catch { Write-EndpointLog ("policy parse failed: {0}" -f $_.Exception.Message) } } +function Apply-PolicyFromBundle { + param( + [Parameter(Mandatory = $true)]$Bundle, + [Parameter(Mandatory = $true)][string]$Source + ) + + if (-not $Bundle.policy) { + throw 'Policy bundle has no policy payload.' + } + + $tempPath = [System.IO.Path]::GetTempFileName() + try { + $Bundle.policy | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $tempPath -Encoding UTF8 + Load-DlpPolicy -Path $tempPath + $script:PolicySource = $Source + $script:PolicyVersion = if ($Bundle.PSObject.Properties.Name -contains 'version') { [string]$Bundle.version } else { $null } + $script:PolicyChecksum = if ($Bundle.PSObject.Properties.Name -contains 'checksum') { [string]$Bundle.checksum } else { $null } + } + finally { + Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue + } +} + +function Refresh-DlpPolicyFromServer { + if (-not $script:PolicyEngineEnabled) { + return $false + } + if (-not $script:PolicyClientAvailable) { + Write-EndpointLog 'policy client module unavailable, cannot use server mode' + return $false + } + + try { + $bundle = Get-RemoteDlpPolicyBundle -ApiBase $script:PolicyApiBase -TimeoutSec 10 + Save-CachedDlpPolicyBundle -Bundle $bundle -CachePath $script:PolicyCachePath + Apply-PolicyFromBundle -Bundle $bundle -Source 'server' + $script:LastPolicyRefreshAt = (Get-Date).ToUniversalTime() + Write-EndpointLog ("policy refreshed from server version={0} checksum={1}" -f $script:PolicyVersion, $script:PolicyChecksum) + return $true + } + catch { + Write-EndpointLog ("policy refresh failed: {0}" -f $_.Exception.Message) + return $false + } +} + +function Initialize-DlpPolicy { + if ($script:PolicyMode -eq 'server') { + if (Refresh-DlpPolicyFromServer) { + return + } + + if ($script:PolicyClientAvailable) { + $cached = Read-CachedDlpPolicyBundle -CachePath $script:PolicyCachePath + if ($cached) { + try { + Apply-PolicyFromBundle -Bundle $cached -Source 'cache' + Write-EndpointLog ("policy loaded from cache version={0} checksum={1}" -f $script:PolicyVersion, $script:PolicyChecksum) + return + } + catch { + Write-EndpointLog ("cached policy load failed: {0}" -f $_.Exception.Message) + } + } + } + + Load-DlpPolicy -Path $script:LocalPolicyPath + $script:PolicySource = 'local-fallback' + return + } + + Load-DlpPolicy -Path $script:LocalPolicyPath +} + function Should-EmitByCooldown { param( [string]$Fingerprint, @@ -862,6 +963,7 @@ $resolvedServerHost = if ($ServerHost) { $ServerHost } elseif ($deploymentConfig $resolvedServerPort = if ($PSBoundParameters.ContainsKey('ServerPort')) { $ServerPort } elseif ($deploymentConfig) { [int]$deploymentConfig.server.port } else { 5600 } $resolvedServerScheme = if ($ServerScheme) { $ServerScheme } elseif ($deploymentConfig) { [string]$deploymentConfig.server.scheme } else { 'http' } $resolvedPolicyPath = if ($PolicyPath) { $PolicyPath } elseif ($deploymentConfig -and $deploymentConfig.paths.PSObject.Properties.Name -contains 'policyPath') { [string]$deploymentConfig.paths.policyPath } else { 'C:\ProgramData\AWatch-rus\dlp-policy.json' } +$resolvedStateRoot = if ($deploymentConfig -and $deploymentConfig.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$deploymentConfig.paths.stateRoot } else { Split-Path -Path $resolvedPolicyPath -Parent } $resolvedPollSeconds = if ($PSBoundParameters.ContainsKey('PollSeconds')) { $PollSeconds } elseif ($deploymentConfig) { [int]$deploymentConfig.collector.pollSeconds } else { 5 } $resolvedLogsRoot = if ($deploymentConfig) { [string]$deploymentConfig.paths.logsRoot } else { 'C:\ProgramData\AWatch-rus\logs' } $resolvedLogPath = if ($LogPath) { $LogPath } else { Join-Path $resolvedLogsRoot ("endpoint-signals-{0}.log" -f $env:USERNAME) } @@ -869,12 +971,20 @@ $resolvedLocalAgentLogsEnabled = if ($deploymentConfig -and $deploymentConfig.PS $resolvedIncidentArtifactsRoot = if ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'incidentCapture' -and $deploymentConfig.incidentCapture.PSObject.Properties.Name -contains 'artifactsRoot') { [string]$deploymentConfig.incidentCapture.artifactsRoot } else { Join-Path $env:LOCALAPPDATA 'AWatch-rus\\incident-artifacts' } $resolvedIncidentScreenshotEnabled = if ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'incidentCapture' -and $deploymentConfig.incidentCapture.PSObject.Properties.Name -contains 'screenshotEnabled') { [bool]$deploymentConfig.incidentCapture.screenshotEnabled } else { $true } $resolvedHostname = if ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]$deploymentConfig.awHostname)) { [string]$deploymentConfig.awHostname } else { [string]$env:COMPUTERNAME } +$resolvedPolicyMode = if ($PolicyMode) { [string]$PolicyMode } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'mode') { [string]$deploymentConfig.policyEngine.mode } else { 'local' } +$resolvedPolicyEngineEnabled = if ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'enabled') { [bool]$deploymentConfig.policyEngine.enabled } else { $false } +$resolvedPolicyEngineHost = if ($PolicyEngineHost) { [string]$PolicyEngineHost } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'host') { [string]$deploymentConfig.policyEngine.host } else { $resolvedServerHost } +$resolvedPolicyEnginePort = if ($PSBoundParameters.ContainsKey('PolicyEnginePort')) { $PolicyEnginePort } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'port') { [int]$deploymentConfig.policyEngine.port } else { $resolvedServerPort } +$resolvedPolicyEngineScheme = if ($PolicyEngineScheme) { [string]$PolicyEngineScheme } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'scheme') { [string]$deploymentConfig.policyEngine.scheme } else { $resolvedServerScheme } +$resolvedPolicyRefreshSeconds = if ($PSBoundParameters.ContainsKey('PolicyRefreshSeconds')) { $PolicyRefreshSeconds } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'refreshSeconds') { [int]$deploymentConfig.policyEngine.refreshSeconds } else { 300 } +$resolvedPolicyCachePath = if ($PolicyCachePath) { [string]$PolicyCachePath } elseif ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'policyEngine' -and $deploymentConfig.policyEngine.PSObject.Properties.Name -contains 'cachePath') { [string]$deploymentConfig.policyEngine.cachePath } else { Join-Path $resolvedStateRoot 'dlp-policy-cache.json' } if ($resolvedLocalAgentLogsEnabled -and -not (Test-Path -LiteralPath $resolvedLogsRoot)) { New-Item -Path $resolvedLogsRoot -ItemType Directory -Force | Out-Null } $script:ApiBase = '{0}://{1}:{2}/api/0' -f $resolvedServerScheme, $resolvedServerHost, $resolvedServerPort +$script:PolicyApiBase = '{0}://{1}:{2}/api/0' -f $resolvedPolicyEngineScheme, $resolvedPolicyEngineHost, $resolvedPolicyEnginePort $script:Hostname = $resolvedHostname $script:SessionId = (Get-Process -Id $PID).SessionId $script:KnownBuckets = @{} @@ -891,17 +1001,37 @@ $script:LogPath = $resolvedLogPath $script:IncidentArtifactsRoot = $resolvedIncidentArtifactsRoot $script:IncidentScreenshotEnabled = $resolvedIncidentScreenshotEnabled $script:ScreenshotTypesLoaded = $false +$script:PolicyMode = $resolvedPolicyMode +$script:PolicyEngineEnabled = $resolvedPolicyEngineEnabled +$script:PolicyRefreshSeconds = [Math]::Max($resolvedPolicyRefreshSeconds, 60) +$script:PolicyCachePath = $resolvedPolicyCachePath +$script:LocalPolicyPath = $resolvedPolicyPath +$script:LastPolicyRefreshAt = [datetime]::MinValue +# Integration test flag (backward compatible - defaults to false) +$script:IntegrationTestEnabled = if ($deploymentConfig -and $deploymentConfig.PSObject.Properties.Name -contains 'integrationTestEnabled') { [bool]$deploymentConfig.integrationTestEnabled } else { $false } -Load-DlpPolicy -Path $resolvedPolicyPath +# Integration metadata tracking (backward compatible) +$script:TotalEventsProcessed = 0 +$script:LastEventTime = $null + +Initialize-DlpPolicy Write-EndpointLog ("endpoint collector started against {0}" -f $script:ApiBase) while ($true) { try { + if ($script:PolicyMode -eq 'server' -and (($nowUtc = (Get-Date).ToUniversalTime()) - $script:LastPolicyRefreshAt).TotalSeconds -ge $script:PolicyRefreshSeconds) { + [void](Refresh-DlpPolicyFromServer) + } + $nowUtc = (Get-Date).ToUniversalTime() if (($nowUtc - $script:LastSelfTestAt).TotalSeconds -ge $script:SelfTestIntervalSeconds) { Send-EndpointSignalHeartbeat -SignalType 'self_test' -Data @{ collector = 'dlp-endpoint-signals' policyEnabled = [bool]$script:Policy.defaults.enabled + policyMode = $script:PolicyMode + policySource = $script:PolicySource + policyVersion = $script:PolicyVersion + policyChecksum = $script:PolicyChecksum } $script:LastSelfTestAt = $nowUtc } @@ -921,6 +1051,8 @@ while ($true) { clipboardHash = $clipboardHash clipboardLength = $clipboardText.Length } + $script:TotalEventsProcessed++ + $script:LastEventTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') Evaluate-ClipboardRules -ClipboardText $clipboardText -ClipboardHash $clipboardHash } } @@ -942,6 +1074,8 @@ while ($true) { driveLetter = $deviceId volumeName = $volumeName } + $script:TotalEventsProcessed++ + $script:LastEventTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') Evaluate-UsbRules -DriveLetter $deviceId -VolumeName $volumeName } } @@ -981,6 +1115,8 @@ while ($true) { documentNameOriginal = $documentNameOriginal owner = $owner } + $script:TotalEventsProcessed++ + $script:LastEventTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') Evaluate-PrintRules -PrinterName $printerName -DocumentName $documentName -Owner $owner } @@ -1028,6 +1164,8 @@ while ($true) { eventRecordId = $recordId eventSource = 'printservice-307' } + $script:TotalEventsProcessed++ + $script:LastEventTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') Evaluate-PrintRules -PrinterName $printerName -DocumentName (if ($resolvedDocument) { $resolvedDocument } else { $documentName }) -Owner $owner } @@ -1046,5 +1184,30 @@ while ($true) { Write-EndpointLog ("collector error: {0}" -f $_.Exception.Message) } + # Integration metadata self-test (backward compatible) + if ($script:IntegrationTestEnabled -and (Get-Date).Minute -eq 0) { + try { + $testMetadata = @{ + timestamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') + collector = 'dlp-endpoint-signals' + version = '1.0.0' + hostname = $env:COMPUTERNAME + username = $env:USERNAME + status = 'healthy' + checks = @{ + eventsProcessed = $script:TotalEventsProcessed + lastEventTime = $script:LastEventTime + iocRulesLoaded = if ($script:IocRules) { @($script:IocRules).Count } else { 0 } + policyRulesLoaded = if ($script:Policy -and $script:Policy.endpoint) { (@($script:Policy.endpoint.clipboard).Count + @($script:Policy.endpoint.usb).Count + @($script:Policy.endpoint.print).Count) } else { 0 } + } + } + Send-EndpointSignalHeartbeat -SignalType 'integration_test' -Data $testMetadata + Write-EndpointLog "Integration metadata test sent" + } + catch { + Write-EndpointLog "Integration test failed: $($_.Exception.Message)" + } + } + Start-Sleep -Seconds $resolvedPollSeconds } diff --git a/windows/dlp-policy-client.ps1 b/windows/dlp-policy-client.ps1 new file mode 100644 index 0000000..5b7f9f2 --- /dev/null +++ b/windows/dlp-policy-client.ps1 @@ -0,0 +1,85 @@ +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Invoke-DlpPolicyGetJson { + param( + [Parameter(Mandatory = $true)][string]$Uri, + [int]$TimeoutSec = 10 + ) + + $request = [System.Net.HttpWebRequest]::Create($Uri) + $request.Method = 'GET' + $request.Accept = 'application/json' + $request.KeepAlive = $false + $request.Timeout = $TimeoutSec * 1000 + $request.ReadWriteTimeout = $TimeoutSec * 1000 + + $response = $request.GetResponse() + try { + $stream = $response.GetResponseStream() + $reader = New-Object System.IO.StreamReader($stream, [System.Text.Encoding]::UTF8) + try { + $reader.ReadToEnd() | ConvertFrom-Json + } + finally { + $reader.Close() + } + } + finally { + $response.Close() + } +} + +function Get-RemoteDlpPolicyBundle { + param( + [Parameter(Mandatory = $true)][string]$ApiBase, + [int]$TimeoutSec = 10 + ) + + $bundle = Invoke-DlpPolicyGetJson -Uri ($ApiBase.TrimEnd('/') + '/dlp/policies/active') -TimeoutSec $TimeoutSec + if (-not $bundle) { + throw 'Policy engine returned empty response.' + } + if (-not $bundle.active) { + throw 'Policy engine has no active policy.' + } + if (-not $bundle.policy) { + throw 'Policy engine response has no policy payload.' + } + return $bundle +} + +function Read-CachedDlpPolicyBundle { + param([Parameter(Mandatory = $true)][string]$CachePath) + + if (-not (Test-Path -LiteralPath $CachePath)) { + return $null + } + + try { + return Get-Content -LiteralPath $CachePath -Raw | ConvertFrom-Json + } + catch { + return $null + } +} + +function Save-CachedDlpPolicyBundle { + param( + [Parameter(Mandatory = $true)]$Bundle, + [Parameter(Mandatory = $true)][string]$CachePath + ) + + $directory = Split-Path -Path $CachePath -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + + $json = $Bundle | ConvertTo-Json -Depth 20 + Set-Content -LiteralPath $CachePath -Value $json -Encoding UTF8 +} + +Export-ModuleMember -Function Invoke-DlpPolicyGetJson, Get-RemoteDlpPolicyBundle, Read-CachedDlpPolicyBundle, Save-CachedDlpPolicyBundle diff --git a/windows/email-outbound-collector.ps1 b/windows/email-outbound-collector.ps1 index 7a1600f..96c6461 100644 --- a/windows/email-outbound-collector.ps1 +++ b/windows/email-outbound-collector.ps1 @@ -26,7 +26,7 @@ param( [string]$LogPath, [int]$PollSeconds, [ValidateSet('outlook', 'smtp', 'both')] - [string]$Mode = 'both' + [string]$Mode = 'smtp' ) Set-StrictMode -Version Latest @@ -322,8 +322,23 @@ function Invoke-EmailEnforcement { # --------------------------------------------------------------------------- function Initialize-OutlookCom { + if ($script:OutlookDisabled) { + return $false + } + + if (-not (Test-OutlookProfileConfigured)) { + Write-CollectorLog "Outlook profile not configured for current user, Outlook mode disabled" + $script:OutlookDisabled = $true + return $false + } + + if (-not (Get-Process -Name OUTLOOK -ErrorAction SilentlyContinue | Select-Object -First 1)) { + Write-CollectorLog "Outlook process not running, skipping COM initialization" + return $false + } + try { - $script:OutlookApp = New-Object -ComObject Outlook.Application + $script:OutlookApp = [Runtime.InteropServices.Marshal]::GetActiveObject('Outlook.Application') $script:OutlookNamespace = $script:OutlookApp.GetNamespace('MAPI') $script:SentFolder = $script:OutlookNamespace.GetDefaultFolder(5) # olFolderSentMail Write-CollectorLog "Outlook COM initialized, Sent Items folder opened" @@ -335,6 +350,32 @@ function Initialize-OutlookCom { } } +function Test-OutlookProfileConfigured { + [OutputType([bool])] + $officeRoots = @( + 'HKCU:\Software\Microsoft\Office', + 'HKCU:\Software\WOW6432Node\Microsoft\Office' + ) + + foreach ($root in $officeRoots) { + if (-not (Test-Path -LiteralPath $root)) { continue } + $versions = Get-ChildItem -LiteralPath $root -ErrorAction SilentlyContinue | + Where-Object { $_.PSChildName -match '^\d+\.\d+$' } | + Sort-Object { [version]$_.PSChildName } -Descending + foreach ($ver in $versions) { + $profilesPath = Join-Path $ver.PSPath 'Outlook\Profiles' + if (Test-Path -LiteralPath $profilesPath) { + $profiles = Get-ChildItem -LiteralPath $profilesPath -ErrorAction SilentlyContinue + if ($profiles -and $profiles.Count -gt 0) { + return $true + } + } + } + } + + return $false +} + function Get-OutlookSentItems { param([datetime]$Since) @@ -521,6 +562,7 @@ $script:OutlookApp = $null $script:OutlookNamespace = $null $script:SentFolder = $null $script:OutlookLastPoll = (Get-Date).AddMinutes(-5) +$script:OutlookDisabled = $false Load-EmailPolicy -Path $resolvedPolicyPath Write-CollectorLog ("email collector started mode={0} against {1}" -f $Mode, $script:ApiBase) diff --git a/windows/validate-deployment.ps1 b/windows/validate-deployment.ps1 index 41b6bd9..17d9e19 100644 --- a/windows/validate-deployment.ps1 +++ b/windows/validate-deployment.ps1 @@ -18,6 +18,7 @@ $fileCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'fil $sessionCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$config.paths.sessionCollectorScript } else { Join-Path $stateRoot 'worktime-session-collector.ps1' } $rulesPath = [string]$config.paths.rulesPath $policyPath = if ($config.paths.PSObject.Properties.Name -contains 'policyPath') { [string]$config.paths.policyPath } else { Join-Path $stateRoot 'dlp-policy.json' } +$policyClientScript = if ($config.paths.PSObject.Properties.Name -contains 'policyClientScript') { [string]$config.paths.policyClientScript } else { Join-Path $stateRoot 'dlp-policy-client.ps1' } $launchScript = [string]$config.paths.launchScript $recoveryScript = [string]$config.paths.recoveryScript @@ -44,6 +45,7 @@ $requiredFiles = @( $sessionCollectorScript, $rulesPath, $policyPath, + $policyClientScript, $launchScript, $recoveryScript, $ConfigPath @@ -59,7 +61,9 @@ if ($windowExpected) { } $missingFiles = @( - $requiredFiles | Where-Object { -not (Test-Path -LiteralPath $_) } + $requiredFiles | + Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } | + Where-Object { -not (Test-Path -LiteralPath $_) } ) $processNames = @()