feat(dlp): add compliance reporting scheduler and admin CLI

This commit is contained in:
igor04091968
2026-05-13 03:31:40 +03:00
parent 28a4dae67f
commit 070a8f8f7e
11 changed files with 481 additions and 16 deletions
+66
View File
@@ -308,6 +308,9 @@
AW_DLP_CASE_BIND_HOST={{ aw_dlp_case_bind_host }} AW_DLP_CASE_BIND_HOST={{ aw_dlp_case_bind_host }}
AW_DLP_CASE_PORT={{ aw_dlp_case_port }} AW_DLP_CASE_PORT={{ aw_dlp_case_port }}
AW_DLP_CASE_DB_PATH={{ aw_dlp_case_db_path }} AW_DLP_CASE_DB_PATH={{ aw_dlp_case_db_path }}
AW_DLP_COMPLIANCE_REPORT_DIR={{ aw_dlp_compliance_report_dir }}
AW_DLP_COMPLIANCE_TEMPLATE={{ aw_dlp_compliance_template_path }}
AW_SERVER_URL=http://127.0.0.1:5600/api/0
XDG_DATA_HOME={{ aw_server_data_dir }}/.local/share XDG_DATA_HOME={{ aw_server_data_dir }}/.local/share
XDG_CONFIG_HOME={{ aw_server_data_dir }}/.config XDG_CONFIG_HOME={{ aw_server_data_dir }}/.config
@@ -534,6 +537,55 @@
mode: "0644" mode: "0644"
when: aw_dlp_case_management_enabled | default(true) | bool when: aw_dlp_case_management_enabled | default(true) | bool
- name: Создать каталоги DLP compliance
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
loop:
- /opt/activitywatch/dlp-compliance
- /opt/activitywatch/dlp-compliance/templates
- "{{ aw_dlp_compliance_report_dir }}"
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Скопировать файлы DLP compliance
ansible.builtin.copy:
src: "{{ aw_repo_root }}/aw-server/dlp-compliance/{{ item.src }}"
dest: "{{ item.dest }}"
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "{{ item.mode }}"
loop:
- { src: "report_generator.py", dest: "/opt/activitywatch/dlp-compliance/report_generator.py", mode: "0755" }
- { src: "requirements.txt", dest: "/opt/activitywatch/dlp-compliance/requirements.txt", mode: "0644" }
- { src: "templates/152-fz-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html", mode: "0644" }
- { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" }
- { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" }
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Создать virtualenv DLP compliance
ansible.builtin.command:
cmd: python3 -m venv /opt/activitywatch/dlp-compliance/.venv
args:
creates: /opt/activitywatch/dlp-compliance/.venv/bin/python
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Установить зависимости DLP compliance
ansible.builtin.pip:
requirements: /opt/activitywatch/dlp-compliance/requirements.txt
virtualenv: /opt/activitywatch/dlp-compliance/.venv
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Установить dlp-admin-cli
ansible.builtin.copy:
src: "{{ aw_repo_root }}/scripts/dlp-admin-cli.py"
dest: /usr/local/bin/dlp-admin-cli.py
owner: root
group: root
mode: "0755"
- name: Установить скрипт AW worktime API - name: Установить скрипт AW worktime API
ansible.builtin.copy: ansible.builtin.copy:
src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py"
@@ -638,6 +690,20 @@
state: restarted state: restarted
when: aw_dlp_case_management_enabled | default(true) | bool when: aw_dlp_case_management_enabled | default(true) | bool
- name: Включить и перезапустить timer DLP compliance report
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.timer
enabled: true
state: restarted
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Выполнить разовый прогон DLP compliance report
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.service
state: started
failed_when: false
when: aw_dlp_compliance_enabled | default(true) | bool
- name: Включить и перезапустить AW worktime API - name: Включить и перезапустить AW worktime API
ansible.builtin.systemd: ansible.builtin.systemd:
name: aw-worktime-api.service name: aw-worktime-api.service
+1
View File
@@ -11,3 +11,4 @@
- role: dlp-content-analysis - role: dlp-content-analysis
- role: dlp-integrations - role: dlp-integrations
- role: dlp-case-management - role: dlp-case-management
- role: dlp-compliance
+3
View File
@@ -46,4 +46,7 @@ aw_dlp_case_management_enabled: true
aw_dlp_case_bind_host: "0.0.0.0" aw_dlp_case_bind_host: "0.0.0.0"
aw_dlp_case_port: 5602 aw_dlp_case_port: 5602
aw_dlp_case_db_path: "/opt/activitywatch/dlp-case-management/cases.db" aw_dlp_case_db_path: "/opt/activitywatch/dlp-case-management/cases.db"
aw_dlp_compliance_enabled: true
aw_dlp_compliance_report_dir: "/opt/activitywatch/dlp-compliance/reports"
aw_dlp_compliance_template_path: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html"
aw_server_post_deploy_health_check_enabled: true aw_server_post_deploy_health_check_enabled: true
+3
View File
@@ -37,6 +37,9 @@ aw_dlp_case_management_enabled: true
aw_dlp_case_bind_host: "0.0.0.0" aw_dlp_case_bind_host: "0.0.0.0"
aw_dlp_case_port: 5602 aw_dlp_case_port: 5602
aw_dlp_case_db_path: "/opt/activitywatch/dlp-case-management/cases.db" aw_dlp_case_db_path: "/opt/activitywatch/dlp-case-management/cases.db"
aw_dlp_compliance_enabled: true
aw_dlp_compliance_report_dir: "/opt/activitywatch/dlp-compliance/reports"
aw_dlp_compliance_template_path: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html"
aw_server_post_deploy_health_check_enabled: true aw_server_post_deploy_health_check_enabled: true
aw_worktime_from: "08:00" aw_worktime_from: "08:00"
@@ -0,0 +1,53 @@
---
- name: Ensure DLP compliance directory tree
ansible.builtin.file:
path: "{{ item }}"
state: directory
owner: "{{ aw_server_user | default('activitywatch') }}"
group: "{{ aw_server_group | default('activitywatch') }}"
mode: "0755"
loop:
- /opt/activitywatch/dlp-compliance
- /opt/activitywatch/dlp-compliance/templates
- /opt/activitywatch/dlp-compliance/reports
- name: Copy DLP compliance files
ansible.builtin.copy:
src: "{{ playbook_dir }}/../aw-server/dlp-compliance/{{ item.src }}"
dest: "{{ item.dest }}"
owner: "{{ aw_server_user | default('activitywatch') }}"
group: "{{ aw_server_group | default('activitywatch') }}"
mode: "{{ item.mode }}"
loop:
- { src: "report_generator.py", dest: "/opt/activitywatch/dlp-compliance/report_generator.py", mode: "0755" }
- { src: "requirements.txt", dest: "/opt/activitywatch/dlp-compliance/requirements.txt", mode: "0644" }
- { src: "templates/152-fz-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html", mode: "0644" }
- { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" }
- { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" }
- name: Create venv for DLP compliance
ansible.builtin.command:
cmd: python3 -m venv /opt/activitywatch/dlp-compliance/.venv
args:
creates: /opt/activitywatch/dlp-compliance/.venv/bin/python
- name: Install Python dependencies for DLP compliance
ansible.builtin.pip:
requirements: /opt/activitywatch/dlp-compliance/requirements.txt
virtualenv: /opt/activitywatch/dlp-compliance/.venv
- name: Reload systemd
ansible.builtin.systemd:
daemon_reload: true
- name: Enable and start DLP compliance timer
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.timer
enabled: true
state: restarted
- name: Run DLP compliance report once
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.service
state: started
@@ -0,0 +1,12 @@
[Unit]
Description=AWatch DLP 152-FZ compliance report generator
After=network-online.target
[Service]
Type=oneshot
EnvironmentFile=-/etc/activitywatch/aw-server.env
WorkingDirectory=/opt/activitywatch/dlp-compliance
ExecStart=/opt/activitywatch/dlp-compliance/.venv/bin/python /opt/activitywatch/dlp-compliance/report_generator.py
User=activitywatch
Group=activitywatch
@@ -0,0 +1,12 @@
[Unit]
Description=Monthly AWatch DLP 152-FZ compliance report schedule
[Timer]
OnCalendar=monthly
Persistent=true
RandomizedDelaySec=5m
Unit=aw-dlp-report-scheduler.service
[Install]
WantedBy=timers.target
+163 -8
View File
@@ -1,20 +1,175 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from __future__ import annotations from __future__ import annotations
from datetime import datetime import argparse
import json
import os
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from urllib.parse import quote
from urllib.request import Request, urlopen
def render_html(period: str) -> str: def _env(name: str, default: str) -> str:
return f"""<html><body><h1>Отчет 152-ФЗ</h1><p>Период: {period}</p><p>Сгенерирован: {datetime.now().isoformat()}</p></body></html>""" value = os.environ.get(name)
return value if value not in (None, "") else default
AW_API_BASE = _env("AW_SERVER_URL", "http://127.0.0.1:5600/api/0").rstrip("/")
OUTPUT_DIR = Path(_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports"))
TEMPLATE_PATH = Path(_env("AW_DLP_COMPLIANCE_TEMPLATE", "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html"))
@dataclass
class ReportStats:
total_incidents: int
high: int
medium: int
low: int
by_host: dict[str, int]
channels: dict[str, int]
def _http_json(url: str) -> object:
req = Request(url, headers={"Accept": "application/json"})
with urlopen(req, timeout=30) as response:
return json.loads(response.read().decode("utf-8"))
def _parse_ts(value: str | None) -> datetime | None:
if not value:
return None
text = value.replace("Z", "+00:00")
try:
return datetime.fromisoformat(text).astimezone(UTC)
except ValueError:
return None
def _load_incidents(start: datetime, end: datetime) -> list[dict]:
buckets = _http_json(f"{AW_API_BASE}/buckets")
if not isinstance(buckets, dict):
return []
bucket_ids = sorted([bid for bid in buckets.keys() if str(bid).startswith("aw-dlp-incidents_")])
incidents: list[dict] = []
for bucket_id in bucket_ids:
encoded = quote(str(bucket_id), safe="")
events = _http_json(f"{AW_API_BASE}/buckets/{encoded}/events?limit=2000")
if not isinstance(events, list):
continue
for event in events:
if not isinstance(event, dict):
continue
ts = _parse_ts(event.get("timestamp"))
if ts is None or ts < start or ts > end:
continue
incidents.append(event)
return incidents
def _build_stats(incidents: list[dict]) -> ReportStats:
by_host: dict[str, int] = {}
channels: dict[str, int] = {}
high = medium = low = 0
for event in incidents:
data = event.get("data") or {}
if not isinstance(data, dict):
data = {}
host = str(data.get("hostname") or "unknown")
by_host[host] = by_host.get(host, 0) + 1
severity = str(data.get("severity") or "low").lower()
if severity == "high":
high += 1
elif severity == "medium":
medium += 1
else:
low += 1
channel = str(data.get("signalType") or data.get("source") or "unknown")
channels[channel] = channels.get(channel, 0) + 1
return ReportStats(
total_incidents=len(incidents),
high=high,
medium=medium,
low=low,
by_host=dict(sorted(by_host.items(), key=lambda item: item[1], reverse=True)),
channels=dict(sorted(channels.items(), key=lambda item: item[1], reverse=True)),
)
def _render_table(title: str, rows: list[tuple[str, int]]) -> str:
if not rows:
return f"<h3>{title}</h3><p>Нет данных</p>"
body = "".join([f"<tr><td>{name}</td><td>{count}</td></tr>" for name, count in rows])
return f"<h3>{title}</h3><table><thead><tr><th>Параметр</th><th>Значение</th></tr></thead><tbody>{body}</tbody></table>"
def _render_html(period_label: str, stats: ReportStats, generated_at: str) -> str:
template = TEMPLATE_PATH.read_text(encoding="utf-8")
return (
template.replace("{{PERIOD}}", period_label)
.replace("{{GENERATED_AT}}", generated_at)
.replace("{{TOTAL}}", str(stats.total_incidents))
.replace("{{HIGH}}", str(stats.high))
.replace("{{MEDIUM}}", str(stats.medium))
.replace("{{LOW}}", str(stats.low))
.replace("{{HOST_TABLE}}", _render_table("Инциденты по хостам", list(stats.by_host.items())))
.replace("{{CHANNEL_TABLE}}", _render_table("Инциденты по каналам", list(stats.channels.items())))
)
def _period_bounds(month: str | None) -> tuple[datetime, datetime, str]:
if month:
start = datetime.fromisoformat(f"{month}-01T00:00:00+00:00").astimezone(UTC)
else:
now = datetime.now(UTC)
start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
if start.month == 12:
end = start.replace(year=start.year + 1, month=1)
else:
end = start.replace(month=start.month + 1)
end = end.replace(second=0, microsecond=0)
return start, end, start.strftime("%Y-%m")
def main() -> None: def main() -> None:
period = datetime.now().strftime("%Y-%m") parser = argparse.ArgumentParser(description="Generate 152-FZ compliance report from AW DLP incidents")
out = Path("/opt/activitywatch/dlp-compliance/reports") parser.add_argument("--month", help="Month in YYYY-MM format (default: current month)")
out.mkdir(parents=True, exist_ok=True) parser.add_argument("--stdout-json", action="store_true", help="Print report metadata as JSON")
html = out / f"152-fz-{period}.html" args = parser.parse_args()
html.write_text(render_html(period), encoding="utf-8")
start, end, period_label = _period_bounds(args.month)
incidents = _load_incidents(start, end)
stats = _build_stats(incidents)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
generated_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
html_out = OUTPUT_DIR / f"152-fz-{period_label}.html"
html_out.write_text(_render_html(period_label, stats, generated_at), encoding="utf-8")
metadata = {
"period": period_label,
"generated_at": generated_at,
"aw_api_base": AW_API_BASE,
"report_path": str(html_out),
"stats": {
"total_incidents": stats.total_incidents,
"high": stats.high,
"medium": stats.medium,
"low": stats.low,
},
}
(OUTPUT_DIR / f"152-fz-{period_label}.json").write_text(
json.dumps(metadata, ensure_ascii=False, indent=2),
encoding="utf-8",
)
if args.stdout_json:
print(json.dumps(metadata, ensure_ascii=False))
if __name__ == "__main__": if __name__ == "__main__":
@@ -0,0 +1 @@
requests==2.32.3
@@ -0,0 +1,33 @@
<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>Отчёт 152-ФЗ {{PERIOD}}</title>
<style>
body { font-family: Arial, sans-serif; margin: 24px; color: #1f2937; }
h1, h2, h3 { margin: 0 0 12px 0; }
.meta { margin-bottom: 18px; color: #4b5563; }
.cards { display: flex; gap: 12px; margin: 18px 0; }
.card { border: 1px solid #d1d5db; border-radius: 8px; padding: 10px 14px; min-width: 120px; }
.label { color: #6b7280; font-size: 12px; }
.value { font-size: 22px; font-weight: 700; }
table { border-collapse: collapse; width: 100%; margin-bottom: 18px; }
th, td { border: 1px solid #d1d5db; padding: 8px; text-align: left; }
th { background: #f3f4f6; }
</style>
</head>
<body>
<h1>Compliance отчёт 152-ФЗ</h1>
<div class="meta">Период: {{PERIOD}} | Сформирован: {{GENERATED_AT}}</div>
<div class="cards">
<div class="card"><div class="label">Всего инцидентов</div><div class="value">{{TOTAL}}</div></div>
<div class="card"><div class="label">High</div><div class="value">{{HIGH}}</div></div>
<div class="card"><div class="label">Medium</div><div class="value">{{MEDIUM}}</div></div>
<div class="card"><div class="label">Low</div><div class="value">{{LOW}}</div></div>
</div>
{{HOST_TABLE}}
{{CHANNEL_TABLE}}
</body>
</html>
+134 -8
View File
@@ -3,29 +3,155 @@ from __future__ import annotations
import argparse import argparse
import json import json
from datetime import UTC, datetime, timedelta
from urllib import request from urllib import request
from urllib.parse import quote
def get_json(url: str): def get_json(url: str):
with request.urlopen(url, timeout=10) as r: with request.urlopen(url, timeout=30) as r:
return json.loads(r.read().decode("utf-8")) return json.loads(r.read().decode("utf-8"))
def send_json(url: str, method: str, payload: dict | None = None):
body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8")
req = request.Request(url, data=body, method=method, headers={"Content-Type": "application/json"})
with request.urlopen(req, timeout=30) as r:
raw = r.read().decode("utf-8")
return json.loads(raw) if raw else {}
def parse_iso(ts: str | None) -> datetime | None:
if not ts:
return None
try:
return datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone(UTC)
except ValueError:
return None
def main() -> None: def main() -> None:
p = argparse.ArgumentParser(description="AWatch DLP admin CLI") p = argparse.ArgumentParser(description="AWatch DLP admin CLI")
p.add_argument("--server", default="http://127.0.0.1:5601") p.add_argument("--policy-server", default="http://127.0.0.1:5601")
p.add_argument("--case-server", default="http://127.0.0.1:5602")
p.add_argument("--aw-server", default="http://127.0.0.1:5600")
sub = p.add_subparsers(dest="cmd", required=True) sub = p.add_subparsers(dest="cmd", required=True)
sub.add_parser("policies-list") policies = sub.add_parser("policies")
sub.add_parser("health-check") policies_sub = policies.add_subparsers(dest="policies_cmd", required=True)
policies_sub.add_parser("list")
policies_sub.add_parser("active")
incidents = sub.add_parser("incidents")
incidents_sub = incidents.add_subparsers(dest="incidents_cmd", required=True)
incidents_list = incidents_sub.add_parser("list")
incidents_list.add_argument("--host")
incidents_list.add_argument("--severity")
incidents_list.add_argument("--limit", type=int, default=100)
incidents_list.add_argument("--since-hours", type=int, default=24)
cases = sub.add_parser("cases")
cases_sub = cases.add_subparsers(dest="cases_cmd", required=True)
cases_list = cases_sub.add_parser("list")
cases_list.add_argument("--host")
cases_list.add_argument("--status")
cases_list.add_argument("--limit", type=int, default=100)
cases_create = cases_sub.add_parser("create")
cases_create.add_argument("--incident-id", required=True)
cases_create.add_argument("--title", required=True)
cases_create.add_argument("--host")
cases_create.add_argument("--severity", default="medium")
health = sub.add_parser("health")
health_sub = health.add_subparsers(dest="health_cmd", required=True)
health_sub.add_parser("check")
args = p.parse_args() args = p.parse_args()
if args.cmd == "policies-list": if args.cmd == "policies" and args.policies_cmd == "list":
data = get_json(f"{args.server}/api/0/dlp/policies") data = get_json(f"{args.policy_server}/api/0/dlp/policies")
print(json.dumps(data, ensure_ascii=False, indent=2)) print(json.dumps(data, ensure_ascii=False, indent=2))
elif args.cmd == "health-check": return
data = get_json(f"{args.server}/health")
if args.cmd == "policies" and args.policies_cmd == "active":
data = get_json(f"{args.policy_server}/api/0/dlp/policies/active")
print(json.dumps(data, ensure_ascii=False, indent=2)) print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "incidents" and args.incidents_cmd == "list":
bucket_map = get_json(f"{args.aw_server}/api/0/buckets")
if not isinstance(bucket_map, dict):
print("[]")
return
bucket_ids = [x for x in bucket_map.keys() if str(x).startswith("aw-dlp-incidents_")]
if args.host:
bucket_ids = [x for x in bucket_ids if str(x).endswith("_" + args.host)]
after = datetime.now(UTC) - timedelta(hours=max(1, args.since_hours))
rows = []
for bucket_id in sorted(bucket_ids):
encoded = quote(str(bucket_id), safe="")
events = get_json(f"{args.aw_server}/api/0/buckets/{encoded}/events?limit={max(1, args.limit)}")
if not isinstance(events, list):
continue
for ev in events:
if not isinstance(ev, dict):
continue
ts = parse_iso(ev.get("timestamp"))
if ts is None or ts < after:
continue
data = ev.get("data") or {}
if args.severity and str((data or {}).get("severity", "")).lower() != args.severity.lower():
continue
rows.append(ev)
print(json.dumps(rows, ensure_ascii=False, indent=2))
return
if args.cmd == "cases" and args.cases_cmd == "list":
query = []
if args.host:
query.append(f"host={quote(args.host, safe='')}")
if args.status:
query.append(f"status={quote(args.status, safe='')}")
query.append(f"limit={max(1, args.limit)}")
data = get_json(f"{args.case_server}/api/0/dlp/cases?{'&'.join(query)}")
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "cases" and args.cases_cmd == "create":
payload = {
"incident_id": args.incident_id,
"title": args.title,
"host": args.host,
"severity": args.severity,
"evidence": {"source": "dlp-admin-cli"},
}
data = send_json(f"{args.case_server}/api/0/dlp/cases", "POST", payload)
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "health" and args.health_cmd == "check":
out = {}
try:
out["policy"] = get_json(f"{args.policy_server}/healthz")
except Exception as exc:
out["policy"] = {"status": "error", "error": str(exc)}
try:
out["cases"] = get_json(f"{args.case_server}/health")
except Exception as exc:
out["cases"] = {"status": "error", "error": str(exc)}
try:
out["aw"] = get_json(f"{args.aw_server}/api/0/info")
except Exception as exc:
out["aw"] = {"status": "error", "error": str(exc)}
print(json.dumps(out, ensure_ascii=False, indent=2))
return
raise SystemExit("unsupported command")
if __name__ == "__main__": if __name__ == "__main__":