feat(dlp): deploy hayabusa IOC refresh on aw-server with post-deploy checks

2026-05-10 03:13:28 +03:00
parent c43f9b8708
commit 4b5c75bdbb
5 changed files with 218 additions and 1 deletions
@@ -297,6 +297,7 @@
              AW_SERVER_GROUP={{ aw_server_group }}
              AW_WORKTIME_REPORT_BASE={{ aw_worktime_report_base }}
              AW_WORKTIME_TZ={{ aw_worktime_timezone }}
+              AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output
              XDG_DATA_HOME={{ aw_server_data_dir }}/.local/share
              XDG_CONFIG_HOME={{ aw_server_data_dir }}/.config

@@ -632,6 +633,133 @@
            state: started
            daemon_reload: true

+    - name: Настроить IOC enrichment из Hayabusa Sigma
+      when: aw_dlp_ioc_enabled | default(false) | bool
+      block:
+        - name: Установить зависимости IOC refresh
+          ansible.builtin.apt:
+            name:
+              - python3-yaml
+            state: present
+            update_cache: true
+
+        - name: Создать каталог IOC enrichment
+          ansible.builtin.file:
+            path: "{{ aw_dlp_ioc_workdir }}/{{ item }}"
+            state: directory
+            owner: "{{ aw_server_user }}"
+            group: "{{ aw_server_group }}"
+            mode: "0755"
+          loop:
+            - ""
+            - output
+            - tmp
+
+        - name: Установить экстрактор IOC из Sigma
+          ansible.builtin.copy:
+            src: "{{ aw_repo_root }}/scripts/extract_ioc_from_sigma.py"
+            dest: /usr/local/bin/aw-extract-ioc-from-sigma.py
+            owner: root
+            group: root
+            mode: "0755"
+
+        - name: Установить wrapper refresh IOC
+          ansible.builtin.copy:
+            dest: /usr/local/bin/aw-dlp-ioc-refresh.sh
+            owner: root
+            group: root
+            mode: "0755"
+            content: |
+              #!/usr/bin/env bash
+              set -euo pipefail
+              WORKDIR="{{ aw_dlp_ioc_workdir }}"
+              OUTDIR="${WORKDIR}/output"
+              TMPDIR="${WORKDIR}/tmp"
+              RULES_DIR="${TMPDIR}/hayabusa-rules"
+              ZIP_PATH="${TMPDIR}/hayabusa-rules.zip"
+
+              mkdir -p "${OUTDIR}" "${TMPDIR}"
+              rm -rf "${RULES_DIR}"
+              curl -fsSL -o "${ZIP_PATH}" "{{ aw_dlp_ioc_rules_zip_url }}"
+              unzip -q -o "${ZIP_PATH}" -d "${TMPDIR}"
+
+              EXTRACTED_DIR=$(find "${TMPDIR}" -maxdepth 1 -type d -name 'hayabusa-rules-*' | head -n1)
+              if [ -z "${EXTRACTED_DIR}" ]; then
+                echo "ERROR: hayabusa-rules archive extraction failed" >&2
+                exit 1
+              fi
+              mv "${EXTRACTED_DIR}" "${RULES_DIR}"
+
+              /usr/bin/python3 /usr/local/bin/aw-extract-ioc-from-sigma.py \
+                --rules-root "${RULES_DIR}" \
+                --out-dir "${OUTDIR}"
+
+              ln -sfn "${OUTDIR}" "${WORKDIR}/latest"
+
+        - name: Установить systemd unit IOC refresh
+          ansible.builtin.copy:
+            dest: /etc/systemd/system/aw-dlp-ioc-refresh.service
+            owner: root
+            group: root
+            mode: "0644"
+            content: |
+              [Unit]
+              Description=Refresh DLP IOC blacklist from Hayabusa Sigma rules
+              After=network-online.target
+              Wants=network-online.target
+
+              [Service]
+              Type=oneshot
+              ExecStart=/usr/local/bin/aw-dlp-ioc-refresh.sh
+
+        - name: Установить systemd timer IOC refresh
+          ansible.builtin.copy:
+            dest: /etc/systemd/system/aw-dlp-ioc-refresh.timer
+            owner: root
+            group: root
+            mode: "0644"
+            content: |
+              [Unit]
+              Description=Run DLP IOC refresh from Hayabusa rules
+
+              [Timer]
+              OnBootSec={{ aw_dlp_ioc_refresh_on_boot_sec }}
+              OnUnitActiveSec={{ aw_dlp_ioc_refresh_interval }}
+              Persistent=true
+              Unit=aw-dlp-ioc-refresh.service
+
+              [Install]
+              WantedBy=timers.target
+
+        - name: Включить и запустить IOC refresh timer
+          ansible.builtin.systemd:
+            name: aw-dlp-ioc-refresh.timer
+            enabled: true
+            state: started
+            daemon_reload: true
+
+        - name: Выполнить принудительный refresh IOC
+          ansible.builtin.systemd:
+            name: aw-dlp-ioc-refresh.service
+            state: started
+
+        - name: Проверить наличие IOC артефактов после refresh
+          ansible.builtin.stat:
+            path: "{{ aw_dlp_ioc_workdir }}/output/{{ item }}"
+          register: aw_dlp_ioc_artifacts
+          loop:
+            - ioc_blacklist.json
+            - ioc_blacklist.csv
+            - ioc_blacklist.sql
+
+        - name: Assert по IOC артефактам
+          ansible.builtin.assert:
+            that:
+              - item.stat.exists
+              - (item.stat.size | int) > 100
+            fail_msg: "Не сгенерирован IOC артефакт: {{ item.stat.path | default('unknown') }}"
+          loop: "{{ aw_dlp_ioc_artifacts.results }}"
+
    - name: Применить базовые worktime settings (classes)
      ansible.builtin.uri:
        url: "http://127.0.0.1:{{ aw_server_port }}/api/0/settings/classes"
@@ -725,6 +853,37 @@
        - aw_apply_worktime_settings | default(false) | bool
        - (aw_server_landingpage | default('') | string | length) > 0

+    - name: Обновить IOC blacklist из hayabusa (если включено)
+      when:
+        - aw_hayabusa_ioc_refresh_enabled | default(false) | bool
+        - aw_hayabusa_rules_root is defined
+        - aw_hayabusa_ioc_output_dir is defined
+      block:
+        - name: Создать выходную директорию для IOC
+          ansible.builtin.file:
+            path: "{{ aw_hayabusa_ioc_output_dir }}"
+            state: directory
+            mode: "0755"
+            owner: "{{ aw_server_user }}"
+            group: "{{ aw_server_group }}"
+
+        - name: Обновить IOC blacklist из hayabusa правил
+          ansible.builtin.shell:
+            cmd: |
+              set -euo pipefail
+              {{ aw_repo_root }}/scripts/build_dlp_ioc_from_hayabusa.sh \
+                "{{ aw_hayabusa_rules_root }}" \
+                "{{ aw_hayabusa_ioc_output_dir }}"
+          args:
+            chdir: "{{ aw_repo_root }}"
+          register: aw_hayabusa_ioc_refresh_result
+          changed_when: "'IOC artifacts generated' in aw_hayabusa_ioc_refresh_result.stdout"
+
+        - name: Показать результат обновления IOC
+          ansible.builtin.debug:
+            msg: "{{ aw_hayabusa_ioc_refresh_result.stdout }}"
+          when: aw_hayabusa_ioc_refresh_result.stdout is defined
+
  handlers:
    - name: Перезагрузить systemd
      ansible.builtin.systemd:
@@ -22,6 +22,12 @@ aw_server_cors_origins:

 aw_apply_worktime_settings: true

+aw_dlp_ioc_enabled: true
+aw_dlp_ioc_workdir: "/opt/activitywatch/dlp-ioc"
+aw_dlp_ioc_rules_zip_url: "https://github.com/Yamato-Security/hayabusa-rules/archive/refs/heads/main.zip"
+aw_dlp_ioc_refresh_on_boot_sec: "5min"
+aw_dlp_ioc_refresh_interval: "6h"
+
 aw_worktime_from: "08:00"
 aw_worktime_to: "17:00"
 aw_worktime_start_of_day: "{{ aw_worktime_from }}"
@@ -10,6 +10,7 @@ from zoneinfo import ZoneInfo

 AW = "http://127.0.0.1:5600/api/0"
 REPORT_TZ = ZoneInfo(os.environ.get("AW_WORKTIME_TZ", "Europe/Moscow"))
+IOC_DIR = os.environ.get("AW_DLP_IOC_DIR", "/opt/activitywatch/dlp-ioc/output")


 def get(u):
@@ -213,6 +214,32 @@ def render_html(rows):

 class H(BaseHTTPRequestHandler):
    def do_GET(self):
+        if self.path.startswith("/dlp-ioc/"):
+            name = self.path.split("?", 1)[0].rsplit("/", 1)[-1]
+            if name not in {"ioc_blacklist.json", "ioc_blacklist.csv", "ioc_blacklist.sql"}:
+                self.send_response(404)
+                self.end_headers()
+                return
+            path = os.path.join(IOC_DIR, name)
+            if not os.path.isfile(path):
+                self.send_response(404)
+                self.end_headers()
+                return
+            with open(path, "rb") as f:
+                data = f.read()
+            if name.endswith(".json"):
+                ctype = "application/json; charset=utf-8"
+            elif name.endswith(".csv"):
+                ctype = "text/csv; charset=utf-8"
+            else:
+                ctype = "text/plain; charset=utf-8"
+            self.send_response(200)
+            self.send_header("Content-Type", ctype)
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+            return
+
        if not self.path.startswith("/reports/worktime/today"):
            self.send_response(404)
            self.end_headers()
@@ -18,6 +18,26 @@ This adds a safe offline pipeline to preload DLP blacklists from static Sigma in
 - `scripts/extract_ioc_from_sigma.py` (core extractor)
 - `scripts/build_dlp_ioc_from_hayabusa.sh` (wrapper)

+## Production (AW server 10.10.10.13)
+
+IOC enrichment is deployed by `ansible/deploy_aw_server.yml` when `aw_dlp_ioc_enabled=true`.
+
+- systemd service: `aw-dlp-ioc-refresh.service`
+- systemd timer: `aw-dlp-ioc-refresh.timer`
+- refresh interval: `aw_dlp_ioc_refresh_interval` (default `6h`)
+- output dir: `/opt/activitywatch/dlp-ioc/output`
+- HTTP export via existing AW worktime API (`:5610`):
+  - `http://10.10.10.13:5610/dlp-ioc/ioc_blacklist.json`
+  - `http://10.10.10.13:5610/dlp-ioc/ioc_blacklist.csv`
+  - `http://10.10.10.13:5610/dlp-ioc/ioc_blacklist.sql`
+
+Mandatory post-deploy checks in Ansible:
+- `ioc_blacklist.json`
+- `ioc_blacklist.csv`
+- `ioc_blacklist.sql`
+
+Each file must exist and be non-empty, otherwise deploy fails.
+
 ## Run

 ```bash
@@ -50,4 +70,3 @@ bash scripts/build_dlp_ioc_from_hayabusa.sh \

 - This pipeline only creates export artifacts and does not modify running DLP agents.
 - Review and tune false positives before enforcing blocking in production.
-
@@ -95,5 +95,11 @@
        "documentRegex": "(?i)(salary|зарплат|passport|паспорт|договор|contract)"
      }
    ]
+  },
+  "ioc": {
+    "enabled": true,
+    "source": "http://10.10.10.13:5610/dlp-ioc/ioc_blacklist.json",
+    "format": "hayabusa_sigma_v1",
+    "refreshMinutes": 360
  }
 }