From e80ae5fbc361298eb142ffdc772f459ad713feb5 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Mon, 27 Apr 2026 04:41:32 +0300 Subject: [PATCH] feat(pfsense): add AW poller and host group sections --- README.md | 5 + ansible/README.md | 23 ++ ansible/deploy_aw_pfsense_poller.yml | 66 +++++ ansible/deploy_aw_server.yml | 2 + ansible/group_vars/pfsense-poller.example.yml | 30 +++ aw-server/apply_webui_ru_patch.sh | 3 + aw-server/aw-host-groups.json | 30 +++ aw-server/aw-ru-patch.js | 226 +++++++++++++++++- docs/pfsense.md | 59 +++++ .../pfsense-aw-poller.cpython-312.pyc | Bin 0 -> 9059 bytes pfsense/pfsense-aw-poller.example.json | 41 ++++ pfsense/pfsense-aw-poller.py | 205 ++++++++++++++++ pfsense/pfsense-aw-poller.service | 15 ++ 13 files changed, 702 insertions(+), 3 deletions(-) create mode 100644 ansible/deploy_aw_pfsense_poller.yml create mode 100644 ansible/group_vars/pfsense-poller.example.yml create mode 100644 aw-server/aw-host-groups.json create mode 100644 docs/pfsense.md create mode 100644 pfsense/__pycache__/pfsense-aw-poller.cpython-312.pyc create mode 100644 pfsense/pfsense-aw-poller.example.json create mode 100644 pfsense/pfsense-aw-poller.py create mode 100644 pfsense/pfsense-aw-poller.service diff --git a/README.md b/README.md index cfeaaa6..4c9d183 100755 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - `proxmox/` — шаблонные скрипты подготовки и наполнения CT на стороне Proxmox. - `aw-server/` — установочные скрипты, env-шаблон, systemd unit и RU patch для Web UI. - `ansible/` — Ansible-ensemble для автоматизированного сервера (Debian/CT). +- `pfsense/` — внешний poller для pfSense API и systemd unit под Debian/Ubuntu utility VM. - `windows/` — PowerShell toolkit: single-user, domain-users, ensemble orchestration, hardening/recovery, validation, phase-2 DLP telemetry (`aw-dlp-incidents_*`, `aw-dlp-endpoint-signals_*`). - `scripts/quality-gate.sh` — локальный preflight-пайплайн проверок. @@ -37,6 +38,10 @@ - `ansible/deploy_aw_windows_phase2.yml` +Для внешнего pfSense poller'а: + +- `ansible/deploy_aw_pfsense_poller.yml` + Скрипты `proxmox/create-ct.sh` и `proxmox/push-aw-artifacts.sh` по умолчанию читают: - `secrets/deploy.secrets.env` diff --git a/ansible/README.md b/ansible/README.md index d072d1b..b8a82d7 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -5,6 +5,7 @@ - деплой на уже существующий Debian host/CT; - полный цикл с нуля в Proxmox: создание CT + bootstrap + установка ActivityWatch + RU patch. - централизованный деплой Windows phase-2 collectors по WinRM. +- deployment внешнего pfSense poller'а на Debian/Ubuntu utility VM. ## Файлы @@ -12,11 +13,13 @@ - `/home/igor/tmp/AWatch-rus/ansible/provision_proxmox_ct_and_deploy_aw.yml` — full-stack playbook для Proxmox. - `/home/igor/tmp/AWatch-rus/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml` — массовый full-stack playbook (несколько CT). - `/home/igor/tmp/AWatch-rus/ansible/deploy_aw_windows_phase2.yml` — WinRM playbook для развёртывания phase-2 Windows collector'ов. +- `/home/igor/tmp/AWatch-rus/ansible/deploy_aw_pfsense_poller.yml` — deployment pfSense poller'а. - `/home/igor/tmp/AWatch-rus/ansible/inventory.example.ini` — шаблон inventory. - `/home/igor/tmp/AWatch-rus/ansible/group_vars/all.example.yml` — шаблон переменных. - `/home/igor/tmp/AWatch-rus/ansible/group_vars/proxmox.example.yml` — шаблон переменных CT в Proxmox. - `/home/igor/tmp/AWatch-rus/ansible/group_vars/proxmox-matrix.example.yml` — шаблон матрицы CT. - `/home/igor/tmp/AWatch-rus/ansible/group_vars/windows.example.yml` — шаблон переменных Windows phase-2. +- `/home/igor/tmp/AWatch-rus/ansible/group_vars/pfsense-poller.example.yml` — шаблон переменных pfSense poller'а. ## Быстрый запуск @@ -86,11 +89,31 @@ Playbook: - `aw_windows_incident_artifacts_root: 'C:\...\incident-artifacts'` — переопределить путь артефактов; - `aw_windows_skip_hardening: true` — пропустить `hardening-recovery.ps1` внутри ensemble-скрипта. +## pfSense poller rollout + +1. Подготовьте vars: + - `cp /home/igor/tmp/AWatch-rus/ansible/group_vars/pfsense-poller.example.yml /home/igor/tmp/AWatch-rus/ansible/group_vars/pfsense-poller.yml` +2. Добавьте inventory group `[aw_pfsense_pollers]`. +3. Запустите: + +```bash +cd /home/igor/tmp/AWatch-rus/ansible +ansible-playbook -i inventory.ini deploy_aw_pfsense_poller.yml +``` + +Playbook: + +- ставит `python3`; +- копирует `pfsense-aw-poller.py`; +- пишет `/etc/aw-pfsense/poller.json`; +- поднимает `aw-pfsense-poller.service`. + ## Результат - Установлен ActivityWatch Server. - Создан systemd-unit `activitywatch-server.service`. - Установлен RU Web UI patch. - Для Web UI используется checksum-based cache-bust для `ru-patch-v5.js` и `sw-cleanup.js`, чтобы браузер не держал старую DLP/русскую статику после деплоя. +- На `#/home` Web UI делит хосты на `Windows RDP` и `Virtual servers + Proxmox`. - Выполнена валидация API `http://127.0.0.1:5600/api/0/info`. - Для full-stack сценария CT создаётся автоматически через `pct create`. diff --git a/ansible/deploy_aw_pfsense_poller.yml b/ansible/deploy_aw_pfsense_poller.yml new file mode 100644 index 0000000..3eea27c --- /dev/null +++ b/ansible/deploy_aw_pfsense_poller.yml @@ -0,0 +1,66 @@ +--- +- name: Deploy pfSense ActivityWatch poller + hosts: aw_pfsense_pollers + become: true + gather_facts: true + + vars: + aw_pfsense_install_root: "/opt/aw-pfsense" + aw_pfsense_config_dir: "/etc/aw-pfsense" + aw_pfsense_service_name: "aw-pfsense-poller.service" + + tasks: + - name: Install required packages + ansible.builtin.apt: + name: + - python3 + state: present + update_cache: true + + - name: Ensure directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - "{{ aw_pfsense_install_root }}" + - "{{ aw_pfsense_config_dir }}" + + - name: Install pfSense poller script + ansible.builtin.copy: + src: "{{ aw_repo_root }}/pfsense/pfsense-aw-poller.py" + dest: "{{ aw_pfsense_install_root }}/pfsense-aw-poller.py" + mode: "0755" + + - name: Install systemd service + ansible.builtin.copy: + src: "{{ aw_repo_root }}/pfsense/pfsense-aw-poller.service" + dest: "/etc/systemd/system/{{ aw_pfsense_service_name }}" + mode: "0644" + notify: + - Reload systemd + + - name: Write pfSense poller config + ansible.builtin.copy: + dest: "{{ aw_pfsense_config_dir }}/poller.json" + mode: "0600" + content: "{{ aw_pfsense_poller_config | to_nice_json }}" + notify: + - Restart pfSense poller + + - name: Enable and start pfSense poller + ansible.builtin.systemd: + name: "{{ aw_pfsense_service_name }}" + enabled: true + state: restarted + daemon_reload: true + + handlers: + - name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + + - name: Restart pfSense poller + ansible.builtin.systemd: + name: "{{ aw_pfsense_service_name }}" + state: restarted diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index bb558e3..d5f7fda 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -11,6 +11,7 @@ aw_bootstrap_dir: "/tmp/aw-rus-bootstrap" aw_ru_patch_cache_bust: "{{ lookup('file', aw_repo_root + '/aw-server/aw-ru-patch.js') | hash('sha1') | truncate(12, true, '') }}" aw_sw_cleanup_cache_bust: "{{ lookup('file', aw_repo_root + '/aw-server/aw-sw-cleanup.js') | hash('sha1') | truncate(12, true, '') }}" + aw_host_groups_cache_bust: "{{ lookup('file', aw_repo_root + '/aw-server/aw-host-groups.json') | hash('sha1') | truncate(12, true, '') }}" aw_worktime_classes: "{{ lookup('file', aw_repo_root + '/aw-server/settings/classes-worktime.json') | from_json }}" aw_default_views: "{{ lookup('file', aw_repo_root + '/aw-server/settings/views-default.json') | from_json }}" @@ -92,6 +93,7 @@ - { src: "{{ aw_repo_root }}/aw-server/activitywatch-server.service", dest: "/etc/systemd/system/activitywatch-server.service", mode: "0644" } - { src: "{{ aw_repo_root }}/aw-server/aw-ru-patch.js", dest: "{{ aw_server_webui_dir }}/js/ru-patch-v5.js", mode: "0644" } - { src: "{{ aw_repo_root }}/aw-server/aw-sw-cleanup.js", dest: "{{ aw_server_webui_dir }}/js/sw-cleanup.js", mode: "0644" } + - { src: "{{ aw_repo_root }}/aw-server/aw-host-groups.json", dest: "{{ aw_server_webui_dir }}/js/aw-host-groups.json", mode: "0644" } notify: - Reload systemd - Restart activitywatch diff --git a/ansible/group_vars/pfsense-poller.example.yml b/ansible/group_vars/pfsense-poller.example.yml new file mode 100644 index 0000000..87aabaa --- /dev/null +++ b/ansible/group_vars/pfsense-poller.example.yml @@ -0,0 +1,30 @@ +aw_pfsense_poller_config: + poll_interval_seconds: 60 + aw: + server_host: "10.10.10.13" + server_port: 5600 + hostname: "PFSENSE-EDGE01" + pulse_time_seconds: 120 + timeout_seconds: 15 + pfsense: + name: "pfSense Edge 01" + host: "10.10.10.1" + scheme: "https" + verify_tls: false + timeout_seconds: 15 + headers: + X-API-Key: "replace-me" + X-API-Secret: "replace-me" + endpoints: + - name: "system-status" + path: "/api/v2/status/system" + bucket_prefix: "aw-pfsense-health" + bucket_type: "aw.pfsense.health" + - name: "interfaces" + path: "/api/v2/interface" + bucket_prefix: "aw-pfsense-interfaces" + bucket_type: "aw.pfsense.interfaces" + - name: "gateways" + path: "/api/v2/status/gateways" + bucket_prefix: "aw-pfsense-gateways" + bucket_type: "aw.pfsense.gateways" diff --git a/aw-server/apply_webui_ru_patch.sh b/aw-server/apply_webui_ru_patch.sh index 6ea909e..d35c71e 100755 --- a/aw-server/apply_webui_ru_patch.sh +++ b/aw-server/apply_webui_ru_patch.sh @@ -12,6 +12,7 @@ source "$ENV_FILE" WEBUI_DIR="${AW_SERVER_WEBUI_DIR:-/opt/activitywatch/webui-ru}" PATCH_JS_SRC="/root/bootstrap/aw-ru-patch.js" SW_CLEANUP_SRC="/root/bootstrap/aw-sw-cleanup.js" +HOST_GROUPS_SRC="/root/bootstrap/aw-host-groups.json" INDEX_HTML="$WEBUI_DIR/index.html" SERVICE_WORKER="$WEBUI_DIR/service-worker.js" TS=$(date +%Y%m%d%H%M%S) @@ -22,11 +23,13 @@ TIMESPIRAL_REPLACEMENT='start:new Date(Date.now()-12*36e5)' [[ -f "$PATCH_JS_SRC" ]] || { echo "missing $PATCH_JS_SRC" >&2; exit 1; } [[ -f "$SW_CLEANUP_SRC" ]] || { echo "missing $SW_CLEANUP_SRC" >&2; exit 1; } +[[ -f "$HOST_GROUPS_SRC" ]] || { echo "missing $HOST_GROUPS_SRC" >&2; exit 1; } [[ -f "$INDEX_HTML" ]] || { echo "missing $INDEX_HTML" >&2; exit 1; } install -d "$WEBUI_DIR/js" install -m 0644 "$PATCH_JS_SRC" "$WEBUI_DIR/js/aw-ru-patch.js" install -m 0644 "$SW_CLEANUP_SRC" "$WEBUI_DIR/js/aw-sw-cleanup.js" +install -m 0644 "$HOST_GROUPS_SRC" "$WEBUI_DIR/js/aw-host-groups.json" cp "$INDEX_HTML" "$INDEX_HTML.bak.$TS" sed -i '/aw-ru-patch.js/d;/aw-sw-cleanup.js/d' "$INDEX_HTML" diff --git a/aw-server/aw-host-groups.json b/aw-server/aw-host-groups.json new file mode 100644 index 0000000..fbc2e9c --- /dev/null +++ b/aw-server/aw-host-groups.json @@ -0,0 +1,30 @@ +{ + "groups": [ + { + "id": "windows-rdp", + "name": "Windows RDP", + "description": "Пользовательские Windows/RDP хосты с активностью, DLP и рабочим временем.", + "patterns": [ + "^(SHARKON|WIN|RDP|TERM|TS-|WS-)" + ], + "links": [ + { "label": "Активность", "type": "activity" }, + { "label": "DLP", "type": "bucket", "bucket_prefix": "aw-dlp-endpoint-signals_" } + ] + }, + { + "id": "virtual-infra", + "name": "Virtual servers + Proxmox", + "description": "Инфраструктурные VM и сетевые узлы. Здесь должны лежать Proxmox, pfSense, Debian и Ubuntu серверы.", + "patterns": [ + "^(PFSENSE|PVE|PROXMOX|DEBIAN|UBUNTU|LINUX|VM-|SRV-|INFRA-)" + ], + "links": [ + { "label": "pfSense health", "type": "bucket", "bucket_prefix": "aw-pfsense-health_" }, + { "label": "pfSense gateways", "type": "bucket", "bucket_prefix": "aw-pfsense-gateways_" }, + { "label": "Все бакеты", "type": "buckets" } + ] + } + ], + "ungrouped_name": "Прочие хосты" +} diff --git a/aw-server/aw-ru-patch.js b/aw-server/aw-ru-patch.js index e0a690d..2111486 100755 --- a/aw-server/aw-ru-patch.js +++ b/aw-server/aw-ru-patch.js @@ -1,6 +1,6 @@ (function () { - window.__awRuPatchVersion = "template-v4-dlp-review"; - document.documentElement.setAttribute("data-aw-ru-patch", "template-v4-dlp-review"); + window.__awRuPatchVersion = "template-v5-dlp-review-host-groups"; + document.documentElement.setAttribute("data-aw-ru-patch", "template-v5-dlp-review-host-groups"); const exact = new Map([ ["ActivityWatch", "АктивВотч"], @@ -322,7 +322,17 @@ '.aw-ru-dlp-message { margin-top: 8px; font-size: 12px; }', '.aw-ru-dlp-actions { display: flex; gap: 6px; flex-wrap: wrap; }', '.aw-ru-dlp-section { margin-top: 18px; }', - '.aw-ru-dlp-section h5 { margin: 0 0 8px; }' + '.aw-ru-dlp-section h5 { margin: 0 0 8px; }', + '.aw-ru-host-groups { margin: 16px 0; padding: 16px; border: 1px solid rgba(120,120,120,.35); border-radius: 8px; background: rgba(20,20,20,.03); }', + '.aw-ru-host-groups-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; }', + '.aw-ru-host-group-card { border: 1px solid rgba(120,120,120,.25); border-radius: 8px; padding: 12px; background: rgba(255,255,255,.02); }', + '.aw-ru-host-group-card h4 { margin: 0 0 8px; }', + '.aw-ru-host-group-card p { margin: 0 0 12px; font-size: 13px; opacity: .85; }', + '.aw-ru-host-list { display: flex; flex-direction: column; gap: 8px; }', + '.aw-ru-host-item { border: 1px solid rgba(120,120,120,.2); border-radius: 6px; padding: 8px; }', + '.aw-ru-host-item-title { font-weight: 600; margin-bottom: 6px; }', + '.aw-ru-host-links { display: flex; flex-wrap: wrap; gap: 6px; }', + '.aw-ru-host-links a { display: inline-block; padding: 4px 8px; border-radius: 999px; background: rgba(90,140,255,.15); text-decoration: none; }' ].join("\n"); document.head.appendChild(style); } @@ -619,6 +629,215 @@ } } + function isHomeRoute() { + const hash = window.location.hash || ""; + return !hash || /^#\/home(?:[/?#]|$)/i.test(hash); + } + + function getDefaultHostGroupsConfig() { + return { + groups: [ + { + id: "windows-rdp", + name: "Windows RDP", + description: "Пользовательские Windows/RDP хосты.", + patterns: ["^(SHARKON|WIN|RDP|TERM|TS-|WS-)"], + links: [ + { label: "Активность", type: "activity" }, + { label: "DLP", type: "bucket", bucket_prefix: "aw-dlp-endpoint-signals_" } + ] + }, + { + id: "virtual-infra", + name: "Virtual servers + Proxmox", + description: "Инфраструктурные VM, Proxmox и сетевые узлы.", + patterns: ["^(PFSENSE|PVE|PROXMOX|DEBIAN|UBUNTU|LINUX|VM-|SRV-|INFRA-)"], + links: [ + { label: "pfSense health", type: "bucket", bucket_prefix: "aw-pfsense-health_" }, + { label: "pfSense gateways", type: "bucket", bucket_prefix: "aw-pfsense-gateways_" }, + { label: "Все бакеты", type: "buckets" } + ] + } + ], + ungrouped_name: "Прочие хосты" + }; + } + + function getHostGroupsState() { + if (!window.__awRuHostGroupsState) { + window.__awRuHostGroupsState = { + config: null, + buckets: null, + loading: false + }; + } + return window.__awRuHostGroupsState; + } + + async function ensureHostGroupsData() { + const state = getHostGroupsState(); + if (state.loading) return state; + if (state.config && state.buckets) return state; + state.loading = true; + try { + if (!state.config) { + try { + state.config = await awApiJson("/js/aw-host-groups.json?v=" + encodeURIComponent(window.__awRuPatchVersion), { method: "GET", headers: {} }); + } catch (error) { + state.config = getDefaultHostGroupsConfig(); + } + } + if (!state.buckets) { + state.buckets = await awApiJson("/api/0/buckets/", { method: "GET", headers: {} }); + } + } finally { + state.loading = false; + } + return state; + } + + function extractHostFromBucket(bucketId, bucketMeta) { + if (bucketMeta && bucketMeta.hostname) return String(bucketMeta.hostname); + const prefixes = [ + "aw-watcher-window_", + "aw-watcher-afk_", + "aw-dlp-endpoint-signals_", + "aw-dlp-incidents_", + "aw-pfsense-health_", + "aw-pfsense-gateways_", + "aw-pfsense-interfaces_" + ]; + for (const prefix of prefixes) { + if (bucketId.indexOf(prefix) === 0) { + return bucketId.slice(prefix.length); + } + } + return ""; + } + + function buildHostBucketMap(rawBuckets) { + const result = new Map(); + const entries = Array.isArray(rawBuckets) + ? rawBuckets.map(function (item) { return [item.id || "", item]; }) + : Object.entries(rawBuckets || {}); + entries.forEach(function (entry) { + const bucketId = entry[0]; + const meta = entry[1] || {}; + const host = extractHostFromBucket(bucketId, meta); + if (!host) return; + if (!result.has(host)) result.set(host, []); + result.get(host).push(bucketId); + }); + return result; + } + + function matchHostGroup(host, groups) { + for (const group of groups) { + const patterns = Array.isArray(group.patterns) ? group.patterns : []; + for (const pattern of patterns) { + try { + if (new RegExp(pattern, "i").test(host)) { + return group.id; + } + } catch (error) { + } + } + } + return ""; + } + + function buildHostLink(host, hostBuckets, linkDef) { + if (!linkDef || !linkDef.type) return ""; + if (linkDef.type === "activity") { + return '#/activity/' + encodeURIComponent(host) + '/day/' + encodeURIComponent(new Date().toISOString().slice(0, 10)) + '/view/summary'; + } + if (linkDef.type === "buckets") { + return "#/buckets"; + } + if (linkDef.type === "bucket" && linkDef.bucket_prefix) { + const bucketId = String(linkDef.bucket_prefix) + host; + return hostBuckets.indexOf(bucketId) >= 0 ? '#/buckets/' + encodeURIComponent(bucketId) : ""; + } + return ""; + } + + function renderHostGroupCards(state) { + const config = state.config || getDefaultHostGroupsConfig(); + const groups = Array.isArray(config.groups) ? config.groups : []; + const hostBuckets = buildHostBucketMap(state.buckets); + const grouped = new Map(); + + groups.forEach(function (group) { + grouped.set(group.id, []); + }); + grouped.set("__ungrouped__", []); + + Array.from(hostBuckets.keys()).sort().forEach(function (host) { + const groupId = matchHostGroup(host, groups) || "__ungrouped__"; + grouped.get(groupId).push(host); + }); + + const cards = []; + groups.forEach(function (group) { + const hosts = grouped.get(group.id) || []; + const items = hosts.map(function (host) { + const links = (group.links || []).map(function (linkDef) { + const href = buildHostLink(host, hostBuckets.get(host) || [], linkDef); + return href ? '' + escapeHtml(linkDef.label || "Открыть") + '' : ""; + }).filter(Boolean).join(""); + return '
' + + '
' + escapeHtml(host) + '
' + + '' + + '
'; + }).join(""); + cards.push( + '
' + + '

' + escapeHtml(group.name || group.id) + '

' + + '

' + escapeHtml(group.description || "") + '

' + + '
' + (items || '
Хосты пока не обнаружены.
') + '
' + + '
' + ); + }); + + const ungroupedHosts = grouped.get("__ungrouped__") || []; + if (ungroupedHosts.length) { + cards.push( + '
' + + '

' + escapeHtml(config.ungrouped_name || "Прочие хосты") + '

' + + '

Хосты, которые пока не попали под шаблоны группировки.

' + + '
' + + ungroupedHosts.map(function (host) { + return '
' + escapeHtml(host) + '
'; + }).join("") + + '
' + + '
' + ); + } + + return cards.join(""); + } + + async function injectHostGroupsCenter(root) { + if (!isHomeRoute()) return; + const heading = root.querySelector("h3"); + if (!heading) return; + + let center = root.querySelector("[data-aw-ru-host-groups='1']"); + if (!center) { + center = document.createElement("section"); + center.className = "aw-ru-host-groups"; + center.setAttribute("data-aw-ru-host-groups", "1"); + center.innerHTML = + '

Разделы хостов

' + + '

Здесь хосты разделены на пользовательские Windows RDP и инфраструктурные виртуальные серверы/Proxmox.

' + + '

Загрузка...

'; + heading.parentElement.insertBefore(center, heading.nextSibling); + } + + const state = await ensureHostGroupsData(); + center.querySelector("[data-aw-ru-host-groups-grid]").innerHTML = renderHostGroupCards(state); + } + function renderDlpTableRows(center, host) { const state = getSuppressionState(); const tbody = center.querySelector("[data-aw-ru-dlp-events]"); @@ -1144,6 +1363,7 @@ injectDlpNavigation(document.body); injectDlpReviewCenter(document.body); injectDlpAlertsCenter(document.body); + injectHostGroupsCenter(document.body).catch(function () {}); redirectBareTrendsRoute(); } diff --git a/docs/pfsense.md b/docs/pfsense.md new file mode 100644 index 0000000..c64f62e --- /dev/null +++ b/docs/pfsense.md @@ -0,0 +1,59 @@ +# pfSense -> ActivityWatch + +Для `pfSense` используется внешний poller на `Debian/Ubuntu` utility VM. Сам firewall не трогаем Windows-агентами и не превращаем в desktop-client. + +## Схема + +- `pfSense` отвечает по API. +- `Debian/Ubuntu` poller VM опрашивает его с интервалом. +- poller отправляет события в общий `AW server`. + +## Bucket'ы + +Базовая конфигурация пишет: + +- `aw-pfsense-health_` +- `aw-pfsense-interfaces_` +- `aw-pfsense-gateways_` + +## Файлы + +- [pfsense-aw-poller.py](/home/igor/tmp/AWatch-rus/pfsense/pfsense-aw-poller.py) +- [pfsense-aw-poller.service](/home/igor/tmp/AWatch-rus/pfsense/pfsense-aw-poller.service) +- [pfsense-aw-poller.example.json](/home/igor/tmp/AWatch-rus/pfsense/pfsense-aw-poller.example.json) +- [deploy_aw_pfsense_poller.yml](/home/igor/tmp/AWatch-rus/ansible/deploy_aw_pfsense_poller.yml) +- [pfsense-poller.example.yml](/home/igor/tmp/AWatch-rus/ansible/group_vars/pfsense-poller.example.yml) + +## Ручной запуск + +```bash +sudo install -d /opt/aw-pfsense /etc/aw-pfsense +sudo cp /path/to/pfsense-aw-poller.py /opt/aw-pfsense/ +sudo chmod 0755 /opt/aw-pfsense/pfsense-aw-poller.py +sudo cp /path/to/poller.json /etc/aw-pfsense/poller.json +python3 /opt/aw-pfsense/pfsense-aw-poller.py --config /etc/aw-pfsense/poller.json --once +``` + +## Через Ansible + +```bash +cd /home/igor/tmp/AWatch-rus/ansible +cp group_vars/pfsense-poller.example.yml group_vars/pfsense-poller.yml +ansible-playbook -i inventory.ini deploy_aw_pfsense_poller.yml +``` + +Inventory: + +```ini +[aw_pfsense_pollers] +aw-poller-01 ansible_host=10.10.10.50 +``` + +## AW Web + +На `#/home` добавлен отдельный блок: + +- `Windows RDP` +- `Virtual servers + Proxmox` + +Для `pfSense` задавай hostname вида `PFSENSE-EDGE01`, тогда он попадёт во вторую группу. diff --git a/pfsense/__pycache__/pfsense-aw-poller.cpython-312.pyc b/pfsense/__pycache__/pfsense-aw-poller.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e232eea433ddab6690f5ba0e5700df358707a47b GIT binary patch literal 9059 zcmb_CX>1!;dNZ6Ea(GHSbW1j6Nw#T6k{us$96OHfSeB0}j;+|SHVI2}Miyy`WZn!T zOBuFGo1#(^!15-o6BCYX`Z zmmOhg*)n3GUv7k>Uw(wAUtz>bzqayMq+d(IK5ieek2^*jEaKr3&T-d>3-bKC%!peO z0D2@Vz&fd3vc1cW)Jr0iypkQDPjUeCOHP0R$px@MaszCXJOG>IW~uJdW`-fn7Dndc zPn+AQexbhUX1REiEoE3{lAUBj-mkDY6tWni6j5b0HZB_?!l_hJ4zUJ*Dx!`>%VX(> z1Bkz`34m*iTE@pPaSLt#4p7arWa8HEG|+;%mbm@#HiKJcCQ7CJ0V%a2Yjfh>A)^abuq_ zU*l|#cK45^#^wIl#T4mR$J6}>&qdVeSTD&a{pnFfPAanb?Tt+Irc;T8O#0H3h7+)b z0e3j2q(~iL^1x5&1u)I5IQ>`l&F)(eZ@8|z3eKKoXJ6jgr}rNzI0x=1pIUV1iBF?> z=gT_(GI7I*Xyx#rAi$3v>>H5ztp|&}Y0?<;pyD>OuKJbOTNbn3{K6q}Lyv@vI@-Gj zu2__l(UcTRUJUUDA6HUILrA0|l0t9_!%pOg6jtTSDpErls6oL+2MKQ5uw&0K4kArZ z_Q6l-0)R(RcV+MF-Vdeuogc)P8@J>ew=5mi8@Ci1_Z6J`b$Nf?b$*sv;xC3!TGNvPpy zDv6snJkc>ZdMP}XQq*Jwn8beg$m!wm&`U!{4Ch-iiH%N%$5WCFjN~X6)r4Z$%EJ|0 zS+IO)-V2#nLJGrdCUz2p=0^A_{Q#z!FKpgooqxsSUG{9wdo~w5Tc(c|J2p%oz00?* zw07L+z21xYrlQj~Jybf#)$E0NdXPVYt3m;mwMwjHiLqImW+7h97GF(w&>53fp%a$T zxOci$#6VNLa#&^!^yW-zs!4T~)~r<(%~5eGa?Mid8%H)+lQc%NYBr6%#osK?oE3iy z7UaV#?KL zux8RI;tB535Twj_8hTUSm!XX$60r-0jmU3iWJNV>r_Btsq|$QI;8745R!Ih7Arss@ z;N+l{bVHtWAy)oEN}41+n02lqZ^M?3OroSPyz=O%993g)$>H*X$rkJ_5?M*7K;{}8 ziA*R?s0&t8S=NlHYPuwAc0)VvO1TYSnkjmGz#lD}^w4p=`Gn3ld|~qx#g=6;m=}YK zO?Smm(dN``O)I{}Yg?{vnIA6rHZ1!>d0!~Eso;Bh#Yna`Co+9>#a?%1-R!z$`}(|n{i1ED>CRT&zP?}|y3bgIZTjhvqNnl7`0V(5 zsWOIh1yAn`_r->u_th&?vs3e3@Be;o=dT3l{3{&s5WyqG1&jX&Vb5V3_m4K=aJ?_5mBA!<>C7+?o;8|?0ks;u3gl{treyqtFA>Bx zST`jFvK?|ziTsgvKh|EMEF8Pk;6buhP^apI3^j+(hOlLaGm9P zoQ)j#4srz#u1qc^T5*H^&Ja0V8bb!5{VDh<0{}ob@$RCtai(`}@`sIo+5Us}<<1@X z&K*m~?i|uPcN990>cK%>JXWl0xYyWO46H9U2OrpYkK?|B@is&GKm88D`nc4s+}K*f z=>O}+;J!GxH$f8M-cYu!y)Uu{wnv^tuou8oLq9MXQ7?c(JCM#KKy-ja*iZIhhZhhW z1Q4>47cqYXfFa0l;gvdQ?=W_+UZH|&rF6=rL-v4`Bq1M{rXVMw9p_aJ0r=mY^5Ed+*Q56J)sA0>VhoX8*#v`;q6&~53P6uXn1Hm8PoWh_-*EBa_$K| z#7d6=$u2PoOIFZ?nw-x|Ht=y!Za9G3#ZvxbuBO@TG_jiBI11x6NphL3U|F~|>)L&V zJVRxL5Xq=xh6^Z=$Rw<$E`cHufYoEsDcd1h88qAnp-+m$rl_hj#Ec>n%JE`4q9_w7 zA`ujJkJ!Cb#cs;akMgZ{`k$}#|;c8_mv_qDxO z_s+)({>`8H`*Md0o_Og~m+;RD*haT7&N8Q}6YtLSNc7Eipe{=4Y zzdrxT`K775y@$RO8;bRf%k{y0eQ@zeZcDyw>Gc{<;D zT0b*V==@%x{F0 zWk;)XR2?YrIGWXpfe6?IWEB%{9KD-03Cuie71RG|wYCKgR=aA}-!z}vU@oA#YB)0y z{9`-rI^-Do>S^dQ0NUDB)`65=z#il;$&(7`a3EsJPzIkwAD=UXm@1DeV6jEU(+N3D z73>8-5}SiN4q3?akT-<4B8iL)CZ8jw#FB~{Nk(OI7F(>~2dlC~B3R%Oa*|q@WD^$o zL=60Hia-ogJsu8&pQVxzOK_d5eMzs986S^;9FW6hbCHZgm($n{=>&Mp=AGNLu=B1h z_=`a6{4+Q9UElZ7^M$~c+>YEKJK|&d=I*-Z@(fi;YJtJYaD#-iEnzAGnw6 zH|Fa%u6P4bXrGtn72Vqflew%gnW?%^U6~1~mzr~%0itg!u}E8z1!2}|mTNg?5*TVG z-GT#Grb@|5M~a|MsBj18#s&gjL}9GiK_K&4N7Y#|@td@jb=sL#yy?)`nlsxahdHz6 zq$Y>cG!e5dnAt_w;MUwQvs)8kRxayND`qU1DV`@__Gs=3=Gt>ZB&^ay1T$5*S;ot* zk`Jc*ki3!)oW}C0*4&F<6HRYgxfvw5>MSKlnPn5#Fx^i0Y$1}?i@^GK~y8%mR|RT3{=Z<1j3TT7C4 zGo(9Pui2$`tv+7EwN){`Lz7l-QjyfLz(+0TF~Cs&d-Z3^-w9T2Hv0nB7K(LU&GG9O zxLV^UOs6K#7-#V4V1TfJI1Gee1at-!!xEV&`2rX`nd0{L_fzM?;Eo;{4wW5| zE@-M+v4*?sN2G~78oO+;VX_}P97P1)$V6XxzCQ4yNi0ew<&?KvS&amO@PsH32*)%I zo&XTy2ri=J{Ll&kMPr6U!M#=^1kDo&wMpqz3@mJ0N!U$NDF`yyjX=(rnJ zaN}u=uhgN806UO(0fgM-9OluwDS2$@JJC2NmB!S_r(PWSBWN(}gTuq8j^JG653yr{ zRvx(=mD3ow2zki}R?}6WwPZNJ*DD7mz|9yn^VCVi%cc=F@+JpMPF(toYiBuDYVX?b^Pp`xeDqPr<*f7-%a7))kw( zzHwUpjv3p157W}QxIW*!ea2mE>Yht2zPxyOF|G%;&)8SM!gDnhT>t}DPR^d3Z(iJA zaBVkB9s1_y3a))nEV>$@<2Q98D60U|+VfFIvAN}5Ywx}Gu44O!#b@*Fz02*}^X=Q0 zHs0P`Xg>@}tIY=%W=Cgn-NqY<>xtZ+rB@2;o-4L@-#B^wWUg_kqtJc;{NE1WHx8z~ z7tW(?SFvrwgBEv#=Yh@Zb<7CgbTf8XTWNXT@|yIP-rQSD(kI$-|ABn}0sVzx{p|Td z|4700z498}HoN1#k7?;BwrnUiw!$1`E&E?Elgo#Z^krvf-r4z(u?miR&F#g;4aGpq zwc}ThuQayI_k0k4U>E8gzhnfvT@4Qzq1>^^Mq-=+)|9kH6 zQ!j2~{%KpsfS3EJyXDAJ+)p>Mn19NJaF1ue#r@2RH9vC+0KXHcN1K%D@;?9qs_Jrx zL8EiWp-u#S$Ui~O3zF58;LywVv5>W_($lpyjwsp9F{bRza;rvxV~4ZW)Eu?O@~d?J zZyM_aM-P23NR@eoRr+11)LJhxXq(hrf!pj=a;h(a+w^)TSUn;HqAQkuX%>X5^eF>D zTpFU%tT)S=&cwws?|1QxW&@iSoJ|GBP1#WJfeHXiRwfXHFNu<2XsXnt!Bmai10#CM z+8;^B`nMT%Wnom2tr$usDFo{P3>bD(a3~NdLD^__3CXzk;8l0|04dubbm% z+ounK@R&a|p9OZ-+xl{mCClGhe{5aK-jNF1PU!t7KONA0XQoe}fqMGd*{f%D;9OE@ zc)H--F?}3s&(0sqH*8#P*phG9qHo=I`$D1NV8MF`8yn_E=g;RGH!U}A%{Ok%oxI&s zXndjIJqT^SuEpa8U;p%pd!Blbm4Q}d#`(7kfvukfw&%!0h85!Mj93)wu6SlWA7&TN z=3ZTT`_6Iw)QJAt#X{5A=VI(Whn?$AX$5g-Y3g=Zj%yJy5cui7)n@dUKB%g`f+&^%t9P$rM9XthUIG7$igKw+BgQt!K zBS|SZnj#YsA_WyW0S^PR6jULDw%24(O$86)TR}{nq;Io*@B-4P}CH}hgFi1reyZFhIgs!y3Y9 zBBP8^3zE!XmDdo#4lD(~JqO9f%s2#FPhl*a7_NvUg`t(!fn|iXSXSu6YDuVkAE%(^ z8A=5RqZ3$6)!U|4n!(fk@hH{|K}pDR+Tg|`m*IWWN|6$Sr)beGGemPYbma>250ykE zDa4OQVo7ofD*q0C%1;1*6Yg$Uc6aC9-THR*t~&#!5${;$oAP|qydyWP?-|nhrn~%0 z)mkyvuJ3z=)}DoE*s`-d?`$tP*G<3p3wPt3dhPA2Zx`HM(*rAYLutz;33 zr2|Fgfk?%qLVkofSLuPQk6O*8R}8bNe4phG#!vq@{5!MhUzz5IR+c@@J``E@`G=yF^?&Viu-hKCI$7(( zb{l|zll4DruY+9o0LwZap0>~m54-XH28MUdOnt_+6nWS5;XfXHXK;4-y|bV5f%_c8 Ix6;-94_H#oB>(^b literal 0 HcmV?d00001 diff --git a/pfsense/pfsense-aw-poller.example.json b/pfsense/pfsense-aw-poller.example.json new file mode 100644 index 0000000..6ec7097 --- /dev/null +++ b/pfsense/pfsense-aw-poller.example.json @@ -0,0 +1,41 @@ +{ + "poll_interval_seconds": 60, + "aw": { + "server_host": "10.10.10.13", + "server_port": 5600, + "hostname": "PFSENSE-EDGE01", + "pulse_time_seconds": 120, + "timeout_seconds": 15 + }, + "pfsense": { + "name": "pfSense Edge 01", + "host": "10.10.10.1", + "scheme": "https", + "verify_tls": false, + "timeout_seconds": 15, + "headers": { + "X-API-Key": "replace-me", + "X-API-Secret": "replace-me" + } + }, + "endpoints": [ + { + "name": "system-status", + "path": "/api/v2/status/system", + "bucket_prefix": "aw-pfsense-health", + "bucket_type": "aw.pfsense.health" + }, + { + "name": "interfaces", + "path": "/api/v2/interface", + "bucket_prefix": "aw-pfsense-interfaces", + "bucket_type": "aw.pfsense.interfaces" + }, + { + "name": "gateways", + "path": "/api/v2/status/gateways", + "bucket_prefix": "aw-pfsense-gateways", + "bucket_type": "aw.pfsense.gateways" + } + ] +} diff --git a/pfsense/pfsense-aw-poller.py b/pfsense/pfsense-aw-poller.py new file mode 100644 index 0000000..5f96ba8 --- /dev/null +++ b/pfsense/pfsense-aw-poller.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +import argparse +import json +import ssl +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +def utc_now_iso(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def load_json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def build_ssl_context(verify_tls): + if verify_tls: + return None + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + +def http_json(url, method="GET", headers=None, body=None, timeout=15, ssl_context=None): + payload = None + effective_headers = {"Content-Type": "application/json; charset=utf-8"} + if headers: + effective_headers.update(headers) + if body is not None: + payload = json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=payload, headers=effective_headers, method=method) + with urllib.request.urlopen(request, timeout=timeout, context=ssl_context) as response: + raw = response.read().decode("utf-8") + return json.loads(raw) if raw else None + + +def ensure_bucket(aw_base_url, bucket_id, client_name, bucket_type, hostname, timeout, ssl_context): + http_json( + f"{aw_base_url}/buckets/{urllib.parse.quote(bucket_id, safe='')}", + method="POST", + body={ + "client": client_name, + "type": bucket_type, + "hostname": hostname, + }, + timeout=timeout, + ssl_context=ssl_context, + ) + + +def send_heartbeat(aw_base_url, bucket_id, event, pulse_time, timeout, ssl_context): + http_json( + f"{aw_base_url}/buckets/{urllib.parse.quote(bucket_id, safe='')}/heartbeat?pulsetime={pulse_time}", + method="POST", + body=event, + timeout=timeout, + ssl_context=ssl_context, + ) + + +def normalize_headers(config): + headers = dict(config.get("headers") or {}) + auth = config.get("auth") or {} + bearer_token = auth.get("bearer_token") + basic = auth.get("basic") + if bearer_token: + headers["Authorization"] = f"Bearer {bearer_token}" + elif basic and basic.get("username") and basic.get("password"): + import base64 + token = base64.b64encode(f"{basic['username']}:{basic['password']}".encode("utf-8")).decode("ascii") + headers["Authorization"] = f"Basic {token}" + return headers + + +def summarize_payload(payload): + if isinstance(payload, dict): + return { + "keys": sorted(payload.keys())[:50], + "size": len(json.dumps(payload, ensure_ascii=False)), + } + if isinstance(payload, list): + return { + "items": len(payload), + "sample_type": type(payload[0]).__name__ if payload else "none", + "size": len(json.dumps(payload, ensure_ascii=False)), + } + return { + "type": type(payload).__name__, + "value": str(payload)[:400], + } + + +def poll_endpoint(config, endpoint, aw_base_url, aw_timeout, pf_timeout, ssl_context): + pf_host = config["pfsense"]["host"] + scheme = config["pfsense"].get("scheme", "https") + hostname = config["aw"]["hostname"] + path = endpoint["path"] + url = f"{scheme}://{pf_host}{path}" + headers = normalize_headers(config["pfsense"]) + + try: + payload = http_json( + url, + method=endpoint.get("method", "GET"), + headers=headers, + timeout=pf_timeout, + ssl_context=ssl_context, + ) + bucket_prefix = endpoint["bucket_prefix"] + bucket_id = f"{bucket_prefix}_{hostname}" + ensure_bucket( + aw_base_url, + bucket_id, + endpoint.get("client", bucket_prefix), + endpoint.get("bucket_type", "aw.pfsense.metric"), + hostname, + aw_timeout, + ssl_context, + ) + event = { + "timestamp": utc_now_iso(), + "duration": 0, + "data": { + "source": "pfsense-aw-poller", + "target": { + "host": pf_host, + "name": config["pfsense"].get("name", hostname), + }, + "endpoint": { + "path": path, + "method": endpoint.get("method", "GET"), + "name": endpoint.get("name", bucket_prefix), + }, + "summary": summarize_payload(payload), + "payload": payload, + }, + } + send_heartbeat( + aw_base_url, + bucket_id, + event, + endpoint.get("pulse_time_seconds", config["aw"].get("pulse_time_seconds", 120)), + aw_timeout, + ssl_context, + ) + return { + "endpoint": path, + "bucket_id": bucket_id, + "status": "ok", + } + except urllib.error.HTTPError as error: + return { + "endpoint": path, + "status": "http_error", + "code": error.code, + "reason": str(error), + } + except Exception as error: + return { + "endpoint": path, + "status": "error", + "reason": str(error), + } + + +def run_once(config): + aw = config["aw"] + aw_base_url = f"http://{aw['server_host']}:{aw.get('server_port', 5600)}/api/0" + aw_timeout = int(aw.get("timeout_seconds", 15)) + pf_timeout = int(config["pfsense"].get("timeout_seconds", 15)) + verify_tls = bool(config["pfsense"].get("verify_tls", False)) + ssl_context = build_ssl_context(verify_tls) + + results = [] + for endpoint in config.get("endpoints") or []: + results.append(poll_endpoint(config, endpoint, aw_base_url, aw_timeout, pf_timeout, ssl_context)) + return results + + +def main(): + parser = argparse.ArgumentParser(description="Poll pfSense API and forward selected telemetry to ActivityWatch.") + parser.add_argument("--config", required=True, help="Path to JSON config.") + parser.add_argument("--once", action="store_true", help="Run one cycle and exit.") + args = parser.parse_args() + + config = load_json(args.config) + interval = int(config.get("poll_interval_seconds", 60)) + + while True: + results = run_once(config) + print(json.dumps({"timestamp": utc_now_iso(), "results": results}, ensure_ascii=False), flush=True) + if args.once: + return 0 + time.sleep(max(interval, 15)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pfsense/pfsense-aw-poller.service b/pfsense/pfsense-aw-poller.service new file mode 100644 index 0000000..e7becbd --- /dev/null +++ b/pfsense/pfsense-aw-poller.service @@ -0,0 +1,15 @@ +[Unit] +Description=pfSense ActivityWatch poller +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /opt/aw-pfsense/pfsense-aw-poller.py --config /etc/aw-pfsense/poller.json +Restart=always +RestartSec=10 +User=root +Group=root + +[Install] +WantedBy=multi-user.target