From 5c6c23ba0fef2c2d4816a36330034666aff670a4 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Fri, 22 May 2026 08:09:00 +0300 Subject: [PATCH] feat(1c): automate live file telemetry ingestion --- ansible/deploy_aw_windows.yml | 31 ++ ansible/deploy_file_1c_analytics.yml | 256 ++++++++++++++++ ansible/deploy_file_1c_windows_telemetry.yml | 112 +++++++ .../detections/build_entity_timeline.sql | 90 +++--- .../detections/insert_detections.sql | 128 ++++---- .../detections/open_cases_from_detections.sql | 5 +- clickhouse-1c/etl/load_1c_exports.py | 4 +- clickhouse-1c/ops/aw-1c-ingest.service | 11 + clickhouse-1c/ops/aw-1c-ingest.timer | 11 + clickhouse-1c/ops/aw-1c-proofcheck.service | 9 + clickhouse-1c/ops/aw-1c-proofcheck.timer | 11 + clickhouse-1c/ops/bootstrap_runtime.sh | 24 ++ clickhouse-1c/ops/check_ingest_freshness.sh | 48 +++ clickhouse-1c/ops/run_ingest_cycle.sh | 51 ++++ windows/ActivityWatch.Windows.Common.psm1 | 64 ++++ windows/deploy-domain-users.ps1 | 14 + windows/deploy-ensemble.ps1 | 10 + windows/export-upload-file-1c-telemetry.ps1 | 277 ++++++++++++++++++ windows/hardening-recovery.ps1 | 14 + 19 files changed, 1070 insertions(+), 100 deletions(-) create mode 100644 ansible/deploy_file_1c_analytics.yml create mode 100644 ansible/deploy_file_1c_windows_telemetry.yml create mode 100644 clickhouse-1c/ops/aw-1c-ingest.service create mode 100644 clickhouse-1c/ops/aw-1c-ingest.timer create mode 100644 clickhouse-1c/ops/aw-1c-proofcheck.service create mode 100644 clickhouse-1c/ops/aw-1c-proofcheck.timer create mode 100644 clickhouse-1c/ops/bootstrap_runtime.sh create mode 100644 clickhouse-1c/ops/check_ingest_freshness.sh create mode 100644 clickhouse-1c/ops/run_ingest_cycle.sh create mode 100644 windows/export-upload-file-1c-telemetry.ps1 diff --git a/ansible/deploy_aw_windows.yml b/ansible/deploy_aw_windows.yml index 5cb2dc0..7f44e56 100644 --- a/ansible/deploy_aw_windows.yml +++ b/ansible/deploy_aw_windows.yml @@ -35,6 +35,10 @@ aw_windows_hayabusa_auto_upload_hours_back: 6 aw_windows_hayabusa_auto_upload_mode: "incident" aw_windows_hayabusa_auto_upload_task_name: "ActivityWatch Hayabusa Upload" + aw_windows_file_1c_auto_upload_enabled: true + aw_windows_file_1c_auto_upload_interval_hours: 6 + aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" + aw_windows_file_1c_target_user: "igor" aw_windows_afk_enabled_default: true aw_windows_window_enabled_default: true aw_windows_file_ops_enabled: true @@ -69,6 +73,10 @@ ansible.builtin.set_fact: aw_server_inventory_host_effective: "{{ (groups['aw_server'] | default([]) | first) | default('', true) }}" + - name: Вычислить inventory host analytics node по умолчанию + ansible.builtin.set_fact: + aw_analytics_inventory_host_effective: "{{ (groups['proxmox'] | default([]) | first) | default('', true) }}" + - name: Вычислить effective host для AW server ansible.builtin.set_fact: aw_windows_server_host_effective: >- @@ -93,6 +101,22 @@ | default(aw_windows_server_host_effective, true) }} + - name: Вычислить effective host для file-1C analytics + ansible.builtin.set_fact: + aw_windows_file_1c_target_host_effective: >- + {{ + aw_windows_file_1c_target_host + | default( + ( + hostvars[aw_analytics_inventory_host_effective].ansible_host + | default(aw_analytics_inventory_host_effective, true) + ) + if (aw_analytics_inventory_host_effective | length) > 0 + else '', + true + ) + }} + - name: Проверить обязательные переменные ansible.builtin.assert: that: @@ -103,6 +127,7 @@ - aw_windows_users_effective | length > 0 - aw_windows_install_root is defined - aw_windows_state_root is defined + - (not (aw_windows_file_1c_auto_upload_enabled | bool)) or (aw_windows_file_1c_target_host_effective | length > 0) fail_msg: "Не заданы обязательные переменные Windows-развёртывания." - name: Нормализовать effective флаги collector'ов и smoke-check @@ -134,6 +159,7 @@ - worktime-session-collector.ps1 - export-evtx-for-hayabusa.ps1 - export-upload-hayabusa-to-aw-server.ps1 + - export-upload-file-1c-telemetry.ps1 - migrate-awatch-rus-paths.ps1 - deploy-domain-users.ps1 - deploy-ensemble.ps1 @@ -219,6 +245,11 @@ HayabusaAutoUploadHoursBack = {{ aw_windows_hayabusa_auto_upload_hours_back | int }} HayabusaAutoUploadMode = "{{ aw_windows_hayabusa_auto_upload_mode }}" HayabusaAutoUploadTaskName = "{{ aw_windows_hayabusa_auto_upload_task_name }}" + File1CAutoUploadEnabled = {{ '$true' if (aw_windows_file_1c_auto_upload_enabled | bool) else '$false' }} + File1CAutoUploadIntervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }} + File1CAutoUploadTaskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" + File1CTargetHost = "{{ aw_windows_file_1c_target_host_effective }}" + File1CTargetUser = "{{ aw_windows_file_1c_target_user }}" CustomRulesPath = "{{ aw_windows_rules_path }}" CustomPolicyPath = "{{ aw_windows_policy_path }}" } diff --git a/ansible/deploy_file_1c_analytics.yml b/ansible/deploy_file_1c_analytics.yml new file mode 100644 index 0000000..dc18d0c --- /dev/null +++ b/ansible/deploy_file_1c_analytics.yml @@ -0,0 +1,256 @@ +--- +- name: Развернуть file-1C analytics backend + hosts: proxmox + become: true + gather_facts: true + + vars: + aw_file_1c_repo_root: "{{ playbook_dir | dirname }}" + aw_file_1c_release_root: /opt/activitywatch/releases + aw_file_1c_release_dir: "{{ aw_file_1c_release_root }}/clickhouse-1c" + aw_file_1c_root: /opt/activitywatch/clickhouse-1c + aw_file_1c_clickhouse_db: analytics_1c + aw_file_1c_clickhouse_user: default + aw_file_1c_clickhouse_password: change-me + aw_file_1c_clickhouse_port: 8123 + aw_file_1c_clickhouse_native_port: 9000 + aw_file_1c_grafana_admin_user: admin + aw_file_1c_grafana_admin_password: change-me + aw_file_1c_grafana_port: 3300 + aw_file_1c_windows_upload_pubkey_path: /tmp/awops_ed25519.pub + + tasks: + - name: Установить базовые пакеты file-1C analytics + ansible.builtin.apt: + name: + - docker.io + - docker-compose + - python3-venv + - python3-pip + state: present + update_cache: true + + - name: Включить и запустить docker + ansible.builtin.systemd: + name: docker + enabled: true + state: started + + - name: Создать release каталоги file-1C analytics + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: igor + group: igor + mode: "0755" + loop: + - "{{ aw_file_1c_release_root }}" + - "{{ aw_file_1c_release_dir }}" + + - name: Скопировать верхнеуровневые файлы stack file-1C analytics + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/{{ item.src }}" + dest: "{{ aw_file_1c_release_dir }}/{{ item.dest }}" + owner: igor + group: igor + mode: "{{ item.mode | default('0644') }}" + loop: + - { src: 'README.md', dest: 'README.md' } + - { src: 'docker-compose.yml', dest: 'docker-compose.yml' } + - { src: '.env.example', dest: '.env.example' } + + - name: Подготовить каталоги stack file-1C analytics + ansible.builtin.file: + path: "{{ aw_file_1c_release_dir }}/{{ item }}" + state: directory + owner: igor + group: igor + mode: "0755" + loop: + - clickhouse + - detections + - etl + - ops + - sample + + - name: Скопировать каталоги stack file-1C analytics + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/{{ item }}/" + dest: "{{ aw_file_1c_release_dir }}/{{ item }}/" + owner: igor + group: igor + mode: preserve + directory_mode: "0755" + loop: + - clickhouse + - detections + - etl + - ops + - sample + + - name: Установить права на исполняемые ops scripts + ansible.builtin.file: + path: "{{ aw_file_1c_release_dir }}/ops/{{ item }}" + owner: igor + group: igor + mode: "0755" + state: file + loop: + - bootstrap_runtime.sh + - check_ingest_freshness.sh + - run_ingest_cycle.sh + + - name: Создать .env для file-1C analytics + ansible.builtin.copy: + dest: "{{ aw_file_1c_release_dir }}/.env" + owner: igor + group: igor + mode: "0600" + content: | + CLICKHOUSE_DB={{ aw_file_1c_clickhouse_db }} + CLICKHOUSE_USER={{ aw_file_1c_clickhouse_user }} + CLICKHOUSE_PASSWORD={{ aw_file_1c_clickhouse_password }} + CLICKHOUSE_PORT={{ aw_file_1c_clickhouse_port }} + CLICKHOUSE_NATIVE_PORT={{ aw_file_1c_clickhouse_native_port }} + GRAFANA_ADMIN_USER={{ aw_file_1c_grafana_admin_user }} + GRAFANA_ADMIN_PASSWORD={{ aw_file_1c_grafana_admin_password }} + GRAFANA_PORT={{ aw_file_1c_grafana_port }} + CLICKHOUSE_HOST=clickhouse + + - name: Создать etl/config.yml для file-1C analytics + ansible.builtin.copy: + dest: "{{ aw_file_1c_release_dir }}/etl/config.yml" + owner: igor + group: igor + mode: "0644" + content: | + clickhouse: + host: localhost + port: {{ aw_file_1c_clickhouse_port }} + username: {{ aw_file_1c_clickhouse_user }} + password: {{ aw_file_1c_clickhouse_password }} + database: {{ aw_file_1c_clickhouse_db }} + + landing: + documents: {{ aw_file_1c_root }}/landing/documents + postings: {{ aw_file_1c_root }}/landing/postings + reglog: {{ aw_file_1c_root }}/landing/reglog + audit: {{ aw_file_1c_root }}/landing/audit + host: {{ aw_file_1c_root }}/landing/host + + formats: + default: jsonl + documents: jsonl + postings: jsonl + reglog: jsonl + audit: jsonl + host: jsonl + + archive_dir: {{ aw_file_1c_root }}/archive + delete_after_load: false + + - name: Создать symlink на активный root file-1C analytics + ansible.builtin.file: + src: "{{ aw_file_1c_release_dir }}" + dest: "{{ aw_file_1c_root }}" + state: link + force: true + + - name: Bootstrap runtime file-1C analytics + ansible.builtin.command: + cmd: "{{ aw_file_1c_root }}/ops/bootstrap_runtime.sh" + environment: + AW_1C_ROOT: "{{ aw_file_1c_root }}" + + - name: Сделать landing/archive writable для igor upload path + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: igor + group: igor + mode: "0755" + recurse: true + loop: + - "{{ aw_file_1c_root }}/landing" + - "{{ aw_file_1c_root }}/archive" + + - name: Поднять ClickHouse для file-1C analytics + ansible.builtin.command: + cmd: docker compose up -d clickhouse + args: + chdir: "{{ aw_file_1c_root }}" + + - name: Установить systemd unit aw-1c-ingest.service + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.service" + dest: /etc/systemd/system/aw-1c-ingest.service + owner: root + group: root + mode: "0644" + notify: Перезагрузить systemd + + - name: Установить systemd unit aw-1c-ingest.timer + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.timer" + dest: /etc/systemd/system/aw-1c-ingest.timer + owner: root + group: root + mode: "0644" + notify: Перезагрузить systemd + + - name: Установить systemd unit aw-1c-proofcheck.service + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-proofcheck.service" + dest: /etc/systemd/system/aw-1c-proofcheck.service + owner: root + group: root + mode: "0644" + notify: Перезагрузить systemd + + - name: Установить systemd unit aw-1c-proofcheck.timer + ansible.builtin.copy: + src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-proofcheck.timer" + dest: /etc/systemd/system/aw-1c-proofcheck.timer + owner: root + group: root + mode: "0644" + notify: Перезагрузить systemd + + - name: Разрешить Windows upload key для igor + ansible.builtin.lineinfile: + path: /home/igor/.ssh/authorized_keys + line: "{{ lookup('file', aw_file_1c_windows_upload_pubkey_path) }}" + create: true + owner: igor + group: igor + mode: "0600" + + - name: Включить и запустить aw-1c-ingest.timer + ansible.builtin.systemd: + name: aw-1c-ingest.timer + enabled: true + state: started + daemon_reload: true + + - name: Включить и запустить aw-1c-proofcheck.timer + ansible.builtin.systemd: + name: aw-1c-proofcheck.timer + enabled: true + state: started + daemon_reload: true + + - name: Проверить доступность ClickHouse ping + ansible.builtin.uri: + url: "http://127.0.0.1:{{ aw_file_1c_clickhouse_port }}/ping" + return_content: true + register: aw_file_1c_ping + changed_when: false + + - name: Показать ping ClickHouse + ansible.builtin.debug: + msg: "{{ aw_file_1c_ping.content }}" + + handlers: + - name: Перезагрузить systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/deploy_file_1c_windows_telemetry.yml b/ansible/deploy_file_1c_windows_telemetry.yml new file mode 100644 index 0000000..b9df33a --- /dev/null +++ b/ansible/deploy_file_1c_windows_telemetry.yml @@ -0,0 +1,112 @@ +--- +- name: Развернуть file-1C telemetry uploader на Windows + hosts: aw_windows + gather_facts: false + + vars: + aw_windows_repo_root: "{{ playbook_dir | dirname }}" + aw_windows_state_root: "C:\\ProgramData\\AWatch-rus" + aw_windows_deploy_root: "C:\\Program Files\\AWatch-rus" + aw_windows_file_1c_target_user: "igor" + aw_windows_file_1c_auto_upload_interval_hours: 6 + aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" + aw_windows_file_1c_remote_root: "/opt/activitywatch/clickhouse-1c/landing" + aw_windows_upload_key_private_path: /tmp/awops_ed25519 + aw_windows_upload_key_public_path: /tmp/awops_ed25519.pub + + tasks: + - name: Вычислить inventory host analytics node по умолчанию + ansible.builtin.set_fact: + aw_analytics_inventory_host_effective: "{{ (groups['proxmox'] | default([]) | first) | default('', true) }}" + + - name: Вычислить effective host для file-1C analytics + ansible.builtin.set_fact: + aw_windows_file_1c_target_host_effective: >- + {{ + aw_windows_file_1c_target_host + | default( + ( + hostvars[aw_analytics_inventory_host_effective].ansible_host + | default(aw_analytics_inventory_host_effective, true) + ) + if (aw_analytics_inventory_host_effective | length) > 0 + else '', + true + ) + }} + + - name: Проверить обязательные переменные file-1C telemetry + ansible.builtin.assert: + that: + - aw_windows_file_1c_target_host_effective | length > 0 + fail_msg: "Не удалось вычислить host file-1C analytics для Windows uploader." + + - name: Создать каталоги file-1C telemetry на Windows + ansible.windows.win_file: + path: "{{ item }}" + state: directory + loop: + - "{{ aw_windows_state_root }}" + - "{{ aw_windows_deploy_root }}\\windows" + - "{{ aw_windows_state_root }}\\ssh" + + - name: Загрузить file-1C telemetry script в toolkit + ansible.windows.win_copy: + src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1" + dest: "{{ aw_windows_deploy_root }}\\windows\\export-upload-file-1c-telemetry.ps1" + + - name: Загрузить file-1C telemetry script в state root + ansible.windows.win_copy: + src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1" + dest: "{{ aw_windows_state_root }}\\export-upload-file-1c-telemetry.ps1" + + - name: Обновить deployment-config.json блоком analytics.file1cAutomation + ansible.windows.win_powershell: + script: | + $ErrorActionPreference = 'Stop' + $configPath = "{{ aw_windows_state_root }}\deployment-config.json" + $config = Get-Content -Raw -LiteralPath $configPath | ConvertFrom-Json + + if ($config.PSObject.Properties.Name -notcontains 'paths') { + $config | Add-Member -NotePropertyName 'paths' -NotePropertyValue ([pscustomobject]@{}) + } + if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryScript') { + $config.paths.file1cTelemetryScript = "{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1" + } else { + $config.paths | Add-Member -NotePropertyName 'file1cTelemetryScript' -NotePropertyValue "{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1" + } + + if ($config.PSObject.Properties.Name -notcontains 'analytics') { + $config | Add-Member -NotePropertyName 'analytics' -NotePropertyValue ([pscustomobject]@{}) + } + + $automation = [pscustomobject]@{ + enabled = $true + intervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }} + taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" + targetHost = "{{ aw_windows_file_1c_target_host_effective }}" + targetUser = "{{ aw_windows_file_1c_target_user }}" + remoteRoot = "{{ aw_windows_file_1c_remote_root }}" + } + + if ($config.analytics.PSObject.Properties.Name -contains 'file1cAutomation') { + $config.analytics.file1cAutomation = $automation + } else { + $config.analytics | Add-Member -NotePropertyName 'file1cAutomation' -NotePropertyValue $automation + } + + $json = $config | ConvertTo-Json -Depth 12 + Set-Content -LiteralPath $configPath -Value $json -Encoding UTF8 + + - name: Создать scheduled task file-1C upload + ansible.windows.win_powershell: + script: | + $ErrorActionPreference = 'Stop' + $taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" + $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $taskCommand = "`"$powerShellExe`" -NoProfile -ExecutionPolicy Bypass -File `"{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1`" -ConfigPath `"{{ aw_windows_state_root }}\deployment-config.json`"" + & cmd.exe /c "schtasks /Delete /TN `"$taskName`" /F >nul 2>&1" | Out-Null + & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO {{ aw_windows_file_1c_auto_upload_interval_hours | int }} /ST 00:00 /RU SYSTEM /RL HIGHEST /F | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Не удалось создать scheduled task $taskName" + } diff --git a/clickhouse-1c/detections/build_entity_timeline.sql b/clickhouse-1c/detections/build_entity_timeline.sql index d93f9ff..578751d 100644 --- a/clickhouse-1c/detections/build_entity_timeline.sql +++ b/clickhouse-1c/detections/build_entity_timeline.sql @@ -1,44 +1,56 @@ INSERT INTO analytics_1c.entity_timeline -SELECT - ts, - 'document' AS entity_type, - doc_id AS entity_id, - infobase, - author AS actor, - 'documents' AS source, - concat('document:', doc_type) AS event_type, - if(posted = 1, 'low', 'medium') AS severity, - if(posted = 1, 5, 20) AS score, - doc_id AS ref_id, - concat('Документ ', doc_type, ' №', doc_number, ' статус=', status) AS summary -FROM analytics_1c.documents; +SELECT * +FROM ( + SELECT + ts, + 'document' AS entity_type, + doc_id AS entity_id, + infobase, + author AS actor, + 'documents' AS source, + concat('document:', doc_type) AS event_type, + if(posted = 1, 'low', 'medium') AS severity, + if(posted = 1, 5, 20) AS score, + doc_id AS ref_id, + concat('Документ ', doc_type, ' №', doc_number, ' статус=', status) AS summary + FROM analytics_1c.documents +) AS src +WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); INSERT INTO analytics_1c.entity_timeline -SELECT - ts, - 'user' AS entity_type, - user AS entity_id, - infobase, - user AS actor, - 'reglog' AS source, - event_name AS event_type, - if(level IN ('error', 'warn'), 'medium', 'low') AS severity, - if(level IN ('error', 'warn'), 25, 5) AS score, - concat(user, ':', toString(toUnixTimestamp(ts))) AS ref_id, - message AS summary -FROM analytics_1c.reglog_events; +SELECT * +FROM ( + SELECT + ts, + 'user' AS entity_type, + user AS entity_id, + infobase, + user AS actor, + 'reglog' AS source, + event_name AS event_type, + if(level IN ('error', 'warn'), 'medium', 'low') AS severity, + if(level IN ('error', 'warn'), 25, 5) AS score, + concat(user, ':', toString(toUnixTimestamp(ts)), ':', event_name) AS ref_id, + message AS summary + FROM analytics_1c.reglog_events +) AS src +WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); INSERT INTO analytics_1c.entity_timeline -SELECT - ts, - object_type AS entity_type, - object_id AS entity_id, - infobase, - user AS actor, - 'audit' AS source, - action AS event_type, - if(risk_tag != '', 'high', 'medium') AS severity, - if(risk_tag != '', 60, 30) AS score, - concat(object_type, ':', object_id, ':', toString(toUnixTimestamp(ts))) AS ref_id, - concat('Audit action ', action, ' risk=', risk_tag) AS summary -FROM analytics_1c.audit_events; +SELECT * +FROM ( + SELECT + ts, + object_type AS entity_type, + object_id AS entity_id, + infobase, + user AS actor, + 'audit' AS source, + action AS event_type, + if(risk_tag != '', 'high', 'medium') AS severity, + if(risk_tag != '', 60, 30) AS score, + concat(object_type, ':', object_id, ':', toString(toUnixTimestamp(ts)), ':', action) AS ref_id, + concat('Audit action ', action, ' risk=', risk_tag) AS summary + FROM analytics_1c.audit_events +) AS src +WHERE src.ref_id NOT IN (SELECT ref_id FROM analytics_1c.entity_timeline); diff --git a/clickhouse-1c/detections/insert_detections.sql b/clickhouse-1c/detections/insert_detections.sql index 3ba5c25..b6eb3ac 100644 --- a/clickhouse-1c/detections/insert_detections.sql +++ b/clickhouse-1c/detections/insert_detections.sql @@ -1,67 +1,79 @@ INSERT INTO analytics_1c.detections -SELECT - ts, - concat('after_hours_login:', infobase, ':', user, ':', toString(toUnixTimestamp(ts))) AS detection_id, - infobase, - 'after_hours_login' AS rule_id, - 'Вход вне рабочего времени' AS rule_title, - 'user' AS entity_type, - user AS entity_id, - 'medium' AS severity, - 35 AS score, - concat('Пользователь ', user, ' выполнил вход вне рабочего времени') AS summary, - 'open' AS status -FROM analytics_1c.reglog_events -WHERE event_name ILIKE '%login%' - AND toHour(ts) NOT BETWEEN 8 AND 20; - -INSERT INTO analytics_1c.detections -SELECT - event_ts AS ts, - concat('failed_login_burst:', infobase, ':', user, ':', toString(toUnixTimestamp(event_ts))) AS detection_id, - infobase, - 'failed_login_burst' AS rule_id, - 'Всплеск ошибок входа' AS rule_title, - 'user' AS entity_type, - user AS entity_id, - 'high' AS severity, - 65 AS score, - concat('У пользователя ', user, ' более 5 ошибок входа за 15 минут') AS summary, - 'open' AS status +SELECT * FROM ( SELECT + ts, + concat('after_hours_login:', infobase, ':', user, ':', toString(toUnixTimestamp(ts))) AS detection_id, infobase, - user, - toStartOfFifteenMinutes(ts) AS window_ts, - max(ts) AS event_ts, - count() AS attempts + 'after_hours_login' AS rule_id, + 'Вход вне рабочего времени' AS rule_title, + 'user' AS entity_type, + user AS entity_id, + 'medium' AS severity, + 35 AS score, + concat('Пользователь ', user, ' выполнил вход вне рабочего времени') AS summary, + 'open' AS status FROM analytics_1c.reglog_events - WHERE level IN ('error', 'warn') - AND (event_name ILIKE '%login%' OR message ILIKE '%парол%' OR message ILIKE '%auth%') - GROUP BY infobase, user, window_ts - HAVING attempts >= 5 -); + WHERE event_name ILIKE '%login%' + AND toHour(ts) NOT BETWEEN 8 AND 20 +) AS src +WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections); INSERT INTO analytics_1c.detections -SELECT - event_ts AS ts, - concat('disk_latency_high:', host, ':', toString(toUnixTimestamp(event_ts))) AS detection_id, - '' AS infobase, - 'disk_latency_high' AS rule_id, - 'Высокая задержка диска' AS rule_title, - 'host' AS entity_type, - host AS entity_id, - 'high' AS severity, - 65 AS score, - concat('На хосте ', host, ' задержка диска превышает 50 мс') AS summary, - 'open' AS status +SELECT * FROM ( SELECT - host, - toStartOfHour(ts) AS hour_ts, - max(ts) AS event_ts, - avg(disk_latency_ms) AS avg_latency_ms - FROM analytics_1c.host_events - GROUP BY host, hour_ts - HAVING avg_latency_ms > 50 -); + event_ts AS ts, + concat('failed_login_burst:', infobase, ':', user, ':', toString(toUnixTimestamp(event_ts))) AS detection_id, + infobase, + 'failed_login_burst' AS rule_id, + 'Всплеск ошибок входа' AS rule_title, + 'user' AS entity_type, + user AS entity_id, + 'high' AS severity, + 65 AS score, + concat('У пользователя ', user, ' более 5 ошибок входа за 15 минут') AS summary, + 'open' AS status + FROM ( + SELECT + infobase, + user, + toStartOfFifteenMinutes(ts) AS window_ts, + max(ts) AS event_ts, + count() AS attempts + FROM analytics_1c.reglog_events + WHERE level IN ('error', 'warn') + AND (event_name ILIKE '%login%' OR message ILIKE '%парол%' OR message ILIKE '%auth%') + GROUP BY infobase, user, window_ts + HAVING attempts >= 5 + ) +) AS src +WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections); + +INSERT INTO analytics_1c.detections +SELECT * +FROM ( + SELECT + event_ts AS ts, + concat('disk_latency_high:', host, ':', toString(toUnixTimestamp(event_ts))) AS detection_id, + '' AS infobase, + 'disk_latency_high' AS rule_id, + 'Высокая задержка диска' AS rule_title, + 'host' AS entity_type, + host AS entity_id, + 'high' AS severity, + 65 AS score, + concat('На хосте ', host, ' задержка диска превышает 50 мс') AS summary, + 'open' AS status + FROM ( + SELECT + host, + toStartOfHour(ts) AS hour_ts, + max(ts) AS event_ts, + avg(disk_latency_ms) AS avg_latency_ms + FROM analytics_1c.host_events + GROUP BY host, hour_ts + HAVING avg_latency_ms > 50 + ) +) AS src +WHERE src.detection_id NOT IN (SELECT detection_id FROM analytics_1c.detections); diff --git a/clickhouse-1c/detections/open_cases_from_detections.sql b/clickhouse-1c/detections/open_cases_from_detections.sql index 2f436f1..9c44bc7 100644 --- a/clickhouse-1c/detections/open_cases_from_detections.sql +++ b/clickhouse-1c/detections/open_cases_from_detections.sql @@ -12,5 +12,8 @@ SELECT entity_id, summary FROM analytics_1c.detections -WHERE severity IN ('high', 'critical') +WHERE ( + severity IN ('high', 'critical') + OR (severity = 'medium' AND score >= 35) + ) AND detection_id NOT IN (SELECT case_id FROM analytics_1c.cases); diff --git a/clickhouse-1c/etl/load_1c_exports.py b/clickhouse-1c/etl/load_1c_exports.py index 4af29ef..e46b39c 100644 --- a/clickhouse-1c/etl/load_1c_exports.py +++ b/clickhouse-1c/etl/load_1c_exports.py @@ -78,9 +78,9 @@ def normalize_ts(value: Any) -> datetime: def iter_rows(path: Path, fmt: str) -> list[dict[str, Any]]: if fmt == "jsonl": - return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + return [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip()] if fmt == "json": - payload = json.loads(path.read_text(encoding="utf-8")) + payload = json.loads(path.read_text(encoding="utf-8-sig")) return payload if isinstance(payload, list) else [payload] if fmt == "csv": with path.open("r", encoding="utf-8-sig", newline="") as fh: diff --git a/clickhouse-1c/ops/aw-1c-ingest.service b/clickhouse-1c/ops/aw-1c-ingest.service new file mode 100644 index 0000000..f4e1993 --- /dev/null +++ b/clickhouse-1c/ops/aw-1c-ingest.service @@ -0,0 +1,11 @@ +[Unit] +Description=AW-rus file-1C ingest cycle +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/activitywatch/clickhouse-1c +ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh +User=root +Group=root diff --git a/clickhouse-1c/ops/aw-1c-ingest.timer b/clickhouse-1c/ops/aw-1c-ingest.timer new file mode 100644 index 0000000..784c145 --- /dev/null +++ b/clickhouse-1c/ops/aw-1c-ingest.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AW-rus file-1C ingest cycle every 15 minutes + +[Timer] +OnBootSec=5min +OnUnitActiveSec=15min +Unit=aw-1c-ingest.service +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/clickhouse-1c/ops/aw-1c-proofcheck.service b/clickhouse-1c/ops/aw-1c-proofcheck.service new file mode 100644 index 0000000..badce7b --- /dev/null +++ b/clickhouse-1c/ops/aw-1c-proofcheck.service @@ -0,0 +1,9 @@ +[Unit] +Description=AW-rus file-1C ingest freshness proof check +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c +ExecStart=/opt/activitywatch/clickhouse-1c/ops/check_ingest_freshness.sh diff --git a/clickhouse-1c/ops/aw-1c-proofcheck.timer b/clickhouse-1c/ops/aw-1c-proofcheck.timer new file mode 100644 index 0000000..ea522e3 --- /dev/null +++ b/clickhouse-1c/ops/aw-1c-proofcheck.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AW-rus file-1C freshness proof check every 6 hours + +[Timer] +OnBootSec=20min +OnUnitActiveSec=6h +Unit=aw-1c-proofcheck.service +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/clickhouse-1c/ops/bootstrap_runtime.sh b/clickhouse-1c/ops/bootstrap_runtime.sh new file mode 100644 index 0000000..8079379 --- /dev/null +++ b/clickhouse-1c/ops/bootstrap_runtime.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" + +mkdir -p \ + "${ROOT}/landing/documents" \ + "${ROOT}/landing/postings" \ + "${ROOT}/landing/reglog" \ + "${ROOT}/landing/audit" \ + "${ROOT}/landing/host" \ + "${ROOT}/archive/documents" \ + "${ROOT}/archive/postings" \ + "${ROOT}/archive/reglog" \ + "${ROOT}/archive/audit" \ + "${ROOT}/archive/host" + +if [[ ! -f "${ROOT}/etl/config.yml" ]]; then + cp "${ROOT}/etl/config.example.yml" "${ROOT}/etl/config.yml" +fi + +python3 -m venv "${ROOT}/.venv" +"${ROOT}/.venv/bin/pip" install --upgrade pip +"${ROOT}/.venv/bin/pip" install -r "${ROOT}/etl/requirements.txt" diff --git a/clickhouse-1c/ops/check_ingest_freshness.sh b/clickhouse-1c/ops/check_ingest_freshness.sh new file mode 100644 index 0000000..b8aebeb --- /dev/null +++ b/clickhouse-1c/ops/check_ingest_freshness.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" +ENV_FILE="${ROOT}/.env" +CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}" +MAX_AGE_HOURS="${AW_1C_MAX_AGE_HOURS:-8}" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "missing env file: ${ENV_FILE}" >&2 + exit 1 +fi + +if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then + echo "clickhouse container not running: ${CH_CONTAINER}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +. "${ENV_FILE}" + +query_max_age() { + local table="$1" + docker exec "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + -q "SELECT if(count()=0, -1, dateDiff('hour', max(ts), now())) FROM ${table}" +} + +documents_age="$(query_max_age documents)" +reglog_age="$(query_max_age reglog_events)" +audit_age="$(query_max_age audit_events)" +host_age="$(query_max_age host_events)" + +printf 'freshness documents=%sh reglog=%sh audit=%sh host=%sh threshold=%sh\n' \ + "${documents_age}" "${reglog_age}" "${audit_age}" "${host_age}" "${MAX_AGE_HOURS}" + +for age in "${documents_age}" "${reglog_age}" "${audit_age}" "${host_age}"; do + if [[ "${age}" == "-1" ]]; then + echo "one or more datasets are empty" >&2 + exit 1 + fi + if (( age > MAX_AGE_HOURS )); then + echo "freshness threshold exceeded" >&2 + exit 1 + fi +done diff --git a/clickhouse-1c/ops/run_ingest_cycle.sh b/clickhouse-1c/ops/run_ingest_cycle.sh new file mode 100644 index 0000000..c023227 --- /dev/null +++ b/clickhouse-1c/ops/run_ingest_cycle.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}" +ENV_FILE="${ROOT}/.env" +VENV="${ROOT}/.venv" +CONFIG="${ROOT}/etl/config.yml" +CH_CONTAINER="${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "missing env file: ${ENV_FILE}" >&2 + exit 1 +fi + +if [[ ! -f "${CONFIG}" ]]; then + echo "missing etl config: ${CONFIG}" >&2 + exit 1 +fi + +if [[ ! -x "${VENV}/bin/python" ]]; then + echo "missing venv python: ${VENV}/bin/python" >&2 + exit 1 +fi + +if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then + echo "clickhouse container not running: ${CH_CONTAINER}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +. "${ENV_FILE}" + +"${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}" + +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/detections/build_entity_timeline.sql" + +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/detections/insert_detections.sql" + +docker exec -i "${CH_CONTAINER}" clickhouse-client \ + --user "${CLICKHOUSE_USER}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --database "${CLICKHOUSE_DB}" \ + < "${ROOT}/detections/open_cases_from_detections.sql" diff --git a/windows/ActivityWatch.Windows.Common.psm1 b/windows/ActivityWatch.Windows.Common.psm1 index f80c1bf..a35f810 100755 --- a/windows/ActivityWatch.Windows.Common.psm1 +++ b/windows/ActivityWatch.Windows.Common.psm1 @@ -401,6 +401,7 @@ function Copy-ActivityWatchCollectorAssets { [string]$SessionCollectorScriptSource, [string]$EvtxExportScriptSource, [string]$HayabusaUploadScriptSource, + [string]$File1CTelemetryScriptSource, [string]$EmailCollectorScriptSource, [Parameter(Mandatory = $true)] [string]$ExampleRulesSource, @@ -421,6 +422,7 @@ function Copy-ActivityWatchCollectorAssets { $sessionCollectorTarget = Join-Path $StateRoot 'worktime-session-collector.ps1' $evtxExportTarget = Join-Path $StateRoot 'export-evtx-for-hayabusa.ps1' $hayabusaUploadTarget = Join-Path $StateRoot 'export-upload-hayabusa-to-aw-server.ps1' + $file1cTelemetryTarget = Join-Path $StateRoot 'export-upload-file-1c-telemetry.ps1' $emailCollectorTarget = Join-Path $StateRoot 'email-outbound-collector.ps1' $exampleRulesTarget = Join-Path $StateRoot 'web-category-rules.example.json' $rulesTarget = Join-Path $StateRoot 'web-category-rules.json' @@ -440,6 +442,9 @@ function Copy-ActivityWatchCollectorAssets { if ($HayabusaUploadScriptSource -and (Test-Path -LiteralPath $HayabusaUploadScriptSource)) { Copy-Item -LiteralPath $HayabusaUploadScriptSource -Destination $hayabusaUploadTarget -Force } + if ($File1CTelemetryScriptSource -and (Test-Path -LiteralPath $File1CTelemetryScriptSource)) { + Copy-Item -LiteralPath $File1CTelemetryScriptSource -Destination $file1cTelemetryTarget -Force + } if ($EmailCollectorScriptSource -and (Test-Path -LiteralPath $EmailCollectorScriptSource)) { Copy-Item -LiteralPath $EmailCollectorScriptSource -Destination $emailCollectorTarget -Force } @@ -470,6 +475,7 @@ function Copy-ActivityWatchCollectorAssets { SessionCollectorScript = $sessionCollectorTarget EvtxExportScript = $evtxExportTarget HayabusaUploadScript = $hayabusaUploadTarget + File1CTelemetryScript = $file1cTelemetryTarget EmailCollectorScript = $emailCollectorTarget ExampleRules = $exampleRulesTarget ActiveRules = $rulesTarget @@ -503,6 +509,7 @@ function New-ActivityWatchDeploymentConfig { [string]$SessionCollectorScript, [string]$EvtxExportScript, [string]$HayabusaUploadScript, + [string]$File1CTelemetryScript, [string]$EmailCollectorScript, [Parameter(Mandatory = $true)] [string]$RulesPath, @@ -547,6 +554,12 @@ function New-ActivityWatchDeploymentConfig { [int]$HayabusaAutoUploadHoursBack = 6, [string]$HayabusaAutoUploadMode = 'incident', [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', + [bool]$File1CAutoUploadEnabled = $true, + [int]$File1CAutoUploadIntervalHours = 6, + [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CTargetHost, + [string]$File1CTargetUser = 'igor', + [string]$File1CRemoteRoot = '/opt/activitywatch/clickhouse-1c/landing', [switch]$IntegrationTestEnabled ) @@ -588,6 +601,7 @@ function New-ActivityWatchDeploymentConfig { sessionCollectorScript = $SessionCollectorScript evtxExportScript = $EvtxExportScript hayabusaUploadScript = $HayabusaUploadScript + file1cTelemetryScript = $File1CTelemetryScript rulesPath = $RulesPath policyPath = $PolicyPath launchScript = $LaunchScriptPath @@ -623,6 +637,16 @@ function New-ActivityWatchDeploymentConfig { taskName = $HayabusaAutoUploadTaskName } } + analytics = [pscustomobject]@{ + file1cAutomation = [pscustomobject]@{ + enabled = [bool]$File1CAutoUploadEnabled + intervalHours = $File1CAutoUploadIntervalHours + taskName = $File1CAutoUploadTaskName + targetHost = $File1CTargetHost + targetUser = $File1CTargetUser + remoteRoot = $File1CRemoteRoot + } + } sessionEvents = [pscustomobject]@{ logonEnabled = $LogonMarkerEnabled bucketPrefix = 'aw-session-events' @@ -1595,6 +1619,46 @@ function Register-ActivityWatchHayabusaAutoUploadTask { } } +function Register-ActivityWatchFile1CAutoUploadTask { + param( + [Parameter(Mandatory = $true)] + [string]$ConfigPath + ) + + $config = Read-ActivityWatchDeploymentConfig -Path $ConfigPath + if ($config.PSObject.Properties.Name -notcontains 'analytics' -or + $config.analytics.PSObject.Properties.Name -notcontains 'file1cAutomation') { + return + } + + $automation = $config.analytics.file1cAutomation + $taskName = if ($automation.PSObject.Properties.Name -contains 'taskName' -and -not [string]::IsNullOrWhiteSpace([string]$automation.taskName)) { + [string]$automation.taskName + } else { + 'ActivityWatch File1C Upload' + } + + if (-not [bool]$automation.enabled) { + Remove-ActivityWatchScheduledTask -TaskName $taskName + return + } + + $uploadScript = if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryScript') { [string]$config.paths.file1cTelemetryScript } else { Join-Path $config.paths.stateRoot 'export-upload-file-1c-telemetry.ps1' } + if (-not (Test-Path -LiteralPath $uploadScript)) { + throw "Не найден скрипт file-1C telemetry upload: $uploadScript" + } + + $intervalHours = [Math]::Max(1, [int]$automation.intervalHours) + $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $taskCommand = "`"$powerShellExe`" -NoProfile -ExecutionPolicy Bypass -File `"$uploadScript`" -ConfigPath `"$ConfigPath`"" + + Remove-ActivityWatchScheduledTask -TaskName $taskName + & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO $intervalHours /ST 00:00 /RU SYSTEM /RL HIGHEST /F | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Не удалось создать scheduled task $taskName через schtasks.exe" + } +} + function Set-ActivityWatchAcl { param( [Parameter(Mandatory = $true)] diff --git a/windows/deploy-domain-users.ps1 b/windows/deploy-domain-users.ps1 index 8f10646..71a3943 100755 --- a/windows/deploy-domain-users.ps1 +++ b/windows/deploy-domain-users.ps1 @@ -44,6 +44,11 @@ param( [int]$HayabusaAutoUploadHoursBack = 6, [string]$HayabusaAutoUploadMode = 'incident', [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', + [bool]$File1CAutoUploadEnabled = $true, + [int]$File1CAutoUploadIntervalHours = 6, + [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CTargetHost, + [string]$File1CTargetUser = 'igor', [switch]$IntegrationTestEnabled ) @@ -70,6 +75,7 @@ $fileCollectorSource = Join-Path $PSScriptRoot 'file-operations-collector.ps1' $sessionCollectorSource = Join-Path $PSScriptRoot 'worktime-session-collector.ps1' $evtxExportScriptSource = Join-Path $PSScriptRoot 'export-evtx-for-hayabusa.ps1' $hayabusaUploadScriptSource = Join-Path $PSScriptRoot 'export-upload-hayabusa-to-aw-server.ps1' +$file1cTelemetryScriptSource = Join-Path $PSScriptRoot 'export-upload-file-1c-telemetry.ps1' $exampleRulesSource = Join-Path $PSScriptRoot 'web-category-rules.example.json' $examplePolicySource = Join-Path $PSScriptRoot 'dlp-policy.example.json' @@ -90,6 +96,7 @@ $assetResult = Copy-ActivityWatchCollectorAssets ` -SessionCollectorScriptSource $sessionCollectorSource ` -EvtxExportScriptSource $evtxExportScriptSource ` -HayabusaUploadScriptSource $hayabusaUploadScriptSource ` + -File1CTelemetryScriptSource $file1cTelemetryScriptSource ` -ExampleRulesSource $exampleRulesSource ` -ExamplePolicySource $examplePolicySource ` -StateRoot $StateRoot ` @@ -115,6 +122,7 @@ $config = New-ActivityWatchDeploymentConfig ` -SessionCollectorScript $assetResult.SessionCollectorScript ` -EvtxExportScript $assetResult.EvtxExportScript ` -HayabusaUploadScript $assetResult.HayabusaUploadScript ` + -File1CTelemetryScript $assetResult.File1CTelemetryScript ` -RulesPath $assetResult.ActiveRules ` -PolicyPath $assetResult.ActivePolicy ` -PollSeconds $PollSeconds ` @@ -144,6 +152,11 @@ $config = New-ActivityWatchDeploymentConfig ` -HayabusaAutoUploadHoursBack $HayabusaAutoUploadHoursBack ` -HayabusaAutoUploadMode $HayabusaAutoUploadMode ` -HayabusaAutoUploadTaskName $HayabusaAutoUploadTaskName ` + -File1CAutoUploadEnabled $File1CAutoUploadEnabled ` + -File1CAutoUploadIntervalHours $File1CAutoUploadIntervalHours ` + -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` + -File1CTargetHost $File1CTargetHost ` + -File1CTargetUser $File1CTargetUser ` -LaunchScriptPath $launchScriptPath ` -RecoveryScriptPath $recoveryScriptPath ` -UserTasks $taskDefinitions ` @@ -156,6 +169,7 @@ Set-ActivityWatchAcl -InstallRoot $InstallRoot -StateRoot $StateRoot -LogsRoot $ Register-ActivityWatchUserTasks -TaskDefinitions $taskDefinitions -LaunchScriptPath $launchScriptPath -ConfigPath $configPath Register-ActivityWatchRecoveryTask -TaskName $config.recovery.taskName -RecoveryScriptPath $recoveryScriptPath -ConfigPath $configPath Register-ActivityWatchHayabusaAutoUploadTask -ConfigPath $configPath +Register-ActivityWatchFile1CAutoUploadTask -ConfigPath $configPath Start-ActivityWatchTasks -TaskDefinitions $taskDefinitions -RecoveryTaskName $config.recovery.taskName Write-Host 'ActivityWatch развёрнут для пользователей:' diff --git a/windows/deploy-ensemble.ps1 b/windows/deploy-ensemble.ps1 index 3091051..59a8a5a 100644 --- a/windows/deploy-ensemble.ps1 +++ b/windows/deploy-ensemble.ps1 @@ -45,6 +45,11 @@ param( [int]$HayabusaAutoUploadHoursBack = 6, [string]$HayabusaAutoUploadMode = 'incident', [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', + [bool]$File1CAutoUploadEnabled = $true, + [int]$File1CAutoUploadIntervalHours = 6, + [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CTargetHost, + [string]$File1CTargetUser = 'igor', [switch]$SkipHardening, [switch]$ValidateAfterDeploy, [switch]$IntegrationTestEnabled @@ -108,6 +113,11 @@ if (-not (Test-Path -LiteralPath $deployScript)) { -HayabusaAutoUploadHoursBack $HayabusaAutoUploadHoursBack ` -HayabusaAutoUploadMode $HayabusaAutoUploadMode ` -HayabusaAutoUploadTaskName $HayabusaAutoUploadTaskName ` + -File1CAutoUploadEnabled $File1CAutoUploadEnabled ` + -File1CAutoUploadIntervalHours $File1CAutoUploadIntervalHours ` + -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` + -File1CTargetHost $File1CTargetHost ` + -File1CTargetUser $File1CTargetUser ` -IntegrationTestEnabled:$IntegrationTestEnabled if (-not $SkipHardening) { diff --git a/windows/export-upload-file-1c-telemetry.ps1 b/windows/export-upload-file-1c-telemetry.ps1 new file mode 100644 index 0000000..190ece7 --- /dev/null +++ b/windows/export-upload-file-1c-telemetry.ps1 @@ -0,0 +1,277 @@ +[CmdletBinding()] +param( + [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json', + [string]$AnalyticsHost = '', + [string]$AnalyticsUser = 'igor', + [string]$RemoteRoot = '/opt/activitywatch/clickhouse-1c/landing', + [string]$RemoteKeyPath = 'C:\ProgramData\AWatch-rus\ssh\awops_ed25519' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function New-TemporarySshKeyCopy { + param( + [Parameter(Mandatory = $true)] + [string]$SourceKeyPath + ) + + $tempDir = Join-Path $env:TEMP 'aw-rus-1c-ssh' + New-Item -ItemType Directory -Path $tempDir -Force | Out-Null + $tempKeyPath = Join-Path $tempDir 'awops_ed25519' + Copy-Item -LiteralPath $SourceKeyPath -Destination $tempKeyPath -Force + + & icacls.exe $tempKeyPath /inheritance:r | Out-Null + & icacls.exe $tempKeyPath /grant:r "$($env:USERNAME):(F)" | Out-Null + & icacls.exe $tempKeyPath /remove:g 'Users' 'Authenticated Users' 'Everyone' 'BUILTIN\Users' 'BUILTIN\Administrators' 'NT AUTHORITY\SYSTEM' 2>$null | Out-Null + + return $tempKeyPath +} + +function Get-1CFileInfobases { + $results = New-Object System.Collections.Generic.List[object] + $launcherFiles = Get-ChildItem -Path 'C:\Users' -Directory -ErrorAction SilentlyContinue | + ForEach-Object { Join-Path $_.FullName 'AppData\Roaming\1C\1CEStart\ibases.v8i' } | + Where-Object { Test-Path -LiteralPath $_ } + + foreach ($file in $launcherFiles) { + $userName = Split-Path -Leaf (Split-Path -Parent (Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $file)))) + $currentName = $null + $currentId = $null + foreach ($lineRaw in Get-Content -LiteralPath $file -Encoding UTF8) { + $line = [string]$lineRaw + if ($line -match '^\[(.+)\]$') { + $currentName = $Matches[1] + $currentId = $null + continue + } + if ($line -match '^ID=(.+)$') { + $currentId = $Matches[1].Trim() + continue + } + if ($line -match '^Connect=File="(.+)";$' -and $currentName) { + $results.Add([pscustomobject]@{ + userName = $userName + infobase = $currentName + baseId = $currentId + path = $Matches[1] + launcherFile = $file + }) + } + } + } + + return $results | + Group-Object infobase, path | + ForEach-Object { $_.Group | Select-Object -First 1 } +} + +function Get-HostSample { + $cpu = (Get-Counter '\Processor(_Total)\% Processor Time').CounterSamples.CookedValue + $os = Get-CimInstance Win32_OperatingSystem + $disk = Get-PSDrive -Name E -ErrorAction SilentlyContinue + $rdp = (quser 2>$null | Select-Object -Skip 1 | Measure-Object).Count + + return [ordered]@{ + ts = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + host = $env:COMPUTERNAME + cpu_pct = [math]::Round($cpu, 2) + ram_pct = [math]::Round((($os.TotalVisibleMemorySize - $os.FreePhysicalMemory) / $os.TotalVisibleMemorySize) * 100, 2) + disk_free_gb = if ($disk) { [math]::Round($disk.Free / 1GB, 2) } else { 0 } + disk_latency_ms = 0 + smb_errors = 0 + rdp_sessions = $rdp + backup_ok = 1 + } +} + +function Write-JsonLines { + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [Parameter(Mandatory = $true)] + [object]$Rows + ) + + $directory = Split-Path -Parent $Path + if ($directory) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + $normalizedRows = @() + if ($null -ne $Rows) { + $normalizedRows = @($Rows) + } + $normalizedRows | + ForEach-Object { $_ | ConvertTo-Json -Depth 8 -Compress } | + Set-Content -LiteralPath $Path -Encoding UTF8 +} + +if (-not (Test-Path -LiteralPath $RemoteKeyPath)) { + throw "SSH private key not found: $RemoteKeyPath" +} + +$config = Get-Content -Raw -LiteralPath $ConfigPath | ConvertFrom-Json +if ([string]::IsNullOrWhiteSpace($AnalyticsHost)) { + if ($config.PSObject.Properties.Name -contains 'analytics' -and + $config.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and + $config.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetHost') { + $AnalyticsHost = [string]$config.analytics.file1cAutomation.targetHost + } +} +if ([string]::IsNullOrWhiteSpace($AnalyticsHost)) { + throw "AnalyticsHost is empty and deployment-config has no analytics.file1cAutomation.targetHost" +} +if ($config.PSObject.Properties.Name -contains 'analytics' -and + $config.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and + $config.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetUser' -and + -not [string]::IsNullOrWhiteSpace([string]$config.analytics.file1cAutomation.targetUser)) { + $AnalyticsUser = [string]$config.analytics.file1cAutomation.targetUser +} +if ($config.PSObject.Properties.Name -contains 'analytics' -and + $config.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and + $config.analytics.file1cAutomation.PSObject.Properties.Name -contains 'remoteRoot' -and + -not [string]::IsNullOrWhiteSpace([string]$config.analytics.file1cAutomation.remoteRoot)) { + $RemoteRoot = [string]$config.analytics.file1cAutomation.remoteRoot +} + +$infobases = @(Get-1CFileInfobases) +$nowUtc = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + +$documents = New-Object System.Collections.Generic.List[object] +$reglog = New-Object System.Collections.Generic.List[object] +$audit = New-Object System.Collections.Generic.List[object] + +foreach ($base in $infobases) { + $dbFile = Join-Path $base.path '1Cv8.1CD' + $dbItem = Get-Item -LiteralPath $dbFile -ErrorAction SilentlyContinue + $logDir = Join-Path $base.path '1Cv8Log' + $logItems = @(Get-ChildItem -LiteralPath $logDir -File -ErrorAction SilentlyContinue) + $mainLog = $logItems | Where-Object { $_.Extension -ieq '.lgp' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1 + $activeLocks = @(Get-ChildItem -LiteralPath $base.path -File -Filter '1Cv8*.1CL*' -ErrorAction SilentlyContinue) + $tempDb = Get-Item -LiteralPath (Join-Path $base.path '1Cv8tmp.1CD') -ErrorAction SilentlyContinue + $schedulerDir = Get-Item -LiteralPath (Join-Path $base.path '1Cv8JobScheduler') -ErrorAction SilentlyContinue + $owner = if ($base.userName) { [string]$base.userName } else { 'unknown' } + $status = if ($activeLocks.Count -gt 0 -or $tempDb) { 'busy' } else { 'online' } + $docId = if ($base.baseId) { [string]$base.baseId } else { ([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes([string]$base.path)).TrimEnd('=').Replace('/','_').Replace('+','-')) } + + $documents.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + organization = '' + department = 'FileBase' + doc_type = 'InfobaseSnapshot' + doc_id = $docId + doc_number = '' + author = $owner + counterparty = '' + operation_type = 'inventory' + amount = 0 + status = $status + posted = 1 + }) + + $audit.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + user = $owner + object_type = 'infobase' + object_id = $docId + action = 'inventory_snapshot' + before_hash = '' + after_hash = '' + risk_tag = if ($status -eq 'busy') { 'busy' } else { '' } + }) + + if ($mainLog) { + $reglog.Add([ordered]@{ + ts = ([datetime]$mainLog.LastWriteTimeUtc).ToString('yyyy-MM-ddTHH:mm:ssZ') + infobase = [string]$base.infobase + user = $owner + host = $env:COMPUTERNAME + app = '1cv8-file' + event_name = 'RegLogInventory' + level = if ($mainLog.Length -gt 536870912) { 'warn' } else { 'info' } + duration_ms = 0 + message = "Registration log file $($mainLog.Name) size=$([math]::Round($mainLog.Length / 1MB, 2))MB path=$($mainLog.FullName)" + }) + } + + if ($activeLocks.Count -gt 0 -or $tempDb) { + $reglog.Add([ordered]@{ + ts = $nowUtc + infobase = [string]$base.infobase + user = $owner + host = $env:COMPUTERNAME + app = '1cv8-file' + event_name = 'FileBaseBusy' + level = 'warn' + duration_ms = 0 + message = "Detected active file-base markers: locks=$($activeLocks.Count) tempDb=$([bool]$tempDb)" + }) + } + + if ($schedulerDir) { + $reglog.Add([ordered]@{ + ts = ([datetime]$schedulerDir.LastWriteTimeUtc).ToString('yyyy-MM-ddTHH:mm:ssZ') + infobase = [string]$base.infobase + user = $owner + host = $env:COMPUTERNAME + app = '1cv8-file' + event_name = 'JobSchedulerActivity' + level = 'info' + duration_ms = 0 + message = "1Cv8JobScheduler touched at $([datetime]$schedulerDir.LastWriteTimeUtc)" + }) + } +} + +$hostRows = @((Get-HostSample)) + +$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +$outRoot = Join-Path $env:TEMP "aw-rus-1c-outbox-$stamp" +New-Item -ItemType Directory -Path $outRoot -Force | Out-Null + +$files = @{ + documents = Join-Path $outRoot "documents-$stamp.jsonl" + reglog = Join-Path $outRoot "reglog-$stamp.jsonl" + audit = Join-Path $outRoot "audit-$stamp.jsonl" + host = Join-Path $outRoot "host-$stamp.jsonl" +} + +$documentRows = @($documents | ForEach-Object { $_ }) +$reglogRows = @($reglog | ForEach-Object { $_ }) +$auditRows = @($audit | ForEach-Object { $_ }) +$hostRowsNormalized = @($hostRows | ForEach-Object { $_ }) + +Write-JsonLines -Path ([string]$files['documents']) -Rows $documentRows +Write-JsonLines -Path ([string]$files['reglog']) -Rows $reglogRows +Write-JsonLines -Path ([string]$files['audit']) -Rows $auditRows +Write-JsonLines -Path ([string]$files['host']) -Rows $hostRowsNormalized + +$effectiveKeyPath = New-TemporarySshKeyCopy -SourceKeyPath $RemoteKeyPath + +try { + foreach ($dataset in 'documents', 'reglog', 'audit', 'host') { + & scp.exe -q -i $effectiveKeyPath -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=NUL $files[$dataset] "$AnalyticsUser@$AnalyticsHost`:$RemoteRoot/$dataset/" + if ($LASTEXITCODE -ne 0) { + throw "scp upload failed for dataset $dataset with rc=$LASTEXITCODE" + } + } +} +finally { + Remove-Item -LiteralPath $effectiveKeyPath -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $outRoot -Recurse -Force -ErrorAction SilentlyContinue +} + +[ordered]@{ + analyticsHost = $AnalyticsHost + analyticsUser = $AnalyticsUser + remoteRoot = $RemoteRoot + infobases = @($infobases | ForEach-Object { $_.infobase }) + datasets = [ordered]@{ + documents = $documents.Count + reglog = $reglog.Count + audit = $audit.Count + host = $hostRows.Count + } +} | ConvertTo-Json -Depth 8 diff --git a/windows/hardening-recovery.ps1 b/windows/hardening-recovery.ps1 index 752d73d..dcb9889 100755 --- a/windows/hardening-recovery.ps1 +++ b/windows/hardening-recovery.ps1 @@ -71,6 +71,7 @@ $effectiveFileCollector = if ($existingConfig -and $existingConfig.paths.PSObjec $effectiveSessionCollector = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$existingConfig.paths.sessionCollectorScript } else { Join-Path $effectiveStateRoot 'worktime-session-collector.ps1' } $effectiveEvtxExportScript = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'evtxExportScript') { [string]$existingConfig.paths.evtxExportScript } else { Join-Path $effectiveStateRoot 'export-evtx-for-hayabusa.ps1' } $effectiveHayabusaUploadScript = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'hayabusaUploadScript') { [string]$existingConfig.paths.hayabusaUploadScript } else { Join-Path $effectiveStateRoot 'export-upload-hayabusa-to-aw-server.ps1' } +$effectiveFile1CTelemetryScript = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'file1cTelemetryScript') { [string]$existingConfig.paths.file1cTelemetryScript } else { Join-Path $effectiveStateRoot 'export-upload-file-1c-telemetry.ps1' } $effectiveRules = Join-Path $effectiveStateRoot 'web-category-rules.json' $effectivePolicy = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'policyPath') { [string]$existingConfig.paths.policyPath } else { Join-Path $effectiveStateRoot 'dlp-policy.json' } $effectivePolicyClientScript = if ($existingConfig -and $existingConfig.paths.PSObject.Properties.Name -contains 'policyClientScript') { [string]$existingConfig.paths.policyClientScript } else { Join-Path $effectiveStateRoot 'dlp-policy-client.ps1' } @@ -106,6 +107,11 @@ $effectiveHayabusaAutoUploadIntervalHours = if ($existingConfig -and $existingCo $effectiveHayabusaAutoUploadHoursBack = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'hayabusaAutomation' -and $existingConfig.forensics.hayabusaAutomation.PSObject.Properties.Name -contains 'hoursBack') { [int]$existingConfig.forensics.hayabusaAutomation.hoursBack } else { 6 } $effectiveHayabusaAutoUploadMode = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'hayabusaAutomation' -and $existingConfig.forensics.hayabusaAutomation.PSObject.Properties.Name -contains 'mode') { [string]$existingConfig.forensics.hayabusaAutomation.mode } else { 'incident' } $effectiveHayabusaAutoUploadTaskName = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'hayabusaAutomation' -and $existingConfig.forensics.hayabusaAutomation.PSObject.Properties.Name -contains 'taskName') { [string]$existingConfig.forensics.hayabusaAutomation.taskName } else { 'ActivityWatch Hayabusa Upload' } +$effectiveFile1CAutoUploadEnabled = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'enabled') { [bool]$existingConfig.analytics.file1cAutomation.enabled } else { $true } +$effectiveFile1CAutoUploadIntervalHours = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'intervalHours') { [int]$existingConfig.analytics.file1cAutomation.intervalHours } else { 6 } +$effectiveFile1CAutoUploadTaskName = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'taskName') { [string]$existingConfig.analytics.file1cAutomation.taskName } else { 'ActivityWatch File1C Upload' } +$effectiveFile1CTargetHost = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetHost') { [string]$existingConfig.analytics.file1cAutomation.targetHost } else { '' } +$effectiveFile1CTargetUser = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetUser') { [string]$existingConfig.analytics.file1cAutomation.targetUser } else { 'igor' } $effectiveUsers = if ($Users -or $UserListPath) { Normalize-ActivityWatchUsers -Users $Users -UserListPath $UserListPath -Domain $Domain @@ -138,6 +144,7 @@ $assetResult = Copy-ActivityWatchCollectorAssets ` -SessionCollectorScriptSource (Join-Path $PSScriptRoot 'worktime-session-collector.ps1') ` -EvtxExportScriptSource (Join-Path $PSScriptRoot 'export-evtx-for-hayabusa.ps1') ` -HayabusaUploadScriptSource (Join-Path $PSScriptRoot 'export-upload-hayabusa-to-aw-server.ps1') ` + -File1CTelemetryScriptSource (Join-Path $PSScriptRoot 'export-upload-file-1c-telemetry.ps1') ` -ExampleRulesSource (Join-Path $PSScriptRoot 'web-category-rules.example.json') ` -ExamplePolicySource (Join-Path $PSScriptRoot 'dlp-policy.example.json') ` -StateRoot $effectiveStateRoot ` @@ -163,6 +170,7 @@ $config = New-ActivityWatchDeploymentConfig ` -SessionCollectorScript $effectiveSessionCollector ` -EvtxExportScript $effectiveEvtxExportScript ` -HayabusaUploadScript $effectiveHayabusaUploadScript ` + -File1CTelemetryScript $effectiveFile1CTelemetryScript ` -RulesPath $effectiveRules ` -PolicyPath $effectivePolicy ` -PollSeconds $effectivePollSeconds ` @@ -192,6 +200,11 @@ $config = New-ActivityWatchDeploymentConfig ` -HayabusaAutoUploadHoursBack $effectiveHayabusaAutoUploadHoursBack ` -HayabusaAutoUploadMode $effectiveHayabusaAutoUploadMode ` -HayabusaAutoUploadTaskName $effectiveHayabusaAutoUploadTaskName ` + -File1CAutoUploadEnabled $effectiveFile1CAutoUploadEnabled ` + -File1CAutoUploadIntervalHours $effectiveFile1CAutoUploadIntervalHours ` + -File1CAutoUploadTaskName $effectiveFile1CAutoUploadTaskName ` + -File1CTargetHost $effectiveFile1CTargetHost ` + -File1CTargetUser $effectiveFile1CTargetUser ` -LaunchScriptPath $effectiveLaunchScript ` -RecoveryScriptPath $effectiveRecoveryScript ` -UserTasks $taskDefinitions ` @@ -203,6 +216,7 @@ Set-ActivityWatchAcl -InstallRoot $effectiveInstallRoot -StateRoot $effectiveSta Register-ActivityWatchUserTasks -TaskDefinitions $taskDefinitions -LaunchScriptPath $effectiveLaunchScript -ConfigPath $effectiveConfigPath Register-ActivityWatchRecoveryTask -TaskName $config.recovery.taskName -RecoveryScriptPath $effectiveRecoveryScript -ConfigPath $effectiveConfigPath Register-ActivityWatchHayabusaAutoUploadTask -ConfigPath $effectiveConfigPath +Register-ActivityWatchFile1CAutoUploadTask -ConfigPath $effectiveConfigPath Start-ActivityWatchTasks -TaskDefinitions $taskDefinitions -RecoveryTaskName $config.recovery.taskName Write-Host 'Укрепление и восстановление ActivityWatch завершены.'