feat(1c): automate live file telemetry ingestion

2026-05-22 08:09:00 +03:00
parent 41f7a869e5
commit 5c6c23ba0f
19 changed files with 1070 additions and 100 deletions
@@ -35,6 +35,10 @@
    aw_windows_hayabusa_auto_upload_hours_back: 6
    aw_windows_hayabusa_auto_upload_mode: "incident"
    aw_windows_hayabusa_auto_upload_task_name: "ActivityWatch Hayabusa Upload"
+    aw_windows_file_1c_auto_upload_enabled: true
+    aw_windows_file_1c_auto_upload_interval_hours: 6
+    aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload"
+    aw_windows_file_1c_target_user: "igor"
    aw_windows_afk_enabled_default: true
    aw_windows_window_enabled_default: true
    aw_windows_file_ops_enabled: true
@@ -69,6 +73,10 @@
      ansible.builtin.set_fact:
        aw_server_inventory_host_effective: "{{ (groups['aw_server'] | default([]) | first) | default('', true) }}"

+    - name: Вычислить inventory host analytics node по умолчанию
+      ansible.builtin.set_fact:
+        aw_analytics_inventory_host_effective: "{{ (groups['proxmox'] | default([]) | first) | default('', true) }}"
+
    - name: Вычислить effective host для AW server
      ansible.builtin.set_fact:
        aw_windows_server_host_effective: >-
@@ -93,6 +101,22 @@
            | default(aw_windows_server_host_effective, true)
          }}

+    - name: Вычислить effective host для file-1C analytics
+      ansible.builtin.set_fact:
+        aw_windows_file_1c_target_host_effective: >-
+          {{
+            aw_windows_file_1c_target_host
+            | default(
+                (
+                  hostvars[aw_analytics_inventory_host_effective].ansible_host
+                  | default(aw_analytics_inventory_host_effective, true)
+                )
+                if (aw_analytics_inventory_host_effective | length) > 0
+                else '',
+                true
+              )
+          }}
+
    - name: Проверить обязательные переменные
      ansible.builtin.assert:
        that:
@@ -103,6 +127,7 @@
          - aw_windows_users_effective | length > 0
          - aw_windows_install_root is defined
          - aw_windows_state_root is defined
+          - (not (aw_windows_file_1c_auto_upload_enabled | bool)) or (aw_windows_file_1c_target_host_effective | length > 0)
        fail_msg: "Не заданы обязательные переменные Windows-развёртывания."

    - name: Нормализовать effective флаги collector'ов и smoke-check
@@ -134,6 +159,7 @@
        - worktime-session-collector.ps1
        - export-evtx-for-hayabusa.ps1
        - export-upload-hayabusa-to-aw-server.ps1
+        - export-upload-file-1c-telemetry.ps1
        - migrate-awatch-rus-paths.ps1
        - deploy-domain-users.ps1
        - deploy-ensemble.ps1
@@ -219,6 +245,11 @@
            HayabusaAutoUploadHoursBack = {{ aw_windows_hayabusa_auto_upload_hours_back | int }}
            HayabusaAutoUploadMode = "{{ aw_windows_hayabusa_auto_upload_mode }}"
            HayabusaAutoUploadTaskName = "{{ aw_windows_hayabusa_auto_upload_task_name }}"
+            File1CAutoUploadEnabled = {{ '$true' if (aw_windows_file_1c_auto_upload_enabled | bool) else '$false' }}
+            File1CAutoUploadIntervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }}
+            File1CAutoUploadTaskName = "{{ aw_windows_file_1c_auto_upload_task_name }}"
+            File1CTargetHost = "{{ aw_windows_file_1c_target_host_effective }}"
+            File1CTargetUser = "{{ aw_windows_file_1c_target_user }}"
            CustomRulesPath = "{{ aw_windows_rules_path }}"
            CustomPolicyPath = "{{ aw_windows_policy_path }}"
          }
@@ -0,0 +1,256 @@
+---
+- name: Развернуть file-1C analytics backend
+  hosts: proxmox
+  become: true
+  gather_facts: true
+
+  vars:
+    aw_file_1c_repo_root: "{{ playbook_dir | dirname }}"
+    aw_file_1c_release_root: /opt/activitywatch/releases
+    aw_file_1c_release_dir: "{{ aw_file_1c_release_root }}/clickhouse-1c"
+    aw_file_1c_root: /opt/activitywatch/clickhouse-1c
+    aw_file_1c_clickhouse_db: analytics_1c
+    aw_file_1c_clickhouse_user: default
+    aw_file_1c_clickhouse_password: change-me
+    aw_file_1c_clickhouse_port: 8123
+    aw_file_1c_clickhouse_native_port: 9000
+    aw_file_1c_grafana_admin_user: admin
+    aw_file_1c_grafana_admin_password: change-me
+    aw_file_1c_grafana_port: 3300
+    aw_file_1c_windows_upload_pubkey_path: /tmp/awops_ed25519.pub
+
+  tasks:
+    - name: Установить базовые пакеты file-1C analytics
+      ansible.builtin.apt:
+        name:
+          - docker.io
+          - docker-compose
+          - python3-venv
+          - python3-pip
+        state: present
+        update_cache: true
+
+    - name: Включить и запустить docker
+      ansible.builtin.systemd:
+        name: docker
+        enabled: true
+        state: started
+
+    - name: Создать release каталоги file-1C analytics
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: igor
+        group: igor
+        mode: "0755"
+      loop:
+        - "{{ aw_file_1c_release_root }}"
+        - "{{ aw_file_1c_release_dir }}"
+
+    - name: Скопировать верхнеуровневые файлы stack file-1C analytics
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/{{ item.src }}"
+        dest: "{{ aw_file_1c_release_dir }}/{{ item.dest }}"
+        owner: igor
+        group: igor
+        mode: "{{ item.mode | default('0644') }}"
+      loop:
+        - { src: 'README.md', dest: 'README.md' }
+        - { src: 'docker-compose.yml', dest: 'docker-compose.yml' }
+        - { src: '.env.example', dest: '.env.example' }
+
+    - name: Подготовить каталоги stack file-1C analytics
+      ansible.builtin.file:
+        path: "{{ aw_file_1c_release_dir }}/{{ item }}"
+        state: directory
+        owner: igor
+        group: igor
+        mode: "0755"
+      loop:
+        - clickhouse
+        - detections
+        - etl
+        - ops
+        - sample
+
+    - name: Скопировать каталоги stack file-1C analytics
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/{{ item }}/"
+        dest: "{{ aw_file_1c_release_dir }}/{{ item }}/"
+        owner: igor
+        group: igor
+        mode: preserve
+        directory_mode: "0755"
+      loop:
+        - clickhouse
+        - detections
+        - etl
+        - ops
+        - sample
+
+    - name: Установить права на исполняемые ops scripts
+      ansible.builtin.file:
+        path: "{{ aw_file_1c_release_dir }}/ops/{{ item }}"
+        owner: igor
+        group: igor
+        mode: "0755"
+        state: file
+      loop:
+        - bootstrap_runtime.sh
+        - check_ingest_freshness.sh
+        - run_ingest_cycle.sh
+
+    - name: Создать .env для file-1C analytics
+      ansible.builtin.copy:
+        dest: "{{ aw_file_1c_release_dir }}/.env"
+        owner: igor
+        group: igor
+        mode: "0600"
+        content: |
+          CLICKHOUSE_DB={{ aw_file_1c_clickhouse_db }}
+          CLICKHOUSE_USER={{ aw_file_1c_clickhouse_user }}
+          CLICKHOUSE_PASSWORD={{ aw_file_1c_clickhouse_password }}
+          CLICKHOUSE_PORT={{ aw_file_1c_clickhouse_port }}
+          CLICKHOUSE_NATIVE_PORT={{ aw_file_1c_clickhouse_native_port }}
+          GRAFANA_ADMIN_USER={{ aw_file_1c_grafana_admin_user }}
+          GRAFANA_ADMIN_PASSWORD={{ aw_file_1c_grafana_admin_password }}
+          GRAFANA_PORT={{ aw_file_1c_grafana_port }}
+          CLICKHOUSE_HOST=clickhouse
+
+    - name: Создать etl/config.yml для file-1C analytics
+      ansible.builtin.copy:
+        dest: "{{ aw_file_1c_release_dir }}/etl/config.yml"
+        owner: igor
+        group: igor
+        mode: "0644"
+        content: |
+          clickhouse:
+            host: localhost
+            port: {{ aw_file_1c_clickhouse_port }}
+            username: {{ aw_file_1c_clickhouse_user }}
+            password: {{ aw_file_1c_clickhouse_password }}
+            database: {{ aw_file_1c_clickhouse_db }}
+
+          landing:
+            documents: {{ aw_file_1c_root }}/landing/documents
+            postings: {{ aw_file_1c_root }}/landing/postings
+            reglog: {{ aw_file_1c_root }}/landing/reglog
+            audit: {{ aw_file_1c_root }}/landing/audit
+            host: {{ aw_file_1c_root }}/landing/host
+
+          formats:
+            default: jsonl
+            documents: jsonl
+            postings: jsonl
+            reglog: jsonl
+            audit: jsonl
+            host: jsonl
+
+          archive_dir: {{ aw_file_1c_root }}/archive
+          delete_after_load: false
+
+    - name: Создать symlink на активный root file-1C analytics
+      ansible.builtin.file:
+        src: "{{ aw_file_1c_release_dir }}"
+        dest: "{{ aw_file_1c_root }}"
+        state: link
+        force: true
+
+    - name: Bootstrap runtime file-1C analytics
+      ansible.builtin.command:
+        cmd: "{{ aw_file_1c_root }}/ops/bootstrap_runtime.sh"
+      environment:
+        AW_1C_ROOT: "{{ aw_file_1c_root }}"
+
+    - name: Сделать landing/archive writable для igor upload path
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        owner: igor
+        group: igor
+        mode: "0755"
+        recurse: true
+      loop:
+        - "{{ aw_file_1c_root }}/landing"
+        - "{{ aw_file_1c_root }}/archive"
+
+    - name: Поднять ClickHouse для file-1C analytics
+      ansible.builtin.command:
+        cmd: docker compose up -d clickhouse
+      args:
+        chdir: "{{ aw_file_1c_root }}"
+
+    - name: Установить systemd unit aw-1c-ingest.service
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.service"
+        dest: /etc/systemd/system/aw-1c-ingest.service
+        owner: root
+        group: root
+        mode: "0644"
+      notify: Перезагрузить systemd
+
+    - name: Установить systemd unit aw-1c-ingest.timer
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.timer"
+        dest: /etc/systemd/system/aw-1c-ingest.timer
+        owner: root
+        group: root
+        mode: "0644"
+      notify: Перезагрузить systemd
+
+    - name: Установить systemd unit aw-1c-proofcheck.service
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-proofcheck.service"
+        dest: /etc/systemd/system/aw-1c-proofcheck.service
+        owner: root
+        group: root
+        mode: "0644"
+      notify: Перезагрузить systemd
+
+    - name: Установить systemd unit aw-1c-proofcheck.timer
+      ansible.builtin.copy:
+        src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-proofcheck.timer"
+        dest: /etc/systemd/system/aw-1c-proofcheck.timer
+        owner: root
+        group: root
+        mode: "0644"
+      notify: Перезагрузить systemd
+
+    - name: Разрешить Windows upload key для igor
+      ansible.builtin.lineinfile:
+        path: /home/igor/.ssh/authorized_keys
+        line: "{{ lookup('file', aw_file_1c_windows_upload_pubkey_path) }}"
+        create: true
+        owner: igor
+        group: igor
+        mode: "0600"
+
+    - name: Включить и запустить aw-1c-ingest.timer
+      ansible.builtin.systemd:
+        name: aw-1c-ingest.timer
+        enabled: true
+        state: started
+        daemon_reload: true
+
+    - name: Включить и запустить aw-1c-proofcheck.timer
+      ansible.builtin.systemd:
+        name: aw-1c-proofcheck.timer
+        enabled: true
+        state: started
+        daemon_reload: true
+
+    - name: Проверить доступность ClickHouse ping
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:{{ aw_file_1c_clickhouse_port }}/ping"
+        return_content: true
+      register: aw_file_1c_ping
+      changed_when: false
+
+    - name: Показать ping ClickHouse
+      ansible.builtin.debug:
+        msg: "{{ aw_file_1c_ping.content }}"
+
+  handlers:
+    - name: Перезагрузить systemd
+      ansible.builtin.systemd:
+        daemon_reload: true
@@ -0,0 +1,112 @@
+---
+- name: Развернуть file-1C telemetry uploader на Windows
+  hosts: aw_windows
+  gather_facts: false
+
+  vars:
+    aw_windows_repo_root: "{{ playbook_dir | dirname }}"
+    aw_windows_state_root: "C:\\ProgramData\\AWatch-rus"
+    aw_windows_deploy_root: "C:\\Program Files\\AWatch-rus"
+    aw_windows_file_1c_target_user: "igor"
+    aw_windows_file_1c_auto_upload_interval_hours: 6
+    aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload"
+    aw_windows_file_1c_remote_root: "/opt/activitywatch/clickhouse-1c/landing"
+    aw_windows_upload_key_private_path: /tmp/awops_ed25519
+    aw_windows_upload_key_public_path: /tmp/awops_ed25519.pub
+
+  tasks:
+    - name: Вычислить inventory host analytics node по умолчанию
+      ansible.builtin.set_fact:
+        aw_analytics_inventory_host_effective: "{{ (groups['proxmox'] | default([]) | first) | default('', true) }}"
+
+    - name: Вычислить effective host для file-1C analytics
+      ansible.builtin.set_fact:
+        aw_windows_file_1c_target_host_effective: >-
+          {{
+            aw_windows_file_1c_target_host
+            | default(
+                (
+                  hostvars[aw_analytics_inventory_host_effective].ansible_host
+                  | default(aw_analytics_inventory_host_effective, true)
+                )
+                if (aw_analytics_inventory_host_effective | length) > 0
+                else '',
+                true
+              )
+          }}
+
+    - name: Проверить обязательные переменные file-1C telemetry
+      ansible.builtin.assert:
+        that:
+          - aw_windows_file_1c_target_host_effective | length > 0
+        fail_msg: "Не удалось вычислить host file-1C analytics для Windows uploader."
+
+    - name: Создать каталоги file-1C telemetry на Windows
+      ansible.windows.win_file:
+        path: "{{ item }}"
+        state: directory
+      loop:
+        - "{{ aw_windows_state_root }}"
+        - "{{ aw_windows_deploy_root }}\\windows"
+        - "{{ aw_windows_state_root }}\\ssh"
+
+    - name: Загрузить file-1C telemetry script в toolkit
+      ansible.windows.win_copy:
+        src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1"
+        dest: "{{ aw_windows_deploy_root }}\\windows\\export-upload-file-1c-telemetry.ps1"
+
+    - name: Загрузить file-1C telemetry script в state root
+      ansible.windows.win_copy:
+        src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1"
+        dest: "{{ aw_windows_state_root }}\\export-upload-file-1c-telemetry.ps1"
+
+    - name: Обновить deployment-config.json блоком analytics.file1cAutomation
+      ansible.windows.win_powershell:
+        script: |
+          $ErrorActionPreference = 'Stop'
+          $configPath = "{{ aw_windows_state_root }}\deployment-config.json"
+          $config = Get-Content -Raw -LiteralPath $configPath | ConvertFrom-Json
+
+          if ($config.PSObject.Properties.Name -notcontains 'paths') {
+            $config | Add-Member -NotePropertyName 'paths' -NotePropertyValue ([pscustomobject]@{})
+          }
+          if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryScript') {
+            $config.paths.file1cTelemetryScript = "{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1"
+          } else {
+            $config.paths | Add-Member -NotePropertyName 'file1cTelemetryScript' -NotePropertyValue "{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1"
+          }
+
+          if ($config.PSObject.Properties.Name -notcontains 'analytics') {
+            $config | Add-Member -NotePropertyName 'analytics' -NotePropertyValue ([pscustomobject]@{})
+          }
+
+          $automation = [pscustomobject]@{
+            enabled = $true
+            intervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }}
+            taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}"
+            targetHost = "{{ aw_windows_file_1c_target_host_effective }}"
+            targetUser = "{{ aw_windows_file_1c_target_user }}"
+            remoteRoot = "{{ aw_windows_file_1c_remote_root }}"
+          }
+
+          if ($config.analytics.PSObject.Properties.Name -contains 'file1cAutomation') {
+            $config.analytics.file1cAutomation = $automation
+          } else {
+            $config.analytics | Add-Member -NotePropertyName 'file1cAutomation' -NotePropertyValue $automation
+          }
+
+          $json = $config | ConvertTo-Json -Depth 12
+          Set-Content -LiteralPath $configPath -Value $json -Encoding UTF8
+
+    - name: Создать scheduled task file-1C upload
+      ansible.windows.win_powershell:
+        script: |
+          $ErrorActionPreference = 'Stop'
+          $taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}"
+          $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
+          $taskCommand = "`"$powerShellExe`" -NoProfile -ExecutionPolicy Bypass -File `"{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1`" -ConfigPath `"{{ aw_windows_state_root }}\deployment-config.json`""
+          & cmd.exe /c "schtasks /Delete /TN `"$taskName`" /F >nul 2>&1" | Out-Null
+          & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO {{ aw_windows_file_1c_auto_upload_interval_hours | int }} /ST 00:00 /RU SYSTEM /RL HIGHEST /F | Out-Null
+          if ($LASTEXITCODE -ne 0) {
+            throw "Не удалось создать scheduled task $taskName"
+          }