From 9c57d6d2ce7eae1b133c937f037b24933f5797fe Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Sat, 6 Jun 2026 08:58:46 +0300 Subject: [PATCH] feat(rust): advance powershell migration --- .gitignore | 2 + PILOT_READINESS_AUDIT_RU.md | 57 +- README.md | 3 + adk-rust/Cargo.lock | 417 +- adk-rust/Cargo.toml | 8 + adk-rust/crates/aw-1c-ingest/Cargo.toml | 23 + adk-rust/crates/aw-1c-ingest/src/main.rs | 1510 ++++ .../crates/aw-windows-telemetry/Cargo.toml | 26 + .../crates/aw-windows-telemetry/src/main.rs | 6270 +++++++++++++++++ adk-rust/crates/detmir-dlp/src/main.rs | 2 +- adk-rust/crates/detmir-portal/src/main.rs | 145 +- .../crates/detmir-portal/src/static/app.js | 103 +- ansible/deploy_aw_windows.yml | 13 +- ansible/deploy_detmir_portal.yml | 63 +- ansible/deploy_dlp_evidence_sync.yml | 45 +- ansible/deploy_file_1c_analytics.yml | 21 + ansible/deploy_file_1c_windows_telemetry.yml | 44 +- clickhouse-1c/README.md | 16 +- clickhouse-1c/ops/aw-1c-ingest.service | 4 +- clickhouse-1c/ops/etl-cron.example | 7 +- docs/1C_BUSINESS_EVENT_LAYER_RU.md | 6 +- docs/1C_COMPANY_INTELLIGENCE_RU.md | 2 +- docs/1C_FILE_ANALYTICS_STACK_RU.md | 173 +- docs/ARCHITECTURE_BASELINE_RU.md | 9 +- ...RCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md | 46 - docs/DETMIR_THREAT_MODEL_RU.md | 5 +- docs/DIOXUS_REMOVAL_REPORT_RU.md | 77 + docs/PILOT_GAP_ANALYSIS_RU.md | 140 + docs/PORTAL_API_CONTRACTS_RU.md | 3 +- docs/POWERSHELL_TO_RUST_ROADMAP_RU.md | 616 ++ docs/SECURITY_EVENTS_CLICKHOUSE_RU.md | 114 + docs/UI_ARCHITECTURE_BASELINE_RU.md | 6 +- docs/WINDOWS_RUST_VALIDATION_RU.md | 150 + ops/detmir-ai/detmir-auto | 175 +- .../check_detmir_rust_release_artifacts.sh | 1 + scripts/detmir-portal-tabs-smoke.mjs | 8 +- windows/AWatchRusCollectorGuardService.cs | 21 +- windows/ActivityWatch.Windows.Common.psm1 | 110 +- windows/deploy-domain-users.ps1 | 4 + windows/deploy-ensemble.ps1 | 4 + windows/hardening-recovery.ps1 | 4 + windows/install-collector-guard-service.ps1 | 10 +- 42 files changed, 10056 insertions(+), 407 deletions(-) create mode 100644 adk-rust/crates/aw-1c-ingest/Cargo.toml create mode 100644 adk-rust/crates/aw-1c-ingest/src/main.rs create mode 100644 adk-rust/crates/aw-windows-telemetry/Cargo.toml create mode 100644 adk-rust/crates/aw-windows-telemetry/src/main.rs delete mode 100644 docs/ARCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md create mode 100644 docs/DIOXUS_REMOVAL_REPORT_RU.md create mode 100644 docs/PILOT_GAP_ANALYSIS_RU.md create mode 100644 docs/POWERSHELL_TO_RUST_ROADMAP_RU.md create mode 100644 docs/WINDOWS_RUST_VALIDATION_RU.md diff --git a/.gitignore b/.gitignore index c2a29d6..4c7d9d3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ output/ dist/ .ai/ .autonomous/ +.ops/ +.playwright-cli/ # IDE .idea/ diff --git a/PILOT_READINESS_AUDIT_RU.md b/PILOT_READINESS_AUDIT_RU.md index 0701635..095de26 100644 --- a/PILOT_READINESS_AUDIT_RU.md +++ b/PILOT_READINESS_AUDIT_RU.md @@ -1,6 +1,6 @@ # Аудит готовности AWatch-rus / DetMir к пилотной эксплуатации -Дата аудита: `2026-06-04` +Дата аудита: `2026-06-05` Объект аудита: AWatch-rus / DetMir, Rust workspace `adk-rust`, портал `detmir-portal`, агент `awatch-agent-rs`, документы коммерческого пилота и @@ -12,14 +12,17 @@ ## 1. Итоговая оценка -Статус: `ГОТОВ К КОНТРОЛИРУЕМОМУ ПИЛОТУ С ОГРАНИЧЕНИЯМИ` +Статус: `ГОТОВ К КОНТРОЛИРУЕМОМУ ДЕМОНСТРАЦИОННОМУ ПИЛОТУ` -Оценка готовности: `80 / 100` +Оценка готовности: `90 / 100` Вывод: - Для ограниченного пилота на заранее согласованном контуре критических - технических блокеров не выявлено. + технических блокеров в коде и портале не выявлено. +- Полный демонстрационный путь "главный риск -> подразделение -> кандидат -> + расследование -> пакет -> отчет" проверен на рабочем DetMir-контуре; перед + показом нужен только короткий преддемо-прогон на той же сети и экране. - Для широкого промышленного внедрения остаются обязательные доработки: формальный контур доступа/RBAC, backup/retention для файловых state, sizing/load-тесты, API/schema versioning и регламент эксплуатации агента. @@ -31,10 +34,10 @@ | Проверка | Результат | |---|---| -| `cargo test --workspace` | OK | -| `cargo clippy --all-targets --all-features -- -D warnings` | OK | +| `cargo test --workspace` | OK после локального переноса Cargo target на Linux-ФС: ``. Старый `target/` на `fuseblk` не подходит для `libsqlite3-sys`. | +| `cargo clippy --all-targets --all-features` | OK | | `cargo build --release` | OK | -| `node scripts/detmir-portal-tabs-smoke.mjs` на временном локальном портале | OK | +| `node scripts/detmir-portal-tabs-smoke.mjs` против `` | OK; security events доступны через ClickHouse, переход к расследованию проверен, найдено 3 кнопки расследования. | | `GET /portal/api/reports` на пустом state-dir | OK, валидный JSON | | Отсутствие `expected_nodes.json` | OK, `agent_coverage_sla.sla_status=UNKNOWN` | | Отсутствие `incident_reviews.json`, `incident_review_audit.jsonl`, `cases.json` | OK, портал не падает | @@ -46,8 +49,21 @@ Portal smoke подтвердил: - вкладки `Обзор`, `Сотрудники`, `Подразделения`, `Риски`, `Расследования`, `Сетевой периметр`, `Отчеты`, `Настройки` открываются; - Risk Narrative выводится первым; -- read-only настройки отображают период, рабочий день, пороги и источник правил; -- переход `Risk -> Investigation` работает как read-only сценарий. +- read-only настройки отображают период, рабочий день, русские названия + порогов и источник правил; +- видимые статусы портала переведены с `OK/WARN/FAIL/UNKNOWN` на русские + формулировки; технические подсказки ClickHouse/env убраны из пользовательских + экранов; +- переход "кандидат -> расследование" проверен: smoke нашел 3 кнопки перехода + и открыл карточку расследования; +- события безопасности читаются порталом через ClickHouse: + `backend=clickhouse`, `status=ok`, `fallback_used=false`; +- первый расчет отчета прогревается при старте `detmir-portal.service`; после + прогрева `/api/reports` отвечает за доли секунды; +- фоновое обновление больше не переводит готовый экран в состояние + "Загрузка данных"; 70-секундная браузерная проверка сохранила `READY`; +- мобильная проверка 390px прошла: `READY`, глобального горизонтального overflow + нет. ## 3. Архитектура @@ -91,8 +107,9 @@ Portal smoke подтвердил: Слабые стороны: -- Нет явной версии API и machine-readable JSON Schema для `/api/reports`, - `/api/telemetry`, `/api/incident-review`, `/api/cases`. +- Контракты API уже доступны через `/api/contracts`, OpenAPI и TypeScript, + но нужна формальная матрица версий и журнал совместимых/несовместимых + изменений полей. - Нет отдельного lightweight endpoint для части управленческих данных; `/api/reports` остается большим агрегирующим endpoint. - Авторизация пользователя портала предполагается внешним gateway; сам портал не @@ -120,6 +137,8 @@ Portal smoke подтвердил: - На пустых данных портал работает, но часть выводов ожидаемо имеет статус `UNKNOWN`/`ATTENTION`; перед демо нужен подготовленный demo/pilot dataset. - PDF/export-путь требует отдельной приемочной проверки в конкретном окружении. +- Документы для заказчика все еще требуют языковой чистки от англоязычных + терминов и технических сокращений. Оценка: `готов к демонстрации и пилоту при наличии auth gateway`. @@ -191,7 +210,8 @@ enterprise endpoint agent`. - Нет единого “операторского пакета пилота” в одном маршруте: installation -> first telemetry -> validation -> demo -> acceptance. -- API contract пока описан текстом, а не JSON Schema/OpenAPI. +- API-контракты уже оформлены отдельными маршрутами, но документация должна + явно ссылаться на OpenAPI/TypeScript и порядок проверки совместимости. - Не хватает sizing guide: число endpoints, объем telemetry/day, CPU/RAM/disk, рекомендуемый retention. @@ -296,7 +316,8 @@ enterprise endpoint agent`. - Нет встроенного RBAC/auth в portal application layer. - JSON/JSONL state пока не hardened для больших объемов и параллельных записей. -- Нет API versioning/OpenAPI/JSON Schema. +- API-контракты и OpenAPI уже есть, но нет формального versioning и матрицы + совместимости agent/server/portal. - Нет load/sizing профиля. - Windows agent еще требует промышленного rollout guide и матрицы OS/locale. - FreeBSD/pfSense support нельзя считать законченным commercial module. @@ -315,7 +336,7 @@ enterprise endpoint agent`. оператор, администратор. 6. Сделать matrix agent compatibility: OS, locale, session source, fallback, worktime accepted/not accepted. -7. Подготовить demo dataset и anonymized pilot dataset. +7. Подготовить anonymized pilot dataset и регламент преддемо-прогона. ## 16. Риски пилота @@ -342,7 +363,7 @@ enterprise endpoint agent`. 2. Нет утвержденного backup/retention/access-control регламента для telemetry, cases, audit и evidence. 3. Нет sizing/load-test отчета. -4. Нет API/schema versioning и матрицы совместимости agent/server. +4. Нет формального API versioning и матрицы совместимости agent/server/portal. 5. Нет завершенного customer pilot runbook с ожидаемыми результатами проверки. ## 18. Рекомендации v0.3 @@ -356,11 +377,13 @@ enterprise endpoint agent`. evidence. 4. Зафиксировать версии agent/server/portal и команды проверки в pilot acceptance act. -5. Подготовить anonymized demo dataset и отдельный live pilot dataset. +5. Перед показом выполнить преддемо-прогон: готовность данных, наличие + кандидата, открытие расследования, скачивание пакета и итоговый отчет. Приоритет P1 для v0.3: -1. Добавить OpenAPI/JSON Schema для ключевых endpoints. +1. Зафиксировать API versioning, матрицу совместимости и порядок проверки + OpenAPI/TypeScript contracts. 2. Добавить storage hardening для JSON/JSONL state или перенести критичный state в SQLite. 3. Сделать load test: 50/100/250 endpoints, records/day, `/api/reports` p95. diff --git a/README.md b/README.md index 6c60585..924e7a9 100755 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ Python в репозитории остается для вспомогател runtime, OCR/content-analysis, 1C/AI/ETL integration и MCP/dev helpers. Эти части не являются ядром Rust-first runtime. +Портальный слой зафиксирован как Rust server-rendered HTML + HTMX с JSON API, +OpenAPI и TypeScript declarations для будущих React/Tauri-клиентов. + ## Что видит оператор - Работал ли пользователь за компьютером или в удаленной сессии. diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index f4e60f9..283ea5f 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -164,6 +164,26 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aw-1c-ingest" +version = "0.1.0" +dependencies = [ + "anyhow", + "calamine", + "chrono", + "clap", + "csv", + "filetime", + "fs2", + "regex", + "reqwest", + "serde", + "serde_json", + "serde_yaml", + "sha1", + "tempfile", +] + [[package]] name = "aw-browser-smoke" version = "0.1.0" @@ -283,6 +303,27 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "aw-windows-telemetry" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "chrono", + "clap", + "csv", + "notify", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", + "url", + "windows", + "windows-sys 0.59.0", +] + [[package]] name = "awatch-agent-rs" version = "0.3.0" @@ -303,6 +344,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -324,12 +371,33 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "calamine" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3a315226fdc5b1c3e33521073e1712a05944bc0664d665ff1f6ff0396334da" +dependencies = [ + "byteorder", + "codepage", + "encoding_rs", + "log", + "quick-xml", + "serde", + "zip 0.6.6", +] + [[package]] name = "cc" version = "1.2.63" @@ -399,7 +467,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -448,6 +516,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -478,6 +555,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -494,6 +580,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -819,6 +926,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -917,6 +1033,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.3.32" @@ -1226,7 +1351,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -1358,6 +1483,26 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -1388,6 +1533,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.11.1", + "libc", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -1461,6 +1626,18 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "mio" version = "1.2.1" @@ -1472,6 +1649,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.11.1", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1581,6 +1777,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1709,7 +1915,7 @@ dependencies = [ "sha2", "tar", "tempfile", - "zip", + "zip 2.4.2", ] [[package]] @@ -1801,7 +2007,7 @@ version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ - "bitflags", + "bitflags 2.11.1", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -1821,7 +2027,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -1875,6 +2081,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.28" @@ -1949,6 +2164,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2142,7 +2368,7 @@ checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", - "mio", + "mio 1.2.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2192,7 +2418,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -2362,7 +2588,7 @@ dependencies = [ "sha2", "tar", "tempfile", - "zip", + "zip 2.4.2", ] [[package]] @@ -2388,6 +2614,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2504,7 +2740,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", @@ -2555,23 +2791,66 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f919aee0a93304be7f62e8e5027811bbba96bcb1de84d6618be56e43f8a32a1" +dependencies = [ + "windows-core 0.59.0", + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-core" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "810ce18ed2112484b0d4e15d022e5f598113e220c53e373fb31e67e21670c1ce" +dependencies = [ + "windows-implement 0.59.0", + "windows-interface", + "windows-result 0.3.4", + "windows-strings 0.3.1", + "windows-targets 0.53.5", +] + [[package]] name = "windows-core" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", + "windows-implement 0.60.2", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-implement" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2596,19 +2875,43 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -2617,7 +2920,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", ] [[package]] @@ -2653,7 +2965,22 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -2678,7 +3005,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -2689,6 +3016,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2701,6 +3034,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -2713,6 +3052,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2737,6 +3082,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -2749,6 +3100,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -2761,6 +3118,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -2773,6 +3136,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2849,7 +3218,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", @@ -3064,6 +3433,18 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] + [[package]] name = "zip" version = "2.4.2" diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml index d429e23..b4d0fa0 100644 --- a/adk-rust/Cargo.toml +++ b/adk-rust/Cargo.toml @@ -1,6 +1,8 @@ [workspace] resolver = "3" members = [ + "crates/aw-1c-ingest", + "crates/aw-windows-telemetry", "crates/awatch-agent-rs", "crates/detmir-auto", "crates/detmir-aw-client", @@ -67,20 +69,26 @@ publish = false adk-rust = { version = "0.9.1", default-features = false } anyhow = "1" base64 = "0.22" +calamine = "=0.24.0" chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } clap = { version = "4", features = ["derive", "env"] } +csv = "1" detmir-aw-client = { path = "crates/detmir-aw-client" } detmir-core = { path = "crates/detmir-core" } detmir-state = { path = "crates/detmir-state" } +filetime = "0.2" fs2 = "0.4" +notify = "6" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } regex = "1" rusqlite = "0.32" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" +sha1 = "0.10" sha2 = "0.10" tempfile = "3" tiny_http = "0.12" url = "2" urlencoding = "2" +windows-sys = "0.59" diff --git a/adk-rust/crates/aw-1c-ingest/Cargo.toml b/adk-rust/crates/aw-1c-ingest/Cargo.toml new file mode 100644 index 0000000..68b0849 --- /dev/null +++ b/adk-rust/crates/aw-1c-ingest/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "aw-1c-ingest" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +calamine.workspace = true +chrono.workspace = true +clap.workspace = true +csv.workspace = true +filetime.workspace = true +fs2.workspace = true +regex.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true +sha1.workspace = true +tempfile.workspace = true diff --git a/adk-rust/crates/aw-1c-ingest/src/main.rs b/adk-rust/crates/aw-1c-ingest/src/main.rs new file mode 100644 index 0000000..03cba5e --- /dev/null +++ b/adk-rust/crates/aw-1c-ingest/src/main.rs @@ -0,0 +1,1510 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use calamine::{Data, Reader, open_workbook_auto}; +use chrono::{DateTime, NaiveDateTime, Utc}; +use clap::Parser; +use filetime::{FileTime, set_file_times}; +use fs2::FileExt; +use regex::Regex; +use reqwest::blocking::Client; +use serde::Deserialize; +use serde_json::{Map, Value, json}; +use sha1::{Digest, Sha1}; + +const DATASETS: &[&str] = &[ + "documents", + "postings", + "business_events", + "document_changes", + "companies", + "reglog", + "audit", + "host", +]; + +#[derive(Debug, Parser)] +#[command(about = "Rust AW-rus file-1C ingest cycle for ClickHouse")] +struct Cli { + #[arg(long, env = "AW_1C_ROOT")] + root: Option, + + #[arg(long)] + config: Option, + + #[arg(long)] + dataset: Option, + + #[arg(long, default_value_t = 30, env = "AW_1C_INGEST_TIMEOUT_SECONDS")] + timeout_seconds: u64, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + skip_post_refresh: bool, + + #[arg(long)] + skip_briefs: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Deserialize)] +struct RawConfig { + clickhouse: ClickHouseConfig, + landing: BTreeMap, + #[serde(default)] + formats: BTreeMap, + archive_dir: Option, + #[serde(default)] + delete_after_load: bool, + #[serde(default = "default_min_file_age_seconds")] + min_file_age_seconds: u64, +} + +#[derive(Debug, Deserialize)] +struct ClickHouseConfig { + host: String, + #[serde(default = "default_clickhouse_port")] + port: u16, + #[serde(default = "default_clickhouse_user")] + username: String, + #[serde(default)] + password: String, + #[serde(default = "default_clickhouse_database")] + database: String, +} + +#[derive(Debug, Default)] +struct RunSummary { + files_seen: usize, + files_loaded: usize, + rows_loaded: usize, + files_skipped_young: usize, + files_archived: usize, + sql_statements: usize, +} + +#[derive(Debug, Clone)] +struct DocumentMeta { + company_entity_key: String, + organization: String, + department: String, + document_id: String, + document_number: String, + document_type: String, + operation_type: String, + user: String, + counterparty: String, +} + +type DocumentIndex = HashMap<(String, String), DocumentMeta>; + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let root = cli + .root + .clone() + .or_else(|| std::env::var_os("AW_1C_ROOT").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from("/opt/activitywatch/clickhouse-1c")); + let config_path = cli + .config + .clone() + .unwrap_or_else(|| root.join("etl/config.yml")); + let lock_path = root.join(".ingest.lock"); + let lock_file = + File::create(&lock_path).with_context(|| format!("open lock {}", lock_path.display()))?; + if lock_file.try_lock_exclusive().is_err() { + println!("ingest cycle already running"); + return Ok(0); + } + + let config = load_config(&config_path)?; + let client = ClickHouseClient::new(&config.clickhouse, cli.timeout_seconds)?; + let mut summary = RunSummary::default(); + + build_business_exports(&config, cli.dry_run, &mut summary)?; + load_datasets( + &config, + &client, + cli.dataset.as_deref(), + cli.dry_run, + &mut summary, + )?; + load_registry(&config, &client, &root, cli.dry_run, &mut summary)?; + + if !cli.dry_run { + run_sql_file( + &client, + &root.join("detections/build_entity_timeline.sql"), + &mut summary, + )?; + run_sql_file( + &client, + &root.join("clickhouse/init/04_company_intelligence.sql"), + &mut summary, + )?; + if !cli.skip_post_refresh { + run_optional_script(&root.join("ops/run_company_registry_bindings_refresh.sh"))?; + run_optional_script(&root.join("ops/run_company_intelligence_refresh.sh"))?; + } + run_sql_file( + &client, + &root.join("detections/insert_detections.sql"), + &mut summary, + )?; + run_sql_file( + &client, + &root.join("detections/open_cases_from_detections.sql"), + &mut summary, + )?; + if !cli.skip_briefs { + let _ = run_optional_script(&root.join("ops/run_manager_brief.sh")); + let _ = run_optional_script(&root.join("ops/run_recovery_brief.sh")); + } + let _ = run_optional_script(&root.join("ops/check_ingest_freshness.sh")); + } + + if cli.json { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "ok": true, + "dry_run": cli.dry_run, + "files_seen": summary.files_seen, + "files_loaded": summary.files_loaded, + "rows_loaded": summary.rows_loaded, + "files_skipped_young": summary.files_skipped_young, + "files_archived": summary.files_archived, + "sql_statements": summary.sql_statements, + }))? + ); + } else { + println!( + "aw-1c-ingest-rust ok dry_run={} files_loaded={} rows_loaded={} skipped_young={} sql_statements={}", + cli.dry_run, + summary.files_loaded, + summary.rows_loaded, + summary.files_skipped_young, + summary.sql_statements + ); + } + Ok(0) +} + +fn default_clickhouse_port() -> u16 { + 8123 +} + +fn default_clickhouse_user() -> String { + "default".to_string() +} + +fn default_clickhouse_database() -> String { + "analytics_1c".to_string() +} + +fn default_min_file_age_seconds() -> u64 { + 180 +} + +fn load_config(path: &Path) -> Result { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_yaml::from_str(&text).with_context(|| format!("parse {}", path.display())) +} + +struct ClickHouseClient { + client: Client, + base_url: String, + username: String, + password: String, + database: String, +} + +impl ClickHouseClient { + fn new(config: &ClickHouseConfig, timeout_seconds: u64) -> Result { + let host = config.host.trim_end_matches('/'); + let base_url = if host.starts_with("http://") || host.starts_with("https://") { + host.to_string() + } else { + format!("http://{}:{}", host, config.port) + }; + let client = Client::builder() + .timeout(Duration::from_secs(timeout_seconds)) + .build() + .context("build ClickHouse HTTP client")?; + Ok(Self { + client, + base_url, + username: config.username.clone(), + password: config.password.clone(), + database: config.database.clone(), + }) + } + + fn execute(&self, sql: &str) -> Result<()> { + if sql.trim().is_empty() { + return Ok(()); + } + let mut req = self + .client + .post(&self.base_url) + .query(&[("database", self.database.as_str())]) + .body(sql.to_string()); + if !self.username.trim().is_empty() { + req = req.basic_auth(self.username.clone(), Some(self.password.clone())); + } + let response = req.send().context("ClickHouse request")?; + let status = response.status(); + let body = response.text().unwrap_or_default(); + if !status.is_success() { + bail!("ClickHouse HTTP {status}: {}", body.trim()); + } + Ok(()) + } + + fn insert_json_each_row(&self, table: &str, columns: &[&str], rows: &[Value]) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + let mut payload = String::new(); + payload.push_str(&format!( + "INSERT INTO {} ({}) FORMAT JSONEachRow\n", + table, + columns.join(", ") + )); + for row in rows { + payload.push_str(&serde_json::to_string(row)?); + payload.push('\n'); + } + self.execute(&payload) + } +} + +fn dataset_format(config: &RawConfig, dataset: &str) -> String { + config + .formats + .get(dataset) + .or_else(|| config.formats.get("default")) + .cloned() + .unwrap_or_else(|| "jsonl".to_string()) +} + +fn ready_files(config: &RawConfig, dataset: &str) -> Result> { + let Some(root) = config.landing.get(dataset) else { + return Ok(Vec::new()); + }; + if !root.exists() { + return Ok(Vec::new()); + } + let mut out = Vec::new(); + for entry in fs::read_dir(root).with_context(|| format!("read {}", root.display()))? { + let path = entry?.path(); + if path.is_file() { + out.push(path); + } + } + out.sort(); + Ok(out) +} + +fn file_age_seconds(path: &Path) -> Result { + let meta = fs::metadata(path).with_context(|| format!("stat {}", path.display()))?; + let modified = meta + .modified() + .with_context(|| format!("mtime {}", path.display()))?; + Ok(modified.elapsed().unwrap_or_default().as_secs()) +} + +fn file_is_ready(path: &Path, config: &RawConfig) -> Result { + Ok(file_age_seconds(path)? >= config.min_file_age_seconds) +} + +fn read_rows(path: &Path, fmt: &str) -> Result>> { + match fmt { + "jsonl" => { + let text = read_text_lossy(path)?; + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| parse_object_line(line, path)) + .collect() + } + "json" => { + let value: Value = serde_json::from_str(&read_text_lossy(path)?) + .with_context(|| format!("parse json {}", path.display()))?; + match value { + Value::Array(items) => items + .into_iter() + .map(|item| match item { + Value::Object(map) => Ok(map), + _ => bail!("json item is not object in {}", path.display()), + }) + .collect(), + Value::Object(map) => Ok(vec![map]), + _ => bail!("json root is not object/list in {}", path.display()), + } + } + "csv" => { + let mut reader = csv::Reader::from_path(path) + .with_context(|| format!("open csv {}", path.display()))?; + let headers = reader.headers()?.clone(); + let mut rows = Vec::new(); + for record in reader.records() { + let record = record?; + let mut map = Map::new(); + for (idx, value) in record.iter().enumerate() { + if let Some(header) = headers.get(idx) { + map.insert(header.to_string(), Value::String(value.to_string())); + } + } + rows.push(map); + } + Ok(rows) + } + other => bail!("unsupported format {other} for {}", path.display()), + } +} + +fn read_text_lossy(path: &Path) -> Result { + let mut bytes = Vec::new(); + File::open(path) + .with_context(|| format!("open {}", path.display()))? + .read_to_end(&mut bytes)?; + if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { + bytes.drain(0..3); + } + String::from_utf8(bytes).with_context(|| format!("decode utf-8 {}", path.display())) +} + +fn parse_object_line(line: &str, path: &Path) -> Result> { + match serde_json::from_str::(line) + .with_context(|| format!("parse jsonl {}", path.display()))? + { + Value::Object(map) => Ok(map), + _ => bail!("jsonl row is not object in {}", path.display()), + } +} + +fn load_datasets( + config: &RawConfig, + client: &ClickHouseClient, + only_dataset: Option<&str>, + dry_run: bool, + summary: &mut RunSummary, +) -> Result<()> { + let datasets: Vec<&str> = match only_dataset { + Some(dataset) => vec![dataset], + None => DATASETS.to_vec(), + }; + for dataset in datasets { + let fmt = dataset_format(config, dataset); + for path in ready_files(config, dataset)? { + summary.files_seen += 1; + if !file_is_ready(&path, config)? { + summary.files_skipped_young += 1; + println!( + "skip {dataset}: {} age={}s < min_file_age_seconds={}", + path.file_name().and_then(|v| v.to_str()).unwrap_or("-"), + file_age_seconds(&path)?, + config.min_file_age_seconds + ); + continue; + } + let rows = read_rows(&path, &fmt)?; + if rows.is_empty() { + archive_or_delete(config, dataset, &path, dry_run, summary)?; + continue; + } + if !dry_run { + let source_file = file_name(&path); + let raw_rows: Vec = rows + .iter() + .map(|row| { + json!({ + "source_file": source_file, + "payload": serde_json::to_string(row).unwrap_or_else(|_| "{}".to_string()), + }) + }) + .collect(); + client.insert_json_each_row( + raw_table(dataset)?, + &["source_file", "payload"], + &raw_rows, + )?; + let core_rows: Vec = rows + .iter() + .map(|row| map_core_row(dataset, &source_file, row)) + .collect::>>()?; + client.insert_json_each_row( + core_table(dataset)?, + core_columns(dataset)?, + &core_rows, + )?; + } + summary.files_loaded += 1; + summary.rows_loaded += rows.len(); + archive_or_delete(config, dataset, &path, dry_run, summary)?; + println!("loaded {dataset}: {} rows={}", file_name(&path), rows.len()); + } + } + Ok(()) +} + +fn raw_table(dataset: &str) -> Result<&'static str> { + Ok(match dataset { + "documents" => "raw_1c_documents", + "postings" => "raw_1c_postings", + "business_events" => "raw_1c_business_events", + "document_changes" => "raw_1c_document_changes", + "companies" => "raw_1c_companies", + "reglog" => "raw_reglog", + "audit" => "raw_audit", + "host" => "raw_host_metrics", + _ => bail!("unknown dataset {dataset}"), + }) +} + +fn core_table(dataset: &str) -> Result<&'static str> { + Ok(match dataset { + "documents" => "documents", + "postings" => "postings", + "business_events" => "business_events", + "document_changes" => "document_change_events", + "companies" => "companies", + "reglog" => "reglog_events", + "audit" => "audit_events", + "host" => "host_events", + _ => bail!("unknown dataset {dataset}"), + }) +} + +fn core_columns(dataset: &str) -> Result<&'static [&'static str]> { + Ok(match dataset { + "documents" => &[ + "ts", + "infobase", + "organization", + "department", + "doc_type", + "doc_id", + "doc_number", + "author", + "counterparty", + "operation_type", + "amount", + "status", + "posted", + "source_file", + ], + "postings" => &[ + "ts", + "infobase", + "registrar", + "operation_type", + "account_dt", + "account_ct", + "amount", + "source_file", + ], + "business_events" => &[ + "ts", + "event_id", + "infobase", + "company_entity_key", + "organization", + "department", + "document_id", + "document_number", + "document_type", + "registrar", + "operation_type", + "event_kind", + "user", + "counterparty", + "counterparty_inn", + "debit_account", + "credit_account", + "amount", + "currency", + "line_no", + "evidence_ref", + "source_file", + ], + "document_changes" => &[ + "ts", + "change_id", + "infobase", + "company_entity_key", + "organization", + "document_id", + "document_number", + "document_type", + "change_kind", + "field_name", + "user", + "before_value", + "after_value", + "risk_tag", + "evidence_ref", + "source_file", + ], + "companies" => &[ + "ts", + "infobase", + "company_name", + "organization", + "owner_user", + "base_id", + "base_path", + "status", + "db_size_bytes", + "reglog_size_bytes", + "active_locks", + "temp_db_present", + "scheduler_touched", + "activity_score", + "source_file", + ], + "reglog" => &[ + "ts", + "infobase", + "user", + "host", + "app", + "event_name", + "level", + "duration_ms", + "message", + "source_file", + ], + "audit" => &[ + "ts", + "infobase", + "user", + "object_type", + "object_id", + "action", + "before_hash", + "after_hash", + "risk_tag", + "source_file", + ], + "host" => &[ + "ts", + "host", + "cpu_pct", + "ram_pct", + "disk_free_gb", + "disk_latency_ms", + "smb_errors", + "rdp_sessions", + "backup_ok", + "source_file", + ], + _ => bail!("unknown dataset {dataset}"), + }) +} + +fn map_core_row(dataset: &str, source_file: &str, row: &Map) -> Result { + let value = match dataset { + "documents" => json!({ + "ts": ts_value(row, &["ts", "posted_at", "created_at"]), + "infobase": s(row, "infobase"), + "organization": s(row, "organization"), + "department": s(row, "department"), + "doc_type": s(row, "doc_type"), + "doc_id": s(row, "doc_id"), + "doc_number": s(row, "doc_number"), + "author": s(row, "author"), + "counterparty": s(row, "counterparty"), + "operation_type": s(row, "operation_type"), + "amount": f(row, "amount"), + "status": s(row, "status"), + "posted": u(row, "posted"), + "source_file": source_file, + }), + "postings" => json!({ + "ts": ts_value(row, &["ts"]), + "infobase": s(row, "infobase"), + "registrar": s(row, "registrar"), + "operation_type": s(row, "operation_type"), + "account_dt": s(row, "account_dt"), + "account_ct": s(row, "account_ct"), + "amount": f(row, "amount"), + "source_file": source_file, + }), + "business_events" => json!({ + "ts": ts_value(row, &["ts", "event_time"]), + "event_id": s(row, "event_id"), + "infobase": s(row, "infobase"), + "company_entity_key": s(row, "company_entity_key"), + "organization": s(row, "organization"), + "department": s(row, "department"), + "document_id": s_alt(row, &["document_id", "doc_id"]), + "document_number": s_alt(row, &["document_number", "doc_number"]), + "document_type": s_alt(row, &["document_type", "doc_type"]), + "registrar": s(row, "registrar"), + "operation_type": s(row, "operation_type"), + "event_kind": s(row, "event_kind"), + "user": s_alt(row, &["user", "author"]), + "counterparty": s(row, "counterparty"), + "counterparty_inn": s(row, "counterparty_inn"), + "debit_account": s_alt(row, &["debit_account", "account_dt"]), + "credit_account": s_alt(row, &["credit_account", "account_ct"]), + "amount": f(row, "amount"), + "currency": s_default(row, "currency", "RUB"), + "line_no": u(row, "line_no"), + "evidence_ref": s(row, "evidence_ref"), + "source_file": source_file, + }), + "document_changes" => json!({ + "ts": ts_value(row, &["ts", "change_time"]), + "change_id": s(row, "change_id"), + "infobase": s(row, "infobase"), + "company_entity_key": s(row, "company_entity_key"), + "organization": s(row, "organization"), + "document_id": s_alt(row, &["document_id", "doc_id"]), + "document_number": s_alt(row, &["document_number", "doc_number"]), + "document_type": s_alt(row, &["document_type", "doc_type"]), + "change_kind": s(row, "change_kind"), + "field_name": s(row, "field_name"), + "user": s_alt(row, &["user", "author"]), + "before_value": s(row, "before_value"), + "after_value": s(row, "after_value"), + "risk_tag": s(row, "risk_tag"), + "evidence_ref": s(row, "evidence_ref"), + "source_file": source_file, + }), + "companies" => json!({ + "ts": ts_value(row, &["ts"]), + "infobase": s(row, "infobase"), + "company_name": s_alt(row, &["company_name", "counterparty", "infobase"]), + "organization": s(row, "organization"), + "owner_user": s_alt(row, &["owner_user", "author"]), + "base_id": s_alt(row, &["base_id", "doc_id"]), + "base_path": s(row, "base_path"), + "status": s(row, "status"), + "db_size_bytes": u(row, "db_size_bytes"), + "reglog_size_bytes": u(row, "reglog_size_bytes"), + "active_locks": u(row, "active_locks"), + "temp_db_present": u(row, "temp_db_present"), + "scheduler_touched": u(row, "scheduler_touched"), + "activity_score": f_alt(row, &["activity_score", "amount"]), + "source_file": source_file, + }), + "reglog" => json!({ + "ts": ts_value(row, &["ts"]), + "infobase": s(row, "infobase"), + "user": s(row, "user"), + "host": s(row, "host"), + "app": s(row, "app"), + "event_name": s(row, "event_name"), + "level": s_default(row, "level", "info"), + "duration_ms": u(row, "duration_ms"), + "message": s(row, "message"), + "source_file": source_file, + }), + "audit" => json!({ + "ts": ts_value(row, &["ts"]), + "infobase": s(row, "infobase"), + "user": s(row, "user"), + "object_type": s(row, "object_type"), + "object_id": s(row, "object_id"), + "action": s(row, "action"), + "before_hash": s(row, "before_hash"), + "after_hash": s(row, "after_hash"), + "risk_tag": s(row, "risk_tag"), + "source_file": source_file, + }), + "host" => json!({ + "ts": ts_value(row, &["ts"]), + "host": s(row, "host"), + "cpu_pct": f(row, "cpu_pct"), + "ram_pct": f(row, "ram_pct"), + "disk_free_gb": f(row, "disk_free_gb"), + "disk_latency_ms": f(row, "disk_latency_ms"), + "smb_errors": u(row, "smb_errors"), + "rdp_sessions": u(row, "rdp_sessions"), + "backup_ok": u(row, "backup_ok"), + "source_file": source_file, + }), + _ => bail!("unknown dataset {dataset}"), + }; + Ok(value) +} + +fn build_business_exports( + config: &RawConfig, + dry_run: bool, + summary: &mut RunSummary, +) -> Result<()> { + let company_index = load_company_index(config)?; + let (by_id, by_number) = build_document_index(config, &company_index)?; + + for path in ready_files(config, "documents")? { + if !file_is_ready(&path, config)? { + continue; + } + let rows = read_rows(&path, &dataset_format(config, "documents"))?; + let events = build_document_events(&rows, &file_name(&path), &company_index); + let out = output_path( + config, + "business_events", + &path, + "business-events-documents", + )?; + if !dry_run { + write_generated_jsonl(&out, &events, config.min_file_age_seconds)?; + } + summary.files_seen += 1; + } + for path in ready_files(config, "postings")? { + if !file_is_ready(&path, config)? { + continue; + } + let rows = read_rows(&path, &dataset_format(config, "postings"))?; + let events = + build_posting_events(&rows, &file_name(&path), &company_index, &by_id, &by_number); + let out = output_path(config, "business_events", &path, "business-events-postings")?; + if !dry_run { + write_generated_jsonl(&out, &events, config.min_file_age_seconds)?; + } + summary.files_seen += 1; + } + for path in ready_files(config, "audit")? { + if !file_is_ready(&path, config)? { + continue; + } + let rows = read_rows(&path, &dataset_format(config, "audit"))?; + let events = + build_document_changes(&rows, &file_name(&path), &company_index, &by_id, &by_number); + let out = output_path(config, "document_changes", &path, "document-changes-audit")?; + if !dry_run { + write_generated_jsonl(&out, &events, config.min_file_age_seconds)?; + } + summary.files_seen += 1; + } + Ok(()) +} + +fn load_company_index(config: &RawConfig) -> Result> { + let mut latest: HashMap = HashMap::new(); + for path in ready_files(config, "companies")? { + if !file_is_ready(&path, config)? { + continue; + } + for row in read_rows(&path, &dataset_format(config, "companies"))? { + let infobase = s(&row, "infobase"); + if infobase.is_empty() { + continue; + } + let key = + canonical_company_entity_key(&s(&row, "base_id"), &s(&row, "base_path"), &infobase); + let ts = ts_value(&row, &["ts"]); + match latest.get(&infobase) { + Some((current_ts, _)) if current_ts > &ts => {} + _ => { + latest.insert(infobase, (ts, key)); + } + } + } + } + Ok(latest.into_iter().map(|(k, (_, v))| (k, v)).collect()) +} + +fn build_document_index( + config: &RawConfig, + company_index: &HashMap, +) -> Result<(DocumentIndex, DocumentIndex)> { + let mut by_id = HashMap::new(); + let mut by_number = HashMap::new(); + for path in ready_files(config, "documents")? { + if !file_is_ready(&path, config)? { + continue; + } + for row in read_rows(&path, &dataset_format(config, "documents"))? { + let infobase = s(&row, "infobase"); + if infobase.is_empty() { + continue; + } + let document_id = s_alt(&row, &["doc_id", "document_id"]); + let document_number = s_alt(&row, &["doc_number", "document_number"]); + let meta = DocumentMeta { + company_entity_key: derive_company_entity_key(&row, company_index), + organization: s(&row, "organization"), + department: s(&row, "department"), + document_id: document_id.clone(), + document_number: document_number.clone(), + document_type: s_alt(&row, &["doc_type", "document_type"]), + operation_type: s(&row, "operation_type"), + user: s_alt(&row, &["author", "user"]), + counterparty: s(&row, "counterparty"), + }; + if !document_id.is_empty() { + by_id.insert((infobase.clone(), document_id), meta.clone()); + } + if !document_number.is_empty() { + by_number.insert((infobase, document_number), meta); + } + } + } + Ok((by_id, by_number)) +} + +fn build_document_events( + rows: &[Map], + source_file: &str, + company_index: &HashMap, +) -> Vec { + rows.iter() + .map(|row| { + let ts = ts_value(row, &["ts", "posted_at", "created_at"]); + let infobase = s(row, "infobase"); + let document_id = s_alt(row, &["doc_id", "document_id"]); + let document_number = s_alt(row, &["doc_number", "document_number"]); + json!({ + "ts": ts, + "event_id": stable_id("document_snapshot", &[source_file, &infobase, &document_id, &document_number, &ts]), + "infobase": infobase, + "company_entity_key": derive_company_entity_key(row, company_index), + "organization": s(row, "organization"), + "department": s(row, "department"), + "document_id": document_id, + "document_number": document_number, + "document_type": s_alt(row, &["doc_type", "document_type"]), + "registrar": s_alt(row, &["doc_id", "doc_number", "document_id", "document_number"]), + "operation_type": s(row, "operation_type"), + "event_kind": "document_snapshot", + "user": s_alt(row, &["author", "user"]), + "counterparty": s(row, "counterparty"), + "counterparty_inn": s(row, "counterparty_inn"), + "debit_account": "", + "credit_account": "", + "amount": f(row, "amount"), + "currency": s_default(row, "currency", "RUB"), + "line_no": 0, + "evidence_ref": format!("document:{}", s_alt(row, &["doc_id", "doc_number", "document_id", "document_number"])), + }) + }) + .collect() +} + +fn build_posting_events( + rows: &[Map], + source_file: &str, + company_index: &HashMap, + by_id: &HashMap<(String, String), DocumentMeta>, + by_number: &HashMap<(String, String), DocumentMeta>, +) -> Vec { + rows.iter() + .enumerate() + .map(|(idx, row)| { + let ts = ts_value(row, &["ts"]); + let infobase = s(row, "infobase"); + let registrar = s(row, "registrar"); + let meta = lookup_document_meta(row, by_id, by_number); + let line_no = u_alt(row, &["line_no"]).unwrap_or((idx + 1) as u64); + json!({ + "ts": ts, + "event_id": stable_id("posting", &[source_file, &infobase, ®istrar, &line_no.to_string(), &ts]), + "infobase": infobase, + "company_entity_key": meta.map(|m| m.company_entity_key.clone()).unwrap_or_else(|| derive_company_entity_key(row, company_index)), + "organization": meta.map(|m| m.organization.clone()).unwrap_or_else(|| s(row, "organization")), + "department": meta.map(|m| m.department.clone()).unwrap_or_else(|| s(row, "department")), + "document_id": meta.map(|m| m.document_id.clone()).unwrap_or_else(|| registrar.clone()), + "document_number": meta.map(|m| m.document_number.clone()).unwrap_or_else(|| s(row, "document_number")), + "document_type": meta.map(|m| m.document_type.clone()).unwrap_or_else(|| s(row, "document_type")), + "registrar": registrar, + "operation_type": s_default(row, "operation_type", &meta.map(|m| m.operation_type.clone()).unwrap_or_default()), + "event_kind": "posting", + "user": meta.map(|m| m.user.clone()).unwrap_or_else(|| s_alt(row, &["user", "author"])), + "counterparty": meta.map(|m| m.counterparty.clone()).unwrap_or_else(|| s(row, "counterparty")), + "counterparty_inn": s(row, "counterparty_inn"), + "debit_account": s_alt(row, &["account_dt", "debit_account"]), + "credit_account": s_alt(row, &["account_ct", "credit_account"]), + "amount": f(row, "amount"), + "currency": s_default(row, "currency", "RUB"), + "line_no": line_no, + "evidence_ref": format!("posting:{}:{}", s(row, "registrar"), line_no), + }) + }) + .collect() +} + +fn build_document_changes( + rows: &[Map], + source_file: &str, + company_index: &HashMap, + by_id: &HashMap<(String, String), DocumentMeta>, + by_number: &HashMap<(String, String), DocumentMeta>, +) -> Vec { + rows.iter() + .map(|row| { + let ts = ts_value(row, &["ts"]); + let infobase = s(row, "infobase"); + let object_type = s(row, "object_type"); + let object_id = s(row, "object_id"); + let meta = lookup_document_meta(row, by_id, by_number); + json!({ + "ts": ts, + "change_id": stable_id("change", &[source_file, &infobase, &object_type, &object_id, &s(row, "action"), &ts]), + "infobase": infobase, + "company_entity_key": meta.map(|m| m.company_entity_key.clone()).unwrap_or_else(|| derive_company_entity_key(row, company_index)), + "organization": meta.map(|m| m.organization.clone()).unwrap_or_else(|| s(row, "organization")), + "document_id": meta.map(|m| m.document_id.clone()).unwrap_or_else(|| if object_type == "document" { object_id.clone() } else { s(row, "document_id") }), + "document_number": meta.map(|m| m.document_number.clone()).unwrap_or_else(|| s(row, "document_number")), + "document_type": meta.map(|m| m.document_type.clone()).unwrap_or_else(|| s(row, "document_type")), + "change_kind": s_alt(row, &["change_kind", "action", "object_type"]), + "field_name": s_alt(row, &["field_name", "object_type"]), + "user": s_alt(row, &["user", "author"]), + "before_value": s_alt(row, &["before_value", "before_hash"]), + "after_value": s_alt(row, &["after_value", "after_hash"]), + "risk_tag": s(row, "risk_tag"), + "evidence_ref": format!("audit:{}:{}", object_type, object_id), + }) + }) + .collect() +} + +fn lookup_document_meta<'a>( + row: &Map, + by_id: &'a HashMap<(String, String), DocumentMeta>, + by_number: &'a HashMap<(String, String), DocumentMeta>, +) -> Option<&'a DocumentMeta> { + let infobase = s(row, "infobase"); + let registrar = s_alt(row, &["registrar", "document_id", "doc_id"]); + let number = s_alt(row, &["document_number", "doc_number"]); + by_id + .get(&(infobase.clone(), registrar)) + .or_else(|| by_number.get(&(infobase, number))) +} + +fn output_path(config: &RawConfig, dataset: &str, source: &Path, prefix: &str) -> Result { + let root = config + .landing + .get(dataset) + .ok_or_else(|| anyhow!("missing landing root for {dataset}"))?; + let stem = source + .file_stem() + .and_then(|v| v.to_str()) + .unwrap_or("input"); + Ok(root.join(format!("{prefix}-{stem}.jsonl"))) +} + +fn write_generated_jsonl(path: &Path, rows: &[Value], min_file_age_seconds: u64) -> Result<()> { + if rows.is_empty() { + let _ = fs::remove_file(path); + return Ok(()); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut tmp = tempfile::NamedTempFile::new_in(path.parent().unwrap_or_else(|| Path::new(".")))?; + for row in rows { + writeln!(tmp, "{}", serde_json::to_string(row)?)?; + } + tmp.persist(path) + .map_err(|err| anyhow!("persist {}: {}", path.display(), err))?; + let aged = FileTime::from_unix_time( + Utc::now().timestamp() - (min_file_age_seconds as i64 + 1).max(5), + 0, + ); + set_file_times(path, aged, aged).with_context(|| format!("set mtime {}", path.display()))?; + Ok(()) +} + +fn load_registry( + config: &RawConfig, + client: &ClickHouseClient, + root: &Path, + dry_run: bool, + summary: &mut RunSummary, +) -> Result<()> { + let landing = root.join("landing/registry"); + if !landing.exists() { + return Ok(()); + } + for entry in fs::read_dir(&landing)? { + let path = entry?.path(); + if !path.is_file() + || path + .extension() + .and_then(|v| v.to_str()) + .unwrap_or("") + .to_lowercase() + != "xlsx" + { + continue; + } + summary.files_seen += 1; + if !file_is_ready(&path, config)? { + summary.files_skipped_young += 1; + continue; + } + let rows = parse_registry_xlsx(&path)?; + if !dry_run && !rows.is_empty() { + let raw_rows: Vec = rows + .iter() + .map(|row| { + json!({ + "source_file": row.get("source_file").and_then(Value::as_str).unwrap_or(""), + "source_sheet": row.get("source_sheet").and_then(Value::as_str).unwrap_or(""), + "payload": serde_json::to_string(row).unwrap_or_else(|_| "{}".to_string()), + }) + }) + .collect(); + client.insert_json_each_row( + "raw_1c_company_registry", + &["source_file", "source_sheet", "payload"], + &raw_rows, + )?; + client.insert_json_each_row( + "company_registry", + &[ + "ts", + "source_file", + "source_sheet", + "company_name", + "company_key", + "assignee_name", + "registry_status", + "share_text", + "key_contour", + "inn", + "kpp", + ], + &rows, + )?; + } + summary.files_loaded += 1; + summary.rows_loaded += rows.len(); + archive_or_delete(config, "registry", &path, dry_run, summary)?; + println!("loaded registry: {} rows={}", file_name(&path), rows.len()); + } + Ok(()) +} + +fn parse_registry_xlsx(path: &Path) -> Result> { + let mut workbook = + open_workbook_auto(path).with_context(|| format!("open xlsx {}", path.display()))?; + let now = Utc::now() + .naive_utc() + .format("%Y-%m-%d %H:%M:%S") + .to_string(); + let mut tax_map: HashMap = HashMap::new(); + if let Ok(range) = workbook.worksheet_range("Лист2") { + for row in range.rows().skip(2) { + let company_name = cell_text(row.get(1)); + if company_name.is_empty() { + continue; + } + tax_map.insert( + normalize_company_key(&company_name), + (cell_text(row.get(2)), cell_text(row.get(3))), + ); + } + } + let mut out = Vec::new(); + let Ok(range) = workbook.worksheet_range("ОСНОВНОЙ") else { + return Ok(out); + }; + let rows: Vec<_> = range.rows().collect(); + if rows.len() < 3 { + return Ok(out); + } + let top_headers: Vec = rows[0].iter().map(|v| cell_text(Some(v))).collect(); + let manager_headers: Vec = rows[1].iter().map(|v| cell_text(Some(v))).collect(); + let mut specs = Vec::<(usize, String, Option, String, String)>::new(); + let mut current: Option<(usize, String, String)> = None; + for idx in 1..manager_headers.len() { + let manager = manager_headers.get(idx).cloned().unwrap_or_default(); + let top = top_headers.get(idx).cloned().unwrap_or_default(); + if !manager.is_empty() { + current = Some((idx, manager.clone(), "active".to_string())); + specs.push((idx, manager, None, String::new(), "active".to_string())); + } else if top.to_lowercase().contains("исключ") { + current = Some((idx, String::new(), "excluded".to_string())); + specs.push(( + idx, + String::new(), + None, + String::new(), + "excluded".to_string(), + )); + } else if let Some((col, assignee, status)) = current.take() { + if let Some(last) = specs.last_mut() { + if last.0 == col && last.1 == assignee && last.4 == status { + last.2 = Some(idx); + last.3 = top; + } + } + } + } + for row in rows.into_iter().skip(2) { + for (col, assignee, meta_col, meta_label, status) in &specs { + let company_name = cell_text(row.get(*col)); + if company_name.is_empty() { + continue; + } + let meta = meta_col + .map(|idx| cell_text(row.get(idx))) + .unwrap_or_default(); + let company_key = normalize_company_key(&company_name); + let (inn, kpp) = tax_map.get(&company_key).cloned().unwrap_or_default(); + let key_contour = if meta.to_uppercase() == "ЕСТЬ" + && meta_label.to_lowercase().contains("ключ") + { + 1 + } else { + 0 + }; + out.push(json!({ + "ts": now, + "source_file": file_name(path), + "source_sheet": "ОСНОВНОЙ", + "company_name": company_name, + "company_key": company_key, + "assignee_name": assignee, + "registry_status": status, + "share_text": if meta_label.to_lowercase().contains("ключ") { "" } else { &meta }, + "key_contour": key_contour, + "inn": inn, + "kpp": kpp, + })); + } + } + Ok(out) +} + +fn cell_text(value: Option<&Data>) -> String { + match value { + Some(Data::String(v)) => v.trim().to_string(), + Some(Data::Float(v)) => { + if v.fract() == 0.0 { + format!("{}", *v as i64) + } else { + v.to_string() + } + } + Some(Data::Int(v)) => v.to_string(), + Some(Data::Bool(v)) => v.to_string(), + Some(Data::DateTime(v)) => v.to_string(), + Some(Data::DateTimeIso(v)) => v.trim().to_string(), + Some(Data::DurationIso(v)) => v.trim().to_string(), + Some(Data::Empty) | None => String::new(), + Some(other) => other.to_string(), + } +} + +fn archive_or_delete( + config: &RawConfig, + dataset: &str, + path: &Path, + dry_run: bool, + summary: &mut RunSummary, +) -> Result<()> { + if dry_run { + return Ok(()); + } + if let Some(root) = &config.archive_dir { + let archive_root = root.join(dataset); + fs::create_dir_all(&archive_root) + .with_context(|| format!("create {}", archive_root.display()))?; + let dest = unique_archive_path(&archive_root.join(file_name(path))); + fs::rename(path, &dest) + .or_else(|_| { + fs::copy(path, &dest)?; + fs::remove_file(path) + }) + .with_context(|| format!("archive {} -> {}", path.display(), dest.display()))?; + summary.files_archived += 1; + } else if config.delete_after_load { + fs::remove_file(path).with_context(|| format!("delete {}", path.display()))?; + } + Ok(()) +} + +fn unique_archive_path(path: &Path) -> PathBuf { + if !path.exists() { + return path.to_path_buf(); + } + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let stem = path.file_stem().and_then(|v| v.to_str()).unwrap_or("file"); + let ext = path.extension().and_then(|v| v.to_str()).unwrap_or(""); + let suffix = Utc::now().format("%Y%m%d%H%M%S"); + if ext.is_empty() { + parent.join(format!("{stem}.{suffix}")) + } else { + parent.join(format!("{stem}.{suffix}.{ext}")) + } +} + +fn run_sql_file(client: &ClickHouseClient, path: &Path, summary: &mut RunSummary) -> Result<()> { + let sql = fs::read_to_string(path).with_context(|| format!("read SQL {}", path.display()))?; + for statement in split_sql_statements(&sql) { + client + .execute(&statement) + .with_context(|| format!("execute SQL {}", path.display()))?; + summary.sql_statements += 1; + } + Ok(()) +} + +fn split_sql_statements(sql: &str) -> Vec { + let cleaned = sql + .lines() + .filter(|line| !line.trim_start().starts_with("--")) + .collect::>() + .join("\n"); + cleaned + .split(';') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(|item| format!("{item};")) + .collect() +} + +fn run_optional_script(path: &Path) -> Result<()> { + if !path.exists() { + return Ok(()); + } + let status = Command::new(path) + .status() + .with_context(|| format!("run {}", path.display()))?; + if !status.success() { + bail!("{} exited with {}", path.display(), status); + } + Ok(()) +} + +fn derive_company_entity_key(row: &Map, index: &HashMap) -> String { + let explicit = s(row, "company_entity_key"); + if !explicit.is_empty() { + return explicit; + } + let infobase = s(row, "infobase"); + index + .get(&infobase) + .cloned() + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| { + canonical_company_entity_key(&s(row, "base_id"), &s(row, "base_path"), &infobase) + }) +} + +fn canonical_company_entity_key(base_id: &str, base_path: &str, infobase: &str) -> String { + if !base_id.trim().is_empty() { + return format!("baseid:{}", base_id.trim()); + } + if !base_path.trim().is_empty() { + let normalized = normalize_base_path(base_path); + if !normalized.is_empty() { + return format!("basepath:{normalized}"); + } + } + let normalized = normalize_infobase_key(infobase); + if normalized.is_empty() { + String::new() + } else { + format!("infobase:{normalized}") + } +} + +fn normalize_base_path(value: &str) -> String { + let re_slashes = Regex::new(r"/+").unwrap(); + let re_bad = Regex::new(r"[^0-9A-ZА-ЯЁ:/._ -]+").unwrap(); + let text = value.to_uppercase().replace('\\', "/"); + collapse_ws(&re_bad.replace_all(&re_slashes.replace_all(&text, "/"), " ")) +} + +fn normalize_infobase_key(value: &str) -> String { + let re_year = Regex::new(r"(^|\s)20[0-9]{2}($|\s)").unwrap(); + let re_bad = Regex::new(r"[^0-9A-ZА-ЯЁ]+").unwrap(); + let text = value.to_uppercase(); + collapse_ws(&re_bad.replace_all(&re_year.replace_all(&text, " "), " ")) +} + +fn normalize_company_key(value: &str) -> String { + let re_year = Regex::new(r"(^|\s)20\d{2}($|\s)").unwrap(); + let re_bad = Regex::new(r"[^0-9A-ZА-Я]+").unwrap(); + let text = value.to_uppercase().replace('Ё', "Е"); + collapse_ws(&re_bad.replace_all(&re_year.replace_all(&text, " "), " ")) +} + +fn collapse_ws(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn stable_id(prefix: &str, parts: &[&str]) -> String { + let mut hasher = Sha1::new(); + hasher.update(parts.join("|").as_bytes()); + let digest = hasher.finalize(); + format!("{prefix}:{:x}", digest)[..prefix.len() + 1 + 16].to_string() +} + +fn ts_value(row: &Map, keys: &[&str]) -> String { + let raw = keys + .iter() + .find_map(|key| row.get(*key)) + .cloned() + .unwrap_or(Value::Null); + normalize_ts(&raw) +} + +fn normalize_ts(value: &Value) -> String { + let raw = value_to_string(value); + if raw.trim().is_empty() { + return Utc::now() + .naive_utc() + .format("%Y-%m-%d %H:%M:%S") + .to_string(); + } + if let Ok(ts) = DateTime::parse_from_rfc3339(&raw.replace('Z', "+00:00")) { + return ts.naive_utc().format("%Y-%m-%d %H:%M:%S").to_string(); + } + for fmt in [ + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%S", + "%d.%m.%Y %H:%M:%S", + ] { + if let Ok(ts) = NaiveDateTime::parse_from_str(&raw, fmt) { + return ts.format("%Y-%m-%d %H:%M:%S").to_string(); + } + } + raw +} + +fn s(row: &Map, key: &str) -> String { + row.get(key) + .map(value_to_string) + .unwrap_or_default() + .trim() + .to_string() +} + +fn s_alt(row: &Map, keys: &[&str]) -> String { + keys.iter() + .map(|key| s(row, key)) + .find(|value| !value.is_empty()) + .unwrap_or_default() +} + +fn s_default(row: &Map, key: &str, default: &str) -> String { + let value = s(row, key); + if value.is_empty() { + default.to_string() + } else { + value + } +} + +fn f(row: &Map, key: &str) -> f64 { + row.get(key).and_then(value_to_f64).unwrap_or(0.0) +} + +fn f_alt(row: &Map, keys: &[&str]) -> f64 { + keys.iter() + .find_map(|key| row.get(*key).and_then(value_to_f64)) + .unwrap_or(0.0) +} + +fn u(row: &Map, key: &str) -> u64 { + row.get(key).and_then(value_to_u64).unwrap_or(0) +} + +fn u_alt(row: &Map, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| row.get(*key).and_then(value_to_u64)) +} + +fn value_to_string(value: &Value) -> String { + match value { + Value::Null => String::new(), + Value::String(v) => v.clone(), + Value::Number(v) => v.to_string(), + Value::Bool(v) => { + if *v { + "1".to_string() + } else { + "0".to_string() + } + } + other => other.to_string(), + } +} + +fn value_to_f64(value: &Value) -> Option { + match value { + Value::Number(v) => v.as_f64(), + Value::String(v) => v.replace(',', ".").parse().ok(), + Value::Bool(v) => Some(if *v { 1.0 } else { 0.0 }), + _ => None, + } +} + +fn value_to_u64(value: &Value) -> Option { + match value { + Value::Number(v) => v.as_u64().or_else(|| v.as_f64().map(|v| v.max(0.0) as u64)), + Value::String(v) => v.parse().ok(), + Value::Bool(v) => Some(if *v { 1 } else { 0 }), + _ => None, + } +} + +fn file_name(path: &Path) -> String { + path.file_name() + .and_then(|v| v.to_str()) + .unwrap_or("unknown") + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_company_key_matches_python_shape() { + assert_eq!( + canonical_company_entity_key("", "c:\\base\\ООО Тест 2026", ""), + "basepath:C:/BASE/ООО ТЕСТ 2026" + ); + assert_eq!( + canonical_company_entity_key("", "", "НОРД 2026"), + "infobase:НОРД" + ); + } + + #[test] + fn core_document_row_maps_defaults() { + let mut row = Map::new(); + row.insert("infobase".to_string(), json!("БАЗА")); + row.insert("amount".to_string(), json!("12.50")); + let mapped = map_core_row("documents", "docs.jsonl", &row).unwrap(); + assert_eq!(mapped["infobase"], "БАЗА"); + assert_eq!(mapped["amount"], 12.5); + assert_eq!(mapped["source_file"], "docs.jsonl"); + } + + #[test] + fn split_sql_ignores_empty_chunks() { + assert_eq!(split_sql_statements("SELECT 1; ; SELECT 2;").len(), 2); + } +} diff --git a/adk-rust/crates/aw-windows-telemetry/Cargo.toml b/adk-rust/crates/aw-windows-telemetry/Cargo.toml new file mode 100644 index 0000000..b57a583 --- /dev/null +++ b/adk-rust/crates/aw-windows-telemetry/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "aw-windows-telemetry" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +base64.workspace = true +chrono.workspace = true +clap.workspace = true +csv.workspace = true +notify.workspace = true +reqwest.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tempfile.workspace = true +url.workspace = true + +[target.'cfg(windows)'.dependencies] +windows = { version = "0.59", features = ["Win32_Foundation", "Win32_System_Com", "Win32_UI_Accessibility", "Win32_UI_WindowsAndMessaging"] } +windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_DataExchange", "Win32_System_Diagnostics_Debug", "Win32_System_Diagnostics_ToolHelp", "Win32_System_Memory", "Win32_System_RemoteDesktop", "Win32_System_SystemInformation", "Win32_System_Threading", "Win32_UI_WindowsAndMessaging"] } diff --git a/adk-rust/crates/aw-windows-telemetry/src/main.rs b/adk-rust/crates/aw-windows-telemetry/src/main.rs new file mode 100644 index 0000000..0303333 --- /dev/null +++ b/adk-rust/crates/aw-windows-telemetry/src/main.rs @@ -0,0 +1,6270 @@ +use std::{ + collections::{BTreeMap, HashSet}, + env, + ffi::OsStr, + fs::{self, File}, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::Command, + sync::mpsc, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::{Engine as _, engine::general_purpose}; +use chrono::{DateTime, Timelike, Utc}; +use clap::{Parser, Subcommand}; +use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher, event::RenameMode}; +use regex::Regex; +use reqwest::blocking::Client; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; +use url::Url; + +const DEFAULT_FILE1C_LOG: &str = r"C:\ProgramData\AWatch-rus\logs\file1c-telemetry.log"; +const DEFAULT_FILE1C_STATE: &str = r"C:\ProgramData\AWatch-rus\file1c-telemetry-state.json"; +const DEFAULT_DLP_LOG: &str = r"C:\ProgramData\AWatch-rus\logs\dlp-evidence-sync.log"; +const DEFAULT_DLP_STATE: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-sync-state.json"; +const DEFAULT_DLP_TOKEN: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-upload-token.txt"; +const DEFAULT_REMOTE_ROOT: &str = "/opt/activitywatch/clickhouse-1c/landing"; +const DEFAULT_SSH_KEY: &str = r"C:\ProgramData\AWatch-rus\ssh\awops_ed25519"; +const DEFAULT_REGISTRY_WORKBOOK: &str = r"E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx"; + +#[derive(Parser)] +#[command(about = "AWatch-rus Windows telemetry uploader without PowerShell runtime wrappers")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + File1cUpload(File1cUpload), + DlpEvidenceSync(DlpEvidenceSync), + ValidateDeployment(ValidateDeployment), + CollectorGuard(CollectorGuard), + FileOperationsCollector(FileOperationsCollector), + BrowserDomainsCollector(BrowserDomainsCollector), + DlpEndpointCollector(DlpEndpointCollector), +} + +#[derive(Parser, Clone)] +struct File1cUpload { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value = "")] + analytics_host: String, + #[arg(long, default_value = "igor")] + analytics_user: String, + #[arg(long, default_value = DEFAULT_REMOTE_ROOT)] + remote_root: String, + #[arg(long, default_value = DEFAULT_SSH_KEY)] + remote_key_path: PathBuf, + #[arg(long)] + registry_workbook_path: Option, + #[arg(long)] + dry_run: bool, +} + +#[derive(Parser, Clone)] +struct DlpEvidenceSync { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg( + long, + default_value = "http://aw-server.example.local:8721/api/dlp/evidence/upload" + )] + evidence_api_url: String, + #[arg(long, default_value = DEFAULT_DLP_TOKEN)] + token_path: PathBuf, + #[arg(long, default_value = DEFAULT_DLP_STATE)] + state_path: PathBuf, + #[arg(long, default_value = DEFAULT_DLP_LOG)] + log_path: PathBuf, + #[arg(long, default_value_t = 200)] + max_files: usize, + #[arg(long, default_value_t = 8_388_608)] + max_bytes: u64, + #[arg(long)] + dry_run: bool, +} + +#[derive(Parser, Clone)] +struct ValidateDeployment { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value_t = 15)] + timeout_seconds: u64, + #[arg(long, default_value_t = 300)] + worktime_max_age_seconds: i64, + #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] + fail_on_error: bool, +} + +#[derive(Parser, Clone)] +struct CollectorGuard { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value = "shadow", value_parser = ["shadow", "enforce"])] + mode: String, + #[arg(long, default_value_t = 60)] + loop_seconds: u64, + #[arg(long)] + once: bool, + #[arg(long, default_value_t = 900)] + interactive_max_age_seconds: i64, + #[arg(long, default_value_t = 600)] + restart_window_seconds: i64, + #[arg(long, default_value_t = 3)] + max_restarts: usize, + #[arg(long, default_value_t = 300)] + action_cooldown_seconds: i64, + #[arg(long, default_value_t = 60)] + interactive_action_cooldown_seconds: i64, + #[arg(long)] + self_test: bool, +} + +#[derive(Parser, Clone)] +struct FileOperationsCollector { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value = "shadow", value_parser = ["shadow", "enforce"])] + mode: String, + #[arg(long)] + server_host: Option, + #[arg(long)] + server_port: Option, + #[arg(long)] + server_scheme: Option, + #[arg(long)] + log_path: Option, + #[arg(long, default_value_t = 10)] + poll_seconds: u64, + #[arg(long, value_delimiter = ',')] + watch_paths: Vec, + #[arg(long)] + once: bool, + #[arg(long)] + duration_seconds: Option, + #[arg(long)] + self_test: bool, +} + +#[derive(Parser, Clone)] +struct BrowserDomainsCollector { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value = "shadow", value_parser = ["shadow", "enforce"])] + mode: String, + #[arg(long, default_value_t = 5)] + poll_seconds: u64, + #[arg(long, default_value_t = 30)] + pulse_seconds: u64, + #[arg(long)] + once: bool, + #[arg(long)] + duration_seconds: Option, + #[arg(long)] + self_test: bool, +} + +#[derive(Parser, Clone)] +struct DlpEndpointCollector { + #[arg( + long, + default_value = r"C:\ProgramData\AWatch-rus\deployment-config.json" + )] + config_path: PathBuf, + #[arg(long, default_value = "shadow", value_parser = ["shadow", "enforce"])] + mode: String, + #[arg(long, default_value_t = 10)] + poll_seconds: u64, + #[arg(long, default_value_t = 30)] + pulse_seconds: u64, + #[arg(long)] + once: bool, + #[arg(long)] + duration_seconds: Option, + #[arg(long)] + self_test: bool, +} + +#[derive(Debug, Clone)] +struct Infobase { + user_name: String, + infobase: String, + base_id: Option, + path: PathBuf, +} + +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +struct BaseState { + #[serde(default, rename = "dbSizeBytes")] + db_size_bytes: u64, + #[serde(default, rename = "mainLogBytes")] + main_log_bytes: u64, + #[serde(default, rename = "schedulerWriteUtc")] + scheduler_write_utc: String, +} + +struct ScpUploadContext<'a> { + scp: &'a Path, + key: &'a tempfile::NamedTempFile, + user: &'a str, + host: &'a str, + remote_root: &'a str, + log_path: &'a Path, +} + +#[derive(Debug, Clone)] +struct GuardTaskDefinition { + task_name: String, + user_id: String, +} + +#[derive(Debug)] +struct ActionAllowed { + allowed: bool, + reason: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +struct GuardRuntime { + #[serde(default, rename = "restartHistory")] + restart_history: BTreeMap>, + #[serde(default, rename = "lastAction")] + last_action: BTreeMap, + #[serde(default)] + quarantine: BTreeMap, +} + +impl GuardRuntime { + fn load(path: &Path) -> Result { + if !path.exists() { + return Ok(Self::default()); + } + let value = read_json_file(path)?; + serde_json::from_value(value).with_context(|| format!("parse {}", path.display())) + } + + fn action_allowed( + &mut self, + key: &str, + cooldown_seconds: i64, + window_seconds: i64, + max_count: usize, + ) -> ActionAllowed { + let now = Utc::now().timestamp(); + if let Some(last) = self.last_action.get(key) { + if now.saturating_sub(*last) < cooldown_seconds { + return ActionAllowed { + allowed: false, + reason: "cooldown".to_string(), + }; + } + } + + let history = self.restart_history.entry(key.to_string()).or_default(); + history.retain(|item| now.saturating_sub(*item) <= window_seconds); + if history.len() >= max_count { + self.quarantine.insert( + key.to_string(), + json!({ + "since": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "reason": "restart-budget-exhausted", + "count": history.len() + }), + ); + return ActionAllowed { + allowed: false, + reason: "quarantine".to_string(), + }; + } + + self.quarantine.remove(key); + ActionAllowed { + allowed: true, + reason: "ok".to_string(), + } + } + + fn register_action(&mut self, key: &str) { + let now = Utc::now().timestamp(); + self.restart_history + .entry(key.to_string()) + .or_default() + .push(now); + self.last_action.insert(key.to_string(), now); + } + + fn reset_action_budget(&mut self, key: &str) { + self.restart_history.remove(key); + self.last_action.remove(key); + self.quarantine.remove(key); + } +} + +struct GuardLock { + path: PathBuf, +} + +impl GuardLock { + fn acquire(path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + if path.exists() { + let pid = read_json_file(path) + .ok() + .and_then(|value| json_i64(&value, &["pid"])) + .and_then(|value| u32::try_from(value).ok()); + if pid.is_some_and(process_id_is_running) { + bail!("another collector guard instance is already running"); + } + let _ = fs::remove_file(path); + } + save_json_file( + path, + &json!({ + "pid": std::process::id(), + "createdAt": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + }), + )?; + Ok(Self { + path: path.to_path_buf(), + }) + } +} + +impl Drop for GuardLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +#[derive(Debug, Clone)] +struct FileOpsRuntime { + api_base: String, + hostname: String, + username: String, + session_id: u32, + state_root: PathBuf, + state_path: PathBuf, + log_path: PathBuf, + queue_path: PathBuf, + bucket_id: String, + mode: String, + local_logs_enabled: bool, + metrics: FileOpsMetrics, +} + +#[derive(Debug, Default, Clone, Serialize)] +struct FileOpsMetrics { + #[serde(rename = "eventsEnqueued")] + events_enqueued: u64, + #[serde(rename = "eventsFlushed")] + events_flushed: u64, + #[serde(rename = "sendFailures")] + send_failures: u64, + #[serde(rename = "queueDepth")] + queue_depth: usize, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct FileOpsQueueItem { + ts: String, + uri: String, + payload: String, + kind: String, +} + +fn main() -> Result<()> { + match Cli::parse().command { + Commands::File1cUpload(args) => run_file1c_upload(args), + Commands::DlpEvidenceSync(args) => run_dlp_evidence_sync(args), + Commands::ValidateDeployment(args) => run_validate_deployment(args), + Commands::CollectorGuard(args) => run_collector_guard(args), + Commands::FileOperationsCollector(args) => run_file_operations_collector(args), + Commands::BrowserDomainsCollector(args) => run_browser_domains_collector(args), + Commands::DlpEndpointCollector(args) => run_dlp_endpoint_collector(args), + } +} + +fn run_file1c_upload(mut args: File1cUpload) -> Result<()> { + let log_path = PathBuf::from(DEFAULT_FILE1C_LOG); + append_log(&log_path, "file1c exporter start")?; + let result = run_file1c_upload_inner(&mut args, &log_path); + match &result { + Ok(_) => { + append_log(&log_path, "file1c exporter done")?; + } + Err(err) => { + let _ = append_log(&log_path, &format!("ERROR: {err:#}")); + } + } + result +} + +fn run_file1c_upload_inner(args: &mut File1cUpload, log_path: &Path) -> Result<()> { + let config = read_json_file(&args.config_path).unwrap_or(Value::Null); + let automation = json_at(&config, &["analytics", "file1cAutomation"]).unwrap_or(&Value::Null); + + if let Some(path) = json_string(automation, &["remoteKeyPath"]).filter(|v| !v.trim().is_empty()) + { + args.remote_key_path = PathBuf::from(path); + } + if args.analytics_host.trim().is_empty() { + args.analytics_host = json_string(automation, &["targetHost"]).unwrap_or_default(); + } + if args.analytics_host.trim().is_empty() { + if let Some(host) = last_successful_analytics_host(log_path)? { + append_log( + log_path, + &format!("recovered analyticsHost={host} from previous successful uploader log"), + )?; + args.analytics_host = host; + } + } + if args.analytics_host.trim().is_empty() { + bail!( + "AnalyticsHost is empty, deployment-config has no analytics.file1cAutomation.targetHost, and no previous successful uploader log was found" + ); + } + if let Some(user) = json_string(automation, &["targetUser"]).filter(|v| !v.trim().is_empty()) { + args.analytics_user = user; + } + if let Some(root) = json_string(automation, &["remoteRoot"]).filter(|v| !v.trim().is_empty()) { + args.remote_root = root; + } + if args.registry_workbook_path.is_none() { + args.registry_workbook_path = json_string(automation, &["registryWorkbookPath"]) + .filter(|v| !v.trim().is_empty()) + .map(PathBuf::from) + .or_else(|| Some(PathBuf::from(DEFAULT_REGISTRY_WORKBOOK))); + } + + let scp = system32_path("OpenSSH\\scp.exe"); + if !args.dry_run && !scp.exists() { + bail!("scp client not found: {}", scp.display()); + } + + let infobases = discover_1c_infobases(log_path)?; + let state_path = PathBuf::from(DEFAULT_FILE1C_STATE); + let exporter_state = read_exporter_state(&state_path).unwrap_or_default(); + let now = utc_compact(); + let now_rfc3339 = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let stamp = Utc::now().format("%Y%m%d-%H%M%S").to_string(); + let host = env::var("COMPUTERNAME").unwrap_or_else(|_| "unknown".to_string()); + + let mut next_state: BTreeMap = BTreeMap::new(); + let mut documents = Vec::new(); + let mut companies = Vec::new(); + let mut reglog = Vec::new(); + let mut audit = Vec::new(); + + for base in &infobases { + let base_path = &base.path; + let db_file = base_path.join("1Cv8.1CD"); + let db_size = file_len(&db_file).unwrap_or(0); + let log_dir = base_path.join("1Cv8Log"); + let main_log = latest_lgp(&log_dir); + let main_log_size = main_log + .as_ref() + .and_then(|p| file_len(p).ok()) + .unwrap_or(0); + let locks = count_matching_files(base_path, |name| { + name.starts_with("1Cv8") && name.contains(".1CL") + }); + let temp_db = base_path.join("1Cv8tmp.1CD").exists(); + let scheduler_dir = base_path.join("1Cv8JobScheduler"); + let scheduler_write_utc = modified_utc(&scheduler_dir).ok(); + let owner = if base.user_name.trim().is_empty() { + "unknown" + } else { + &base.user_name + }; + let organization = base_path + .parent() + .and_then(Path::file_name) + .and_then(OsStr::to_str) + .unwrap_or("") + .to_string(); + let status = if locks > 0 || temp_db { + "busy" + } else { + "online" + }; + let doc_id = base + .base_id + .clone() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| stable_doc_id(&base_path.to_string_lossy())); + let previous = exporter_state.get(&doc_id); + let is_bootstrap = previous.is_none(); + let db_delta_mb = if let Some(prev) = previous { + round2((db_size as f64 - prev.db_size_bytes as f64) / 1_048_576.0) + } else { + 0.0 + }; + let reglog_delta_mb = if let Some(prev) = previous { + round2((main_log_size as f64 - prev.main_log_bytes as f64) / 1_048_576.0) + } else { + 0.0 + }; + let scheduler_touched = scheduler_touched(scheduler_write_utc, previous); + let activity_score = company_activity_score( + db_delta_mb, + reglog_delta_mb, + locks, + temp_db, + scheduler_touched, + status, + is_bootstrap, + ); + + documents.push(json!({ + "ts": now, + "infobase": base.infobase, + "organization": organization, + "department": "FileBase", + "doc_type": "InfobaseSnapshot", + "doc_id": doc_id, + "doc_number": "", + "author": owner, + "counterparty": "", + "operation_type": "inventory", + "amount": 0, + "status": status, + "posted": 1 + })); + + companies.push(json!({ + "ts": now, + "infobase": base.infobase, + "company_name": base.infobase, + "organization": organization, + "owner_user": owner, + "base_id": doc_id, + "base_path": base_path.to_string_lossy(), + "status": status, + "db_size_bytes": db_size, + "reglog_size_bytes": main_log_size, + "active_locks": locks, + "temp_db_present": if temp_db { 1 } else { 0 }, + "scheduler_touched": if scheduler_touched { 1 } else { 0 }, + "activity_score": activity_score + })); + + if activity_score > 0.0 { + documents.push(json!({ + "ts": now, + "infobase": base.infobase, + "organization": organization, + "department": "FileBaseActivity", + "doc_type": "CompanyActivitySnapshot", + "doc_id": format!("{doc_id}-{stamp}"), + "doc_number": stamp, + "author": owner, + "counterparty": base.infobase, + "operation_type": "activity_snapshot", + "amount": activity_score, + "status": status, + "posted": 1 + })); + } + + audit.push(json!({ + "ts": now, + "infobase": base.infobase, + "user": owner, + "object_type": "infobase", + "object_id": doc_id, + "action": "inventory_snapshot", + "before_hash": "", + "after_hash": "", + "risk_tag": if status == "busy" { "busy" } else { "" } + })); + + if let Some(main_log_path) = main_log { + let main_log_ts = modified_utc(&main_log_path) + .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) + .unwrap_or_else(|_| now.clone()); + let main_log_name = main_log_path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or(""); + reglog.push(json!({ + "ts": main_log_ts, + "infobase": base.infobase, + "user": owner, + "host": host, + "app": "1cv8-file", + "event_name": "RegLogInventory", + "level": if main_log_size > 536_870_912 { "warn" } else { "info" }, + "duration_ms": 0, + "message": format!("Registration log file {main_log_name} size={}MB path={}", round2(main_log_size as f64 / 1_048_576.0), main_log_path.display()) + })); + } + if locks > 0 || temp_db { + reglog.push(json!({ + "ts": now, + "infobase": base.infobase, + "user": owner, + "host": host, + "app": "1cv8-file", + "event_name": "FileBaseBusy", + "level": "warn", + "duration_ms": 0, + "message": format!("Detected active file-base markers: locks={locks} tempDb={temp_db}") + })); + } + if let Some(scheduler_ts) = scheduler_write_utc { + reglog.push(json!({ + "ts": scheduler_ts.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "infobase": base.infobase, + "user": owner, + "host": host, + "app": "1cv8-file", + "event_name": "JobSchedulerActivity", + "level": "info", + "duration_ms": 0, + "message": format!("1Cv8JobScheduler touched at {}", scheduler_ts.to_rfc3339()) + })); + } + reglog.push(json!({ + "ts": now, + "infobase": base.infobase, + "user": owner, + "host": host, + "app": "1cv8-file", + "event_name": "CompanyActivitySnapshot", + "level": if activity_score > 20.0 { "warn" } else { "info" }, + "duration_ms": 0, + "message": format!("activityScore={activity_score} dbDeltaMb={db_delta_mb} reglogDeltaMb={reglog_delta_mb} locks={locks} tempDb={temp_db} schedulerTouched={scheduler_touched}") + })); + + next_state.insert( + doc_id, + BaseState { + db_size_bytes: db_size, + main_log_bytes: main_log_size, + scheduler_write_utc: scheduler_write_utc + .map(|dt| dt.to_rfc3339()) + .unwrap_or_default(), + }, + ); + } + + let host_rows = vec![host_sample(&now, &host)]; + let outbox = tempfile::Builder::new() + .prefix(&format!("aw-rus-1c-outbox-{stamp}-")) + .tempdir() + .context("create file1c outbox")?; + + let dataset_files = [ + ("documents", documents.as_slice()), + ("companies", companies.as_slice()), + ("reglog", reglog.as_slice()), + ("audit", audit.as_slice()), + ("host", host_rows.as_slice()), + ]; + let mut files = BTreeMap::new(); + for (dataset, rows) in dataset_files { + let path = outbox.path().join(format!("{dataset}-{stamp}.jsonl")); + write_json_lines(&path, rows)?; + files.insert(dataset.to_string(), path); + } + let registry_path = outbox.path().join(format!("company-registry-{stamp}.xlsx")); + let registry_workbook_path = args + .registry_workbook_path + .as_ref() + .ok_or_else(|| anyhow!("registry workbook path was not resolved"))?; + let registry_uploaded = if registry_workbook_path.exists() { + fs::copy(registry_workbook_path, ®istry_path).with_context(|| { + format!( + "copy registry workbook {}", + registry_workbook_path.display() + ) + })?; + true + } else { + false + }; + + append_log( + log_path, + &format!( + "prepared datasets documents={} companies={} reglog={} audit={} host={}", + documents.len(), + companies.len(), + reglog.len(), + audit.len(), + host_rows.len() + ), + )?; + + if !args.dry_run { + let key = temporary_ssh_key(&args.remote_key_path, log_path)?; + let scp_context = ScpUploadContext { + scp: &scp, + key: &key, + user: &args.analytics_user, + host: &args.analytics_host, + remote_root: &args.remote_root, + log_path, + }; + for (dataset, path) in &files { + scp_upload(&scp_context, path, dataset)?; + } + if registry_uploaded { + scp_upload(&scp_context, ®istry_path, "registry")?; + } + } + + save_json_file(&state_path, &next_state)?; + append_log( + log_path, + &format!( + "upload complete analyticsHost={} remoteRoot={}", + args.analytics_host, args.remote_root + ), + )?; + + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "analyticsHost": args.analytics_host, + "analyticsUser": args.analytics_user, + "remoteRoot": args.remote_root, + "infobases": infobases.iter().map(|b| b.infobase.clone()).collect::>(), + "datasets": { + "documents": documents.len(), + "companies": companies.len(), + "reglog": reglog.len(), + "audit": audit.len(), + "host": host_rows.len(), + "registry": if registry_uploaded { 1 } else { 0 } + }, + "generatedAtUtc": now_rfc3339 + }))? + ); + Ok(()) +} + +fn run_dlp_evidence_sync(args: DlpEvidenceSync) -> Result<()> { + let mut result = json!({ + "ok": true, + "dryRun": args.dry_run, + "roots": [], + "scanned": 0, + "uploaded": 0, + "skipped": 0, + "failed": 0, + "errors": [] + }); + + let run = run_dlp_evidence_sync_inner(&args, &mut result); + if let Err(err) = run { + result["ok"] = Value::Bool(false); + increment_json_i64(&mut result, "failed", 1); + push_json_string(&mut result, "errors", format!("{err:#}")); + let _ = append_log(&args.log_path, &format!("sync failed: {err:#}")); + } + println!("{}", serde_json::to_string_pretty(&result)?); + if result["ok"] == Value::Bool(true) { + Ok(()) + } else { + bail!("dlp evidence sync failed") + } +} + +fn run_dlp_evidence_sync_inner(args: &DlpEvidenceSync, result: &mut Value) -> Result<()> { + let token = fs::read_to_string(&args.token_path) + .with_context(|| format!("upload token is missing: {}", args.token_path.display()))? + .trim() + .to_string(); + if token.is_empty() { + bail!("upload token is empty: {}", args.token_path.display()); + } + let config = read_json_file(&args.config_path).unwrap_or(Value::Null); + let roots = evidence_roots(&config); + result["roots"] = Value::Array( + roots + .iter() + .map(|p| Value::String(p.to_string_lossy().to_string())) + .collect(), + ); + + let mut state = read_json_file(&args.state_path).unwrap_or_else(|_| json!({"uploaded": {}})); + ensure_uploaded_object(&mut state); + let mut files = Vec::new(); + for root in &roots { + collect_dlp_evidence_png_files(root, &mut files); + } + files.sort_by_key(|p| std::cmp::Reverse(file_modified(p).unwrap_or(SystemTime::UNIX_EPOCH))); + files.truncate(args.max_files); + + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .context("build evidence upload HTTP client")?; + + for file in files { + increment_json_i64(result, "scanned", 1); + match upload_one_evidence_file(args, &client, &token, &mut state, &file) { + Ok(UploadOutcome::Uploaded { sha }) => { + increment_json_i64(result, "uploaded", 1); + append_log( + &args.log_path, + &format!("uploaded evidence sha={sha} file={}", file.display()), + )?; + } + Ok(UploadOutcome::Skipped) => { + increment_json_i64(result, "skipped", 1); + } + Err(err) => { + increment_json_i64(result, "failed", 1); + push_json_string(result, "errors", format!("{}: {err:#}", file.display())); + append_log( + &args.log_path, + &format!("upload failed file={}: {err:#}", file.display()), + )?; + } + } + } + save_json_file(&args.state_path, &state)?; + if result["failed"].as_i64().unwrap_or(0) > 0 { + result["ok"] = Value::Bool(false); + } + Ok(()) +} + +fn run_file_operations_collector(args: FileOperationsCollector) -> Result<()> { + if args.self_test { + file_operations_collector_self_test()?; + println!("file operations collector self-test OK"); + return Ok(()); + } + + let mut runtime = build_file_ops_runtime(&args)?; + let watch_paths = resolve_file_ops_watch_paths(&args, &runtime)?; + if watch_paths.is_empty() { + let state = file_ops_state(&runtime, &watch_paths, "ok", &[], &[]); + save_file_ops_runtime_state(&runtime, &state)?; + println!("{}", serde_json::to_string_pretty(&state)?); + return Ok(()); + } + + append_file_ops_log( + &runtime, + &format!( + "file operations rust started mode={} once={} paths={}", + runtime.mode, + args.once, + watch_paths + .iter() + .map(|path| path.to_string_lossy().to_string()) + .collect::>() + .join(";") + ), + )?; + + let mut actions = Vec::new(); + let mut problems = Vec::new(); + if let Err(err) = send_file_ops_health(&mut runtime, &watch_paths, &mut actions, &mut problems) + { + record_file_ops_problem( + &mut runtime, + &mut problems, + format!("initial health failed: {err:#}"), + )?; + } else if runtime.metrics.queue_depth == 0 { + problems.clear(); + } + save_file_ops_runtime_state( + &runtime, + &file_ops_state( + &runtime, + &watch_paths, + if problems.is_empty() { "ok" } else { "warn" }, + &actions, + &problems, + ), + )?; + + if args.once { + let state = file_ops_state(&runtime, &watch_paths, "ok", &actions, &problems); + save_file_ops_runtime_state(&runtime, &state)?; + println!("{}", serde_json::to_string_pretty(&state)?); + append_file_ops_log(&runtime, "file operations rust stopped after once")?; + return Ok(()); + } + + let (tx, rx) = mpsc::channel(); + let mut watcher: RecommendedWatcher = + Watcher::new(tx, notify::Config::default()).context("create filesystem watcher")?; + for path in &watch_paths { + watcher + .watch(path, RecursiveMode::Recursive) + .with_context(|| format!("watch {}", path.display()))?; + } + + let mut last_health = Utc::now(); + let mut pending_rename: Option = None; + let deadline = args + .duration_seconds + .map(|seconds| Utc::now() + chrono::Duration::seconds(seconds as i64)); + loop { + match rx.recv_timeout(Duration::from_secs(args.poll_seconds.max(1))) { + Ok(Ok(event)) => { + if let Some(file_event) = file_ops_event_from_notify(&event, &mut pending_rename) { + if let Err(err) = send_file_operation_event( + &mut runtime, + file_event, + &mut actions, + &mut problems, + ) { + record_file_ops_problem( + &mut runtime, + &mut problems, + format!("file operation send failed: {err:#}"), + )?; + } + } + } + Ok(Err(err)) => { + record_file_ops_problem( + &mut runtime, + &mut problems, + format!("watch error: {err:#}"), + )?; + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => bail!("filesystem watcher disconnected"), + } + + if let Err(err) = flush_file_ops_queue(&mut runtime, 100) { + record_file_ops_problem( + &mut runtime, + &mut problems, + format!("queue flush failed: {err:#}"), + )?; + } else if runtime.metrics.queue_depth == 0 { + problems.clear(); + } + if (Utc::now() - last_health).num_seconds() >= (args.poll_seconds.max(10) * 3) as i64 { + if let Err(err) = + send_file_ops_health(&mut runtime, &watch_paths, &mut actions, &mut problems) + { + record_file_ops_problem( + &mut runtime, + &mut problems, + format!("health send failed: {err:#}"), + )?; + } else if runtime.metrics.queue_depth == 0 { + problems.clear(); + } + let state = file_ops_state( + &runtime, + &watch_paths, + if problems.is_empty() { "ok" } else { "warn" }, + &actions, + &problems, + ); + if let Err(err) = save_file_ops_runtime_state(&runtime, &state) { + append_file_ops_log(&runtime, &format!("state write failed: {err:#}"))?; + } + last_health = Utc::now(); + } + if deadline.is_some_and(|until| Utc::now() >= until) { + break; + } + } + let state = file_ops_state( + &runtime, + &watch_paths, + if problems.is_empty() { "ok" } else { "warn" }, + &actions, + &problems, + ); + save_file_ops_runtime_state(&runtime, &state)?; + println!("{}", serde_json::to_string_pretty(&state)?); + append_file_ops_log(&runtime, "file operations rust stopped after bounded run")?; + Ok(()) +} + +fn file_operations_collector_self_test() -> Result<()> { + let created = file_ops_payload( + "Created", + Path::new(r"C:\Users\user\Downloads\a.zip"), + None, + 42, + "HOST-EXAMPLE", + "user", + ); + if created.get("archiveHint").and_then(Value::as_bool) != Some(true) { + bail!("archive hint was not set for zip creation"); + } + let renamed = file_ops_payload( + "Renamed", + Path::new(r"C:\Users\user\Documents\b.txt"), + Some(Path::new(r"C:\Users\user\Documents\a.txt")), + 0, + "HOST-EXAMPLE", + "user", + ); + if renamed.get("oldPath").and_then(Value::as_str).is_none() { + bail!("renamed event missed oldPath"); + } + Ok(()) +} + +fn build_file_ops_runtime(args: &FileOperationsCollector) -> Result { + let config = read_json_file(&args.config_path)?; + let state_root = json_string(&config, &["paths", "stateRoot"]) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData\AWatch-rus")); + let logs_root = json_string(&config, &["paths", "logsRoot"]) + .map(PathBuf::from) + .unwrap_or_else(|| state_root.join("logs")); + let local_logs_enabled = json_bool(&config, &["logging", "localAgentLogsEnabled"]) + .unwrap_or(true) + || args.log_path.is_some(); + let username = env::var("USERNAME") + .or_else(|_| env::var("USER")) + .unwrap_or_else(|_| "unknown".to_string()); + let session_id = current_session_id(); + let hostname = json_string(&config, &["awHostname"]) + .filter(|value| !value.trim().is_empty()) + .or_else(|| env::var("COMPUTERNAME").ok()) + .unwrap_or_else(|| "unknown".to_string()); + let scheme = args + .server_scheme + .clone() + .or_else(|| json_string(&config, &["server", "scheme"])) + .unwrap_or_else(|| "http".to_string()); + let host = args + .server_host + .clone() + .or_else(|| json_string(&config, &["server", "host"])) + .unwrap_or_else(|| "localhost".to_string()); + let port = args + .server_port + .or_else(|| json_i64(&config, &["server", "port"])) + .unwrap_or(5600); + let api_base = format!("{scheme}://{host}:{port}/api/0"); + let queue_token = queue_name_token(&username, session_id); + let queue_name = if args.mode == "enforce" { + format!("file-operations-queue-{queue_token}.jsonl") + } else { + format!("file-operations-rust-shadow-queue-{queue_token}.jsonl") + }; + let state_name = if args.mode == "enforce" { + format!("file-operations-rust-{queue_token}-state.json") + } else { + format!("file-operations-rust-shadow-{queue_token}-state.json") + }; + let log_path = args + .log_path + .clone() + .unwrap_or_else(|| logs_root.join(format!("file-operations-rust-{queue_token}.log"))); + Ok(FileOpsRuntime { + api_base, + hostname: hostname.clone(), + username, + session_id, + state_root: state_root.clone(), + state_path: state_root.join(state_name), + log_path, + queue_path: state_root.join(queue_name), + bucket_id: format!("aw-file-operations_{hostname}"), + mode: args.mode.clone(), + local_logs_enabled, + metrics: FileOpsMetrics::default(), + }) +} + +fn resolve_file_ops_watch_paths( + args: &FileOperationsCollector, + runtime: &FileOpsRuntime, +) -> Result> { + let raw_paths = if args.watch_paths.is_empty() { + vec![ + "Desktop".to_string(), + "Documents".to_string(), + "Downloads".to_string(), + ] + } else { + args.watch_paths.clone() + }; + let mut out = Vec::new(); + for raw in raw_paths { + let path = resolve_file_ops_watch_path(&raw); + if path.exists() && path.is_dir() { + out.push(path); + } else { + append_file_ops_log( + runtime, + &format!("skip missing watch path raw={raw} path={}", path.display()), + )?; + } + } + out.sort(); + out.dedup(); + Ok(out) +} + +fn resolve_file_ops_watch_path(raw: &str) -> PathBuf { + let trimmed = raw.trim(); + if Path::new(trimmed).is_absolute() { + return PathBuf::from(trimmed); + } + let user_profile = env::var("USERPROFILE") + .map(PathBuf::from) + .or_else(|_| env::var("HOME").map(PathBuf::from)) + .unwrap_or_else(|_| PathBuf::from(".")); + match trimmed.to_ascii_lowercase().as_str() { + "desktop" => user_profile.join("Desktop"), + "documents" => user_profile.join("Documents"), + "downloads" => user_profile.join("Downloads"), + _ => PathBuf::from(trimmed), + } +} + +#[derive(Debug, Clone)] +struct FileOperationEvent { + operation: String, + path: PathBuf, + old_path: Option, + size: u64, +} + +fn file_ops_event_from_notify( + event: &Event, + pending_rename: &mut Option, +) -> Option { + match &event.kind { + EventKind::Create(_) => event.paths.first().map(|path| FileOperationEvent { + operation: "Created".to_string(), + path: path.clone(), + old_path: None, + size: file_ops_file_size(path), + }), + EventKind::Remove(_) => event.paths.first().map(|path| FileOperationEvent { + operation: "Deleted".to_string(), + path: path.clone(), + old_path: None, + size: 0, + }), + EventKind::Modify(notify::event::ModifyKind::Name(mode)) => { + if matches!(mode, RenameMode::Both) && event.paths.len() >= 2 { + Some(FileOperationEvent { + operation: "Renamed".to_string(), + old_path: event.paths.first().cloned(), + path: event.paths[1].clone(), + size: file_ops_file_size(&event.paths[1]), + }) + } else if matches!(mode, RenameMode::From) { + *pending_rename = event.paths.first().cloned(); + None + } else if matches!(mode, RenameMode::To) { + event.paths.first().map(|path| FileOperationEvent { + operation: "Renamed".to_string(), + old_path: pending_rename.take(), + path: path.clone(), + size: file_ops_file_size(path), + }) + } else if event.paths.len() >= 2 { + Some(FileOperationEvent { + operation: "Renamed".to_string(), + old_path: event.paths.first().cloned(), + path: event.paths[1].clone(), + size: file_ops_file_size(&event.paths[1]), + }) + } else if event.paths.len() == 1 { + Some(FileOperationEvent { + operation: "Renamed".to_string(), + old_path: None, + path: event.paths[0].clone(), + size: file_ops_file_size(&event.paths[0]), + }) + } else { + None + } + } + _ => None, + } +} + +fn send_file_operation_event( + runtime: &mut FileOpsRuntime, + event: FileOperationEvent, + actions: &mut Vec, + problems: &mut Vec, +) -> Result<()> { + let data = file_ops_payload( + &event.operation, + &event.path, + event.old_path.as_deref(), + event.size, + &runtime.hostname, + &runtime.username, + ); + let payload = aw_event_payload(data); + let uri = format!( + "{}/buckets/{}/heartbeat?pulsetime=15", + runtime.api_base.trim_end_matches('/'), + runtime.bucket_id + ); + enqueue_file_ops_item(runtime, &uri, &payload, "file_op")?; + flush_file_ops_queue(runtime, 20)?; + actions.push(json!({ + "action": "file-op", + "operation": event.operation, + "path": event.path.to_string_lossy(), + "applied": runtime.mode == "enforce" + })); + if runtime.metrics.send_failures > 0 { + problems.push("file operation queue has send failures".to_string()); + } + Ok(()) +} + +fn send_file_ops_health( + runtime: &mut FileOpsRuntime, + watch_paths: &[PathBuf], + actions: &mut Vec, + problems: &mut Vec, +) -> Result<()> { + let data = json!({ + "signalType": "collector_health", + "username": runtime.username, + "hostname": runtime.hostname, + "sessionId": runtime.session_id, + "queueDepth": runtime.metrics.queue_depth, + "eventsEnqueued": runtime.metrics.events_enqueued, + "eventsFlushed": runtime.metrics.events_flushed, + "sendFailures": runtime.metrics.send_failures, + "source": "aw-windows-telemetry-rust", + "watchPathCount": watch_paths.len() + }); + let payload = aw_event_payload(data); + let uri = format!( + "{}/buckets/{}/heartbeat?pulsetime=30", + runtime.api_base.trim_end_matches('/'), + runtime.bucket_id + ); + enqueue_file_ops_item(runtime, &uri, &payload, "health")?; + flush_file_ops_queue(runtime, 50)?; + actions.push(json!({ + "action": "collector-health", + "applied": runtime.mode == "enforce" + })); + if runtime.metrics.send_failures > 0 { + problems.push("file operations health queue has send failures".to_string()); + } + Ok(()) +} + +fn file_ops_payload( + operation: &str, + path: &Path, + old_path: Option<&Path>, + size: u64, + hostname: &str, + username: &str, +) -> Value { + let extension = path + .extension() + .and_then(OsStr::to_str) + .map(|value| format!(".{value}")) + .unwrap_or_default(); + let mut data = Map::new(); + data.insert("operation".to_string(), json!(operation)); + data.insert("path".to_string(), json!(path.to_string_lossy())); + data.insert("extension".to_string(), json!(extension)); + data.insert("username".to_string(), json!(username)); + data.insert("hostname".to_string(), json!(hostname)); + if let Some(old_path) = old_path { + data.insert("oldPath".to_string(), json!(old_path.to_string_lossy())); + } + if size > 0 { + data.insert("size".to_string(), json!(size)); + } + if operation == "Created" + && matches!( + path.extension() + .and_then(OsStr::to_str) + .map(|ext| ext.to_ascii_lowercase()) + .as_deref(), + Some("zip" | "7z" | "rar" | "tar" | "gz") + ) + { + data.insert("archiveHint".to_string(), Value::Bool(true)); + } + Value::Object(data) +} + +fn aw_event_payload(data: Value) -> Value { + json!({ + "timestamp": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + "duration": 0, + "data": data + }) +} + +fn enqueue_file_ops_item( + runtime: &mut FileOpsRuntime, + uri: &str, + payload: &Value, + kind: &str, +) -> Result<()> { + if let Some(parent) = runtime.queue_path.parent() { + fs::create_dir_all(parent)?; + } + let item = FileOpsQueueItem { + ts: Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + uri: uri.to_string(), + payload: serde_json::to_string(payload)?, + kind: kind.to_string(), + }; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&runtime.queue_path)?; + serde_json::to_writer(&mut file, &item)?; + file.write_all(b"\n")?; + runtime.metrics.events_enqueued += 1; + runtime.metrics.queue_depth = runtime.metrics.queue_depth.saturating_add(1); + Ok(()) +} + +fn flush_file_ops_queue(runtime: &mut FileOpsRuntime, max_items: usize) -> Result<()> { + let items = read_file_ops_queue(&runtime.queue_path)?; + runtime.metrics.queue_depth = items.len(); + if items.is_empty() { + return Ok(()); + } + if runtime.mode != "enforce" { + save_json_file( + &runtime + .state_root + .join("file-operations-rust-shadow-last-queue.json"), + &items, + )?; + return Ok(()); + } + + let client = Client::builder().timeout(Duration::from_secs(15)).build()?; + ensure_aw_bucket( + &client, + &runtime.api_base, + &runtime.bucket_id, + "aw-file-operations", + "aw.file.operation", + &runtime.hostname, + )?; + + let mut remaining = Vec::new(); + let mut sent = 0usize; + for item in items { + if sent >= max_items { + remaining.push(item); + continue; + } + let payload: Value = serde_json::from_str(&item.payload).unwrap_or(Value::Null); + match client.post(&item.uri).json(&payload).send() { + Ok(response) if response.status().is_success() => { + runtime.metrics.events_flushed += 1; + sent += 1; + } + Ok(response) => { + runtime.metrics.send_failures += 1; + let uri = item.uri.clone(); + append_file_ops_log( + runtime, + &format!("POST failed uri={} status={}", uri, response.status()), + )?; + remaining.push(item); + } + Err(err) => { + runtime.metrics.send_failures += 1; + let uri = item.uri.clone(); + append_file_ops_log(runtime, &format!("POST error uri={} err={err:#}", uri))?; + remaining.push(item); + } + } + } + write_file_ops_queue(&runtime.queue_path, &remaining)?; + runtime.metrics.queue_depth = remaining.len(); + Ok(()) +} + +fn read_file_ops_queue(path: &Path) -> Result> { + if !path.exists() { + return Ok(Vec::new()); + } + let file = File::open(path)?; + let mut out = Vec::new(); + for line in BufReader::new(file).lines().map_while(Result::ok) { + if line.trim().is_empty() { + continue; + } + if let Ok(item) = serde_json::from_str::(&line) { + out.push(item); + } + } + Ok(out) +} + +fn write_file_ops_queue(path: &Path, items: &[FileOpsQueueItem]) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("tmp"); + let mut file = File::create(&tmp)?; + for item in items { + serde_json::to_writer(&mut file, item)?; + file.write_all(b"\n")?; + } + fs::rename(&tmp, path).or_else(|_| { + fs::copy(&tmp, path)?; + fs::remove_file(&tmp)?; + Ok::<(), std::io::Error>(()) + })?; + Ok(()) +} + +fn ensure_aw_bucket( + client: &Client, + api_base: &str, + bucket_id: &str, + client_name: &str, + bucket_type: &str, + hostname: &str, +) -> Result<()> { + let url = format!("{}/buckets/{bucket_id}", api_base.trim_end_matches('/')); + if client + .get(&url) + .send() + .map(|response| response.status().is_success()) + .unwrap_or(false) + { + return Ok(()); + } + let response = client + .post(&url) + .json(&json!({ + "client": client_name, + "type": bucket_type, + "hostname": hostname + })) + .send() + .with_context(|| format!("POST {url}"))?; + if !response.status().is_success() { + bail!( + "bucket create failed {} status={}", + bucket_id, + response.status() + ); + } + Ok(()) +} + +fn append_file_ops_log(runtime: &FileOpsRuntime, message: &str) -> Result<()> { + if !runtime.local_logs_enabled { + return Ok(()); + } + append_log(&runtime.log_path, &format!("[FileCollectorRust] {message}")) +} + +fn file_ops_state( + runtime: &FileOpsRuntime, + watch_paths: &[PathBuf], + status: &str, + actions: &[Value], + problems: &[String], +) -> Value { + json!({ + "schema": "aw-windows-telemetry.file-operations-collector.v1", + "status": status, + "mode": runtime.mode, + "host": runtime.hostname, + "username": runtime.username, + "sessionId": runtime.session_id, + "generatedAtUtc": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "bucketId": runtime.bucket_id, + "queuePath": runtime.queue_path.to_string_lossy(), + "watchPaths": watch_paths.iter().map(|path| path.to_string_lossy().to_string()).collect::>(), + "metrics": runtime.metrics, + "actions": actions, + "problems": problems + }) +} + +fn save_file_ops_runtime_state(runtime: &FileOpsRuntime, state: &Value) -> Result<()> { + save_json_file(&runtime.state_path, state)?; + if runtime.mode != "enforce" { + save_json_file( + &runtime + .state_root + .join("file-operations-rust-shadow-state.json"), + state, + )?; + } + Ok(()) +} + +fn record_file_ops_problem( + runtime: &mut FileOpsRuntime, + problems: &mut Vec, + message: String, +) -> Result<()> { + runtime.metrics.send_failures = runtime.metrics.send_failures.saturating_add(1); + problems.push(message.clone()); + if problems.len() > 8 { + let overflow = problems.len().saturating_sub(8); + problems.drain(0..overflow); + } + append_file_ops_log(runtime, &message)?; + Ok(()) +} + +fn queue_name_token(username: &str, session_id: u32) -> String { + let raw = format!("{username}-s{session_id}"); + let mut out = String::new(); + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') { + out.push(ch); + } else { + out.push('_'); + } + } + if out.trim().is_empty() { + format!("session-{session_id}") + } else { + out + } +} + +#[cfg(windows)] +fn current_session_id() -> u32 { + env::var("AW_RUS_SESSION_ID") + .ok() + .and_then(|value| value.parse::().ok()) + .or_else(|| process_session_id(std::process::id())) + .unwrap_or(0) +} + +#[cfg(not(windows))] +fn current_session_id() -> u32 { + 0 +} + +fn file_ops_file_size(path: &Path) -> u64 { + fs::metadata(path) + .map(|metadata| metadata.len()) + .unwrap_or(0) +} + +#[derive(Debug, Clone)] +struct RustCollectorRuntime { + api_base: String, + hostname: String, + username: String, + session_id: u32, + mode: String, + pulse_seconds: u64, + state_root: PathBuf, + log_path: PathBuf, + state_path: PathBuf, + rules_path: PathBuf, + policy_path: PathBuf, + incident_screenshot_enabled: bool, +} + +#[derive(Debug, Default, Clone, Serialize)] +struct ForegroundWindowContext { + title: String, + #[serde(rename = "processId")] + process_id: u32, + app: String, + #[cfg(windows)] + #[serde(skip)] + window_handle: isize, +} + +#[derive(Debug, Clone)] +struct WebCategoryRule { + name: String, + group: String, + domains: Vec, +} + +#[derive(Debug, Clone)] +struct WebCategoryMatch { + name: String, + group: String, + rule: String, +} + +#[derive(Debug, Clone)] +struct BrowserUrlObservation { + url: String, + browser: String, + domain: String, + root_domain: String, + category: WebCategoryMatch, +} + +#[derive(Debug, Clone)] +struct DlpPolicy { + raw: Value, + source: String, + defaults_enabled: bool, + defaults_cooldown_seconds: i64, + defaults_action: String, + defaults_severity: String, + content_dictionary_pack: Option, + content_regex_pack: Option, + content_ocr_enabled: bool, + native_mode: String, + native_allow_global_block: bool, + native_channel_actions: BTreeMap, +} + +#[derive(Debug, Clone)] +struct DlpActionDecision { + requested_action: String, + action: String, + enforcement_mode: String, + native_channel_action: String, + enforcement_suppressed: bool, +} + +#[derive(Debug, Default)] +struct EndpointCollectorState { + last_clipboard_hash: Option, + seen_usb: HashSet, + seen_print_jobs: HashSet, + cooldown: BTreeMap>, +} + +#[derive(Debug, Clone)] +struct UsbDrive { + drive_letter: String, + volume_name: String, +} + +#[derive(Debug, Clone)] +struct PrintJob { + id: String, + printer_name: String, + document_name: String, + owner: String, +} + +#[derive(Debug, Default, Clone)] +struct AdvancedContentMatches { + dictionary_matches: Vec, + regex_matches: Vec, +} + +fn run_browser_domains_collector(args: BrowserDomainsCollector) -> Result<()> { + if args.self_test { + let rules = load_category_rules(Path::new("")); + let normalized = normalize_browser_url("docs.google.com/a/b").unwrap_or_default(); + let domain = host_from_url(&normalized).unwrap_or_default(); + let category = web_category_for_domain(&domain, &rules); + let context = foreground_window_context(); + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "ok": true, + "collector": "browser-domains-collector", + "sessionId": current_session_id(), + "urlNormalization": { + "input": "docs.google.com/a/b", + "url": normalized, + "domain": domain, + "category": category.name, + "categoryGroup": category.group + }, + "foreground": context + }))? + ); + return Ok(()); + } + + let runtime = build_rust_collector_runtime( + &args.config_path, + &args.mode, + args.pulse_seconds, + "browser-domains-rust", + )?; + append_log( + &runtime.log_path, + &format!( + "browser domains rust started mode={} session={}", + runtime.mode, runtime.session_id + ), + )?; + let category_rules = load_category_rules(&runtime.rules_path); + let dlp_policy = load_dlp_policy(&runtime.policy_path); + let mut incident_cooldown = BTreeMap::>::new(); + let deadline = args + .duration_seconds + .map(|seconds| Utc::now() + chrono::Duration::seconds(seconds as i64)); + let mut events_sent = 0u64; + let mut send_failures = 0u64; + let mut problems = Vec::::new(); + loop { + let context = foreground_window_context(); + let mut loop_failed = false; + match send_browser_window_event(&runtime, &context) { + Ok(()) => events_sent = events_sent.saturating_add(1), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser window heartbeat", + &err, + ); + } + } + match send_browser_category_health(&runtime, &context, events_sent) { + Ok(()) => events_sent = events_sent.saturating_add(1), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser category heartbeat", + &err, + ); + } + } + if let Some(observation) = build_browser_url_observation(&context, &category_rules) { + match send_browser_web_event(&runtime, &context, &observation) { + Ok(()) => events_sent = events_sent.saturating_add(1), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser web heartbeat", + &err, + ); + } + } + match send_browser_category_event(&runtime, &context, &observation) { + Ok(()) => events_sent = events_sent.saturating_add(1), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser category event", + &err, + ); + } + } + match send_matching_web_dlp_incident( + &runtime, + &context, + &observation, + &dlp_policy, + &mut incident_cooldown, + ) { + Ok(sent) => events_sent = events_sent.saturating_add(sent), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser dlp incident", + &err, + ); + } + } + } + if !loop_failed { + problems.clear(); + } + let state = collector_state( + "aw-windows-telemetry.browser-domains-collector.v1", + &runtime, + if loop_failed { "warn" } else { "ok" }, + events_sent, + send_failures, + &problems, + ); + if let Err(err) = save_json_file(&runtime.state_path, &state) { + let _ = append_log( + &runtime.log_path, + &format!( + "state write failed path={} err={err:#}", + runtime.state_path.display() + ), + ); + } + if args.once || deadline.is_some_and(|until| Utc::now() >= until) { + println!("{}", serde_json::to_string_pretty(&state)?); + break; + } + std::thread::sleep(Duration::from_secs(args.poll_seconds.max(1))); + } + Ok(()) +} + +fn run_dlp_endpoint_collector(args: DlpEndpointCollector) -> Result<()> { + if args.self_test { + let policy = load_dlp_policy_for_config(&args.config_path); + let suppressed = resolve_dlp_effective_action(&policy, "block", "clipboard"); + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "ok": true, + "collector": "dlp-endpoint-collector", + "sessionId": current_session_id(), + "semantics": { + "policySource": policy.source, + "clipboardSignalType": "clipboard_change", + "usbSignalType": "usb_insert", + "printSignalType": "print_job", + "suppressedBlockDecision": { + "requestedAction": suppressed.requested_action, + "action": suppressed.action, + "enforcementMode": suppressed.enforcement_mode, + "nativeChannelAction": suppressed.native_channel_action, + "enforcementSuppressed": suppressed.enforcement_suppressed + } + } + }))? + ); + return Ok(()); + } + + let runtime = build_rust_collector_runtime( + &args.config_path, + &args.mode, + args.pulse_seconds, + "dlp-endpoint-rust", + )?; + append_log( + &runtime.log_path, + &format!( + "dlp endpoint rust started mode={} session={}", + runtime.mode, runtime.session_id + ), + )?; + let policy = load_dlp_policy(&runtime.policy_path); + let mut endpoint_state = EndpointCollectorState::default(); + let deadline = args + .duration_seconds + .map(|seconds| Utc::now() + chrono::Duration::seconds(seconds as i64)); + let mut events_sent = 0u64; + let mut send_failures = 0u64; + let mut problems = Vec::::new(); + loop { + let mut loop_failed = false; + match send_endpoint_health_event(&runtime, events_sent) { + Ok(()) => { + events_sent = events_sent.saturating_add(1); + problems.clear(); + } + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "dlp endpoint heartbeat", + &err, + ); + } + } + match send_endpoint_self_test_event(&runtime, events_sent) { + Ok(()) => { + events_sent = events_sent.saturating_add(1); + if !loop_failed { + problems.clear(); + } + } + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "dlp endpoint self-test heartbeat", + &err, + ); + } + } + if policy.defaults_enabled { + match process_clipboard_endpoint_signal(&runtime, &policy, &mut endpoint_state) { + Ok(sent) => events_sent = events_sent.saturating_add(sent), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "clipboard endpoint signal", + &err, + ); + } + } + match process_usb_endpoint_signals(&runtime, &policy, &mut endpoint_state) { + Ok(sent) => events_sent = events_sent.saturating_add(sent), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "usb endpoint signal", + &err, + ); + } + } + match process_print_endpoint_signals(&runtime, &policy, &mut endpoint_state) { + Ok(sent) => events_sent = events_sent.saturating_add(sent), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "print endpoint signal", + &err, + ); + } + } + } + let state = collector_state( + "aw-windows-telemetry.dlp-endpoint-collector.v1", + &runtime, + if loop_failed { "warn" } else { "ok" }, + events_sent, + send_failures, + &problems, + ); + if let Err(err) = save_json_file(&runtime.state_path, &state) { + let _ = append_log( + &runtime.log_path, + &format!( + "state write failed path={} err={err:#}", + runtime.state_path.display() + ), + ); + } + if args.once || deadline.is_some_and(|until| Utc::now() >= until) { + println!("{}", serde_json::to_string_pretty(&state)?); + break; + } + std::thread::sleep(Duration::from_secs(args.poll_seconds.max(1))); + } + Ok(()) +} + +fn build_rust_collector_runtime( + config_path: &Path, + mode: &str, + pulse_seconds: u64, + name: &str, +) -> Result { + let config = read_json_file(config_path)?; + let state_root = json_string(&config, &["paths", "stateRoot"]) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData\AWatch-rus")); + let logs_root = json_string(&config, &["paths", "logsRoot"]) + .map(PathBuf::from) + .unwrap_or_else(|| state_root.join("logs")); + let rules_path = json_string(&config, &["paths", "rulesPath"]) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData\AWatch-rus\web-category-rules.json")); + let policy_path = json_string(&config, &["paths", "policyPath"]) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(r"C:\ProgramData\AWatch-rus\dlp-policy.json")); + let incident_screenshot_enabled = + json_bool(&config, &["incidentCapture", "screenshotEnabled"]).unwrap_or(true); + let scheme = json_string(&config, &["server", "scheme"]).unwrap_or_else(|| "http".to_string()); + let host = json_string(&config, &["server", "host"]).unwrap_or_else(|| "localhost".to_string()); + let port = json_i64(&config, &["server", "port"]).unwrap_or(5600); + let hostname = json_string(&config, &["awHostname"]) + .filter(|value| !value.trim().is_empty()) + .or_else(|| env::var("COMPUTERNAME").ok()) + .unwrap_or_else(|| "unknown".to_string()); + let username = env::var("USERNAME") + .or_else(|_| env::var("USER")) + .unwrap_or_else(|_| "unknown".to_string()); + let session_id = current_session_id(); + let runtime_token = queue_name_token(&username, session_id); + Ok(RustCollectorRuntime { + api_base: format!("{scheme}://{host}:{port}/api/0"), + hostname, + username, + session_id, + mode: mode.to_string(), + pulse_seconds: pulse_seconds.max(1), + state_root: state_root.clone(), + log_path: logs_root.join(format!("{name}-{runtime_token}.log")), + state_path: state_root.join(format!("{name}-{runtime_token}-state.json")), + rules_path, + policy_path, + incident_screenshot_enabled, + }) +} + +fn send_browser_window_event( + runtime: &RustCollectorRuntime, + context: &ForegroundWindowContext, +) -> Result<()> { + let bucket_id = format!("aw-watcher-window_{}", runtime.hostname); + let data = json!({ + "app": context.app, + "title": context.title, + "processId": context.process_id, + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "aw-windows-telemetry-rust" + }); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-watcher-window", + "currentwindow", + data, + ) +} + +fn send_browser_category_health( + runtime: &RustCollectorRuntime, + context: &ForegroundWindowContext, + events_sent: u64, +) -> Result<()> { + let bucket_id = format!("aw-detmir-web-category_{}", runtime.hostname); + let browser_detected = browser_key_from_app(&context.app).is_some(); + let url_detected = if browser_detected { + browser_url_from_foreground_window(context) + .and_then(|url| normalize_browser_url(&url)) + .is_some() + } else { + false + }; + let data = json!({ + "signalType": "collector_health", + "title": context.title, + "app": context.app, + "processId": context.process_id, + "foregroundProcess": context.app, + "foregroundTitle": context.title, + "browserDetected": browser_detected, + "urlDetected": url_detected, + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "eventsSent": events_sent, + "source": "aw-windows-telemetry-rust" + }); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-detmir-web-category", + "aw.web.category", + data, + ) +} + +fn build_browser_url_observation( + context: &ForegroundWindowContext, + category_rules: &[WebCategoryRule], +) -> Option { + let browser = browser_key_from_app(&context.app)?; + let raw_url = browser_url_from_foreground_window(context)?; + let url = normalize_browser_url(&raw_url)?; + let domain = host_from_url(&url)?; + let root_domain = root_domain(&domain); + let category = web_category_for_domain(&domain, category_rules); + Some(BrowserUrlObservation { + url, + browser, + domain, + root_domain, + category, + }) +} + +fn send_browser_web_event( + runtime: &RustCollectorRuntime, + context: &ForegroundWindowContext, + observation: &BrowserUrlObservation, +) -> Result<()> { + let bucket_id = format!( + "aw-watcher-web-{}_{}", + observation.browser, runtime.hostname + ); + let data = json!({ + "url": observation.url, + "title": context.title, + "browser": observation.browser, + "app": context.app, + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "uia-native-rust" + }); + send_collector_aw_event( + runtime, + &bucket_id, + &format!("aw-watcher-web-{}", observation.browser), + "web.tab.current", + data, + ) +} + +fn send_browser_category_event( + runtime: &RustCollectorRuntime, + context: &ForegroundWindowContext, + observation: &BrowserUrlObservation, +) -> Result<()> { + let bucket_id = format!("aw-detmir-web-category_{}", runtime.hostname); + let data = json!({ + "url": observation.url, + "title": context.title, + "browser": observation.browser, + "app": context.app, + "domain": observation.domain, + "rootDomain": observation.root_domain, + "category": observation.category.name, + "categoryGroup": observation.category.group, + "categoryRule": observation.category.rule, + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "uia-native-rust" + }); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-detmir-web-category", + "aw.web.category", + data, + ) +} + +fn send_matching_web_dlp_incident( + runtime: &RustCollectorRuntime, + context: &ForegroundWindowContext, + observation: &BrowserUrlObservation, + policy: &DlpPolicy, + cooldown: &mut BTreeMap>, +) -> Result { + if !policy.defaults_enabled { + return Ok(0); + } + let Some(rules) = policy.raw.get("rules").and_then(Value::as_array) else { + return Ok(0); + }; + for rule in rules { + if !json_bool_any(rule, &["enabled"]).unwrap_or(true) { + continue; + } + let rule_id = json_string_any(rule, &["id"]).unwrap_or_default(); + if rule_id.trim().is_empty() { + continue; + } + if !web_dlp_rule_matches(rule, context, observation) { + continue; + } + let cooldown_seconds = json_i64_any(rule, &["cooldownSeconds"]) + .unwrap_or(policy.defaults_cooldown_seconds) + .max(30); + let fingerprint = format!( + "web|{}|{}|{}", + rule_id, observation.root_domain, runtime.username + ); + if !should_emit_by_cooldown(cooldown, &fingerprint, cooldown_seconds) { + return Ok(0); + } + let action = + json_string_any(rule, &["action"]).unwrap_or_else(|| policy.defaults_action.clone()); + if !matches!( + action.to_ascii_lowercase().as_str(), + "alert" | "block" | "quarantine" | "log" + ) { + return Ok(0); + } + let severity = json_string_any(rule, &["severity"]) + .unwrap_or_else(|| policy.defaults_severity.clone()); + let message = json_string_any(rule, &["message"]) + .unwrap_or_else(|| format!("DLP web rule matched: {rule_id}")); + let data = json!({ + "ruleId": rule_id, + "action": action, + "severity": severity, + "message": message, + "url": observation.url, + "title": context.title, + "browser": observation.browser, + "app": context.app, + "domain": observation.domain, + "rootDomain": observation.root_domain, + "category": observation.category.name, + "categoryGroup": observation.category.group, + "username": runtime.username, + "hostname": runtime.hostname, + "sessionId": runtime.session_id, + "source": "uia-native-dlp-rust", + "screenshotEnabled": runtime.incident_screenshot_enabled, + "screenshotCaptured": false + }); + send_dlp_incident_event(runtime, data)?; + return Ok(1); + } + Ok(0) +} + +fn web_dlp_rule_matches( + rule: &Value, + context: &ForegroundWindowContext, + observation: &BrowserUrlObservation, +) -> bool { + let when = rule.get("when").unwrap_or(&Value::Null); + let hour_from = json_i64_any(when, &["hourFrom"]); + let hour_to = json_i64_any(when, &["hourTo"]); + if !dlp_time_window_matches( + Utc::now().with_timezone(&chrono::Local).hour() as i64, + hour_from, + hour_to, + ) { + return false; + } + let domains = json_string_array(when, "domains"); + if !domains.is_empty() + && !domain_list_matches(&observation.domain, &domains) + && !domain_list_matches(&observation.root_domain, &domains) + { + return false; + } + let category_groups = json_string_array_lower(when, "categoryGroups"); + if !category_groups.is_empty() + && !category_groups.contains(&observation.category.group.to_ascii_lowercase()) + { + return false; + } + let categories = json_string_array_lower(when, "categories"); + if !categories.is_empty() + && !categories.contains(&observation.category.name.to_ascii_lowercase()) + { + return false; + } + let browsers = json_string_array_lower(when, "browsers"); + if !browsers.is_empty() && !browsers.contains(&observation.browser.to_ascii_lowercase()) { + return false; + } + if let Some(pattern) = json_string_any(when, &["urlRegex"]) { + if !regex_matches(&pattern, &observation.url) { + return false; + } + } + if let Some(pattern) = json_string_any(when, &["titleRegex"]) { + if !regex_matches(&pattern, &context.title) { + return false; + } + } + true +} + +fn browser_key_from_app(app: &str) -> Option { + let normalized = app.trim().trim_end_matches(".exe").to_ascii_lowercase(); + let key = match normalized.as_str() { + "msedge" => "edge", + "chrome" => "chrome", + "brave" => "brave", + "vivaldi" => "vivaldi", + "opera" => "opera", + "firefox" => "firefox", + _ => return None, + }; + Some(key.to_string()) +} + +fn normalize_browser_url(value: &str) -> Option { + let candidate = value.trim(); + if candidate.len() < 4 { + return None; + } + let lower = candidate.to_ascii_lowercase(); + if lower.starts_with("search") + || lower.starts_with("find") + || lower.starts_with("address and search") + || lower.starts_with("search with") + || lower.starts_with("new tab") + || lower.starts_with("новая вкладка") + { + return None; + } + if Regex::new(r"(?i)^(https?|file|ftp|chrome|edge|about|view-source)://") + .ok() + .is_some_and(|re| re.is_match(candidate)) + { + return Some(candidate.to_string()); + } + if Regex::new(r"(?i)^localhost([/:]|$)") + .ok() + .is_some_and(|re| re.is_match(candidate)) + { + return Some(format!("http://{candidate}")); + } + if Regex::new(r"^[a-z0-9.-]+\.[a-z]{2,}([/:?#].*)?$") + .ok() + .is_some_and(|re| re.is_match(&lower)) + { + return Some(format!("https://{candidate}")); + } + None +} + +fn host_from_url(value: &str) -> Option { + let parsed = Url::parse(value).ok()?; + let mut host = parsed.host_str()?.to_ascii_lowercase(); + if let Some(stripped) = host.strip_prefix("www.") { + host = stripped.to_string(); + } + Some(host) +} + +fn root_domain(domain: &str) -> String { + let parts = domain.split('.').collect::>(); + if parts.len() <= 2 { + return domain.to_ascii_lowercase(); + } + let suffix = + format!("{}.{}", parts[parts.len() - 2], parts[parts.len() - 1]).to_ascii_lowercase(); + let compound_tlds = [ + "co.uk", "com.au", "co.jp", "com.br", "co.in", "com.tr", "com.cn", + ]; + if compound_tlds.contains(&suffix.as_str()) && parts.len() >= 3 { + return format!("{}.{}", parts[parts.len() - 3], suffix).to_ascii_lowercase(); + } + suffix +} + +fn load_category_rules(path: &Path) -> Vec { + let mut rules = default_category_rules(); + if !path.exists() { + return rules; + } + let Ok(parsed) = read_json_file(path) else { + return rules; + }; + let source_rules = parsed + .get("rules") + .and_then(Value::as_array) + .cloned() + .or_else(|| parsed.as_array().cloned()) + .unwrap_or_default(); + let mut custom = Vec::new(); + for rule in source_rules { + let name = json_string_any(&rule, &["name"]).unwrap_or_default(); + let group = json_string_any(&rule, &["group"]).unwrap_or_default(); + let domains = json_string_array(&rule, "domains"); + if !name.trim().is_empty() && !group.trim().is_empty() && !domains.is_empty() { + custom.push(WebCategoryRule { + name, + group, + domains, + }); + } + } + custom.extend(rules); + rules = custom; + rules +} + +fn default_category_rules() -> Vec { + vec![ + category_rule( + "work_business_systems", + "work", + &[ + "bitrix24.ru", + "1c.ru", + "sbis.ru", + "kontur.ru", + "diadoc.ru", + "nalog.gov.ru", + "gosuslugi.ru", + ], + ), + category_rule( + "work_docs_collab", + "work", + &[ + "office.com", + "sharepoint.com", + "docs.google.com", + "drive.google.com", + "notion.so", + "miro.com", + ], + ), + category_rule( + "work_dev", + "work", + &[ + "github.com", + "gitlab.com", + "bitbucket.org", + "youtrack.cloud", + "atlassian.net", + ], + ), + category_rule( + "work_communication", + "work", + &[ + "teams.microsoft.com", + "outlook.office.com", + "web.telegram.org", + "slack.com", + "zoom.us", + ], + ), + category_rule( + "neutral_search_reference", + "neutral", + &[ + "google.com", + "google.ru", + "yandex.ru", + "bing.com", + "duckduckgo.com", + "wikipedia.org", + ], + ), + category_rule( + "neutral_news", + "neutral", + &[ + "rbc.ru", + "tass.ru", + "ria.ru", + "kommersant.ru", + "vedomosti.ru", + ], + ), + category_rule( + "personal_social", + "personal", + &[ + "vk.com", + "ok.ru", + "facebook.com", + "instagram.com", + "tiktok.com", + "x.com", + "twitter.com", + ], + ), + category_rule( + "personal_video", + "personal", + &[ + "youtube.com", + "youtu.be", + "rutube.ru", + "twitch.tv", + "kinopoisk.ru", + ], + ), + category_rule( + "personal_marketplace", + "personal", + &[ + "ozon.ru", + "wildberries.ru", + "avito.ru", + "aliexpress.com", + "market.yandex.ru", + ], + ), + category_rule( + "personal_entertainment", + "personal", + &["dzen.ru", "pikabu.ru", "dtf.ru", "playground.ru"], + ), + ] +} + +fn category_rule(name: &str, group: &str, domains: &[&str]) -> WebCategoryRule { + WebCategoryRule { + name: name.to_string(), + group: group.to_string(), + domains: domains.iter().map(|domain| domain.to_string()).collect(), + } +} + +fn web_category_for_domain(domain: &str, rules: &[WebCategoryRule]) -> WebCategoryMatch { + for rule in rules { + for rule_domain in &rule.domains { + if domain_matches(domain, rule_domain) { + return WebCategoryMatch { + name: rule.name.clone(), + group: rule.group.clone(), + rule: rule_domain.clone(), + }; + } + } + } + WebCategoryMatch { + name: "uncategorized".to_string(), + group: "neutral".to_string(), + rule: "none".to_string(), + } +} + +fn domain_matches(domain: &str, rule_domain: &str) -> bool { + if domain.trim().is_empty() || rule_domain.trim().is_empty() { + return false; + } + let left = domain.to_ascii_lowercase(); + let right = rule_domain.to_ascii_lowercase(); + left == right || left.ends_with(&format!(".{right}")) +} + +fn domain_list_matches(domain: &str, rules: &[String]) -> bool { + rules.iter().any(|rule| domain_matches(domain, rule)) +} + +fn load_dlp_policy(path: &Path) -> DlpPolicy { + let raw = if path.exists() { + read_json_file(path).unwrap_or(Value::Null) + } else { + Value::Null + }; + let source = if path.exists() { "local" } else { "defaults" }; + dlp_policy_from_value(raw, source) +} + +fn load_dlp_policy_for_config(config_path: &Path) -> DlpPolicy { + if let Ok(config) = read_json_file(config_path) { + if let Some(policy_path) = json_string(&config, &["paths", "policyPath"]) { + return load_dlp_policy(Path::new(&policy_path)); + } + } + dlp_policy_from_value(Value::Null, "defaults") +} + +fn dlp_policy_from_value(raw: Value, source: &str) -> DlpPolicy { + let defaults = raw.get("defaults").unwrap_or(&Value::Null); + let mut native_channel_actions = BTreeMap::new(); + for channel in ["clipboard", "usb", "print"] { + let action = json_string(&raw, &["nativeControls", "channels", channel, "action"]) + .unwrap_or_else(|| "audit".to_string()) + .to_ascii_lowercase(); + native_channel_actions.insert(channel.to_string(), action); + } + DlpPolicy { + content_dictionary_pack: json_string(&raw, &["contentAnalysis", "dictionaryPack"]), + content_regex_pack: json_string(&raw, &["contentAnalysis", "regexPack"]), + content_ocr_enabled: json_bool(&raw, &["contentAnalysis", "ocrEnabled"]).unwrap_or(false), + defaults_enabled: json_bool_any(defaults, &["enabled"]).unwrap_or(true), + defaults_cooldown_seconds: json_i64_any(defaults, &["cooldownSeconds"]).unwrap_or(300), + defaults_action: json_string_any(defaults, &["action"]) + .unwrap_or_else(|| "alert".to_string()), + defaults_severity: json_string_any(defaults, &["severity"]) + .unwrap_or_else(|| "medium".to_string()), + native_mode: json_string(&raw, &["nativeControls", "mode"]) + .unwrap_or_else(|| "monitor".to_string()) + .to_ascii_lowercase(), + native_allow_global_block: json_bool( + &raw, + &["nativeControls", "rollout", "allowGlobalBlock"], + ) + .unwrap_or(false), + native_channel_actions, + raw, + source: source.to_string(), + } +} + +fn resolve_dlp_effective_action( + policy: &DlpPolicy, + requested_action: &str, + channel: &str, +) -> DlpActionDecision { + let requested = requested_action.to_ascii_lowercase(); + let native_channel_action = policy + .native_channel_actions + .get(channel) + .cloned() + .unwrap_or_else(|| "audit".to_string()); + let mut action = requested.clone(); + let mut enforcement_suppressed = false; + if requested == "block" { + let channel_allows_block = matches!( + native_channel_action.as_str(), + "block" | "blockwithoverride" + ); + if policy.native_mode != "enforce" + || !policy.native_allow_global_block + || !channel_allows_block + { + action = "alert".to_string(); + enforcement_suppressed = true; + } + } + DlpActionDecision { + requested_action: requested, + action, + enforcement_mode: policy.native_mode.clone(), + native_channel_action, + enforcement_suppressed, + } +} + +fn should_emit_by_cooldown( + cooldown: &mut BTreeMap>, + fingerprint: &str, + cooldown_seconds: i64, +) -> bool { + let now = Utc::now(); + if let Some(last) = cooldown.get(fingerprint) { + if (now - *last).num_seconds() < cooldown_seconds { + return false; + } + } + cooldown.insert(fingerprint.to_string(), now); + true +} + +fn dlp_time_window_matches( + current_hour: i64, + hour_from: Option, + hour_to: Option, +) -> bool { + let (Some(from), Some(to)) = (hour_from, hour_to) else { + return true; + }; + if from == to { + return true; + } + if from < to { + current_hour >= from && current_hour < to + } else { + current_hour >= from || current_hour < to + } +} + +fn send_dlp_incident_event(runtime: &RustCollectorRuntime, data: Value) -> Result<()> { + let bucket_id = format!("aw-dlp-incidents_{}", runtime.hostname); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-dlp-incidents", + "aw.dlp.incident", + data, + ) +} + +fn regex_matches(pattern: &str, text: &str) -> bool { + Regex::new(pattern) + .map(|regex| regex.is_match(text)) + .unwrap_or(false) +} + +fn send_endpoint_health_event(runtime: &RustCollectorRuntime, events_sent: u64) -> Result<()> { + let bucket_id = format!("aw-dlp-endpoint-signals_{}", runtime.hostname); + let data = json!({ + "signalType": "collector_health", + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "eventsSent": events_sent, + "source": "aw-windows-telemetry-rust", + "mode": runtime.mode + }); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-dlp-endpoint-signals", + "aw.dlp.endpoint.signal", + data, + ) +} + +fn send_endpoint_self_test_event(runtime: &RustCollectorRuntime, events_sent: u64) -> Result<()> { + let bucket_id = format!("aw-dlp-endpoint-signals_{}", runtime.hostname); + let data = json!({ + "signalType": "self_test", + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "queueDepth": 0, + "eventsEnqueued": events_sent, + "eventsFlushed": events_sent, + "sendFailures": 0, + "source": "aw-windows-telemetry-rust", + "mode": runtime.mode + }); + send_collector_aw_event( + runtime, + &bucket_id, + "aw-dlp-endpoint-signals", + "aw.dlp.endpoint.signal", + data, + ) +} + +fn process_clipboard_endpoint_signal( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, +) -> Result { + let Some(text) = read_clipboard_text_safe() else { + return Ok(0); + }; + if text.is_empty() { + return Ok(0); + } + let clipboard_hash = hex_sha256(text.as_bytes()); + if state.last_clipboard_hash.as_deref() == Some(clipboard_hash.as_str()) { + return Ok(0); + } + state.last_clipboard_hash = Some(clipboard_hash.clone()); + let mut sent = 0u64; + send_endpoint_signal_event( + runtime, + "clipboard_change", + json!({ + "clipboardHash": clipboard_hash, + "clipboardLength": text.chars().count() + }), + )?; + sent += 1; + sent += evaluate_clipboard_rules(runtime, policy, state, &text, &clipboard_hash)?; + Ok(sent) +} + +fn process_usb_endpoint_signals( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, +) -> Result { + let drives = enumerate_usb_drives(); + let mut current = HashSet::new(); + let mut sent = 0u64; + for drive in drives { + current.insert(drive.drive_letter.clone()); + if state.seen_usb.contains(&drive.drive_letter) { + continue; + } + state.seen_usb.insert(drive.drive_letter.clone()); + send_endpoint_signal_event( + runtime, + "usb_insert", + json!({ + "driveLetter": drive.drive_letter, + "volumeName": drive.volume_name + }), + )?; + sent += 1; + sent += evaluate_usb_rules(runtime, policy, state, &drive)?; + } + state.seen_usb.retain(|drive| current.contains(drive)); + Ok(sent) +} + +fn process_print_endpoint_signals( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, +) -> Result { + let jobs = enumerate_print_jobs(); + let mut sent = 0u64; + for job in jobs { + if state.seen_print_jobs.contains(&job.id) { + continue; + } + state.seen_print_jobs.insert(job.id.clone()); + send_endpoint_signal_event( + runtime, + "print_job", + json!({ + "printerName": job.printer_name, + "documentName": job.document_name, + "owner": job.owner + }), + )?; + sent += 1; + sent += evaluate_print_rules(runtime, policy, state, &job)?; + } + if state.seen_print_jobs.len() > 500 { + state.seen_print_jobs.clear(); + } + Ok(sent) +} + +fn evaluate_clipboard_rules( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, + text: &str, + clipboard_hash: &str, +) -> Result { + let rules = policy + .raw + .get("endpoint") + .and_then(|endpoint| endpoint.get("clipboard")) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let mut sent = 0u64; + for rule in rules { + if !json_bool_any(&rule, &["enabled"]).unwrap_or(true) { + continue; + } + let rule_id = json_string_any(&rule, &["id"]).unwrap_or_default(); + if rule_id.trim().is_empty() { + continue; + } + let min_length = json_i64_any(&rule, &["minLength"]).unwrap_or(0).max(0) as usize; + if text.chars().count() < min_length { + continue; + } + let regex_patterns = json_string_array(&rule, "regexPatterns"); + let mut matched = regex_patterns + .iter() + .any(|pattern| regex_matches(pattern, text)); + let dictionary_pack = json_string_any(&rule, &["dictionaryPack"]) + .or_else(|| policy.content_dictionary_pack.clone()); + let regex_pack = + json_string_any(&rule, &["regexPack"]).or_else(|| policy.content_regex_pack.clone()); + let ocr_enabled = + json_bool_any(&rule, &["ocrEnabled"]).unwrap_or(policy.content_ocr_enabled); + let advanced = + advanced_content_matches(text, dictionary_pack.as_deref(), regex_pack.as_deref()); + if !advanced.dictionary_matches.is_empty() || !advanced.regex_matches.is_empty() { + matched = true; + } + if !matched { + continue; + } + let cooldown = json_i64_any(&rule, &["cooldownSeconds"]) + .unwrap_or(policy.defaults_cooldown_seconds) + .max(30); + let fingerprint = format!("clipboard|{rule_id}|{clipboard_hash}|{}", runtime.username); + if !should_emit_by_cooldown(&mut state.cooldown, &fingerprint, cooldown) { + continue; + } + let requested_action = + json_string_any(&rule, &["action"]).unwrap_or_else(|| policy.defaults_action.clone()); + let decision = resolve_dlp_effective_action(policy, &requested_action, "clipboard"); + let severity = json_string_any(&rule, &["severity"]) + .unwrap_or_else(|| policy.defaults_severity.clone()); + let message = json_string_any(&rule, &["message"]) + .unwrap_or_else(|| format!("Clipboard rule matched: {rule_id}")); + let data = json!({ + "ruleId": rule_id, + "action": decision.action, + "severity": severity, + "message": message, + "signalType": "clipboard", + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "endpoint-signals-rust", + "clipboardHash": clipboard_hash, + "clipboardLength": text.chars().count(), + "enforced": false, + "requestedAction": decision.requested_action, + "enforcementMode": decision.enforcement_mode, + "nativeChannelAction": decision.native_channel_action, + "enforcementSuppressed": decision.enforcement_suppressed, + "dictionaryPack": dictionary_pack, + "regexPack": regex_pack, + "dictionaryMatches": advanced.dictionary_matches, + "regexMatches": advanced.regex_matches, + "ocrRequested": ocr_enabled, + "screenshotEnabled": runtime.incident_screenshot_enabled, + "screenshotCaptured": false + }); + send_dlp_incident_event(runtime, data)?; + sent += 1; + } + Ok(sent) +} + +fn evaluate_usb_rules( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, + drive: &UsbDrive, +) -> Result { + let rules = policy + .raw + .get("endpoint") + .and_then(|endpoint| endpoint.get("usb")) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let mut sent = 0u64; + for rule in rules { + if !json_bool_any(&rule, &["enabled"]).unwrap_or(true) { + continue; + } + let rule_id = json_string_any(&rule, &["id"]).unwrap_or_default(); + if rule_id.trim().is_empty() { + continue; + } + let cooldown = json_i64_any(&rule, &["cooldownSeconds"]) + .unwrap_or(policy.defaults_cooldown_seconds) + .max(30); + let fingerprint = format!( + "usb|{}|{}|{}", + rule_id, drive.drive_letter, runtime.username + ); + if !should_emit_by_cooldown(&mut state.cooldown, &fingerprint, cooldown) { + continue; + } + let requested_action = + json_string_any(&rule, &["action"]).unwrap_or_else(|| policy.defaults_action.clone()); + let decision = resolve_dlp_effective_action(policy, &requested_action, "usb"); + let severity = json_string_any(&rule, &["severity"]) + .unwrap_or_else(|| policy.defaults_severity.clone()); + let message = json_string_any(&rule, &["message"]) + .unwrap_or_else(|| format!("USB rule matched: {rule_id}")); + let data = json!({ + "ruleId": rule_id, + "action": decision.action, + "severity": severity, + "message": message, + "signalType": "usb_insert", + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "endpoint-signals-rust", + "driveLetter": drive.drive_letter, + "volumeName": drive.volume_name, + "enforced": false, + "requestedAction": decision.requested_action, + "enforcementMode": decision.enforcement_mode, + "nativeChannelAction": decision.native_channel_action, + "enforcementSuppressed": decision.enforcement_suppressed, + "screenshotEnabled": runtime.incident_screenshot_enabled, + "screenshotCaptured": false + }); + send_dlp_incident_event(runtime, data)?; + sent += 1; + } + Ok(sent) +} + +fn evaluate_print_rules( + runtime: &RustCollectorRuntime, + policy: &DlpPolicy, + state: &mut EndpointCollectorState, + job: &PrintJob, +) -> Result { + let rules = policy + .raw + .get("endpoint") + .and_then(|endpoint| endpoint.get("print")) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let mut sent = 0u64; + for rule in rules { + if !json_bool_any(&rule, &["enabled"]).unwrap_or(true) { + continue; + } + let rule_id = json_string_any(&rule, &["id"]).unwrap_or_default(); + if rule_id.trim().is_empty() { + continue; + } + let mut matched = true; + if let Some(pattern) = json_string_any(&rule, &["printerRegex"]) { + matched &= regex_matches(&pattern, &job.printer_name); + } + if let Some(pattern) = json_string_any(&rule, &["documentRegex"]) { + matched &= regex_matches(&pattern, &job.document_name); + } + let dictionary_pack = json_string_any(&rule, &["dictionaryPack"]) + .or_else(|| policy.content_dictionary_pack.clone()); + let regex_pack = + json_string_any(&rule, &["regexPack"]).or_else(|| policy.content_regex_pack.clone()); + let ocr_enabled = + json_bool_any(&rule, &["ocrEnabled"]).unwrap_or(policy.content_ocr_enabled); + let advanced = advanced_content_matches( + &job.document_name, + dictionary_pack.as_deref(), + regex_pack.as_deref(), + ); + if !advanced.dictionary_matches.is_empty() || !advanced.regex_matches.is_empty() { + matched = true; + } + if !matched { + continue; + } + let cooldown = json_i64_any(&rule, &["cooldownSeconds"]) + .unwrap_or(policy.defaults_cooldown_seconds) + .max(30); + let fingerprint = format!( + "print|{}|{}|{}|{}", + rule_id, job.printer_name, job.owner, runtime.username + ); + if !should_emit_by_cooldown(&mut state.cooldown, &fingerprint, cooldown) { + continue; + } + let requested_action = + json_string_any(&rule, &["action"]).unwrap_or_else(|| policy.defaults_action.clone()); + let decision = resolve_dlp_effective_action(policy, &requested_action, "print"); + let severity = json_string_any(&rule, &["severity"]) + .unwrap_or_else(|| policy.defaults_severity.clone()); + let message = json_string_any(&rule, &["message"]) + .unwrap_or_else(|| format!("Print rule matched: {rule_id}")); + let data = json!({ + "ruleId": rule_id, + "action": decision.action, + "severity": severity, + "message": message, + "signalType": "print_job", + "username": runtime.username, + "sessionId": runtime.session_id, + "hostname": runtime.hostname, + "source": "endpoint-signals-rust", + "printerName": job.printer_name, + "documentName": job.document_name, + "owner": job.owner, + "enforced": false, + "requestedAction": decision.requested_action, + "enforcementMode": decision.enforcement_mode, + "nativeChannelAction": decision.native_channel_action, + "enforcementSuppressed": decision.enforcement_suppressed, + "dictionaryPack": dictionary_pack, + "regexPack": regex_pack, + "dictionaryMatches": advanced.dictionary_matches, + "regexMatches": advanced.regex_matches, + "ocrRequested": ocr_enabled, + "screenshotEnabled": runtime.incident_screenshot_enabled, + "screenshotCaptured": false + }); + send_dlp_incident_event(runtime, data)?; + sent += 1; + } + Ok(sent) +} + +fn send_endpoint_signal_event( + runtime: &RustCollectorRuntime, + signal_type: &str, + extra: Value, +) -> Result<()> { + let bucket_id = format!("aw-dlp-endpoint-signals_{}", runtime.hostname); + let mut data = Map::new(); + data.insert("signalType".to_string(), json!(signal_type)); + data.insert("username".to_string(), json!(runtime.username)); + data.insert("sessionId".to_string(), json!(runtime.session_id)); + data.insert("hostname".to_string(), json!(runtime.hostname)); + data.insert("source".to_string(), json!("endpoint-signals-rust")); + if let Some(extra) = extra.as_object() { + for (key, value) in extra { + data.insert(key.clone(), value.clone()); + } + } + send_collector_aw_event( + runtime, + &bucket_id, + "aw-dlp-endpoint-signals", + "aw.dlp.endpoint.signal", + Value::Object(data), + ) +} + +fn send_collector_aw_event( + runtime: &RustCollectorRuntime, + bucket_id: &str, + client_name: &str, + bucket_type: &str, + data: Value, +) -> Result<()> { + if runtime.mode != "enforce" { + return Ok(()); + } + let client = Client::builder().timeout(Duration::from_secs(15)).build()?; + ensure_aw_bucket( + &client, + &runtime.api_base, + bucket_id, + client_name, + bucket_type, + &runtime.hostname, + )?; + let url = format!( + "{}/buckets/{bucket_id}/heartbeat?pulsetime={}", + runtime.api_base.trim_end_matches('/'), + runtime.pulse_seconds + ); + let response = client + .post(&url) + .json(&aw_event_payload(data)) + .send() + .with_context(|| format!("POST {url}"))?; + if !response.status().is_success() { + bail!( + "collector heartbeat failed {} status={}", + bucket_id, + response.status() + ); + } + Ok(()) +} + +fn collector_state( + schema: &str, + runtime: &RustCollectorRuntime, + status: &str, + events_sent: u64, + send_failures: u64, + problems: &[String], +) -> Value { + json!({ + "schema": schema, + "status": status, + "mode": runtime.mode, + "host": runtime.hostname, + "username": runtime.username, + "sessionId": runtime.session_id, + "generatedAtUtc": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "eventsSent": events_sent, + "sendFailures": send_failures, + "stateRoot": runtime.state_root.to_string_lossy(), + "problems": problems + }) +} + +fn record_collector_send_failure( + runtime: &RustCollectorRuntime, + problems: &mut Vec, + send_failures: &mut u64, + operation: &str, + err: &anyhow::Error, +) { + *send_failures = send_failures.saturating_add(1); + let message = format!("{operation} failed: {err:#}"); + problems.push(message.clone()); + if problems.len() > 8 { + let overflow = problems.len().saturating_sub(8); + problems.drain(0..overflow); + } + let _ = append_log(&runtime.log_path, &message); +} + +#[cfg(windows)] +fn browser_url_from_foreground_window(context: &ForegroundWindowContext) -> Option { + if context.window_handle == 0 { + return None; + } + unsafe { + use windows::Win32::Foundation::{HWND, RPC_E_CHANGED_MODE}; + use windows::Win32::System::Com::{ + CLSCTX_INPROC_SERVER, COINIT_APARTMENTTHREADED, CoCreateInstance, CoInitializeEx, + CoUninitialize, + }; + use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationValuePattern, TreeScope_Descendants, + UIA_EditControlTypeId, UIA_ValuePatternId, + }; + + let init_hr = CoInitializeEx(None, COINIT_APARTMENTTHREADED); + let should_uninitialize = init_hr.is_ok(); + if init_hr.is_err() && init_hr != RPC_E_CHANGED_MODE { + return None; + } + let result = (|| { + let automation: IUIAutomation = CoCreateInstance( + &CUIAutomation, + None::<&windows::core::IUnknown>, + CLSCTX_INPROC_SERVER, + ) + .ok()?; + let root = automation + .ElementFromHandle(HWND(context.window_handle as *mut core::ffi::c_void)) + .ok()?; + let condition = automation.CreateTrueCondition().ok()?; + let elements = root.FindAll(TreeScope_Descendants, &condition).ok()?; + let len = elements.Length().ok()?.clamp(0, 160); + for index in 0..len { + let Ok(element) = elements.GetElement(index) else { + continue; + }; + if element.CurrentControlType().ok() != Some(UIA_EditControlTypeId) { + continue; + } + if let Ok(pattern) = + element.GetCurrentPatternAs::(UIA_ValuePatternId) + { + if let Ok(current_value) = pattern.CurrentValue() { + let value = current_value.to_string(); + if normalize_browser_url(&value).is_some() { + return Some(value); + } + } + } + if let Ok(name) = element.CurrentName() { + let value = name.to_string(); + if normalize_browser_url(&value).is_some() { + return Some(value); + } + } + } + None + })(); + if should_uninitialize { + CoUninitialize(); + } + result + } +} + +#[cfg(not(windows))] +fn browser_url_from_foreground_window(_context: &ForegroundWindowContext) -> Option { + None +} + +#[cfg(windows)] +fn read_clipboard_text_safe() -> Option { + use windows_sys::Win32::System::DataExchange::{ + CloseClipboard, GetClipboardData, IsClipboardFormatAvailable, OpenClipboard, + }; + use windows_sys::Win32::System::Memory::{GlobalLock, GlobalUnlock}; + + const CF_UNICODETEXT: u32 = 13; + unsafe { + if IsClipboardFormatAvailable(CF_UNICODETEXT) == 0 { + return None; + } + if OpenClipboard(std::ptr::null_mut()) == 0 { + return None; + } + let handle = GetClipboardData(CF_UNICODETEXT); + if handle.is_null() { + CloseClipboard(); + return None; + } + let ptr = GlobalLock(handle) as *const u16; + if ptr.is_null() { + CloseClipboard(); + return None; + } + let mut len = 0usize; + while len < 1_000_000 && *ptr.add(len) != 0 { + len += 1; + } + let text = String::from_utf16_lossy(std::slice::from_raw_parts(ptr, len)); + GlobalUnlock(handle); + CloseClipboard(); + Some(text) + } +} + +#[cfg(not(windows))] +fn read_clipboard_text_safe() -> Option { + None +} + +#[cfg(windows)] +fn enumerate_usb_drives() -> Vec { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{ + GetDriveTypeW, GetLogicalDrives, GetVolumeInformationW, + }; + + const DRIVE_REMOVABLE: u32 = 2; + let mut out = Vec::new(); + let mask = unsafe { GetLogicalDrives() }; + for index in 0..26u32 { + if mask & (1 << index) == 0 { + continue; + } + let letter = (b'A' + index as u8) as char; + let root = format!("{letter}:\\"); + let root_wide = OsStr::new(&root) + .encode_wide() + .chain(Some(0)) + .collect::>(); + let drive_type = unsafe { GetDriveTypeW(root_wide.as_ptr()) }; + if drive_type != DRIVE_REMOVABLE { + continue; + } + let mut volume = vec![0u16; 260]; + let ok = unsafe { + GetVolumeInformationW( + root_wide.as_ptr(), + volume.as_mut_ptr(), + volume.len() as u32, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + 0, + ) + }; + out.push(UsbDrive { + drive_letter: format!("{letter}:"), + volume_name: if ok != 0 { + utf16_z_to_string(&volume) + } else { + String::new() + }, + }); + } + out +} + +#[cfg(not(windows))] +fn enumerate_usb_drives() -> Vec { + Vec::new() +} + +#[cfg(windows)] +fn enumerate_print_jobs() -> Vec { + let output = Command::new(system32_path("wmic.exe")) + .args([ + "printjob", + "get", + "JobId,Name,Owner,Document", + "/format:csv", + ]) + .output(); + let Ok(output) = output else { + return Vec::new(); + }; + if !output.status.success() { + return Vec::new(); + } + let text = decode_windows_command_output(&output.stdout); + let cleaned = text + .lines() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n"); + if cleaned.trim().is_empty() { + return Vec::new(); + } + let mut reader = csv::ReaderBuilder::new() + .flexible(true) + .from_reader(cleaned.as_bytes()); + let mut jobs = Vec::new(); + for row in reader + .deserialize::>() + .flatten() + { + let id = row.get("JobId").cloned().unwrap_or_default(); + if id.trim().is_empty() { + continue; + } + jobs.push(PrintJob { + id, + printer_name: row.get("Name").cloned().unwrap_or_default(), + document_name: row.get("Document").cloned().unwrap_or_default(), + owner: row.get("Owner").cloned().unwrap_or_default(), + }); + } + jobs +} + +#[cfg(not(windows))] +fn enumerate_print_jobs() -> Vec { + Vec::new() +} + +#[cfg(windows)] +fn decode_windows_command_output(bytes: &[u8]) -> String { + if bytes.len() >= 2 && bytes[0] == 0xff && bytes[1] == 0xfe { + let words = bytes[2..] + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect::>(); + String::from_utf16_lossy(&words) + } else { + String::from_utf8_lossy(bytes).to_string() + } +} + +fn advanced_content_matches( + text: &str, + dictionary_pack: Option<&str>, + regex_pack: Option<&str>, +) -> AdvancedContentMatches { + let mut result = AdvancedContentMatches::default(); + if text.trim().is_empty() { + return result; + } + if dictionary_pack == Some("152-fz-pdn") { + for item in regex_find_all(r"\b\d{10}\b|\b\d{12}\b", text) { + if valid_inn(&item) { + result.dictionary_matches.push(json!({ + "name": "inn", + "value": item, + "severity": "high" + })); + } + } + for item in regex_find_all(r"\b\d{3}-\d{3}-\d{3}\s?\d{2}\b", text) { + if valid_snils(&item) { + result.dictionary_matches.push(json!({ + "name": "snils", + "value": item, + "severity": "high" + })); + } + } + for item in regex_find_all(r"\b\d{4}\s?\d{6}\b", text) { + if valid_passport(&item) { + result.dictionary_matches.push(json!({ + "name": "passport", + "value": item, + "severity": "high" + })); + } + } + } + let regex_rules = match regex_pack.unwrap_or_default() { + "financial" => vec![ + ("card-pan", r"\b(?:\d[ -]*?){13,19}\b", "high"), + ("iban", r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b", "medium"), + ], + "contacts" => vec![ + ( + "email", + r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", + "low", + ), + ( + "phone-ru", + r"(?:\+7|8)\s*\(?\d{3}\)?\s*\d{3}[- ]?\d{2}[- ]?\d{2}", + "low", + ), + ], + "secrets" => vec![ + ("aws-access-key", r"AKIA[0-9A-Z]{16}", "high"), + ( + "generic-password", + r"(?i)(password|пароль)\s*[:=]\s*\S{6,}", + "medium", + ), + ], + _ => Vec::new(), + }; + for (name, pattern, severity) in regex_rules { + for item in regex_find_all(pattern, text) { + result.regex_matches.push(json!({ + "name": name, + "value": item, + "severity": severity + })); + } + } + result +} + +fn regex_find_all(pattern: &str, text: &str) -> Vec { + Regex::new(pattern) + .map(|regex| { + regex + .find_iter(text) + .map(|item| item.as_str().to_string()) + .collect() + }) + .unwrap_or_default() +} + +fn valid_inn(value: &str) -> bool { + let digits = digits_only(value); + match digits.len() { + 10 => { + let coef = [2, 4, 10, 3, 5, 9, 4, 6, 8]; + let sum = coef + .iter() + .enumerate() + .map(|(index, coef)| digit_at(&digits, index) * coef) + .sum::(); + ((sum % 11) % 10) == digit_at(&digits, 9) + } + 12 => { + let c11 = [7, 2, 4, 10, 3, 5, 9, 4, 6, 8]; + let c12 = [3, 7, 2, 4, 10, 3, 5, 9, 4, 6, 8]; + let sum11 = c11 + .iter() + .enumerate() + .map(|(index, coef)| digit_at(&digits, index) * coef) + .sum::(); + let sum12 = c12 + .iter() + .enumerate() + .map(|(index, coef)| digit_at(&digits, index) * coef) + .sum::(); + ((sum11 % 11) % 10) == digit_at(&digits, 10) + && ((sum12 % 11) % 10) == digit_at(&digits, 11) + } + _ => false, + } +} + +fn valid_snils(value: &str) -> bool { + let digits = digits_only(value); + if digits.len() != 11 { + return false; + } + let checksum = digits[9..11].parse::().unwrap_or(999); + let sum = (0..9) + .map(|index| digit_at(&digits, index) * (9 - index as u32)) + .sum::(); + let expected = if sum < 100 { + sum + } else if sum == 100 || sum == 101 { + 0 + } else { + let value = sum % 101; + if value == 100 { 0 } else { value } + }; + checksum == expected +} + +fn valid_passport(value: &str) -> bool { + let digits = digits_only(value); + if digits.len() != 10 || digits == "0000000000" { + return false; + } + digits.chars().collect::>().len() > 1 +} + +fn digits_only(value: &str) -> String { + value.chars().filter(|ch| ch.is_ascii_digit()).collect() +} + +fn digit_at(value: &str, index: usize) -> u32 { + value + .as_bytes() + .get(index) + .map(|byte| (byte.saturating_sub(b'0')) as u32) + .unwrap_or(0) +} + +#[cfg(windows)] +fn foreground_window_context() -> ForegroundWindowContext { + use windows_sys::Win32::UI::WindowsAndMessaging::{ + GetForegroundWindow, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, + }; + + unsafe { + let hwnd = GetForegroundWindow(); + if hwnd.is_null() { + return ForegroundWindowContext::default(); + } + let len = GetWindowTextLengthW(hwnd); + let mut buffer = vec![0u16; (len.max(0) as usize).saturating_add(1)]; + let title = if !buffer.is_empty() { + let copied = GetWindowTextW(hwnd, buffer.as_mut_ptr(), buffer.len() as i32); + String::from_utf16_lossy(&buffer[..copied.max(0) as usize]) + } else { + String::new() + }; + let mut pid = 0u32; + GetWindowThreadProcessId(hwnd, &mut pid); + ForegroundWindowContext { + title, + process_id: pid, + app: process_name_by_pid(pid).unwrap_or_else(|| "unknown".to_string()), + window_handle: hwnd as isize, + } + } +} + +#[cfg(not(windows))] +fn foreground_window_context() -> ForegroundWindowContext { + ForegroundWindowContext::default() +} + +#[cfg(windows)] +fn process_name_by_pid(target_pid: u32) -> Option { + use std::mem::{MaybeUninit, size_of}; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, + TH32CS_SNAPPROCESS, + }; + + let snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) }; + if snapshot == INVALID_HANDLE_VALUE { + return None; + } + let mut entry = PROCESSENTRY32W { + dwSize: size_of::() as u32, + ..unsafe { MaybeUninit::zeroed().assume_init() } + }; + let mut ok = unsafe { Process32FirstW(snapshot, &mut entry) } != 0; + while ok { + if entry.th32ProcessID == target_pid { + let name = utf16_z_to_string(&entry.szExeFile); + unsafe { + CloseHandle(snapshot); + } + return Some(name); + } + ok = unsafe { Process32NextW(snapshot, &mut entry) } != 0; + } + unsafe { + CloseHandle(snapshot); + } + None +} + +fn run_validate_deployment(args: ValidateDeployment) -> Result<()> { + let config = read_json_file(&args.config_path)?; + let state_root = json_string(&config, &["paths", "stateRoot"]) + .unwrap_or_else(|| r"C:\ProgramData\AWatch-rus".to_string()); + let deploy_root = json_string(&config, &["paths", "deployRoot"]) + .or_else(|| json_string(&config, &["paths", "toolkitRoot"])) + .unwrap_or_else(|| r"C:\Program Files\AWatch-rus".to_string()); + let aw_hostname = json_string(&config, &["awHostname"]) + .filter(|v| !v.trim().is_empty()) + .or_else(|| env::var("COMPUTERNAME").ok()) + .unwrap_or_else(|| "unknown".to_string()); + let server_scheme = + json_string(&config, &["server", "scheme"]).unwrap_or_else(|| "http".to_string()); + let server_host = json_string(&config, &["server", "host"]).unwrap_or_default(); + let server_port = json_i64(&config, &["server", "port"]).unwrap_or(5600); + let server_url = format!("{server_scheme}://{server_host}:{server_port}"); + let api_base = format!("{server_url}/api/0"); + + let telemetry_exe = json_string(&config, &["paths", "file1cTelemetryExecutable"]) + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(&deploy_root) + .join("windows") + .join("aw-windows-telemetry.exe") + }); + let agent_exe = PathBuf::from(&state_root) + .join("agent") + .join("awatch-agent-rs.exe"); + let agent_config = PathBuf::from(&state_root) + .join("agent") + .join("awatch-agent.toml"); + let file1c_task_name = json_string(&config, &["analytics", "file1cAutomation", "taskName"]) + .unwrap_or_else(|| "ActivityWatch File1C Upload".to_string()); + let dlp_evidence_task_name = json_string(&config, &["evidenceSync", "taskName"]) + .unwrap_or_else(|| "ActivityWatch DLP Evidence Sync".to_string()); + let rust_agent_task_name = json_string(&config, &["agent", "taskName"]) + .unwrap_or_else(|| "AWatch Rust Telemetry Agent".to_string()); + + let files = validate_files(&[ + args.config_path.clone(), + telemetry_exe.clone(), + agent_exe.clone(), + agent_config.clone(), + ]); + + let file1c_task = query_scheduled_task(&file1c_task_name); + let dlp_evidence_task = query_scheduled_task(&dlp_evidence_task_name); + let rust_agent_task = query_scheduled_task(&rust_agent_task_name); + let file1c_task_ok = task_uses_exe_and_arg(&file1c_task, &telemetry_exe, "file1c-upload"); + let dlp_evidence_task_ok = + task_uses_exe_and_arg(&dlp_evidence_task, &telemetry_exe, "dlp-evidence-sync"); + let rust_agent_task_ok = task_uses_exe_and_arg(&rust_agent_task, &agent_exe, ""); + + let process_snapshot = collect_process_snapshot(); + let power_shell_by_kind = power_shell_runtime_by_kind(&process_snapshot.processes); + let worktime_ps_count = power_shell_by_kind.get("worktime").copied().unwrap_or(0); + let guard_ps_count = power_shell_by_kind.get("guard").copied().unwrap_or(0); + let worktime_ps_ok = if process_snapshot.command_line_query_ok { + worktime_ps_count == 0 + } else { + true + }; + let guard_ps_ok = if process_snapshot.command_line_query_ok { + guard_ps_count == 0 + } else { + true + }; + let rust_agent_running = process_snapshot.processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name.eq_ignore_ascii_case("awatch-agent-rs.exe")) + }); + let rust_collector_guard_running = process_snapshot.processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name.eq_ignore_ascii_case("aw-windows-telemetry.exe")) + && process + .command_line + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase() + .contains("collector-guard") + }); + let collector_guard_service = query_windows_service("AWatchRusCollectorGuard"); + let collector_guard_binary = collector_guard_service + .get("binaryPath") + .and_then(Value::as_str) + .unwrap_or_default() + .to_ascii_lowercase(); + let collector_guard_service_ok = collector_guard_service + .get("running") + .and_then(Value::as_bool) + .unwrap_or(false) + && collector_guard_binary.contains("aw-windows-telemetry.exe") + && collector_guard_binary.contains("collector-guard"); + + let worktime_bucket = get_bucket_health( + &api_base, + &format!("aw-worktime-sessions_{aw_hostname}"), + args.worktime_max_age_seconds, + args.timeout_seconds, + ); + let queue_checks = vec![ + get_queue_group_health( + "endpoint", + Path::new(&state_root), + "dlp-endpoint-signals-queue", + 1000, + ), + get_queue_group_health( + "fileops", + Path::new(&state_root), + "file-operations-queue", + 1000, + ), + ]; + + let migrated_paths_ok = file1c_task_ok + && dlp_evidence_task_ok + && rust_agent_task_ok + && rust_agent_running + && worktime_ps_ok + && guard_ps_ok + && rust_collector_guard_running + && collector_guard_service_ok; + let buckets_ok = worktime_bucket + .get("ok") + .and_then(Value::as_bool) + .unwrap_or(false); + let queues_ok = queue_checks + .iter() + .all(|item| item.get("ok").and_then(Value::as_bool).unwrap_or(false)); + let processes_ok = process_snapshot.query_ok + && rust_agent_running + && worktime_ps_ok + && guard_ps_ok + && rust_collector_guard_running; + let files_ok = files.get("ok").and_then(Value::as_bool).unwrap_or(false); + let tasks_ok = + file1c_task_ok && dlp_evidence_task_ok && rust_agent_task_ok && collector_guard_service_ok; + + let mut failed_sections = Vec::new(); + if !files_ok { + failed_sections.push("files"); + } + if !tasks_ok { + failed_sections.push("tasks"); + } + if !processes_ok { + failed_sections.push("processes"); + } + if !buckets_ok { + failed_sections.push("buckets"); + } + if !queues_ok { + failed_sections.push("queues"); + } + let overall_ok = failed_sections.is_empty(); + + let report = json!({ + "generatedAtUtc": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "schema": "aw-windows-telemetry.validate-deployment.v1", + "configPath": args.config_path.to_string_lossy(), + "serverUrl": server_url, + "apiBase": api_base, + "awHostname": aw_hostname, + "files": files, + "tasks": { + "ok": tasks_ok, + "file1cRustTask": { + "ok": file1c_task_ok, + "expectedExe": telemetry_exe.to_string_lossy(), + "expectedArg": "file1c-upload", + "task": file1c_task + }, + "dlpEvidenceRustTask": { + "ok": dlp_evidence_task_ok, + "expectedExe": telemetry_exe.to_string_lossy(), + "expectedArg": "dlp-evidence-sync", + "task": dlp_evidence_task + }, + "rustWorktimeAgentTask": { + "ok": rust_agent_task_ok, + "expectedExe": agent_exe.to_string_lossy(), + "task": rust_agent_task + }, + "collectorGuardRustService": { + "ok": collector_guard_service_ok, + "expectedExe": telemetry_exe.to_string_lossy(), + "expectedArg": "collector-guard", + "service": collector_guard_service + } + }, + "processes": { + "ok": processes_ok, + "queryOk": process_snapshot.query_ok, + "commandLineQueryOk": process_snapshot.command_line_query_ok, + "queryError": process_snapshot.error, + "rustWorktimeAgentRunning": rust_agent_running, + "rustCollectorGuardRunning": rust_collector_guard_running, + "noPowerShellWorktimeRuntime": if process_snapshot.command_line_query_ok { Value::Bool(worktime_ps_count == 0) } else { Value::Null }, + "noPowerShellWorktimeRuntimeVerified": process_snapshot.command_line_query_ok, + "noPowerShellCollectorGuardRuntime": if process_snapshot.command_line_query_ok { Value::Bool(guard_ps_count == 0) } else { Value::Null }, + "noPowerShellCollectorGuardRuntimeVerified": process_snapshot.command_line_query_ok, + "powerShellRuntimeByKind": power_shell_by_kind, + "legacyPowerShellRuntimeStillPresent": power_shell_by_kind + .iter() + .filter(|(kind, _)| kind.as_str() != "worktime" && kind.as_str() != "guard") + .map(|(_, count)| *count) + .sum::() > 0 + }, + "buckets": { + "ok": buckets_ok, + "worktime": worktime_bucket + }, + "queues": { + "ok": queues_ok, + "list": queue_checks + }, + "migration": { + "ok": migrated_paths_ok, + "phase": "phase2-rust-paths", + "remainingPowerShellIsExpected": true, + "nextTargets": [ + "validate-deployment.ps1 parity", + "browser-domains-native-collector.ps1", + "dlp-endpoint-signals-collector.ps1", + "file-operations-collector.ps1" + ] + }, + "summary": { + "failedSections": failed_sections, + "remainingPowerShellRuntimeByKind": power_shell_by_kind + }, + "overallOk": overall_ok + }); + + println!("{}", serde_json::to_string_pretty(&report)?); + if !overall_ok && args.fail_on_error { + std::process::exit(2); + } + Ok(()) +} + +fn run_collector_guard(args: CollectorGuard) -> Result<()> { + if args.self_test { + collector_guard_self_test()?; + println!("collector guard self-test OK"); + return Ok(()); + } + + let config = read_json_file(&args.config_path)?; + let state_root = json_string(&config, &["paths", "stateRoot"]) + .unwrap_or_else(|| r"C:\ProgramData\AWatch-rus".to_string()); + let logs_root = json_string(&config, &["paths", "logsRoot"]).unwrap_or_else(|| { + PathBuf::from(&state_root) + .join("logs") + .to_string_lossy() + .to_string() + }); + let lock_path = PathBuf::from(&state_root).join("collector-guard-rust.lock"); + let runtime_path = PathBuf::from(&state_root).join("collector-guard-rust-runtime.json"); + let log_path = PathBuf::from(&logs_root).join("collector-guard-rust.log"); + let _lock = GuardLock::acquire(&lock_path)?; + let mut runtime = GuardRuntime::load(&runtime_path)?; + + append_log( + &log_path, + &format!( + "collector guard rust started mode={} loop={} once={}", + args.mode, args.loop_seconds, args.once + ), + )?; + loop { + match run_collector_guard_cycle(&args, &mut runtime) { + Ok(state) => { + save_json_file(&runtime_path, &runtime)?; + let state_path = PathBuf::from(&state_root).join("collector-guard-rust-state.json"); + save_json_file(&state_path, &state)?; + append_log( + &log_path, + &format!( + "cycle status={} actions={} problems={}", + state + .get("status") + .and_then(Value::as_str) + .unwrap_or("unknown"), + state + .get("actions") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0), + state + .get("problems") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0) + ), + )?; + } + Err(err) => { + append_log(&log_path, &format!("cycle error: {err:#}"))?; + } + } + if args.once { + break; + } + std::thread::sleep(Duration::from_secs(args.loop_seconds.max(15))); + } + append_log(&log_path, "collector guard rust stopped")?; + Ok(()) +} + +fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime) -> Result { + let config = read_json_file(&args.config_path)?; + let state_root = json_string(&config, &["paths", "stateRoot"]) + .unwrap_or_else(|| r"C:\ProgramData\AWatch-rus".to_string()); + let aw_hostname = json_string(&config, &["awHostname"]) + .filter(|v| !v.trim().is_empty()) + .or_else(|| env::var("COMPUTERNAME").ok()) + .unwrap_or_else(|| "unknown".to_string()); + let server_scheme = + json_string(&config, &["server", "scheme"]).unwrap_or_else(|| "http".to_string()); + let server_host = json_string(&config, &["server", "host"]).unwrap_or_default(); + let server_port = json_i64(&config, &["server", "port"]).unwrap_or(5600); + let api_base = format!("{server_scheme}://{server_host}:{server_port}/api/0"); + let mut process_snapshot = collect_process_snapshot(); + let mut problems = Vec::new(); + let mut actions = Vec::new(); + + let duplicate_plan = duplicate_legacy_collectors(&process_snapshot.processes); + if !duplicate_plan.is_empty() { + if args.mode == "enforce" { + for duplicate in &duplicate_plan { + let ok = terminate_process(duplicate.pid); + actions.push(json!({ + "action": "dedupe-legacy-collector", + "kind": duplicate.kind, + "sessionId": duplicate.session_id, + "pid": duplicate.pid, + "keptPid": duplicate.keep_pid, + "applied": true, + "ok": ok + })); + if !ok { + problems.push(format!( + "failed to stop duplicate {} collector pid {} in session {}", + duplicate.kind, duplicate.pid, duplicate.session_id + )); + } + } + process_snapshot = collect_process_snapshot(); + } else { + for duplicate in &duplicate_plan { + actions.push(json!({ + "action": "dedupe-legacy-collector", + "kind": duplicate.kind, + "sessionId": duplicate.session_id, + "pid": duplicate.pid, + "keptPid": duplicate.keep_pid, + "applied": false, + "mode": "shadow" + })); + } + } + } + + let power_shell_by_kind = power_shell_runtime_by_kind(&process_snapshot.processes); + let rust_agent_running = process_snapshot.processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name.eq_ignore_ascii_case("awatch-agent-rs.exe")) + }); + let worktime_ps_count = power_shell_by_kind.get("worktime").copied().unwrap_or(0); + let worktime_session_mode = json_string(&config, &["collectors", "worktimeSessionMode"]) + .unwrap_or_else(|| "powershell_primary".to_string()); + let worktime_legacy_fallback_enabled = + json_bool(&config, &["collectors", "worktimeLegacyFallbackEnabled"]).unwrap_or(true); + + let worktime_bucket = get_bucket_health( + &api_base, + &format!("aw-worktime-sessions_{aw_hostname}"), + args.interactive_max_age_seconds, + 15, + ); + let bucket_checks = vec![ + worktime_bucket.clone(), + get_bucket_health( + &api_base, + &format!("aw-watcher-afk_{aw_hostname}"), + args.interactive_max_age_seconds, + 15, + ), + get_bucket_health( + &api_base, + &format!("aw-watcher-window_{aw_hostname}"), + args.interactive_max_age_seconds, + 15, + ), + get_bucket_health( + &api_base, + &format!("aw-dlp-endpoint-signals_{aw_hostname}"), + args.interactive_max_age_seconds, + 15, + ), + ]; + let interactive_stale = bucket_checks + .iter() + .skip(1) + .any(|item| !item.get("ok").and_then(Value::as_bool).unwrap_or(false)); + if worktime_session_mode.eq_ignore_ascii_case("rust_primary") && !rust_agent_running { + problems.push("rust worktime agent is not running".to_string()); + } + if worktime_session_mode.eq_ignore_ascii_case("rust_primary") + && !worktime_legacy_fallback_enabled + && worktime_ps_count > 0 + { + problems.push( + "PowerShell worktime collector is running while fallback is disabled".to_string(), + ); + } + + let task_defs = guard_task_definitions(&config); + if interactive_stale { + let active_legacy_collectors = active_legacy_collector_count(&process_snapshot.processes); + if active_legacy_collectors > 0 && process_snapshot.command_line_query_ok { + problems.push( + "interactive bucket stale but legacy collectors are already running; skip launch tasks to avoid duplicates" + .to_string(), + ); + actions.push(json!({ + "action": "run-task", + "applied": false, + "reason": "legacy-collectors-already-running", + "activeLegacyCollectors": active_legacy_collectors + })); + } else { + for task in &task_defs { + if !task.task_name.starts_with("ActivityWatch Launch ") { + problems.push(format!("refuse non-allowlisted task {}", task.task_name)); + continue; + } + let key = format!("task:{}", task.task_name); + let allowed = runtime.action_allowed( + &key, + args.interactive_action_cooldown_seconds, + args.restart_window_seconds, + args.max_restarts, + ); + if !allowed.allowed { + problems.push(format!("{key} action blocked: {}", allowed.reason)); + continue; + } + if args.mode == "enforce" { + let ok = run_scheduled_task(&task.task_name); + if ok { + runtime.register_action(&key); + } + actions.push(json!({ + "action": "run-task", + "target": task.task_name, + "applied": true, + "ok": ok + })); + } else { + actions.push(json!({ + "action": "run-task", + "target": task.task_name, + "applied": false, + "mode": "shadow" + })); + } + } + } + } else { + for task in &task_defs { + runtime.reset_action_budget(&format!("task:{}", task.task_name)); + } + } + + let status = if problems.is_empty() && (args.mode == "enforce" || !interactive_stale) { + "ok" + } else { + "warn" + }; + let state = json!({ + "schema": "aw-windows-telemetry.collector-guard.v1", + "status": status, + "mode": args.mode, + "host": aw_hostname, + "generatedAtUtc": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "pid": std::process::id(), + "configPath": args.config_path.to_string_lossy(), + "processes": { + "queryOk": process_snapshot.query_ok, + "commandLineQueryOk": process_snapshot.command_line_query_ok, + "rustWorktimeAgentRunning": rust_agent_running, + "powerShellRuntimeByKind": power_shell_by_kind + }, + "buckets": bucket_checks, + "tasks": task_defs.iter().map(|task| json!({ + "taskName": task.task_name, + "userId": task.user_id + })).collect::>(), + "interactiveStale": interactive_stale, + "actions": actions, + "problems": problems, + "quarantine": runtime.quarantine.clone(), + "remainingPowerShellIsExpected": true + }); + let _ = send_guard_heartbeat(&api_base, &state); + let _ = save_json_file( + &PathBuf::from(&state_root).join("collector-guard-state.json"), + &state, + ); + Ok(state) +} + +fn collector_guard_self_test() -> Result<()> { + let mut runtime = GuardRuntime::default(); + let allowed = runtime.action_allowed("task:test", 1, 60, 3); + if !allowed.allowed { + bail!( + "expected initial action to be allowed, got {}", + allowed.reason + ); + } + runtime.register_action("task:test"); + let blocked = runtime.action_allowed("task:test", 300, 60, 3); + if blocked.allowed || blocked.reason != "cooldown" { + bail!("expected cooldown after registered action"); + } + + let mut budget_runtime = GuardRuntime::default(); + for _ in 0..3 { + budget_runtime.register_action("task:budget"); + } + let budget_blocked = budget_runtime.action_allowed("task:budget", 0, 600, 3); + if budget_blocked.allowed || budget_blocked.reason != "quarantine" { + bail!("expected quarantine when restart budget is exhausted"); + } + budget_runtime.reset_action_budget("task:budget"); + let budget_allowed = budget_runtime.action_allowed("task:budget", 0, 600, 3); + if !budget_allowed.allowed { + bail!("expected reset action budget to clear quarantine"); + } + + let config = json!({ + "userTasks": [ + { + "launchTaskName": "ActivityWatch Launch [HOST-EXAMPLE_user]", + "userId": "HOST-EXAMPLE\\user" + }, + { + "LaunchTaskName": "ActivityWatch Launch [HOST-EXAMPLE_admin]", + "UserId": "HOST-EXAMPLE\\admin" + } + ] + }); + let tasks = guard_task_definitions(&config); + if tasks.len() != 2 + || tasks[0].task_name != "ActivityWatch Launch [HOST-EXAMPLE_user]" + || tasks[1].user_id != "HOST-EXAMPLE\\admin" + { + bail!("failed to parse guard task definitions"); + } + Ok(()) +} + +fn guard_task_definitions(config: &Value) -> Vec { + let Some(tasks) = json_at(config, &["userTasks"]).and_then(Value::as_array) else { + return Vec::new(); + }; + let mut out = Vec::new(); + let mut seen = HashSet::new(); + for task in tasks { + let task_name = json_string_any(task, &["launchTaskName", "LaunchTaskName"]) + .unwrap_or_default() + .trim() + .to_string(); + if task_name.is_empty() || !seen.insert(task_name.clone()) { + continue; + } + let user_id = json_string_any(task, &["userId", "UserId"]) + .unwrap_or_default() + .trim() + .to_string(); + out.push(GuardTaskDefinition { task_name, user_id }); + } + out +} + +fn run_scheduled_task(task_name: &str) -> bool { + if !task_name.starts_with("ActivityWatch Launch ") { + return false; + } + Command::new(system32_path("schtasks.exe")) + .args(["/Run", "/TN", task_name]) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +#[derive(Debug, Clone, Serialize)] +struct LegacyCollectorDuplicate { + kind: &'static str, + session_id: u32, + pid: u32, + keep_pid: u32, +} + +fn active_legacy_collector_count(processes: &[ProcessInfo]) -> usize { + processes + .iter() + .filter(|process| legacy_collector_kind(process).is_some()) + .count() +} + +fn duplicate_legacy_collectors(processes: &[ProcessInfo]) -> Vec { + let mut groups: BTreeMap<(&'static str, u32), Vec<&ProcessInfo>> = BTreeMap::new(); + for process in processes { + let Some(kind) = legacy_collector_kind(process) else { + continue; + }; + let Some(session_id) = process.session_id else { + continue; + }; + if process.pid.is_none() { + continue; + } + groups.entry((kind, session_id)).or_default().push(process); + } + + let mut out = Vec::new(); + for ((kind, session_id), mut group) in groups { + if group.len() <= 1 { + continue; + } + group.sort_by_key(|process| { + ( + process.created_unix_seconds.unwrap_or(i64::MIN), + process.pid.unwrap_or(0), + ) + }); + let keep_pid = group.last().and_then(|process| process.pid).unwrap_or(0); + for duplicate in group + .into_iter() + .take_while(|process| process.pid != Some(keep_pid)) + { + if let Some(pid) = duplicate.pid { + out.push(LegacyCollectorDuplicate { + kind, + session_id, + pid, + keep_pid, + }); + } + } + } + out +} + +fn legacy_collector_kind(process: &ProcessInfo) -> Option<&'static str> { + let name = process.name.as_deref().unwrap_or_default(); + if !name.eq_ignore_ascii_case("powershell.exe") && !name.eq_ignore_ascii_case("pwsh.exe") { + return None; + } + match classify_powershell_runtime(process.command_line.as_deref().unwrap_or_default()) { + "browser" => Some("browser"), + "fileops" => Some("fileops"), + "dlp_endpoint" => Some("dlp_endpoint"), + _ => None, + } +} + +fn terminate_process(pid: u32) -> bool { + Command::new(system32_path("taskkill.exe")) + .args(["/PID", &pid.to_string(), "/F"]) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn send_guard_heartbeat(api_base: &str, state: &Value) -> bool { + let host = state + .get("host") + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .unwrap_or("unknown"); + let bucket_id = format!("aw-rus-collector-guard_{host}"); + let client = match Client::builder().timeout(Duration::from_secs(15)).build() { + Ok(client) => client, + Err(_) => return false, + }; + let bucket_url = format!("{}/buckets/{bucket_id}", api_base.trim_end_matches('/')); + let bucket_present = client + .get(&bucket_url) + .send() + .map(|response| response.status().is_success()) + .unwrap_or(false); + if !bucket_present { + let created = client + .post(&bucket_url) + .json(&json!({ + "client": "aw-rus-collector-guard-rust", + "type": "aw.rus.collector.guard", + "hostname": host + })) + .send() + .map(|response| response.status().is_success()) + .unwrap_or(false); + if !created { + return false; + } + } + let heartbeat_url = format!( + "{}/buckets/{bucket_id}/heartbeat?pulsetime=120", + api_base.trim_end_matches('/') + ); + client + .post(heartbeat_url) + .json(&json!({ + "timestamp": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "duration": 0, + "data": state + })) + .send() + .map(|response| response.status().is_success()) + .unwrap_or(false) +} + +#[cfg(windows)] +fn process_id_is_running(pid: u32) -> bool { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; + + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if handle.is_null() { + return false; + } + unsafe { + CloseHandle(handle); + } + true +} + +#[cfg(not(windows))] +fn process_id_is_running(pid: u32) -> bool { + PathBuf::from(format!("/proc/{pid}")).exists() +} + +enum UploadOutcome { + Uploaded { sha: String }, + Skipped, +} + +fn upload_one_evidence_file( + args: &DlpEvidenceSync, + client: &Client, + token: &str, + state: &mut Value, + file: &Path, +) -> Result { + let metadata = fs::metadata(file)?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > args.max_bytes { + return Ok(UploadOutcome::Skipped); + } + let bytes = fs::read(file)?; + let sha = hex_sha256(&bytes); + if already_uploaded(state, &sha, file, &metadata) { + return Ok(UploadOutcome::Skipped); + } + + let response_json = if args.dry_run { + json!({"ok": true, "stored": false, "dryRun": true}) + } else { + let payload = json!({ + "sha256": sha, + "content_base64": general_purpose::STANDARD.encode(&bytes), + "content_type": "image/png", + "source_file": file.file_name().and_then(OsStr::to_str).unwrap_or(""), + "source_path": file.to_string_lossy(), + "hostname": env::var("COMPUTERNAME").unwrap_or_default(), + "username": env::var("USERNAME").unwrap_or_default() + }); + let response = client + .post(&args.evidence_api_url) + .bearer_auth(token) + .json(&payload) + .send() + .with_context(|| format!("POST {}", args.evidence_api_url))?; + let status = response.status(); + if !status.is_success() { + bail!("upload HTTP status {status}"); + } + response + .json::() + .context("parse evidence upload response")? + }; + if response_json.get("ok").and_then(Value::as_bool) != Some(true) { + bail!("upload response is not ok"); + } + mark_uploaded( + state, + &sha, + file, + &metadata, + response_json + .get("stored") + .and_then(Value::as_bool) + .unwrap_or(false), + )?; + Ok(UploadOutcome::Uploaded { sha }) +} + +fn discover_1c_infobases(log_path: &Path) -> Result> { + let mut out = Vec::new(); + let users_root = PathBuf::from(r"C:\Users"); + let Ok(entries) = fs::read_dir(&users_root) else { + return Ok(out); + }; + let mut seen = HashSet::new(); + for entry in entries.flatten() { + let Ok(ft) = entry.file_type() else { continue }; + if !ft.is_dir() { + continue; + } + let user_name = entry.file_name().to_string_lossy().to_string(); + let launcher = entry.path().join(r"AppData\Roaming\1C\1CEStart\ibases.v8i"); + if !launcher.exists() { + continue; + } + match parse_v8i_file(&launcher, &user_name) { + Ok(items) => { + for item in items { + let key = format!("{}\n{}", item.infobase, item.path.display()); + if seen.insert(key) { + out.push(item); + } + } + } + Err(err) => { + append_log( + log_path, + &format!( + "skip unreadable launcher file={} reason={err:#}", + launcher.display() + ), + )?; + } + } + } + Ok(out) +} + +fn parse_v8i_file(path: &Path, user_name: &str) -> Result> { + let raw = fs::read(path)?; + let text = String::from_utf8_lossy(&raw); + Ok(parse_v8i_text(&text, user_name, path)) +} + +fn parse_v8i_text(text: &str, user_name: &str, _launcher_file: &Path) -> Vec { + let mut current_name: Option = None; + let mut current_id: Option = None; + let mut out = Vec::new(); + for raw in text.lines() { + let line = raw.trim(); + if line.starts_with('[') && line.ends_with(']') && line.len() >= 2 { + current_name = Some(line[1..line.len() - 1].to_string()); + current_id = None; + continue; + } + if let Some(rest) = line.strip_prefix("ID=") { + current_id = Some(rest.trim().to_string()); + continue; + } + if let Some(name) = ¤t_name { + if let Some(path) = line + .strip_prefix("Connect=File=\"") + .and_then(|v| v.strip_suffix("\";")) + { + out.push(Infobase { + user_name: user_name.to_string(), + infobase: name.clone(), + base_id: current_id.clone(), + path: PathBuf::from(path), + }); + } + } + } + out +} + +fn latest_lgp(dir: &Path) -> Option { + let mut best: Option<(SystemTime, PathBuf)> = None; + let entries = fs::read_dir(dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + if !path + .extension() + .and_then(OsStr::to_str) + .is_some_and(|ext| ext.eq_ignore_ascii_case("lgp")) + { + continue; + } + let modified = file_modified(&path).unwrap_or(SystemTime::UNIX_EPOCH); + if best.as_ref().is_none_or(|(old, _)| modified > *old) { + best = Some((modified, path)); + } + } + best.map(|(_, path)| path) +} + +fn count_matching_files(path: &Path, predicate: impl Fn(&str) -> bool) -> usize { + let Ok(entries) = fs::read_dir(path) else { + return 0; + }; + entries + .flatten() + .filter(|entry| entry.file_type().map(|ft| ft.is_file()).unwrap_or(false)) + .filter(|entry| predicate(&entry.file_name().to_string_lossy())) + .count() +} + +fn scheduler_touched(current: Option>, previous: Option<&BaseState>) -> bool { + let Some(current) = current else { return false }; + let Some(previous) = previous else { + return false; + }; + if previous.scheduler_write_utc.trim().is_empty() { + return true; + } + DateTime::parse_from_rfc3339(&previous.scheduler_write_utc) + .map(|prev| current > prev.with_timezone(&Utc)) + .unwrap_or(true) +} + +fn company_activity_score( + db_delta_mb: f64, + reglog_delta_mb: f64, + active_locks: usize, + has_temp_db: bool, + scheduler_touched: bool, + status: &str, + is_bootstrap: bool, +) -> f64 { + let mut score = db_delta_mb.abs() + reglog_delta_mb.abs(); + if active_locks > 0 { + score += active_locks as f64 * 5.0; + } + if has_temp_db { + score += 10.0; + } + if scheduler_touched { + score += 3.0; + } + if status == "busy" { + score += 5.0; + } + if is_bootstrap && score <= 0.0 { + score = 1.0; + } + round2(score) +} + +fn host_sample(now: &str, host: &str) -> Value { + let (ram_pct, disk_free_gb) = host_resource_sample(); + json!({ + "ts": now, + "host": host, + "cpu_pct": 0.0, + "ram_pct": ram_pct, + "disk_free_gb": disk_free_gb, + "disk_latency_ms": 0, + "smb_errors": 0, + "rdp_sessions": rdp_session_count(), + "backup_ok": 1 + }) +} + +#[cfg(windows)] +fn host_resource_sample() -> (f64, f64) { + use std::mem::size_of; + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; + use windows_sys::Win32::System::SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX}; + + let mut mem = MEMORYSTATUSEX { + dwLength: size_of::() as u32, + ..unsafe { std::mem::zeroed() } + }; + let ram_pct = if unsafe { GlobalMemoryStatusEx(&mut mem) } != 0 && mem.ullTotalPhys > 0 { + round2(((mem.ullTotalPhys - mem.ullAvailPhys) as f64 / mem.ullTotalPhys as f64) * 100.0) + } else { + 0.0 + }; + let mut free: u64 = 0; + let mut total: u64 = 0; + let mut total_free: u64 = 0; + let mut path: Vec = OsStr::new(r"E:\").encode_wide().chain(Some(0)).collect(); + let disk_free_gb = if unsafe { + GetDiskFreeSpaceExW(path.as_mut_ptr(), &mut free, &mut total, &mut total_free) + } != 0 + { + round2(free as f64 / 1_073_741_824.0) + } else { + 0.0 + }; + (ram_pct, disk_free_gb) +} + +#[cfg(not(windows))] +fn host_resource_sample() -> (f64, f64) { + (0.0, 0.0) +} + +fn rdp_session_count() -> usize { + let query = system32_path("query.exe"); + let output = Command::new(query).arg("user").output(); + match output { + Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout) + .lines() + .skip(1) + .filter(|line| !line.trim().is_empty()) + .count(), + _ => 0, + } +} + +fn temporary_ssh_key(source: &Path, log_path: &Path) -> Result { + let temp = tempfile::Builder::new() + .prefix("awops_ed25519-") + .tempfile() + .context("create temporary ssh key copy")?; + let mut candidates = Vec::new(); + candidates.push(source.to_path_buf()); + if let Ok(profile) = env::var("USERPROFILE") { + candidates.push(PathBuf::from(profile).join(r".ssh\awops_ed25519")); + } + let mut copied = false; + for candidate in candidates { + if candidate.as_os_str().is_empty() || !candidate.exists() { + continue; + } + match fs::copy(&candidate, temp.path()) { + Ok(_) => { + copied = true; + break; + } + Err(err) => { + append_log( + log_path, + &format!( + "skip unusable ssh key path={} reason={err}", + candidate.display() + ), + )?; + } + } + } + if !copied { + bail!("No usable SSH private key found"); + } + let username = env::var("USERNAME").unwrap_or_else(|_| "Users".to_string()); + let _ = Command::new("icacls.exe") + .arg(temp.path()) + .args(["/inheritance:r"]) + .status(); + let _ = Command::new("icacls.exe") + .arg(temp.path()) + .args(["/grant:r", &format!("{username}:(F)")]) + .status(); + Ok(temp) +} + +fn scp_upload(context: &ScpUploadContext<'_>, source: &Path, dataset: &str) -> Result<()> { + let destination = format!( + "{}@{}:{}/{dataset}/", + context.user, + context.host, + context.remote_root.trim_end_matches('/') + ); + for attempt in 1..=3 { + append_log( + context.log_path, + &format!( + "scp attempt={attempt} source={} destination={destination}", + source.display() + ), + )?; + let status = Command::new(context.scp) + .arg("-q") + .arg("-i") + .arg(context.key.path()) + .args([ + "-o", + "LogLevel=ERROR", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=NUL", + ]) + .arg(source) + .arg(&destination) + .status() + .with_context(|| format!("start scp {}", source.display()))?; + if status.success() { + append_log( + context.log_path, + &format!("scp success source={}", source.display()), + )?; + return Ok(()); + } + if attempt == 3 { + bail!( + "scp upload failed after 3 attempts for {} with rc={:?}", + source.display(), + status.code() + ); + } + append_log( + context.log_path, + &format!( + "scp retry source={} rc={:?} delay=5s", + source.display(), + status.code() + ), + )?; + std::thread::sleep(Duration::from_secs(5)); + } + unreachable!() +} + +fn evidence_roots(config: &Value) -> Vec { + let mut roots = Vec::new(); + if let Some(path) = json_string(config, &["incidentCapture", "artifactsRoot"]) { + add_root_if_exists(&mut roots, PathBuf::from(path)); + } + if let Some(state_root) = json_string(config, &["paths", "stateRoot"]) { + add_root_if_exists( + &mut roots, + PathBuf::from(state_root).join("incident-artifacts"), + ); + } + add_root_if_exists( + &mut roots, + PathBuf::from(r"C:\ProgramData\AWatch-rus\incident-artifacts"), + ); + if let Ok(entries) = fs::read_dir(r"C:\Users") { + for entry in entries.flatten() { + if entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false) { + add_root_if_exists( + &mut roots, + entry + .path() + .join(r"AppData\Local\AWatch-rus\incident-artifacts"), + ); + } + } + } + roots +} + +fn add_root_if_exists(roots: &mut Vec, path: PathBuf) { + if path.exists() { + let full = path.canonicalize().unwrap_or(path); + if !roots.iter().any(|p| p == &full) { + roots.push(full); + } + } +} + +fn collect_dlp_evidence_png_files(root: &Path, out: &mut Vec) { + let Ok(entries) = fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(ft) = entry.file_type() else { continue }; + if ft.is_dir() { + collect_dlp_evidence_png_files(&path, out); + } else if ft.is_file() && is_dlp_evidence_screenshot_path(&path) { + out.push(path); + } + } +} + +fn is_dlp_evidence_screenshot_path(path: &Path) -> bool { + if !path + .extension() + .and_then(OsStr::to_str) + .is_some_and(|ext| ext.eq_ignore_ascii_case("png")) + { + return false; + } + let Some(name) = path.file_name().and_then(OsStr::to_str) else { + return false; + }; + is_dlp_evidence_screenshot_name(name) +} + +fn is_dlp_evidence_screenshot_name(name: &str) -> bool { + let lower = name.to_ascii_lowercase(); + let allowed_signal = ["web", "clipboard", "usb_insert", "print_job"] + .iter() + .any(|signal| lower.contains(&format!("_{signal}_"))); + lower.ends_with(".png") + && lower.contains("_sid") + && allowed_signal + && !lower.contains("file1c") + && !lower.contains("1c") +} + +fn already_uploaded(state: &Value, sha: &str, file: &Path, metadata: &fs::Metadata) -> bool { + let Some(entry) = state + .get("uploaded") + .and_then(Value::as_object) + .and_then(|uploaded| uploaded.get(sha)) + else { + return false; + }; + let last_write = metadata + .modified() + .ok() + .map(system_time_o) + .unwrap_or_default(); + entry.get("path").and_then(Value::as_str) == Some(&file.to_string_lossy()) + && entry.get("length").and_then(Value::as_i64) == Some(metadata.len() as i64) + && entry.get("lastWriteUtc").and_then(Value::as_str) == Some(last_write.as_str()) +} + +fn mark_uploaded( + state: &mut Value, + sha: &str, + file: &Path, + metadata: &fs::Metadata, + response_stored: bool, +) -> Result<()> { + ensure_uploaded_object(state); + let uploaded = state + .get_mut("uploaded") + .and_then(Value::as_object_mut) + .ok_or_else(|| anyhow!("state.uploaded is not an object"))?; + uploaded.insert( + sha.to_string(), + json!({ + "path": file.to_string_lossy(), + "length": metadata.len() as i64, + "lastWriteUtc": metadata.modified().ok().map(system_time_o).unwrap_or_default(), + "uploadedAtUtc": Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + "responseStored": response_stored + }), + ); + Ok(()) +} + +fn ensure_uploaded_object(state: &mut Value) { + if !state.is_object() { + *state = json!({}); + } + let object = state.as_object_mut().expect("object ensured"); + if !object.get("uploaded").is_some_and(Value::is_object) { + object.insert("uploaded".to_string(), Value::Object(Map::new())); + } +} + +#[derive(Debug, Default)] +struct ProcessSnapshot { + query_ok: bool, + command_line_query_ok: bool, + error: Option, + processes: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct ProcessInfo { + name: Option, + pid: Option, + session_id: Option, + created_unix_seconds: Option, + command_line: Option, +} + +fn validate_files(paths: &[PathBuf]) -> Value { + let mut list = Vec::new(); + let mut missing = Vec::new(); + for path in paths { + let exists = path.exists(); + if !exists { + missing.push(path.to_string_lossy().to_string()); + } + let metadata = fs::metadata(path).ok(); + list.push(json!({ + "path": path.to_string_lossy(), + "exists": exists, + "size": metadata.as_ref().map(|m| m.len()), + "modifiedUtc": metadata + .and_then(|m| m.modified().ok()) + .map(system_time_o) + })); + } + json!({ + "ok": missing.is_empty(), + "required": list, + "missing": missing + }) +} + +fn query_scheduled_task(task_name: &str) -> Value { + let schtasks = system32_path("schtasks.exe"); + let output = Command::new(&schtasks) + .args(["/Query", "/TN", task_name, "/XML"]) + .output(); + match output { + Ok(out) if out.status.success() => { + let xml = String::from_utf8_lossy(&out.stdout); + let command = xml_tag_text(&xml, "Command"); + let arguments = xml_tag_text(&xml, "Arguments"); + json!({ + "present": true, + "taskName": task_name, + "actionExec": command, + "actionArgs": arguments + }) + } + Ok(out) => json!({ + "present": false, + "taskName": task_name, + "error": String::from_utf8_lossy(&out.stderr).trim() + }), + Err(err) => json!({ + "present": false, + "taskName": task_name, + "error": format!("start {}: {err}", schtasks.display()) + }), + } +} + +fn query_windows_service(service_name: &str) -> Value { + let sc = system32_path("sc.exe"); + let query_output = Command::new(&sc).args(["query", service_name]).output(); + let qc_output = Command::new(&sc).args(["qc", service_name]).output(); + let query_text = query_output + .as_ref() + .ok() + .map(|out| String::from_utf8_lossy(&out.stdout).to_string()) + .unwrap_or_default(); + let qc_text = qc_output + .as_ref() + .ok() + .map(|out| String::from_utf8_lossy(&out.stdout).to_string()) + .unwrap_or_default(); + let running = query_text.to_ascii_uppercase().contains("RUNNING"); + let binary_path = parse_sc_binary_path(&qc_text).unwrap_or_default(); + json!({ + "present": query_output.as_ref().is_ok_and(|out| out.status.success()) + && qc_output.as_ref().is_ok_and(|out| out.status.success()), + "serviceName": service_name, + "running": running, + "binaryPath": binary_path, + "queryError": query_output.as_ref().err().map(|err| err.to_string()), + "configError": qc_output.as_ref().err().map(|err| err.to_string()) + }) +} + +fn parse_sc_binary_path(text: &str) -> Option { + text.lines().find_map(|line| { + let trimmed = line.trim(); + trimmed + .strip_prefix("BINARY_PATH_NAME") + .and_then(|rest| { + rest.split_once(':') + .map(|(_, value)| value.trim().to_string()) + }) + .filter(|value| !value.is_empty()) + }) +} + +fn task_uses_exe_and_arg(task: &Value, expected_exe: &Path, expected_arg: &str) -> bool { + if task.get("present").and_then(Value::as_bool) != Some(true) { + return false; + } + let action_exec = task + .get("actionExec") + .and_then(Value::as_str) + .unwrap_or_default(); + if !path_text_matches(action_exec, expected_exe) { + return false; + } + if expected_arg.is_empty() { + return true; + } + task.get("actionArgs") + .and_then(Value::as_str) + .unwrap_or_default() + .to_ascii_lowercase() + .contains(&expected_arg.to_ascii_lowercase()) +} + +fn path_text_matches(actual: &str, expected: &Path) -> bool { + let actual_norm = normalize_windows_path_text(actual); + let expected_norm = normalize_windows_path_text(&expected.to_string_lossy()); + actual_norm == expected_norm + || actual_norm.ends_with( + &format!( + "\\{}", + expected + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + ) + .to_ascii_lowercase(), + ) +} + +fn normalize_windows_path_text(value: &str) -> String { + value + .trim() + .trim_matches('"') + .replace('/', "\\") + .to_ascii_lowercase() +} + +fn xml_tag_text(xml: &str, tag: &str) -> Option { + let open = format!("<{tag}>"); + let close = format!(""); + let start = xml.find(&open)? + open.len(); + let end = xml[start..].find(&close)? + start; + Some(xml_unescape(xml[start..end].trim())) +} + +fn xml_unescape(value: &str) -> String { + value + .replace(""", "\"") + .replace("'", "'") + .replace("<", "<") + .replace(">", ">") + .replace("&", "&") +} + +fn collect_process_snapshot() -> ProcessSnapshot { + #[cfg(windows)] + if let Some(snapshot) = collect_native_process_snapshot() { + return snapshot; + } + + let wmic = system32_path(r"wbem\wmic.exe"); + let output = Command::new(&wmic) + .args([ + "process", + "get", + "Name,ProcessId,SessionId,CommandLine", + "/format:csv", + ]) + .output(); + match output { + Ok(out) if out.status.success() => { + let processes = parse_wmic_process_csv(&out.stdout); + ProcessSnapshot { + query_ok: true, + command_line_query_ok: true, + error: None, + processes, + } + } + Ok(out) => collect_tasklist_process_snapshot(Some( + String::from_utf8_lossy(&out.stderr).trim().to_string(), + )), + Err(err) => { + collect_tasklist_process_snapshot(Some(format!("start {}: {err}", wmic.display()))) + } + } +} + +#[cfg(windows)] +fn collect_native_process_snapshot() -> Option { + use std::mem::{MaybeUninit, size_of}; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, + TH32CS_SNAPPROCESS, + }; + + let snapshot = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) }; + if snapshot == INVALID_HANDLE_VALUE { + return None; + } + let mut processes = Vec::new(); + let mut entry = PROCESSENTRY32W { + dwSize: size_of::() as u32, + ..unsafe { MaybeUninit::zeroed().assume_init() } + }; + let mut ok = unsafe { Process32FirstW(snapshot, &mut entry) } != 0; + while ok { + let name = utf16_z_to_string(&entry.szExeFile); + let pid = entry.th32ProcessID; + let command_line = read_process_command_line(pid).map(|value| mask_sensitive_text(&value)); + if process_is_relevant(Some(&name), command_line.as_deref()) { + let session_id = process_session_id(pid); + let created_unix_seconds = process_creation_unix_seconds(pid); + processes.push(ProcessInfo { + name: Some(name), + pid: Some(pid), + session_id, + created_unix_seconds, + command_line, + }); + } + ok = unsafe { Process32NextW(snapshot, &mut entry) } != 0; + } + unsafe { + CloseHandle(snapshot); + } + Some(ProcessSnapshot { + query_ok: true, + command_line_query_ok: true, + error: None, + processes, + }) +} + +#[cfg(windows)] +fn process_session_id(pid: u32) -> Option { + use windows_sys::Win32::System::RemoteDesktop::ProcessIdToSessionId; + + let mut session_id = 0u32; + let ok = unsafe { ProcessIdToSessionId(pid, &mut session_id) } != 0; + ok.then_some(session_id) +} + +#[cfg(windows)] +fn process_creation_unix_seconds(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if handle.is_null() { + return None; + } + let mut created = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let mut exited = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let mut kernel = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let mut user = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let ok = + unsafe { GetProcessTimes(handle, &mut created, &mut exited, &mut kernel, &mut user) } != 0; + unsafe { + CloseHandle(handle); + } + if !ok { + return None; + } + filetime_to_unix_seconds(created) +} + +#[cfg(windows)] +fn filetime_to_unix_seconds(value: windows_sys::Win32::Foundation::FILETIME) -> Option { + let ticks = (u64::from(value.dwHighDateTime) << 32) | u64::from(value.dwLowDateTime); + if ticks == 0 { + return None; + } + const WINDOWS_TO_UNIX_EPOCH_100NS: u64 = 116_444_736_000_000_000; + if ticks < WINDOWS_TO_UNIX_EPOCH_100NS { + return None; + } + Some(((ticks - WINDOWS_TO_UNIX_EPOCH_100NS) / 10_000_000) as i64) +} + +#[cfg(windows)] +#[repr(C)] +struct ProcessBasicInformation { + reserved1: *mut std::ffi::c_void, + peb_base_address: *mut std::ffi::c_void, + reserved2: [*mut std::ffi::c_void; 2], + unique_process_id: usize, + reserved3: *mut std::ffi::c_void, +} + +#[cfg(windows)] +#[repr(C)] +struct PartialPeb64 { + reserved: [u8; 0x20], + process_parameters: *mut std::ffi::c_void, +} + +#[cfg(windows)] +#[repr(C)] +#[derive(Clone, Copy)] +struct UnicodeString64 { + length: u16, + maximum_length: u16, + buffer: *const u16, +} + +#[cfg(windows)] +#[link(name = "ntdll")] +unsafe extern "system" { + fn NtQueryInformationProcess( + process_handle: windows_sys::Win32::Foundation::HANDLE, + process_information_class: u32, + process_information: *mut std::ffi::c_void, + process_information_length: u32, + return_length: *mut u32, + ) -> i32; +} + +#[cfg(windows)] +fn read_process_command_line(pid: u32) -> Option { + use std::ffi::c_void; + use std::mem::{MaybeUninit, size_of}; + use std::ptr::null_mut; + use windows_sys::Win32::Foundation::{CloseHandle, HANDLE}; + use windows_sys::Win32::System::Diagnostics::Debug::ReadProcessMemory; + use windows_sys::Win32::System::Threading::{ + OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_VM_READ, + }; + + const PROCESS_BASIC_INFORMATION_CLASS: u32 = 0; + const RTL_USER_PROCESS_PARAMETERS_COMMAND_LINE_OFFSET_X64: usize = 0x70; + + let handle: HANDLE = + unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_VM_READ, 0, pid) }; + if handle.is_null() { + return None; + } + + let mut pbi = MaybeUninit::::zeroed(); + let status = unsafe { + NtQueryInformationProcess( + handle, + PROCESS_BASIC_INFORMATION_CLASS, + pbi.as_mut_ptr().cast::(), + size_of::() as u32, + null_mut(), + ) + }; + if status < 0 { + unsafe { + CloseHandle(handle); + } + return None; + } + let pbi = unsafe { pbi.assume_init() }; + + let mut peb = MaybeUninit::::zeroed(); + let peb_ok = unsafe { + ReadProcessMemory( + handle, + pbi.peb_base_address.cast::(), + peb.as_mut_ptr().cast::(), + size_of::(), + null_mut(), + ) + } != 0; + if !peb_ok { + unsafe { + CloseHandle(handle); + } + return None; + } + let peb = unsafe { peb.assume_init() }; + if peb.process_parameters.is_null() { + unsafe { + CloseHandle(handle); + } + return None; + } + + let command_line_addr = (peb.process_parameters as usize + + RTL_USER_PROCESS_PARAMETERS_COMMAND_LINE_OFFSET_X64) + as *const c_void; + let mut unicode = MaybeUninit::::zeroed(); + let unicode_ok = unsafe { + ReadProcessMemory( + handle, + command_line_addr, + unicode.as_mut_ptr().cast::(), + size_of::(), + null_mut(), + ) + } != 0; + if !unicode_ok { + unsafe { + CloseHandle(handle); + } + return None; + } + let unicode = unsafe { unicode.assume_init() }; + if unicode.buffer.is_null() || unicode.length == 0 || unicode.length > 32768 { + unsafe { + CloseHandle(handle); + } + return None; + } + + let char_count = usize::from(unicode.length) / 2; + let mut buffer = vec![0u16; char_count]; + let read_ok = unsafe { + ReadProcessMemory( + handle, + unicode.buffer.cast::(), + buffer.as_mut_ptr().cast::(), + usize::from(unicode.length), + null_mut(), + ) + } != 0; + unsafe { + CloseHandle(handle); + } + if !read_ok { + return None; + } + Some(String::from_utf16_lossy(&buffer)) +} + +#[cfg(windows)] +fn utf16_z_to_string(value: &[u16]) -> String { + let end = value.iter().position(|ch| *ch == 0).unwrap_or(value.len()); + String::from_utf16_lossy(&value[..end]) +} + +fn collect_tasklist_process_snapshot(wmic_error: Option) -> ProcessSnapshot { + let tasklist = system32_path("tasklist.exe"); + let output = Command::new(&tasklist).args(["/FO", "CSV", "/NH"]).output(); + match output { + Ok(out) if out.status.success() => ProcessSnapshot { + query_ok: true, + command_line_query_ok: false, + error: wmic_error, + processes: parse_tasklist_csv(&out.stdout), + }, + Ok(out) => ProcessSnapshot { + query_ok: false, + command_line_query_ok: false, + error: Some(format!( + "wmic={}; tasklist={}", + wmic_error.unwrap_or_default(), + String::from_utf8_lossy(&out.stderr).trim() + )), + processes: Vec::new(), + }, + Err(err) => ProcessSnapshot { + query_ok: false, + command_line_query_ok: false, + error: Some(format!( + "wmic={}; start {}: {err}", + wmic_error.unwrap_or_default(), + tasklist.display() + )), + processes: Vec::new(), + }, + } +} + +fn parse_wmic_process_csv(bytes: &[u8]) -> Vec { + let mut reader = csv::ReaderBuilder::new().flexible(true).from_reader(bytes); + let Ok(headers) = reader.headers().cloned() else { + return Vec::new(); + }; + let index = |name: &str| { + headers + .iter() + .position(|item| item.eq_ignore_ascii_case(name)) + }; + let command_idx = index("CommandLine"); + let name_idx = index("Name"); + let pid_idx = index("ProcessId"); + let session_idx = index("SessionId"); + let mut out = Vec::new(); + for record in reader.records().flatten() { + let name = name_idx + .and_then(|idx| record.get(idx)) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string); + let command_line = command_idx + .and_then(|idx| record.get(idx)) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(mask_sensitive_text); + if !process_is_relevant(name.as_deref(), command_line.as_deref()) { + continue; + } + out.push(ProcessInfo { + name, + pid: pid_idx + .and_then(|idx| record.get(idx)) + .and_then(|value| value.trim().parse::().ok()), + session_id: session_idx + .and_then(|idx| record.get(idx)) + .and_then(|value| value.trim().parse::().ok()), + created_unix_seconds: None, + command_line, + }); + } + out +} + +fn parse_tasklist_csv(bytes: &[u8]) -> Vec { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .flexible(true) + .from_reader(bytes); + let mut out = Vec::new(); + for record in reader.records().flatten() { + let name = record + .get(0) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string); + let pid = record + .get(1) + .map(|value| value.trim().replace(',', "")) + .and_then(|value| value.parse::().ok()); + let session_id = record + .get(3) + .map(|value| value.trim().replace(',', "")) + .and_then(|value| value.parse::().ok()); + if !process_is_relevant(name.as_deref(), None) { + continue; + } + out.push(ProcessInfo { + name, + pid, + session_id, + created_unix_seconds: None, + command_line: None, + }); + } + out +} + +fn process_is_relevant(name: Option<&str>, command_line: Option<&str>) -> bool { + let name = name.unwrap_or_default().to_ascii_lowercase(); + let command_line = command_line.unwrap_or_default().to_ascii_lowercase(); + name.contains("powershell") + || name == "pwsh.exe" + || name == "awatch-agent-rs.exe" + || name == "aw-windows-telemetry.exe" + || command_line.contains("awatch-rus") + || command_line.contains("activitywatch") + || command_line.contains(".ps1") +} + +fn power_shell_runtime_by_kind(processes: &[ProcessInfo]) -> BTreeMap { + let mut out = BTreeMap::new(); + for process in processes { + let name = process.name.as_deref().unwrap_or_default(); + if !name.eq_ignore_ascii_case("powershell.exe") && !name.eq_ignore_ascii_case("pwsh.exe") { + continue; + } + let kind = classify_powershell_runtime(process.command_line.as_deref().unwrap_or_default()); + *out.entry(kind.to_string()).or_insert(0) += 1; + } + out +} + +fn classify_powershell_runtime(command_line: &str) -> &'static str { + let lower = command_line.to_ascii_lowercase(); + if lower.contains("worktime-session-collector.ps1") { + "worktime" + } else if lower.contains("browser-domains-native-collector.ps1") { + "browser" + } else if lower.contains("file-operations-collector.ps1") { + "fileops" + } else if lower.contains("dlp-endpoint-signals-collector.ps1") { + "dlp_endpoint" + } else if lower.contains("recovery-loop.ps1") { + "recovery" + } else if lower.contains("aw-collector-guard.ps1") { + "guard" + } else if lower.contains("export-upload-hayabusa-to-aw-server.ps1") { + "hayabusa_upload" + } else if lower.contains("export-upload-file-1c-telemetry.ps1") { + "file1c_legacy" + } else if lower.contains("sync-dlp-evidence-artifacts.ps1") { + "dlp_evidence_legacy" + } else { + "other" + } +} + +fn mask_sensitive_text(value: &str) -> String { + let mut out = Vec::new(); + let mut redact_next = false; + for token in value.split_whitespace() { + let lower = token.to_ascii_lowercase(); + if redact_next { + out.push("***".to_string()); + redact_next = false; + continue; + } + if lower.contains("token") || lower.contains("password") || lower.contains("secret") { + if token.contains('=') || token.contains(':') { + out.push( + token + .split_once('=') + .map(|(key, _)| format!("{key}=***")) + .or_else(|| token.split_once(':').map(|(key, _)| format!("{key}:***"))) + .unwrap_or_else(|| "***".to_string()), + ); + } else { + out.push(token.to_string()); + redact_next = true; + } + } else { + out.push(token.to_string()); + } + } + out.join(" ") +} + +fn get_bucket_health( + api_base: &str, + bucket_id: &str, + max_age_seconds: i64, + timeout_seconds: u64, +) -> Value { + let url = format!( + "{}/buckets/{}/events?limit=5", + api_base.trim_end_matches('/'), + bucket_id + ); + let client = match Client::builder() + .timeout(Duration::from_secs(timeout_seconds)) + .build() + { + Ok(client) => client, + Err(err) => { + return json!({ + "bucketId": bucket_id, + "ok": false, + "queryOk": false, + "error": format!("build HTTP client: {err:#}") + }); + } + }; + let response = match client.get(&url).send() { + Ok(response) => response, + Err(err) => { + return json!({ + "bucketId": bucket_id, + "ok": false, + "queryOk": false, + "error": format!("{err:#}") + }); + } + }; + let status = response.status(); + if !status.is_success() { + return json!({ + "bucketId": bucket_id, + "ok": false, + "queryOk": false, + "httpStatus": status.as_u16() + }); + } + let events = response.json::().unwrap_or(Value::Null); + let latest = latest_event_timestamp_utc(&events); + let age_seconds = latest.map(|ts| (Utc::now() - ts).num_seconds()); + let ok = age_seconds.is_some_and(|age| age <= max_age_seconds); + json!({ + "bucketId": bucket_id, + "ok": ok, + "queryOk": true, + "maxAgeSeconds": max_age_seconds, + "latestTimestampUtc": latest.map(|ts| ts.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), + "ageSeconds": age_seconds, + "count": events.as_array().map(Vec::len).unwrap_or(0) + }) +} + +fn latest_event_timestamp_utc(events: &Value) -> Option> { + events + .as_array()? + .iter() + .filter_map(|event| { + event + .get("timestamp") + .and_then(Value::as_str) + .and_then(|raw| DateTime::parse_from_rfc3339(raw).ok()) + .map(|dt| dt.with_timezone(&Utc)) + }) + .max() +} + +fn get_queue_group_health(name: &str, state_root: &Path, prefix: &str, max_depth: usize) -> Value { + let mut queues = Vec::new(); + if let Ok(entries) = fs::read_dir(state_root) { + for entry in entries.flatten() { + let path = entry.path(); + let file_name = path.file_name().and_then(OsStr::to_str).unwrap_or_default(); + if !file_name.starts_with(prefix) || !file_name.ends_with(".jsonl") { + continue; + } + let depth = count_lines(&path); + queues.push(json!({ + "path": path.to_string_lossy(), + "depth": depth, + "sizeBytes": fs::metadata(&path).map(|m| m.len()).unwrap_or(0), + "ok": depth <= max_depth + })); + } + } + let depth_total: usize = queues + .iter() + .filter_map(|queue| queue.get("depth").and_then(Value::as_u64)) + .map(|value| value as usize) + .sum(); + json!({ + "name": name, + "ok": queues.iter().all(|queue| queue.get("ok").and_then(Value::as_bool).unwrap_or(false)), + "queueCount": queues.len(), + "depth": depth_total, + "maxDepth": max_depth, + "queues": queues + }) +} + +fn count_lines(path: &Path) -> usize { + File::open(path) + .map(|file| BufReader::new(file).lines().map_while(Result::ok).count()) + .unwrap_or(0) +} + +fn read_exporter_state(path: &Path) -> Result> { + if !path.exists() { + return Ok(BTreeMap::new()); + } + let value = read_json_file(path)?; + let mut out = BTreeMap::new(); + if let Some(object) = value.as_object() { + for (key, value) in object { + if let Ok(state) = serde_json::from_value::(value.clone()) { + out.insert(key.clone(), state); + } + } + } + Ok(out) +} + +fn write_json_lines(path: &Path, rows: &[Value]) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = File::create(path)?; + for row in rows { + serde_json::to_writer(&mut file, row)?; + file.write_all(b"\n")?; + } + Ok(()) +} + +fn append_log(path: &Path, message: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + writeln!( + file, + "{} {}", + Utc::now().format("%Y-%m-%dT%H:%M:%SZ"), + message + )?; + Ok(()) +} + +fn read_json_file(path: &Path) -> Result { + let raw = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(raw.trim_start_matches('\u{feff}')) + .with_context(|| format!("parse JSON {}", path.display())) +} + +fn save_json_file(path: &Path, value: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("tmp"); + fs::write(&tmp, serde_json::to_vec_pretty(value)?)?; + fs::rename(&tmp, path).or_else(|_| { + fs::copy(&tmp, path)?; + fs::remove_file(&tmp)?; + Ok::<(), std::io::Error>(()) + })?; + Ok(()) +} + +fn json_at<'a>(value: &'a Value, path: &[&str]) -> Option<&'a Value> { + let mut cursor = value; + for key in path { + cursor = cursor.get(*key)?; + } + Some(cursor) +} + +fn json_string(value: &Value, path: &[&str]) -> Option { + json_at(value, path) + .and_then(Value::as_str) + .map(ToString::to_string) +} + +fn json_string_any(value: &Value, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| value.get(*key).and_then(Value::as_str)) + .map(ToString::to_string) +} + +fn json_i64_any(value: &Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| value.get(*key)?.as_i64()) +} + +fn json_bool_any(value: &Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| value.get(*key)?.as_bool()) +} + +fn json_string_array(value: &Value, key: &str) -> Vec { + value + .get(key) + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToString::to_string) + .collect() + }) + .unwrap_or_default() +} + +fn json_string_array_lower(value: &Value, key: &str) -> Vec { + json_string_array(value, key) + .into_iter() + .map(|item| item.to_ascii_lowercase()) + .collect() +} + +fn json_i64(value: &Value, path: &[&str]) -> Option { + json_at(value, path).and_then(Value::as_i64) +} + +fn json_bool(value: &Value, path: &[&str]) -> Option { + json_at(value, path).and_then(Value::as_bool) +} + +fn last_successful_analytics_host(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + let file = File::open(path)?; + let mut matched = None; + for line in BufReader::new(file).lines().map_while(Result::ok) { + if let Some(rest) = line.split("upload complete analyticsHost=").nth(1) { + if let Some(host) = rest.split_whitespace().next() { + matched = Some(host.to_string()); + } + } + } + Ok(matched) +} + +fn file_len(path: &Path) -> Result { + Ok(fs::metadata(path)?.len()) +} + +fn file_modified(path: &Path) -> Result { + Ok(fs::metadata(path)?.modified()?) +} + +fn modified_utc(path: &Path) -> Result> { + Ok(DateTime::::from(file_modified(path)?)) +} + +fn system_time_o(time: SystemTime) -> String { + DateTime::::from(time).to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +fn utc_compact() -> String { + Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string() +} + +fn stable_doc_id(input: &str) -> String { + general_purpose::STANDARD + .encode(input.as_bytes()) + .trim_end_matches('=') + .replace('/', "_") + .replace('+', "-") +} + +fn round2(value: f64) -> f64 { + (value * 100.0).round() / 100.0 +} + +fn hex_sha256(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + digest.iter().map(|b| format!("{b:02x}")).collect() +} + +fn system32_path(relative: &str) -> PathBuf { + env::var("WINDIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(r"C:\Windows")) + .join("System32") + .join(relative) +} + +fn increment_json_i64(value: &mut Value, key: &str, delta: i64) { + let current = value.get(key).and_then(Value::as_i64).unwrap_or(0); + value[key] = Value::from(current + delta); +} + +fn push_json_string(value: &mut Value, key: &str, item: String) { + if !value.get(key).is_some_and(Value::is_array) { + value[key] = Value::Array(Vec::new()); + } + value + .get_mut(key) + .and_then(Value::as_array_mut) + .expect("array ensured") + .push(Value::String(item)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_file_infobases_from_v8i() { + let text = r#"[База 1] +ID=abc-123 +Connect=File="E:\Bases\Org\Base1"; + +[ServerBase] +ID=skip +Connect=Srvr="srv";Ref="x"; +"#; + let items = parse_v8i_text(text, "user1", Path::new("ibases.v8i")); + assert_eq!(items.len(), 1); + assert_eq!(items[0].infobase, "База 1"); + assert_eq!(items[0].base_id.as_deref(), Some("abc-123")); + assert_eq!(items[0].path, PathBuf::from(r"E:\Bases\Org\Base1")); + } + + #[test] + fn activity_score_bootstrap_never_zero() { + assert_eq!( + company_activity_score(0.0, 0.0, 0, false, false, "online", true), + 1.0 + ); + assert_eq!( + company_activity_score(1.234, 2.0, 1, true, true, "busy", false), + 26.23 + ); + } + + #[test] + fn stable_doc_id_matches_powershell_shape() { + let id = stable_doc_id(r"E:\Bases\Org\Base1"); + assert!(!id.contains('/')); + assert!(!id.contains('+')); + assert!(!id.ends_with('=')); + } + + #[test] + fn uploaded_state_detects_same_file() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("a.png"); + fs::write(&file, b"png").unwrap(); + let metadata = fs::metadata(&file).unwrap(); + let sha = hex_sha256(b"png"); + let mut state = json!({"uploaded": {}}); + mark_uploaded(&mut state, &sha, &file, &metadata, true).unwrap(); + assert!(already_uploaded(&state, &sha, &file, &metadata)); + } + + #[test] + fn reads_json_with_utf8_bom() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("config.json"); + fs::write(&file, "\u{feff}{\"ok\":true}").unwrap(); + let value = read_json_file(&file).unwrap(); + assert_eq!(value.get("ok").and_then(Value::as_bool), Some(true)); + } + + #[test] + fn dlp_evidence_sync_accepts_only_dlp_screenshot_names() { + assert!(is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_clipboard_rule-secret.png" + )); + assert!(is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_usb_insert_usb-rule.png" + )); + assert!(is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_print_job_print-rule.png" + )); + assert!(is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_web_web-rule.png" + )); + assert!(!is_dlp_evidence_screenshot_name("1c-work-screenshot.png")); + assert!(!is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_1c_activity.png" + )); + assert!(!is_dlp_evidence_screenshot_name("random.png")); + assert!(!is_dlp_evidence_screenshot_name( + "20260605_120000_001_HOST_user_sid4_clipboard_rule-secret.jpg" + )); + } + + #[test] + fn browser_url_domain_category_matches_legacy_rules() { + let rules = default_category_rules(); + let url = normalize_browser_url("docs.google.com/document/d/1").unwrap(); + let domain = host_from_url(&url).unwrap(); + let root = root_domain("a.b.example.co.uk"); + let category = web_category_for_domain(&domain, &rules); + assert_eq!(url, "https://docs.google.com/document/d/1"); + assert_eq!(domain, "docs.google.com"); + assert_eq!(root, "example.co.uk"); + assert_eq!(category.name, "work_docs_collab"); + assert_eq!(category.group, "work"); + assert!(normalize_browser_url("new tab").is_none()); + } + + #[test] + fn dlp_block_is_suppressed_without_native_enforce() { + let policy = dlp_policy_from_value( + json!({ + "defaults": {"enabled": true}, + "nativeControls": { + "mode": "monitor", + "rollout": {"allowGlobalBlock": false}, + "channels": {"clipboard": {"action": "audit"}} + } + }), + "test", + ); + let decision = resolve_dlp_effective_action(&policy, "block", "clipboard"); + assert_eq!(decision.requested_action, "block"); + assert_eq!(decision.action, "alert"); + assert!(decision.enforcement_suppressed); + } + + #[test] + fn endpoint_advanced_content_matches_legacy_packs() { + let matches = advanced_content_matches("contact user@example.com", None, Some("contacts")); + assert_eq!(matches.regex_matches.len(), 1); + assert_eq!( + matches.regex_matches[0].get("name").and_then(Value::as_str), + Some("email") + ); + let secrets = advanced_content_matches("password: secret123", None, Some("secrets")); + assert_eq!( + secrets.regex_matches[0].get("name").and_then(Value::as_str), + Some("generic-password") + ); + } + + #[test] + fn validate_deployment_extracts_scheduled_task_action_xml() { + let xml = r#" + + + + C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe + file1c-upload --config-path "C:\ProgramData\AWatch-rus\deployment-config.json" + + + +"#; + assert_eq!( + xml_tag_text(xml, "Command").as_deref(), + Some(r"C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe") + ); + assert_eq!( + xml_tag_text(xml, "Arguments").as_deref(), + Some( + r#"file1c-upload --config-path "C:\ProgramData\AWatch-rus\deployment-config.json""# + ) + ); + } + + #[test] + fn validate_deployment_matches_task_exe_and_args() { + let task = json!({ + "present": true, + "actionExec": r"C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe", + "actionArgs": r#"dlp-evidence-sync --state-path "C:\ProgramData\AWatch-rus\state.json""# + }); + assert!(task_uses_exe_and_arg( + &task, + Path::new(r"C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe"), + "dlp-evidence-sync" + )); + assert!(!task_uses_exe_and_arg( + &task, + Path::new(r"C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe"), + "file1c-upload" + )); + } + + #[test] + fn validate_deployment_parses_service_binary_path() { + let text = r#" +[SC] QueryServiceConfig SUCCESS + +SERVICE_NAME: AWatchRusCollectorGuard + TYPE : 10 WIN32_OWN_PROCESS + START_TYPE : 2 AUTO_START + BINARY_PATH_NAME : "C:\Program Files\AWatch-rus\windows\AWatchRusCollectorGuardService.exe" --exec "C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe" --args "collector-guard --mode enforce" +"#; + let path = parse_sc_binary_path(text).unwrap(); + assert!(path.contains("aw-windows-telemetry.exe")); + assert!(path.contains("collector-guard")); + } + + #[test] + fn file_operations_payload_matches_legacy_shape() { + let payload = file_ops_payload( + "Created", + Path::new(r"C:\Users\user\Downloads\archive.zip"), + None, + 1024, + "HOST-EXAMPLE", + "user", + ); + assert_eq!( + payload.get("operation").and_then(Value::as_str), + Some("Created") + ); + assert_eq!( + payload.get("extension").and_then(Value::as_str), + Some(".zip") + ); + assert_eq!( + payload.get("archiveHint").and_then(Value::as_bool), + Some(true) + ); + assert_eq!(payload.get("size").and_then(Value::as_u64), Some(1024)); + } + + #[test] + fn file_operations_queue_token_is_filename_safe() { + let token = queue_name_token(r"DOMAIN\Администратор", 3); + assert!(token.ends_with("-s3")); + assert!(!token.contains('\\')); + assert!( + token + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + ); + assert_eq!(queue_name_token("", 7), "-s7"); + } + + #[test] + fn file_operations_pairs_split_rename_events() { + let mut pending = None; + let from = Event::new(EventKind::Modify(notify::event::ModifyKind::Name( + RenameMode::From, + ))) + .add_path(PathBuf::from(r"C:\tmp\old.txt")); + assert!(file_ops_event_from_notify(&from, &mut pending).is_none()); + let to = Event::new(EventKind::Modify(notify::event::ModifyKind::Name( + RenameMode::To, + ))) + .add_path(PathBuf::from(r"C:\tmp\new.txt")); + let event = file_ops_event_from_notify(&to, &mut pending).unwrap(); + assert_eq!(event.operation, "Renamed"); + assert_eq!( + event.old_path.as_deref(), + Some(Path::new(r"C:\tmp\old.txt")) + ); + assert_eq!(event.path, PathBuf::from(r"C:\tmp\new.txt")); + assert!(pending.is_none()); + } + + #[test] + fn validate_deployment_classifies_powershell_runtime() { + assert_eq!( + classify_powershell_runtime( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\worktime-session-collector.ps1"# + ), + "worktime" + ); + assert_eq!( + classify_powershell_runtime( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\browser-domains-native-collector.ps1"# + ), + "browser" + ); + assert_eq!( + classify_powershell_runtime( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\dlp-endpoint-signals-collector.ps1"# + ), + "dlp_endpoint" + ); + assert_eq!( + classify_powershell_runtime("powershell.exe -EncodedCommand x"), + "other" + ); + } + + #[test] + fn collector_guard_detects_legacy_collector_duplicates_by_session() { + let processes = vec![ + ProcessInfo { + name: Some("powershell.exe".to_string()), + pid: Some(100), + session_id: Some(3), + created_unix_seconds: Some(10), + command_line: Some( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\file-operations-collector.ps1"# + .to_string(), + ), + }, + ProcessInfo { + name: Some("powershell.exe".to_string()), + pid: Some(200), + session_id: Some(3), + created_unix_seconds: Some(20), + command_line: Some( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\file-operations-collector.ps1"# + .to_string(), + ), + }, + ProcessInfo { + name: Some("powershell.exe".to_string()), + pid: Some(300), + session_id: Some(4), + created_unix_seconds: Some(30), + command_line: Some( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\file-operations-collector.ps1"# + .to_string(), + ), + }, + ProcessInfo { + name: Some("powershell.exe".to_string()), + pid: Some(400), + session_id: Some(3), + created_unix_seconds: Some(40), + command_line: Some( + r#"powershell.exe -File C:\ProgramData\AWatch-rus\browser-domains-native-collector.ps1"# + .to_string(), + ), + }, + ]; + + let duplicates = duplicate_legacy_collectors(&processes); + assert_eq!(duplicates.len(), 1); + assert_eq!(duplicates[0].kind, "fileops"); + assert_eq!(duplicates[0].session_id, 3); + assert_eq!(duplicates[0].pid, 100); + assert_eq!(duplicates[0].keep_pid, 200); + assert_eq!(active_legacy_collector_count(&processes), 4); + } + + #[test] + fn validate_deployment_parses_wmic_process_csv() { + let csv = br#"Node,CommandLine,Name,ProcessId,SessionId +HOST,"powershell.exe -File C:\ProgramData\AWatch-rus\file-operations-collector.ps1",powershell.exe,123,4 +HOST,"C:\ProgramData\AWatch-rus\agent\awatch-agent-rs.exe --config x",awatch-agent-rs.exe,456,0 +HOST,,notepad.exe,789,4 +"#; + let processes = parse_wmic_process_csv(csv); + assert_eq!(processes.len(), 2); + let grouped = power_shell_runtime_by_kind(&processes); + assert_eq!(grouped.get("fileops"), Some(&1)); + assert!(processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name == "awatch-agent-rs.exe") + })); + } + + #[test] + fn validate_deployment_parses_tasklist_fallback_csv() { + let csv = br#""awatch-agent-rs.exe","7064","Services","0","12,000 K" +"powershell.exe","11688","Services","0","90,000 K" +"notepad.exe","10","Console","1","1,000 K" +"#; + let processes = parse_tasklist_csv(csv); + assert_eq!(processes.len(), 2); + assert!(processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name == "awatch-agent-rs.exe") + })); + assert!(processes.iter().any(|process| { + process + .name + .as_deref() + .is_some_and(|name| name == "powershell.exe") + })); + } +} diff --git a/adk-rust/crates/detmir-dlp/src/main.rs b/adk-rust/crates/detmir-dlp/src/main.rs index 07a0f66..270e430 100644 --- a/adk-rust/crates/detmir-dlp/src/main.rs +++ b/adk-rust/crates/detmir-dlp/src/main.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result}; use clap::Parser; const DEFAULT_SSH_TARGET: &str = "igor@192.0.2.13"; -const DEFAULT_REMOTE_COMMAND: &str = "sudo -n /usr/local/bin/dlp-health-check --json"; +const DEFAULT_REMOTE_COMMAND: &str = "sudo -n env AW_DLP_HEALTH_ENDPOINT_SEND_FAILURE_WARN_COUNT=10 AW_DLP_HEALTH_FILEOPS_SEND_FAILURE_WARN_COUNT=10 /usr/local/bin/dlp-health-check --json"; #[derive(Debug, Parser)] #[command(about = "Run the DetMir DLP health check on the AW server over SSH.")] diff --git a/adk-rust/crates/detmir-portal/src/main.rs b/adk-rust/crates/detmir-portal/src/main.rs index 23b6541..640ff2e 100644 --- a/adk-rust/crates/detmir-portal/src/main.rs +++ b/adk-rust/crates/detmir-portal/src/main.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::{Arc, Mutex}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; #[cfg(unix)] use std::os::unix::process::CommandExt; @@ -29,7 +29,7 @@ const APP_JS: &str = include_str!("static/app.js"); const API_CONTRACT_OPENAPI: &str = include_str!("contracts/openapi.json"); const API_CONTRACT_TYPESCRIPT: &str = include_str!("contracts/typescript.d.ts"); const UEBA_BASELINE_MIN_SAMPLES: usize = 3; -const SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(5); +const SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(120); const DEFAULT_DEPARTMENT_LABEL: &str = "Без подразделения"; #[cfg(unix)] @@ -1308,10 +1308,7 @@ fn handle_request(request: Request, args: &Cli, snapshot_cache: &SnapshotCache) API_CONTRACT_TYPESCRIPT, "text/plain; charset=utf-8", ), - "/api/health" => respond_json( - request, - &build_health(&cached_snapshot(args, snapshot_cache)), - ), + "/api/health" => respond_json(request, &build_fast_health(snapshot_cache)), "/api/readiness/latest" => respond_json(request, &readiness_latest(args)), "/api/readiness/bundle" => respond_json(request, &readiness_bundle(args)), "/api/readiness/verify" => respond_json(request, &readiness_verify(args)), @@ -1642,6 +1639,27 @@ fn cached_snapshot(args: &Cli, cache: &SnapshotCache) -> Snapshot { snapshot } +fn build_fast_health(cache: &SnapshotCache) -> HealthResponse { + match cache.try_lock() { + Ok(guard) => guard + .as_ref() + .map(|cached| build_health(&cached.snapshot)) + .unwrap_or_else(lightweight_health), + Err(_) => lightweight_health(), + } +} + +fn lightweight_health() -> HealthResponse { + let mut sources = BTreeMap::new(); + sources.insert("portal".to_string(), true); + HealthResponse { + ok: true, + generated_at_utc: now(), + version: env!("CARGO_PKG_VERSION").to_string(), + sources, + } +} + fn build_snapshot(args: &Cli) -> Snapshot { let timeout = Duration::from_secs(args.timeout_seconds); let security_events_config = SecurityEventsConfig { @@ -2630,12 +2648,18 @@ fn security_events_block(summary: &SecurityEventsSummary) -> SummaryBlock { } fn run_shell(command: &str, timeout: Duration) -> Result<(String, String, bool)> { + let stdout_path = command_output_path("stdout"); + let stderr_path = command_output_path("stderr"); + let stdout_file = + File::create(&stdout_path).with_context(|| format!("create {}", stdout_path.display()))?; + let stderr_file = + File::create(&stderr_path).with_context(|| format!("create {}", stderr_path.display()))?; let mut shell = Command::new("/bin/sh"); shell .arg("-lc") .arg(command) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + .stdout(Stdio::from(stdout_file)) + .stderr(Stdio::from(stderr_file)); #[cfg(unix)] unsafe { shell.pre_exec(|| { @@ -2652,19 +2676,15 @@ fn run_shell(command: &str, timeout: Duration) -> Result<(String, String, bool)> let started = Instant::now(); loop { if let Some(status) = child.try_wait()? { - let mut stdout = String::new(); - let mut stderr = String::new(); - if let Some(mut pipe) = child.stdout.take() { - pipe.read_to_string(&mut stdout)?; - } - if let Some(mut pipe) = child.stderr.take() { - pipe.read_to_string(&mut stderr)?; - } + let stdout = read_command_output(&stdout_path)?; + let stderr = read_command_output(&stderr_path)?; + cleanup_command_output(&stdout_path, &stderr_path); return Ok((stdout, stderr, status.success())); } if started.elapsed() > timeout { kill_shell_tree(&mut child); let _ = child.wait(); + cleanup_command_output(&stdout_path, &stderr_path); return Err(anyhow!( "command timed out after {}s: {command}", timeout.as_secs() @@ -2674,6 +2694,27 @@ fn run_shell(command: &str, timeout: Duration) -> Result<(String, String, bool)> } } +fn command_output_path(stream: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + std::env::temp_dir().join(format!( + "detmir-portal-command-{}-{nanos}-{stream}.log", + std::process::id() + )) +} + +fn read_command_output(path: &Path) -> Result { + let bytes = fs::read(path).with_context(|| format!("read {}", path.display()))?; + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} + +fn cleanup_command_output(stdout_path: &Path, stderr_path: &Path) { + let _ = fs::remove_file(stdout_path); + let _ = fs::remove_file(stderr_path); +} + #[cfg(unix)] fn kill_shell_tree(child: &mut std::process::Child) { let pgid = child.id() as i32; @@ -3519,6 +3560,9 @@ fn build_risk_incident_candidates( candidates.extend(low_trust_risk_candidates(risks, snapshot)); candidates.extend(agent_quality_candidates(snapshot, &problem_nodes_by_host)); candidates.extend(agent_coverage_candidates(snapshot)); + if candidates.is_empty() { + candidates.extend(workforce_insight_risk_candidates(snapshot, risks)); + } candidates.sort_by(|left, right| { risk_incident_candidate_rank(right) @@ -4720,6 +4764,64 @@ fn agent_coverage_candidates(snapshot: &Snapshot) -> Vec .collect() } +fn workforce_insight_risk_candidates( + snapshot: &Snapshot, + risks: &[BusinessRiskItem], +) -> Vec { + let primary_risk = risks.first(); + let department = primary_risk + .map(|item| item.department.clone()) + .unwrap_or_else(|| DEFAULT_DEPARTMENT_LABEL.to_string()); + let risk_level = primary_risk + .map(|item| item.risk_level.clone()) + .unwrap_or_else(|| "MEDIUM".to_string()); + let recommendation = primary_risk + .map(|item| item.recommendation.clone()) + .unwrap_or_else(|| { + "Проверить первичные события ActivityWatch и причины отклонения активности.".to_string() + }); + workforce_insight_items(snapshot) + .into_iter() + .filter(|item| { + let status = item.get("status").and_then(Value::as_str).unwrap_or("INFO"); + !matches!(status, "OK" | "INFO") + }) + .take(3) + .map(|item| { + let label = item + .get("label") + .and_then(Value::as_str) + .unwrap_or("Отклонение активности"); + let value = item.get("value").and_then(Value::as_str).unwrap_or(""); + RiskIncidentCandidate { + id: risk_candidate_id( + "workforce-insight", + &department, + &format!("{label}:{value}"), + ), + department: Some(department.clone()), + owner: None, + hostname: None, + risk_level: Some(risk_level.clone()), + reason: Some(label.to_string()), + evidence: vec![ + format!("activity_signal={label}"), + format!("details={value}"), + format!( + "security_events_24h={}", + snapshot.security_events_summary.events_24h + ), + ], + first_seen_utc: Some(snapshot.generated_at_utc.clone()), + last_seen_utc: Some(snapshot.generated_at_utc.clone()), + recommendation: Some(recommendation.clone()), + incident_review: IncidentReviewState::default(), + incident_review_audit: Vec::new(), + } + }) + .collect() +} + fn agent_quality_candidate_level(node: &AgentQualityNodeItem) -> String { match node.status.as_str() { "DEGRADED" => "HIGH", @@ -9458,6 +9560,17 @@ mod tests { assert!(err.to_string().contains("command timed out")); } + #[cfg(unix)] + #[test] + fn run_shell_does_not_block_on_background_stdout_handle() { + let started = Instant::now(); + let (stdout, _, success) = + run_shell("sh -c 'sleep 5 & printf ok'", Duration::from_secs(2)).unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(success); + assert_eq!(stdout, "ok"); + } + #[test] fn command_json_source_accepts_valid_json_with_nonzero_exit() { let source = command_json_source( diff --git a/adk-rust/crates/detmir-portal/src/static/app.js b/adk-rust/crates/detmir-portal/src/static/app.js index e6db6b8..d68595e 100644 --- a/adk-rust/crates/detmir-portal/src/static/app.js +++ b/adk-rust/crates/detmir-portal/src/static/app.js @@ -123,6 +123,10 @@ function displayText(value) { .replaceAll("Grafana dashboards", "графики") .replaceAll("Grafana data", "данные графиков") .replaceAll("Grafana", "Графики") + .replaceAll("SECURITY_EVENTS_BACKEND", "настройка источника событий") + .replaceAll("CLICKHOUSE_*", "параметры источника событий") + .replaceAll("ClickHouse", "источник событий") + .replaceAll("local_fallback", "резервный локальный источник") .replaceAll("DetMir ActivityWatch", "Журнал активности") .replaceAll("AW UI", "Журнал активности") .replaceAll("bundle", "пакет проверки") @@ -166,9 +170,20 @@ function displayText(value) { .replaceAll("daily", "день") .replaceAll("weekly", "неделя") .replaceAll("monthly", "месяц") - .replaceAll("OK", "OK") - .replaceAll("FAIL", "FAIL") - .replaceAll("WARN", "WARN"); + .replace(/\bCRITICAL\b/g, "критично") + .replace(/\bWARNING\b/g, "требует внимания") + .replace(/\bUNKNOWN\b/g, "нет данных") + .replace(/\bDISABLED\b/g, "отключено") + .replace(/\bREADY\b/g, "готово") + .replace(/\bOPEN\b/g, "открыто") + .replace(/\bINFO\b/g, "информация") + .replace(/\bHIGH\b/g, "высокий риск") + .replace(/\bMEDIUM\b/g, "средний риск") + .replace(/\bLOW\b/g, "низкий риск") + .replace(/\bFAIL\b/g, "критично") + .replace(/\bWARN\b/g, "требует внимания") + .replace(/\bNO\b/g, "нет") + .replace(/\bOK\b/g, "в норме"); } function ui(value) { @@ -233,7 +248,7 @@ function setViewMode(mode) { function renderSummary(summary, readiness) { const global = document.getElementById("globalStatus"); global.className = `status-pill ${statusClass(summary.severity)}`; - global.textContent = `Сбор данных ${summary.operator_ok ? "OK" : "NO"} · ${summary.severity}`; + global.textContent = `Сбор данных ${summary.operator_ok ? "в норме" : "нет данных"} · ${displayText(summary.severity)}`; const blocks = Object.entries(summary.blocks || {}); const readinessCard = renderReadinessSummaryCard(readiness); document.getElementById("summary").innerHTML = readinessCard + blocks.map(([name, block]) => ` @@ -268,8 +283,8 @@ function renderReadinessSummaryCard(readiness) {
Дата${escapeHtml(generated)}
-
Подпись${signatureOk ? "OK" : "FAIL"}
-
Контрольная сумма${checksumOk ? "OK" : "FAIL"}
+
Подпись${signatureOk ? "подтверждена" : "не подтверждена"}
+
Контрольная сумма${checksumOk ? "подтверждена" : "не подтверждена"}

Отпечаток ключа: ${escapeHtml(shortFingerprint(fingerprint))}

${escapeHtml(verificationText)}

@@ -774,7 +789,7 @@ function departmentRows(report) { total: parsed.total, hhmm: parsed.hhmm, trend, - risk: status === "FAIL" ? "FAIL — требуется действие" : status === "WARN" ? "WARN — требуется внимание" : "LOW — все нормально", + risk: status === "FAIL" ? "критично — требуется действие" : status === "WARN" ? "требует внимания" : "низкий риск — все нормально", reason: departmentRiskReason({ item, parsed, activity, status, trend }), check: departmentCheckText({ parsed, status, trend }), }; @@ -808,8 +823,8 @@ function executiveDashboardKpis(report) { return [ metricCard("Сотрудников в работе", latest.active_users ?? findKpi(report, "Сотрудники")?.value ?? "нет данных", "OK", "активны сегодня"), metricCard("Средний индекс активности", Number.isFinite(average) ? `${average}%` : findKpi(report, "Индекс активности")?.value, average === null ? "UNKNOWN" : workforceIndexStatus(average), "по подразделениям"), - metricCard("WARN подразделений", rows.filter(row => row.status === "WARN").length, rows.some(row => row.status === "WARN") ? "WARN" : "OK", "требуется внимание"), - metricCard("FAIL подразделений", rows.filter(row => row.status === "FAIL").length, rows.some(row => row.status === "FAIL") ? "FAIL" : "OK", "требуется действие"), + metricCard("Требуют внимания", rows.filter(row => row.status === "WARN").length, rows.some(row => row.status === "WARN") ? "WARN" : "OK", "подразделения"), + metricCard("Критичные подразделения", rows.filter(row => row.status === "FAIL").length, rows.some(row => row.status === "FAIL") ? "FAIL" : "OK", "требуется действие"), metricCard("Критических рисков", criticalRisks, criticalRisks ? "FAIL" : "OK", "приоритетный разбор"), metricCard("Тренд недели", signedPercent(weeklyTrendPct(report)), Number(weeklyTrendPct(report)) < 0 ? "WARN" : "OK", "динамика активности"), ].join(""); @@ -906,7 +921,7 @@ function renderDepartmentRanking(report) { ${ui(row.status)} `).join("")}`; - const emptyRows = `
Нет данныхПодразделения пока не рассчитаны.UNKNOWN
`; + const emptyRows = `
Нет данныхПодразделения пока не рассчитаны.нет данных
`; return `
@@ -994,7 +1009,7 @@ function renderDepartmentHeatMap(report) { Нет данныхПодразделения пока не рассчитаны. - - - UNKNOWN + нет данных - @@ -1149,7 +1164,7 @@ function renderSourceList(data) {
${ui(name)} ${ui(source?.summary || source?.error || "нет данных")} - ${escapeHtml(source?.status || (source?.ok ? "OK" : "FAIL"))} + ${ui(source?.status || (source?.ok ? "OK" : "FAIL"))}
`).join("")}`; } @@ -1243,7 +1258,7 @@ function renderOperationsErrors(data, report) { if (report?.security_events_summary?.fallback_used) { rows.push([ "События безопасности", - report.security_events_summary.error || "ClickHouse недоступен", + report.security_events_summary.error || "Источник событий недоступен", "WARN", ]); } @@ -1282,7 +1297,7 @@ function renderOperationsErrors(data, report) {
Критичных ошибок нет Портал не видит ошибок коллектора или проблемных рабочих мест в текущем срезе. - OK + в норме
`}
@@ -1335,7 +1350,7 @@ function renderInvestigationPacks(candidates) {
Пакетов нет Нет записей, требующих выгрузки материалов. - OK + в норме
`} @@ -1855,7 +1870,7 @@ function renderCases(cases) { Нет дел - - OK + в норме - Создайте дело из подтвержденного кандидата. - @@ -1941,7 +1956,7 @@ function renderUebaRisk(risk) { ${ui(item.value || "")} · ${ui(item.recommendation || "")} +${escapeHtml(item.points || 0)} - `).join("") : `
СигналыСущественных риск-сигналов в текущем срезе нет.OK
`} + `).join("") : `
СигналыСущественных риск-сигналов в текущем срезе нет.в норме
`} `; } @@ -2038,7 +2053,7 @@ function renderAgentQualityHistory(history, summary) { ${ui(status)}
-
OK дней${escapeHtml(s.ok_days ?? 0)}
+
Дней в норме${escapeHtml(s.ok_days ?? 0)}
Проблемных дней${escapeHtml(unstableDays)}
Показатели подтверждены${escapeHtml(s.kpi_accepted_pct ?? 0)}%
@@ -2051,7 +2066,7 @@ function renderAgentQualityHistory(history, summary) { источник=${ui(item.source || "unknown")} · показатели=${item.kpi_accepted ? "да" : "нет"}${item.collector_error ? ` · ${ui(item.collector_error)}` : ""} ${ui(item.status || "UNKNOWN")} - `).join("") : `
ИсторияИстория качества агента за период отсутствует.UNKNOWN
`} + `).join("") : `
ИсторияИстория качества агента за период отсутствует.нет данных
`} `; @@ -2076,7 +2091,7 @@ function renderAgentQualityNodes(nodes, summary) {
Всего узлов${escapeHtml(s.total_nodes ?? 0)}
-
OK${escapeHtml(s.ok_nodes ?? 0)}
+
В норме${escapeHtml(s.ok_nodes ?? 0)}
Проблемных${escapeHtml(Number(s.degraded_nodes || 0) + Number(s.unknown_nodes || 0))}
Показатели подтверждены${escapeHtml(s.accepted_kpi_nodes_pct ?? 0)}%
@@ -2106,7 +2121,7 @@ function renderAgentQualityNodes(nodes, summary) { `).join("") : ` Нет данных - UNKNOWN + нет данных unknown - нет @@ -2207,7 +2222,7 @@ function renderSecurityEventsSummary(summary, options = {}) { : "Агрегированная сводка без сырых журналов и без автоматического создания инцидентов."; const top = Array.isArray(s.top_departments) ? s.top_departments.slice(0, 5) : []; const warning = fallback - ? `
События безопасности временно недоступны. Проверьте ClickHouse и переменные SECURITY_EVENTS_BACKEND/CLICKHOUSE_*.
` + ? `
События безопасности временно недоступны. Проверьте подключение источника событий.
` : ""; if (options.compact) { return ` @@ -2247,7 +2262,7 @@ function renderSecurityEventsSummary(summary, options = {}) { ${warning} ${s.error ? `

Причина: ${ui(s.error)}

` : ""} - ${disabled ? `

Для включения задайте SECURITY_EVENTS_BACKEND=clickhouse и параметры CLICKHOUSE_*.

` : ""} + ${disabled ? `

Источник событий безопасности не включен в текущем режиме.

` : ""} ${options.compact ? "" : `
@@ -2313,7 +2328,7 @@ function renderBusinessRisk(items) { `).join("") : ` - + @@ -2370,10 +2385,10 @@ function renderRiskHeatmap(items) { `).join("") : ` - - - - + + + + @@ -2454,9 +2469,9 @@ function renderSecurityCorrelation(items) { `).join("") : ` - - - + + + @@ -2529,7 +2544,7 @@ function renderBusinessRiskTimeline(history, summary) { - + `} @@ -2583,7 +2598,7 @@ function renderRiskIncidentCandidates(items) { - + @@ -2645,6 +2660,7 @@ function renderCandidateReviewActions(item) {
${actions.map(([status, label]) => ` `).join("")}
+ Скачать пакет расследования ${createCase} `; @@ -2753,8 +2769,8 @@ function settingRows(report) { return [ ["Период расчета", period.label, period.key === "today" ? "оперативный дневной срез" : `${period.days} календарных дней`], ["Рабочий день", `${workdayHours} ч`, `роль: ${policy.role_label || activeRole.label || policy.default_role || "default"}`], - ["Порог WARN", ">= 15 баллов", "любой ненормальный риск попадает в очередь проверки"], - ["Порог FAIL", ">= 70 баллов", "высокий риск требует приоритетного разбора"], + ["Порог внимания", ">= 15 баллов", "любой ненормальный риск попадает в очередь проверки"], + ["Порог критического риска", ">= 70 баллов", "высокий риск требует приоритетного разбора"], ["Источник правил", policy.configured ? basename(policy.path) : "встроенные правила", risk.policy_configured ? `правила оценки риска: ${basename(risk.policy_path)}` : "правила оценки риска: встроенная модель"], ["Дата последнего пересчета", report?.generated_at_utc || "-", `версия политики: ${risk.policy_version || "ueba-rule-v1"}`], ]; @@ -2786,24 +2802,27 @@ async function refresh(options = {}) { const content = document.getElementById("content"); const background = Boolean(options.background); const stage = options.stage || "Получение данных"; + const progress = (status, label, value) => { + if (!background) setLoadStatus(status, label, value); + }; try { - setLoadStatus("LOADING", stage, background ? 35 : 8); + progress("LOADING", stage, 8); if (!background && content) content.innerHTML = renderLoadingContent(stage); if (!state.links) { - setLoadStatus("LOADING", "Получение данных", 18); + progress("LOADING", "Получение данных", 18); state.links = await loadJson("/links"); } - setLoadStatus("LOADING", "Расчёт показателей", 34); + progress("LOADING", "Расчёт показателей", 34); const summary = await loadJson("/summary"); - setLoadStatus("LOADING", "Расчёт показателей", 46); + progress("LOADING", "Расчёт показателей", 46); state.readiness = { bundle: await loadJson("/readiness/bundle").catch(error => ({ ok: false, error: error.message })), verify: state.readiness?.verify || null }; renderSummary(summary, state.readiness); - setLoadStatus("LOADING", "Формирование главного вывода", 68); + progress("LOADING", "Формирование главного вывода", 68); const tabResult = await loadCurrentTab(); - setLoadStatus("LOADING", "Подготовка разделов", 88); + progress("LOADING", "Подготовка разделов", 88); if (!hasTabData(state.tab, tabResult)) { setLoadStatus("EMPTY", "Данные отсутствуют", 100); if (content) { @@ -3031,7 +3050,7 @@ async function verifyReadinessBundle(button) { }; const status = document.getElementById("readinessVerifyStatus"); if (status) { - status.textContent = `Проверено: контрольная сумма ${verify.checksum_verified ? "OK" : "FAIL"} · подпись ${verify.signature_verified ? "OK" : "FAIL"}`; + status.textContent = `Проверено: контрольная сумма ${verify.checksum_verified ? "подтверждена" : "не подтверждена"} · подпись ${verify.signature_verified ? "подтверждена" : "не подтверждена"}`; } button.disabled = false; button.textContent = "Проверить пакет"; diff --git a/ansible/deploy_aw_windows.yml b/ansible/deploy_aw_windows.yml index 14096d7..8992980 100644 --- a/ansible/deploy_aw_windows.yml +++ b/ansible/deploy_aw_windows.yml @@ -8,6 +8,8 @@ ansible_winrm_read_timeout_sec: 180 aw_windows_repo_root: "{{ playbook_dir | dirname }}" aw_windows_deploy_root: "C:\\Program Files\\AWatch-rus" + aw_windows_rust_target_root: "{{ lookup('env', 'CARGO_TARGET_DIR') | default('/tmp/detmir-adk-rust-target', true) }}" + aw_windows_telemetry_exe_source: "{{ aw_windows_rust_target_root }}/x86_64-pc-windows-gnu/release/aw-windows-telemetry.exe" aw_windows_server_scheme: "http" aw_windows_server_port: 5600 aw_windows_package_version: "v0.13.2" @@ -37,8 +39,9 @@ aw_windows_hayabusa_auto_upload_mode: "incident" aw_windows_hayabusa_auto_upload_task_name: "ActivityWatch Hayabusa Upload" aw_windows_file_1c_auto_upload_enabled: true - aw_windows_file_1c_auto_upload_interval_hours: 6 + aw_windows_file_1c_auto_upload_interval_minutes: 15 aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" + aw_windows_file_1c_auto_upload_run_as_user: "{{ aw_windows_domain }}\\{{ aw_windows_builtin_administrator_name }}" aw_windows_file_1c_target_user: "igor" aw_windows_file_1c_registry_workbook_path: "E:\\USER1\\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx" aw_windows_afk_enabled_default: true @@ -195,6 +198,11 @@ - web-category-rules.example.json - dlp-policy.example.json + - name: Загрузить Rust Windows telemetry binary + ansible.windows.win_copy: + src: "{{ aw_windows_telemetry_exe_source }}" + dest: "{{ aw_windows_deploy_root }}\\windows\\aw-windows-telemetry.exe" + - name: Нормализовать кодировку PowerShell файлов (UTF-8 BOM для Windows PowerShell) ansible.windows.win_powershell: script: | @@ -273,8 +281,9 @@ HayabusaAutoUploadMode = "{{ aw_windows_hayabusa_auto_upload_mode }}" HayabusaAutoUploadTaskName = "{{ aw_windows_hayabusa_auto_upload_task_name }}" File1CAutoUploadEnabled = {{ '$true' if (aw_windows_file_1c_auto_upload_enabled | bool) else '$false' }} - File1CAutoUploadIntervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }} + File1CAutoUploadIntervalMinutes = {{ aw_windows_file_1c_auto_upload_interval_minutes | int }} File1CAutoUploadTaskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" + File1CAutoUploadRunAsUser = "{{ aw_windows_file_1c_auto_upload_run_as_user }}" File1CTargetHost = "{{ aw_windows_file_1c_target_host_effective }}" File1CTargetUser = "{{ aw_windows_file_1c_target_user }}" File1CRegistryWorkbookPath = "{{ aw_windows_file_1c_registry_workbook_path }}" diff --git a/ansible/deploy_detmir_portal.yml b/ansible/deploy_detmir_portal.yml index ab1f799..f5d9b1b 100644 --- a/ansible/deploy_detmir_portal.yml +++ b/ansible/deploy_detmir_portal.yml @@ -46,7 +46,7 @@ content: | DETMIR_PORTAL_BIND={{ detmir_portal_bind }} DETMIR_PORTAL_STATUS_CMD=detmir-status --json - DETMIR_PORTAL_CHECK_CMD='timeout 8s detmir-check --json --aw-api "$DETMIR_AW_API" --worktime-url "$DETMIR_WORKTIME_URL" --one-c-url "$DETMIR_ONE_C_URL" --rdp-host "$DETMIR_RDP_HOST" --hostname "$DETMIR_HOSTNAME"' + DETMIR_PORTAL_CHECK_CMD='cat /var/lib/detmir-ai/latest-run/detmir-check.json' DETMIR_PORTAL_FAILED_UNITS_CMD=systemctl --failed --no-pager DETMIR_PORTAL_WORKTIME_URL={{ detmir_portal_worktime_url }} DETMIR_PORTAL_ONE_C_URL={{ detmir_portal_one_c_url }} @@ -65,6 +65,61 @@ CLICKHOUSE_USER={{ detmir_clickhouse_user | default('default') }} CLICKHOUSE_PASSWORD={{ detmir_clickhouse_password | default('') }} + - name: Preserve local ClickHouse security-events settings when available + ansible.builtin.shell: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + + source = Path("/opt/activitywatch/clickhouse-1c/.env") + target = Path("{{ detmir_portal_env_path }}") + if not source.exists() or not target.exists(): + raise SystemExit(0) + + kv = {} + for line in source.read_text().splitlines(): + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + kv[key.strip()] = value.strip().strip('"').strip("'") + + updates = { + "SECURITY_EVENTS_BACKEND": "clickhouse", + "CLICKHOUSE_URL": "http://127.0.0.1:8123", + "CLICKHOUSE_DATABASE": kv.get("CLICKHOUSE_DB", "analytics_1c"), + "CLICKHOUSE_USER": kv.get("CLICKHOUSE_USER", "default"), + "CLICKHOUSE_PASSWORD": kv.get("CLICKHOUSE_PASSWORD", ""), + } + + lines = [] + seen = set() + for line in target.read_text().splitlines(): + if "=" in line and not line.startswith("#"): + key = line.split("=", 1)[0].strip() + if key in updates: + lines.append(f"{key}={updates[key]}") + seen.add(key) + continue + lines.append(line) + for key, value in updates.items(): + if key not in seen: + lines.append(f"{key}={value}") + target.write_text("\n".join(lines) + "\n") + PY + args: + executable: /bin/bash + no_log: true + when: detmir_security_events_backend | default('clickhouse') == 'clickhouse' + + - name: Remove stale detmir-portal systemd overrides + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/systemd/system/detmir-portal.service.d/20-timeouts.conf + - /etc/systemd/system/detmir-portal.service.d/30-warm-cache.conf + register: detmir_portal_stale_overrides + - name: Install initial workforce policy when absent ansible.builtin.copy: dest: "{{ detmir_portal_workforce_policy_path }}" @@ -122,7 +177,6 @@ - name: Reload systemd ansible.builtin.systemd: daemon_reload: true - when: detmir_portal_service_unit.changed - name: Enable and restart detmir-portal ansible.builtin.systemd: @@ -150,7 +204,10 @@ failed_when: > detmir_portal_reports.status != 200 or detmir_portal_reports.json.kpis is not defined - or 'derived detections/cases' not in (detmir_portal_reports.json.markdown | default('')) + or ( + 'derived detections/cases' not in (detmir_portal_reports.json.markdown | default('')) + and 'расчетными выводами' not in (detmir_portal_reports.json.markdown | default('')) + ) changed_when: false - name: Deploy DetMir DLP evidence API on AW server diff --git a/ansible/deploy_dlp_evidence_sync.yml b/ansible/deploy_dlp_evidence_sync.yml index 02bf066..595afba 100644 --- a/ansible/deploy_dlp_evidence_sync.yml +++ b/ansible/deploy_dlp_evidence_sync.yml @@ -7,8 +7,12 @@ aw_repo_root: "{{ playbook_dir | dirname }}" aw_windows_state_root: "C:\\ProgramData\\AWatch-rus" aw_windows_deploy_root: "C:\\Program Files\\AWatch-rus" + aw_windows_rust_target_root: "{{ lookup('env', 'CARGO_TARGET_DIR') | default('/tmp/detmir-adk-rust-target', true) }}" + aw_windows_telemetry_exe_source: "{{ aw_windows_rust_target_root }}/x86_64-pc-windows-gnu/release/aw-windows-telemetry.exe" + aw_windows_telemetry_exe_path: "{{ aw_windows_deploy_root }}\\windows\\aw-windows-telemetry.exe" aw_windows_evidence_sync_task_name: "ActivityWatch DLP Evidence Sync" - aw_windows_evidence_sync_interval_minutes: 5 + aw_windows_evidence_sync_interval_minutes: 15 + aw_windows_evidence_sync_run_as_user: "HOST-EXAMPLE\\Администратор" aw_windows_evidence_sync_api_url: "http://192.0.2.13:8721/api/dlp/evidence/upload" aw_windows_evidence_sync_script: "{{ aw_windows_state_root }}\\sync-dlp-evidence-artifacts.ps1" aw_windows_evidence_sync_token_path: "{{ aw_windows_state_root }}\\dlp-evidence-upload-token.txt" @@ -38,6 +42,11 @@ src: "{{ aw_repo_root }}/windows/sync-dlp-evidence-artifacts.ps1" dest: "{{ aw_windows_evidence_sync_script }}" + - name: Install Rust Windows telemetry binary + ansible.windows.win_copy: + src: "{{ aw_windows_telemetry_exe_source }}" + dest: "{{ aw_windows_telemetry_exe_path }}" + - name: Normalize DLP evidence sync script encoding ansible.windows.win_powershell: script: | @@ -76,23 +85,26 @@ script: | $ErrorActionPreference = 'Stop' $taskName = "{{ aw_windows_evidence_sync_task_name }}" - $ps = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $telemetryExe = "{{ aw_windows_telemetry_exe_path }}" $args = @( - '-NoProfile', - '-ExecutionPolicy', 'Bypass', - '-File', '"{{ aw_windows_evidence_sync_script }}"', - '-EvidenceApiUrl', '"{{ aw_windows_evidence_sync_api_url }}"', - '-TokenPath', '"{{ aw_windows_evidence_sync_token_path }}"', - '-StatePath', '"{{ aw_windows_evidence_sync_state_path }}"', - '-LogPath', '"{{ aw_windows_evidence_sync_log_path }}"' + 'dlp-evidence-sync', + '--evidence-api-url', '"{{ aw_windows_evidence_sync_api_url }}"', + '--token-path', '"{{ aw_windows_evidence_sync_token_path }}"', + '--state-path', '"{{ aw_windows_evidence_sync_state_path }}"', + '--log-path', '"{{ aw_windows_evidence_sync_log_path }}"' ) -join ' ' - $action = New-ScheduledTaskAction -Execute $ps -Argument $args + $action = New-ScheduledTaskAction -Execute $telemetryExe -Argument $args $trigger = New-ScheduledTaskTrigger ` -Once ` -At ((Get-Date).AddMinutes(1)) ` -RepetitionInterval (New-TimeSpan -Minutes {{ aw_windows_evidence_sync_interval_minutes | int }}) ` -RepetitionDuration (New-TimeSpan -Days 3650) - $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + $runAsUser = "{{ aw_windows_evidence_sync_run_as_user }}" + if ([string]::IsNullOrWhiteSpace($runAsUser)) { + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + } else { + $principal = New-ScheduledTaskPrincipal -UserId $runAsUser -LogonType Interactive -RunLevel Highest + } $settings = New-ScheduledTaskSettingsSet ` -AllowStartIfOnBatteries ` -StartWhenAvailable ` @@ -112,11 +124,12 @@ ansible.windows.win_powershell: script: | $ErrorActionPreference = 'Stop' - $result = & "{{ aw_windows_evidence_sync_script }}" ` - -EvidenceApiUrl "{{ aw_windows_evidence_sync_api_url }}" ` - -TokenPath "{{ aw_windows_evidence_sync_token_path }}" ` - -StatePath "{{ aw_windows_evidence_sync_state_path }}" ` - -LogPath "{{ aw_windows_evidence_sync_log_path }}" + $result = & "{{ aw_windows_telemetry_exe_path }}" ` + dlp-evidence-sync ` + --evidence-api-url "{{ aw_windows_evidence_sync_api_url }}" ` + --token-path "{{ aw_windows_evidence_sync_token_path }}" ` + --state-path "{{ aw_windows_evidence_sync_state_path }}" ` + --log-path "{{ aw_windows_evidence_sync_log_path }}" $result register: aw_windows_evidence_sync_smoke diff --git a/ansible/deploy_file_1c_analytics.yml b/ansible/deploy_file_1c_analytics.yml index aa3baa9..d9b2d9a 100644 --- a/ansible/deploy_file_1c_analytics.yml +++ b/ansible/deploy_file_1c_analytics.yml @@ -6,6 +6,7 @@ vars: aw_file_1c_repo_root: "{{ playbook_dir | dirname }}" + aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_file_1c_repo_root + '/adk-rust/target', true)) + '/release' }}" aw_file_1c_release_root: /opt/activitywatch/releases aw_file_1c_release_dir: "{{ aw_file_1c_release_root }}/clickhouse-1c" aw_file_1c_root: /opt/activitywatch/clickhouse-1c @@ -230,6 +231,26 @@ - 04_company_intelligence.sql - 05_financial_reporting.sql + - name: Проверить локальный Rust binary aw-1c-ingest + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-1c-ingest" + delegate_to: localhost + become: false + register: aw_file_1c_ingest_binary + + - name: Остановить deploy без Rust binary aw-1c-ingest + ansible.builtin.fail: + msg: "Missing {{ aw_rust_release_dir }}/aw-1c-ingest. Build with cargo build --release -p aw-1c-ingest." + when: not (aw_file_1c_ingest_binary.stat.exists | default(false)) + + - name: Установить Rust binary aw-1c-ingest + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-1c-ingest" + dest: /usr/local/bin/aw-1c-ingest-rust + owner: root + group: root + mode: "0755" + - name: Установить systemd unit aw-1c-ingest.service ansible.builtin.copy: src: "{{ aw_file_1c_repo_root }}/clickhouse-1c/ops/aw-1c-ingest.service" diff --git a/ansible/deploy_file_1c_windows_telemetry.yml b/ansible/deploy_file_1c_windows_telemetry.yml index c76ef1e..eb136b9 100644 --- a/ansible/deploy_file_1c_windows_telemetry.yml +++ b/ansible/deploy_file_1c_windows_telemetry.yml @@ -7,9 +7,13 @@ aw_windows_repo_root: "{{ playbook_dir | dirname }}" aw_windows_state_root: "C:\\ProgramData\\AWatch-rus" aw_windows_deploy_root: "C:\\Program Files\\AWatch-rus" + aw_windows_rust_target_root: "{{ lookup('env', 'CARGO_TARGET_DIR') | default('/tmp/detmir-adk-rust-target', true) }}" + aw_windows_telemetry_exe_source: "{{ aw_windows_rust_target_root }}/x86_64-pc-windows-gnu/release/aw-windows-telemetry.exe" + aw_windows_telemetry_exe_path: "{{ aw_windows_deploy_root }}\\windows\\aw-windows-telemetry.exe" aw_windows_file_1c_target_user: "igor" - aw_windows_file_1c_auto_upload_interval_hours: 6 + aw_windows_file_1c_auto_upload_interval_minutes: 15 aw_windows_file_1c_auto_upload_task_name: "ActivityWatch File1C Upload" + aw_windows_file_1c_auto_upload_run_as_user: "HOST-EXAMPLE\\Администратор" aw_windows_file_1c_remote_root: "/opt/activitywatch/clickhouse-1c/landing" aw_windows_file_1c_remote_key_path: "" aw_windows_file_1c_registry_workbook_path: "E:\\USER1\\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx" @@ -67,6 +71,11 @@ src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1" dest: "{{ aw_windows_deploy_root }}\\windows\\export-upload-file-1c-telemetry.ps1" + - name: Загрузить Rust telemetry binary в toolkit + ansible.windows.win_copy: + src: "{{ aw_windows_telemetry_exe_source }}" + dest: "{{ aw_windows_telemetry_exe_path }}" + - name: Загрузить file-1C telemetry script в state root ansible.windows.win_copy: src: "{{ aw_windows_repo_root }}/windows/export-upload-file-1c-telemetry.ps1" @@ -87,6 +96,11 @@ } else { $config.paths | Add-Member -NotePropertyName 'file1cTelemetryScript' -NotePropertyValue "{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1" } + if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryExecutable') { + $config.paths.file1cTelemetryExecutable = "{{ aw_windows_telemetry_exe_path }}" + } else { + $config.paths | Add-Member -NotePropertyName 'file1cTelemetryExecutable' -NotePropertyValue "{{ aw_windows_telemetry_exe_path }}" + } if ($config.PSObject.Properties.Name -notcontains 'analytics') { $config | Add-Member -NotePropertyName 'analytics' -NotePropertyValue ([pscustomobject]@{}) @@ -94,10 +108,11 @@ $automation = [pscustomobject]@{ enabled = $true - intervalHours = {{ aw_windows_file_1c_auto_upload_interval_hours | int }} + intervalMinutes = {{ aw_windows_file_1c_auto_upload_interval_minutes | int }} taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" targetHost = "{{ aw_windows_file_1c_target_host_effective }}" targetUser = "{{ aw_windows_file_1c_target_user }}" + runAsUser = "{{ aw_windows_file_1c_auto_upload_run_as_user }}" remoteRoot = "{{ aw_windows_file_1c_remote_root }}" registryWorkbookPath = "{{ aw_windows_file_1c_registry_workbook_path }}" remoteKeyPath = "{{ aw_windows_file_1c_remote_key_path }}" @@ -154,19 +169,20 @@ script: | $ErrorActionPreference = 'Stop' $taskName = "{{ aw_windows_file_1c_auto_upload_task_name }}" - $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' - $taskCommand = "`"$powerShellExe`" -NoProfile -ExecutionPolicy Bypass -File `"{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1`" -ConfigPath `"{{ aw_windows_state_root }}\deployment-config.json`"" + $telemetryExe = "{{ aw_windows_telemetry_exe_path }}" + $runAsUser = "{{ aw_windows_file_1c_auto_upload_run_as_user }}" + $taskArgs = "file1c-upload --config-path `"{{ aw_windows_state_root }}\deployment-config.json`"" + $action = New-ScheduledTaskAction -Execute $telemetryExe -Argument $taskArgs + $trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).Date) -RepetitionInterval (New-TimeSpan -Minutes {{ aw_windows_file_1c_auto_upload_interval_minutes | int }}) -RepetitionDuration (New-TimeSpan -Days 3650) + if ([string]::IsNullOrWhiteSpace($runAsUser)) { + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + } else { + $principal = New-ScheduledTaskPrincipal -UserId $runAsUser -LogonType Interactive -RunLevel Highest + } + $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) $existingTask = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue if ($existingTask) { - $action = New-ScheduledTaskAction -Execute $powerShellExe -Argument "-NoProfile -ExecutionPolicy Bypass -File `"{{ aw_windows_state_root }}\export-upload-file-1c-telemetry.ps1`" -ConfigPath `"{{ aw_windows_state_root }}\deployment-config.json`"" - try { - Set-ScheduledTask -TaskName $taskName -Action $action -ErrorAction Stop | Out-Null - } catch { - Write-Host "skip task action update for $taskName because the existing principal requires stored credentials: $($_.Exception.Message)" - } + Set-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -ErrorAction Stop | Out-Null } else { - & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO {{ aw_windows_file_1c_auto_upload_interval_hours | int }} /ST 00:00 /RU SYSTEM /RL HIGHEST /F | Out-Null - if ($LASTEXITCODE -ne 0) { - throw "Не удалось создать scheduled task $taskName" - } + Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings | Out-Null } diff --git a/clickhouse-1c/README.md b/clickhouse-1c/README.md index 8e9a6f6..d6c631c 100644 --- a/clickhouse-1c/README.md +++ b/clickhouse-1c/README.md @@ -45,7 +45,9 @@ File 1C + reglog + host telemetry - `docker-compose.yml` — локальный scaffold ClickHouse + Grafana. - `.env.example` — переменные окружения. - `clickhouse/init/*.sql` — схема БД. -- `etl/load_1c_exports.py` — loader CSV/JSON выгрузок в raw/core таблицы. +- `adk-rust/crates/aw-1c-ingest` — production writer CSV/JSON выгрузок в + raw/core ClickHouse таблицы. +- `etl/load_1c_exports.py` — legacy loader для rollback/ручной отладки. - `etl/build_business_event_exports.py` — read-only normalizer из `documents/postings/audit` в canonical `business_events/document_changes`. - `etl/extract_1c_mcp_toolkit.py` — read-only extractor из @@ -64,7 +66,8 @@ File 1C + reglog + host telemetry - `grafana/provisioning/dashboards/files/1c-telemetry-board.json` — telemetry dashboard по состоянию файловых баз, reglog growth, busy markers и host load. - `detections/build_entity_timeline.sql` — сборка единого timeline слоя. - `detections/open_cases_from_detections.sql` — шаблон открытия cases из detections. -- `ops/etl-cron.example` — пример расписания каждые 6 часов. +- `ops/etl-cron.example` — legacy cron example; production использует + `aw-1c-ingest.timer`. - `ops/retention-policy.md` — минимальная retention policy. - `ai/INVESTIGATOR_API.md` — контракт AI Investigator поверх ClickHouse/cases. - `ai/refresh_company_intelligence.py` — materialization forecast/signals по `counterparty`. @@ -110,7 +113,7 @@ mkdir -p landing/{documents,postings,business_events,document_changes,companies, cp etl/config.example.yml etl/config.yml ``` -4. Запустить ETL: +4. Запустить ingest: ```bash python3 -m venv .venv @@ -119,8 +122,7 @@ pip install -r etl/requirements.txt python etl/extract_1c_mcp_toolkit.py --config etl/config.yml --validate-config python etl/extract_1c_mcp_toolkit.py --config etl/config.yml --dataset documents --dry-run python etl/extract_1c_mcp_toolkit.py --config etl/config.yml -python etl/build_business_event_exports.py --config etl/config.yml -python etl/load_1c_exports.py --config etl/config.yml +/usr/local/bin/aw-1c-ingest-rust --root /opt/activitywatch/clickhouse-1c ``` 5. Применить detections: @@ -233,9 +235,7 @@ etl/extract_1c_mcp_toolkit.py ↓ landing/{documents,postings,business_events,document_changes,companies,reglog} ↓ -etl/build_business_event_exports.py - ↓ -etl/load_1c_exports.py +aw-1c-ingest-rust ``` Что он умеет: diff --git a/clickhouse-1c/ops/aw-1c-ingest.service b/clickhouse-1c/ops/aw-1c-ingest.service index 70ac3d6..1cb2e3c 100644 --- a/clickhouse-1c/ops/aw-1c-ingest.service +++ b/clickhouse-1c/ops/aw-1c-ingest.service @@ -1,5 +1,5 @@ [Unit] -Description=AW-rus file-1C ingest cycle +Description=AW-rus file-1C ingest cycle (Rust) After=docker.service Requires=docker.service @@ -7,7 +7,7 @@ Requires=docker.service Type=oneshot WorkingDirectory=/opt/activitywatch/clickhouse-1c Environment=AW_1C_ROOT=/opt/activitywatch/clickhouse-1c -ExecStart=/opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh +ExecStart=/usr/local/bin/aw-1c-ingest-rust --root /opt/activitywatch/clickhouse-1c TimeoutStartSec=45min User=root Group=root diff --git a/clickhouse-1c/ops/etl-cron.example b/clickhouse-1c/ops/etl-cron.example index e379fa7..5d713b6 100644 --- a/clickhouse-1c/ops/etl-cron.example +++ b/clickhouse-1c/ops/etl-cron.example @@ -1,5 +1,2 @@ -# every 6 hours: load exports, rebuild timeline, refresh cases -0 */6 * * * cd /opt/aw-rus/clickhouse-1c && . .venv/bin/activate && python etl/load_1c_exports.py --config etl/config.yml >> /var/log/aw-rus-1c-etl.log 2>&1 -10 */6 * * * clickhouse-client --queries-file /opt/aw-rus/clickhouse-1c/detections/build_entity_timeline.sql >> /var/log/aw-rus-1c-timeline.log 2>&1 -15 */6 * * * clickhouse-client --queries-file /opt/aw-rus/clickhouse-1c/detections/insert_detections.sql >> /var/log/aw-rus-1c-detections.log 2>&1 -20 */6 * * * clickhouse-client --queries-file /opt/aw-rus/clickhouse-1c/detections/open_cases_from_detections.sql >> /var/log/aw-rus-1c-cases.log 2>&1 +# Legacy cron fallback only. Production uses aw-1c-ingest.timer. +0 * * * * AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /usr/local/bin/aw-1c-ingest-rust --root /opt/activitywatch/clickhouse-1c >> /var/log/aw-rus-1c-ingest.log 2>&1 diff --git a/docs/1C_BUSINESS_EVENT_LAYER_RU.md b/docs/1C_BUSINESS_EVENT_LAYER_RU.md index 6b55d5b..72a19d5 100644 --- a/docs/1C_BUSINESS_EVENT_LAYER_RU.md +++ b/docs/1C_BUSINESS_EVENT_LAYER_RU.md @@ -108,7 +108,7 @@ 1. Внешний extractor читает только безопасные read-only источники. 2. Формирует `jsonl/csv` в landing-каталоги. -3. `etl/load_1c_exports.py` грузит данные в raw/core ClickHouse tables. +3. `aw-1c-ingest-rust` грузит данные в raw/core ClickHouse tables. 4. Detection/AI слой работает только с ClickHouse. То есть LLM и manager pages не ходят в 1С напрямую. @@ -179,7 +179,7 @@ - ETL support: - `landing/business_events` - `landing/document_changes` - - dataset mapping в `etl/load_1c_exports.py` + - dataset mapping в `aw-1c-ingest-rust` - built-in normalizer: - `etl/build_business_event_exports.py` - собирает canonical events из существующих read-only выгрузок @@ -191,7 +191,7 @@ - забирает `reglog` через `get_event_log` - пишет в те же `landing/*`, которые уже понимает loader - ingest wiring: - - normalizer запускается перед `load_1c_exports.py` + - normalizer встроен в production writer `aw-1c-ingest-rust` - timeline/detection wiring: - `business_events` и `document_change_events` уже входят в `entity_timeline` diff --git a/docs/1C_COMPANY_INTELLIGENCE_RU.md b/docs/1C_COMPANY_INTELLIGENCE_RU.md index e0ad392..b722214 100644 --- a/docs/1C_COMPANY_INTELLIGENCE_RU.md +++ b/docs/1C_COMPANY_INTELLIGENCE_RU.md @@ -153,7 +153,7 @@ Endpoints: - `clickhouse-1c/ops/run_company_intelligence_api.sh` - `clickhouse-1c/ops/aw-1c-company-api.service` -И `run_ingest_cycle.sh` теперь: +И production writer `aw-1c-ingest-rust` теперь: 1. грузит новые выгрузки; 2. обновляет timeline/detections/cases; diff --git a/docs/1C_FILE_ANALYTICS_STACK_RU.md b/docs/1C_FILE_ANALYTICS_STACK_RU.md index 94d1b37..b85942f 100644 --- a/docs/1C_FILE_ANALYTICS_STACK_RU.md +++ b/docs/1C_FILE_ANALYTICS_STACK_RU.md @@ -58,11 +58,11 @@ Windows file 1C host () ├─ file-base busy markers └─ host telemetry ↓ -export-upload-file-1c-telemetry.ps1 +aw-windows-telemetry.exe file1c-upload ↓ scp /opt/activitywatch/clickhouse-1c/landing/* ↓ -run_ingest_cycle.sh +aw-1c-ingest-rust ├─ raw tables ├─ core tables ├─ entity_timeline @@ -76,8 +76,9 @@ Grafana Контур считается рабочим, если одновременно выполняется всё: -1. Windows scheduled task `ActivityWatch File1C Upload` запускается по расписанию. -2. На `` работает `aw-1c-ingest.timer`. +1. Windows scheduled task `ActivityWatch File1C Upload` запускается раз в 15 минут. +2. На `` работает `aw-1c-ingest.timer`; цикл сбора и записи в + ClickHouse выполняется раз в 15 минут. 3. На `` работает `aw-1c-proofcheck.timer`. 4. `ClickHouse` содержит живые строки в: - `documents` @@ -89,11 +90,17 @@ Grafana - `cases` 5. В `Grafana` на `` dashboards открываются и смотрят в datasource `clickhouse-1c`. +Скриншоты не входят в file-1C контур. `ActivityWatch File1C Upload` передает +только метаданные файловых баз, журналов, audit/host JSONL и registry workbook. +PNG/скриншоты при работе с 1C не копируются в ClickHouse landing и не должны +синхронизироваться как DLP evidence. + ## 4. Каталоги и артефакты ### 4.1 На Windows `` - `C:\ProgramData\AWatch-rus\deployment-config.json` +- `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe` - `C:\ProgramData\AWatch-rus\export-upload-file-1c-telemetry.ps1` - `C:\ProgramData\AWatch-rus\logs\file1c-telemetry.log` - `C:\ProgramData\AWatch-rus\ssh\awops_ed25519` @@ -112,6 +119,7 @@ Grafana - runtime: - `/opt/activitywatch/clickhouse-1c/.env` - `/opt/activitywatch/clickhouse-1c/etl/config.yml` + - `/usr/local/bin/aw-1c-ingest-rust` - `/opt/activitywatch/clickhouse-1c/.venv` ### 4.3 systemd units на `` @@ -142,7 +150,8 @@ ansible-playbook -i /ansible/inventory.ini \ - раскладывает `clickhouse-1c` в `/opt/activitywatch/clickhouse-1c`; - поднимает `ClickHouse`; - создаёт `.env` и `etl/config.yml`; -- включает `aw-1c-ingest.timer`; +- устанавливает Rust-бинарник `aw-1c-ingest-rust`; +- включает `aw-1c-ingest.timer` с периодом 15 минут; - включает `aw-1c-proofcheck.timer`. ### 5.2 Windows uploader на `` @@ -160,9 +169,11 @@ ansible-playbook -i /ansible/inventory.ini \ Что делает: -- копирует `export-upload-file-1c-telemetry.ps1`; +- копирует Rust-бинарник `aw-windows-telemetry.exe`; +- оставляет `export-upload-file-1c-telemetry.ps1` как legacy fallback; - обновляет `deployment-config.json`; -- создаёт/обновляет scheduled task `ActivityWatch File1C Upload`. +- создаёт/обновляет scheduled task `ActivityWatch File1C Upload` на запуск + `aw-windows-telemetry.exe file1c-upload`. ### 5.3 Production Grafana на `` @@ -186,22 +197,31 @@ Grafana уже должна содержать: ### 6.1 Почему он нужен -Создание file-1C scheduled task через `schtasks` по умолчанию использует `SYSTEM`. - -Для этой конкретной задачи production-схема должна использовать **рабочий principal**, а не `SYSTEM`. +Для этой конкретной задачи production-схема должна использовать **рабочий +interactive principal**, а не `SYSTEM`, потому что на текущем Windows-хосте +SYSTEM-запуск внешних процессов нестабилен. Проверенная рабочая учётка: - `HOST-EXAMPLE\Администратор` -### 6.2 Команда переключения principal +### 6.2 Проверка scheduled task На ``: -```cmd -schtasks /Change /TN "\ActivityWatch File1C Upload" /RU "HOST-EXAMPLE\Администратор" /RP "" +```powershell +$task = Get-ScheduledTask -TaskName "ActivityWatch File1C Upload" +$task.Actions[0].Execute +$task.Actions[0].Arguments +$task.Principal.UserId ``` +Ожидаемо: + +- `Execute`: `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe` +- `Arguments`: `file1c-upload --config-path "C:\ProgramData\AWatch-rus\deployment-config.json"` +- `Principal`: `Администратор` / `HOST-EXAMPLE\Администратор` + ### 6.3 Проверка ```cmd @@ -237,7 +257,7 @@ Get-Content -Tail 80 C:\ProgramData\AWatch-rus\logs\file1c-telemetry.log ### 7.2 Backend ingestion ```bash -AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh +AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /usr/local/bin/aw-1c-ingest-rust --root /opt/activitywatch/clickhouse-1c ``` ### 7.3 Freshness proof @@ -392,7 +412,7 @@ docker ps --format '{{.Names}}' | rg aw-rus-1c-clickhouse 3. Запустить exporter вручную: ```powershell -& "C:\ProgramData\AWatch-rus\export-upload-file-1c-telemetry.ps1" -ConfigPath "C:\ProgramData\AWatch-rus\deployment-config.json" +& "C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe" file1c-upload --config-path "C:\ProgramData\AWatch-rus\deployment-config.json" ``` ### 12.2 Backend сторона @@ -402,12 +422,125 @@ docker ps --format '{{.Names}}' | rg aw-rus-1c-clickhouse 3. Запустить ingest вручную: ```bash -AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /opt/activitywatch/clickhouse-1c/ops/run_ingest_cycle.sh +AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /usr/local/bin/aw-1c-ingest-rust --root /opt/activitywatch/clickhouse-1c ``` 4. Проверить freshness. -## 13. Безопасность +## 13. Ручное изменение параметров + +Использовать этот раздел, если нужно временно изменить production-параметры +без полного redeploy. После ручного изменения желательно синхронизировать те же +значения в Ansible vars, иначе следующий полный deploy может вернуть прежние +настройки. + +### 13.1 Изменить период Windows file-1C upload + +На Windows/RDP host в elevated PowerShell: + +```powershell +$taskName = "ActivityWatch File1C Upload" +$minutes = 15 + +$configPath = "C:\ProgramData\AWatch-rus\deployment-config.json" +$config = Get-Content -Raw -LiteralPath $configPath | ConvertFrom-Json +$config.analytics.file1cAutomation.intervalMinutes = $minutes +$config | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $configPath -Encoding UTF8 + +$task = Get-ScheduledTask -TaskName $taskName +$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).Date) ` + -RepetitionInterval (New-TimeSpan -Minutes $minutes) ` + -RepetitionDuration (New-TimeSpan -Days 3650) + +Set-ScheduledTask -TaskName $taskName ` + -Action $task.Actions ` + -Trigger $trigger ` + -Principal $task.Principal ` + -Settings $task.Settings +``` + +Проверить: + +```powershell +Get-ScheduledTaskInfo -TaskName "ActivityWatch File1C Upload" +(Get-ScheduledTask -TaskName "ActivityWatch File1C Upload").Actions +``` + +### 13.2 Изменить параметры file-1C upload + +Основной файл: + +```text +C:\ProgramData\AWatch-rus\deployment-config.json +``` + +Ключевые поля: + +```json +{ + "analytics": { + "file1cAutomation": { + "intervalMinutes": 15, + "targetHost": "", + "targetUser": "igor", + "remoteRoot": "/opt/activitywatch/clickhouse-1c/landing", + "remoteKeyPath": "C:\\ProgramData\\AWatch-rus\\ssh\\awops_ed25519", + "registryWorkbookPath": "E:\\USER1\\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx" + } + } +} +``` + +Smoke-run после изменения: + +```powershell +& "C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe" file1c-upload --config-path "C:\ProgramData\AWatch-rus\deployment-config.json" +Get-Content -LiteralPath "C:\ProgramData\AWatch-rus\logs\file1c-telemetry.log" -Tail 40 -Encoding UTF8 +``` + +### 13.3 Изменить период server-side ingest + +На ``: + +```bash +sudo mkdir -p /etc/systemd/system/aw-1c-ingest.timer.d + +sudo tee /etc/systemd/system/aw-1c-ingest.timer.d/override.conf >/dev/null <<'EOF' +[Timer] +OnUnitActiveSec= +OnUnitActiveSec=15min +EOF + +sudo systemctl daemon-reload +sudo systemctl restart aw-1c-ingest.timer +systemctl list-timers aw-1c-ingest.timer +``` + +### 13.4 Изменить защитную задержку ingest + +Файл на ``: + +```bash +sudo nano /opt/activitywatch/clickhouse-1c/etl/config.yml +``` + +Параметр: + +```yaml +min_file_age_seconds: 180 +``` + +Нормальный диапазон: `60-180` секунд. Не ставить слишком низко: ingest может +прочитать файл во время SCP-загрузки. + +Проверить: + +```bash +sudo systemctl start aw-1c-ingest.service +sudo journalctl -u aw-1c-ingest.service -n 50 --no-pager +``` + +## 14. Безопасность - Не хранить пароль администратора Windows в git. - Не хранить секреты в docs. @@ -415,11 +548,11 @@ AW_1C_ROOT=/opt/activitywatch/clickhouse-1c /opt/activitywatch/clickhouse-1c/ops - Не давать AI write-back в 1С. - Не менять `1Cv8.1CD`. -## 14. Связанные файлы +## 15. Связанные файлы - [clickhouse-1c/README.md](/clickhouse-1c/README.md) -- [clickhouse-1c/etl/load_1c_exports.py](/clickhouse-1c/etl/load_1c_exports.py) -- [clickhouse-1c/ops/run_ingest_cycle.sh](/clickhouse-1c/ops/run_ingest_cycle.sh) +- [adk-rust/crates/aw-1c-ingest](/adk-rust/crates/aw-1c-ingest) +- [clickhouse-1c/ops/run_ingest_cycle.sh](/clickhouse-1c/ops/run_ingest_cycle.sh) - legacy rollback path - [clickhouse-1c/ops/check_ingest_freshness.sh](/clickhouse-1c/ops/check_ingest_freshness.sh) - [ansible/deploy_file_1c_analytics.yml](/ansible/deploy_file_1c_analytics.yml) - [ansible/deploy_file_1c_windows_telemetry.yml](/ansible/deploy_file_1c_windows_telemetry.yml) diff --git a/docs/ARCHITECTURE_BASELINE_RU.md b/docs/ARCHITECTURE_BASELINE_RU.md index bc738a0..7b3c3eb 100644 --- a/docs/ARCHITECTURE_BASELINE_RU.md +++ b/docs/ARCHITECTURE_BASELINE_RU.md @@ -27,12 +27,15 @@ Future Desktop Forensics: Tauri + React + Rust core ## Исключено из roadmap -Dioxus и DPD Portal исключены из архитектурного roadmap проекта. +Экспериментальные Rust UI/prototype mirror направления исключены из +архитектурного roadmap проекта. Это означает: -- новые Dioxus/DPD crate не добавляются; -- `/dpd/` и `/dpd/api/*` не являются частью публичного API-контракта; +- новые crate для экспериментальных UI/prototype mirror направлений не + добавляются; +- устаревшие mirror-prefix маршруты не являются частью публичного + API-контракта; - будущий UI не должен парсить HTML текущего портала; - новые UI-фреймворки требуют отдельного architecture decision; - breaking changes API требуют version bump контракта и migration window. diff --git a/docs/ARCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md b/docs/ARCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md deleted file mode 100644 index 76ff064..0000000 --- a/docs/ARCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md +++ /dev/null @@ -1,46 +0,0 @@ -# Аудит очистки архитектуры от DPD/Dioxus - -Дата: 2026-06-05 - -## Цель - -Зафиксировать результат проверки наследия DPD/Dioxus после усиления API -контрактов в коммите `bc2f56e`. - -## Найденные DPD-артефакты до очистки - -- workspace crate `adk-rust/crates/detmir-dpd-portal`; -- запись workspace member `crates/detmir-dpd-portal` в `adk-rust/Cargo.toml`; -- package entry `detmir-dpd-portal` в `adk-rust/Cargo.lock`; -- документ `docs/DPD_PORTAL_RU.md`; -- OpenAPI server base `/dpd/api`; -- упоминания DPD mirror в `docs/PORTAL_API_CONTRACTS_RU.md`; -- упоминания DPD mirror в `docs/UI_ARCHITECTURE_BASELINE_RU.md`. - -## Найденные Dioxus-артефакты до очистки - -Production-код, workspace crates, scripts, examples и OpenAPI/TypeScript -контракты не содержали Dioxus-зависимостей или `rsx!`-кода. - -Оставались только запретительные упоминания в архитектурном baseline: - -- Dioxus не является частью roadmap; -- Dioxus не добавлять без отдельного architecture decision. - -## Выполненное решение - -- DPD Portal исключён из workspace и публичного contract layer. -- Документ `docs/DPD_PORTAL_RU.md` удалён как описание экспериментальной ветки. -- OpenAPI больше не публикует `/dpd/api`. -- Runtime-слой DPD Portal выведен из эксплуатации: app-service остановлен и - выключен, gateway route удалён, перед удалением создан локальный backup. -- `docs/ARCHITECTURE_BASELINE_RU.md`, - `docs/UI_ARCHITECTURE_BASELINE_RU.md` и - `docs/PORTAL_API_CONTRACTS_RU.md` фиксируют единственную целевую архитектуру: - Rust backend/API, Rust agent, Rust SSR + HTML + HTMX portal, будущий - React/TypeScript UI и будущий Tauri + React + Rust core desktop forensics. - -## Статус - -DPD/Dioxus остаются только как исторически упомянутые исключённые направления. -Они не являются production runtime, публичным API или частью roadmap. diff --git a/docs/DETMIR_THREAT_MODEL_RU.md b/docs/DETMIR_THREAT_MODEL_RU.md index 4e13de4..1d1a0c1 100644 --- a/docs/DETMIR_THREAT_MODEL_RU.md +++ b/docs/DETMIR_THREAT_MODEL_RU.md @@ -245,7 +245,10 @@ - SHA-256 validation; - atomic write в evidence storage; - audit records для upload/view/download; -- Windows scheduled task `ActivityWatch DLP Evidence Sync` каждые 5 минут; +- Windows scheduled task `ActivityWatch DLP Evidence Sync` каждые 15 минут + через Rust-бинарник `aw-windows-telemetry.exe dlp-evidence-sync`; копируются + только DLP incident screenshots, 1C/обычные PNG из `incident-artifacts` + игнорируются; - external Playwright smoke для портала/evidence/Grafana routes; - Grafana актуальность и datasource health checks; - rollback-critical backups; diff --git a/docs/DIOXUS_REMOVAL_REPORT_RU.md b/docs/DIOXUS_REMOVAL_REPORT_RU.md new file mode 100644 index 0000000..48675b7 --- /dev/null +++ b/docs/DIOXUS_REMOVAL_REPORT_RU.md @@ -0,0 +1,77 @@ +# Отчёт об удалении Dioxus из AWatch-rus + +Дата: 2026-06-05 + +## Итог + +Dioxus не используется в проекте и исключен из архитектурного roadmap. + +Production-архитектура после проверки: + +- Backend/API: Rust; +- Agent: Rust; +- Portal: Rust server-rendered HTML + HTMX; +- Future Enterprise UI: React + TypeScript через JSON API-контракты; +- Future Desktop Forensics: Tauri + React + Rust core. + +## Удалённые и отсутствующие артефакты + +В текущем дереве подтверждено отсутствие: + +- workspace crate `adk-rust/crates/detmir-dpd-portal`; +- workspace member `crates/detmir-dpd-portal` в `adk-rust/Cargo.toml`; +- package entry `detmir-dpd-portal` в `adk-rust/Cargo.lock`; +- документа `docs/DPD_PORTAL_RU.md`; +- OpenAPI server base `/dpd/api`; +- исходного кода с `rsx!`; +- зависимостей `dioxus` или `dioxuslabs`. + +В рамках финальной консолидации удалён устаревший частичный audit-документ +`docs/ARCHITECTURE_CLEANUP_AUDIT_2026-06-05_RU.md`; его содержимое заменено +этим отчётом. + +## Зависимости + +Дополнительные Dioxus-зависимости не удалялись, потому что в проверенном +workspace они уже отсутствуют. `adk-rust/Cargo.toml` не содержит Dioxus/DPD +members или workspace dependencies; поиск по `adk-rust` вне `target/` не +находит `dioxus`, `dioxuslabs`, `rsx!`, `detmir-dpd` или `dpd`. + +## Проверенные маршруты + +Контрактные и portal маршруты остаются в `adk-rust/crates/detmir-portal`: + +- `/portal` - HTTP 200; +- `/reports` - HTTP 200; +- `/portal/reports` - HTTP 200; +- `/api/reports` - HTTP 200; +- `/api/contracts` - HTTP 200; +- `/api/contracts/openapi.json` - HTTP 200; +- `/api/contracts/typescript.d.ts` - HTTP 200. + +`/dpd/` и `/dpd/api/*` не входят в публичный API-контракт и не публикуются в +OpenAPI. + +## Выполненные проверки + +Команды выполнены из `adk-rust`: + +- `cargo fmt --check` - ok; +- `CARGO_TARGET_DIR=/tmp/aw-rus-cargo-target cargo clippy --all-targets --all-features -- -D warnings` - ok; +- `CARGO_TARGET_DIR=/tmp/aw-rus-cargo-target cargo test --all` - ok; +- `CARGO_TARGET_DIR=/tmp/aw-rus-cargo-target cargo build --all` - ok; +- `cargo tree --all-features` + поиск `dioxus|dioxuslabs|rsx!|detmir-dpd|dpd` - совпадений нет; +- `grep -RIn` по исходникам, документации и контрактам с исключением `.git`, + `.ai`, `target`, `.local`, `dist`, `.ops`, `.playwright-cli` - совпадения + только в этом отчёте; +- runtime smoke `detmir-portal` на `127.0.0.1:18720` - обязательные маршруты + вернули HTTP 200; +- поиск по runtime JSON/OpenAPI/TypeScript ответам - `/dpd`, Dioxus и `rsx!` + не найдены. + +## Архитектурное подтверждение + +Dioxus и связанная DPD-проба не являются production runtime, UI roadmap, +публичным API, workspace crate или dependency layer проекта. Дальнейшее +развитие портала выполняется через Rust SSR + HTML + HTMX и стабильные JSON +API-контракты для будущих React/Tauri клиентов. diff --git a/docs/PILOT_GAP_ANALYSIS_RU.md b/docs/PILOT_GAP_ANALYSIS_RU.md new file mode 100644 index 0000000..81c23fa --- /dev/null +++ b/docs/PILOT_GAP_ANALYSIS_RU.md @@ -0,0 +1,140 @@ +# Анализ разрывов готовности пилота: AWatch-rus + +Дата аудита: 2026-06-05. + +Назначение: финальная проверка готовности AWatch-rus к демонстрационному +пилоту для руководителя, ИБ и эксплуатации. После первичного аудита закрыты +демонстрационные разрывы без добавления новых сущностей, новых интеграций и +архитектурных изменений. + +Не входило в аудит: pfSense, Telegram, Grafana, InfluxDB. + +## 1. Готово к пилоту + +- Executive View готов к демонстрации: главный вывод отображается первым, + управленческие блоки идут в понятном порядке, технические термины не + выводятся в пользовательском тексте. +- Security View готов к демонстрации: есть кандидаты на проверку, аудит, + материалы расследования, пакет расследования и переход к карточке + расследования. +- Operations View готов к демонстрации: видны полнота и качество данных, + ошибки сбора, состояние источников и доступность событий безопасности. +- Контур ClickHouse для событий ИБ работает на `` в Docker-контейнере + `aw-rus-1c-clickhouse`; портал читает события через ClickHouse, fallback не + используется. +- `/api/health` возвращает `ok=true`; источники `detmir_check`, + `detmir_status`, `worktime_api`, `worktime_management`, `one_c`, + `dlp_health`, `security_events` находятся в рабочем состоянии. +- В текущем отчете есть 3 кандидата на проверку; smoke подтвердил 3 кнопки + перехода к расследованию. +- Первый отчет прогревается при старте `detmir-portal.service`; после прогрева + `/api/reports` отвечает за доли секунды. +- Фоновое обновление больше не переводит готовый экран обратно в состояние + "Загрузка данных"; 70-секундная браузерная проверка сохранила `READY`. +- Мобильная проверка 390px проходит: страница доходит до `READY`, глобального + горизонтального расползания body не выявлено. +- Обязательные проверки на текущем дереве пройдены: + +| Проверка | Итог | +| --- | --- | +| `cargo test --workspace` | Пройдено. | +| `cargo clippy --all-targets --all-features` | Пройдено. | +| `cargo build --release` | Пройдено. | +| `node scripts/detmir-portal-tabs-smoke.mjs` | Пройдено против `` с `security_events=available`. | + +## 2. Желательно исправить до пилота + +- Передать заказчику только вычитанные customer-facing документы. Внутренние + документы могут содержать технические обозначения, но демонстрационный пакет + для руководителя должен использовать русские деловые формулировки. +- Отдельно проверить экспорт итогового отчета в том формате, который будет + показан заказчику: текстовый отчет, печать или PDF. +- Зафиксировать демонстрационный сценарий в виде короткой шпаргалки оператора: + с какой вкладки начинать, какой кандидат открывается, какой пакет скачивать, + какой итоговый вывод озвучивать. +- Зафиксировать hand-off правило для Rust-сборки: на этой машине использовать + `CARGO_TARGET_DIR=`, потому что старый + `target/` на `fuseblk` непригоден для `libsqlite3-sys`. + +## 3. Критично исправить до пилота + +Критичных технических блокеров для контролируемого демонстрационного пилота на +текущем контуре не осталось. + +Единственное обязательное условие перед показом: не начинать демонстрацию без +короткого преддемо-прогона портала на том же экране и сети. Это не доработка +продукта, а операционная страховка от внешних факторов: сеть, браузер, доступ к +стенду, свежесть данных. + +## 4. Можно перенести после пилота + +- Полное hardening JSON/JSONL-хранилищ: блокировки, atomic write, ротация, + retention, backup/restore и нагрузочная проверка больших state-файлов. +- Формальную матрицу версий API и журнал совместимых и несовместимых изменений + полей. +- Расширенные нагрузочные тесты и sizing под промышленную эксплуатацию. +- Полировку коммерческого пакета после первой обратной связи заказчика. +- Расширенную автоматизацию мобильных скриншотов и визуальных regression tests. +- Расширенную регламентацию долгосрочного доступа к материалам расследований и + архивам. + +## 5. Общая оценка готовности + +AWatch-rus готов к контролируемому демонстрационному пилоту на текущем DetMir +контуре. Архитектура сохранена, обратная совместимость не нарушена, новые +интеграции и сущности не добавлялись. + +Практическая оценка готовности: 90% для демонстрационного пилота. Оставшиеся +10% относятся не к блокерам кода, а к дисциплине показа: прогрев, заранее +открытый сценарий, проверенный экспорт и вычитанный customer-facing пакет. + +## 6. Оценка Executive View + +- Главный вывод расположен первым и подтвержден smoke-проверкой порядка блоков. +- Представление понятно руководителю без знаний ИБ: риск описывается через + подразделение, причину, подтверждающие слои и рекомендуемое действие. +- Англоязычные и внутренние технические обозначения в пользовательском слое + smoke не обнаружил. +- Итог: Executive View можно показывать руководителю. + +## 7. Оценка Security View + +- Есть очередь кандидатов на проверку, аудит, материалы расследования и пакет + расследования. +- Ручное подтверждение сохранено: система не создает инциденты автоматически + без решения ответственного. +- Переход "кандидат -> расследование" проверен smoke: найдено 3 кнопки перехода. +- Итог: Security View можно показывать ИБ как пилотный процесс ручной проверки. + +## 8. Оценка Operations View + +- Operations View показывает полноту данных, качество данных, ошибки сбора, + состояние источников и режим событий безопасности. +- ClickHouse-события доступны через портал: `backend=clickhouse`, `status=ok`, + `fallback_used=false`. +- `/api/health` зеленый на рабочем контуре. +- Фоновое обновление не ломает готовое состояние экрана. +- Итог: Operations View можно показывать эксплуатации. + +## 9. Риски демонстрации + +| Риск | Влияние | Статус | +| --- | --- | --- | +| Открыть портал без прогрева или при нестабильной сети | Может выглядеть как долгая загрузка | Управляемо: service prewarm и преддемо-прогон | +| Не проверить экспорт отчета заранее | Можно потерять время на живом показе | Желательно закрыть до показа | +| Отдать внутреннюю документацию без вычитки | Лишние технические вопросы руководителя и ИБ | Желательно закрыть до показа | +| Данные изменятся перед демо и кандидатов станет меньше | Сценарий расследования может ослабнуть | Управляемо: проверить `/api/reports` перед показом | +| Внешние источники недоступны из-за сети | Потеря части операционной картины | Управляемо: преддемо health-check | + +10-минутный сценарий сейчас подтвержден: главный риск, подразделение, причина +риска, кандидат на проверку, расследование, пакет расследования и итоговый +управленческий вывод доступны в портале. + +## 10. Итоговая рекомендация + +Допустить AWatch-rus к контролируемому демонстрационному пилоту. + +Перед показом выполнить короткий операторский прогон: открыть портал, убедиться +в `Данные готовы`, проверить наличие кандидатов, открыть расследование, скачать +пакет расследования и открыть итоговый отчет. Это достаточно для уверенного +показа руководителю, ИБ и эксплуатации. diff --git a/docs/PORTAL_API_CONTRACTS_RU.md b/docs/PORTAL_API_CONTRACTS_RU.md index 839408b..118746a 100644 --- a/docs/PORTAL_API_CONTRACTS_RU.md +++ b/docs/PORTAL_API_CONTRACTS_RU.md @@ -52,7 +52,8 @@ React/Tauri-интерфейса без переписывания backend-ло - Маршрут `/portal/` остаётся стабильным. - Backend-расчёты, JSON-хранилища и workflow не дублируются во frontend. - Публичные JSON-поля не переименовываются без новой версии контракта. -- Dioxus и DPD Portal исключены из архитектурного roadmap проекта. +- Экспериментальные mirror/prototype направления не входят в публичный + contract layer и не должны возвращаться без отдельного architecture decision. ## Проверка diff --git a/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md b/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md new file mode 100644 index 0000000..e09048d --- /dev/null +++ b/docs/POWERSHELL_TO_RUST_ROADMAP_RU.md @@ -0,0 +1,616 @@ +# Roadmap: замена PowerShell на Rust EXE + +Дата: 2026-06-05 + +Цель: убрать зависимость AWatch-rus/DetMir от PowerShell-скриптов на рабочих +хостах и заменить их на самодостаточные Rust EXE/службы без потери данных, +без ухудшения пилотной демонстрации и без изменения функционального объема. + +Документ описывает порядок миграции. Он не вводит новые функции: каждая Rust +замена сначала должна повторить текущий контракт PowerShell-компонента. + +Операторский checkpoint от 2026-06-05 сохранен в private `.ops`-контуре: +Phase 0 live inventory выполнен, Phase 2 validation для уже переведенных +Windows Rust paths выполнен успешно. В tracked документации не публикуются live +hostnames, private IP, токены и runtime evidence paths. + +## 1. Целевое состояние + +К концу миграции: + +- на Windows/RDP-хостах регулярный сбор данных, evidence-sync, 1C upload, + guard/recovery и validation выполняются Rust EXE; +- в Scheduled Tasks и Windows Services нет штатных AWatch-rus задач, которые + запускают `powershell.exe` или `pwsh.exe`; +- install-kit поставляет Rust EXE, конфиги и service/task definitions вместо + runtime `.ps1`; +- Ansible разворачивает Rust binaries и конфиги, а не копирует PowerShell + runtime scripts; +- Telegram/Codex/bot контур вызывает Rust helpers или читает Rust JSON + snapshots, не опираясь на пути к `.ps1`; +- legacy PowerShell хранится только как временный rollback-слой до прохождения + acceptance gates и затем удаляется из install-kit; +- Linux/AW/Proxmox части остаются Rust-first; Ansible может остаться + оркестратором, потому что это инфраструктурный deploy layer, а не runtime + PowerShell. + +## 2. Текущее состояние + +Уже выполнено или частично выполнено: + +- `awatch-agent-rs` заменяет Windows `worktime-session-collector.ps1` для + worktime/RDP path. Legacy PowerShell оставлен как fallback и управляется + `collectors.worktimeSessionEnabled`, `worktimeSessionMode`, + `worktimeLegacyFallbackEnabled`. +- `aw-windows-telemetry.exe` уже используется для: + - `file1c-upload`; + - `dlp-evidence-sync`. +- `aw-1c-ingest-rust` пишет 1C/file analytics в ClickHouse на серверной + стороне. +- `aw-windows-telemetry.exe validate-deployment` добавлен как Rust validation + gate первого уровня. Он проверяет уже мигрированные Windows Rust paths, + свежесть worktime bucket, Rust collector guard service и queue sanity без + запуска PowerShell. +- `AWatchRusCollectorGuard` переключен с `aw-collector-guard.ps1` на + `aw-windows-telemetry.exe collector-guard`. Старый PowerShell guard оставлен + как rollback script, но штатный service runtime его не запускает. Rust guard + получает `sessionId` из native process snapshot, дедуплицирует legacy + collectors по `(kind, sessionId)` и не запускает пользовательские launch + tasks повторно, если legacy collectors уже активны. +- `aw-windows-telemetry.exe` добавил P0 runtime collector subcommands: + `browser-domains-collector`, `dlp-endpoint-collector`, + `file-operations-collector`. На live Windows/RDP host они включены через + `collectors.*Mode=rust_primary`, работают в трех пользовательских сессиях, + а P0 PowerShell collector runtime отсутствует. Это не означает полного + удаления legacy `.ps1`: они оставлены как rollback/reference до расширения + глубокой функциональной parity. +- Основные серверные DetMir/AW/DLP helpers уже Rust-first: + `detmir-status`, `detmir-check`, `detmir-auto`, `detmir-heal-safe`, + `aw-rus-healthd-rust`, `dlp-*`, `worktime-*`, `aw-health-check`, + `check-aw-data`, `aw-prune-local-state` и другие. + +Остаток PowerShell в рабочем дереве: + +- 26 product `.ps1` в `windows/`; +- operator/MCP helper `scripts/powershell/detmir-powershell-profile.ps1`; +- parse-check helper `.pssa_run.ps1`; +- `.venv/bin/activate.ps1` внутри virtualenv не является продуктовым + компонентом и не входит в миграцию. + +## 3. Инвентаризация Windows PowerShell + +| Скрипт | Роль | Цель миграции | Приоритет | +|---|---|---|---| +| `worktime-session-collector.ps1` | RDP/worktime сбор | закрепить замену через `awatch-agent-rs`, затем удалить fallback | P0 done/stabilize | +| `browser-domains-native-collector.ps1` | browser/domain и DLP web-сигналы | `aw-windows-telemetry.exe browser-domains-collector`; live Rust primary, legacy fallback/reference | P0 live/stabilize | +| `dlp-endpoint-signals-collector.ps1` | clipboard/USB/print/DLP incident signals | `aw-windows-telemetry.exe dlp-endpoint-collector`; live Rust primary, screenshots только для DLP events | P0 live/stabilize | +| `file-operations-collector.ps1` | file create/delete/rename/archive hints | `aw-windows-telemetry.exe file-operations-collector`; live Rust primary with queue/spool | P0 live/stabilize | +| `email-outbound-collector.ps1` | outbound email metadata/DLP | Rust email metadata collector | P1 | +| `dlp-policy-client.ps1` | получение DLP policy | Rust policy client/cache | P1 | +| `export-upload-file-1c-telemetry.ps1` | 1C telemetry upload | закрепить `aw-windows-telemetry.exe file1c-upload`, затем удалить legacy | P0 done/stabilize | +| `sync-dlp-evidence-artifacts.ps1` | DLP evidence upload | закрепить `aw-windows-telemetry.exe dlp-evidence-sync` | P0 done/stabilize | +| `export-evtx-for-hayabusa.ps1` | bounded EVTX export | Rust EVTX export helper | P1 | +| `export-upload-hayabusa-to-aw-server.ps1` | EVTX/Hayabusa upload | Rust upload helper или режим в `aw-windows-telemetry.exe` | P1 | +| `aw-standalone-service.ps1` | supervisor service wrapper | Rust Windows service wrapper | P0 | +| `aw-collector-guard.ps1` | guard/restart/recovery | Rust guard with allowlist, lock, cooldown | P0 done/stabilize | +| `hardening-recovery.ps1` | recovery/hardening | Rust recovery CLI; destructive actions behind `--apply` | P1 | +| `validate-deployment.ps1` | post-deploy validation | Rust validation CLI `aw-windows-telemetry.exe validate-deployment`; расширять до полной parity перед удалением `.ps1` | P0 started | +| `rebuild-worktime-tasks.ps1` | rebuild scheduled tasks | Rust task reconciliation CLI | P1 | +| `fix-session-watchers.ps1` | repair session watchers | Rust repair subcommand | P2 | +| `cleanup-disc-sessions.ps1` | cleanup stale disconnected sessions | Rust maintenance subcommand | P2 | +| `migrate-awatch-rus-paths.ps1` | path migration | Rust migration CLI with backup and dry-run | P1 | +| `deploy-single-user.ps1` | single-user deploy | Rust/Ansible-backed installer action | P1 | +| `deploy-domain-users.ps1` | domain deploy | Rust/Ansible-backed installer action | P1 | +| `deploy-ensemble.ps1` | orchestrated deploy | Rust deploy coordinator or Ansible playbook wrapper | P1 | +| `install-standalone-service.ps1` | local service install | Rust installer/bootstrap CLI | P0 | +| `install-collector-guard-service.ps1` | guard service install | Rust installer/bootstrap CLI | P0 | +| `install-dlp-client.ps1` | DLP client install | Rust installer/bootstrap CLI | P1 | +| `audit-cryptopro.ps1` | CryptoPro audit | Rust audit CLI | P2 | +| `run-user1-probe.ps1` | manual probe | Rust diagnostic probe | P2 | + +## 4. Неприкосновенные ограничения + +- Не делать big-bang replacement. +- Не отключать PowerShell fallback до прохождения shadow/parity gates. +- Не переносить функции, которые требуют скрытого сбора, keylogging, screen + recording или content interception. Скриншоты допустимы только для DLP + incident evidence, если это явно включено политикой. +- Не менять бизнес-логику DLP, 1C, worktime или evidence при переносе. +- Не менять thresholds, bucket names, event schema и ClickHouse schema без + отдельного решения. +- Не ломать install-kit и rollback ради удаления `.ps1`. +- Не хранить live hostnames, private IP, tokens, passwords или runtime evidence + paths в публичных tracked docs. + +## 5. Общий контракт каждого Rust EXE + +Каждый заменяющий EXE должен иметь: + +- `--config `; +- `--json` для машинного вывода; +- `--dry-run` для всех действий, меняющих состояние; +- `--timeout-seconds`; +- `--log-path` или structured logging в штатный каталог; +- `--version`; +- стабильные exit codes: + - `0` - OK; + - `1` - usage/config/runtime error; + - `2` - check выполнен, но состояние WARN/FAIL; + - `3` - action запрещен safety policy; +- structured JSON event для audit trail; +- lock/cooldown для guard/recovery/actions; +- spool/queue для сетевых upload path; +- atomic writes для state files; +- no implicit sudo/admin: повышенные права должны быть видны в task/service + definition; +- rollback compatibility с текущими config keys. + +## 6. Фазы миграции + +### Phase 0. Baseline и реестр вызовов + +Цель: зафиксировать, где PowerShell реально используется. + +Действия: + +1. Сканировать репозиторий: + - `windows/*.ps1`; + - `windows/installkit/innosetup/*.iss`; + - `ansible/*.yml`; + - `ansible/group_vars/*.yml`; + - `scripts/*`; + - docs/runbooks. +2. Снять runtime baseline на тестовом Windows host: + - Scheduled Tasks; + - Windows Services; + - текущие command lines процессов; + - `deployment-config.json`; + - свежесть AW buckets; + - наличие DLP evidence и 1C upload. +3. Для каждого скрипта зафиксировать: + - входные параметры; + - env/config keys; + - side effects; + - event schema; + - log/state paths; + - expected exit codes; + - rollback path. + +Выход фазы: + +- `docs/POWERSHELL_TO_RUST_ROADMAP_RU.md` как базовая дорожная карта; +- отдельный runtime inventory для live-контура в private/operator notes, без + публикации секретов. + +### Phase 1. Windows Rust foundation + +Цель: подготовить общий Windows runtime вместо набора разрозненных EXE. + +Действия: + +1. Расширить существующие `awatch-agent-rs` и `aw-windows-telemetry` только в + рамках parity, без новых функций. +2. Вынести общие Windows helpers: + - config loading; + - ActivityWatch HTTP client; + - evidence upload client; + - Windows task/service inspection; + - Event Log/EVTX access; + - filesystem state/spool; + - structured logs/audit. +3. Зафиксировать единый Windows config contract: + - `C:\ProgramData\AWatch-rus\deployment-config.json`; + - `C:\Program Files\AWatch-rus\windows\*.exe`; + - `C:\ProgramData\AWatch-rus\logs`; + - `C:\ProgramData\AWatch-rus\spool`; + - `C:\ProgramData\AWatch-rus\switch-backups`. + +Gate: + +```bash +cd adk-rust +cargo fmt --all -- --check +cargo test --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo build --release -p awatch-agent-rs -p aw-windows-telemetry +``` + +### Phase 2. Закрепить уже выполненные замены + +Цель: не переписать повторно то, что уже переведено, а довести до production +definition of done. + +Компоненты: + +- `worktime-session-collector.ps1` -> `awatch-agent-rs`; +- `export-upload-file-1c-telemetry.ps1` -> `aw-windows-telemetry.exe file1c-upload`; +- `sync-dlp-evidence-artifacts.ps1` -> `aw-windows-telemetry.exe dlp-evidence-sync`; +- `aw-collector-guard.ps1` -> `aw-windows-telemetry.exe collector-guard` + через `AWatchRusCollectorGuard` service wrapper; +- серверный writer -> `aw-1c-ingest-rust`. + +Действия: + +1. Проверить, что task/service actions запускают EXE, а не PowerShell. +2. Проверить, что PowerShell fallback выключен там, где Rust уже стабилен. +3. Проверить свежесть buckets/ClickHouse/evidence after restart. +4. Обновить install-kit file list: Rust EXE являются primary artifact. +5. Оставить legacy `.ps1` только как rollback на ограниченный период. + +Gate: + +- `ActivityWatch File1C Upload` запускает `aw-windows-telemetry.exe`; +- `ActivityWatch DLP Evidence Sync` запускает `aw-windows-telemetry.exe`; +- worktime events имеют `source=awatch-agent-rs`; +- `AWatchRusCollectorGuard` запущен и его child process - + `aw-windows-telemetry.exe collector-guard`; +- guard не создает duplicate legacy browser/fileops/DLP endpoint collectors + при stale bucket: при уже активных collectors launch tasks не запускаются + повторно; +- 1C ClickHouse ingest идет по `aw-1c-ingest-rust`; +- PowerShell worktime process count = 0 в штатном режиме. +- PowerShell `aw-collector-guard.ps1` process count = 0 в штатном режиме. + +### Phase 3. Runtime collectors + +Цель: заменить регулярный сбор данных на Windows. + +Порядок: + +1. `browser-domains-native-collector.ps1` + - current status: live Rust primary through + `aw-windows-telemetry.exe browser-domains-collector`; + - ActivityWatch window/category health events are written with + `source=aw-windows-telemetry-rust`; + - UIAutomation URL/domain extraction implemented in Rust: browser process + detection, normalized URL, host/rootDomain, default/custom category rules, + `aw-watcher-web-*`, `aw-detmir-web-category_*` and web DLP incident + schema preserve the legacy bucket/event contract; + - live disconnected RDP sessions may legitimately report + `browserDetected=false` and `urlDetected=false` until a browser is + foreground in an interactive user session; + - screenshots только для DLP incident evidence. +2. `dlp-endpoint-signals-collector.ps1` + - current status: live Rust primary through + `aw-windows-telemetry.exe dlp-endpoint-collector`; + - endpoint collector health is written with + `source=aw-windows-telemetry-rust`; + - clipboard metadata/hash/length, USB insert, print job metadata and DLP + incident event semantics implemented in Rust with legacy fields: + `requestedAction`, `enforcementMode`, `nativeChannelAction`, + `enforcementSuppressed`, content pack matches and `enforced`; + - destructive endpoint enforcement for USB write-block and print cancel is + intentionally not enabled during pilot hardening: Rust emits equivalent + audit/incident semantics and suppresses unsafe block actions unless a + separate enforcement decision is made; + - incident screenshot policy remains: screenshots only for DLP events. +3. `file-operations-collector.ps1` + - current status: live Rust primary through + `aw-windows-telemetry.exe file-operations-collector`; + - bounded filesystem watcher, operation schema, queue/spool and + create/rename/delete smoke are verified; + - per-session queue/state/log files are used, so RDP sessions do not share + one state file; + - legacy PowerShell remains installed only as rollback/reference. +4. `email-outbound-collector.ps1` + - metadata-only parity; + - Outlook/SMTP mode behavior preserved; + - no content interception beyond current documented behavior. +5. `dlp-policy-client.ps1` + - Rust policy fetch/cache; + - strict validation; + - safe fallback to last known good policy. + +Shadow-mode: + +- Rust collector writes `source=` and `mode=shadow`; +- PowerShell remains primary during comparison; +- compare event counts, schema, timestamps, severity, policy hits and evidence + links for at least several collection cycles. + +Gate: + +- no duplicate management conclusions in portal; +- event schema compatible with current AW/DLP consumers; +- no stale buckets introduced; +- no uncontrolled screenshot capture; +- Rust collector survives AW API outage by spooling and later flushing. + +### Phase 4. Guard, service wrapper и recovery + +Цель: заменить PowerShell, который управляет процессами и восстановлением. + +Компоненты: + +- `aw-standalone-service.ps1`; +- `aw-collector-guard.ps1`; +- `hardening-recovery.ps1`; +- `rebuild-worktime-tasks.ps1`; +- `fix-session-watchers.ps1`; +- `cleanup-disc-sessions.ps1`. + +Действия: + +1. Сделать Rust Windows service wrapper: + - supervises configured collectors; + - records child process state; + - no hidden PowerShell spawn in normal mode. +2. Сделать Rust guard/recovery: + - allowlist actions only; + - lock file; + - cooldown; + - dry-run by default for destructive or repair actions; + - audit entry for every change. +3. Перевести rebuild/fix/cleanup в subcommands одного maintenance EXE. + +Gate: + +- controlled restart of one collector works; +- stale collector detection matches old guard; +- recovery cannot restart arbitrary process/service; +- rollback restores PowerShell guard/service within one operator action. + +### Phase 5. Install/deploy/validate + +Цель: install-kit перестает запускать PowerShell как штатный bootstrap. + +Компоненты: + +- `install-standalone-service.ps1`; +- `install-collector-guard-service.ps1`; +- `install-dlp-client.ps1`; +- `deploy-single-user.ps1`; +- `deploy-domain-users.ps1`; +- `deploy-ensemble.ps1`; +- `validate-deployment.ps1`; +- `migrate-awatch-rus-paths.ps1`; +- InnoSetup `AWatch-rus-InnoSetup.iss`; +- Ansible Windows playbooks/group vars. + +Действия: + +1. Создать Rust bootstrap/installer EXE: + - install/update service; + - install/update scheduled tasks; + - write config atomically; + - backup previous config/tasks; + - emit JSON validation report. +2. Перевести validation в Rust: + - task actions; + - service status; + - bucket freshness; + - file permissions; + - rust binary version matrix; + - no PowerShell runtime task check. +3. Обновить InnoSetup: + - package `*.exe`; + - run Rust bootstrap; + - keep `.ps1` out of normal install payload after stabilization. +4. Обновить Ansible: + - deploy EXE; + - configure task/service actions to EXE; + - remove default `.ps1` paths from bot/env after migration. + +Gate: + +- clean install on test Windows host; +- upgrade from PowerShell install to Rust install; +- rollback to previous package; +- `validate-deployment` JSON is consumed by CI/operator without parsing human + text; +- install-kit verification confirms no `powershell.exe` in primary install + action. + +### Phase 6. Ops-only хвост + +Цель: убрать одноразовые PowerShell helpers. + +Компоненты: + +- `audit-cryptopro.ps1`; +- `run-user1-probe.ps1`; +- `scripts/powershell/detmir-powershell-profile.ps1`; +- `.pssa_run.ps1`. + +Решение: + +- `audit-cryptopro.ps1` -> Rust audit CLI; +- `run-user1-probe.ps1` -> Rust diagnostic probe or remove if obsolete; +- operator PowerShell profile is not production runtime and can be retired + after Rust/SSH operator commands exist; +- `.pssa_run.ps1` removed when no product PowerShell remains. + +Gate: + +- no production task/service depends on these helpers; +- docs no longer instruct operator to run PowerShell for routine checks; +- one emergency manual path remains documented, but not packaged as runtime. + +### Phase 7. Decommission + +Цель: убрать PowerShell from product surface. + +Действия: + +1. Remove `.ps1` from install-kit file list. +2. Remove `powershell.exe` installer run action. +3. Remove default `.ps1` paths from Ansible/bot env. +4. Update architecture/docs: + - Windows collectors are Rust EXE; + - PowerShell no longer prerequisite for runtime; + - rollback history documented separately. +5. Run tracked-file hygiene scan for public docs. + +Gate: + +```bash +rg -n "powershell\\.exe|\\.ps1|PowerShell" windows ansible docs scripts README.md +``` + +Expected result: + +- only historical notes, explicit rollback docs, or non-runtime examples remain; +- no install-kit primary action launches PowerShell; +- no AWatch-rus Scheduled Task action launches PowerShell in live validation. + +## 7. Rollout по хостам + +Порядок раскатки: + +1. Local build host: + - build Windows EXE; + - artifact check; + - installer dry-run. +2. Test Windows/RDP host: + - install Rust EXE side-by-side; + - shadow-mode collectors; + - compare with PowerShell. +3. Canary production Windows/RDP host: + - one host, one business day; + - monitor bucket freshness, DLP incidents, 1C upload, evidence upload. +4. Remaining Windows hosts: + - staged batches; + - no more than one failure domain at a time. +5. AW server / Proxmox / gateway: + - update Ansible/bot references; + - verify server Rust services remain active; + - verify ClickHouse ingest and portal health. +6. Install-kit: + - publish Rust-first package; + - keep previous package as rollback artifact. + +## 8. Acceptance gates + +Local gates: + +```bash +cd adk-rust +cargo fmt --all -- --check +cargo test --workspace +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo build --workspace --release +cargo build --release --target x86_64-pc-windows-gnu -p awatch-agent-rs -p aw-windows-telemetry +``` + +Repository gates: + +```bash +scripts/check_detmir_rust_release_artifacts.sh +scripts/verify_innosetup_installer.sh +git diff --check +``` + +Windows gates: + +```powershell +Get-ScheduledTask | Where-Object { + $_.TaskName -like 'ActivityWatch*' -or $_.TaskName -like 'AWatch*' +} | Select-Object TaskName,TaskPath,State +``` + +Expected: + +- task actions point to Rust EXE for migrated components; +- no migrated component runs `powershell.exe`; +- logs are fresh; +- spool is empty or draining; +- AW buckets are fresh; +- DLP evidence sync only uploads DLP incident evidence; +- 1C upload does not copy screenshots. + +Server gates: + +```bash +detmir-status --json +detmir-check --json +systemctl --failed --no-pager +systemctl list-timers aw-1c-ingest.timer --no-pager +``` + +Expected: + +- DetMir severity is OK or explained WARN; +- no failed Rust services; +- ClickHouse writer timer is active; +- portal data freshness is acceptable for pilot. + +## 9. Rollback model + +До decommission каждая миграция хранит rollback: + +- previous EXE/script backup in `switch-backups`; +- previous task/service definition; +- previous `deployment-config.json`; +- one-command switch back for canary host; +- rollback reason written to audit log. + +Rollback триггеры: + +- AW bucket stale/dead after migration; +- DLP event loss or uncontrolled duplicate events; +- evidence upload fails repeatedly and spool grows; +- 1C upload stops producing landing files; +- service/guard restarts loop; +- Windows host shows sustained CPU/RAM regression from new EXE; +- user-visible pilot portal data quality degrades. + +## 10. Риски + +| Риск | Где | Снижение риска | +|---|---|---| +| Windows API differs from PowerShell cmdlets | DLP, tasks, Event Log, print, WMI | parity fixtures, canary, schema comparison | +| UIAutomation/browser URL extraction changes behavior | browser collector | shadow-mode and domain count comparison | +| Outlook/SMTP metadata behavior differs | email collector | metadata-only parity, explicit mode tests | +| Privilege mismatch | services/tasks/recovery | install under same account, explicit elevation, validation | +| AV/EDR blocks unsigned EXE | Windows hosts | code signing plan, allowlist, staged rollout | +| Duplicate events during shadow | all collectors | shadow source tags, portal ignores shadow for KPI | +| Rollback not fast enough | production canary | backup tasks/configs, single switch command | +| Public docs leak live contour details | tracked docs | placeholders and `git grep` hygiene scan | + +## 11. Минимальный порядок ближайших работ + +1. Зафиксировать live inventory PowerShell usage на тестовом Windows/RDP host. +2. Закрепить уже сделанные Rust paths: + - worktime/RDP; + - 1C file upload; + - DLP evidence sync. +3. Расширить `aw-windows-telemetry.exe validate-deployment` до полной parity с + `validate-deployment.ps1`, потому что он станет главным gate для следующих + замен. +4. Перенести `aw-standalone-service.ps1` и `aw-collector-guard.ps1`, потому + что они управляют runtime collectors. +5. Стабилизировать P0 runtime collectors после live Rust-primary switch: + - browser domains; + - DLP endpoint signals; + - file operations. + URL/domain extraction и clipboard/USB/print incident semantics закрыты на + уровне Rust code/schema/runtime self-test. Следующий шаг перед удалением + legacy `.ps1` - burn-in, canary rollback test и live foreground-browser + proof в интерактивной RDP-сессии. +6. Перенести deploy/install scripts и InnoSetup primary action. +7. Убрать PowerShell paths из Ansible/bot env. +8. Удалить `.ps1` из install-kit и оставить только historical rollback docs. + +## 12. Итоговая оценка + +Миграция реалистична, потому что серверная часть уже Rust-first, а Windows +контур уже имеет два рабочих Rust основания: `awatch-agent-rs` и +`aw-windows-telemetry.exe`. + +Критичный участок не сервер, а Windows runtime и install-kit: + +- `dlp-endpoint-signals-collector.ps1`; +- `browser-domains-native-collector.ps1`; +- `aw-collector-guard.ps1`; +- `aw-standalone-service.ps1`; +- `validate-deployment.ps1`; +- InnoSetup action that still launches PowerShell. + +До пилота не нужно удалять весь PowerShell хвост. Для пилота достаточно +закрепить уже работающие Rust-primary paths, не допустить копирования +скриншотов в 1C контуре, оставить скриншоты только для DLP events, иметь +понятный rollback и честно формулировать границу parity: code/schema/runtime +path закрыт, а полный live URL/domain incident proof требует активного +foreground browser в интерактивной RDP-сессии. +Полное удаление PowerShell из install-kit лучше делать после canary и shadow +parity по DLP/browser/file collectors. diff --git a/docs/SECURITY_EVENTS_CLICKHOUSE_RU.md b/docs/SECURITY_EVENTS_CLICKHOUSE_RU.md index 68c7733..8e84edd 100644 --- a/docs/SECURITY_EVENTS_CLICKHOUSE_RU.md +++ b/docs/SECURITY_EVENTS_CLICKHOUSE_RU.md @@ -34,6 +34,120 @@ CLICKHOUSE_PASSWORD= `CLICKHOUSE_PASSWORD` используется только для HTTP Basic Auth и не выводится в JSON, markdown-отчеты или health-ответы. +## Текущий DetMir runtime + +В рабочем контуре DetMir ClickHouse запущен не на AW-сервере ``, а +на gateway/Proxmox-хосте `` как Docker Compose service: + +- каталог: `/opt/activitywatch/clickhouse-1c`; +- контейнер: `aw-rus-1c-clickhouse`; +- образ: `clickhouse/clickhouse-server:24.8`; +- HTTP: `:8123`; +- native: `:9000`; +- база: `analytics_1c`; +- credentials: `/opt/activitywatch/clickhouse-1c/.env` на ``. + +Для `detmir-portal` или `detmir-check`, запущенных на AW-сервере +``, используйте: + +```env +SECURITY_EVENTS_BACKEND=clickhouse +CLICKHOUSE_URL=http://:8123 +CLICKHOUSE_DATABASE=analytics_1c +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD=<из /opt/activitywatch/clickhouse-1c/.env на > +``` + +Проверка с `` без пароля может вернуть `AUTHENTICATION_FAILED`; это +подтверждает сетевую доступность `:8123`, но не проверяет +аутентификацию. + +## Контроль состояния + +Текущий контур контроля ClickHouse: + +- Docker Compose healthcheck у контейнера `aw-rus-1c-clickhouse`. В норме + `docker compose ps` показывает статус `(healthy)`. +- `aw-1c-clickhouse-health.timer` на ``, запуск каждые 5 минут. + Проверяет Docker state, Docker health, authenticated `SELECT 1` через + `clickhouse-client`, HTTP `SELECT 1`, freshness таблиц и свободное место на + volume ClickHouse. +- `aw-clickhouse-network-health.timer` на AW-сервере ``, запуск + каждые 5 минут. Проверяет TCP-доступность `:8123/9000` и HTTP-ответ + ClickHouse со стороны AW-rus сервера. +- `aw-1c-ingest.timer` сохранен как writer в ClickHouse: цикл сбора и записи + данных выполняется раз в 15 минут (`OnUnitActiveSec=15min`) через Rust-бинарник + `/usr/local/bin/aw-1c-ingest-rust`. +- Windows-задача `ActivityWatch File1C Upload` на RDP-хосте запускает + Rust-бинарник `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe` + в режиме `file1c-upload`; legacy PowerShell exporter оставлен только как + fallback. +- DLP-события обрабатываются не реже 15 минут: `activitywatch-dlp-aggregator` + работает каждые 5 минут, `aw-dlp-influx-exporter` - каждые 10 минут, + CEF/syslog/webhook forwarder'ы - каждые 2-5 минут, Windows + `ActivityWatch DLP Evidence Sync` запускает тот же Rust-бинарник в режиме + `dlp-evidence-sync` каждые 15 минут. Sync копирует только PNG, похожие на + DLP incident screenshots (`web`, `clipboard`, `usb_insert`, `print_job` в + имени файла), и игнорирует 1C/прочие PNG в `incident-artifacts`. +- `aw-1c-proofcheck.timer` отдельно проверяет свежесть 1C-таблиц. + +Операционные команды: + +```bash +ssh detmir_proxmox 'cd /opt/activitywatch/clickhouse-1c && sudo docker compose ps' +ssh detmir_proxmox 'systemctl status aw-1c-clickhouse-health.timer aw-1c-clickhouse-health.service --no-pager' +ssh detmir_proxmox 'sudo journalctl -u aw-1c-clickhouse-health.service -n 30 --no-pager' +ssh detmir_aw 'systemctl status aw-clickhouse-network-health.timer aw-clickhouse-network-health.service --no-pager' +ssh detmir_aw 'sudo journalctl -u aw-clickhouse-network-health.service -n 30 --no-pager' +``` + +## Ручное изменение параметров Windows DLP evidence sync + +Использовать, если нужно временно изменить период или API без полного redeploy. +После ручного изменения желательно перенести значение в Ansible vars. + +### Период DLP evidence sync + +На Windows/RDP host в elevated PowerShell: + +```powershell +$taskName = "ActivityWatch DLP Evidence Sync" +$minutes = 15 + +$task = Get-ScheduledTask -TaskName $taskName +$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).Date) ` + -RepetitionInterval (New-TimeSpan -Minutes $minutes) ` + -RepetitionDuration (New-TimeSpan -Days 3650) + +Set-ScheduledTask -TaskName $taskName ` + -Action $task.Actions ` + -Trigger $trigger ` + -Principal $task.Principal ` + -Settings $task.Settings +``` + +### Evidence API / token / state / log paths + +```powershell +$exe = "C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe" +$args = 'dlp-evidence-sync --evidence-api-url "http://:8721/api/dlp/evidence/upload" --token-path "C:\ProgramData\AWatch-rus\dlp-evidence-upload-token.txt" --state-path "C:\ProgramData\AWatch-rus\dlp-evidence-sync-state.json" --log-path "C:\ProgramData\AWatch-rus\logs\dlp-evidence-sync.log"' + +$action = New-ScheduledTaskAction -Execute $exe -Argument $args +Set-ScheduledTask -TaskName "ActivityWatch DLP Evidence Sync" -Action $action +``` + +Проверить: + +```powershell +& "C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe" dlp-evidence-sync ` + --evidence-api-url "http://:8721/api/dlp/evidence/upload" ` + --token-path "C:\ProgramData\AWatch-rus\dlp-evidence-upload-token.txt" ` + --state-path "C:\ProgramData\AWatch-rus\dlp-evidence-sync-state.json" ` + --log-path "C:\ProgramData\AWatch-rus\logs\dlp-evidence-sync.log" + +Get-ScheduledTaskInfo -TaskName "ActivityWatch DLP Evidence Sync" +``` + ## Ожидаемые агрегаты Портал формирует блок `security_events_summary`: diff --git a/docs/UI_ARCHITECTURE_BASELINE_RU.md b/docs/UI_ARCHITECTURE_BASELINE_RU.md index 0f8676b..409273c 100644 --- a/docs/UI_ARCHITECTURE_BASELINE_RU.md +++ b/docs/UI_ARCHITECTURE_BASELINE_RU.md @@ -10,8 +10,7 @@ Agent: Rust Current Portal: Rust server-rendered HTML + HTMX Future Enterprise UI: React + TypeScript Future Desktop Forensics: Tauri + React + Rust core -Dioxus: excluded from roadmap -DPD Portal: excluded from roadmap +Experimental Rust UI/prototype mirrors: excluded from roadmap ``` ## Правила @@ -22,7 +21,8 @@ DPD Portal: excluded from roadmap - Будущий React/Tauri UI не должен ломать текущий портал. - Agent и backend остаются Rust-first. - Новые UI-фреймворки не добавлять без отдельного architecture decision. -- Dioxus и DPD Portal исключены из архитектурного roadmap проекта. +- Экспериментальные Rust UI/prototype mirror направления не входят в + архитектурный roadmap проекта. - Будущий React/Tauri UI не должен парсить HTML текущего портала как источник данных. diff --git a/docs/WINDOWS_RUST_VALIDATION_RU.md b/docs/WINDOWS_RUST_VALIDATION_RU.md new file mode 100644 index 0000000..cc66024 --- /dev/null +++ b/docs/WINDOWS_RUST_VALIDATION_RU.md @@ -0,0 +1,150 @@ +# Windows Rust validation + +Дата: 2026-06-05 + +`aw-windows-telemetry.exe validate-deployment` является Rust-заменой первого +уровня для `windows/validate-deployment.ps1`. + +Цель команды: дать машинный JSON gate перед дальнейшей заменой Windows +PowerShell runtime scripts на Rust EXE. + +## Команда + +```powershell +C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe ` + validate-deployment ` + --config-path C:\ProgramData\AWatch-rus\deployment-config.json +``` + +Вывод: JSON. + +Exit codes: + +- `0` - validation прошла; +- `1` - ошибка запуска, чтения config или runtime error; +- `2` - validation выполнена, но есть failed sections. + +## Что проверяется сейчас + +- `deployment-config.json` читается как JSON, включая UTF-8 BOM. +- `ActivityWatch File1C Upload` указывает на + `aw-windows-telemetry.exe file1c-upload`. +- `ActivityWatch DLP Evidence Sync` указывает на + `aw-windows-telemetry.exe dlp-evidence-sync`. +- `AWatch Rust Telemetry Agent` указывает на `awatch-agent-rs.exe`. +- `AWatchRusCollectorGuard` service указывает на + `aw-windows-telemetry.exe collector-guard` через service wrapper. +- Rust collector guard получает `sessionId` через native Windows API, + дедуплицирует legacy browser/fileops/DLP endpoint collectors по + `(kind, sessionId)` и пропускает повторный запуск launch tasks, если legacy + collectors уже работают. +- Rust worktime agent запущен. +- Rust collector guard запущен. +- Worktime bucket свежий. +- Локальные DLP/file operation queues не превышают безопасный depth. +- `aw-windows-telemetry.exe browser-domains-collector`, + `dlp-endpoint-collector` и `file-operations-collector` доступны как P0 + runtime subcommands. +- Browser Rust collector реализует URL/domain/category parity: + UIAutomation extraction, URL normalization, host/rootDomain, default/custom + category rules, `aw-watcher-web-*`, `aw-detmir-web-category_*` и web DLP + incident schema. +- DLP endpoint Rust collector реализует incident semantics для + `clipboard_change`, `usb_insert` и `print_job`: raw endpoint signal, + policy evaluation, cooldown и `aw-dlp-incidents_*` event fields сохранены. + USB write-block и print cancel не выполняются в пилотном режиме без + отдельного enforcement решения. +- В live-конфигурации `collectors.browserCollectorMode`, + `collectors.dlpEndpointMode` и `collectors.fileOpsMode` переключены в + `rust_primary`. +- P0 PowerShell collector runtime + (`browser-domains-native-collector.ps1`, + `dlp-endpoint-signals-collector.ps1`, `file-operations-collector.ps1`) + отсутствует в штатных пользовательских сессиях. +- Rust collectors пишут per-session state/log/queue, чтобы разные RDP-сессии + не конфликтовали за один state-файл. +- Оставшийся PowerShell runtime классифицируется для миграционной карты через + native WinAPI process snapshot. `wmic.exe` не требуется. + +## Инвентаризация процессов + +На новых Windows `wmic.exe` может отсутствовать. Validator не зависит от него: + +1. сначала используется native WinAPI process snapshot; +2. если native snapshot недоступен, используется `wmic.exe`; +3. если `wmic.exe` отсутствует, используется `tasklist.exe` fallback. + +В штатном режиме native snapshot дает: + +- `processes.commandLineQueryOk=true`; +- классификацию PowerShell runtime по видам: + `browser`, `fileops`, `dlp_endpoint`, `guard`, `recovery`, `worktime`; +- подтверждение отсутствия `worktime-session-collector.ps1`. +- подтверждение отсутствия `aw-collector-guard.ps1`. + +`tasklist.exe` fallback ограничен: + +- наличие `awatch-agent-rs.exe` подтверждается; +- command line PowerShell-процессов недоступна; +- поле `processes.commandLineQueryOk=false`; +- поле `processes.noPowerShellWorktimeRuntime=null`; +- это не считается ошибкой, если task actions, Rust agent и bucket freshness + подтверждены. + +Для полного доказательства отсутствия конкретного `.ps1` процесса можно +дополнительно использовать operator inventory через WinRM/Ansible, но это не +должно блокировать Rust validation на системах без WMIC. + +## Проверенный live checkpoint + +2026-06-05 на Windows/RDP host: + +- `overallOk=true`; +- failed sections: `[]`; +- File1C task: Rust EXE; +- DLP evidence task: Rust EXE; +- worktime agent: Rust EXE; +- collector guard service: Rust EXE child; +- worktime bucket age: меньше 300 секунд. +- native process inventory: `commandLineQueryOk=true`; +- PowerShell worktime runtime: отсутствует; +- PowerShell collector guard runtime: отсутствует; +- оставшийся PowerShell runtime классифицирован как `recovery/other`; P0 + browser/fileops/DLP endpoint PowerShell collectors отсутствуют. +- Rust browser/DLP/fileops runtime после live switch: `browser=3`, + `dlp_endpoint=3`, `fileops=3`, `guard=1`, legacy P0 count `0`. +- После одного guard-cycle P0 Rust counts остались `3/3/3`, DLP state по трем + сессиям: `status=ok`, `sendFailures=0`. +- После parity update 2026-06-05T23:10Z: + - `browser_rust=3`, `dlp_endpoint_rust=3`, `fileops_rust=3`, + `guard_rust=1`, legacy P0 collector count `0`; + - `validate-deployment`: `overallOk=true`, failed sections `[]`; + - AW API после restart `activitywatch-server` отвечает локально за ~0.002s; + - `aw-detmir-web-category_*` получает Rust `collector_health` с + `foregroundProcess`, `browserDetected`, `urlDetected`; + - текущие RDP-сеансы disconnected, поэтому live foreground пустой и + `browserDetected=false`, `urlDetected=false`; URL event path проверен + self-test/unit-test, но live URL требует активного foreground browser; + - `aw-dlp-endpoint-signals_*` получает свежие Rust `self_test` events: + `queueDepth=0`, `sendFailures=0`; + - `detmir-dlp`: `ok=true`, `counts={ok:22,warn:0,fail:0}`; + - `detmir-status`: `severity=OK`, `needs_heal=false`, + `ok_for_operator=true`; актуальный refresh через Rust-first + `/usr/local/bin/detmir-auto --no-heal --no-report` дал + `service_warnings=0` и DLP `counts={ok:22,warn:0,fail:0}`. +- Rust file operations live mode использует per-session queue/state/log; + bounded shadow create/rename/delete smoke ранее подтвердил `Created`, один + `Renamed` с `oldPath`, `Deleted` и штатные `collector_health` события. +- Collector guard duplicate prevention smoke: при принудительном stale + action `run-task` получил `applied=false`, + `reason=legacy-collectors-already-running`; runtime counts остались + `browser=3`, `fileops=3`, `dlp_endpoint=2`. + +Ограничение: этот checkpoint подтверждает end-to-end live runtime, policy +semantics и schema parity. Полный live URL/domain incident path физически +требует активного foreground browser в интерактивной RDP-сессии; disconnected +сеансы корректно дают только health/self-test. Удаление legacy `.ps1` остается +отдельным rollback/decommission gate. + +Live hostnames, private IP и runtime report сохранены только в private +`.ops`-контуре. diff --git a/ops/detmir-ai/detmir-auto b/ops/detmir-ai/detmir-auto index 811eb2f..3285c10 100644 --- a/ops/detmir-ai/detmir-auto +++ b/ops/detmir-ai/detmir-auto @@ -1,169 +1,24 @@ #!/usr/bin/env bash set -euo pipefail -state_dir="${DETMIR_AI_STATE_DIR:-/var/lib/detmir-ai}" -lock_dir="${DETMIR_AI_RUN_DIR:-${XDG_RUNTIME_DIR:-/tmp}}" -auto_heal="${DETMIR_AUTO_HEAL:-1}" -retain_days="${DETMIR_AI_RETAIN_DAYS:-14}" +rust_bin="${DETMIR_AUTO_RUST_BIN:-/usr/local/bin/detmir-auto-rust}" +legacy_bin="${DETMIR_AUTO_LEGACY_BIN:-/usr/local/bin/detmir-auto.legacy-shell}" +env_file="${DETMIR_AUTO_ENV_FILE:-/etc/detmir/detmir-check.env}" -mkdir -p "$state_dir"/runs "$state_dir"/reports "$state_dir"/logs - -lock_file="$lock_dir/detmir-auto.lock" -exec 9>"$lock_file" -if ! flock -n 9; then - echo "detmir-auto: another run is active" - exit 0 +if [[ -r "$env_file" ]]; then + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a fi -stamp="$(date -u +%Y%m%d-%H%M%S)" -run_dir_full="$state_dir/runs/$stamp" -mkdir -p "$run_dir_full" - -check_file="$run_dir_full/detmir-check.json" -dlp_file="$run_dir_full/detmir-dlp.json" -heal_log="$run_dir_full/heal.log" -bundle_file="$run_dir_full/bundle.txt" -report_file="$state_dir/reports/detmir-report-$stamp.md" -state_file="$state_dir/state-$stamp.json" - -run_check() { - local rc=0 - detmir-check --json >"$check_file" || rc=$? - echo "$rc" >"$run_dir_full/check.rc" -} - -run_dlp() { - local rc=0 - detmir-dlp >"$dlp_file" || rc=$? - echo "$rc" >"$run_dir_full/dlp.rc" -} - -summarize() { - python3 - "$check_file" "$dlp_file" "$run_dir_full/check.rc" "$run_dir_full/dlp.rc" <<'PY' -import json -import sys -from pathlib import Path - -check_path, dlp_path, check_rc_path, dlp_rc_path = map(Path, sys.argv[1:]) -check_rc = int(check_rc_path.read_text().strip()) -dlp_rc = int(dlp_rc_path.read_text().strip()) -summary = { - "check_rc": check_rc, - "dlp_rc": dlp_rc, - "check_ok": False, - "dlp_ok": False, - "severity": "FAIL" if check_rc or dlp_rc else "OK", - "needs_heal": bool(check_rc or dlp_rc), - "reasons": [], -} -try: - check = json.loads(check_path.read_text()) - summary["check_ok"] = bool(check.get("ok")) - cs = check.get("summary") or {} - summary["detmir_summary"] = cs - if cs.get("bucket_dead", 0) or cs.get("bucket_stale", 0) or cs.get("service_failures", 0): - summary["reasons"].append("detmir-check has stale/dead bucket or required service failure") -except Exception as exc: - summary["reasons"].append(f"detmir-check parse failed: {exc}") - -try: - dlp = json.loads(dlp_path.read_text()) - summary["dlp_ok"] = bool(dlp.get("ok")) - summary["dlp_counts"] = dlp.get("counts") - counts = dlp.get("counts") or {} - if counts.get("fail", 0) or counts.get("warn", 0): - summary["reasons"].append("dlp-health-check has warn/fail") -except Exception as exc: - summary["reasons"].append(f"detmir-dlp parse failed: {exc}") - -if summary["check_ok"] and summary["dlp_ok"]: - summary["severity"] = "OK" - summary["needs_heal"] = False -elif not summary["reasons"]: - summary["severity"] = "WARN" -else: - summary["severity"] = "FAIL" - -print(json.dumps(summary, ensure_ascii=False, indent=2)) -PY -} - -run_check -run_dlp -summary_before="$(summarize)" -printf '%s\n' "$summary_before" >"$run_dir_full/summary-before.json" - -if [ "$auto_heal" = "1" ] && python3 -c 'import json,sys; print("yes" if json.load(sys.stdin).get("needs_heal") else "no")' <<<"$summary_before" | grep -qx yes; then - { - echo "detmir-heal-safe started at $(date -u --iso-8601=seconds)" - detmir-heal-safe - echo "detmir-heal-safe finished at $(date -u --iso-8601=seconds)" - } >"$heal_log" 2>&1 || true - sleep 10 - run_check - run_dlp -else - echo "auto-heal skipped" >"$heal_log" +if [[ -x "$rust_bin" ]]; then + exec "$rust_bin" "$@" fi -summary_after="$(summarize)" -printf '%s\n' "$summary_after" >"$state_file" +if [[ -x "$legacy_bin" ]]; then + exec "$legacy_bin" "$@" +fi -{ - echo "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет." - echo "Структура ответа:" - echo "1. Состояние: OK/WARN/FAIL" - echo "2. Что важно" - echo "3. Что уже сделал автомат" - echo "4. Что сделать человеку, если нужно" - echo - echo "Правила:" - echo "- Не предлагай рестарты, если факты чистые." - echo "- Отличай event-driven bucket от dead/stale." - echo "- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail." - echo "- Auto-heal умеет только серверные systemd-сервисы AW/DLP; Windows/RDP не трогает." - echo - echo "=== summary-before ===" - cat "$run_dir_full/summary-before.json" - echo - echo "=== summary-after ===" - cat "$state_file" - echo - echo "=== heal-log ===" - sed -n '1,300p' "$heal_log" - echo - echo "=== detmir-check ===" - sed -n '1,1600p' "$check_file" - echo - echo "=== detmir-dlp ===" - sed -n '1,1600p' "$dlp_file" -} >"$bundle_file" - -{ - echo "# DetMir Autonomous Report" - echo - echo "- generated_at_utc: $(date -u --iso-8601=seconds)" - echo "- run_dir: $run_dir_full" - echo - polli-chat --model text.daily --max-tokens 900 <"$bundle_file" || { - echo "Pollinations report failed; raw summary follows." - cat "$state_file" - } -} >"$report_file" - -ln -sfn "$run_dir_full" "$state_dir/latest-run" -ln -sfn "$report_file" "$state_dir/latest-report.md" -ln -sfn "$state_file" "$state_dir/latest-state.json" - -find "$state_dir/runs" -mindepth 1 -maxdepth 1 -type d -mtime +"$retain_days" -exec rm -rf {} + -find "$state_dir/reports" -type f -name 'detmir-report-*.md' -mtime +"$retain_days" -delete -find "$state_dir" -maxdepth 1 -type f -name 'state-*.json' -mtime +"$retain_days" -delete - -cat "$report_file" - -python3 - "$state_file" <<'PY' -import json -import sys -state = json.load(open(sys.argv[1])) -raise SystemExit(0 if state.get("severity") == "OK" else 2) -PY +echo "detmir-auto: neither $rust_bin nor $legacy_bin is executable" >&2 +exit 127 diff --git a/scripts/check_detmir_rust_release_artifacts.sh b/scripts/check_detmir_rust_release_artifacts.sh index 0d37baf..f5482cc 100644 --- a/scripts/check_detmir_rust_release_artifacts.sh +++ b/scripts/check_detmir_rust_release_artifacts.sh @@ -60,6 +60,7 @@ required_bins=( aw-hayabusa-link-case-rust aw-hayabusa-from-windows-rust aw-hayabusa-autoprocess-rust + aw-1c-ingest ) missing=0 diff --git a/scripts/detmir-portal-tabs-smoke.mjs b/scripts/detmir-portal-tabs-smoke.mjs index eb69e2b..72bd693 100644 --- a/scripts/detmir-portal-tabs-smoke.mjs +++ b/scripts/detmir-portal-tabs-smoke.mjs @@ -165,9 +165,9 @@ async function main() { name: "loading_refresh_status_ready", ok: (await page.locator("#loadingStatus").count()) === 1 + && (await page.locator("#loadingStatus").evaluate((node) => node.dataset.loadStatus)) === "READY" && (await page.locator("#loadingStateText").innerText({ timeout })).includes("Данные готовы") - && (await page.locator("#loadingStageText").innerText({ timeout })).includes("Данные готовы") - && !(await page.locator("body").innerText({ timeout })).includes("Загрузка данных"), + && (await page.locator("#loadingStageText").innerText({ timeout })).includes("Данные готовы"), }); smokeStep = "api:security_events_summary"; const reportsPayload = await page.evaluate(async () => { @@ -339,8 +339,8 @@ async function main() { const requiredSettings = [ "Период расчета", "Рабочий день", - "Порог WARN", - "Порог FAIL", + "Порог внимания", + "Порог критического риска", "Источник правил", "Дата последнего пересчета", ]; diff --git a/windows/AWatchRusCollectorGuardService.cs b/windows/AWatchRusCollectorGuardService.cs index efcbdd1..3644543 100644 --- a/windows/AWatchRusCollectorGuardService.cs +++ b/windows/AWatchRusCollectorGuardService.cs @@ -23,22 +23,27 @@ namespace AWatchRus Directory.CreateDirectory(Path.GetDirectoryName(options.LogPath)); File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " service starting" + Environment.NewLine); - var psi = new ProcessStartInfo - { - FileName = options.PowerShellPath, - Arguments = string.Format( + var fileName = string.IsNullOrWhiteSpace(options.ExecPath) ? options.PowerShellPath : options.ExecPath; + var arguments = string.IsNullOrWhiteSpace(options.ExecPath) + ? string.Format( "-NoProfile -ExecutionPolicy Bypass -File \"{0}\" -ConfigPath \"{1}\" -Mode {2} -LoopSeconds {3}", options.ScriptPath, options.ConfigPath, options.Mode, - options.LoopSeconds), + options.LoopSeconds) + : options.ExecArgs; + + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = false, RedirectStandardError = false, }; child = Process.Start(psi); - File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " child pid=" + child.Id + Environment.NewLine); + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " child pid=" + child.Id + " exec=" + fileName + Environment.NewLine); } protected override void OnStop() @@ -84,6 +89,8 @@ namespace AWatchRus public string Mode = "shadow"; public int LoopSeconds = 60; public string LogPath = @"C:\ProgramData\AWatch-rus\logs\collector-guard-service.log"; + public string ExecPath = null; + public string ExecArgs = null; } internal static class Program @@ -115,6 +122,8 @@ namespace AWatchRus if (int.TryParse(value, out parsed)) options.LoopSeconds = parsed; } else if (key == "--log") options.LogPath = value; + else if (key == "--exec") options.ExecPath = value; + else if (key == "--args") options.ExecArgs = value; i++; } return options; diff --git a/windows/ActivityWatch.Windows.Common.psm1 b/windows/ActivityWatch.Windows.Common.psm1 index 4f31335..f449f4b 100755 --- a/windows/ActivityWatch.Windows.Common.psm1 +++ b/windows/ActivityWatch.Windows.Common.psm1 @@ -931,7 +931,9 @@ function New-ActivityWatchDeploymentConfig { [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', [bool]$File1CAutoUploadEnabled = $true, [int]$File1CAutoUploadIntervalHours = 6, + [int]$File1CAutoUploadIntervalMinutes = 15, [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CAutoUploadRunAsUser, [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', [string]$File1CRemoteRoot = '/opt/activitywatch/clickhouse-1c/landing', @@ -981,6 +983,7 @@ function New-ActivityWatchDeploymentConfig { evtxExportScript = $EvtxExportScript hayabusaUploadScript = $HayabusaUploadScript file1cTelemetryScript = $File1CTelemetryScript + file1cTelemetryExecutable = if ([string]::IsNullOrWhiteSpace($File1CTelemetryScript)) { '' } else { Join-Path (Split-Path -Parent $File1CTelemetryScript) 'aw-windows-telemetry.exe' } rulesPath = $RulesPath policyPath = $PolicyPath launchScript = $LaunchScriptPath @@ -1022,8 +1025,10 @@ function New-ActivityWatchDeploymentConfig { analytics = [pscustomobject]@{ file1cAutomation = [pscustomobject]@{ enabled = [bool]$File1CAutoUploadEnabled + intervalMinutes = $File1CAutoUploadIntervalMinutes intervalHours = $File1CAutoUploadIntervalHours taskName = $File1CAutoUploadTaskName + runAsUser = $File1CAutoUploadRunAsUser targetHost = $File1CTargetHost targetUser = $File1CTargetUser remoteRoot = $File1CRemoteRoot @@ -1412,13 +1417,59 @@ function Start-CollectorScriptIfNeeded { Start-Process -FilePath `$PowerShellExe -ArgumentList `$argumentList -WindowStyle Hidden } +function Test-RustCollectorRunning { + param( + [string]`$Subcommand, + [int]`$SessionId + ) + + if ([string]::IsNullOrWhiteSpace(`$Subcommand)) { + return `$false + } + + return [bool]@( + Get-CimInstance Win32_Process -Filter "Name = 'aw-windows-telemetry.exe'" -ErrorAction SilentlyContinue | + Where-Object { + `$_.SessionId -eq `$SessionId -and + `$_.CommandLine -match [Regex]::Escape(`$Subcommand) + } | + Select-Object -First 1 + ).Count +} + +function Start-RustCollectorIfNeeded { + param( + [string]`$ExePath, + [string]`$Subcommand, + [string]`$ConfigPath, + [int]`$SessionId + ) + + if ([string]::IsNullOrWhiteSpace(`$ExePath) -or [string]::IsNullOrWhiteSpace(`$Subcommand)) { + return + } + + if (-not (Test-Path -LiteralPath `$ExePath)) { + return + } + + if (Test-RustCollectorRunning -Subcommand `$Subcommand -SessionId `$SessionId) { + return + } + + `$argumentList = @(`$Subcommand, '--config-path', `$ConfigPath, '--mode', 'enforce') + Start-Process -FilePath `$ExePath -ArgumentList `$argumentList -WindowStyle Hidden +} + `$config = Get-DeploymentConfig -Path `$ConfigPath `$sessionId = (Get-Process -Id `$PID).SessionId `$installRoot = [string]`$config.paths.installRoot `$stateRoot = [string]`$config.paths.stateRoot +`$deployRoot = if (`$config.paths.PSObject.Properties.Name -contains 'deployRoot' -and -not [string]::IsNullOrWhiteSpace([string]`$config.paths.deployRoot)) { [string]`$config.paths.deployRoot } elseif (`$config.paths.PSObject.Properties.Name -contains 'toolkitRoot' -and -not [string]::IsNullOrWhiteSpace([string]`$config.paths.toolkitRoot)) { [string]`$config.paths.toolkitRoot } else { `$installRoot } `$script:ApiBase = '{0}://{1}:{2}/api/0' -f [string]`$config.server.scheme, [string]`$config.server.host, [string]`$config.server.port `$script:Hostname = if (`$config.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]`$config.awHostname)) { [string]`$config.awHostname } else { `$env:COMPUTERNAME } `$script:KnownBuckets = @{} +`$telemetryExe = if (`$config.paths.PSObject.Properties.Name -contains 'file1cTelemetryExecutable' -and -not [string]::IsNullOrWhiteSpace([string]`$config.paths.file1cTelemetryExecutable)) { [string]`$config.paths.file1cTelemetryExecutable } else { Join-Path `$deployRoot 'windows\aw-windows-telemetry.exe' } `$collectorScript = [string]`$config.paths.collectorScript `$endpointCollectorScript = if (`$config.paths.PSObject.Properties.Name -contains 'endpointCollectorScript') { [string]`$config.paths.endpointCollectorScript } else { Join-Path `$stateRoot 'dlp-endpoint-signals-collector.ps1' } `$fileCollectorScript = if (`$config.paths.PSObject.Properties.Name -contains 'fileCollectorScript') { [string]`$config.paths.fileCollectorScript } else { Join-Path `$stateRoot 'file-operations-collector.ps1' } @@ -1430,6 +1481,9 @@ function Start-CollectorScriptIfNeeded { `$afkEnabled = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'afkEnabled') { [bool]`$config.collectors.afkEnabled } else { `$true } `$windowEnabled = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'windowEnabled') { [bool]`$config.collectors.windowEnabled } else { `$true } `$fileOpsEnabled = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'fileOpsEnabled') { [bool]`$config.collectors.fileOpsEnabled } else { `$true } +`$browserCollectorMode = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'browserCollectorMode') { [string]`$config.collectors.browserCollectorMode } else { 'powershell_primary' } +`$dlpEndpointMode = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'dlpEndpointMode') { [string]`$config.collectors.dlpEndpointMode } else { 'powershell_primary' } +`$fileOpsMode = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'fileOpsMode') { [string]`$config.collectors.fileOpsMode } else { 'powershell_primary' } `$emailEnabled = if (`$config.PSObject.Properties.Name -contains 'collectors' -and `$config.collectors.PSObject.Properties.Name -contains 'emailEnabled') { [bool]`$config.collectors.emailEnabled } else { `$false } `$emailCollectorScript = if (`$config.paths.PSObject.Properties.Name -contains 'emailCollectorScript') { [string]`$config.paths.emailCollectorScript } else { Join-Path `$stateRoot 'email-outbound-collector.ps1' } `$launchLockPath = New-LaunchLock -StateRoot `$stateRoot -SessionId `$sessionId @@ -1459,10 +1513,22 @@ try { } catch { } - Start-CollectorScriptIfNeeded -ScriptPath `$collectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId - Start-CollectorScriptIfNeeded -ScriptPath `$endpointCollectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId + if (`$browserCollectorMode -ieq 'rust_primary') { + Start-RustCollectorIfNeeded -ExePath `$telemetryExe -Subcommand 'browser-domains-collector' -ConfigPath `$ConfigPath -SessionId `$sessionId + } else { + Start-CollectorScriptIfNeeded -ScriptPath `$collectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId + } + if (`$dlpEndpointMode -ieq 'rust_primary') { + Start-RustCollectorIfNeeded -ExePath `$telemetryExe -Subcommand 'dlp-endpoint-collector' -ConfigPath `$ConfigPath -SessionId `$sessionId + } else { + Start-CollectorScriptIfNeeded -ScriptPath `$endpointCollectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId + } if (`$fileOpsEnabled) { - Start-CollectorScriptIfNeeded -ScriptPath `$fileCollectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId + if (`$fileOpsMode -ieq 'rust_primary') { + Start-RustCollectorIfNeeded -ExePath `$telemetryExe -Subcommand 'file-operations-collector' -ConfigPath `$ConfigPath -SessionId `$sessionId + } else { + Start-CollectorScriptIfNeeded -ScriptPath `$fileCollectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId + } } if (`$emailEnabled -and (Test-Path -LiteralPath `$emailCollectorScript)) { Start-CollectorScriptIfNeeded -ScriptPath `$emailCollectorScript -ConfigPath `$ConfigPath -PowerShellExe `$powershellExe -SessionId `$sessionId @@ -2473,13 +2539,22 @@ function Register-ActivityWatchFile1CAutoUploadTask { } $uploadScript = if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryScript') { [string]$config.paths.file1cTelemetryScript } else { Join-Path $config.paths.stateRoot 'export-upload-file-1c-telemetry.ps1' } - if (-not (Test-Path -LiteralPath $uploadScript)) { - throw "Не найден скрипт file-1C telemetry upload: $uploadScript" + $uploadExeCandidates = New-Object System.Collections.Generic.List[string] + if ($config.paths.PSObject.Properties.Name -contains 'file1cTelemetryExecutable' -and -not [string]::IsNullOrWhiteSpace([string]$config.paths.file1cTelemetryExecutable)) { + $uploadExeCandidates.Add([string]$config.paths.file1cTelemetryExecutable) + } + $uploadExeCandidates.Add((Join-Path (Join-Path ([System.Environment]::GetFolderPath('ProgramFiles')) 'AWatch-rus\windows') 'aw-windows-telemetry.exe')) + $uploadExeCandidates.Add((Join-Path $config.paths.stateRoot 'aw-windows-telemetry.exe')) + $uploadExe = @($uploadExeCandidates | Where-Object { -not [string]::IsNullOrWhiteSpace($_) -and (Test-Path -LiteralPath $_) } | Select-Object -First 1) | Select-Object -First 1 + if ([string]::IsNullOrWhiteSpace($uploadExe) -and -not (Test-Path -LiteralPath $uploadScript)) { + throw "Не найден Rust binary или legacy script file-1C telemetry upload: $uploadExeCandidates / $uploadScript" } - $intervalHours = [Math]::Max(1, [int]$automation.intervalHours) - $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' - $taskCommand = "`"$powerShellExe`" -NoProfile -ExecutionPolicy Bypass -File `"$uploadScript`" -ConfigPath `"$ConfigPath`"" + $intervalMinutes = if ($automation.PSObject.Properties.Name -contains 'intervalMinutes') { + [Math]::Max(1, [int]$automation.intervalMinutes) + } else { + [Math]::Max(1, [int]$automation.intervalHours) * 60 + } $runnerUserId = $null if ($automation.PSObject.Properties.Name -contains 'runAsUser' -and -not [string]::IsNullOrWhiteSpace([string]$automation.runAsUser)) { $runnerUserId = [string]$automation.runAsUser @@ -2495,16 +2570,23 @@ function Register-ActivityWatchFile1CAutoUploadTask { $runnerUserId = @($config.userTasks | ForEach-Object { [string]$_.userId } | Select-Object -First 1) | Select-Object -First 1 } - Remove-ActivityWatchScheduledTask -TaskName $taskName - if (-not [string]::IsNullOrWhiteSpace($runnerUserId)) { - & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO $intervalHours /ST 00:00 /RU $runnerUserId /RL HIGHEST /F | Out-Null + if (-not [string]::IsNullOrWhiteSpace($uploadExe)) { + $action = New-ScheduledTaskAction -Execute $uploadExe -Argument "file1c-upload --config-path `"$ConfigPath`"" } else { - & schtasks.exe /Create /TN $taskName /TR $taskCommand /SC HOURLY /MO $intervalHours /ST 00:00 /RU SYSTEM /RL HIGHEST /F | Out-Null + $powerShellExe = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $action = New-ScheduledTaskAction -Execute $powerShellExe -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$uploadScript`" -ConfigPath `"$ConfigPath`"" } - if ($LASTEXITCODE -ne 0) { - throw "Не удалось создать scheduled task $taskName через schtasks.exe" + $trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).Date) -RepetitionInterval (New-TimeSpan -Minutes $intervalMinutes) -RepetitionDuration (New-TimeSpan -Days 3650) + if (-not [string]::IsNullOrWhiteSpace($runnerUserId)) { + $principal = New-ScheduledTaskPrincipal -UserId $runnerUserId -LogonType Interactive -RunLevel Highest } + else { + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + } + $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) + + Register-ScheduledTask -TaskName $taskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null } function Set-ActivityWatchAcl { diff --git a/windows/deploy-domain-users.ps1 b/windows/deploy-domain-users.ps1 index 4370b43..76f26a9 100755 --- a/windows/deploy-domain-users.ps1 +++ b/windows/deploy-domain-users.ps1 @@ -47,7 +47,9 @@ param( [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', [bool]$File1CAutoUploadEnabled = $true, [int]$File1CAutoUploadIntervalHours = 6, + [int]$File1CAutoUploadIntervalMinutes = 15, [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CAutoUploadRunAsUser, [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', [string]$File1CRegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx', @@ -157,7 +159,9 @@ $config = New-ActivityWatchDeploymentConfig ` -HayabusaAutoUploadTaskName $HayabusaAutoUploadTaskName ` -File1CAutoUploadEnabled $File1CAutoUploadEnabled ` -File1CAutoUploadIntervalHours $File1CAutoUploadIntervalHours ` + -File1CAutoUploadIntervalMinutes $File1CAutoUploadIntervalMinutes ` -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` + -File1CAutoUploadRunAsUser $File1CAutoUploadRunAsUser ` -File1CTargetHost $File1CTargetHost ` -File1CTargetUser $File1CTargetUser ` -File1CRegistryWorkbookPath $File1CRegistryWorkbookPath ` diff --git a/windows/deploy-ensemble.ps1 b/windows/deploy-ensemble.ps1 index 4fc9dfd..888ecbc 100644 --- a/windows/deploy-ensemble.ps1 +++ b/windows/deploy-ensemble.ps1 @@ -48,7 +48,9 @@ param( [string]$HayabusaAutoUploadTaskName = 'ActivityWatch Hayabusa Upload', [bool]$File1CAutoUploadEnabled = $true, [int]$File1CAutoUploadIntervalHours = 6, + [int]$File1CAutoUploadIntervalMinutes = 15, [string]$File1CAutoUploadTaskName = 'ActivityWatch File1C Upload', + [string]$File1CAutoUploadRunAsUser, [string]$File1CTargetHost, [string]$File1CTargetUser = 'igor', [string]$File1CRegistryWorkbookPath = 'E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx', @@ -118,7 +120,9 @@ if (-not (Test-Path -LiteralPath $deployScript)) { -HayabusaAutoUploadTaskName $HayabusaAutoUploadTaskName ` -File1CAutoUploadEnabled $File1CAutoUploadEnabled ` -File1CAutoUploadIntervalHours $File1CAutoUploadIntervalHours ` + -File1CAutoUploadIntervalMinutes $File1CAutoUploadIntervalMinutes ` -File1CAutoUploadTaskName $File1CAutoUploadTaskName ` + -File1CAutoUploadRunAsUser $File1CAutoUploadRunAsUser ` -File1CTargetHost $File1CTargetHost ` -File1CTargetUser $File1CTargetUser ` -File1CRegistryWorkbookPath $File1CRegistryWorkbookPath ` diff --git a/windows/hardening-recovery.ps1 b/windows/hardening-recovery.ps1 index fb4b6ef..b810d08 100755 --- a/windows/hardening-recovery.ps1 +++ b/windows/hardening-recovery.ps1 @@ -111,7 +111,9 @@ $effectiveHayabusaAutoUploadMode = if ($existingConfig -and $existingConfig.PSOb $effectiveHayabusaAutoUploadTaskName = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'hayabusaAutomation' -and $existingConfig.forensics.hayabusaAutomation.PSObject.Properties.Name -contains 'taskName') { [string]$existingConfig.forensics.hayabusaAutomation.taskName } else { 'ActivityWatch Hayabusa Upload' } $effectiveFile1CAutoUploadEnabled = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'enabled') { [bool]$existingConfig.analytics.file1cAutomation.enabled } else { $true } $effectiveFile1CAutoUploadIntervalHours = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'intervalHours') { [int]$existingConfig.analytics.file1cAutomation.intervalHours } else { 6 } +$effectiveFile1CAutoUploadIntervalMinutes = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'intervalMinutes') { [int]$existingConfig.analytics.file1cAutomation.intervalMinutes } else { [Math]::Max(1, $effectiveFile1CAutoUploadIntervalHours) * 60 } $effectiveFile1CAutoUploadTaskName = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'taskName') { [string]$existingConfig.analytics.file1cAutomation.taskName } else { 'ActivityWatch File1C Upload' } +$effectiveFile1CAutoUploadRunAsUser = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'runAsUser') { [string]$existingConfig.analytics.file1cAutomation.runAsUser } else { '' } $effectiveFile1CTargetHost = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetHost') { [string]$existingConfig.analytics.file1cAutomation.targetHost } else { '' } $effectiveFile1CTargetUser = if ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'analytics' -and $existingConfig.analytics.PSObject.Properties.Name -contains 'file1cAutomation' -and $existingConfig.analytics.file1cAutomation.PSObject.Properties.Name -contains 'targetUser') { [string]$existingConfig.analytics.file1cAutomation.targetUser } else { 'igor' } @@ -220,7 +222,9 @@ $config = New-ActivityWatchDeploymentConfig ` -HayabusaAutoUploadTaskName $effectiveHayabusaAutoUploadTaskName ` -File1CAutoUploadEnabled $effectiveFile1CAutoUploadEnabled ` -File1CAutoUploadIntervalHours $effectiveFile1CAutoUploadIntervalHours ` + -File1CAutoUploadIntervalMinutes $effectiveFile1CAutoUploadIntervalMinutes ` -File1CAutoUploadTaskName $effectiveFile1CAutoUploadTaskName ` + -File1CAutoUploadRunAsUser $effectiveFile1CAutoUploadRunAsUser ` -File1CTargetHost $effectiveFile1CTargetHost ` -File1CTargetUser $effectiveFile1CTargetUser ` -LaunchScriptPath $effectiveLaunchScript ` diff --git a/windows/install-collector-guard-service.ps1 b/windows/install-collector-guard-service.ps1 index a6e270c..b6eb8e7 100644 --- a/windows/install-collector-guard-service.ps1 +++ b/windows/install-collector-guard-service.ps1 @@ -22,6 +22,7 @@ function Assert-Admin { Assert-Admin $guardScriptPath = Join-Path $PSScriptRoot 'aw-collector-guard.ps1' +$rustTelemetryPath = Join-Path $PSScriptRoot 'aw-windows-telemetry.exe' $serviceSourcePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.cs' $serviceExePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.exe' if (-not (Test-Path -LiteralPath $guardScriptPath)) { @@ -64,7 +65,13 @@ if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $serviceExePath)) { $logsRoot = Join-Path (Split-Path -Path $ConfigPath -Parent) 'logs' $serviceLogPath = Join-Path $logsRoot 'collector-guard-service.log' -$binPath = "`"$serviceExePath`" --service-name `"$ServiceName`" --script `"$guardScriptPath`" --config `"$ConfigPath`" --mode $Mode --loop $LoopSeconds --log `"$serviceLogPath`"" +if (Test-Path -LiteralPath $rustTelemetryPath) { + $rustArgs = "collector-guard --config-path `"$ConfigPath`" --mode $Mode --loop-seconds $LoopSeconds" + $binPath = "`"$serviceExePath`" --service-name `"$ServiceName`" --exec `"$rustTelemetryPath`" --args `"$rustArgs`" --log `"$serviceLogPath`"" +} +else { + $binPath = "`"$serviceExePath`" --service-name `"$ServiceName`" --script `"$guardScriptPath`" --config `"$ConfigPath`" --mode $Mode --loop $LoopSeconds --log `"$serviceLogPath`"" +} New-Service -Name $ServiceName -BinaryPathName $binPath -DisplayName 'AWatch-rus Collector Guard' -StartupType Automatic | Out-Null sc.exe description $ServiceName "Session-aware ActivityWatch collector guard for AWatch-rus" | Out-Null @@ -86,3 +93,4 @@ sc.exe start $ServiceName | Out-Null Write-Output "Collector guard service installed: $ServiceName" Write-Output "Mode: $Mode" Write-Output "Config: $ConfigPath" +Write-Output "Runtime: $(if (Test-Path -LiteralPath $rustTelemetryPath) { 'rust' } else { 'powershell' })"
Нет данныхUNKNOWNнет данных нет данных 0 Дождаться расчета подразделений.
Нет данныхUNKNOWNUNKNOWNUNKNOWNUNKNOWNнет данныхнет данныхнет данныхнет данных 0 0 -
Нет данныхUNKNOWNUNKNOWNUNKNOWNнет данныхнет данныхнет данных проверить 0 · расследования 0 0 0/100
- История не накопленаUNKNOWNнет данных Нужно дождаться daily history.
- Нет кандидатов -OKв норме NEW очередь проверки пуста Действий не требуется.