diff --git a/adk-rust/.gitignore b/adk-rust/.gitignore new file mode 100644 index 0000000..55509a8 --- /dev/null +++ b/adk-rust/.gitignore @@ -0,0 +1,4 @@ +target/ +.env +.env.* +!.env.example diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock new file mode 100644 index 0000000..4fca3cf --- /dev/null +++ b/adk-rust/Cargo.lock @@ -0,0 +1,3007 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adk-core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "257e0d5a39ef6130bc5fe7f0a8d9c1b28a3fbb690f28854525f9493ebf3df4f5" +dependencies = [ + "async-trait", + "chrono", + "futures", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "adk-rust" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ce7ff944c94f27123ecfcd137ce4a8601a81409af9c7979b770fa0d1ffc43c" +dependencies = [ + "adk-core", + "anyhow", + "async-trait", + "futures", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aw-browser-smoke" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "tempfile", +] + +[[package]] +name = "aw-contour-smoke" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde_json", + "tempfile", + "url", +] + +[[package]] +name = "aw-db-health" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "rusqlite", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "aw-db-maintenance" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "rusqlite", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "aw-health-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "reqwest", + "serde_json", + "tempfile", +] + +[[package]] +name = "aw-prune-local-state" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "aw-rus-healthd" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-core", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "aw-slo-monitor" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "tempfile", + "urlencoding", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "check-aw-data" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde_json", + "tempfile", +] + +[[package]] +name = "check-aw-full" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde_json", + "tempfile", +] + +[[package]] +name = "check-install-kit-vs-repo" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "sha2", + "tempfile", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "detmir-auto" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-core", + "detmir-state", + "fs2", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "detmir-aw-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "detmir-core", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "detmir-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-aw-client", + "detmir-core", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "detmir-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "serde", +] + +[[package]] +name = "detmir-dlp" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", +] + +[[package]] +name = "detmir-heal-safe" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "detmir-core", + "serde", + "serde_json", +] + +[[package]] +name = "detmir-state" +version = "0.1.0" +dependencies = [ + "anyhow", + "detmir-core", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "detmir-status" +version = "0.1.0" +dependencies = [ + "adk-rust", + "anyhow", + "clap", + "detmir-state", + "serde", + "serde_json", +] + +[[package]] +name = "diag-and-manual-restart" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "tempfile", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dlp-admin-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde_json", + "urlencoding", +] + +[[package]] +name = "dlp-aggregator" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-aw-client", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "urlencoding", +] + +[[package]] +name = "dlp-case-management" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "rusqlite", + "serde_json", + "sha2", + "tiny_http", + "urlencoding", +] + +[[package]] +name = "dlp-cef-exporter" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "dlp-compliance" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "urlencoding", +] + +[[package]] +name = "dlp-content-analyzer" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "regex", + "serde", + "serde_json", +] + +[[package]] +name = "dlp-health-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-core", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "dlp-influx-exporter" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "urlencoding", +] + +[[package]] +name = "dlp-policy-engine" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "rusqlite", + "serde_json", + "sha2", + "tiny_http", + "urlencoding", +] + +[[package]] +name = "dlp-syslog-forwarder" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "dlp-webhook-sender" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "extract-ioc-from-sigma" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "regex", + "serde", + "serde_json", + "serde_yaml", + "tempfile", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "hayabusa-tools" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "fs2", + "regex", + "reqwest", + "serde", + "serde_json", + "tempfile", + "urlencoding", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "merge-aw-server-dbs" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "rusqlite", + "serde_json", + "tempfile", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prod-backup-restore" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "prod-rollout" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "quality-gate" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "tempfile", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rdp-worktime-report" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "rebuild-install-kit" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "flate2", + "sha2", + "tar", + "tempfile", + "zip", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tsj-guardian-status" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-state", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "tsj-guardian-watchdog" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "detmir-core", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "validate-install-kit" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "flate2", + "serde", + "serde_json", + "sha2", + "tar", + "tempfile", + "zip", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "verify-innosetup-installer" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "worktime-api" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "tiny_http", + "url", + "urlencoding", +] + +[[package]] +name = "worktime-autoheal" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "worktime-influx-exporter" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "urlencoding", +] + +[[package]] +name = "worktime-prewarm" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", + "urlencoding", +] + +[[package]] +name = "worktime-ui-bridge" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml new file mode 100644 index 0000000..507acb1 --- /dev/null +++ b/adk-rust/Cargo.toml @@ -0,0 +1,79 @@ +[workspace] +resolver = "3" +members = [ + "crates/detmir-auto", + "crates/detmir-aw-client", + "crates/aw-db-health", + "crates/aw-db-maintenance", + "crates/aw-health-check", + "crates/aw-prune-local-state", + "crates/check-aw-data", + "crates/check-aw-full", + "crates/check-install-kit-vs-repo", + "crates/validate-install-kit", + "crates/verify-innosetup-installer", + "crates/rebuild-install-kit", + "crates/quality-gate", + "crates/extract-ioc-from-sigma", + "crates/merge-aw-server-dbs", + "crates/prod-backup-restore", + "crates/prod-rollout", + "crates/rdp-worktime-report", + "crates/aw-contour-smoke", + "crates/aw-browser-smoke", + "crates/diag-and-manual-restart", + "crates/aw-slo-monitor", + "crates/aw-rus-healthd", + "crates/detmir-check", + "crates/detmir-core", + "crates/dlp-health-check", + "crates/dlp-content-analyzer", + "crates/dlp-admin-cli", + "crates/dlp-policy-engine", + "crates/dlp-case-management", + "crates/dlp-compliance", + "crates/hayabusa-tools", + "crates/dlp-cef-exporter", + "crates/dlp-aggregator", + "crates/dlp-influx-exporter", + "crates/worktime-influx-exporter", + "crates/worktime-autoheal", + "crates/worktime-api", + "crates/worktime-prewarm", + "crates/worktime-ui-bridge", + "crates/dlp-syslog-forwarder", + "crates/dlp-webhook-sender", + "crates/detmir-dlp", + "crates/detmir-heal-safe", + "crates/detmir-status", + "crates/detmir-state", + "crates/tsj-guardian-status", + "crates/tsj-guardian-watchdog", +] + +[workspace.package] +edition = "2024" +rust-version = "1.85" +license = "Apache-2.0" +publish = false + +[workspace.dependencies] +adk-rust = { version = "0.9.1", default-features = false } +anyhow = "1" +chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } +clap = { version = "4", features = ["derive", "env"] } +detmir-aw-client = { path = "crates/detmir-aw-client" } +detmir-core = { path = "crates/detmir-core" } +detmir-state = { path = "crates/detmir-state" } +fs2 = "0.4" +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +regex = "1" +rusqlite = "0.32" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +sha2 = "0.10" +tempfile = "3" +tiny_http = "0.12" +url = "2" +urlencoding = "2" diff --git a/adk-rust/README.md b/adk-rust/README.md new file mode 100644 index 0000000..9075563 --- /dev/null +++ b/adk-rust/README.md @@ -0,0 +1,89 @@ +# DetMir ADK-Rust Workspace + +This directory is the Rust workspace for replacing operational Python and shell +scripts with durable standalone Rust modules. + +## Layout + +- `Cargo.toml` - workspace manifest and shared dependency versions. +- `Cargo.lock` - pinned dependency graph for reproducible builds. +- `crates/` - one binary or library module per operational function. +- `target/` - local build output, ignored by git. + +## Current Modules + +- `detmir-auto` - no-heal autonomous orchestration shadow binary for running + `detmir-check`, `detmir-dlp`, state/report writes, latest symlink updates, and + retention cleanup. +- `detmir-core` - shared status levels, exit codes, and UTC timestamp helpers. +- `detmir-state` - DetMir autonomous state models, normalization, and atomic JSON writes. +- `detmir-aw-client` - small blocking ActivityWatch HTTP client and event timestamp helpers. +- `detmir-check` - read-only DetMir contour check replacement for the legacy Python command. +- `detmir-dlp` - SSH wrapper replacement for remote DLP health JSON collection. +- `dlp-health-check` - AW server DLP health check replacement. +- `aw-db-maintenance` - guarded weekly SQLite maintenance for old allowlisted + process-level session events, with backup-before-delete. +- `check-aw-full` - read-only local AW/RDP full check replacement for the + legacy shell helper. +- `dlp-aggregator` - AW server DLP warehouse aggregator replacement. +- `dlp-influx-exporter` - AW server DLP InfluxDB line protocol exporter replacement. +- `worktime-autoheal` - AW server worktime autoheal and backfill replacement. +- `worktime-influx-exporter` - AW server worktime InfluxDB line protocol exporter replacement. +- `worktime-prewarm` - AW server worktime report cache prewarm replacement. +- `worktime-ui-bridge` - AW server worktime sessions to AFK/window bridge replacement. +- `dlp-syslog-forwarder` - AW server DLP syslog integration replacement. +- `dlp-webhook-sender` - AW server DLP webhook integration replacement. +- `dlp-cef-exporter` - AW server DLP CEF/syslog exporter replacement. +- `extract-ioc-from-sigma` - offline Sigma/Hayabusa IOC export replacement + used by the DLP IOC preload wrapper. +- `merge-aw-server-dbs` - ActivityWatch SQLite DB merge replacement used by + legacy root DB recovery/deploy tooling. +- `prod-backup-restore` - safe planner/checker for the destructive production + backup-restore flow; apply remains explicit legacy-only at this stage. +- `prod-rollout` - safe planner/orchestrator for production AW server/Windows + rollout; normal script runs are plan-only and real rollout requires + explicit `--apply`. +- `rdp-worktime-report` - local RDP worktime CSV/JSON report helper + replacement for the legacy shell/Python script. +- `aw-contour-smoke` - Rust replacement for the Proxmox-side DetMir contour + smoke checks, used through a Rust-first project wrapper. +- `diag-and-manual-restart` - Rust replacement for the AW/DLP diagnostic and + explicit manual restart helper, with conservative no-restart healthy path. +- `aw-browser-smoke` - Rust launcher for the browser smoke test; Playwright + remains the execution engine and the legacy Node script remains fallback. +- `detmir-status` - read-only DetMir state normalizer with text, JSON, and ADK + `Content` output. Also builds `detmir-adk-status` as a compatibility binary. + +## Migration Runbook + +Use `RUNBOOK.md` as the operational plan for replacing Python and shell modules +with Rust. It defines migration phases, safety gates, rollback rules, and the +order in which DetMir modules should be moved. + +## Commands + +```bash +cd adk-rust +cargo fmt --all +cargo check --workspace +cargo build --release --workspace +``` + +Run the current status module: + +```bash +cd adk-rust +cargo run -p detmir-status -- --json +cargo run -p detmir-status -- --adk-json +cargo run -p detmir-status -- status --json +``` + +## Migration Rules + +- New replacements go under `crates/` and are added to workspace `members`. +- Keep modules standalone: config comes from files, environment variables, or + explicit CLI flags; no laptop-only assumptions. +- Default to read-only behavior first; add mutation/recovery paths only with + tests and explicit operator-safe failure modes. +- Do not embed secrets in binaries, source files, or examples. +- Every module should expose machine-readable JSON output where practical. diff --git a/adk-rust/RUNBOOK.md b/adk-rust/RUNBOOK.md new file mode 100644 index 0000000..362af88 --- /dev/null +++ b/adk-rust/RUNBOOK.md @@ -0,0 +1,1510 @@ +# Runbook: перевод DetMir на Rust / ADK-Rust + +Дата фиксации: `2026-06-01` + +Цель: постепенно заменить хрупкие Python/shell operational scripts на +самодостаточные Rust-бинарники, не ломая текущий production-контур DetMir. + +Этот runbook является рабочим планом миграции. Если фактический runtime +расходится с этим документом, сначала фиксируется baseline, затем обновляется +runbook. + +Текущий resume snapshot проекта хранится в +`.ai/runtime/detmir-current-session.md`. Перед продолжением миграции после +перерыва или context reset сначала читать его, затем `RUNBOOK.md` и +`/home/igor/.codex/skills/detmir-rust-migration/references/current-state.md`. + +## 1. Целевое состояние + +В результате миграции DetMir должен иметь: + +- единый Rust workspace в `adk-rust/`; +- один standalone-бинарник или crate на каждую операционную функцию; +- стабильные machine-readable JSON-контракты вместо парсинга human text; +- одинаковые exit codes для старых и новых команд; +- systemd units/timers, не зависящие от ноутбука, venv, pip и локальных путей; +- безопасный auto-heal с dry-run, lock, cooldown, allowlist и rollback; +- ADK-compatible envelopes там, где вывод передается агентам/боту/LLM. + +Rust не должен использоваться как самоцель. Если компонент надежнее оставить в +Ansible, PowerShell или Playwright, он остается там до появления практической +причины для переноса. + +## 2. Текущая отправная точка + +Уже создано: + +- `adk-rust/Cargo.toml` - workspace; +- `adk-rust/crates/detmir-status` - read-only status module with text, JSON and ADK JSON output; +- `/usr/local/bin/detmir-status` на Proxmox, проверен против + `/var/lib/detmir-ai/latest-state.json`; +- `/usr/local/bin/detmir-adk-status` на Proxmox, проверен против + `/var/lib/detmir-ai/latest-state.json` как compatibility binary. + +Ключевые текущие legacy-компоненты: + +| Компонент | Текущая реализация | Риск переноса | +|---|---|---| +| `detmir-status` | Rust binary + thin compatibility wrapper | выполнено | +| `detmir-check` | Rust binary + thin compatibility wrapper | выполнено | +| `detmir-dlp` | Rust SSH wrapper + thin compatibility wrapper | выполнено | +| `detmir-auto` | Rust production via systemd drop-in + legacy script retained | switched | +| `detmir-heal-safe` | Rust binary deployed + legacy script retained | switched for Rust auto | +| `tsj_guardian_watchdog.sh` | Rust service via systemd drop-in + legacy shell retained | switched | +| Telegram `/status`/decision backend | Rust helper + permanent Python bot runtime | backend only | +| `aw-rus-healthd.py` | Rust production via systemd drop-in + legacy Python retained | switched | +| `scripts/dlp-health-check.py` | Rust production binary + legacy Python backup retained | switched | +| DLP aggregator | Rust production via systemd drop-in + legacy Python retained | switched | +| AW DLP Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched | +| AW worktime Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched | +| AW worktime prewarm | Rust production via systemd drop-in + legacy shell retained | switched | +| AW worktime API | Rust production via systemd drop-in + legacy Python retained | switched | +| DLP syslog forwarder | Rust production via systemd drop-in + legacy Python retained | switched | +| DLP webhook sender | Rust production via systemd drop-in + legacy Python retained | switched | +| DLP CEF exporter | Rust production via systemd drop-in + legacy Python retained | switched | +| `tsj_guardian_bot.py` | Permanent Python Telegram runtime | не переносить | +| AW remaining worktime modules | mostly Rust production; inspect leftovers before next item | средний | +| deploy/install scripts | shell/Ansible/PowerShell | высокий, переносить последними | + +## 3. Архитектура workspace + +Целевая структура: + +```text +adk-rust/ + Cargo.toml + Cargo.lock + RUNBOOK.md + crates/ + detmir-core/ + detmir-aw-client/ + detmir-systemd/ + detmir-state/ + detmir-status/ + detmir-check/ + detmir-dlp/ + detmir-auto/ + detmir-heal-safe/ + tsj-guardian-status/ + tsj-guardian-watchdog/ +``` + +Назначение shared crates: + +| Crate | Назначение | +|---|---| +| `detmir-core` | ошибки, CLI output, timeouts, retry, exit codes, config loading | +| `detmir-aw-client` | ActivityWatch HTTP API, buckets, events, timestamps | +| `detmir-systemd` | безопасный wrapper вокруг `systemctl`, allowlist, dry-run | +| `detmir-state` | чтение/запись `/var/lib/detmir-ai`, atomic writes, retention | +| `detmir-report` | Markdown/JSON reports, ADK envelopes, report bundle assembly | + +Бинарные модули должны быть маленькими и собираться из shared crates. Логику не +дублировать между `check`, `status`, `auto` и ботом. + +## 4. Общие контракты для каждого Rust-модуля + +Каждый новый модуль обязан иметь: + +- `--json` для машинного вывода; +- `--pretty` или обычный text output для человека, если команда операторская; +- `--config ` либо documented env vars; +- `--dry-run` для любых действий, которые меняют состояние; +- `--timeout` там, где есть сеть/SSH/HTTP; +- `--no-color` если вывод может попадать в systemd/Telegram/report; +- стабильные exit codes: + - `0` - OK; + - `1` - usage/config/runtime error; + - `2` - проверка выполнена, но состояние WARN/FAIL; + - `3` - action запрещен safety policy; +- structured logs через `tracing` для daemon/action-команд; +- unit tests на чистую логику; +- fixture tests на реальные JSON samples; +- README с примером локального и серверного запуска. + +## 5. Safety policy + +Нельзя сразу заменять управляющие скрипты без shadow-mode. + +Обязательные правила: + +- сначала read-only parity, потом mutation; +- старый и новый модуль должны некоторое время работать параллельно; +- новый модуль не получает право писать state, рестартить сервисы или удалять + файлы до прохождения acceptance gates; +- все state writes только через atomic temp file + rename; +- heal-команды только по allowlist units; +- no implicit sudo: если нужен `sudo`, он должен быть явно виден в deploy/unit; +- lock file обязателен для `auto` и `heal`; +- cooldown обязателен для restart/start actions; +- каждый risky action пишет audit entry; +- rollback должен быть одной командой systemd/service symlink switch. + +## 6. Фазы миграции + +### Phase 0. Baseline и фиксация контрактов + +Цель: перед переносом зафиксировать, что именно считается корректным поведением. + +Действия: + +1. Снять текущие outputs legacy-команд: + - `detmir-status`; + - `detmir-check --json`; + - `detmir-dlp`; + - `detmir-auto` на зеленом контуре; + - `detmir-heal-safe` в dry-safe сценарии или на mock host. +2. Сохранить sanitized fixtures в `adk-rust/fixtures/`. +3. Зафиксировать JSON schemas: + - status summary; + - DetMir check result; + - DLP health result; + - auto run state; + - heal action log. +4. Зафиксировать exit code matrix старых команд. +5. Зафиксировать runtime paths: + - `/var/lib/detmir-ai`; + - `/usr/local/bin`; + - relevant systemd units/timers. + +Гейт готовности: + +```bash +cd adk-rust +cargo fmt --all -- --check +cargo check --workspace +``` + +Результат фазы: миграция не начинается вслепую; есть baseline для сравнения. + +### Phase 1. Rust foundation + +Цель: создать shared crates, чтобы не плодить разные реализации HTTP, времени, +ошибок, JSON и systemd. + +Модули: + +- `detmir-core`; +- `detmir-state`; +- `detmir-aw-client`; +- `detmir-systemd`. + +Минимальная функциональность: + +- RFC3339/UTC timestamp parsing; +- HTTP JSON client with timeout/retry; +- TCP check; +- atomic JSON write; +- retention cleanup; +- exit-code helper; +- common `StatusLevel`: `OK`, `WARN`, `FAIL`; +- bucket mode enum: `fresh`, `inactive_ok`, `event_driven`; +- systemd read-only checks; +- systemd action allowlist, но без включения mutation by default. + +Гейт: + +```bash +cd adk-rust +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +### Phase 2. Read-only operator modules + +Цель: заменить самые безопасные команды, не меняющие состояние. + +Порядок: + +1. `detmir-status` + - объединить с совместимым `detmir-adk-status`; + - читать `/var/lib/detmir-ai/latest-state.json`; + - выдавать text, JSON, ADK JSON; + - заменить shell wrapper после parity. +2. `detmir-check` + - перенести Python checks в Rust; + - сохранить семантику `FRESH`, `INACTIVE`, `EVENT-DRIVEN`, `STALE`, `DEAD`; + - сохранить `summary.bucket_ok/stale/dead/service_failures/service_warnings`; + - не менять thresholds без отдельного решения. +3. `detmir-dlp` + - сначала переписать SSH wrapper; + - затем отдельно переносить `dlp-health-check.py` на AW server. + +Shadow-mode для `detmir-check`: + +```bash +detmir-check --json > /tmp/detmir-check.old.json +adk-rust/target/release/detmir-check --json > /tmp/detmir-check.new.json +jq -S '.summary' /tmp/detmir-check.old.json > /tmp/old.summary.json +jq -S '.summary' /tmp/detmir-check.new.json > /tmp/new.summary.json +diff -u /tmp/old.summary.json /tmp/new.summary.json +``` + +Гейт: + +- old/new agree on `ok`; +- old/new agree on summary counters; +- new module returns same exit code class; +- no network side effects; +- server run succeeds from systemd-like environment. + +### Phase 3. State/report orchestration + +Цель: заменить `detmir-auto` без немедленной замены heal. + +Первый Rust `detmir-auto` должен: + +- брать lock; +- создавать run directory; +- запускать `detmir-check` и `detmir-dlp`; +- писать `summary-before.json`; +- писать `state-*.json`; +- обновлять symlinks: + - `latest-run`; + - `latest-state.json`; + - `latest-report.md`; +- выполнять retention cleanup; +- уметь `--no-heal`; +- по умолчанию на первом этапе не выполнять heal, а только писать + `would_heal=true`. + +Pollinations/LLM report: + +- не должен быть hard dependency для статуса; +- failure LLM report не должен ломать state update; +- raw JSON summary всегда должен сохраняться даже при ошибке report generation. + +Гейт: + +- Rust `detmir-auto --no-heal` дает такой же `latest-state.json` по смыслу; +- symlink updates atomic enough for readers; +- timer можно прогнать вручную без изменения heal behavior; +- `detmir-status` видит новое state без изменений. + +Текущее состояние: + +- `detmir-auto-rust` установлен на Proxmox как shadow binary + `/usr/local/bin/detmir-auto-rust`; +- `detmir-auto-rust` умеет вызывать Rust heal через `--enable-heal` или + `DETMIR_AUTO_HEAL=1`, но shadow unit явно запускается с `--no-heal`; +- production `detmir-auto.service` переключен на `/usr/local/bin/detmir-auto-rust` + через drop-in `/etc/systemd/system/detmir-auto.service.d/20-rust-switch.conf`; +- legacy `/usr/local/bin/detmir-auto` сохранен для rollback; +- отдельный `detmir-auto-rust-shadow.timer` отключен после успешного + production timer observation; unit-файлы и shadow state сохранены; +- shadow пишет только в `/var/lib/detmir-ai/shadow/detmir-auto-rust`; +- `detmir-auto-rust-shadow.service` использует `SuccessExitStatus=2`, чтобы + найденный shadow FAIL фиксировался в JSON, но не загрязнял + `systemctl --failed`; +- перед стартом shadow service проверяет, что production `detmir-auto.service` + не активен; +- последняя systemd shadow-проверка после retry в AW client: + `severity=OK`, `check_rc=0`, `dlp_rc=0`, buckets `8/0/0`, DLP `22/0/0`. +- production Rust start проверен: `ExecStart=/usr/local/bin/detmir-auto-rust + --command-timeout-seconds 180`, latest-state `OK`, `systemctl --failed` + пустой. +- scheduled production timer cycle проверен после switch: process status `0`, + latest-state `OK`, report generated, `systemctl --failed` пустой. + +### Phase 4. Safe heal + +Цель: перенести `detmir-heal-safe`, но только после read-only parity. + +Обязательные guardrails: + +- `--dry-run` default на первых deploy; +- allowlist units: + - `activitywatch-server.service`; + - `aw-worktime-api.service`; + - `aw-worktime-ui-bridge.timer`; + - `activitywatch-dlp-aggregator.timer` только если unit существует; +- запрет wildcard restart; +- `systemctl reset-failed` только для allowlist; +- cooldown между restart попытками; +- audit log в run directory; +- hard timeout на SSH/systemctl; +- never touch Windows/RDP recovery from this module. + +Гейт: + +- dry-run показывает exact actions; +- на зеленом контуре не рестартит ничего; +- на mock/failing unit рестартит только allowlisted unit; +- после heal всегда запускается повторный check; +- exit code не маскирует FAIL. + +Текущее состояние: + +- создан crate `detmir-heal-safe`; +- binary развернут на Proxmox как `/usr/local/bin/detmir-heal-safe-rust`; +- production `/usr/local/bin/detmir-heal-safe` пока не заменен; +- production Rust auto использует `/usr/local/bin/detmir-heal-safe-rust` через + `DETMIR_HEAL_BIN`; +- default mode: dry-run, mutation только через `--apply`; +- optional timer start выключен по умолчанию, включается только + `--start-optional`; +- green dry-run проверен: restart/start не планируются; +- green apply проверен: выполнены только `reset-failed` и DLP health snapshot, + контур остался `OK`, `systemctl --failed` пустой. +- red/mock heal test проверен на Proxmox в isolated state-dir: + `summary-before=FAIL`, heal вызван, check/dlp повторены, final state `OK`, + process rc `0`. + +### Phase 5. Watchdog и lightweight services + +Цель: заменить простые shell watchdogs и wrappers. + +Порядок: + +1. `tsj-guardian-watchdog.sh` -> `tsj-guardian-watchdog`; +2. lightweight service wrappers; +3. bounded cleanup/retention jobs. + +Особое внимание: + +- bot heartbeat file parsing; +- duplicate `gost` instance dedupe; +- service restart only on stale/missing heartbeat; +- no killing unrelated processes. + +Гейт: + +- dry-run mode показывает, какой PID был бы убит; +- process matching покрыт тестами; +- systemd timer/service rollback сохранен. + +Текущее состояние: + +- создан crate `tsj-guardian-watchdog`; +- binary развернут на Proxmox как + `/usr/local/bin/tsj-guardian-watchdog-rust`; +- production `tsj-guardian-watchdog.service` переключен на Rust через drop-in + `/etc/systemd/system/tsj-guardian-watchdog.service.d/20-rust-switch.conf`; +- legacy `/opt/infra-admin/tsj-bot/tsj_guardian_watchdog.sh` сохранен; +- dry-run выявил bug legacy-подхода: `pgrep -f` не матчился как literal из-за + `+` в `gost` pattern; +- Rust версия использует literal process scan через `ps`; +- one-shot apply удалил лишний duplicate `gost` PID и оставил systemd MainPID; +- service mode работает как: + +```bash +/usr/local/bin/tsj-guardian-watchdog-rust --apply --loop-forever --interval-seconds 60 +``` + +Rollback: + +```bash +sudo rm -f /etc/systemd/system/tsj-guardian-watchdog.service.d/20-rust-switch.conf +sudo systemctl daemon-reload +sudo systemctl restart tsj-guardian-watchdog.service +``` + +### Phase 6. AW server health and DLP modules + +Цель: перенести серверные Python health checks/exporters. + +Порядок: + +1. `aw-server/aw-rus-healthd.py` - выполнено, production через + `aw-rus-healthd.service.d/20-rust-switch.conf`; +2. `scripts/dlp-health-check.py` - выполнено, `/usr/local/bin/dlp-health-check` + заменён Rust-бинарником; +3. `scripts/aggregate_dlp_events.py` - выполнено, production через + `activitywatch-dlp-aggregator.service.d/20-rust-switch.conf`; +4. DLP integrations: + - syslog forwarder - выполнено, production через + `aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf`; + - webhook sender - выполнено, production через + `aw-dlp-webhook-sender.service.d/20-rust-switch.conf`; + - CEF exporter - выполнено, production через + `aw-dlp-cef-exporter.service.d/20-rust-switch.conf`; +5. `aw-server/aw-dlp-influx-exporter.py` - выполнено, production через + `aw-dlp-influx-exporter.service.d/20-rust-switch.conf`; +6. `aw-server/aw-worktime-influx-exporter.py` - выполнено, production через + `aw-worktime-influx-exporter.service.d/20-rust-switch.conf`; +7. `aw-server/aw-worktime-prewarm.sh` - выполнено, production через + `aw-worktime-prewarm.service.d/20-rust-switch.conf`; +8. `aw-server/aw-worktime-ui-bridge.py` - выполнено, production через + `aw-worktime-ui-bridge.service.d/20-rust-switch.conf`; +9. `aw-server/aw-worktime-autoheal.sh` - выполнено, production через + `aw-worktime-autoheal.service.d/20-rust-switch.conf`; +10. remaining worktime modules. + +Правила: + +- exporters должны сохранять Prometheus/Influx output format; +- API endpoints не менять без compatibility layer; +- для каждого exporter сначала golden output fixture; +- service unit меняется только после side-by-side run. + +Гейт: + +- old/new metrics names match; +- dashboard datasource health не ломается; +- unit tests старого поведения перенесены или сохранены; +- `systemctl --failed` clean после deploy. + +### Phase 7. Telegram bot backend helpers + +Цель: не переносить Telegram bot runtime на Rust. Python остается постоянным +production runtime для Telegram polling/sending, proxy, retries и side effects. +На Rust выносятся только backend helpers, status aggregation, decision/gating и +безопасные read-only/action contracts, которые бот вызывает как внешние команды. + +Порядок: + +1. `[done]` Бот продолжает жить на Python. +2. `[started]` Команды `/status`, `/detmir`, health summaries начинают читать Rust JSON. + Первым вынесен `tsj-guardian-status`: read-only helper для строки + `detmir_auto` из `/var/lib/detmir-ai/latest-state.json`; вторым шагом тот же + helper начал рендерить `aw_rus_slo` из `AW_RUS_SLO_SUMMARY_CMD`. +3. Recovery/actions вызывают Rust binaries с dry-run/audit. +4. `[decision]` Bot runtime на Rust не переносится; новые улучшения Telegram + делаются в Python runtime или через Rust backend helpers. + +Почему так: + +- Telegram runtime, proxy, retries и async edge cases уже стабильно покрыты + Python-кодом и тестами; +- перенос runtime даст мало пользы и высокий риск регрессий; +- максимальная польза от Rust сначала в backend-командах и safety contracts. + +Гейт: + +- `/status` показывает те же строки или лучше; +- smoke message проходит; +- watchdog активен; +- restart bot не теряет config/secrets; +- секреты не попадают в logs/tests. + +### Phase 8. Deploy/install scripts + +Цель: переносить последними, только после стабилизации runtime binaries. + +Что может остаться не на Rust: + +- Ansible playbooks; +- Windows PowerShell collector deployment; +- Playwright browser smoke; +- packaging scripts, если они надежны и редко исполняются. + +Что имеет смысл перенести: + +- validation CLI; +- install-kit consistency checker; +- local report generators; +- deterministic packaging helpers. + +Гейт: + +- install-kit output byte/content expected; +- rollback installer path documented; +- production deploy не зависит от laptop-only paths. + +## 7. Deployment workflow + +Сборка: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +cargo build --release --workspace +``` + +Установка одного бинарника на Proxmox: + +```bash +install -o root -g root -m 0755 \ + target/release/ \ + /usr/local/bin/ +``` + +Через Ansible: + +```bash +ansible proxmox -i ansible/inventory.ini -m copy -a \ + 'src=/mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust/target/release/ dest=/usr/local/bin/ owner=root group=root mode=0755 backup=yes' +``` + +Проверка на сервере: + +```bash + --version || true + --help + --json +systemctl --failed +``` + +## 8. Rollback workflow + +Для каждого заменяемого компонента должен быть rollback: + +1. Старый script остается на сервере как `.legacy` или в package backup. +2. systemd unit меняется через drop-in или controlled template. +3. Перед switch: + - сохранить `systemctl cat `; + - сохранить checksum старого binary/script; + - сохранить latest known good command. +4. Rollback command документируется в PR/run note. + +Текущий rollback для `detmir-auto` после Rust switch: + +```bash +sudo rm -f /etc/systemd/system/detmir-auto.service.d/20-rust-switch.conf +sudo systemctl daemon-reload +sudo systemctl restart detmir-auto.service +``` + +Перед switch сохранены: + +```bash +/var/lib/detmir-ai/switch-backups/detmir-auto.service.before-rust-20260531-192809.txt +/var/lib/detmir-ai/switch-backups/detmir-auto.sha256.before-rust-20260531-192809.txt +``` + +Пример: + +```bash +cp -a /usr/local/bin/detmir-check /usr/local/bin/detmir-check.legacy +install -o root -g root -m 0755 target/release/detmir-check /usr/local/bin/detmir-check +detmir-check --json +``` + +Если новый binary провален: + +```bash +mv /usr/local/bin/detmir-check.legacy /usr/local/bin/detmir-check +systemctl restart detmir-auto.timer || true +``` + +## 9. Acceptance checklist + +Перед заменой legacy-команды: + +- [ ] old/new CLI documented; +- [ ] old/new JSON compared on real fixture; +- [ ] exit codes match by class; +- [ ] unit tests pass; +- [ ] integration smoke pass on target host; +- [ ] no secrets in logs; +- [ ] systemd environment tested; +- [ ] rollback command known; +- [ ] docs updated; +- [ ] operator command example added. + +Перед включением mutation/recovery: + +- [ ] `--dry-run` verified; +- [ ] allowlist verified; +- [ ] lock verified; +- [ ] cooldown verified; +- [ ] audit log verified; +- [ ] green contour causes no action; +- [ ] failing contour causes only intended action; +- [ ] post-action check verified; +- [ ] rollback tested. + +## 10. Quality gates + +Локально: + +```bash +cd adk-rust +cargo fmt --all -- --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace +``` + +На Proxmox: + +```bash +detmir-status --json +detmir-check --json +detmir-dlp +detmir-auto --no-heal +systemctl --failed +``` + +На AW server: + +```bash +aw-rus-healthd --json +dlp-health-check --json +/usr/local/bin/aw-slo-monitor-rust --state-dir /var/lib/activitywatch/slo-rust-shadow --json +systemctl --failed +``` + +Для Telegram: + +```bash +systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg +``` + +## 11. Приоритет ближайших работ + +Немедленный порядок: + +1. `[done]` Создать `detmir-core`, `detmir-state`, `detmir-aw-client`. +2. `[done]` Объединить `detmir-status` и `detmir-adk-status` в один Rust binary. +3. `[done]` Перенести `detmir-check`. +4. `[done]` Сделать shadow compare `detmir-check` old/new. +5. `[done]` Перенести `detmir-dlp` wrapper. +6. `[done]` Начать `detmir-auto --no-heal`. +7. `[done]` Накопить scheduled shadow/prod parity для `detmir-auto-rust`, затем выполнить controlled switch. +8. `[done]` Перенести `detmir-heal-safe` с dry-run/apply guardrails и подключить к Rust auto. +9. `[done]` Наблюдать timer цикл Rust production и отключить redundant shadow timer. +10. `[done]` Перенести `tsj-guardian-watchdog.sh` на Rust service loop. +11. `[done]` Перенести AW-RUS SLO monitor/summary path на Rust: + `aw-slo-monitor-rust` пишет `aw-slo-samples.jsonl`, + `aw-slo-summary.json` и `aw-slo-summary.txt`; production включен через + `/etc/systemd/system/aw-slo-monitor.service.d/20-rust-switch.conf`. +12. `[done]` Выполнить controlled correction старых ложных SLO bad-сэмплов: + backup создан, 133 записи с единственной причиной + `worktime_today_html body too small ... < 5000` исправлены, 59 + не-ложных/неоднозначных bad-сэмплов оставлены в истории. +13. `[done]` Вынести Telegram `/status` aggregation layer в Rust: + `tsj-guardian-status --status-text` собирает полный текст статуса, Python + runtime Telegram оставлен и использует старую сборку только как fallback. +14. `[done]` Вынести incident/escalation decision path в Rust: + `tsj-guardian-status` принимает JSON через stdin и решает suggestions, + transient quorum/defer и timeout escalation/fallback. Python runtime + выполняет только side effects и сохраняет fallback на старую логику. +15. `[done]` Вынести operator action routing/gating в Rust: + `tsj-guardian-status --operator-action-decision` нормализует action aliases, + назначает handler/risk, блокирует неизвестные action и защищает + update install/rollback confirm без pending-запроса. Python runtime + продолжает выполнять команды и сохраняет fallback. +16. `[done]` Вынести DLP policy/mode decision path в Rust: + `tsj-guardian-status --dlp-policy-decision` определяет monitor/enforce/mixed, + считает block/total по endpoint groups, строит toggle/target policy plan и + changed_rules. Python runtime продолжает выполнять API PUT и Windows policy + sync, fallback на старую Python-логику сохранен. +17. `[done]` Вынести confirmation state machines в Rust: + `tsj-guardian-status --confirmation-decision` решает TTL/expire, cancel, + first_confirm и apply code/stage validation для pfSense, OpenVPN, + Proxmox selection и Proxmox restore. Python runtime продолжает создавать + pending-запросы и выполнять side effects после `allowed=true`, fallback на + старую Python-валидацию сохранен. +18. `[done]` Вынести autoheal/action plan extraction в Rust: + `tsj-guardian-status --autoheal-plan-decision` классифицирует AW-Rus + failures в Windows collector heal, server-side DLP heal, worktime rebuild, + SLO no-direct-target и задержку перед повторной проверкой. Python runtime + продолжает выполнять recovery-команды и повторный probe, fallback на старую + Python-маршрутизацию сохранен. Production deploy проверен на Proxmox: + sample decisions OK, `tsj-guardian-bot`, `tsj-guardian-watchdog` и + `gost-tg` active, `detmir-status` OK / `ok_for_operator=true`, свежий + journal после правильной выкладки helper без повторных unexpected-argument + warnings. +19. `[done]` Укрепить deploy artifacts после миграции: + `deploy_aw_server.yml` переведен на единый `aw_rust_release_dir` для Rust + binary `stat`/`copy`, чтобы `CARGO_TARGET_DIR` работал одинаково для + Proxmox/Telegram и AW-server deploy. Добавлен + `scripts/check_detmir_rust_release_artifacts.sh`; проверено + `CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target cargo build + --release --workspace`, artifact check OK, `deploy_aw_server.yml + --syntax-check` OK. +20. `[done]` Прогнать production AW-server deploy contract и закрыть найденные + эксплуатационные дефекты: + - Influx tokens для deploy берутся из окружения; при восстановлении с + сервера значения не печатать. + - Legacy DB merge выключен по умолчанию через + `aw_legacy_db_merge_enabled: false`, потому что backup SQLite на + 12G root FS уперся в `No space left on device`. Повторять merge только + после отдельного disk plan. + - Browser smoke по умолчанию принудительно использует `chromium-cli`; + `node-playwright` на текущем Node ломается `rimraf: callback function + required`. + - `aw-rus-healthd-rust` теперь проверяет AW interactive buckets по + `metadata.end`, а event buckets выбирают свежайшее событие из окна + `limit=20` с учетом `timestamp + duration`; это устраняет ложные + stale/warn на длинных ActivityWatch событиях и guard heartbeat ordering. + - После reset runtime budget `AWatchRusCollectorGuard` и однократного + перезапуска AW watcher tasks production verification зеленый: + `aw-rus-healthd-rust` `ok=14 warn=0 fail=0`, `detmir-status` OK, + `systemctl --failed` на AW и Proxmox пусто, `deploy_aw_server.yml + --syntax-check` OK. +21. `[done]` Выполнить безопасную уборку AW app data и защитить timers от + лавинного восстановления: + - удалены только старые browser-smoke run-директории, manual profile, + неиспользуемый `/opt/activitywatch/releases/v0.13.2` и неиспользуемый + `/opt/activitywatch/aw-rus-ops/venv`; + - текущий `aw-server-rust-v0.13.2`, SQLite DB/WAL/SHM и rollback-critical + backups не удалялись; + - root FS улучшен примерно с `96%` до `85%`; + - AW timers переведены на staggered `OnCalendar`, чтобы после ручного + restart/daemon-reload не запускать все missed persistent jobs одной + пачкой; + - legacy `aw_to_influx_exporter.service` удален из systemd, потому что + production уже использует Rust DLP/worktime Influx exporters; + - `aw-rus-healthd-rust` получил timeout для wrapper-команд, чтобы + `aw-health-check`/`dlp-health-check` не могли подвесить healthd. +22. `[done]` Перенести DLP content analyzer text path в Rust: + - добавлен `dlp-content-analyzer` crate; + - CLI совместим с `aw-dlp-content-analyzer --text --dictionary-pack + --regex-pack`; + - реализованы dictionary/regex pack matching и checksum validators для + ИНН, СНИЛС, паспорта РФ; + - позиции совпадений считаются в Python-compatible character offsets, не + byte offsets; + - image/OCR path оставлен через legacy Python fallback, чтобы не тащить + OCR/Tesseract в Rust на этом шаге; + - `/usr/local/bin/aw-dlp-content-analyzer` теперь wrapper, который + предпочитает `/usr/local/bin/aw-dlp-content-analyzer-rust`, но сохраняет + Python fallback; + - parity на AW server проверен на `152-fz-pdn`, `contacts`, `secrets`, + `financial`: match counts, values и offsets совпали; + - artifact check расширен `dlp-content-analyzer`. +23. `[done]` Перенести DLP admin CLI в Rust: + - добавлен `dlp-admin-cli` crate; + - сохранены команды `policies list/active`, `incidents list`, + `cases list/create`, `health check`; + - HTTP client использует no-proxy для локальных AW/DLP сервисов; + - `/usr/local/bin/dlp-admin-cli` установлен как Rust production CLI, + legacy `/usr/local/bin/dlp-admin-cli.py` оставлен для rollback/reference; + - parity на AW server совпал с Python для `health`, `policies active`, + `incidents list --since-hours 24 --limit 5`, `cases list --limit 5`; + - production verification зеленый: AW failed units 0, + `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` 22/0/0, + `detmir-status` OK, `detmir-check` OK, Proxmox failed units 0; + - artifact check расширен `dlp-admin-cli`. +24. `[done]` Перенести DLP policy engine service в Rust: + - добавлен `dlp-policy-engine` crate; + - сохранены существующая SQLite schema/DB и API: policies + list/create/get/update/delete, active bundle/version, + submit/approve/draft/activate, rollback, agent heartbeat/desired, + audit; + - checksum policy совместим с Python: + `json.dumps(..., ensure_ascii=False, sort_keys=True, + separators=(",", ":"))` + SHA-256; + - env contract сохранен: + `AW_DLP_POLICY_ENGINE_BIND_HOST`, `AW_DLP_POLICY_ENGINE_PORT`, + `AW_DLP_POLICY_ENGINE_DB_PATH`; + - shadow parity на AW server совпал с Python для list/active/version и + agent heartbeat/desired; + - Rust исправляет documented `/api/0/dlp/policies/audit?limit=N`: legacy + FastAPI возвращал 422 из-за route-order конфликта с `{policy_id}`; + - production `/usr/local/bin/aw-dlp-policy-engine-rust` подключен через + `/etc/systemd/system/aw-dlp-policy-engine.service.d/20-rust-switch.conf`; + - rollback: удалить этот drop-in, `systemctl daemon-reload`, restart + `aw-dlp-policy-engine.service`; + - production verification зеленый: active policy `default-policy` v6, + audit endpoint OK, AW failed units 0, `aw-rus-healthd-rust` 14/0/0, + `dlp-health-check` 22/0/0, `detmir-status` OK, `detmir-check` OK, + Proxmox failed units 0; + - artifact check расширен `dlp-policy-engine`. +25. `[done]` Перенести DLP case management service в Rust: + - добавлен `dlp-case-management` crate; + - сохранены существующая SQLite schema/DB и API: `/health`, cases + list/create/get/update, comments list/create, Hayabusa forensics link; + - сохранены self-test rejection и evidence SHA-256 contract: + `json.dumps(..., ensure_ascii=False, sort_keys=True, + separators=(",", ":"))` + SHA-256; + - env contract сохранен: + `AW_DLP_CASE_BIND_HOST`, `AW_DLP_CASE_PORT`, `AW_DLP_CASE_DB_PATH`; + - local HTTP smoke на temp DB проверил create/list/patch/comment/hayabusa + link/self-test reject; + - shadow parity на AW server совпал с Python для health, case list/filter, + case detail и comments на копии production DB; + - production `/usr/local/bin/aw-dlp-case-management-rust` подключен через + `/etc/systemd/system/aw-dlp-case-management.service.d/20-rust-switch.conf`; + - rollback: удалить этот drop-in, `systemctl daemon-reload`, restart + `aw-dlp-case-management.service`; + - production verification зеленый: case list OK/latest id 35, + AW failed units 0, `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` + 22/0/0, `detmir-status` OK, `detmir-check` OK, Proxmox failed units 0; + - artifact check расширен `dlp-case-management`. +26. `[done]` Перенести DLP compliance report scheduler/generator в Rust: + - добавлен `dlp-compliance` crate; + - сохранены CLI `--month`, `--profile`, `--profiles`, `--stdout-json`; + - сохранены env fallback для AW API: `AW_DLP_AW_API_BASE`, + `AW_SERVER_URL`, default `http://127.0.0.1:5600`; + - сохранены output/template envs и artifact names: + `-.html/json`; + - shadow parity на AW server в temp output dirs совпал с Python для + `152-fz,pci-dss`: profile/period/aw_api_base/stats и наличие artifacts; + - production `/usr/local/bin/aw-dlp-compliance-rust` подключен через + `/etc/systemd/system/aw-dlp-report-scheduler.service.d/20-rust-switch.conf`; + - rollback: удалить этот drop-in и `systemctl daemon-reload`; + - production oneshot run успешен: `152-fz-2026-06` и `pci-dss-2026-06` + html/json созданы, timer active, AW failed units 0, + `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` 22/0/0, + `detmir-status` OK, `detmir-check` OK, Proxmox failed units 0; + - artifact check расширен `dlp-compliance`. +27. `[done]` Перенести Hayabusa case alert/link helpers в Rust: + - добавлен `hayabusa-tools` crate с бинарниками + `aw-hayabusa-case-alert-rust` и `aw-hayabusa-link-case-rust`; + - сохранены CLI/env контракты legacy helpers: + `--case-id`, `--intake-json`, `--case-api-base`, `--mode`, + `--link-source`, `AW_HAYABUSA_*`; + - `case-alert` сохраняет scoring/severity, `top_rules`, comment text, + case create/update/link path и Telegram alert text; + - Telegram request errors в Rust не печатают bot token/URL; + - `/usr/local/bin/aw-hayabusa-case-alert` и + `/usr/local/bin/aw-hayabusa-link-case` теперь wrapper-команды, которые + предпочитают Rust binaries и fallback на Python из + `/opt/activitywatch/aw-rus-ops/hayabusa`; + - shadow compare на реальном `/opt/hayabusa/state/latest-intake.json` с + `AW_HAYABUSA_AUTO_CASE_ENABLED=false` и + `AW_HAYABUSA_TELEGRAM_ENABLED=false` дал `summary_equal=true`; + - production verification зеленый: AW failed units 0, + `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` 22/0/0, + `detmir-status` OK, Proxmox failed units 0; + - artifact check расширен `aw-hayabusa-case-alert-rust` и + `aw-hayabusa-link-case-rust`. +28. `[done]` Перенести remaining Hayabusa offline helpers в Rust: + - `hayabusa-tools` расширен бинарниками + `aw-hayabusa-from-windows-rust` и `aw-hayabusa-autoprocess-rust`; + - `from-windows` сохраняет server-side workflow: + Ansible WinRM export, latest zip discovery, fetch в drop-dir, + `aw-hayabusa accept`, `process-inbox`, optional link-case и печать + `LATEST_INTAKE`; + - `autoprocess` сохраняет drop-dir scanner, lock file, + `.caseid`/`.meta.json` sidecars, host/mode/link_source inference, + case-alert capture, sidecar/archive move в report dir; + - `/usr/local/bin/aw-hayabusa-from-windows` и + `/usr/local/bin/aw-hayabusa-autoprocess` теперь wrapper-команды, + которые предпочитают Rust и fallback на Python из + `/opt/activitywatch/aw-rus-ops/hayabusa`; + - `aw-hayabusa-drop.service` теперь запускает + `/usr/local/bin/aw-hayabusa-autoprocess` напрямую, без `python3`, чтобы + wrapper мог выбрать Rust; + - no-mutation smoke на AW server: `--help` для всех Hayabusa helpers, + empty drop-dir для `autoprocess`, `systemctl cat` ExecStart OK; + - production verification зеленый: AW failed units 0, + `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` 22/0/0, + `detmir-status` OK, Proxmox failed units 0; + - artifact check расширен `aw-hayabusa-from-windows-rust` и + `aw-hayabusa-autoprocess-rust`. +29. `[done]` Перенести AW ops helper `aw-health-check` в Rust: + - добавлен crate `aw-health-check`; + - сохранены проверки legacy shell: systemd services/timer, + ActivityWatch `/api/0/info`, Worktime `/health`, DLP transport + freshness через `dlp-health-check --json`, и drift checks для + `startOfDay`, `always_active_pattern`, `landingpage`; + - env contract читается из процесса и `/etc/activitywatch/aw-server.env`; + - production `/usr/local/bin/aw-health-check` теперь wrapper, который + предпочитает `/usr/local/bin/aw-health-check-rust` и fallback на + `/opt/activitywatch/aw-rus-ops/health-check.sh`; + - shadow parity на AW server совпал с shell по exit code и ключевым + health-строкам; + - production verification зеленый: `aw-health-check` OK, + `aw-rus-healthd-rust` 14/0/0, `dlp-health-check` 22/0/0, + `detmir-status` OK, `ok_for_operator=true`, failed units 0; + - artifact check расширен `aw-health-check`. +30. `[done]` Перенести AW ops helper `check-aw-data` в Rust: + - добавлен crate `check-aw-data`; + - корневой `check-aw-data.sh` стал Rust-first wrapper с fallback на + `scripts/legacy/check-aw-data.sh`; + - на AW server production `/usr/local/bin/check-aw-data` теперь wrapper, + который предпочитает `/usr/local/bin/check-aw-data-rust`, по умолчанию + использует `http://127.0.0.1:5600` и fallback на + `/opt/activitywatch/aw-rus-ops/check-aw-data.sh`; + - сохранены операторские статусы `FRESH`, `STALE`, `DEAD`, `EMPTY`, + `EVENT-DRIVEN`, `INACTIVE` и CORS probe; + - быстрый режим по умолчанию использует `metadata.end` из + `/api/0/buckets`; глубокое чтение event ids включается флагом + `--with-event-ids`; + - production verification: `/usr/local/bin/check-aw-data --no-color` + завершился за 2s, buckets fresh/event-driven/inactive as expected, + `CORS: OK (HTTP 200)`; + - final gates зеленые: AW failed units 0, `aw-rus-healthd-rust` 14/0/0, + `dlp-health-check` 22/0/0, `detmir-auto --no-heal` OK, + `detmir-status` OK и `ok_for_operator=true`; + - artifact check расширен `check-aw-data`. +31. `[done]` Перенести AW maintenance helper `aw-prune-local-state` в Rust: + - добавлен crate `aw-prune-local-state`; + - `/usr/local/bin/aw-prune-local-state.sh` теперь Rust-first wrapper, + который без аргументов запускает Rust binary в legacy-compatible + `--apply` режиме и fallback на shell из + `/opt/activitywatch/aw-rus-ops/aw-prune-local-state.sh`; + - Rust binary по умолчанию работает как dry-run, поддерживает `--json`, + `--apply`, retention/keep параметры и строгий allowlist путей; + - safety policy: не удалять SQLite DB вне `backups/db`, не удалять + `switch-backups`, `before-rust`, `rollback` и корневые state каталоги; + - cleanup расширен на старые `browser-smoke` run-директории как project + app data; rollback-critical backups и AW SQLite DB не трогались; + - production recovery во время проверки: rootfs AW server был 100% + заполнен, `aw-server-rust` попал в `poisoned lock`; через Rust helper + удалены старые browser-smoke runs на 347 MB суммарно, затем + `activitywatch-server.service` восстановлен без удаления DB; + - final gates зеленые: AW failed units 0, `aw-rus-healthd-rust` 14/0/0, + `dlp-health-check` 22/0/0, `check-aw-data` OK, `detmir-auto --no-heal` + OK, `detmir-status` OK и `ok_for_operator=true`; + - Telegram bot/watchdog временно остановлены по операторской команде и не + перезапускались в рамках этого шага; + - artifact check расширен `aw-prune-local-state`. +32. `[done]` Устранить нехватку места на AW server через Proxmox resize: + - CT `203` (`aw-server`, `10.10.10.13`) rootfs расширен через Proxmox + `pct resize 203 rootfs +20G`; + - перед resize сохранен config backup: + `/var/lib/detmir-ai/switch-backups/ct203-aw-server.before-rootfs-resize-20260602T054826Z.conf`; + - rootfs изменился с `12G` на `32G`, свободное место стало около `20G`; + - Proxmox storage `local-btrfs` после resize имеет около `106G` + доступного места; + - AW `/api/0/info` и `/api/0/buckets` проверены с AW server и Proxmox; + - final gates зеленые: AW health 14/0/0, DLP health 22/0/0, + `detmir-check` service_warnings=0, `detmir-auto --no-heal` OK, + `detmir-status` OK и `ok_for_operator=true`, failed units 0; + - Telegram bot/watchdog оставлены `inactive` по операторской команде. +33. `[done]` Остановить неконтролируемый рост `aw-session-events`: + - read-only SQLite audit показал, что `/var/lib/activitywatch/aw-server-rust/sqlite.db` + занимает около `6.8G`, `freelist_count=0`; VACUUM сам по себе не + освободит место, потому что размер занят live events; + - основной источник роста: `aw-session-events_SHARKON2025` - около + `6.9M` строк и `~5GB` payload, с пиками `1.2M-2.4M` process-level + событий в сутки за 2026-05-29..2026-06-01; + - production live config на RDP был `pollSeconds=5` и + `sessionEvents.processEventsEnabled=true`; + - отключена постоянная process-level публикация: + `aw_windows_process_events_enabled=false` в production/example/default + Ansible vars и `sessionEvents.processEventsEnabled=false` в live + `C:\ProgramData\AWatch-rus\deployment-config.json`; + - live config backup сохранен на RDP: + `C:\ProgramData\AWatch-rus\switch-backups\deployment-config.before-disable-process-events-20260602T061836Z.json`; + - старый `worktime-session-collector.ps1` PID `8476` остановлен, collector + поднят заново штатными `ActivityWatch Launch [...]` tasks/guard; + - delta-gate: `metadata.end` bucket `aw-session-events_SHARKON2025` + остался `2026-06-02T06:27:11.197Z` через 75 секунд, постоянный поток + остановлен; + - базовый сбор не сломан: `detmir-check --json` OK, + `detmir-status` severity OK, AW failed units 0, Proxmox failed units 0, + AW API и Worktime API отвечают; + - SQLite row deletion/compact не выполнялись на этом этапе: сначала + стабилизирован источник роста, retention/trim старых process events - + отдельный destructive этап с backup. +34. `[done]` Controlled trim старых process-level `aw-session-events`: + - перед mutation сохранен полный rollback backup SQLite DB/WAL/SHM: + `/var/lib/activitywatch/backups/db/aw-sqlite-before-session-events-trim-20260602T064127Z`; + - на время операции остановлены AW-related timers/services и + `activitywatch-server.service`, чтобы не было writer'ов к SQLite; + - scoped delete удалил только события bucket + `aw-session-events_SHARKON2025` с `eventType=process_start` или + `eventType=process_stop`; + - удалено `6,906,190` шумных process-level событий; + - сохранены logon events: после trim в `aw-session-events_SHARKON2025` + осталось `174` события, recent samples имеют `eventType=logon`; + - `PRAGMA integrity_check` до и после `VACUUM`: `ok`; + - DB уменьшилась с `6.8G` до `350M`, rootfs AW server вернулся к + `38%` использования и около `19G` free; + - после restart AW был `poisoned lock` из-за timer запросов во время + старта; выполнен controlled restart при остановленных timers, затем + `/api/0/info` и `/events?limit=1` вернулись OK; + - timers включены обратно, failed units на AW и Proxmox: `0`; + - final gates зеленые: `aw-health-check` OK, `dlp-health-check` + `22/0/0`, `detmir-check` bucket_ok=8 stale=0 dead=0, + `detmir-auto --no-heal` OK, `detmir-status` severity OK и + `ok_for_operator=true`; + - Telegram bot/watchdog оставлены `inactive` по операторской команде. +35. `[done]` Добавить Rust guard от повторного роста AW SQLite: + - добавлен crate `aw-db-health`; + - helper read-only открывает `/var/lib/activitywatch/aw-server-rust/sqlite.db` + через SQLite read-only flags и не меняет DB; + - проверяет: + - размер `sqlite.db` (`warn=2GiB`, `fail=5GiB` по умолчанию); + - размер `sqlite.db-wal` (`warn=256MiB`, `fail=1GiB`); + - число строк `aw-session-events_` (`warn=10000`, `fail=100000`); + - recent process-level events за окно 600 секунд + (`warn=1`, `fail=100`); + - latest `eventType/source` для быстрой диагностики; + - опционально `--windows-config` и + `sessionEvents.processEventsEnabled`; + - поддерживает text output и `--json`; + - `aw-health-check-rust` теперь вызывает `/usr/local/bin/aw-db-health --json` + и включает DB growth guard в общий AW health path; + - `scripts/check_detmir_rust_release_artifacts.sh` требует + `aw-db-health`; + - `ansible/deploy_aw_server.yml` устанавливает + `/usr/local/bin/aw-db-health` и обновленный + `/usr/local/bin/aw-health-check-rust`; установка health helpers + продублирована до Influx token assert, чтобы DB guard обновлялся даже + если playbook позже останавливается на пустом внешнем Influx token; + - production `/usr/local/bin/aw-db-health --json`: + `ok=true`, `db=352.3MiB`, `wal=4.0MiB`, session rows `174`, + recent process events `0`, latest `eventType=logon`; + - production `/usr/local/bin/aw-health-check`: `AW DB growth guard passed`; + - local gates: `cargo fmt --all -- --check`, `cargo test -p aw-db-health` + `3 passed`, `cargo test -p aw-health-check` `2 passed`, + release build OK, artifact check OK; + - final gates зеленые: `dlp-health-check` `22/0/0`, `detmir-check` OK, + `detmir-auto --no-heal` OK, `detmir-status` severity OK, + failed units на AW/Proxmox `0`; + - Telegram bot/watchdog оставлены `inactive`. +35.1. `[done]` Добавить автономную guarded чистку AW SQLite: + - добавлен crate `aw-db-maintenance`; + - helper dry-run по умолчанию, mutation только с `--apply`; + - allowlist удаления: только bucket `aw-session-events_` и только + события `eventType=process_start` или `eventType=process_stop`; + - logon/session events не удаляются; + - retention по умолчанию `7` дней; + - перед удалением создается SQLite backup в + `/var/lib/activitywatch/backups/db/aw-sqlite-before-db-maintenance-*.db`; + если строк к удалению нет, backup не создается и DB не пишется; + - удаление chunked, без `VACUUM`, чтобы не создавать длительный downtime; + - добавлены `aw-server/aw-db-maintenance.service` и + `aw-server/aw-db-maintenance.timer`; + - timer включен на AW server: `OnCalendar=Sun *-*-* 03:30:00`, + `RandomizedDelaySec=15m`, `Persistent=true`; + - production dry-run: `planned_delete_rows=0`, `deleted_rows=0`; + - ручной service smoke: `Result=success`, `ExecMainStatus=0`, + `planned_delete_rows=0`, `deleted_rows=0`, `backup_created=false`; + - production timer active; ближайший запуск по remote `systemctl + list-timers`: `Sun 2026-06-07 03:40:05 UTC`; + - gates: `cargo fmt --all -- --check`, `cargo test -p aw-db-maintenance` + `3 passed`, `cargo clippy -p aw-db-maintenance --all-targets -- -D + warnings`, release build OK, artifact check OK, `quality-gate.sh` OK, + `ansible-playbook deploy_aw_server.yml --syntax-check` OK; + - final production gates: AW failed units `0`, `aw-db-health` OK + (`sqlite.db=359.6MiB`, WAL `4.0MiB`, session rows `174`, recent process + events `0`, latest `eventType=logon`), DetMir status OK with + `dlp_counts={ok:22,warn:0,fail:0}`, `ok_for_operator=true`. +36. `[done]` Устранить blocker полного AW server deploy на Influx token: + - проблема: `deploy_aw_server.yml` падал на assert + `aw_worktime_influx_enabled=true`, потому что локальные env + `AW_WORKTIME_INFLUX_TOKEN` и `AW_DLP_INFLUX_TOKEN` пустые; + - runtime `/etc/activitywatch/aw-server.env` на AW server уже содержал оба + token, поэтому правильный путь - сохранить remote secrets, а не + отключать exporters; + - playbook теперь через `slurp` с `no_log: true` читает текущий + `/etc/activitywatch/aw-server.env`, выбирает effective token: + local env token если задан, иначе existing remote token; + - asserts проверяют effective tokens и не печатают секреты; + - запись `/etc/activitywatch/aw-server.env` использует effective tokens, + чтобы full deploy не затирал рабочие Influx credentials пустыми + значениями; + - полный `ansible-playbook -i inventory.ini deploy_aw_server.yml + -e aw_rust_release_dir=/home/igor/.cache/detmir-adk-rust-target/release` + прошел до конца: `failed=0`, `ok=282`; + - final gates после deploy зеленые: `aw-db-health` OK, + `aw-health-check` OK, `dlp-health-check` `22/0/0`, + `detmir-check` OK, `detmir-auto --no-heal` OK, + `detmir-status` severity OK, failed units на AW/Proxmox `0`; + - Telegram bot/watchdog оставлены `inactive`. +37. `[done]` Закрепить safe default для Windows `processEventsEnabled`: + - исправлены Windows deploy/recovery defaults, чтобы новый deploy, + ensemble/domain deploy, hardening recovery и общий + `New-ActivityWatchDeploymentConfig` не включали process-level + `aw-session-events` без явного параметра; + - изменены `windows/deploy-single-user.ps1`, + `windows/deploy-ensemble.ps1`, `windows/deploy-domain-users.ps1`, + `windows/hardening-recovery.ps1`, + `windows/ActivityWatch.Windows.Common.psm1` и + `windows/validate-deployment.ps1`; + - синхронизированы текстовые копии в + `install-kit-awindows-20260427-211240/windows/`; + - invariant: existing live config сохраняется, но missing/new config + default всегда `sessionEvents.processEventsEnabled=false`; включать + можно только явно и временно для forensic/debug окна; + - local gates: PowerShell AST parse для измененных scripts/modules OK, + `ansible-playbook -i inventory.ini deploy_aw_windows.yml --syntax-check` + OK; + - production deploy: обновленные PowerShell файлы скопированы в + `C:\Program Files\AWatch-rus\windows` с UTF-8 BOM для Windows + PowerShell 5.1; remote backup перед заменой: + `C:\ProgramData\AWatch-rus\switch-backups\windows-safe-default-20260602T091213Z`; + - `hardening-recovery.ps1` выполнен по existing + `C:\ProgramData\AWatch-rus\deployment-config.json`; после recovery + config остался `processEventsEnabled=false`, `logonEnabled=true`, + `pollSeconds=5`, users включают `SHARKON2025\Администратор`; + - exact task check: `ActivityWatch Launch [SHARKON2025_Администратор]` + существует, ошибочный + `ActivityWatch Launch [SHARKON2025_Administrator]` отсутствует, + `ActivityWatch Recovery` существует; + - `AWatchRusCollectorGuard` running/automatic; Windows + `validate-deployment.ps1` вернул `overallOk=True`; + - production AW DB guard: session rows `174`, recent process events `0`, + latest `eventType=logon`, DB около `353MiB`; + - final gates зеленые: `check-aw-data` buckets fresh/event-driven OK, + `aw-health-check` OK, `dlp-health-check` `22/0/0`, + `detmir-check` bucket_ok `8`, stale `0`, dead `0`, + `detmir-auto --no-heal` OK, `detmir-status` severity OK и + `ok_for_operator=true`, failed units на AW/Proxmox `0`; + - краткий transient после recovery: один прогон `detmir-auto` увидел + `file-operations sendFailuresDelta`; повтор через 30 секунд показал + `sendFailuresDelta=0`, поэтому коррекция collectors не потребовалась; + - Telegram bot/watchdog оставлены `inactive` по принятому решению. +38. `[done]` Пересобрать и проверить Windows install kit: + - выполнен штатный `./scripts/rebuild_install_kit.sh`; + - пересобраны: + `install-kit-awindows-20260427-211240/`, + `install-kit-awindows-20260427-211240.zip`, + `install-kit-awindows-20260427-211240.tar.gz` и `MANIFEST.txt`; + - `./scripts/check_install_kit_vs_repo.sh`: compared `62`, + missing `0`, mismatched `0`, PowerShell mismatches `0`; + - manifest checksum verification: OK, entries `66`; + - archives verification: zip и tar.gz распакованы во временные каталоги, + опасные defaults не найдены; подтверждены + `[bool]$ProcessEventsEnabled = $false`, + `effectiveProcessEventsEnabled ... else { $false }`, + `sessionProcessEventsEnabled ... else { $false }`, + `aw_windows_process_events_enabled: false`; + - PowerShell AST parse для kit scripts/modules OK; + - `ansible-playbook -i inventory.example.ini deploy_aw_windows.yml + --syntax-check` из install-kit ansible каталога OK. +39. `[done]` Перенести install-kit consistency checker на Rust: + - добавлен crate `check-install-kit-vs-repo`; + - `scripts/check_install_kit_vs_repo.sh` теперь Rust-first wrapper: + ищет `CHECK_INSTALL_KIT_VS_REPO_RUST`, + `$CARGO_TARGET_DIR/release/check-install-kit-vs-repo`, + `adk-rust/target/release/check-install-kit-vs-repo`, + `/usr/local/bin/check-install-kit-vs-repo`, затем использует Python + fallback; + - сохранен текстовый output contract: + `Compared files`, `Missing in repo`, `Mismatched content`, + `PowerShell mismatches`; добавлен `--json`; + - поведение усилено: missing/mismatch теперь дают non-zero exit code, + чтобы рассинхрон install-kit не проходил молча; + - `scripts/check_detmir_rust_release_artifacts.sh` теперь требует + `check-install-kit-vs-repo`; + - после обновления wrapper выполнен `./scripts/rebuild_install_kit.sh`, + чтобы install kit нес актуальный Rust-first wrapper; + - gates: `cargo fmt --all -- --check`, `cargo test -p + check-install-kit-vs-repo` (`3 passed`), `cargo clippy -p + check-install-kit-vs-repo --all-targets -- -D warnings`, release build + OK, `./scripts/check_install_kit_vs_repo.sh` OK, negative mismatch smoke + rc `1`, `./scripts/validate_install_kit.sh` OK, artifact check OK. +40. `[done]` Перенести install-kit validator на Rust: + - добавлен crate `validate-install-kit`; + - `scripts/validate_install_kit.sh` теперь Rust-first wrapper: + ищет `VALIDATE_INSTALL_KIT_RUST`, + `$CARGO_TARGET_DIR/release/validate-install-kit`, + `adk-rust/target/release/validate-install-kit`, + `/usr/local/bin/validate-install-kit`, затем использует legacy + Bash/Python fallback; + - сохранен stage output: + `[1/4] Required files presence`, + `[2/4] Manifest checksum verification`, + `[3/4] Manifest completeness`, + `[4/4] Archive composition check`, + `validate_install_kit: OK`; добавлен `--json`; + - Rust validator проверяет required files, SHA256 из `MANIFEST.txt`, + полноту manifest относительно install-kit directory, совпадение состава + zip/tar.gz и корректный archive prefix; + - `scripts/rebuild_install_kit.sh` теперь включает + `scripts/validate_install_kit.sh` в install-kit `scripts/`; + - `scripts/check_detmir_rust_release_artifacts.sh` теперь требует + `validate-install-kit`; + - после wrapper change выполнен `./scripts/rebuild_install_kit.sh`; + - gates: `cargo fmt --all -- --check`, `cargo test -p + validate-install-kit` (`4 passed`), `cargo clippy -p + validate-install-kit --all-targets -- -D warnings`, release build OK, + Rust-first `./scripts/validate_install_kit.sh` OK + (`MANIFEST complete: 67 files tracked`, `Archives match: 68 files`), + `./scripts/check_install_kit_vs_repo.sh` OK (`compared 62`, + mismatches `0`), negative manifest checksum smoke rc `1`, negative + archive mismatch smoke rc `1`, artifact check OK. +41. `[done]` Перенести InnoSetup installer verifier на Rust: + - добавлен crate `verify-innosetup-installer`; + - `scripts/verify_innosetup_installer.sh` теперь Rust-first wrapper: + ищет `VERIFY_INNOSETUP_INSTALLER_RUST`, + `$CARGO_TARGET_DIR/release/verify-innosetup-installer`, + `adk-rust/target/release/verify-innosetup-installer`, + `/usr/local/bin/verify-innosetup-installer`, затем использует legacy + Wine/Bash fallback; + - Rust verifier создает чистый Wine prefix, silent-install'ит + `windows/installkit/innosetup/AWatch-rus-InstallKit.exe` в + `C:\AWatchRusExtract`, сверяет payload guard/policy файлов с + репозиторием и проверяет self-test marker + `collector guard self-test OK`; + - первый прогон корректно поймал устаревший installer payload: + `aw-collector-guard.ps1` и `install-collector-guard-service.ps1` + отличались от repo после safe-default hardening; + - installer пересобран штатным `windows/installkit/innosetup/build_with_wine.sh`, + после чего Rust-first verifier завершился `verify_innosetup_installer: + OK`; + - `scripts/check_detmir_rust_release_artifacts.sh` теперь требует + `verify-innosetup-installer`. +42. `[done]` Перенести install-kit rebuild path на Rust: + - добавлен crate `rebuild-install-kit`; + - `scripts/rebuild_install_kit.sh` теперь Rust-first wrapper: + ищет `REBUILD_INSTALL_KIT_RUST`, + `$CARGO_TARGET_DIR/release/rebuild-install-kit`, + `adk-rust/target/release/rebuild-install-kit`, + `/usr/local/bin/rebuild-install-kit`, затем использует legacy Bash + fallback; + - Rust rebuild сохраняет `*.deployment-config.json`, пересобирает + directory tree, `MANIFEST.txt`, `.zip` и `.tar.gz`; + - копирование реализовано через read + replace-write, а не `fs::copy`, + потому что текущий mounted-диск возвращал `Operation not permitted` на + truncate/copy existing files; + - install-kit теперь включает сам `scripts/rebuild_install_kit.sh`, чтобы + комплект был самодостаточным для дальнейшей пересборки; + - Rust-first rebuild verified через `validate_install_kit`, + `check_install_kit_vs_repo`, archive script `cmp`, and release artifact + check. +43. `[done]` Перенести `quality-gate.sh` на Rust-first orchestrator: + - добавлен crate `quality-gate`; + - `scripts/quality-gate.sh` теперь Rust-first wrapper: + ищет `QUALITY_GATE_RUST`, + `$CARGO_TARGET_DIR/release/quality-gate`, + `adk-rust/target/release/quality-gate`, `/usr/local/bin/quality-gate`, + затем использует legacy Bash fallback; + - Rust orchestrator сохраняет stage contract: Bash syntax, + ShellCheck when available, Node syntax when available, PowerShell parse + and collector guard self-test when `pwsh` available, Ansible syntax when + `ansible-playbook` available; + - final gates: `cargo fmt --all -- --check`, targeted tests for + `verify-innosetup-installer`, `rebuild-install-kit`, `quality-gate`, + `validate-install-kit`, clippy `-D warnings`, release builds, + Rust-first `quality-gate: OK`, install-kit validate/check OK, + InnoSetup verifier OK, and DetMir read-only status OK with + `ok_for_operator=true`. +44. `[done]` Перенести offline Sigma/Hayabusa IOC extraction path на Rust: + - добавлен crate `extract-ioc-from-sigma`; + - `scripts/build_dlp_ioc_from_hayabusa.sh` теперь Rust-first wrapper: + ищет `EXTRACT_IOC_FROM_SIGMA_RUST`, + `$CARGO_TARGET_DIR/release/extract-ioc-from-sigma`, + `adk-rust/target/release/extract-ioc-from-sigma`, + `/usr/local/bin/extract-ioc-from-sigma`, затем использует legacy Python + `scripts/extract_ioc_from_sigma.py`; + - Rust binary сохраняет CLI-контракт `--rules-root`, `--out-dir`, + `--table-name` и output files `ioc_blacklist.json`, + `ioc_blacklist.csv`, `ioc_blacklist.sql`; + - extraction contract сохранен: `Image|endswith`, + `CommandLine|contains`, `OriginalFileName`, `Hashes|SHA256` и SHA256 + values embedded in `Hashes` strings; + - parity fix: Rust использует Unicode lowercase и PyYAML-compatible + `True`/`False` rendering для YAML 1.1 boolean-like literals, чтобы не + расширять IOC set относительно legacy Python; + - real Hayabusa rules parity: + `rules_scanned=4963`, `iocs_extracted=13510`, + `commandline_contains=10192`, `original_filename=457`, + `process_image_endswith=1171`, `sha256=1690`, + `missing_in_rust=0`, `extra_in_rust=0`; + - gates: `cargo fmt --all -- --check`, `cargo test -p + extract-ioc-from-sigma` (`3 passed`), `cargo clippy -p + extract-ioc-from-sigma --all-targets -- -D warnings`, release build OK, + `scripts/check_detmir_rust_release_artifacts.sh` OK, + Rust-first `scripts/quality-gate.sh` OK. +45. `[done]` Перенести `scripts/rdp-worktime-report.sh` на Rust-first helper: + - добавлен crate `rdp-worktime-report`; + - `scripts/rdp-worktime-report.sh` теперь Rust-first wrapper: + ищет `RDP_WORKTIME_REPORT_RUST`, + `$CARGO_TARGET_DIR/release/rdp-worktime-report`, + `adk-rust/target/release/rdp-worktime-report`, + `/usr/local/bin/rdp-worktime-report`, затем использует legacy embedded + Python fallback; + - Rust binary сохраняет CLI/env contract: `--day today|yesterday`, + `--from YYYY-MM-DD --to YYYY-MM-DD`, `AW_BASE_URL`, + `AW_WORKTIME_HOST`, `AW_WORKTIME_DEFAULT_SAMPLE_SECONDS`, + `AW_WORKTIME_MAX_SAMPLE_SECONDS`, `OUT_DIR`; + - output contract сохранен: `rdp-worktime-_.csv`, + `rdp-worktime-_.json`, stdout paths plus `CSV:`/`JSON:`; + - calculation contract сохранен: bucket + `aw-worktime-sessions_`, active samples by `active=true` or + Russian/English active state, interval merge, sampleSeconds/pollSeconds, + duration and next-timestamp fallback, canonical `HOST\user` userId; + - live parity on yesterday against Python fallback: + `rust_csv_rows=3`, `python_csv_rows=3`, `csv_equal=True`, + JSON `host/bucket_id/from/to/rows` all equal; + - gates: `cargo fmt --all -- --check`, `cargo test -p + rdp-worktime-report` (`3 passed`), `cargo clippy -p + rdp-worktime-report --all-targets -- -D warnings`, release build OK, + `scripts/check_detmir_rust_release_artifacts.sh` OK, + `scripts/quality-gate.sh` OK, DetMir read-only status OK with + `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. +46. `[done]` Перенести Proxmox DetMir contour smoke на Rust-first helper: + - добавлен crate `aw-contour-smoke`; + - `scripts/aw-contour-smoke-10.10.10.2.sh` теперь Rust-first wrapper: + ищет `AW_CONTOUR_SMOKE_RUST`, + `$CARGO_TARGET_DIR/release/aw-contour-smoke`, + `adk-rust/target/release/aw-contour-smoke`, + `/usr/local/sbin/aw-contour-smoke`, + `/usr/local/bin/aw-contour-smoke`, затем использует legacy Bash + fallback; + - `scripts/aw-contour-smoke-local.sh` умеет временно доставлять release + binary на Proxmox для shadow smoke без постоянного platform change; + - remote parity: Rust и legacy оба вернули `OK=30 WARN=0 FAIL=3 SKIP=0`, + `rc=2`; все 3 fail совпали и относятся к существующему gateway-auth + `HTTP 401` на `/go/proxmox-gui`, `/go/file1c-brief`, + `/go/file1c-actions`, не к Rust regression; + - gates: `cargo fmt --all -- --check`, `cargo test -p + aw-contour-smoke` (`2 passed`), `cargo clippy -p aw-contour-smoke + --all-targets -- -D warnings`, release build OK, + `bash -n` wrappers OK, artifact check OK, `quality-gate.sh` OK. +47. `[done]` Перенести `scripts/diag_and_manual_restart.sh` на Rust-first + diagnostic helper: + - добавлен crate `diag-and-manual-restart`; + - wrapper ищет `DIAG_AND_MANUAL_RESTART_RUST`, + `$CARGO_TARGET_DIR/release/diag-and-manual-restart`, + `adk-rust/target/release/diag-and-manual-restart`, + `/usr/local/bin/diag-and-manual-restart`, затем использует legacy Bash + fallback; + - здоровый контур не перезапускается: helper запускает `aw-health-check` + и `dlp-health-check`, печатает `Diagnostics: healthy. Restart not + needed.` и выходит `0`; + - mutation path требует явного подтверждения или `--yes`; live restart не + запускался, потому что контур был зеленый; + - gates: fmt/test/clippy/release OK, `bash -n` OK, Rust-first healthy path + rc `0`, legacy fallback healthy path rc `0`, artifact check OK, + `quality-gate.sh` OK, DetMir read-only OK. +48. `[done]` Перенести browser smoke launcher на Rust-first helper: + - добавлен crate `aw-browser-smoke`; + - `scripts/aw-webui-browser-smoke.sh` теперь Rust-first wrapper: + ищет `AW_BROWSER_SMOKE_RUST`, + `$CARGO_TARGET_DIR/release/aw-browser-smoke`, + `adk-rust/target/release/aw-browser-smoke`, + `/usr/local/bin/aw-browser-smoke`, затем использует legacy Node + fallback; + - Playwright/Chromium логика остается в + `scripts/aw-webui-browser-smoke.mjs`; Rust отвечает за устойчивый запуск, + `NODE_PATH`, passthrough args и сохранение child exit code; + - fake-node проверки подтвердили passthrough `.mjs` path, user args, + `NODE_PATH` и exit-code propagation для Rust launcher и legacy fallback; + - live browser smoke через Rust-first wrapper: `ok=true`, страницы + `aw_webui_home`, `worktime_today_html`, + `worktime_management_html` все OK; + - gates: `cargo fmt --all -- --check`, `cargo test -p aw-browser-smoke` + (`3 passed`), `cargo clippy -p aw-browser-smoke --all-targets -- -D + warnings`, release build OK, `bash -n` OK, artifact check OK, + `quality-gate.sh` OK, DetMir read-only OK with + `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. +49. `[done]` Перенести `check-aw-full.sh` на Rust-first read-only helper: + - добавлен crate `check-aw-full`; + - root wrapper `check-aw-full.sh` ищет `CHECK_AW_FULL_RUST`, + `$CARGO_TARGET_DIR/release/check-aw-full`, + `adk-rust/target/release/check-aw-full`, `/usr/local/bin/check-aw-full`, + затем выполняет embedded legacy Bash fallback; + - для прямого legacy compare добавлен `CHECK_AW_FULL_FORCE_LEGACY=1`; + - Rust helper сохраняет операторский контракт: AW server connectivity, + CORS probe, 8 bucket freshness rows, RDP WinRM/SSH TCP checks, summary + `FRESH/STALE/DEAD`, recovery hint when stale/dead collectors exist; + - live parity: Rust-first и legacy оба показали connectivity OK, CORS OK, + WinRM OK, SSH OK, `FRESH=8 STALE=0 DEAD=0`; bucket statuses совпали, + один live event id у `Window watcher` ожидаемо сдвинулся между + последовательными запусками; + - gates: `cargo fmt --all -- --check`, `cargo test -p check-aw-full` + (`4 passed`), `cargo clippy -p check-aw-full --all-targets -- -D + warnings`, release build OK, `bash -n` OK, artifact check OK, + `quality-gate.sh` OK, DetMir read-only OK with + `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. +50. `[done]` Перенести `scripts/merge_aw_server_dbs.py` merge engine на Rust: + - добавлен crate `merge-aw-server-dbs`; + - `scripts/merge_aw_server_dbs.py` остается совместимым с текущим + `python3 merge_aw_server_dbs.py` вызовом, но в начале exec'ает Rust + binary при наличии `MERGE_AW_SERVER_DBS_RUST`, + `$CARGO_TARGET_DIR/release/merge-aw-server-dbs`, + `adk-rust/target/release/merge-aw-server-dbs` или + `/usr/local/bin/merge-aw-server-dbs`; + - `MERGE_AW_SERVER_DBS_FORCE_LEGACY=1` форсирует Python fallback; + - `ansible/deploy_aw_server.yml` теперь optional устанавливает + `/usr/local/bin/merge-aw-server-dbs`, если release artifact существует, + сохраняя прежний `python3 /usr/local/bin/merge_aw_server_dbs.py` command; + - Rust сохраняет контракт `--base`, `--overlay`, `--output`, SQLite + backup-copy base DB, bucket matching by id/key/name, duplicate event + suppression by `(starttime,endtime,data)`, JSON summary; + - fallback bugfix: Python legacy больше не падает на `UNIQUE(name)` + name-conflict/update path и считает только реально вставленные buckets; + - offline parity fixture: Rust и Python fallback дали одинаковый JSON + `inserted_buckets=1`, `inserted_events=3` и одинаковые SQLite table rows + (`buckets=3`, `events=5`) на кейсе duplicate event + name-conflict + bucket; + - gates: `cargo fmt --all -- --check`, `cargo test -p + merge-aw-server-dbs` (`2 passed`), `cargo clippy -p + merge-aw-server-dbs --all-targets -- -D warnings`, release build OK, + `python3 -m py_compile scripts/merge_aw_server_dbs.py`, artifact check + OK, `quality-gate.sh` OK, DetMir read-only OK after DLP baseline retry + with `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. +51. `[done]` Перенести `scripts/prod_backup_restore.sh` в safe-by-default + Rust planner/checker: + - добавлен crate `prod-backup-restore`; + - `scripts/prod_backup_restore.sh` теперь Rust-first wrapper: + ищет `PROD_BACKUP_RESTORE_RUST`, + `$CARGO_TARGET_DIR/release/prod-backup-restore`, + `adk-rust/target/release/prod-backup-restore`, + `/usr/local/bin/prod-backup-restore`; + - обычный запуск больше не выполняет restore при наличии Rust artifact: + он печатает plan-only flow и помечает destructive steps; + - если Rust artifact отсутствует, wrapper отказывается запускать + destructive legacy случайно и просит собрать Rust planner; + - старый destructive flow доступен только явно: + `scripts/prod_backup_restore.sh --apply-legacy`; + - Rust `--apply` намеренно запрещен на этом этапе; + - `--check-inputs --json` валидирует env/files/commands без вывода + секретов, проверяет `AW_SSH_PASSWORD`, `AW_WINRM_PASSWORD`, `sshpass`, + `ansible-playbook`, inventory и `merge_aw_server_dbs.py`; + - план сохраняет порядок legacy flow: scp merge script, remote backup dir, + DB existence checks, DB backups, stop `activitywatch-server`, merge DB, + install merged DB, 3 ansible playbooks, final AW validation; + - safe smoke: plan text OK, JSON valid (`steps=12`, `missing_count=0`), + destructive steps present and marked, password names/values не попадают в + planned commands, Rust `--apply` exits rc `1`; + - gates: `cargo fmt --all -- --check`, `cargo test -p + prod-backup-restore` (`3 passed`), `cargo clippy -p + prod-backup-restore --all-targets -- -D warnings`, release build OK, + `bash -n scripts/prod_backup_restore.sh`, artifact check OK, + `quality-gate.sh` OK, DetMir read-only OK with + `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`; + transient `service_warnings=1` disappeared on immediate + `detmir-check --json` repeat (`service_warnings=0`). +52. `[done]` Перенести `scripts/prod_rollout.sh` в Rust-first safe + planner/orchestrator: + - добавлен crate `prod-rollout`; + - `scripts/prod_rollout.sh` теперь Rust-first wrapper: + ищет `PROD_ROLLOUT_RUST`, `$CARGO_TARGET_DIR/release/prod-rollout`, + `adk-rust/target/release/prod-rollout`, `/usr/local/bin/prod-rollout`; + - обычный запуск больше не стартует production deploy автоматически: + он показывает plan-only rollout flow; + - реальный rollout через Rust доступен только явно: + `scripts/prod_rollout.sh --apply`; + - старый Bash rollout сохранен только явно: + `scripts/prod_rollout.sh --apply-legacy`; + - `--check-inputs --json` валидирует `AW_SSH_PASSWORD`, + `AW_WINRM_PASSWORD`, `git`, `ansible`, `ansible-playbook`, inventory, + `quality-gate.sh`, `deploy_aw_server.yml`, `deploy_aw_windows.yml` и + `post_validate_aw_windows.yml` без вывода секретов; + - план сохраняет legacy порядок: `quality-gate`, Ansible ping AW server, + Ansible win_ping AW Windows, `deploy_aw_server --check --diff`, real + `deploy_aw_server`, `deploy_aw_windows --check --diff`, real + `deploy_aw_windows`, `post_validate_aw_windows`; + - mutation в плане помечена только для `deploy-aw-server` и + `deploy-aw-windows`; + - apply-runner пишет stdout/stderr каждого шага в + `.rollout-logs//`, общий ход - в `rollout.log`; + - safe smoke: `scripts/prod_rollout.sh --check-inputs --json` вернул + `mode=plan-only`, `missing_count=0`, `steps=8`, mutations + `deploy-aw-server` и `deploy-aw-windows`; + - gates: `cargo fmt --all -- --check`, `cargo test -p prod-rollout` + (`3 passed`), `cargo clippy -p prod-rollout --all-targets -- -D + warnings`, release build OK, `bash -n scripts/prod_rollout.sh`, + artifact check OK. + +Отложить: + +- перенос Telegram bot runtime снят с плана: Python остается постоянным + runtime, Rust используется только для backend helpers; +- перенос оставшихся install/runtime scripts на Rust; +- переписывание PowerShell Windows collector path; +- любые network/firewall/VPN изменения; +- pfSense полностью frozen/no-touch: текущего потенциала достаточно, NAT/gateway + tooling не переносить и не дергать без отдельной явной команды. + +## 12. Stop conditions + +Миграцию остановить и откатить конкретный модуль, если: + +- новый binary дает иной `severity` без объяснимой причины; +- меняет exit code class на зеленом или красном контуре; +- пишет неполный/corrupt state; +- вызывает restart на зеленом контуре; +- ломает Telegram `/status`; +- ломает Grafana/Influx/Prometheus metric names; +- требует laptop-only path или интерактивный shell; +- выводит секреты в stdout/journald/report. + +## 13. Рабочий принцип + +Правильный перенос на Rust - это не переписывание строк один-в-один. + +Для каждого legacy script надо сделать: + +1. понять контракт; +2. зафиксировать fixture; +3. написать typed model; +4. реализовать read-only parity; +5. включить shadow-mode; +6. заменить production command; +7. только потом добавлять action/mutation; +8. удалить legacy только после периода стабильной эксплуатации. diff --git a/adk-rust/crates/aw-browser-smoke/Cargo.toml b/adk-rust/crates/aw-browser-smoke/Cargo.toml new file mode 100644 index 0000000..1645204 --- /dev/null +++ b/adk-rust/crates/aw-browser-smoke/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "aw-browser-smoke" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-browser-smoke/src/main.rs b/adk-rust/crates/aw-browser-smoke/src/main.rs new file mode 100644 index 0000000..355c22a --- /dev/null +++ b/adk-rust/crates/aw-browser-smoke/src/main.rs @@ -0,0 +1,146 @@ +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use clap::Parser; + +const PLAYWRIGHT_NODE_MODULES: &str = "/home/igor/.agents/skills/playwright/node_modules"; + +#[derive(Debug, Parser)] +#[command( + about = "Rust launcher for the ActivityWatch-Russian browser smoke test", + trailing_var_arg = true +)] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long)] + script: Option, + + #[arg(long)] + node: Option, + + #[arg(last = true)] + args: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct LaunchPlan { + node: PathBuf, + script: PathBuf, + args: Vec, + node_path: Option, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let plan = build_launch_plan(&cli)?; + run_child(&plan) +} + +fn build_launch_plan(cli: &Cli) -> Result { + let root = cli.root.clone(); + let script = cli + .script + .clone() + .unwrap_or_else(|| root.join("scripts").join("aw-webui-browser-smoke.mjs")); + if !script.is_file() { + bail!("browser smoke script not found: {}", script.display()); + } + + Ok(LaunchPlan { + node: cli.node.clone().unwrap_or_else(|| PathBuf::from("node")), + script, + args: cli.args.clone(), + node_path: default_node_path(std::env::var_os("NODE_PATH").as_deref()), + }) +} + +fn default_node_path(current: Option<&OsStr>) -> Option { + if current.is_some_and(|value| !value.is_empty()) { + return None; + } + let path = Path::new(PLAYWRIGHT_NODE_MODULES); + if path.is_dir() { + Some(path.as_os_str().to_os_string()) + } else { + None + } +} + +fn run_child(plan: &LaunchPlan) -> Result { + let mut command = Command::new(&plan.node); + command.arg(&plan.script).args(&plan.args); + if let Some(node_path) = &plan.node_path { + command.env("NODE_PATH", node_path); + } + + let status = command + .status() + .with_context(|| format!("run {}", plan.node.display()))?; + Ok(status.code().unwrap_or(1)) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + + #[test] + fn keeps_existing_node_path_untouched() { + assert_eq!( + default_node_path(Some(OsStr::new("/custom/node_modules"))), + None + ); + } + + #[test] + fn launch_plan_uses_default_script_and_passes_args() { + let dir = tempfile::tempdir().unwrap(); + let scripts = dir.path().join("scripts"); + fs::create_dir(&scripts).unwrap(); + let script = scripts.join("aw-webui-browser-smoke.mjs"); + fs::write(&script, "console.log('ok')\n").unwrap(); + + let cli = Cli { + root: dir.path().to_path_buf(), + script: None, + node: Some(PathBuf::from("/usr/bin/node")), + args: vec![OsString::from("--probe"), OsString::from("value")], + }; + + let plan = build_launch_plan(&cli).unwrap(); + assert_eq!(plan.node, PathBuf::from("/usr/bin/node")); + assert_eq!(plan.script, script); + assert_eq!( + plan.args, + vec![OsString::from("--probe"), OsString::from("value")] + ); + } + + #[test] + fn missing_script_is_an_error() { + let cli = Cli { + root: PathBuf::from("/tmp/no-such-aw-browser-root"), + script: None, + node: None, + args: vec![], + }; + let err = build_launch_plan(&cli).unwrap_err().to_string(); + assert!(err.contains("browser smoke script not found")); + } +} diff --git a/adk-rust/crates/aw-contour-smoke/Cargo.toml b/adk-rust/crates/aw-contour-smoke/Cargo.toml new file mode 100644 index 0000000..5cb1b61 --- /dev/null +++ b/adk-rust/crates/aw-contour-smoke/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aw-contour-smoke" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde_json.workspace = true +url.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-contour-smoke/src/main.rs b/adk-rust/crates/aw-contour-smoke/src/main.rs new file mode 100644 index 0000000..938c89e --- /dev/null +++ b/adk-rust/crates/aw-contour-smoke/src/main.rs @@ -0,0 +1,461 @@ +use std::net::{TcpStream, ToSocketAddrs}; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result}; +use clap::{Parser, ValueEnum}; +use reqwest::blocking::Client; +use reqwest::redirect::Policy; +use serde_json::Value; + +#[derive(Debug, Parser)] +#[command(about = "ActivityWatch-Russian contour smoke checks")] +struct Cli { + #[arg(long, value_enum, default_value_t = Mode::ProxmoxRemote)] + mode: Mode, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum Mode { + ProxmoxRemote, +} + +#[derive(Debug, Default)] +struct Counts { + ok: usize, + warn: usize, + fail: usize, + skip: usize, +} + +impl Counts { + fn pass(&mut self, msg: impl AsRef) { + self.ok += 1; + println!("[OK] {}", msg.as_ref()); + } + + fn fail(&mut self, msg: impl AsRef) { + self.fail += 1; + println!("[FAIL] {}", msg.as_ref()); + } + + fn skip(&mut self, msg: impl AsRef) { + self.skip += 1; + println!("[SKIP] {}", msg.as_ref()); + } +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + match cli.mode { + Mode::ProxmoxRemote => run_proxmox_remote(), + } +} + +fn run_proxmox_remote() -> Result { + let mut counts = Counts::default(); + let http = Client::builder() + .no_proxy() + .danger_accept_invalid_certs(true) + .timeout(Duration::from_secs(15)) + .build() + .context("build HTTP client")?; + let no_redirect_http = Client::builder() + .no_proxy() + .danger_accept_invalid_certs(true) + .timeout(Duration::from_secs(15)) + .redirect(Policy::none()) + .build() + .context("build no-redirect HTTP client")?; + + section("Host"); + print_command("hostnamectl", &["hostnamectl"]); + print_command("date", &["date", "-Is"]); + print_command("uptime", &["uptime"]); + + section("Core Services"); + for unit in [ + "nginx.service", + "pveproxy.service", + "pvedaemon.service", + "pvestatd.service", + "pve-cluster.service", + "docker.service", + "aw-1c-company-api.service", + "aw-pve-webadmin-logger.service", + ] { + check_service(&mut counts, unit); + } + + section("Timers"); + for unit in [ + "aw-1c-ingest.timer", + "aw-1c-proofcheck.timer", + "aw-1c-manager-brief.timer", + "aw-1c-recovery-brief.timer", + "aw-1c-weekly-digest.timer", + ] { + check_timer(&mut counts, unit); + } + if let Ok(out) = command_output("systemctl", &["list-timers", "--all", "--no-pager"]) { + print_filtered_lines(&out, &["aw-1c", "NEXT", "LEFT", "PASSED"], 40); + } + + section("Ports"); + check_tcp(&mut counts, "nginx http", "127.0.0.1", 80); + check_tcp(&mut counts, "nginx https", "127.0.0.1", 443); + check_tcp(&mut counts, "proxmox web", "127.0.0.1", 8006); + check_tcp(&mut counts, "1C company API", "10.10.10.2", 8710); + check_tcp(&mut counts, "clickhouse native", "127.0.0.1", 9000); + check_tcp(&mut counts, "clickhouse http", "127.0.0.1", 8123); + if let Ok(out) = command_output("ss", &["-tulpn"]) { + print_filtered_lines( + &out, + &[":80", ":443", ":8006", ":8710", ":8123", ":9000"], + 40, + ); + } + + section("Gateway HTTP"); + check_http_code( + &mut counts, + &http, + "nginx healthz", + "https://127.0.0.1/healthz", + &[200], + ); + check_http_redirect( + &mut counts, + &no_redirect_http, + "go proxmox gui", + "https://127.0.0.1/go/proxmox-gui", + &[301, 302, 307, 308], + Some("https://10.10.10.2:8006/"), + ); + check_http_redirect( + &mut counts, + &no_redirect_http, + "go file1c brief", + "https://127.0.0.1/go/file1c-brief", + &[301, 302, 307, 308], + Some("http://10.10.10.2:8710/manager/brief"), + ); + check_http_redirect( + &mut counts, + &no_redirect_http, + "go file1c actions", + "https://127.0.0.1/go/file1c-actions", + &[301, 302, 307, 308], + Some("http://10.10.10.2:8710/manager/actions"), + ); + + section("1C Company API"); + check_http_code( + &mut counts, + &no_redirect_http, + "1C root redirect", + "http://10.10.10.2:8710/", + &[307], + ); + for (name, url) in [ + ("1C /health", "http://10.10.10.2:8710/health"), + ("1C /api/health", "http://10.10.10.2:8710/api/health"), + ("1C manager brief", "http://10.10.10.2:8710/manager/brief"), + ( + "1C manager actions", + "http://10.10.10.2:8710/manager/actions", + ), + ( + "1C manager recovery", + "http://10.10.10.2:8710/manager/recovery", + ), + ( + "1C weekly digest", + "http://10.10.10.2:8710/manager/digest/weekly", + ), + ] { + check_http_code(&mut counts, &http, name, url, &[200]); + } + + section("ClickHouse"); + check_docker_container(&mut counts, "aw-rus-1c-clickhouse"); + check_http_code( + &mut counts, + &http, + "ClickHouse ping", + "http://127.0.0.1:8123/ping", + &[200], + ); + if command_exists("docker") && docker_container_running("aw-rus-1c-clickhouse") { + check_command( + &mut counts, + "ClickHouse SELECT 1", + "docker", + &[ + "exec", + "aw-rus-1c-clickhouse", + "clickhouse-client", + "--query", + "SELECT 1", + ], + ); + } + + section("System Capacity"); + print_command("df", &["df", "-h", "/", "/var", "/opt"]); + print_command("free", &["free", "-h"]); + + section("Summary"); + println!( + "OK={} WARN={} FAIL={} SKIP={}", + counts.ok, counts.warn, counts.fail, counts.skip + ); + Ok(if counts.fail > 0 { 2 } else { 0 }) +} + +fn section(name: &str) { + println!(); + println!("== {name} =="); +} + +fn command_exists(cmd: &str) -> bool { + Command::new("sh") + .arg("-c") + .arg(format!("command -v {cmd} >/dev/null 2>&1")) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn command_output(cmd: &str, args: &[&str]) -> Result { + let output = Command::new(cmd) + .args(args) + .output() + .with_context(|| format!("run {cmd}"))?; + let mut text = String::new(); + text.push_str(&String::from_utf8_lossy(&output.stdout)); + text.push_str(&String::from_utf8_lossy(&output.stderr)); + if output.status.success() { + Ok(text) + } else { + Err(anyhow::anyhow!("{cmd} failed: {text}")) + } +} + +fn print_command(label: &str, command: &[&str]) { + if let Some((cmd, args)) = command.split_first() { + match command_output(cmd, args) { + Ok(out) => print_indented(&out, 80), + Err(err) => println!(" {label}: {err:#}"), + } + } +} + +fn print_indented(text: &str, max_lines: usize) { + for line in text.lines().take(max_lines) { + println!(" {line}"); + } +} + +fn print_filtered_lines(text: &str, patterns: &[&str], max_lines: usize) { + for line in text + .lines() + .filter(|line| patterns.iter().any(|pattern| line.contains(pattern))) + .take(max_lines) + { + println!(" {line}"); + } +} + +fn check_service(counts: &mut Counts, unit: &str) { + check_systemd_unit(counts, unit, "service"); +} + +fn check_timer(counts: &mut Counts, unit: &str) { + check_systemd_unit(counts, unit, "timer"); +} + +fn check_systemd_unit(counts: &mut Counts, unit: &str, kind: &str) { + if !Command::new("systemctl") + .args(["list-unit-files", unit]) + .status() + .map(|status| status.success()) + .unwrap_or(false) + { + counts.skip(format!("{unit} is not installed")); + return; + } + if Command::new("systemctl") + .args(["is-active", "--quiet", unit]) + .status() + .map(|status| status.success()) + .unwrap_or(false) + { + counts.pass(format!("{unit} active")); + } else { + counts.fail(format!("{unit} inactive or failed")); + if let Ok(out) = command_output("systemctl", &["--no-pager", "--lines=8", "status", unit]) { + print_indented(&out, 30); + } else { + let _ = kind; + } + } +} + +fn check_tcp(counts: &mut Counts, name: &str, host: &str, port: u16) { + let addr = format!("{host}:{port}"); + let ok = addr + .to_socket_addrs() + .ok() + .and_then(|mut addrs| addrs.next()) + .and_then(|addr| TcpStream::connect_timeout(&addr, Duration::from_secs(4)).ok()) + .is_some(); + if ok { + counts.pass(format!("{name} TCP {host}:{port}")); + } else { + counts.fail(format!("{name} TCP {host}:{port}")); + } +} + +fn check_http_code(counts: &mut Counts, client: &Client, name: &str, url: &str, expected: &[u16]) { + match client.get(url).send() { + Ok(response) => { + let code = response.status().as_u16(); + if expected.contains(&code) { + counts.pass(format!("{name} HTTP {code} {url}")); + } else { + counts.fail(format!("{name} HTTP {code} {url}")); + if let Ok(text) = response.text() { + print_indented(&text, 40); + } + } + } + Err(err) => counts.fail(format!("{name} HTTP error {url}: {err}")), + } +} + +fn check_http_redirect( + counts: &mut Counts, + client: &Client, + name: &str, + url: &str, + expected: &[u16], + expected_location: Option<&str>, +) { + match client.head(url).send() { + Ok(response) => { + let code = response.status().as_u16(); + let location = response + .headers() + .get(reqwest::header::LOCATION) + .and_then(|value| value.to_str().ok()) + .unwrap_or(""); + let location_ok = expected_location.is_none_or(|expected| location.contains(expected)); + if expected.contains(&code) && location_ok { + counts.pass(format!( + "{name} HTTP {code} {}", + if location.is_empty() { url } else { location } + )); + } else { + counts.fail(format!( + "{name} HTTP {code} {}", + if location.is_empty() { url } else { location } + )); + } + } + Err(err) => counts.fail(format!("{name} HTTP error {url}: {err}")), + } +} + +fn check_command(counts: &mut Counts, name: &str, cmd: &str, args: &[&str]) { + match command_output(cmd, args) { + Ok(out) => { + counts.pass(name); + print_indented(&out, 40); + } + Err(err) => counts.fail(format!("{name}: {err:#}")), + } +} + +fn check_docker_container(counts: &mut Counts, name: &str) { + if !command_exists("docker") { + counts.skip("docker command unavailable"); + return; + } + if docker_container_running(name) { + counts.pass(format!("docker container {name} running")); + if let Ok(out) = command_output( + "docker", + &[ + "ps", + "--filter", + &format!("name=^/{name}$"), + "--format", + "{{.Names}} {{.Status}} {{.Ports}}", + ], + ) { + print_indented(&out, 20); + } + } else { + counts.fail(format!("docker container {name} not running")); + if let Ok(out) = command_output( + "docker", + &[ + "ps", + "-a", + "--filter", + &format!("name=^/{name}$"), + "--format", + "{{.Names}} {{.Status}} {{.Ports}}", + ], + ) { + print_indented(&out, 20); + } + } +} + +fn docker_container_running(name: &str) -> bool { + command_output("docker", &["ps", "--format", "{{.Names}}"]) + .map(|out| out.lines().any(|line| line == name)) + .unwrap_or(false) +} + +#[allow(dead_code)] +fn parse_json_key_present(value: &Value, key: &str) -> bool { + value.get(key).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_exit_code_matches_failures() { + let counts = Counts { + ok: 1, + warn: 1, + fail: 0, + skip: 1, + }; + assert_eq!(if counts.fail > 0 { 2 } else { 0 }, 0); + let counts = Counts { fail: 1, ..counts }; + assert_eq!(if counts.fail > 0 { 2 } else { 0 }, 2); + } + + #[test] + fn default_counts_are_zero() { + let counts = Counts::default(); + assert_eq!(counts.ok + counts.warn + counts.fail + counts.skip, 0); + } +} diff --git a/adk-rust/crates/aw-db-health/Cargo.toml b/adk-rust/crates/aw-db-health/Cargo.toml new file mode 100644 index 0000000..3d1eae0 --- /dev/null +++ b/adk-rust/crates/aw-db-health/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aw-db-health" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-db-health/src/main.rs b/adk-rust/crates/aw-db-health/src/main.rs new file mode 100644 index 0000000..0a6e440 --- /dev/null +++ b/adk-rust/crates/aw-db-health/src/main.rs @@ -0,0 +1,562 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow}; +use chrono::Utc; +use clap::Parser; +use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; +use serde::Serialize; +use serde_json::{Value, json}; + +const DEFAULT_DB_PATH: &str = "/var/lib/activitywatch/aw-server-rust/sqlite.db"; +const DEFAULT_HOST: &str = "SHARKON2025"; + +#[derive(Debug, Parser)] +#[command(author, version, about = "Read-only ActivityWatch SQLite growth guard")] +struct Cli { + #[arg(long, default_value = DEFAULT_DB_PATH, env = "AW_DB_HEALTH_DB_PATH")] + db_path: PathBuf, + + #[arg(long, env = "AW_DB_HEALTH_SESSION_BUCKET")] + session_bucket: Option, + + #[arg(long, env = "AW_WORKTIME_HOST")] + host: Option, + + #[arg(long, env = "AW_DB_HEALTH_WINDOWS_CONFIG")] + windows_config: Option, + + #[arg(long, default_value_t = gib(2), env = "AW_DB_HEALTH_DB_WARN_BYTES")] + db_warn_bytes: u64, + + #[arg(long, default_value_t = gib(5), env = "AW_DB_HEALTH_DB_FAIL_BYTES")] + db_fail_bytes: u64, + + #[arg(long, default_value_t = mib(256), env = "AW_DB_HEALTH_WAL_WARN_BYTES")] + wal_warn_bytes: u64, + + #[arg(long, default_value_t = gib(1), env = "AW_DB_HEALTH_WAL_FAIL_BYTES")] + wal_fail_bytes: u64, + + #[arg(long, default_value_t = 10_000, env = "AW_DB_HEALTH_SESSION_ROWS_WARN")] + session_rows_warn: i64, + + #[arg( + long, + default_value_t = 100_000, + env = "AW_DB_HEALTH_SESSION_ROWS_FAIL" + )] + session_rows_fail: i64, + + #[arg( + long, + default_value_t = 600, + env = "AW_DB_HEALTH_RECENT_PROCESS_WINDOW_SECONDS" + )] + recent_process_window_seconds: i64, + + #[arg(long, default_value_t = 1, env = "AW_DB_HEALTH_RECENT_PROCESS_WARN")] + recent_process_warn: i64, + + #[arg(long, default_value_t = 100, env = "AW_DB_HEALTH_RECENT_PROCESS_FAIL")] + recent_process_fail: i64, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +enum Status { + Ok, + Warn, + Fail, +} + +#[derive(Debug, Serialize)] +struct CheckResult { + name: String, + status: Status, + summary: String, + details: Value, +} + +#[derive(Debug, Serialize)] +struct Report { + ok: bool, + generated_at_utc: String, + counts: Counts, + results: Vec, +} + +#[derive(Debug, Default, Serialize)] +struct Counts { + ok: usize, + warn: usize, + fail: usize, +} + +const fn mib(value: u64) -> u64 { + value * 1024 * 1024 +} + +const fn gib(value: u64) -> u64 { + value * 1024 * 1024 * 1024 +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let report = build_report(&cli)?; + if cli.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print_text(&report); + } + Ok(if report.ok { 0 } else { 2 }) +} + +fn build_report(cli: &Cli) -> Result { + let mut results = Vec::new(); + let db_path = &cli.db_path; + let wal_path = PathBuf::from(format!("{}-wal", db_path.display())); + let shm_path = PathBuf::from(format!("{}-shm", db_path.display())); + + let db_size = file_size(db_path)?; + results.push(threshold_result( + "sqlite:file-size", + db_size, + cli.db_warn_bytes, + cli.db_fail_bytes, + "ActivityWatch SQLite DB size", + json!({ + "path": db_path, + "size_bytes": db_size, + "warn_bytes": cli.db_warn_bytes, + "fail_bytes": cli.db_fail_bytes, + }), + )); + + let wal_size = file_size_optional(&wal_path)?; + results.push(threshold_result( + "sqlite:wal-size", + wal_size, + cli.wal_warn_bytes, + cli.wal_fail_bytes, + "ActivityWatch SQLite WAL size", + json!({ + "path": wal_path, + "size_bytes": wal_size, + "warn_bytes": cli.wal_warn_bytes, + "fail_bytes": cli.wal_fail_bytes, + }), + )); + + let shm_size = file_size_optional(&shm_path)?; + results.push(CheckResult { + name: "sqlite:shm-size".to_string(), + status: Status::Ok, + summary: format!("SHM size is {}", human_bytes(shm_size)), + details: json!({ + "path": shm_path, + "size_bytes": shm_size, + }), + }); + + let conn = open_readonly(db_path)?; + let bucket = cli.session_bucket.clone().unwrap_or_else(|| { + format!( + "aw-session-events_{}", + cli.host.as_deref().unwrap_or(DEFAULT_HOST) + ) + }); + match bucket_row(&conn, &bucket)? { + Some(bucketrow) => { + let total_rows = count_session_rows(&conn, bucketrow)?; + results.push(threshold_result_i64( + "aw-session-events:rows", + total_rows, + cli.session_rows_warn, + cli.session_rows_fail, + "aw-session-events row count", + json!({ + "bucket": bucket, + "bucketrow": bucketrow, + "rows": total_rows, + "warn_rows": cli.session_rows_warn, + "fail_rows": cli.session_rows_fail, + }), + )); + + let cutoff_ns = now_ns()? - cli.recent_process_window_seconds.max(1) * 1_000_000_000; + let recent_process = count_recent_process_events(&conn, bucketrow, cutoff_ns)?; + results.push(threshold_result_i64( + "aw-session-events:recent-process-events", + recent_process, + cli.recent_process_warn, + cli.recent_process_fail, + "recent process-level aw-session-events", + json!({ + "bucket": bucket, + "bucketrow": bucketrow, + "recent_process_events": recent_process, + "window_seconds": cli.recent_process_window_seconds, + "cutoff_ns": cutoff_ns, + "warn_events": cli.recent_process_warn, + "fail_events": cli.recent_process_fail, + }), + )); + + let latest = latest_session_event(&conn, bucketrow)?; + results.push(CheckResult { + name: "aw-session-events:latest".to_string(), + status: Status::Ok, + summary: latest + .as_ref() + .map(|event| format!("latest eventType={}", event.event_type)) + .unwrap_or_else(|| "no session events".to_string()), + details: json!({ + "bucket": bucket, + "latest": latest, + }), + }); + } + None => { + results.push(CheckResult { + name: "aw-session-events:bucket".to_string(), + status: Status::Warn, + summary: format!("bucket {bucket} not found"), + details: json!({ "bucket": bucket }), + }); + } + } + + if let Some(path) = &cli.windows_config { + results.push(check_windows_config(path)?); + } + + let counts = count_statuses(&results); + Ok(Report { + ok: counts.fail == 0, + generated_at_utc: Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + counts, + results, + }) +} + +fn open_readonly(path: &Path) -> Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("open SQLite DB read-only: {}", path.display()))?; + conn.busy_timeout(std::time::Duration::from_secs(5))?; + Ok(conn) +} + +fn bucket_row(conn: &Connection, bucket: &str) -> Result> { + conn.query_row("SELECT id FROM buckets WHERE name=?", [bucket], |row| { + row.get(0) + }) + .optional() + .context("lookup bucket row") +} + +fn count_session_rows(conn: &Connection, bucketrow: i64) -> Result { + conn.query_row( + "SELECT COUNT(*) FROM events WHERE bucketrow=?", + [bucketrow], + |row| row.get(0), + ) + .context("count session event rows") +} + +fn count_recent_process_events(conn: &Connection, bucketrow: i64, cutoff_ns: i64) -> Result { + conn.query_row( + r#" + SELECT COUNT(*) + FROM events + WHERE bucketrow=? + AND endtime >= ? + AND (data LIKE ? OR data LIKE ?) + "#, + params![ + bucketrow, + cutoff_ns, + "%\"eventType\":\"process_start\"%", + "%\"eventType\":\"process_stop\"%", + ], + |row| row.get(0), + ) + .context("count recent process-level session events") +} + +#[derive(Debug, Serialize)] +struct LatestEvent { + id: i64, + endtime_ns: i64, + event_type: String, + source: String, +} + +fn latest_session_event(conn: &Connection, bucketrow: i64) -> Result> { + let row = conn + .query_row( + "SELECT id, endtime, data FROM events WHERE bucketrow=? ORDER BY id DESC LIMIT 1", + [bucketrow], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional() + .context("read latest session event")?; + Ok(row.map(|(id, endtime_ns, data)| LatestEvent { + id, + endtime_ns, + event_type: json_field(&data, "eventType").unwrap_or_else(|| "unknown".to_string()), + source: json_field(&data, "source").unwrap_or_else(|| "unknown".to_string()), + })) +} + +fn check_windows_config(path: &Path) -> Result { + let text = fs::read_to_string(path) + .with_context(|| format!("read Windows deployment config {}", path.display()))?; + let payload: Value = serde_json::from_str(&text).context("parse Windows deployment config")?; + let enabled = payload + .pointer("/sessionEvents/processEventsEnabled") + .and_then(Value::as_bool); + let status = match enabled { + Some(false) => Status::Ok, + Some(true) => Status::Fail, + None => Status::Warn, + }; + let summary = match enabled { + Some(false) => "processEventsEnabled=false".to_string(), + Some(true) => { + "processEventsEnabled=true; high-volume process stream is enabled".to_string() + } + None => "processEventsEnabled missing".to_string(), + }; + Ok(CheckResult { + name: "windows-config:process-events".to_string(), + status, + summary, + details: json!({ + "path": path, + "processEventsEnabled": enabled, + }), + }) +} + +fn threshold_result( + name: &str, + value: u64, + warn: u64, + fail: u64, + label: &str, + details: Value, +) -> CheckResult { + let status = status_for_u64(value, warn, fail); + CheckResult { + name: name.to_string(), + status, + summary: format!("{label}: {}", human_bytes(value)), + details, + } +} + +fn threshold_result_i64( + name: &str, + value: i64, + warn: i64, + fail: i64, + label: &str, + details: Value, +) -> CheckResult { + let status = status_for_i64(value, warn, fail); + CheckResult { + name: name.to_string(), + status, + summary: format!("{label}: {value}"), + details, + } +} + +fn status_for_u64(value: u64, warn: u64, fail: u64) -> Status { + if value >= fail { + Status::Fail + } else if value >= warn { + Status::Warn + } else { + Status::Ok + } +} + +fn status_for_i64(value: i64, warn: i64, fail: i64) -> Status { + if value >= fail { + Status::Fail + } else if value >= warn { + Status::Warn + } else { + Status::Ok + } +} + +fn count_statuses(results: &[CheckResult]) -> Counts { + let mut counts = Counts::default(); + for result in results { + match result.status { + Status::Ok => counts.ok += 1, + Status::Warn => counts.warn += 1, + Status::Fail => counts.fail += 1, + } + } + counts +} + +fn file_size(path: &Path) -> Result { + Ok(fs::metadata(path) + .with_context(|| format!("stat {}", path.display()))? + .len()) +} + +fn file_size_optional(path: &Path) -> Result { + match fs::metadata(path) { + Ok(metadata) => Ok(metadata.len()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(err) => Err(err).with_context(|| format!("stat {}", path.display())), + } +} + +fn now_ns() -> Result { + Utc::now() + .timestamp_nanos_opt() + .ok_or_else(|| anyhow!("current timestamp out of range")) +} + +fn json_field(data: &str, key: &str) -> Option { + let value = serde_json::from_str::(data).ok()?; + value.get(key)?.as_str().map(ToString::to_string) +} + +fn human_bytes(value: u64) -> String { + const UNITS: &[&str] = &["B", "KiB", "MiB", "GiB", "TiB"]; + let mut size = value as f64; + let mut unit = 0usize; + while size >= 1024.0 && unit + 1 < UNITS.len() { + size /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{value} {}", UNITS[unit]) + } else { + format!("{size:.1} {}", UNITS[unit]) + } +} + +fn print_text(report: &Report) { + println!("=== AW DB Health ==="); + println!("Timestamp: {}", report.generated_at_utc); + for result in &report.results { + let mark = match result.status { + Status::Ok => "✓", + Status::Warn => "⚠", + Status::Fail => "✗", + }; + println!("{mark} {}: {}", result.name, result.summary); + } + println!( + "Summary: ok={} warn={} fail={}", + report.counts.ok, report.counts.warn, report.counts.fail + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::params; + + #[test] + fn classifies_thresholds() { + assert_eq!(status_for_u64(10, 20, 30), Status::Ok); + assert_eq!(status_for_u64(20, 20, 30), Status::Warn); + assert_eq!(status_for_u64(30, 20, 30), Status::Fail); + assert_eq!(status_for_i64(0, 1, 100), Status::Ok); + assert_eq!(status_for_i64(1, 1, 100), Status::Warn); + assert_eq!(status_for_i64(101, 0, 100), Status::Fail); + } + + #[test] + fn reads_session_metrics() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + let conn = Connection::open(tmp.path()).unwrap(); + conn.execute( + "CREATE TABLE buckets (id INTEGER PRIMARY KEY, name TEXT)", + [], + ) + .unwrap(); + conn.execute( + "CREATE TABLE events (id INTEGER PRIMARY KEY, bucketrow INTEGER, endtime INTEGER, data TEXT)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO buckets (id, name) VALUES (15, 'aw-session-events_TEST')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO events (bucketrow, endtime, data) VALUES (15, ?, ?)", + params![ + now_ns().unwrap(), + r#"{"eventType":"process_start","source":"worktime-session-collector"}"# + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO events (bucketrow, endtime, data) VALUES (15, ?, ?)", + params![ + now_ns().unwrap(), + r#"{"eventType":"logon","source":"launch-watchers-awatch-rus"}"# + ], + ) + .unwrap(); + drop(conn); + + let conn = open_readonly(tmp.path()).unwrap(); + let bucketrow = bucket_row(&conn, "aw-session-events_TEST") + .unwrap() + .unwrap(); + assert_eq!(count_session_rows(&conn, bucketrow).unwrap(), 2); + assert_eq!( + count_recent_process_events(&conn, bucketrow, now_ns().unwrap() - 60_000_000_000) + .unwrap(), + 1 + ); + let latest = latest_session_event(&conn, bucketrow).unwrap().unwrap(); + assert_eq!(latest.event_type, "logon"); + } + + #[test] + fn checks_windows_config_flag() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + fs::write( + tmp.path(), + r#"{"sessionEvents":{"processEventsEnabled":false}}"#, + ) + .unwrap(); + let result = check_windows_config(tmp.path()).unwrap(); + assert_eq!(result.status, Status::Ok); + } +} diff --git a/adk-rust/crates/aw-db-maintenance/Cargo.toml b/adk-rust/crates/aw-db-maintenance/Cargo.toml new file mode 100644 index 0000000..7a4ecf7 --- /dev/null +++ b/adk-rust/crates/aw-db-maintenance/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aw-db-maintenance" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +rusqlite = { workspace = true, features = ["backup"] } +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-db-maintenance/src/main.rs b/adk-rust/crates/aw-db-maintenance/src/main.rs new file mode 100644 index 0000000..6a03156 --- /dev/null +++ b/adk-rust/crates/aw-db-maintenance/src/main.rs @@ -0,0 +1,359 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use rusqlite::{Connection, DatabaseName, OpenFlags, params}; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_DB_PATH: &str = "/var/lib/activitywatch/aw-server-rust/sqlite.db"; +const DEFAULT_BACKUP_DIR: &str = "/var/lib/activitywatch/backups/db"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const ALLOWED_EVENT_TYPES: &[&str] = &["process_start", "process_stop"]; + +#[derive(Debug, Parser)] +#[command(about = "Safe ActivityWatch SQLite maintenance for old process-level session events")] +struct Cli { + #[arg(long, default_value = DEFAULT_DB_PATH, env = "AW_DB_MAINTENANCE_DB_PATH")] + db_path: PathBuf, + + #[arg(long, default_value = DEFAULT_BACKUP_DIR, env = "AW_DB_MAINTENANCE_BACKUP_DIR")] + backup_dir: PathBuf, + + #[arg(long, env = "AW_DB_MAINTENANCE_SESSION_BUCKET")] + session_bucket: Option, + + #[arg(long, env = "AW_WORKTIME_HOST")] + host: Option, + + #[arg(long, default_value_t = 7, env = "AW_DB_MAINTENANCE_RETENTION_DAYS")] + retention_days: i64, + + #[arg(long, default_value_t = 1000, env = "AW_DB_MAINTENANCE_CHUNK_SIZE")] + chunk_size: usize, + + #[arg(long)] + apply: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Serialize)] +struct Report { + apply: bool, + generated_at_utc: String, + db_path: String, + backup_path: Option, + bucket: String, + bucketrow: Option, + retention_days: i64, + cutoff_ns: i64, + allowed_event_types: Vec<&'static str>, + planned_delete_rows: usize, + deleted_rows: usize, + backup_created: bool, + skipped_reason: Option, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let report = build_report(&cli)?; + if cli.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print_text(&report); + } + Ok(0) +} + +fn build_report(cli: &Cli) -> Result { + if cli.retention_days < 1 { + bail!("retention_days must be >= 1"); + } + if cli.chunk_size == 0 { + bail!("chunk_size must be > 0"); + } + let bucket = cli.session_bucket.clone().unwrap_or_else(|| { + format!( + "aw-session-events_{}", + cli.host.as_deref().unwrap_or(DEFAULT_HOST) + ) + }); + let cutoff_ns = (Utc::now().timestamp() - cli.retention_days * 86_400) * 1_000_000_000; + let conn = open_connection(&cli.db_path, cli.apply)?; + let bucketrow = bucket_row(&conn, &bucket)?; + let Some(bucketrow) = bucketrow else { + return Ok(base_report( + cli, + bucket, + None, + cutoff_ns, + 0, + 0, + None, + false, + Some("session bucket not found".to_string()), + )); + }; + + let delete_ids = find_deletable_event_ids(&conn, bucketrow, cutoff_ns)?; + let planned = delete_ids.len(); + let mut backup_file = None; + let mut backup_created = false; + let mut deleted = 0; + if cli.apply && planned > 0 { + fs::create_dir_all(&cli.backup_dir) + .with_context(|| format!("create backup dir {}", cli.backup_dir.display()))?; + let backup = backup_path(&cli.backup_dir); + copy_sqlite_via_backup(&cli.db_path, &backup)?; + backup_file = Some(backup); + backup_created = true; + deleted = delete_events(&conn, &delete_ids, cli.chunk_size)?; + } + + Ok(base_report( + cli, + bucket, + Some(bucketrow), + cutoff_ns, + planned, + deleted, + backup_file, + backup_created, + None, + )) +} + +#[allow(clippy::too_many_arguments)] +fn base_report( + cli: &Cli, + bucket: String, + bucketrow: Option, + cutoff_ns: i64, + planned_delete_rows: usize, + deleted_rows: usize, + backup_path: Option, + backup_created: bool, + skipped_reason: Option, +) -> Report { + Report { + apply: cli.apply, + generated_at_utc: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + db_path: cli.db_path.display().to_string(), + backup_path: backup_path.map(|path| path.display().to_string()), + bucket, + bucketrow, + retention_days: cli.retention_days, + cutoff_ns, + allowed_event_types: ALLOWED_EVENT_TYPES.to_vec(), + planned_delete_rows, + deleted_rows, + backup_created, + skipped_reason, + } +} + +fn open_connection(path: &Path, writable: bool) -> Result { + let flags = if writable { + OpenFlags::SQLITE_OPEN_READ_WRITE + } else { + OpenFlags::SQLITE_OPEN_READ_ONLY + }; + let conn = Connection::open_with_flags(path, flags) + .with_context(|| format!("open SQLite DB {}", path.display()))?; + conn.busy_timeout(Duration::from_secs(10))?; + Ok(conn) +} + +fn bucket_row(conn: &Connection, bucket: &str) -> Result> { + let mut stmt = + conn.prepare("select rowid from buckets where name = ?1 order by rowid limit 1")?; + let mut rows = stmt.query([bucket])?; + Ok(rows.next()?.map(|row| row.get::<_, i64>(0)).transpose()?) +} + +fn find_deletable_event_ids(conn: &Connection, bucketrow: i64, cutoff_ns: i64) -> Result> { + let mut stmt = conn + .prepare("select id, data from events where bucketrow = ?1 and endtime < ?2 order by id")?; + let rows = stmt.query_map(params![bucketrow, cutoff_ns], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })?; + let mut ids = Vec::new(); + for row in rows { + let (id, data) = row?; + if is_allowed_process_event(&data) { + ids.push(id); + } + } + Ok(ids) +} + +fn is_allowed_process_event(data: &str) -> bool { + let Ok(value) = serde_json::from_str::(data) else { + return false; + }; + let event_type = value + .pointer("/eventType") + .or_else(|| value.pointer("/data/eventType")) + .and_then(Value::as_str); + event_type.is_some_and(|event_type| ALLOWED_EVENT_TYPES.contains(&event_type)) +} + +fn copy_sqlite_via_backup(src: &Path, dst: &Path) -> Result<()> { + let source = + Connection::open(src).with_context(|| format!("open backup source {}", src.display()))?; + source + .backup(DatabaseName::Main, dst, None) + .with_context(|| format!("backup {} to {}", src.display(), dst.display())) +} + +fn delete_events(conn: &Connection, ids: &[i64], chunk_size: usize) -> Result { + let mut deleted = 0; + for chunk in ids.chunks(chunk_size) { + let tx = conn.unchecked_transaction()?; + { + let mut stmt = tx.prepare("delete from events where id = ?1")?; + for id in chunk { + deleted += stmt.execute([id])?; + } + } + tx.commit()?; + } + Ok(deleted) +} + +fn backup_path(backup_dir: &Path) -> PathBuf { + backup_dir.join(format!( + "aw-sqlite-before-db-maintenance-{}.db", + Utc::now().format("%Y%m%dT%H%M%SZ") + )) +} + +fn print_text(report: &Report) { + println!( + "aw-db-maintenance: {}", + if report.apply { "apply" } else { "dry-run" } + ); + println!("db_path: {}", report.db_path); + println!("bucket: {}", report.bucket); + println!("retention_days: {}", report.retention_days); + println!("planned_delete_rows: {}", report.planned_delete_rows); + println!("deleted_rows: {}", report.deleted_rows); + println!("backup_created: {}", report.backup_created); + if let Some(path) = &report.backup_path { + println!("backup_path: {path}"); + } + if let Some(reason) = &report.skipped_reason { + println!("skipped_reason: {reason}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_allows_process_start_stop_events() { + assert!(is_allowed_process_event(r#"{"eventType":"process_start"}"#)); + assert!(is_allowed_process_event( + r#"{"data":{"eventType":"process_stop"}}"# + )); + assert!(!is_allowed_process_event(r#"{"eventType":"logon"}"#)); + assert!(!is_allowed_process_event(r#"not-json"#)); + } + + #[test] + fn dry_run_does_not_delete_or_backup() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("aw.db"); + create_fixture_db(&db); + let cli = Cli { + db_path: db.clone(), + backup_dir: dir.path().join("backups"), + session_bucket: Some("aw-session-events_TEST".to_string()), + host: None, + retention_days: 7, + chunk_size: 100, + apply: false, + json: true, + }; + let report = build_report(&cli).unwrap(); + assert_eq!(report.planned_delete_rows, 2); + assert_eq!(report.deleted_rows, 0); + assert!(!report.backup_created); + assert_eq!(count_events(&db), 3); + } + + #[test] + fn apply_deletes_only_old_process_events_and_keeps_logon() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("aw.db"); + create_fixture_db(&db); + let cli = Cli { + db_path: db.clone(), + backup_dir: dir.path().join("backups"), + session_bucket: Some("aw-session-events_TEST".to_string()), + host: None, + retention_days: 7, + chunk_size: 1, + apply: true, + json: true, + }; + let report = build_report(&cli).unwrap(); + assert_eq!(report.planned_delete_rows, 2); + assert_eq!(report.deleted_rows, 2); + assert!(report.backup_created); + assert_eq!(count_events(&db), 1); + } + + fn create_fixture_db(path: &Path) { + let conn = Connection::open(path).unwrap(); + conn.execute_batch( + "create table buckets (name text); + create table events (id integer primary key autoincrement, bucketrow integer, starttime integer, endtime integer, data text);", + ) + .unwrap(); + conn.execute( + "insert into buckets (name) values ('aw-session-events_TEST')", + [], + ) + .unwrap(); + conn.execute( + "insert into events (bucketrow,starttime,endtime,data) values (1,1,2,'{\"eventType\":\"process_start\"}')", + [], + ) + .unwrap(); + conn.execute( + "insert into events (bucketrow,starttime,endtime,data) values (1,3,4,'{\"eventType\":\"process_stop\"}')", + [], + ) + .unwrap(); + conn.execute( + "insert into events (bucketrow,starttime,endtime,data) values (1,5,6,'{\"eventType\":\"logon\"}')", + [], + ) + .unwrap(); + } + + fn count_events(path: &Path) -> i64 { + Connection::open(path) + .unwrap() + .query_row("select count(*) from events", [], |row| row.get(0)) + .unwrap() + } +} diff --git a/adk-rust/crates/aw-health-check/Cargo.toml b/adk-rust/crates/aw-health-check/Cargo.toml new file mode 100644 index 0000000..4ee4a9e --- /dev/null +++ b/adk-rust/crates/aw-health-check/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "aw-health-check" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +reqwest.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-health-check/src/main.rs b/adk-rust/crates/aw-health-check/src/main.rs new file mode 100644 index 0000000..4e01c57 --- /dev/null +++ b/adk-rust/crates/aw-health-check/src/main.rs @@ -0,0 +1,495 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use reqwest::blocking::Client; +use serde_json::Value; + +const ENV_PATH: &str = "/etc/activitywatch/aw-server.env"; +const SERVICES: &[&str] = &[ + "activitywatch-server", + "aw-worktime-api", + "aw-worktime-ui-bridge", +]; +const DLP_TRANSPORT_CHECKS: &[&str] = &[ + "buckets:endpoint-signals", + "buckets:file-operations", + "endpoint-self-test-metrics", +]; +const AW_DB_HEALTH_CHECKS: &[&str] = &[ + "sqlite:file-size", + "sqlite:wal-size", + "aw-session-events:rows", + "aw-session-events:recent-process-events", + "windows-config:process-events", +]; + +#[derive(Debug, Default)] +struct HealthState { + unhealthy: Vec, + warnings: Vec, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let env = load_effective_env(Path::new(ENV_PATH))?; + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .no_proxy() + .build() + .context("build HTTP client")?; + let mut state = HealthState::default(); + + println!("=== AW Services Health Check ==="); + println!( + "Timestamp: {}", + command_stdout("date", &[]).unwrap_or_else(|_| "unknown".to_string()) + ); + println!(); + + for service in SERVICES { + check_service(&mut state, service)?; + } + println!(); + + let worktime_base = env_first( + &env, + "AW_RUS_HEALTH_WORKTIME_API", + &env_first(&env, "AW_WORKTIME_REPORT_BASE", "http://127.0.0.1:5610"), + ); + let worktime_url = format!("{}/health", worktime_base.trim_end_matches('/')); + let worktime_timeout = env_i64(&env, "AW_RUS_HEALTH_WORKTIME_TIMEOUT_SECONDS", 15); + let aw_timeout = env_i64(&env, "AW_RUS_HEALTH_AW_TIMEOUT_SECONDS", 15); + let aw_attempts = env_i64(&env, "AW_RUS_HEALTH_AW_ATTEMPTS", 3); + let settings_timeout = env_i64(&env, "AW_RUS_HEALTH_SETTINGS_TIMEOUT_SECONDS", 15); + let settings_attempts = env_i64(&env, "AW_RUS_HEALTH_SETTINGS_ATTEMPTS", 3); + + check_api_endpoint( + &client, + &mut state, + "http://127.0.0.1:5600/api/0/info", + "activitywatch-server", + aw_timeout, + aw_attempts, + ); + check_api_endpoint( + &client, + &mut state, + &worktime_url, + "aw-worktime-api", + worktime_timeout, + 2, + ); + check_dlp_transport_freshness(&mut state, &env); + check_aw_db_health(&mut state, &env); + check_expected_setting( + &client, + &mut state, + "startOfDay", + env_first(&env, "AW_EXPECT_START_OF_DAY", ""), + "startOfDay", + settings_timeout, + settings_attempts, + ); + check_expected_setting( + &client, + &mut state, + "always_active_pattern", + env_first(&env, "AW_EXPECT_ALWAYS_ACTIVE_PATTERN", ""), + "always_active_pattern", + settings_timeout, + settings_attempts, + ); + check_expected_setting( + &client, + &mut state, + "landingpage", + env_first(&env, "AW_EXPECT_LANDINGPAGE", ""), + "landingpage", + settings_timeout, + settings_attempts, + ); + + println!(); + if state.unhealthy.is_empty() { + println!("✓ All services are healthy"); + if !state.warnings.is_empty() { + println!("⚠ Warnings: {}", state.warnings.join(" ")); + } + Ok(0) + } else { + println!("✗ Unhealthy services: {}", state.unhealthy.join(" ")); + Ok(1) + } +} + +fn check_service(state: &mut HealthState, service: &str) -> Result<()> { + if service == "aw-worktime-ui-bridge" { + let active = systemctl_success(&["is-active", "--quiet", "aw-worktime-ui-bridge.timer"])?; + let enabled = systemctl_success(&["is-enabled", "--quiet", "aw-worktime-ui-bridge.timer"])?; + if active && enabled { + println!("✓ aw-worktime-ui-bridge.timer is running and enabled"); + } else { + println!("✗ aw-worktime-ui-bridge.timer is not active/enabled"); + state + .unhealthy + .push("aw-worktime-ui-bridge.timer".to_string()); + } + return Ok(()); + } + + if systemctl_success(&["is-active", "--quiet", service])? { + println!("✓ {service} is running"); + } else { + println!("✗ {service} is not running"); + state.unhealthy.push(service.to_string()); + } + Ok(()) +} + +fn systemctl_success(args: &[&str]) -> Result { + Ok(Command::new("systemctl") + .args(args) + .status() + .with_context(|| format!("run systemctl {}", args.join(" ")))? + .success()) +} + +fn check_api_endpoint( + client: &Client, + state: &mut HealthState, + url: &str, + service_name: &str, + timeout_seconds: i64, + attempts: i64, +) { + for attempt in 1..=attempts.max(1) { + let result = client + .get(url) + .timeout(Duration::from_secs(timeout_seconds.max(1) as u64)) + .send() + .and_then(|resp| resp.error_for_status()) + .map(|_| ()); + if result.is_ok() { + println!("✓ {service_name} API endpoint is responding"); + return; + } + if attempt < attempts { + std::thread::sleep(Duration::from_secs(1)); + } + } + println!("✗ {service_name} API endpoint is not responding"); + state.unhealthy.push(format!("{service_name}-api")); +} + +fn check_expected_setting( + client: &Client, + state: &mut HealthState, + key: &str, + expected: String, + label: &str, + timeout_seconds: i64, + attempts: i64, +) { + if expected.is_empty() { + println!("⚠ expected value for {label} is not configured, skipping drift check"); + state.warnings.push(format!("{key}-expected-missing")); + return; + } + + match read_setting_value(client, key, timeout_seconds, attempts) { + Ok(actual) if actual == expected => { + println!("✓ {label} matches expected value ({expected})"); + } + Ok(actual) => { + println!("✗ {label} drift detected: actual='{actual}' expected='{expected}'"); + state.unhealthy.push(format!("setting-{key}")); + } + Err(_) => { + println!("✗ failed to read setting {label}"); + state.unhealthy.push(format!("setting-{key}")); + } + } +} + +fn read_setting_value( + client: &Client, + key: &str, + timeout_seconds: i64, + attempts: i64, +) -> Result { + let url = format!("http://127.0.0.1:5600/api/0/settings/{key}"); + for attempt in 1..=attempts.max(1) { + let result = client + .get(&url) + .timeout(Duration::from_secs(timeout_seconds.max(1) as u64)) + .send() + .and_then(|resp| resp.error_for_status()) + .and_then(|resp| resp.json::()); + match result { + Ok(value) => return Ok(json_value_to_shell_print(value)), + Err(err) if attempt >= attempts => return Err(anyhow!(err)), + Err(_) => std::thread::sleep(Duration::from_secs(1)), + } + } + Err(anyhow!("setting read exhausted")) +} + +fn check_dlp_transport_freshness(state: &mut HealthState, env: &HashMap) { + let dlp_health = env_first(env, "DLP_HEALTH_BIN", "/usr/local/bin/dlp-health-check"); + if !is_executable(Path::new(&dlp_health)) { + println!("⚠ dlp-health-check is not available, skipping DLP transport freshness checks"); + state.warnings.push("dlp-health-check-missing".to_string()); + return; + } + + let output = match Command::new(&dlp_health).arg("--json").output() { + Ok(output) => output, + Err(_) => { + println!( + "⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks" + ); + state.warnings.push("dlp-health-check-empty".to_string()); + return; + } + }; + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if stdout.is_empty() { + println!("⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks"); + state.warnings.push("dlp-health-check-empty".to_string()); + return; + } + let payload = match serde_json::from_str::(&stdout) { + Ok(payload) => payload, + Err(_) => { + println!( + "⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks" + ); + state.warnings.push("dlp-health-check-empty".to_string()); + return; + } + }; + + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for result in payload + .get("results") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let name = result.get("name").and_then(Value::as_str).unwrap_or(""); + if !DLP_TRANSPORT_CHECKS.contains(&name) { + continue; + } + let status = result.get("status").and_then(Value::as_str).unwrap_or(""); + let summary = result.get("summary").and_then(Value::as_str).unwrap_or(""); + if status == "fail" { + errors.push(format!("{name}:{summary}")); + } else if status == "warn" { + warnings.push(format!("{name}:{summary}")); + } + } + + if errors.is_empty() { + println!("✓ DLP transport freshness check passed"); + } else { + println!("✗ DLP transport freshness check failed"); + state.unhealthy.push("dlp-transport".to_string()); + } + if !errors.is_empty() { + println!(" errors: {}", errors.join(", ")); + } + if !warnings.is_empty() { + let text = warnings.join(", "); + println!(" warnings: {text}"); + state.warnings.push(text); + } +} + +fn check_aw_db_health(state: &mut HealthState, env: &HashMap) { + let aw_db_health = env_first(env, "AW_DB_HEALTH_BIN", "/usr/local/bin/aw-db-health"); + if !is_executable(Path::new(&aw_db_health)) { + println!("⚠ aw-db-health is not available, skipping AW DB growth checks"); + state.warnings.push("aw-db-health-missing".to_string()); + return; + } + + let output = match Command::new(&aw_db_health).arg("--json").output() { + Ok(output) => output, + Err(_) => { + println!("⚠ aw-db-health did not return JSON, skipping AW DB growth checks"); + state.warnings.push("aw-db-health-empty".to_string()); + return; + } + }; + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if stdout.is_empty() { + println!("⚠ aw-db-health did not return JSON, skipping AW DB growth checks"); + state.warnings.push("aw-db-health-empty".to_string()); + return; + } + let payload = match serde_json::from_str::(&stdout) { + Ok(payload) => payload, + Err(_) => { + println!("⚠ aw-db-health did not return JSON, skipping AW DB growth checks"); + state.warnings.push("aw-db-health-empty".to_string()); + return; + } + }; + + let mut errors = Vec::new(); + let mut warnings = Vec::new(); + for result in payload + .get("results") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let name = result.get("name").and_then(Value::as_str).unwrap_or(""); + if !AW_DB_HEALTH_CHECKS.contains(&name) { + continue; + } + let status = result.get("status").and_then(Value::as_str).unwrap_or(""); + let summary = result.get("summary").and_then(Value::as_str).unwrap_or(""); + if status == "fail" { + errors.push(format!("{name}:{summary}")); + } else if status == "warn" { + warnings.push(format!("{name}:{summary}")); + } + } + + if errors.is_empty() { + println!("✓ AW DB growth guard passed"); + } else { + println!("✗ AW DB growth guard failed"); + state.unhealthy.push("aw-db-health".to_string()); + } + if !errors.is_empty() { + println!(" errors: {}", errors.join(", ")); + } + if !warnings.is_empty() { + let text = warnings.join(", "); + println!(" warnings: {text}"); + state.warnings.push(text); + } +} + +fn command_stdout(program: &str, args: &[&str]) -> Result { + let output = Command::new(program).args(args).output()?; + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +fn is_executable(path: &Path) -> bool { + path.is_file() +} + +fn load_effective_env(path: &Path) -> Result> { + let mut env = std::env::vars().collect::>(); + if path.is_file() { + for (key, value) in parse_env_file(&fs::read_to_string(path)?) { + env.insert(key, value); + } + } + Ok(env) +} + +fn parse_env_file(text: &str) -> HashMap { + let mut values = HashMap::new(); + for raw in text.lines() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') || !line.contains('=') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + continue; + }; + let key = key.trim(); + if key.is_empty() + || !key + .chars() + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) + { + continue; + } + values.insert(key.to_string(), unquote_env_value(value.trim())); + } + values +} + +fn unquote_env_value(value: &str) -> String { + if value.len() >= 2 { + let bytes = value.as_bytes(); + if (bytes[0] == b'"' && bytes[value.len() - 1] == b'"') + || (bytes[0] == b'\'' && bytes[value.len() - 1] == b'\'') + { + return value[1..value.len() - 1].to_string(); + } + } + value.to_string() +} + +fn env_first(env: &HashMap, key: &str, default: &str) -> String { + env.get(key) + .filter(|value| !value.is_empty()) + .cloned() + .unwrap_or_else(|| default.to_string()) +} + +fn env_i64(env: &HashMap, key: &str, default: i64) -> i64 { + env.get(key) + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + +fn json_value_to_shell_print(value: Value) -> String { + match value { + Value::String(value) => value, + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_simple_env_file() { + let parsed = parse_env_file( + r#" + # comment + AW_EXPECT_START_OF_DAY="00:00" + AW_EXPECT_LANDINGPAGE=/#/activity/SHARKON2025/view/ + BAD KEY=value + "#, + ); + assert_eq!(parsed.get("AW_EXPECT_START_OF_DAY").unwrap(), "00:00"); + assert_eq!( + parsed.get("AW_EXPECT_LANDINGPAGE").unwrap(), + "/#/activity/SHARKON2025/view/" + ); + assert!(!parsed.contains_key("BAD KEY")); + } + + #[test] + fn formats_json_setting_like_python_print_json_load() { + assert_eq!( + json_value_to_shell_print(Value::String("00:00".to_string())), + "00:00" + ); + assert_eq!(json_value_to_shell_print(Value::Bool(true)), "true"); + } +} diff --git a/adk-rust/crates/aw-prune-local-state/Cargo.toml b/adk-rust/crates/aw-prune-local-state/Cargo.toml new file mode 100644 index 0000000..bd0bfff --- /dev/null +++ b/adk-rust/crates/aw-prune-local-state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "aw-prune-local-state" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-prune-local-state/src/main.rs b/adk-rust/crates/aw-prune-local-state/src/main.rs new file mode 100644 index 0000000..9b2de29 --- /dev/null +++ b/adk-rust/crates/aw-prune-local-state/src/main.rs @@ -0,0 +1,621 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde::Serialize; + +const DEFAULT_DATA_DIR: &str = "/var/lib/activitywatch"; +const TMP_ARCHIVE_PATTERNS: &[NamePattern] = &[ + NamePattern::PrefixSuffix("activitywatch-", ".zip"), + NamePattern::PrefixSuffix("hayabusa-", ".zip"), + NamePattern::Exact("aw-hayabusa-profiles.txt"), +]; +const TMP_WEBUI_PATTERNS: &[NamePattern] = &[ + NamePattern::Exact("aw-worktime-ui-bridge.py"), + NamePattern::Exact("views-default.json"), + NamePattern::Exact("apply_webui_ru_patch.out"), +]; + +#[derive(Debug, Parser)] +#[command(about = "Safely prune ActivityWatch local app state and temporary artifacts")] +struct Cli { + #[arg(long)] + data_dir: Option, + + #[arg(long)] + backup_dir: Option, + + #[arg(long)] + browser_smoke_dir: Option, + + #[arg(long)] + tmp_dir: Option, + + #[arg(long, default_value_t = 7)] + backup_retention_days: u64, + + #[arg(long, default_value_t = 2)] + backup_keep_last_db: usize, + + #[arg(long, default_value_t = 2)] + backup_keep_last_json: usize, + + #[arg(long, default_value_t = 24)] + browser_smoke_keep_runs: usize, + + #[arg(long, default_value_t = 1)] + browser_smoke_retention_days: u64, + + #[arg(long, default_value_t = 1)] + tmp_archive_retention_days: u64, + + #[arg(long, default_value_t = 2)] + tmp_webui_retention_days: u64, + + #[arg(long, default_value_t = false)] + apply: bool, + + #[arg(long, default_value_t = false)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + data_dir: PathBuf, + backup_dir: PathBuf, + browser_smoke_dir: PathBuf, + tmp_dir: PathBuf, + backup_retention_days: u64, + backup_keep_last_db: usize, + backup_keep_last_json: usize, + browser_smoke_keep_runs: usize, + browser_smoke_retention_days: u64, + tmp_archive_retention_days: u64, + tmp_webui_retention_days: u64, + apply: bool, + json: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum ItemKind { + File, + Directory, +} + +#[derive(Debug, Clone, Serialize)] +struct PruneItem { + path: PathBuf, + kind: ItemKind, + reason: String, + age_days: Option, + size_bytes: u64, +} + +#[derive(Debug, Serialize)] +struct Summary { + apply: bool, + planned: usize, + deleted: usize, + failed: usize, + bytes: u64, + items: Vec, + errors: Vec, +} + +#[derive(Debug, Clone, Copy)] +enum NamePattern { + Exact(&'static str), + PrefixSuffix(&'static str, &'static str), +} + +impl NamePattern { + fn matches(self, name: &str) -> bool { + match self { + Self::Exact(expected) => name == expected, + Self::PrefixSuffix(prefix, suffix) => { + name.starts_with(prefix) && name.ends_with(suffix) + } + } + } +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cfg = Config::from_cli(Cli::parse()); + let mut items = Vec::new(); + fs::create_dir_all(&cfg.backup_dir) + .with_context(|| format!("create backup dir {}", cfg.backup_dir.display()))?; + + plan_backup_group( + &cfg.backup_dir.join("db"), + cfg.backup_keep_last_db, + cfg.backup_retention_days, + "backup_db", + &mut items, + )?; + plan_backup_group( + &cfg.backup_dir, + cfg.backup_keep_last_json, + cfg.backup_retention_days, + "backup_root", + &mut items, + )?; + plan_browser_smoke(&cfg, &mut items)?; + plan_tmp( + &cfg.tmp_dir, + TMP_ARCHIVE_PATTERNS, + cfg.tmp_archive_retention_days, + "tmp_archive", + &mut items, + )?; + plan_tmp( + &cfg.tmp_dir, + TMP_WEBUI_PATTERNS, + cfg.tmp_webui_retention_days, + "tmp_webui", + &mut items, + )?; + + validate_plan(&cfg, &items)?; + let mut summary = Summary { + apply: cfg.apply, + planned: items.len(), + deleted: 0, + failed: 0, + bytes: items.iter().map(|item| item.size_bytes).sum(), + items, + errors: Vec::new(), + }; + if cfg.apply { + apply_plan(&mut summary); + } + print_summary(&summary, cfg.json)?; + if summary.failed == 0 { Ok(0) } else { Ok(1) } +} + +impl Config { + fn from_cli(cli: Cli) -> Self { + let data_dir = cli + .data_dir + .or_else(|| env_path("AW_DATA_DIR")) + .unwrap_or_else(|| PathBuf::from(DEFAULT_DATA_DIR)); + let backup_dir = cli + .backup_dir + .or_else(|| env_path("AW_BACKUP_DIR")) + .unwrap_or_else(|| data_dir.join("backups")); + let browser_smoke_dir = cli + .browser_smoke_dir + .or_else(|| env_path("AW_BROWSER_SMOKE_OUTPUT_DIR")) + .unwrap_or_else(|| data_dir.join("browser-smoke")); + let tmp_dir = cli + .tmp_dir + .or_else(|| env_path("AW_TMP_DIR")) + .unwrap_or_else(|| PathBuf::from("/tmp")); + Self { + data_dir, + backup_dir, + browser_smoke_dir, + tmp_dir, + backup_retention_days: env_u64("AW_BACKUP_RETENTION_DAYS", cli.backup_retention_days), + backup_keep_last_db: env_usize("AW_BACKUP_KEEP_LAST_DB", cli.backup_keep_last_db), + backup_keep_last_json: env_usize("AW_BACKUP_KEEP_LAST_JSON", cli.backup_keep_last_json), + browser_smoke_keep_runs: env_usize( + "AW_BROWSER_SMOKE_KEEP_RUNS", + cli.browser_smoke_keep_runs, + ), + browser_smoke_retention_days: env_u64( + "AW_BROWSER_SMOKE_RETENTION_DAYS", + cli.browser_smoke_retention_days, + ), + tmp_archive_retention_days: env_u64( + "AW_TMP_ARCHIVE_RETENTION_DAYS", + cli.tmp_archive_retention_days, + ), + tmp_webui_retention_days: env_u64( + "AW_TMP_WEBUI_RETENTION_DAYS", + cli.tmp_webui_retention_days, + ), + apply: cli.apply, + json: cli.json, + } + } +} + +fn plan_backup_group( + dir: &Path, + keep_last: usize, + keep_days: u64, + reason: &str, + items: &mut Vec, +) -> Result<()> { + let mut files = list_files(dir)?; + files.sort_by(|a, b| b.modified.cmp(&a.modified)); + let cutoff = cutoff(keep_days); + for (idx, candidate) in files.into_iter().enumerate() { + if idx < keep_last || candidate.modified >= cutoff || is_rollback_critical(&candidate.path) + { + continue; + } + items.push(candidate.into_item(reason)); + } + Ok(()) +} + +fn plan_browser_smoke(cfg: &Config, items: &mut Vec) -> Result<()> { + let mut dirs = list_run_dirs(&cfg.browser_smoke_dir)?; + dirs.sort_by(|a, b| b.modified.cmp(&a.modified)); + let cutoff = cutoff(cfg.browser_smoke_retention_days); + for (idx, candidate) in dirs.into_iter().enumerate() { + if idx < cfg.browser_smoke_keep_runs || candidate.modified >= cutoff { + continue; + } + items.push(candidate.into_item("browser_smoke_run")); + } + Ok(()) +} + +fn plan_tmp( + dir: &Path, + patterns: &[NamePattern], + keep_days: u64, + reason: &str, + items: &mut Vec, +) -> Result<()> { + if !dir.exists() { + return Ok(()); + } + let cutoff = cutoff(keep_days); + for entry in fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry?; + let file_type = entry.file_type()?; + if !file_type.is_file() && !file_type.is_symlink() { + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !patterns.iter().any(|pattern| pattern.matches(&name)) { + continue; + } + let meta = entry.metadata()?; + let modified = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH); + if modified >= cutoff { + continue; + } + items.push( + Candidate { + path: entry.path(), + kind: ItemKind::File, + modified, + size_bytes: meta.len(), + } + .into_item(reason), + ); + } + Ok(()) +} + +#[derive(Debug)] +struct Candidate { + path: PathBuf, + kind: ItemKind, + modified: SystemTime, + size_bytes: u64, +} + +impl Candidate { + fn into_item(self, reason: &str) -> PruneItem { + PruneItem { + path: self.path, + kind: self.kind, + reason: reason.to_string(), + age_days: age_days(self.modified), + size_bytes: self.size_bytes, + } + } +} + +fn list_files(dir: &Path) -> Result> { + if !dir.exists() { + return Ok(Vec::new()); + } + let mut files = Vec::new(); + for entry in fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let meta = entry.metadata()?; + files.push(Candidate { + path: entry.path(), + kind: ItemKind::File, + modified: meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), + size_bytes: meta.len(), + }); + } + Ok(files) +} + +fn list_run_dirs(dir: &Path) -> Result> { + if !dir.exists() { + return Ok(Vec::new()); + } + let mut dirs = Vec::new(); + for entry in fs::read_dir(dir).with_context(|| format!("read {}", dir.display()))? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !looks_like_browser_smoke_run(&name) { + continue; + } + let meta = entry.metadata()?; + dirs.push(Candidate { + path: entry.path(), + kind: ItemKind::Directory, + modified: meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), + size_bytes: dir_size(&entry.path()).unwrap_or(0), + }); + } + Ok(dirs) +} + +fn looks_like_browser_smoke_run(name: &str) -> bool { + name.len() >= 20 + && name.starts_with("20") + && name.contains('T') + && name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_') +} + +fn validate_plan(cfg: &Config, items: &[PruneItem]) -> Result<()> { + for item in items { + if is_rollback_critical(&item.path) { + bail!( + "refusing to delete rollback-critical path {}", + item.path.display() + ); + } + let allowed = match item.reason.as_str() { + "backup_db" => is_under_or_equal(&item.path, &cfg.backup_dir.join("db")), + "backup_root" => is_under_or_equal(&item.path, &cfg.backup_dir), + "browser_smoke_run" => is_under_or_equal(&item.path, &cfg.browser_smoke_dir), + "tmp_archive" | "tmp_webui" => is_under_or_equal(&item.path, &cfg.tmp_dir), + _ => false, + }; + if !allowed { + bail!( + "refusing to delete path outside allowlist {}", + item.path.display() + ); + } + if item.reason == "backup_root" && item.path.parent() != Some(cfg.backup_dir.as_path()) { + bail!("refusing nested backup_root delete {}", item.path.display()); + } + if item.path == cfg.data_dir + || item.path == cfg.backup_dir + || item.path == cfg.browser_smoke_dir + { + bail!("refusing to delete root directory {}", item.path.display()); + } + let name = item + .path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if is_sqlite_db_name(name) && item.reason != "backup_db" { + bail!( + "refusing to delete SQLite DB outside backup_db {}", + item.path.display() + ); + } + } + Ok(()) +} + +fn apply_plan(summary: &mut Summary) { + for item in summary.items.clone() { + let result = match item.kind { + ItemKind::File => fs::remove_file(&item.path), + ItemKind::Directory => fs::remove_dir_all(&item.path), + }; + match result { + Ok(()) => summary.deleted += 1, + Err(err) if !item.path.exists() => { + summary.deleted += 1; + summary + .errors + .push(format!("already gone: {} ({err})", item.path.display())); + } + Err(err) => { + summary.failed += 1; + summary + .errors + .push(format!("{}: {err}", item.path.display())); + } + } + } +} + +fn print_summary(summary: &Summary, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(summary)?); + return Ok(()); + } + let mode = if summary.apply { "apply" } else { "dry-run" }; + println!( + "aw-prune-local-state: mode={mode} planned={} deleted={} failed={} bytes={}", + summary.planned, summary.deleted, summary.failed, summary.bytes + ); + for item in &summary.items { + let verb = if summary.apply { + "DELETE" + } else { + "WOULD_DELETE" + }; + println!( + "{verb} {:?} {} reason={} age_days={} bytes={}", + item.kind, + item.path.display(), + item.reason, + item.age_days + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()), + item.size_bytes + ); + } + for err in &summary.errors { + eprintln!("WARN {err}"); + } + Ok(()) +} + +fn cutoff(days: u64) -> SystemTime { + SystemTime::now() + .checked_sub(Duration::from_secs(days.saturating_mul(86_400))) + .unwrap_or(SystemTime::UNIX_EPOCH) +} + +fn age_days(modified: SystemTime) -> Option { + SystemTime::now() + .duration_since(modified) + .ok() + .map(|age| age.as_secs() / 86_400) +} + +fn dir_size(path: &Path) -> Result { + let mut total = 0; + for entry in fs::read_dir(path).with_context(|| format!("read {}", path.display()))? { + let entry = entry?; + let file_type = entry.file_type()?; + let child = entry.path(); + if file_type.is_dir() { + total += dir_size(&child).unwrap_or(0); + } else if file_type.is_file() || file_type.is_symlink() { + total += entry.metadata().map(|meta| meta.len()).unwrap_or(0); + } + } + Ok(total) +} + +fn is_under_or_equal(path: &Path, root: &Path) -> bool { + path == root || path.starts_with(root) +} + +fn is_rollback_critical(path: &Path) -> bool { + let text = path.to_string_lossy().to_ascii_lowercase(); + text.contains("switch-backups") + || text.contains("before-rust") + || text.contains("rollback") + || text.contains("pre-switch") +} + +fn is_sqlite_db_name(name: &str) -> bool { + name.ends_with(".sqlite") + || name.ends_with(".sqlite3") + || name.ends_with(".db") + || name.ends_with(".db-shm") + || name.ends_with(".db-wal") +} + +fn env_path(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + +fn env_usize(name: &str, default: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + + #[test] + fn browser_smoke_run_name_is_narrow() { + assert!(looks_like_browser_smoke_run("2026-06-02T03-02-19-530Z")); + assert!(!looks_like_browser_smoke_run(".cache")); + assert!(!looks_like_browser_smoke_run("latest-result.json")); + } + + #[test] + fn rollback_critical_paths_are_protected() { + assert!(is_rollback_critical(Path::new( + "/var/lib/activitywatch/health/switch-backups/file" + ))); + assert!(is_rollback_critical(Path::new( + "/var/lib/activitywatch/backups/db/before-rust.sqlite" + ))); + } + + #[test] + fn backup_group_keeps_newest_even_with_zero_retention() { + let tmp = tempfile::tempdir().expect("tempdir"); + let old = tmp.path().join("old.json"); + let new = tmp.path().join("new.json"); + File::create(&old).expect("old"); + std::thread::sleep(Duration::from_millis(5)); + File::create(&new).expect("new"); + let mut items = Vec::new(); + plan_backup_group(tmp.path(), 1, 0, "backup_root", &mut items).expect("plan"); + assert_eq!(items.len(), 1); + assert_eq!(items[0].path, old); + } + + #[test] + fn validation_rejects_sqlite_outside_backup_db() { + let tmp = tempfile::tempdir().expect("tempdir"); + let cfg = Config { + data_dir: tmp.path().to_path_buf(), + backup_dir: tmp.path().join("backups"), + browser_smoke_dir: tmp.path().join("browser-smoke"), + tmp_dir: tmp.path().join("tmp"), + backup_retention_days: 1, + backup_keep_last_db: 1, + backup_keep_last_json: 1, + browser_smoke_keep_runs: 1, + browser_smoke_retention_days: 1, + tmp_archive_retention_days: 1, + tmp_webui_retention_days: 1, + apply: false, + json: false, + }; + let item = PruneItem { + path: cfg.browser_smoke_dir.join("state.db"), + kind: ItemKind::File, + reason: "browser_smoke_run".to_string(), + age_days: Some(2), + size_bytes: 1, + }; + assert!(validate_plan(&cfg, &[item]).is_err()); + } +} diff --git a/adk-rust/crates/aw-rus-healthd/Cargo.toml b/adk-rust/crates/aw-rus-healthd/Cargo.toml new file mode 100644 index 0000000..1e459c3 --- /dev/null +++ b/adk-rust/crates/aw-rus-healthd/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "aw-rus-healthd" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-core.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true diff --git a/adk-rust/crates/aw-rus-healthd/src/main.rs b/adk-rust/crates/aw-rus-healthd/src/main.rs new file mode 100644 index 0000000..233b4c3 --- /dev/null +++ b/adk-rust/crates/aw-rus-healthd/src/main.rs @@ -0,0 +1,1032 @@ +use std::collections::HashMap; +use std::fs; +use std::io::Write; +use std::net::{SocketAddr, TcpStream}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Duration as ChronoDuration, SecondsFormat, Utc}; +use clap::Parser; +use detmir_core::{exit_codes, parse_utc_rfc3339}; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Value, json}; + +const ENV_FILE: &str = "/etc/activitywatch/aw-server.env"; + +#[derive(Debug, Parser)] +#[command(about = "Unified AW-RUS health orchestrator.")] +struct Cli { + #[arg(long, default_value = "http://127.0.0.1:5600")] + aw_server: String, + + #[arg(long, default_value = "http://127.0.0.1:5610")] + worktime_api: String, + + #[arg(long, default_value = "192.168.100.18")] + rdp_host: String, + + #[arg(long, default_value = "SHARKON2025")] + rdp_hostname: String, + + #[arg(long, default_value = "/var/lib/activitywatch/health")] + state_dir: PathBuf, + + #[arg( + long, + default_value = "/var/lib/activitywatch/health/windows-validation" + )] + validation_dir: PathBuf, + + #[arg(long, default_value_t = 900)] + session_max_age_seconds: i64, + + #[arg(long, default_value_t = 900)] + interactive_max_age_seconds: i64, + + #[arg(long, default_value_t = 86400)] + session_events_max_age_seconds: i64, + + #[arg(long, default_value_t = 300)] + guard_max_age_seconds: i64, + + #[arg(long)] + guard_required: bool, + + #[arg(long, default_value_t = 259200)] + validation_max_age_seconds: i64, + + #[arg(long, default_value_t = 3.0)] + tcp_timeout_seconds: f64, + + #[arg(long)] + json: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + load_env_file(Path::new(ENV_FILE)); + if !cli_arg_present("--aw-server") { + self.aw_server = env_string("AW_SERVER_URL").unwrap_or(self.aw_server); + } + if !cli_arg_present("--worktime-api") { + self.worktime_api = env_string("AW_RUS_HEALTH_WORKTIME_API") + .or_else(|| env_string("AW_WORKTIME_REPORT_BASE")) + .unwrap_or(self.worktime_api); + } + if !cli_arg_present("--rdp-host") { + self.rdp_host = env_string("AW_MONITORED_WINDOWS_HOST").unwrap_or(self.rdp_host); + } + if !cli_arg_present("--rdp-hostname") { + self.rdp_hostname = + env_string("AW_MONITORED_WINDOWS_HOSTNAME").unwrap_or(self.rdp_hostname); + } + if !cli_arg_present("--state-dir") { + self.state_dir = env_path("AW_RUS_HEALTH_STATE_DIR").unwrap_or(self.state_dir); + } + if !cli_arg_present("--validation-dir") { + self.validation_dir = + env_path("AW_RUS_HEALTH_VALIDATION_DIR").unwrap_or(self.validation_dir); + } + if !cli_arg_present("--session-max-age-seconds") { + self.session_max_age_seconds = env_i64( + "AW_RUS_HEALTH_SESSION_MAX_AGE_SECONDS", + self.session_max_age_seconds, + ); + } + if !cli_arg_present("--interactive-max-age-seconds") { + self.interactive_max_age_seconds = env_i64( + "AW_RUS_HEALTH_INTERACTIVE_MAX_AGE_SECONDS", + self.interactive_max_age_seconds, + ); + } + if !cli_arg_present("--session-events-max-age-seconds") { + self.session_events_max_age_seconds = env_i64( + "AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS", + self.session_events_max_age_seconds, + ); + } + if !cli_arg_present("--guard-max-age-seconds") { + self.guard_max_age_seconds = env_i64( + "AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS", + self.guard_max_age_seconds, + ); + } + if !cli_arg_present("--guard-required") { + self.guard_required = env_bool("AW_RUS_HEALTH_GUARD_REQUIRED"); + } + if !cli_arg_present("--validation-max-age-seconds") { + self.validation_max_age_seconds = env_i64( + "AW_RUS_HEALTH_VALIDATION_MAX_AGE_SECONDS", + self.validation_max_age_seconds, + ); + } + if !cli_arg_present("--tcp-timeout-seconds") { + self.tcp_timeout_seconds = env_f64( + "AW_RUS_HEALTH_TCP_TIMEOUT_SECONDS", + self.tcp_timeout_seconds, + ); + } + self + } +} + +fn cli_arg_present(name: &str) -> bool { + std::env::args_os().skip(1).any(|arg| { + let Some(value) = arg.to_str() else { + return false; + }; + value == name + || value + .strip_prefix(name) + .is_some_and(|rest| rest.starts_with('=')) + }) +} + +#[derive(Debug, Clone, Serialize)] +struct CheckResult { + name: String, + status: String, + summary: String, + details: Value, +} + +#[derive(Debug, Serialize)] +struct HealthReport { + generated_at_utc: String, + ok: bool, + counts: HashMap, + results: Vec, +} + +#[derive(Default)] +struct ReportBuilder { + results: Vec, +} + +impl ReportBuilder { + fn add( + &mut self, + name: impl Into, + status: &str, + summary: impl Into, + details: Value, + ) { + self.results.push(CheckResult { + name: name.into(), + status: status.to_string(), + summary: summary.into(), + details, + }); + } + + fn build(&self) -> HealthReport { + let mut counts = HashMap::from([ + ("ok".to_string(), 0), + ("warn".to_string(), 0), + ("fail".to_string(), 0), + ]); + for item in &self.results { + *counts.entry(item.status.clone()).or_insert(0) += 1; + } + HealthReport { + generated_at_utc: utc_iso(), + ok: !self.results.iter().any(|item| item.status == "fail"), + counts, + results: self.results.clone(), + } + } +} + +fn env_string(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn env_path(name: &str) -> Option { + env_string(name).map(PathBuf::from) +} + +fn env_i64(name: &str, fallback: i64) -> i64 { + env_string(name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + env_string(name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_bool(name: &str) -> bool { + env_string(name) + .map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "yes")) + .unwrap_or(false) +} + +fn load_env_file(path: &Path) { + let Ok(raw) = fs::read_to_string(path) else { + return; + }; + for line in raw.lines().map(str::trim) { + if line.is_empty() || line.starts_with('#') || !line.contains('=') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + continue; + }; + if std::env::var_os(key.trim()).is_none() { + let value = value + .trim() + .trim_matches('"') + .trim_matches('\'') + .to_string(); + // SAFETY: this binary is single-threaded during configuration loading. + unsafe { std::env::set_var(key.trim(), value) }; + } + } +} + +fn utc_iso() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn parse_ts(value: Option<&str>) -> Option> { + value.and_then(|value| parse_utc_rfc3339(value).ok()) +} + +fn age_seconds(ts: Option>) -> Option { + ts.map(|ts| (Utc::now() - ts).num_seconds().max(0)) +} + +fn event_effective_ts(event: &Value) -> Option> { + let timestamp = parse_ts(event.get("timestamp").and_then(Value::as_str))?; + let duration_ms = event + .get("duration") + .and_then(Value::as_f64) + .filter(|duration| duration.is_finite() && *duration > 0.0) + .map(|duration| (duration * 1000.0).round() as i64) + .unwrap_or(0); + Some(timestamp + ChronoDuration::milliseconds(duration_ms)) +} + +fn http_json(client: &Client, url: &str, attempts: usize) -> Result { + let attempts = attempts.max(1); + let mut last_error = None; + for attempt in 0..attempts { + let result = client + .get(url) + .send() + .with_context(|| format!("HTTP request failed: {url}")) + .and_then(|response| response.error_for_status().context("HTTP status error")) + .and_then(|response| response.json::().context("invalid JSON response")); + match result { + Ok(value) => return Ok(value), + Err(err) => last_error = Some(err), + } + if attempt + 1 < attempts { + std::thread::sleep(Duration::from_millis(500 * (1 << attempt))); + } + } + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("HTTP request failed: {url}"))) +} + +fn latest_bucket_event(client: &Client, api_base: &str, bucket_id: &str) -> Result> { + let events = http_json( + client, + &format!("{api_base}/buckets/{bucket_id}/events?limit=20"), + 2, + )?; + let Some(items) = events.as_array() else { + return Ok(None); + }; + let mut objects = items + .iter() + .filter(|item| item.is_object()) + .cloned() + .collect::>(); + objects.sort_by(|left, right| event_effective_ts(right).cmp(&event_effective_ts(left))); + Ok(objects.into_iter().next()) +} + +fn bucket_metadata_ts(buckets: &Value, bucket_id: &str) -> Option> { + parse_ts( + buckets + .get(bucket_id)? + .get("metadata")? + .get("end")? + .as_str(), + ) +} + +fn host_activity_from_worktime(event: Option<&Value>, max_age_seconds: i64) -> Value { + let Some(event) = event else { + return json!({"fresh": false, "active": false, "age_seconds": null, "timestamp": null}); + }; + let timestamp = event.get("timestamp").and_then(Value::as_str); + let age = age_seconds(parse_ts(timestamp)); + let data = event.get("data").cloned().unwrap_or_else(|| json!({})); + let fresh = age.is_some_and(|age| age <= max_age_seconds); + let active = fresh && data.get("active").and_then(Value::as_bool).unwrap_or(false); + json!({ + "fresh": fresh, + "active": active, + "age_seconds": age, + "timestamp": timestamp, + "data": data, + }) +} + +fn bucket_health( + client: &Client, + api_base: &str, + bucket_id: &str, + max_age_seconds: i64, + missing_status: &str, + stale_status: &str, +) -> (String, String, Value) { + let event = match latest_bucket_event(client, api_base, bucket_id) { + Ok(event) => event, + Err(err) => { + return ( + "fail".to_string(), + format!("bucket query failed: {err}"), + json!({"bucket": bucket_id}), + ); + } + }; + let Some(event) = event else { + return ( + missing_status.to_string(), + "no events".to_string(), + json!({"bucket": bucket_id}), + ); + }; + let timestamp = event.get("timestamp").and_then(Value::as_str); + let effective_ts = event_effective_ts(&event); + let age = age_seconds(effective_ts); + let timestamp_source = if event + .get("duration") + .and_then(Value::as_f64) + .is_some_and(|duration| duration.is_finite() && duration > 0.0) + { + "event.timestamp+duration" + } else { + "event.timestamp" + }; + let details = json!({ + "bucket": bucket_id, + "timestamp": timestamp, + "effective_timestamp": effective_ts.map(|ts| ts.to_rfc3339_opts(SecondsFormat::Secs, true)), + "timestamp_source": timestamp_source, + "age_seconds": age, + }); + match age { + None => ( + "warn".to_string(), + "timestamp parse failed".to_string(), + details, + ), + Some(age) if age > max_age_seconds => { + (stale_status.to_string(), format!("stale ({age}s)"), details) + } + Some(age) => ("ok".to_string(), format!("fresh ({age}s)"), details), + } +} + +fn bucket_timestamp_health( + client: &Client, + api_base: &str, + buckets: &Value, + bucket_id: &str, + max_age_seconds: i64, + missing_status: &str, + stale_status: &str, +) -> (String, String, Value) { + if let Some(ts) = bucket_metadata_ts(buckets, bucket_id) { + let age = age_seconds(Some(ts)); + let timestamp = ts.to_rfc3339_opts(SecondsFormat::Secs, true); + let details = json!({ + "bucket": bucket_id, + "timestamp": timestamp, + "age_seconds": age, + "timestamp_source": "bucket_metadata.end", + }); + return match age { + None => ( + "warn".to_string(), + "timestamp parse failed".to_string(), + details, + ), + Some(age) if age > max_age_seconds => { + (stale_status.to_string(), format!("stale ({age}s)"), details) + } + Some(age) => ("ok".to_string(), format!("fresh ({age}s)"), details), + }; + } + bucket_health( + client, + api_base, + bucket_id, + max_age_seconds, + missing_status, + stale_status, + ) +} + +fn guard_bucket_health( + client: &Client, + api_base: &str, + host: &str, + max_age_seconds: i64, + required: bool, +) -> (String, String, Value) { + let bucket_id = format!("aw-rus-collector-guard_{host}"); + let event = match latest_bucket_event(client, api_base, &bucket_id) { + Ok(event) => event, + Err(err) => { + return ( + if required { "fail" } else { "warn" }.to_string(), + format!("guard bucket query failed: {err}"), + json!({"bucket": bucket_id}), + ); + } + }; + let Some(event) = event else { + return ( + if required { "fail" } else { "warn" }.to_string(), + "no guard heartbeat".to_string(), + json!({"bucket": bucket_id, "required": required}), + ); + }; + let timestamp = event.get("timestamp").and_then(Value::as_str); + let age = age_seconds(parse_ts(timestamp)); + let data = event + .get("data") + .filter(|value| value.is_object()) + .cloned() + .unwrap_or_else(|| json!({})); + let guard_status = data + .get("status") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_ascii_lowercase(); + let details = json!({ + "bucket": bucket_id, + "timestamp": timestamp, + "age_seconds": age, + "required": required, + "guard_status": guard_status, + "mode": data.get("mode").cloned().unwrap_or(Value::Null), + "live_session_count": data.get("liveSessionCount").cloned().unwrap_or(Value::Null), + "problems": data.get("problems").cloned().unwrap_or_else(|| json!([])), + "actions": data.get("actions").cloned().unwrap_or_else(|| json!([])), + }); + let stale_status = if required { "fail" } else { "warn" }; + match age { + None => ( + "warn".to_string(), + "guard timestamp parse failed".to_string(), + details, + ), + Some(age) if age > max_age_seconds => ( + stale_status.to_string(), + format!("guard stale ({age}s)"), + details, + ), + Some(_) if matches!(guard_status.as_str(), "fail" | "error") => ( + if required { "fail" } else { "warn" }.to_string(), + format!("guard reports {guard_status}"), + details, + ), + Some(_) if guard_status == "warn" => ( + "warn".to_string(), + "guard reports warn".to_string(), + details, + ), + Some(age) => ("ok".to_string(), format!("guard fresh ({age}s)"), details), + } +} + +fn run_command(cmd: &[&str]) -> (i32, String) { + let timeout = env_f64("AW_RUS_HEALTH_WRAPPER_TIMEOUT_SECONDS", 20.0).max(1.0); + let mut child = match Command::new(cmd[0]) + .args(&cmd[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(err) => return (1, err.to_string()), + }; + let started = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => match child.wait_with_output() { + Ok(output) => { + let mut text = String::from_utf8_lossy(&output.stdout).to_string(); + if !output.stderr.is_empty() { + text.push_str(&String::from_utf8_lossy(&output.stderr)); + } + return (output.status.code().unwrap_or(1), text.trim().to_string()); + } + Err(err) => return (1, err.to_string()), + }, + Ok(None) if started.elapsed() >= Duration::from_secs_f64(timeout) => { + let _ = child.kill(); + let output = child.wait_with_output(); + let mut text = format!("timed out after {timeout:.1}s"); + if let Ok(output) = output { + if !output.stdout.is_empty() { + text.push('\n'); + text.push_str(&String::from_utf8_lossy(&output.stdout)); + } + if !output.stderr.is_empty() { + text.push('\n'); + text.push_str(&String::from_utf8_lossy(&output.stderr)); + } + } + return (124, text.trim().to_string()); + } + Ok(None) => std::thread::sleep(Duration::from_millis(100)), + Err(err) => { + let _ = child.kill(); + return (1, err.to_string()); + } + } + } +} + +fn check_wrapper( + report: &mut ReportBuilder, + name: &str, + cmd: &[&str], + json_mode: bool, + failure_status: &str, +) { + if !Path::new(cmd[0]).exists() { + report.add(name, "warn", "binary missing", json!({"command": cmd})); + return; + } + let (rc, output) = run_command(cmd); + let mut details = json!({"command": cmd, "returncode": rc}); + if json_mode { + match serde_json::from_str::(&output) { + Ok(payload) => details["payload"] = payload, + Err(_) => { + details["raw_output"] = Value::String(output); + report.add(name, "fail", "invalid JSON output", details); + return; + } + } + } else { + details["output"] = Value::String(output); + } + report.add( + name, + if rc == 0 { "ok" } else { failure_status }, + if rc == 0 { "passed" } else { "failed" }, + details, + ); +} + +fn tcp_connect(host: &str, port: u16, timeout_seconds: f64) -> (bool, String) { + let addr = match format!("{host}:{port}").parse::() { + Ok(addr) => addr, + Err(err) => return (false, err.to_string()), + }; + match TcpStream::connect_timeout(&addr, Duration::from_secs_f64(timeout_seconds)) { + Ok(_) => (true, "connected".to_string()), + Err(err) => (false, err.to_string()), + } +} + +fn latest_validation_report(dir: &Path) -> Option { + let mut entries = fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with("-aw_validate_ansible.json")) + }) + .collect::>(); + entries.sort_by_key(|path| path.metadata().and_then(|meta| meta.modified()).ok()); + entries.pop() +} + +fn write_atomic(path: &Path, content: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + let mut tmp = tempfile::NamedTempFile::new_in(parent)?; + tmp.write_all(content.as_bytes())?; + tmp.persist(path) + .map_err(|err| anyhow::anyhow!("failed to persist {}: {}", path.display(), err.error))?; + Ok(()) +} + +fn chmod_if_possible(path: &Path, mode: u32) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = path.metadata() { + let mut perms = meta.permissions(); + perms.set_mode(mode); + let _ = fs::set_permissions(path, perms); + } + } +} + +fn render_text(report: &HealthReport) -> String { + let mut lines = vec![ + "=== AW-RUS Health ===".to_string(), + format!("Timestamp: {}", utc_iso()), + String::new(), + ]; + for item in &report.results { + lines.push(format!( + "[{}] {}: {}", + item.status.to_ascii_uppercase(), + item.name, + item.summary + )); + } + lines.push(String::new()); + lines.push(format!( + "Counts: ok={} warn={} fail={}", + report.counts.get("ok").copied().unwrap_or(0), + report.counts.get("warn").copied().unwrap_or(0), + report.counts.get("fail").copied().unwrap_or(0) + )); + lines.push(format!( + "Overall: {}", + if report.ok { "OK" } else { "FAIL" } + )); + lines.join("\n") +} + +fn normalize_aw_api_base(aw_server: &str) -> String { + let trimmed = aw_server.trim_end_matches('/'); + if trimmed.ends_with("/api/0") { + trimmed.to_string() + } else { + format!("{trimmed}/api/0") + } +} + +fn validation_check(report: &mut ReportBuilder, validation_dir: &Path, max_age_seconds: i64) { + let Some(path) = latest_validation_report(validation_dir) else { + report.add( + "validation:windows", + "warn", + "no validation report snapshot", + json!({"directory": validation_dir}), + ); + return; + }; + let raw = match fs::read_to_string(&path) { + Ok(raw) => raw.trim_start_matches('\u{feff}').to_string(), + Err(err) => { + report.add( + "validation:windows", + "fail", + format!("invalid validation snapshot: {err}"), + json!({"path": path}), + ); + return; + } + }; + let payload = match serde_json::from_str::(&raw) { + Ok(payload) => payload, + Err(err) => { + report.add( + "validation:windows", + "fail", + format!("invalid validation snapshot: {err}"), + json!({"path": path}), + ); + return; + } + }; + let age = path + .metadata() + .and_then(|meta| meta.modified()) + .ok() + .and_then(|modified| modified.elapsed().ok()) + .map(|elapsed| elapsed.as_secs() as i64); + if age.is_some_and(|age| age > max_age_seconds) { + report.add( + "validation:windows", + "warn", + format!("validation snapshot is stale ({}s)", age.unwrap_or(0)), + json!({ + "path": path, + "overall_ok": payload.get("overallOk").cloned().unwrap_or(Value::Null), + "failed_sections": payload.pointer("/summary/failedSections").cloned().unwrap_or_else(|| json!([])), + }), + ); + } else if payload.get("overallOk").and_then(Value::as_bool) == Some(true) { + report.add( + "validation:windows", + "ok", + "validation snapshot OK", + json!({"path": path, "age_seconds": age}), + ); + } else { + report.add( + "validation:windows", + "fail", + "validation snapshot reports failure", + json!({ + "path": path, + "age_seconds": age, + "failed_sections": payload.pointer("/summary/failedSections").cloned().unwrap_or_else(|| json!([])), + }), + ); + } +} + +fn run(cli: &Cli) -> Result { + let client = Client::builder() + .timeout(Duration::from_secs(25)) + .no_proxy() + .build()?; + let mut report = ReportBuilder::default(); + let aw_api_base = normalize_aw_api_base(&cli.aw_server); + + check_wrapper( + &mut report, + "wrapper:aw-health-check", + &["/usr/local/bin/aw-health-check"], + false, + "warn", + ); + check_wrapper( + &mut report, + "wrapper:dlp-health-check", + &["/usr/local/bin/dlp-health-check", "--json"], + true, + "warn", + ); + + match http_json(&client, &format!("{aw_api_base}/info"), 2) { + Ok(info) => report.add( + "http:aw-server", + "ok", + "activitywatch API responded", + json!({"version": info.get("version").cloned().unwrap_or(Value::Null)}), + ), + Err(err) => report.add( + "http:aw-server", + "fail", + format!("activitywatch API failed: {err}"), + json!({"url": format!("{aw_api_base}/info")}), + ), + } + + match http_json( + &client, + &format!("{}/health", cli.worktime_api.trim_end_matches('/')), + 2, + ) { + Ok(payload) => report.add( + "http:worktime-api", + "ok", + "worktime API responded", + json!({"payload": if payload.is_object() { payload } else { json!({}) }}), + ), + Err(err) => report.add( + "http:worktime-api", + "fail", + format!("worktime API failed: {err}"), + json!({"url": cli.worktime_api}), + ), + } + + for (port, label) in [(5985_u16, "winrm"), (3389_u16, "rdp")] { + let (ok, message) = tcp_connect(&cli.rdp_host, port, cli.tcp_timeout_seconds); + report.add( + format!("tcp:{label}"), + if ok { "ok" } else { "fail" }, + if ok { + message + } else { + format!("unreachable: {message}") + }, + json!({"host": cli.rdp_host, "port": port}), + ); + } + + let buckets = match http_json(&client, &format!("{aw_api_base}/buckets"), 2) { + Ok(buckets) if buckets.is_object() => { + report.add( + "aw:buckets-index", + "ok", + "bucket index loaded", + json!({"total": buckets.as_object().map(|value| value.len()).unwrap_or(0)}), + ); + buckets + } + Ok(_) => { + report.add( + "aw:buckets-index", + "fail", + "failed to load bucket index: bucket index is not a dict", + json!({}), + ); + json!({}) + } + Err(err) => { + report.add( + "aw:buckets-index", + "fail", + format!("failed to load bucket index: {err}"), + json!({}), + ); + json!({}) + } + }; + + let host = &cli.rdp_hostname; + let (status, summary, details) = guard_bucket_health( + &client, + &aw_api_base, + host, + cli.guard_max_age_seconds, + cli.guard_required, + ); + report.add("bucket:collector-guard", &status, summary, details); + + let worktime_bucket = format!("aw-worktime-sessions_{host}"); + let worktime_event = if buckets.is_object() { + latest_bucket_event(&client, &aw_api_base, &worktime_bucket) + .ok() + .flatten() + } else { + None + }; + let activity = + host_activity_from_worktime(worktime_event.as_ref(), cli.session_max_age_seconds); + if worktime_event.is_some() { + let (status, summary, mut details) = bucket_health( + &client, + &aw_api_base, + &worktime_bucket, + cli.session_max_age_seconds, + "fail", + "fail", + ); + details["host_activity"] = activity.clone(); + report.add("bucket:worktime-sessions", &status, summary, details); + } else { + report.add( + "bucket:worktime-sessions", + "fail", + "no events", + json!({"bucket": worktime_bucket, "host_activity": activity}), + ); + } + + let interactive_required = activity + .get("active") + .and_then(Value::as_bool) + .unwrap_or(false); + for (bucket_name, label) in [ + ("aw-watcher-afk", "bucket:afk"), + ("aw-watcher-window", "bucket:window"), + ("aw-dlp-endpoint-signals", "bucket:endpoint-signals"), + ] { + let (mut status, mut summary, mut details) = bucket_timestamp_health( + &client, + &aw_api_base, + &buckets, + &format!("{bucket_name}_{host}"), + cli.interactive_max_age_seconds, + if interactive_required { "fail" } else { "warn" }, + if interactive_required { "fail" } else { "warn" }, + ); + details["interactive_required"] = Value::Bool(interactive_required); + details["host_activity"] = activity.clone(); + if !interactive_required && status != "ok" { + details["inactive_summary"] = Value::String(summary); + status = "ok".to_string(); + summary = "inactive: no active interactive users".to_string(); + } + report.add(label, &status, summary, details); + } + + let (mut status, mut summary, details) = bucket_timestamp_health( + &client, + &aw_api_base, + &buckets, + &format!("aw-session-events_{host}"), + cli.session_events_max_age_seconds, + "fail", + "warn", + ); + if status == "warn" { + if let Some(age) = details.get("age_seconds").and_then(Value::as_i64) { + status = "ok".to_string(); + summary = format!("event-driven ({age}s since last logon marker)"); + } + } + report.add("bucket:session-events", &status, summary, details); + + validation_check( + &mut report, + &cli.validation_dir, + cli.validation_max_age_seconds, + ); + Ok(report.build()) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + let report = run(&cli)?; + let json_text = serde_json::to_string_pretty(&report)? + "\n"; + let text = render_text(&report) + "\n"; + let json_path = cli.state_dir.join("aw-rus-health.json"); + let text_path = cli.state_dir.join("aw-rus-health.txt"); + write_atomic(&json_path, &json_text)?; + write_atomic(&text_path, &text)?; + chmod_if_possible(&json_path, 0o644); + chmod_if_possible(&text_path, 0o644); + if cli.json { + print!("{json_text}"); + } else { + print!("{text}"); + } + std::process::exit(if report.ok { + exit_codes::OK + } else { + exit_codes::ERROR + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn activity_from_worktime_marks_recent_active_session() { + let event = json!({"timestamp": "2026-05-18T10:00:00Z", "data": {"active": true}}); + let activity = host_activity_from_worktime(Some(&event), i64::MAX); + assert_eq!(activity["active"], true); + assert_eq!(activity["fresh"], true); + } + + #[test] + fn heartbeat_missing_activity_is_inactive() { + let activity = host_activity_from_worktime(None, 900); + assert_eq!(activity["active"], false); + assert_eq!(activity["fresh"], false); + } + + #[test] + fn bucket_metadata_ts_reads_activitywatch_end_timestamp() { + let buckets = json!({ + "aw-watcher-window_HOST": { + "metadata": { + "end": "2026-06-01T10:54:37.976Z" + } + } + }); + let ts = bucket_metadata_ts(&buckets, "aw-watcher-window_HOST").unwrap(); + assert_eq!( + ts.to_rfc3339_opts(SecondsFormat::Millis, true), + "2026-06-01T10:54:37.976Z" + ); + } + + #[test] + fn event_effective_ts_adds_duration_to_long_activitywatch_event() { + let event = json!({ + "timestamp": "2026-06-01T10:30:29.573Z", + "duration": 1448.403, + }); + let ts = event_effective_ts(&event).unwrap(); + assert_eq!( + ts.to_rfc3339_opts(SecondsFormat::Millis, true), + "2026-06-01T10:54:37.976Z" + ); + } + + #[test] + fn normalizes_aw_api_base() { + assert_eq!( + normalize_aw_api_base("http://127.0.0.1:5600"), + "http://127.0.0.1:5600/api/0" + ); + assert_eq!( + normalize_aw_api_base("http://127.0.0.1:5600/api/0"), + "http://127.0.0.1:5600/api/0" + ); + } +} diff --git a/adk-rust/crates/aw-slo-monitor/Cargo.toml b/adk-rust/crates/aw-slo-monitor/Cargo.toml new file mode 100644 index 0000000..24b87e6 --- /dev/null +++ b/adk-rust/crates/aw-slo-monitor/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "aw-slo-monitor" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-slo-monitor/src/main.rs b/adk-rust/crates/aw-slo-monitor/src/main.rs new file mode 100644 index 0000000..bb1fe14 --- /dev/null +++ b/adk-rust/crates/aw-slo-monitor/src/main.rs @@ -0,0 +1,1078 @@ +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::{Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + thread, + time::{Duration, Instant}, +}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, SecondsFormat, TimeDelta, Utc}; +use clap::{Parser, ValueEnum}; +use reqwest::{ + blocking::Client, + header::{ACCEPT, CONNECTION, HeaderMap, HeaderValue}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const ENV_FILE: &str = "/etc/activitywatch/aw-server.env"; +const DEFAULT_STATE_DIR: &str = "/var/lib/activitywatch/slo"; +const DEFAULT_HEALTHD_CMD: &str = "/usr/local/bin/aw-rus-healthd.py --json"; +const DEFAULT_HEALTHD_STATE_FILE: &str = "/var/lib/activitywatch/health/aw-rus-health.json"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +enum HealthdMode { + State, + Run, +} + +#[derive(Debug, Parser)] +#[command(about = "AW-RUS rolling SLO sampler")] +struct Cli { + #[arg(long, default_value = DEFAULT_STATE_DIR)] + state_dir: PathBuf, + + #[arg(long, default_value = DEFAULT_HEALTHD_CMD)] + healthd_cmd: String, + + #[arg(long, value_enum, default_value_t = HealthdMode::State)] + healthd_mode: HealthdMode, + + #[arg(long, default_value = DEFAULT_HEALTHD_STATE_FILE)] + healthd_state_file: PathBuf, + + #[arg(long, default_value_t = 180)] + healthd_state_max_age_seconds: i64, + + #[arg(long, default_value = "http://127.0.0.1:5600")] + aw_base: String, + + #[arg(long, default_value = "http://127.0.0.1:5610")] + worktime_base: String, + + #[arg(long, default_value = "SHARKON2025")] + host: String, + + #[arg(long, default_value_t = 99.97)] + target_percent: f64, + + #[arg(long, default_value_t = 15)] + sample_interval_seconds: i64, + + #[arg(long, default_value_t = 35)] + retention_days: i64, + + #[arg(long, default_value_t = 15.0)] + http_timeout_seconds: f64, + + #[arg(long, default_value_t = 90)] + health_timeout_seconds: u64, + + #[arg(long, default_value_t = 500)] + worktime_html_min_bytes: usize, + + #[arg(long)] + json: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + let file_env = load_env_file(Path::new(ENV_FILE)); + if !cli_arg_present("--state-dir") { + self.state_dir = env_path(&file_env, "AW_RUS_SLO_STATE_DIR").unwrap_or(self.state_dir); + } + if !cli_arg_present("--healthd-cmd") { + self.healthd_cmd = + env_string(&file_env, "AW_RUS_SLO_HEALTHD_CMD").unwrap_or(self.healthd_cmd); + } + if !cli_arg_present("--healthd-mode") { + self.healthd_mode = env_string(&file_env, "AW_RUS_SLO_HEALTHD_MODE") + .and_then(|value| match value.as_str() { + "state" => Some(HealthdMode::State), + "run" => Some(HealthdMode::Run), + _ => None, + }) + .unwrap_or(self.healthd_mode); + } + if !cli_arg_present("--healthd-state-file") { + self.healthd_state_file = env_path(&file_env, "AW_RUS_SLO_HEALTHD_STATE_FILE") + .unwrap_or(self.healthd_state_file); + } + if !cli_arg_present("--healthd-state-max-age-seconds") { + self.healthd_state_max_age_seconds = env_i64( + &file_env, + "AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", + self.healthd_state_max_age_seconds, + ); + } + if !cli_arg_present("--aw-base") { + self.aw_base = env_string(&file_env, "AW_RUS_SLO_AW_BASE") + .or_else(|| env_string(&file_env, "AW_SERVER_URL")) + .unwrap_or(self.aw_base); + } + if !cli_arg_present("--worktime-base") { + self.worktime_base = env_string(&file_env, "AW_RUS_SLO_WORKTIME_BASE") + .or_else(|| env_string(&file_env, "AW_RUS_HEALTH_WORKTIME_API")) + .unwrap_or(self.worktime_base); + } + if !cli_arg_present("--host") { + self.host = env_string(&file_env, "AW_RUS_SLO_HOST") + .or_else(|| env_string(&file_env, "AW_MONITORED_WINDOWS_HOSTNAME")) + .unwrap_or(self.host); + } + if !cli_arg_present("--target-percent") { + self.target_percent = + env_f64(&file_env, "AW_RUS_SLO_TARGET_PERCENT", self.target_percent); + } + if !cli_arg_present("--sample-interval-seconds") { + self.sample_interval_seconds = env_i64( + &file_env, + "AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", + self.sample_interval_seconds, + ); + } + if !cli_arg_present("--retention-days") { + self.retention_days = + env_i64(&file_env, "AW_RUS_SLO_RETENTION_DAYS", self.retention_days); + } + if !cli_arg_present("--http-timeout-seconds") { + self.http_timeout_seconds = env_f64( + &file_env, + "AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", + self.http_timeout_seconds, + ); + } + if !cli_arg_present("--health-timeout-seconds") { + self.health_timeout_seconds = env_u64( + &file_env, + "AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", + self.health_timeout_seconds, + ); + } + if !cli_arg_present("--worktime-html-min-bytes") { + self.worktime_html_min_bytes = env_usize( + &file_env, + "AW_RUS_SLO_WORKTIME_HTML_MIN_BYTES", + self.worktime_html_min_bytes, + ); + } + self.aw_base = self.aw_base.trim_end_matches('/').to_string(); + self.worktime_base = self.worktime_base.trim_end_matches('/').to_string(); + self + } + + fn urls(&self) -> ProbeUrls { + let host = urlencoding::encode(&self.host); + ProbeUrls { + aw_webui: format!("{}/", self.aw_base), + today_html: format!( + "{}/reports/worktime/today?format=html&day=today&host={host}&allow_stale=1", + self.worktime_base + ), + management_html: format!( + "{}/reports/worktime/management?format=html&day=today&host={host}&allow_stale=1", + self.worktime_base + ), + today_csv: format!( + "{}/reports/worktime/today?format=csv&day=today&host={host}&allow_stale=1", + self.worktime_base + ), + management_json: format!( + "{}/reports/worktime/management?format=json&day=today&host={host}&allow_stale=1", + self.worktime_base + ), + } + } +} + +struct ProbeUrls { + aw_webui: String, + today_html: String, + management_html: String, + today_csv: String, + management_json: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Sample { + ts: String, + ok: bool, + healthd_ok: bool, + #[serde(default)] + healthd_counts: Value, + probes: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct Summary { + generated_at_utc: String, + target_percent: f64, + sample_interval_seconds: i64, + current_sample: Sample, + windows: Windows, +} + +#[derive(Debug, Serialize)] +struct Windows { + #[serde(rename = "24h")] + day_24h: WindowSummary, + #[serde(rename = "7d")] + day_7d: WindowSummary, + #[serde(rename = "30d")] + day_30d: WindowSummary, +} + +#[derive(Debug, Serialize, PartialEq)] +struct WindowSummary { + window_seconds: i64, + samples: usize, + good_samples: usize, + bad_samples: usize, + availability_percent: Option, + target_percent: f64, + observed_bad_seconds: i64, + budget_seconds: i64, + budget_remaining_seconds: i64, + status: String, +} + +fn main() { + match run() { + Ok(ok) => std::process::exit(if ok { 0 } else { 1 }), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(2); + } + } +} + +fn run() -> Result { + let cli = Cli::parse().apply_env(); + let client = build_client(cli.http_timeout_seconds)?; + let sample = build_sample(&cli, &client); + let sample_path = cli.state_dir.join("aw-slo-samples.jsonl"); + let samples = + append_and_trim_sample(&sample_path, &sample, cli.retention_days.max(1) * 86_400)?; + let generated_at = Utc::now(); + let summary = build_summary(&cli, sample, &samples, generated_at); + let summary_text = render_summary_text(&summary); + + write_atomic( + &cli.state_dir.join("aw-slo-summary.json"), + &(serde_json::to_string_pretty(&summary)? + "\n"), + )?; + write_atomic( + &cli.state_dir.join("aw-slo-summary.txt"), + &(summary_text.clone() + "\n"), + )?; + + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else { + println!("{summary_text}"); + } + Ok(summary.current_sample.ok) +} + +fn build_client(timeout_seconds: f64) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(CONNECTION, HeaderValue::from_static("close")); + Client::builder() + .timeout(Duration::from_secs_f64(timeout_seconds.max(0.001))) + .no_proxy() + .pool_max_idle_per_host(0) + .default_headers(headers) + .build() + .context("build HTTP client") +} + +fn build_sample(cli: &Cli, client: &Client) -> Sample { + let health = match cli.healthd_mode { + HealthdMode::Run => run_healthd(&cli.healthd_cmd, cli.health_timeout_seconds), + HealthdMode::State => { + read_healthd_state(&cli.healthd_state_file, cli.healthd_state_max_age_seconds) + } + }; + let urls = cli.urls(); + let mut probes = BTreeMap::new(); + probes.insert( + "aw_webui_index".to_string(), + html_probe( + client, + &urls.aw_webui, + 1_000, + &["ActivityWatch", "id=\"app\"", "ru-patch-v5.js"], + ), + ); + probes.insert( + "worktime_today_html".to_string(), + html_probe( + client, + &urls.today_html, + cli.worktime_html_min_bytes, + &["AW-rus", ""], + ), + ); + probes.insert( + "worktime_management_html".to_string(), + html_probe( + client, + &urls.management_html, + cli.worktime_html_min_bytes, + &["AW-rus", ""], + ), + ); + probes.insert( + "worktime_today_csv".to_string(), + http_probe(client, &urls.today_csv), + ); + probes.insert( + "worktime_management_json".to_string(), + json_probe( + client, + &urls.management_json, + &[("host", json!(cli.host))], + &["generated_at_utc", "host", "summary", "rows", "workday"], + ), + ); + + let health_ok = value_bool(&health, "ok"); + let ok = health_ok && probes.values().all(|probe| value_bool(probe, "ok")); + Sample { + ts: iso(Utc::now()), + ok, + healthd_ok: health_ok, + healthd_counts: health.get("counts").cloned().unwrap_or_else(|| json!({})), + probes, + } +} + +fn fetch_url(client: &Client, url: &str, accept: Option<&str>, attempts: usize) -> Value { + let started = Instant::now(); + let attempts = attempts.max(1); + let mut last_error = String::new(); + for attempt in 1..=attempts { + let mut request = client.get(url); + if let Some(accept) = accept { + request = request.header(ACCEPT, accept); + } + match request.send() { + Ok(response) => { + let status = response.status().as_u16(); + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + match response.bytes() { + Ok(body) => { + let ok = (200..300).contains(&status); + return json!({ + "ok": ok, + "status": status, + "body_bytes": body.len(), + "content_type": content_type, + "body": String::from_utf8_lossy(&body), + "attempts": attempt, + "latency_ms": started.elapsed().as_millis() as i64, + "url": url, + }); + } + Err(error) => last_error = error.to_string(), + } + } + Err(error) => last_error = error.to_string(), + } + if attempt < attempts { + thread::sleep(Duration::from_secs(1)); + } + } + json!({ + "ok": false, + "error": last_error, + "body_bytes": 0, + "attempts": attempts, + "latency_ms": started.elapsed().as_millis() as i64, + "url": url, + }) +} + +fn public_probe_result(mut result: Value) -> Value { + if let Some(object) = result.as_object_mut() { + object.remove("body"); + } + result +} + +fn http_probe(client: &Client, url: &str) -> Value { + public_probe_result(fetch_url(client, url, None, 2)) +} + +fn html_probe(client: &Client, url: &str, min_bytes: usize, required_markers: &[&str]) -> Value { + let mut result = fetch_url(client, url, Some("text/html"), 2); + if !value_bool(&result, "ok") { + return public_probe_result(result); + } + let body = result + .get("body") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let content_type = value_string(&result, "content_type"); + let body_bytes = result + .get("body_bytes") + .and_then(Value::as_u64) + .unwrap_or(0) as usize; + let missing: Vec<&str> = required_markers + .iter() + .copied() + .filter(|marker| !body.contains(marker)) + .collect(); + if body_bytes < min_bytes { + set_probe_error( + &mut result, + format!("body too small: {body_bytes} < {min_bytes}"), + ); + } else if !content_type.to_ascii_lowercase().contains("text/html") { + set_probe_error( + &mut result, + format!( + "unexpected content-type: {}", + if content_type.is_empty() { + "unknown" + } else { + &content_type + } + ), + ); + } else if !missing.is_empty() { + set_probe_error( + &mut result, + format!("missing markers: {}", missing.join(", ")), + ); + if let Some(object) = result.as_object_mut() { + object.insert("missing_markers".to_string(), json!(missing)); + } + } + public_probe_result(result) +} + +fn json_probe( + client: &Client, + url: &str, + expected_values: &[(&str, Value)], + required_keys: &[&str], +) -> Value { + let mut result = fetch_url(client, url, Some("application/json"), 2); + if !value_bool(&result, "ok") { + return public_probe_result(result); + } + let body = result.get("body").and_then(Value::as_str).unwrap_or(""); + let payload: Value = match serde_json::from_str(body) { + Ok(payload) => payload, + Err(error) => { + set_probe_error(&mut result, format!("invalid json: {error}")); + return public_probe_result(result); + } + }; + let Some(object) = payload.as_object() else { + set_probe_error(&mut result, "json root is not object".to_string()); + return public_probe_result(result); + }; + + let missing_keys: Vec<&str> = required_keys + .iter() + .copied() + .filter(|key| !object.contains_key(*key)) + .collect(); + let mismatched: BTreeMap<&str, Value> = expected_values + .iter() + .filter_map(|(key, expected)| { + let actual = object.get(*key)?; + if actual == expected { + None + } else { + Some((*key, json!({"expected": expected, "actual": actual}))) + } + }) + .collect(); + if !missing_keys.is_empty() { + set_probe_error( + &mut result, + format!("missing json keys: {}", missing_keys.join(", ")), + ); + if let Some(result_object) = result.as_object_mut() { + result_object.insert("missing_keys".to_string(), json!(missing_keys)); + } + } else if !mismatched.is_empty() { + set_probe_error(&mut result, "unexpected json values".to_string()); + if let Some(result_object) = result.as_object_mut() { + result_object.insert("mismatched_values".to_string(), json!(mismatched)); + } + } else if let Some(result_object) = result.as_object_mut() { + let mut keys: Vec<&String> = object.keys().collect(); + keys.sort(); + result_object.insert("json_keys".to_string(), json!(keys)); + } + public_probe_result(result) +} + +fn set_probe_error(result: &mut Value, message: String) { + if let Some(object) = result.as_object_mut() { + object.insert("ok".to_string(), json!(false)); + object.insert("error".to_string(), json!(message)); + } +} + +fn run_healthd(command: &str, timeout_seconds: u64) -> Value { + let mut child = match Command::new("/bin/sh") + .arg("-c") + .arg(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(error) => return json!({"ok": false, "returncode": null, "error": error.to_string()}), + }; + let deadline = Instant::now() + Duration::from_secs(timeout_seconds.max(1)); + loop { + match child.try_wait() { + Ok(Some(status)) => { + let mut output = String::new(); + if let Some(mut stdout) = child.stdout.take() { + let _ = stdout.read_to_string(&mut output); + } + if let Some(mut stderr) = child.stderr.take() { + let mut err = String::new(); + let _ = stderr.read_to_string(&mut err); + output.push_str(&err); + } + let payload: Value = serde_json::from_str(&output).unwrap_or_else(|_| json!({})); + let ok = status.success() && value_bool(&payload, "ok"); + return json!({ + "ok": ok, + "returncode": status.code(), + "counts": payload.get("counts").cloned().unwrap_or_else(|| json!({})), + "payload": payload, + "output_tail": output_tail(&output), + }); + } + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return json!({ + "ok": false, + "returncode": null, + "error": format!("timeout after {timeout_seconds}s"), + "output_tail": "", + }); + } + thread::sleep(Duration::from_millis(100)); + } + Err(error) => { + let _ = child.kill(); + return json!({"ok": false, "returncode": null, "error": error.to_string()}); + } + } + } +} + +fn read_healthd_state(path: &Path, max_age_seconds: i64) -> Value { + let payload: Value = match fs::read_to_string(path) + .with_context(|| format!("read {}", path.display())) + .and_then(|content| serde_json::from_str(&content).context("parse healthd state json")) + { + Ok(payload) => payload, + Err(error) => return json!({"ok": false, "error": error.to_string(), "path": path}), + }; + let generated = payload + .get("generated_at_utc") + .and_then(Value::as_str) + .and_then(parse_ts); + let Some(generated) = generated else { + return json!({"ok": false, "error": "missing or invalid generated_at_utc", "path": path}); + }; + let age_seconds = (Utc::now() - generated).num_seconds().max(0); + json!({ + "ok": value_bool(&payload, "ok") && age_seconds <= max_age_seconds, + "counts": payload.get("counts").cloned().unwrap_or_else(|| json!({})), + "age_seconds": age_seconds, + "path": path, + "payload": payload, + }) +} + +fn append_and_trim_sample( + path: &Path, + sample: &Sample, + retention_seconds: i64, +) -> Result> { + let sample_ts = parse_ts(&sample.ts).unwrap_or_else(Utc::now); + let cutoff = sample_ts - TimeDelta::seconds(retention_seconds.max(1)); + let mut samples = load_samples(path, cutoff); + samples.push(serde_json::from_value(serde_json::to_value(sample)?)?); + + let mut content = String::new(); + for item in &samples { + content.push_str(&serde_json::to_string(item)?); + content.push('\n'); + } + write_atomic(path, &content)?; + Ok(samples) +} + +fn load_samples(path: &Path, cutoff: DateTime) -> Vec { + let Ok(content) = fs::read_to_string(path) else { + return Vec::new(); + }; + content + .lines() + .filter_map(|line| { + let sample: Sample = serde_json::from_str(line).ok()?; + let ts = parse_ts(&sample.ts)?; + (ts >= cutoff).then_some(sample) + }) + .collect() +} + +fn build_summary( + cli: &Cli, + sample: Sample, + samples: &[Sample], + generated_at: DateTime, +) -> Summary { + let windows = Windows { + day_24h: summarize_window( + samples, + generated_at, + 86_400, + cli.sample_interval_seconds, + cli.target_percent, + ), + day_7d: summarize_window( + samples, + generated_at, + 7 * 86_400, + cli.sample_interval_seconds, + cli.target_percent, + ), + day_30d: summarize_window( + samples, + generated_at, + 30 * 86_400, + cli.sample_interval_seconds, + cli.target_percent, + ), + }; + Summary { + generated_at_utc: iso(generated_at), + target_percent: cli.target_percent, + sample_interval_seconds: cli.sample_interval_seconds, + current_sample: sample, + windows, + } +} + +fn summarize_window( + samples: &[Sample], + now: DateTime, + window_seconds: i64, + sample_interval_seconds: i64, + target_percent: f64, +) -> WindowSummary { + let cutoff = now - TimeDelta::seconds(window_seconds); + let window: Vec<&Sample> = samples + .iter() + .filter(|sample| parse_ts(&sample.ts).is_some_and(|ts| ts >= cutoff)) + .collect(); + let total = window.len(); + let good = window.iter().filter(|sample| sample.ok).count(); + let bad = total - good; + let availability_percent = if total == 0 { + None + } else { + Some(round5((good as f64 / total as f64) * 100.0)) + }; + let observed_bad_seconds = bad as i64 * sample_interval_seconds; + let budget_seconds = (window_seconds as f64 * ((100.0 - target_percent) / 100.0)) as i64; + let budget_remaining_seconds = budget_seconds - observed_bad_seconds; + let status = if total == 0 { + "unknown" + } else if budget_remaining_seconds >= 0 { + "ok" + } else { + "burning" + } + .to_string(); + WindowSummary { + window_seconds, + samples: total, + good_samples: good, + bad_samples: bad, + availability_percent, + target_percent, + observed_bad_seconds, + budget_seconds, + budget_remaining_seconds, + status, + } +} + +fn render_summary_text(summary: &Summary) -> String { + let mut lines = vec![ + "=== AW-RUS SLO ===".to_string(), + format!("Timestamp: {}", summary.generated_at_utc), + format!("Target: {}%", summary.target_percent), + String::new(), + ]; + for (name, data) in [ + ("24h", &summary.windows.day_24h), + ("7d", &summary.windows.day_7d), + ("30d", &summary.windows.day_30d), + ] { + let availability = data + .availability_percent + .map(|value| format!("{value:.5}%")) + .unwrap_or_else(|| "n/a".to_string()); + lines.push(format!( + "{name}: {} availability={} samples={} bad={} bad_seconds={} budget_remaining_seconds={}", + data.status, + availability, + data.samples, + data.bad_samples, + data.observed_bad_seconds, + data.budget_remaining_seconds + )); + } + lines.push(String::new()); + lines.push(format!( + "Current sample: {}", + if summary.current_sample.ok { + "OK" + } else { + "FAIL" + } + )); + for name in [ + "aw_webui_index", + "worktime_today_html", + "worktime_management_html", + "worktime_today_csv", + "worktime_management_json", + ] { + let Some(probe) = summary.current_sample.probes.get(name) else { + continue; + }; + let marker = if value_bool(probe, "ok") { + "OK" + } else { + "FAIL" + }; + let detail = probe + .get("status") + .and_then(Value::as_u64) + .map(|status| status.to_string()) + .or_else(|| { + probe + .get("error") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .unwrap_or_default(); + lines.push(format!("- {name}: {marker} {detail}")); + } + lines.join("\n") +} + +fn write_atomic(path: &Path, content: &str) -> Result<()> { + let parent = path + .parent() + .with_context(|| format!("resolve parent for {}", path.display()))?; + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + let tmp = parent.join(format!( + ".{}.tmp.{}", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("aw-slo"), + std::process::id() + )); + { + let mut file = File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?; + file.write_all(content.as_bytes()) + .with_context(|| format!("write {}", tmp.display()))?; + file.sync_all().ok(); + } + fs::set_permissions(&tmp, fs::Permissions::from_mode_ext(0o644)).ok(); + fs::rename(&tmp, path) + .with_context(|| format!("rename {} to {}", tmp.display(), path.display()))?; + Ok(()) +} + +trait PermissionsExtCompat { + fn from_mode_ext(mode: u32) -> Self; +} + +impl PermissionsExtCompat for fs::Permissions { + #[cfg(unix)] + fn from_mode_ext(mode: u32) -> Self { + use std::os::unix::fs::PermissionsExt; + fs::Permissions::from_mode(mode) + } + + #[cfg(not(unix))] + fn from_mode_ext(_mode: u32) -> Self { + fs::metadata(".") + .map(|metadata| metadata.permissions()) + .unwrap_or_else(|_| fs::Permissions::readonly()) + } +} + +fn load_env_file(path: &Path) -> BTreeMap { + let Ok(content) = fs::read_to_string(path) else { + return BTreeMap::new(); + }; + content + .lines() + .filter_map(|raw| { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let (key, value) = line.split_once('=')?; + Some(( + key.trim().to_string(), + value + .trim() + .trim_matches('"') + .trim_matches('\'') + .to_string(), + )) + }) + .collect() +} + +fn cli_arg_present(name: &str) -> bool { + std::env::args_os().skip(1).any(|arg| { + let Some(value) = arg.to_str() else { + return false; + }; + value == name + || value + .strip_prefix(name) + .is_some_and(|rest| rest.starts_with('=')) + }) +} + +fn env_string(file_env: &BTreeMap, name: &str) -> Option { + std::env::var(name) + .ok() + .or_else(|| file_env.get(name).cloned()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn env_path(file_env: &BTreeMap, name: &str) -> Option { + env_string(file_env, name).map(PathBuf::from) +} + +fn env_i64(file_env: &BTreeMap, name: &str, fallback: i64) -> i64 { + env_string(file_env, name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_u64(file_env: &BTreeMap, name: &str, fallback: u64) -> u64 { + env_string(file_env, name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_usize(file_env: &BTreeMap, name: &str, fallback: usize) -> usize { + env_string(file_env, name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_f64(file_env: &BTreeMap, name: &str, fallback: f64) -> f64 { + env_string(file_env, name) + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn iso(dt: DateTime) -> String { + dt.to_rfc3339_opts(SecondsFormat::Micros, true) +} + +fn round5(value: f64) -> f64 { + (value * 100_000.0).round() / 100_000.0 +} + +fn value_bool(value: &Value, key: &str) -> bool { + value.get(key).and_then(Value::as_bool).unwrap_or(false) +} + +fn value_string(value: &Value, key: &str) -> String { + value + .get(key) + .and_then(Value::as_str) + .unwrap_or("") + .to_string() +} + +fn output_tail(output: &str) -> String { + let chars: Vec = output.chars().collect(); + let start = chars.len().saturating_sub(1000); + chars[start..].iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample(ts: &str, ok: bool) -> Sample { + Sample { + ts: ts.to_string(), + ok, + healthd_ok: ok, + healthd_counts: json!({}), + probes: BTreeMap::new(), + } + } + + #[test] + fn summarize_window_calculates_9997_budget() { + let now = parse_ts("2026-05-30T12:00:00Z").unwrap(); + let samples = vec![ + sample("2026-05-30T11:58:00Z", true), + sample("2026-05-30T11:59:00Z", false), + sample("2026-05-30T12:00:00Z", true), + ]; + + let summary = summarize_window(&samples, now, 86_400, 60, 99.97); + + assert_eq!(summary.samples, 3); + assert_eq!(summary.good_samples, 2); + assert_eq!(summary.bad_samples, 1); + assert_eq!(summary.availability_percent, Some(66.66667)); + assert_eq!(summary.budget_seconds, 25); + assert_eq!(summary.budget_remaining_seconds, -35); + assert_eq!(summary.status, "burning"); + } + + #[test] + fn summarize_window_status_uses_remaining_error_budget_for_partial_window() { + let now = parse_ts("2026-05-30T12:00:00Z").unwrap(); + let samples = vec![ + sample("2026-05-30T11:59:30Z", false), + sample("2026-05-30T11:59:45Z", true), + sample("2026-05-30T12:00:00Z", true), + ]; + + let summary = summarize_window(&samples, now, 86_400, 15, 99.97); + + assert_eq!(summary.availability_percent, Some(66.66667)); + assert_eq!(summary.budget_seconds, 25); + assert_eq!(summary.budget_remaining_seconds, 10); + assert_eq!(summary.status, "ok"); + } + + #[test] + fn render_summary_text_includes_budget_remaining() { + let mut probes = BTreeMap::new(); + probes.insert( + "worktime_today_csv".to_string(), + json!({"ok": true, "status": 200}), + ); + let sample = Sample { + ts: "2026-05-30T12:00:00Z".to_string(), + ok: true, + healthd_ok: true, + healthd_counts: json!({}), + probes, + }; + let windows = Windows { + day_24h: WindowSummary { + window_seconds: 86_400, + samples: 10, + good_samples: 10, + bad_samples: 0, + availability_percent: Some(100.0), + target_percent: 99.97, + observed_bad_seconds: 0, + budget_seconds: 25, + budget_remaining_seconds: 25, + status: "ok".to_string(), + }, + day_7d: WindowSummary { + window_seconds: 604_800, + samples: 10, + good_samples: 10, + bad_samples: 0, + availability_percent: Some(100.0), + target_percent: 99.97, + observed_bad_seconds: 0, + budget_seconds: 181, + budget_remaining_seconds: 181, + status: "ok".to_string(), + }, + day_30d: WindowSummary { + window_seconds: 2_592_000, + samples: 10, + good_samples: 10, + bad_samples: 0, + availability_percent: Some(100.0), + target_percent: 99.97, + observed_bad_seconds: 0, + budget_seconds: 777, + budget_remaining_seconds: 777, + status: "ok".to_string(), + }, + }; + let summary = Summary { + generated_at_utc: "2026-05-30T12:00:00Z".to_string(), + target_percent: 99.97, + sample_interval_seconds: 15, + current_sample: sample, + windows, + }; + + let text = render_summary_text(&summary); + + assert!(text.contains("Target: 99.97%")); + assert!(text.contains("24h: ok availability=100.00000%")); + assert!(text.contains("budget_remaining_seconds=25")); + assert!(text.contains("- worktime_today_csv: OK 200")); + } + + #[test] + fn load_samples_skips_bad_and_old_lines() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("samples.jsonl"); + fs::write( + &path, + concat!( + "{\"ts\":\"2026-05-01T00:00:00Z\",\"ok\":true,\"healthd_ok\":true,\"probes\":{}}\n", + "not-json\n", + "{\"ts\":\"2026-05-30T11:59:00Z\",\"ok\":true,\"healthd_ok\":true,\"probes\":{}}\n" + ), + ) + .unwrap(); + + let samples = load_samples(&path, parse_ts("2026-05-30T11:00:00Z").unwrap()); + + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].ts, "2026-05-30T11:59:00Z"); + } +} diff --git a/adk-rust/crates/check-aw-data/Cargo.toml b/adk-rust/crates/check-aw-data/Cargo.toml new file mode 100644 index 0000000..0d74bf0 --- /dev/null +++ b/adk-rust/crates/check-aw-data/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "check-aw-data" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/check-aw-data/src/main.rs b/adk-rust/crates/check-aw-data/src/main.rs new file mode 100644 index 0000000..7455e14 --- /dev/null +++ b/adk-rust/crates/check-aw-data/src/main.rs @@ -0,0 +1,555 @@ +use std::process::Command; +use std::time::Duration; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde_json::Value; + +const DEFAULT_SERVER: &str = "http://10.10.10.13:5600"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const BUCKETS: &[&str] = &[ + "aw-dlp-endpoint-signals", + "aw-dlp-incidents", + "aw-dlp-review", + "aw-dlp-rules", + "aw-watcher-afk", + "aw-watcher-window", + "aw-session-events", + "aw-worktime-sessions", +]; + +#[derive(Debug, Parser)] +#[command(about = "Check ActivityWatch data collection freshness for DetMir/AW-RUS")] +struct Cli { + #[arg(long)] + server: Option, + + #[arg(long)] + host: Option, + + #[arg(long, default_value_t = 15)] + timeout_seconds: u64, + + #[arg(long, default_value_t = 15)] + bucket_timeout_seconds: u64, + + #[arg(long, default_value_t = 3)] + context_timeout_seconds: u64, + + #[arg(long, default_value_t = false)] + with_event_ids: bool, + + #[arg(long, default_value_t = false)] + no_color: bool, +} + +#[derive(Debug, Clone)] +struct Colors { + red: &'static str, + green: &'static str, + yellow: &'static str, + cyan: &'static str, + reset: &'static str, +} + +impl Colors { + fn new(enabled: bool) -> Self { + if enabled { + Self { + red: "\u{1b}[0;31m", + green: "\u{1b}[0;32m", + yellow: "\u{1b}[1;33m", + cyan: "\u{1b}[0;36m", + reset: "\u{1b}[0m", + } + } else { + Self { + red: "", + green: "", + yellow: "", + cyan: "", + reset: "", + } + } + } + + fn paint(&self, color: &str, text: &str) -> String { + format!("{color}{text}{}", self.reset) + } +} + +#[derive(Debug, Default)] +struct ContextState { + host_inactive: bool, + guard_healthy: bool, +} + +#[derive(Debug, PartialEq, Eq)] +enum BucketStatus { + Fresh, + Stale, + Dead, + Empty, + EventDriven, + Inactive, + Unknown, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let server = cli + .server + .or_else(|| env_nonempty("AW_CHECK_SERVER")) + .or_else(|| env_nonempty("AW_SERVER_URL")) + .unwrap_or_else(|| DEFAULT_SERVER.to_string()) + .trim_end_matches('/') + .to_string(); + let host = cli + .host + .or_else(|| env_nonempty("AW_CHECK_HOST")) + .or_else(|| env_nonempty("AW_MONITORED_HOST")) + .or_else(|| env_nonempty("AW_MONITORED_WINDOWS_HOSTNAME")) + .unwrap_or_else(|| DEFAULT_HOST.to_string()); + let colors = Colors::new(!cli.no_color && env_nonempty("NO_COLOR").is_none()); + let client = Client::builder() + .no_proxy() + .build() + .context("build HTTP client")?; + let now = Utc::now(); + + println!("=== ActivityWatch Data Check: {host} ==="); + println!(); + print!("Server connectivity... "); + let info_url = format!("{server}/api/0/info"); + let info = match get_json(&client, &info_url, cli.timeout_seconds) { + Ok(value) => value, + Err(err) => { + eprintln!("check-aw-data: {err:#}"); + println!( + "{} (cannot reach {server})", + colors.paint(colors.red, "FAILED") + ); + return Ok(1); + } + }; + let Some(version) = info.get("version").and_then(Value::as_str) else { + println!( + "{} (cannot reach {server})", + colors.paint(colors.red, "FAILED") + ); + return Ok(1); + }; + println!("{} (aw-server {version})", colors.paint(colors.green, "OK")); + println!(); + + let context = read_context(&server, &host, now, cli.context_timeout_seconds); + let buckets_index = get_json( + &client, + &format!("{server}/api/0/buckets"), + cli.timeout_seconds, + ) + .ok(); + println!("--- Buckets ---"); + println!( + "{:<45} {:<8} {:<22} STATUS", + "BUCKET", "EVENTS", "LAST EVENT" + ); + println!( + "{:<45} {:<8} {:<22} ------", + "---------------------------------------------", "--------", "----------------------" + ); + + for bucket in BUCKETS { + let bucket_full = format!("{bucket}_{host}"); + let event = bucket_event( + &server, + &bucket_full, + buckets_index.as_ref(), + cli.with_event_ids, + cli.bucket_timeout_seconds, + ); + let (last_id, last_ts, age, status) = render_bucket(bucket, event.as_ref(), now, &context); + println!( + "{:<45} {:<8} {:<22} {}", + bucket_full, + last_id, + format!("{last_ts} ({age})"), + render_status(&colors, status) + ); + } + + println!(); + println!("--- CORS Check ---"); + let cors_status = check_cors(&client, &server); + if cors_status == 200 { + println!("{} (HTTP 200)", colors.paint(colors.green, "CORS: OK")); + } else { + println!( + "{} (HTTP {cors_status})", + colors.paint(colors.red, "CORS: FAIL") + ); + } + + println!(); + println!("=== Check Complete ==="); + println!("Timestamp: {}", now.format("%Y-%m-%d %H:%M:%S UTC")); + Ok(0) +} + +fn read_context( + server: &str, + host: &str, + now: DateTime, + timeout_seconds: u64, +) -> ContextState { + let mut state = ContextState::default(); + if let Ok(Some(event)) = get_latest_event( + server, + &format!("aw-worktime-sessions_{host}"), + timeout_seconds, + ) { + if let Some(ts) = event_timestamp(&event) { + let age = (now - ts).num_seconds(); + let active = event + .pointer("/data/active") + .and_then(Value::as_bool) + .unwrap_or(false); + if age >= 0 && age < 900 && !active { + state.host_inactive = true; + } + } + } + if let Ok(Some(event)) = get_latest_event( + server, + &format!("aw-rus-collector-guard_{host}"), + timeout_seconds, + ) { + if let Some(ts) = event_timestamp(&event) { + let age = (now - ts).num_seconds(); + let status = event + .pointer("/data/status") + .and_then(Value::as_str) + .unwrap_or(""); + let problems = event + .pointer("/data/problems") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0); + if age >= 0 && age < 300 && status == "ok" && problems == 0 { + state.guard_healthy = true; + } + } + } + state +} + +fn bucket_event( + server: &str, + bucket: &str, + buckets_index: Option<&Value>, + with_event_ids: bool, + timeout_seconds: u64, +) -> Option { + if with_event_ids { + get_latest_event(server, bucket, timeout_seconds) + .ok() + .flatten() + .or_else(|| metadata_event(buckets_index, bucket)) + } else { + metadata_event(buckets_index, bucket) + } +} + +fn render_bucket( + bucket: &str, + event: Option<&Value>, + now: DateTime, + context: &ContextState, +) -> (String, String, String, BucketStatus) { + let Some(event) = event else { + return ( + "0".to_string(), + "no events".to_string(), + "none".to_string(), + classify_bucket_no_events(bucket, context), + ); + }; + let last_id = event + .get("id") + .map(json_value_to_string) + .unwrap_or_else(|| "0".to_string()); + let Some(ts_raw) = event.get("timestamp").and_then(Value::as_str) else { + return ( + last_id, + "no events".to_string(), + "none".to_string(), + classify_bucket_no_events(bucket, context), + ); + }; + let Some(ts) = parse_ts(ts_raw) else { + return ( + last_id, + ts_raw.to_string(), + "unknown".to_string(), + BucketStatus::Unknown, + ); + }; + let age = (now - ts).num_seconds().max(0); + ( + last_id, + ts_raw.to_string(), + format_age(age), + classify_bucket_age(bucket, age, context), + ) +} + +fn classify_bucket_age(bucket: &str, age_sec: i64, context: &ContextState) -> BucketStatus { + match bucket { + "aw-watcher-window" if context.host_inactive => return BucketStatus::Inactive, + "aw-dlp-endpoint-signals" if context.host_inactive && context.guard_healthy => { + return BucketStatus::Inactive; + } + _ => {} + } + + match bucket { + "aw-dlp-incidents" | "aw-dlp-review" | "aw-dlp-rules" | "aw-session-events" => { + if age_sec < 86_400 { + BucketStatus::Fresh + } else { + BucketStatus::EventDriven + } + } + _ if age_sec < 3_600 => BucketStatus::Fresh, + _ if age_sec < 86_400 => BucketStatus::Stale, + _ => BucketStatus::Dead, + } +} + +fn classify_bucket_no_events(bucket: &str, context: &ContextState) -> BucketStatus { + match bucket { + "aw-watcher-window" if context.host_inactive => BucketStatus::Inactive, + "aw-dlp-endpoint-signals" if context.host_inactive && context.guard_healthy => { + BucketStatus::Inactive + } + "aw-dlp-incidents" | "aw-dlp-review" | "aw-dlp-rules" | "aw-session-events" => { + BucketStatus::EventDriven + } + _ => BucketStatus::Empty, + } +} + +fn render_status(colors: &Colors, status: BucketStatus) -> String { + match status { + BucketStatus::Fresh => colors.paint(colors.green, "FRESH"), + BucketStatus::Stale => colors.paint(colors.yellow, "STALE"), + BucketStatus::Dead => colors.paint(colors.red, "DEAD"), + BucketStatus::Empty => colors.paint(colors.red, "EMPTY"), + BucketStatus::EventDriven => colors.paint(colors.cyan, "EVENT-DRIVEN"), + BucketStatus::Inactive => colors.paint(colors.cyan, "INACTIVE"), + BucketStatus::Unknown => colors.paint(colors.red, "?"), + } +} + +fn get_latest_event(server: &str, bucket: &str, timeout_seconds: u64) -> Result> { + let url = format!("{server}/api/0/buckets/{bucket}/events?limit=1"); + let client = Client::builder() + .timeout(Duration::from_secs(timeout_seconds.max(1))) + .no_proxy() + .pool_max_idle_per_host(0) + .build() + .context("build timed HTTP client")?; + let value = client + .get(&url) + .header("Connection", "close") + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}"))?; + Ok(value.as_array().and_then(|items| items.first()).cloned()) +} + +fn metadata_event(buckets_index: Option<&Value>, bucket: &str) -> Option { + let bucket_info = buckets_index?.get(bucket)?; + let timestamp = bucket_info + .pointer("/metadata/end") + .or_else(|| bucket_info.get("end")) + .and_then(Value::as_str)?; + Some(serde_json::json!({ + "id": 0, + "timestamp": timestamp, + "data": {}, + "_source": "bucket_metadata", + })) +} + +fn get_json(client: &Client, url: &str, _timeout_seconds: u64) -> Result { + client + .get(url) + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn check_cors(_client: &Client, server: &str) -> u16 { + let origin = "http://10.10.10.13:5600"; + let first = curl_status(&format!("{server}/api/0/settings/"), origin); + if first == 200 { + return first; + } + if !server.contains("127.0.0.1") && !server.contains("localhost") { + let fallback = curl_status("http://127.0.0.1:5600/api/0/settings/", origin); + if fallback != 0 { + return fallback; + } + } + first +} + +fn curl_status(url: &str, origin: &str) -> u16 { + let origin_header = format!("Origin: {origin}"); + let args = [ + "-s", + "--connect-timeout", + "3", + "--max-time", + "5", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "-H", + origin_header.as_str(), + url, + ]; + let output = Command::new("/usr/bin/curl") + .args(args) + .output() + .or_else(|_| Command::new("curl").args(args).output()); + let Ok(output) = output else { + return 0; + }; + if !output.status.success() { + return 0; + } + String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .unwrap_or(0) +} + +fn event_timestamp(event: &Value) -> Option> { + event + .get("timestamp") + .and_then(Value::as_str) + .and_then(parse_ts) +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .map(|ts| ts.with_timezone(&Utc)) + .ok() +} + +fn format_age(age_sec: i64) -> String { + if age_sec < 3_600 { + format!("{}m ago", age_sec / 60) + } else if age_sec < 86_400 { + format!("{}h ago", age_sec / 3_600) + } else { + format!("{}d ago", age_sec / 86_400) + } +} + +fn json_value_to_string(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + other => other.to_string(), + } +} + +fn env_nonempty(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn event_driven_buckets_do_not_become_dead_when_old() { + let context = ContextState::default(); + assert_eq!( + classify_bucket_age("aw-dlp-incidents", 100_000, &context), + BucketStatus::EventDriven + ); + } + + #[test] + fn inactive_host_suppresses_window_stale() { + let context = ContextState { + host_inactive: true, + guard_healthy: true, + }; + assert_eq!( + classify_bucket_age("aw-watcher-window", 100_000, &context), + BucketStatus::Inactive + ); + assert_eq!( + classify_bucket_no_events("aw-dlp-endpoint-signals", &context), + BucketStatus::Inactive + ); + } + + #[test] + fn formats_age_like_legacy_script() { + assert_eq!(format_age(59), "0m ago"); + assert_eq!(format_age(3_600), "1h ago"); + assert_eq!(format_age(86_400), "1d ago"); + } + + #[test] + fn metadata_only_bucket_event_skips_missing_deep_event_read() { + let index = serde_json::json!({ + "aw-watcher-window_SHARKON2025": { + "metadata": { + "end": "2026-06-02T00:00:00Z" + } + } + }); + let event = bucket_event( + "http://127.0.0.1:1", + "aw-watcher-window_SHARKON2025", + Some(&index), + false, + 1, + ) + .expect("metadata event"); + assert_eq!( + event.get("timestamp").and_then(Value::as_str), + Some("2026-06-02T00:00:00Z") + ); + assert_eq!(event.get("id").and_then(Value::as_i64), Some(0)); + } +} diff --git a/adk-rust/crates/check-aw-full/Cargo.toml b/adk-rust/crates/check-aw-full/Cargo.toml new file mode 100644 index 0000000..152bff9 --- /dev/null +++ b/adk-rust/crates/check-aw-full/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "check-aw-full" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/check-aw-full/src/main.rs b/adk-rust/crates/check-aw-full/src/main.rs new file mode 100644 index 0000000..56e7c08 --- /dev/null +++ b/adk-rust/crates/check-aw-full/src/main.rs @@ -0,0 +1,537 @@ +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde_json::Value; + +const DEFAULT_SERVER: &str = "http://10.10.10.13:5600"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const DEFAULT_RDP_HOST: &str = "192.168.100.18"; +const BUCKETS: &[(&str, &str)] = &[ + ("aw-watcher-afk", "AFK watcher"), + ("aw-watcher-window", "Window watcher"), + ("aw-worktime-sessions", "Worktime sessions"), + ("aw-session-events", "Session events"), + ("aw-dlp-endpoint-signals", "DLP signals"), + ("aw-dlp-incidents", "DLP incidents"), + ("aw-dlp-review", "DLP review"), + ("aw-dlp-rules", "DLP rules"), +]; + +#[derive(Debug, Parser)] +#[command(about = "Full read-only ActivityWatch check for server, buckets, and RDP host")] +struct Cli { + #[arg(long, default_value = DEFAULT_SERVER)] + server: String, + + #[arg(long, default_value = DEFAULT_HOST)] + host: String, + + #[arg(long, default_value = DEFAULT_RDP_HOST)] + rdp_host: String, + + #[arg(long, default_value_t = 15)] + timeout_seconds: u64, + + #[arg(long)] + no_color: bool, +} + +#[derive(Debug, Clone)] +struct Colors { + red: &'static str, + green: &'static str, + yellow: &'static str, + cyan: &'static str, + reset: &'static str, +} + +impl Colors { + fn new(enabled: bool) -> Self { + if enabled { + Self { + red: "\u{1b}[0;31m", + green: "\u{1b}[0;32m", + yellow: "\u{1b}[1;33m", + cyan: "\u{1b}[0;36m", + reset: "\u{1b}[0m", + } + } else { + Self { + red: "", + green: "", + yellow: "", + cyan: "", + reset: "", + } + } + } + + fn paint(&self, color: &str, text: &str) -> String { + format!("{color}{text}{}", self.reset) + } +} + +#[derive(Debug, Default)] +struct ContextState { + host_inactive: bool, + guard_healthy: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BucketStatus { + Fresh, + Stale, + Dead, + Empty, + EventDriven, + Inactive, + Unknown, +} + +#[derive(Debug, Clone)] +struct BucketRow { + label: &'static str, + last_id: String, + age: String, + status: BucketStatus, +} + +#[derive(Debug, Default)] +struct Summary { + fresh: usize, + stale: usize, + dead: usize, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let server = cli.server.trim_end_matches('/').to_string(); + let colors = Colors::new(!cli.no_color && std::env::var_os("NO_COLOR").is_none()); + let timeout = Duration::from_secs(cli.timeout_seconds.max(1)); + let client = Client::builder() + .timeout(timeout) + .no_proxy() + .pool_max_idle_per_host(0) + .build() + .context("build HTTP client")?; + let now = Utc::now(); + + println!( + "{}", + colors.paint( + colors.cyan, + &format!("=== ActivityWatch Full Check: {} ===", cli.host) + ) + ); + println!(); + + println!( + "{}", + colors.paint(colors.cyan, &format!("--- 1. AW Server ({server}) ---")) + ); + print!(" Connectivity... "); + let info = match get_json(&client, &format!("{server}/api/0/info")) { + Ok(value) => value, + Err(_) => { + println!(" {}", colors.paint(colors.red, "FAILED")); + return Ok(1); + } + }; + let Some(version) = info.get("version").and_then(Value::as_str) else { + println!(" {}", colors.paint(colors.red, "FAILED")); + return Ok(1); + }; + println!( + " {} (aw-server {version})", + colors.paint(colors.green, "OK") + ); + + print!(" CORS... "); + let cors_code = check_cors(&client, &server); + if cors_code == 200 { + println!("{}", colors.paint(colors.green, "OK")); + } else { + println!("{} (HTTP {cors_code})", colors.paint(colors.red, "FAIL")); + } + println!(); + + let context = read_context(&client, &server, &cli.host, now); + println!("{}", colors.paint(colors.cyan, "--- 2. Data Buckets ---")); + println!( + " {:<42} {:<8} {:<20} STATUS", + "BUCKET", "EVENTS", "LAST EVENT" + ); + println!( + " {:<42} {:<8} {:<20} ------", + "------------------------------------------", "--------", "--------------------" + ); + + let mut rows = Vec::new(); + for (bucket, label) in BUCKETS { + let row = read_bucket_row(&client, &server, &cli.host, bucket, label, now, &context); + println!( + " {:<42} {:<8} {:<20} {}", + row.label, + row.last_id, + row.age, + render_status(&colors, row.status) + ); + rows.push(row); + } + println!(); + + println!( + "{}", + colors.paint( + colors.cyan, + &format!("--- 3. RDP Host ({}) ---", cli.rdp_host) + ) + ); + print!(" WinRM (5985)... "); + if tcp_open(&cli.rdp_host, 5985, Duration::from_secs(5)) { + println!("{}", colors.paint(colors.green, "OK")); + } else { + println!("{}", colors.paint(colors.red, "UNREACHABLE")); + } + print!(" SSH (22)... "); + if tcp_open(&cli.rdp_host, 22, Duration::from_secs(5)) { + println!("{}", colors.paint(colors.green, "OK")); + } else { + println!( + "{} (normal for Windows)", + colors.paint(colors.yellow, "CLOSED") + ); + } + println!(); + + let summary = summarize(&rows); + println!("{}", colors.paint(colors.cyan, "--- 4. Summary ---")); + println!( + " FRESH: {}", + colors.paint(colors.green, &summary.fresh.to_string()) + ); + println!( + " STALE: {}", + colors.paint(colors.yellow, &summary.stale.to_string()) + ); + println!( + " DEAD: {}", + colors.paint(colors.red, &summary.dead.to_string()) + ); + + if summary.dead > 0 || summary.stale > 0 { + println!(); + println!( + " {} Some collectors may need restart on RDP host", + colors.paint(colors.red, "WARNING:") + ); + println!( + " Run: {}", + colors.paint( + colors.cyan, + "ansible -i ansible/inventory.ini rdp-prod -m win_shell -a 'schtasks /Run /TN \"ActivityWatch Recovery\"'" + ) + ); + } + + println!(); + println!("{}", colors.paint(colors.cyan, "=== Check Complete ===")); + println!(" Timestamp: {}", now.format("%Y-%m-%d %H:%M:%S UTC")); + Ok(0) +} + +fn read_context(client: &Client, server: &str, host: &str, now: DateTime) -> ContextState { + let mut state = ContextState::default(); + if let Ok(Some(event)) = latest_event(client, server, &format!("aw-worktime-sessions_{host}")) { + if let Some(ts) = event_timestamp(&event) { + let age = (now - ts).num_seconds(); + let active = event + .pointer("/data/active") + .and_then(Value::as_bool) + .unwrap_or(false); + if (0..900).contains(&age) && !active { + state.host_inactive = true; + } + } + } + if let Ok(Some(event)) = latest_event(client, server, &format!("aw-rus-collector-guard_{host}")) + { + if let Some(ts) = event_timestamp(&event) { + let age = (now - ts).num_seconds(); + let status = event + .pointer("/data/status") + .and_then(Value::as_str) + .unwrap_or(""); + let problems = event + .pointer("/data/problems") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0); + if (0..300).contains(&age) && status == "ok" && problems == 0 { + state.guard_healthy = true; + } + } + } + state +} + +fn read_bucket_row( + client: &Client, + server: &str, + host: &str, + bucket: &str, + label: &'static str, + now: DateTime, + context: &ContextState, +) -> BucketRow { + let bucket_full = format!("{bucket}_{host}"); + let event = latest_event(client, server, &bucket_full).ok().flatten(); + let Some(event) = event else { + return BucketRow { + label, + last_id: "0".to_string(), + age: "none".to_string(), + status: classify_bucket_no_events(bucket, context), + }; + }; + + let last_id = event + .get("id") + .map(json_value_to_string) + .unwrap_or_else(|| "0".to_string()); + let Some(ts_raw) = event.get("timestamp").and_then(Value::as_str) else { + return BucketRow { + label, + last_id, + age: "?".to_string(), + status: BucketStatus::Unknown, + }; + }; + let Some(ts) = parse_ts(ts_raw) else { + return BucketRow { + label, + last_id, + age: "?".to_string(), + status: BucketStatus::Unknown, + }; + }; + let age_sec = (now - ts).num_seconds().max(0); + BucketRow { + label, + last_id, + age: format_age(age_sec), + status: classify_bucket_age(bucket, age_sec, context), + } +} + +fn classify_bucket_age(bucket: &str, age_sec: i64, context: &ContextState) -> BucketStatus { + match bucket { + "aw-watcher-window" if context.host_inactive => return BucketStatus::Inactive, + "aw-dlp-endpoint-signals" if context.host_inactive && context.guard_healthy => { + return BucketStatus::Inactive; + } + _ => {} + } + + match bucket { + "aw-dlp-incidents" | "aw-dlp-review" | "aw-dlp-rules" | "aw-session-events" => { + if age_sec < 86_400 { + BucketStatus::Fresh + } else { + BucketStatus::EventDriven + } + } + _ if age_sec < 3_600 => BucketStatus::Fresh, + _ if age_sec < 86_400 => BucketStatus::Stale, + _ => BucketStatus::Dead, + } +} + +fn classify_bucket_no_events(bucket: &str, context: &ContextState) -> BucketStatus { + match bucket { + "aw-watcher-window" if context.host_inactive => BucketStatus::Inactive, + "aw-dlp-endpoint-signals" if context.host_inactive && context.guard_healthy => { + BucketStatus::Inactive + } + "aw-dlp-incidents" | "aw-dlp-review" | "aw-dlp-rules" | "aw-session-events" => { + BucketStatus::EventDriven + } + _ => BucketStatus::Empty, + } +} + +fn summarize(rows: &[BucketRow]) -> Summary { + let mut summary = Summary::default(); + for row in rows { + match row.status { + BucketStatus::Fresh | BucketStatus::EventDriven | BucketStatus::Inactive => { + summary.fresh += 1 + } + BucketStatus::Stale => summary.stale += 1, + BucketStatus::Dead | BucketStatus::Empty | BucketStatus::Unknown => summary.dead += 1, + } + } + summary +} + +fn render_status(colors: &Colors, status: BucketStatus) -> String { + match status { + BucketStatus::Fresh => colors.paint(colors.green, "FRESH"), + BucketStatus::Stale => colors.paint(colors.yellow, "STALE"), + BucketStatus::Dead => colors.paint(colors.red, "DEAD"), + BucketStatus::Empty => colors.paint(colors.red, "EMPTY"), + BucketStatus::EventDriven => colors.paint(colors.cyan, "EVENT-DRIVEN"), + BucketStatus::Inactive => colors.paint(colors.cyan, "INACTIVE"), + BucketStatus::Unknown => colors.paint(colors.red, "?"), + } +} + +fn latest_event(client: &Client, server: &str, bucket: &str) -> Result> { + let url = format!("{server}/api/0/buckets/{bucket}/events?limit=1"); + let value = get_json(client, &url)?; + Ok(value.as_array().and_then(|items| items.first()).cloned()) +} + +fn get_json(client: &Client, url: &str) -> Result { + client + .get(url) + .header("Connection", "close") + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn check_cors(client: &Client, server: &str) -> u16 { + let url = format!("{server}/api/0/settings/"); + client + .get(&url) + .header("Origin", "http://10.10.10.13:5600") + .send() + .map(|response| response.status().as_u16()) + .unwrap_or(0) +} + +fn tcp_open(host: &str, port: u16, timeout: Duration) -> bool { + let Ok(mut addrs) = (host, port).to_socket_addrs() else { + return false; + }; + addrs.any(|addr| TcpStream::connect_timeout(&addr, timeout).is_ok()) +} + +fn event_timestamp(event: &Value) -> Option> { + event + .get("timestamp") + .and_then(Value::as_str) + .and_then(parse_ts) +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .map(|ts| ts.with_timezone(&Utc)) + .ok() +} + +fn format_age(age_sec: i64) -> String { + if age_sec < 3_600 { + format!("{}m", age_sec / 60) + } else if age_sec < 86_400 { + format!("{}h", age_sec / 3_600) + } else { + format!("{}d", age_sec / 86_400) + } +} + +fn json_value_to_string(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn event_driven_bucket_is_not_dead_when_old() { + assert_eq!( + classify_bucket_age("aw-session-events", 100_000, &ContextState::default()), + BucketStatus::EventDriven + ); + } + + #[test] + fn inactive_context_suppresses_expected_idle_buckets() { + let context = ContextState { + host_inactive: true, + guard_healthy: true, + }; + assert_eq!( + classify_bucket_age("aw-watcher-window", 100_000, &context), + BucketStatus::Inactive + ); + assert_eq!( + classify_bucket_no_events("aw-dlp-endpoint-signals", &context), + BucketStatus::Inactive + ); + } + + #[test] + fn summary_treats_event_and_inactive_as_fresh_class() { + let rows = vec![ + BucketRow { + label: "a", + last_id: "0".to_string(), + age: "1m".to_string(), + status: BucketStatus::Fresh, + }, + BucketRow { + label: "b", + last_id: "0".to_string(), + age: "none".to_string(), + status: BucketStatus::EventDriven, + }, + BucketRow { + label: "c", + last_id: "0".to_string(), + age: "none".to_string(), + status: BucketStatus::Inactive, + }, + BucketRow { + label: "d", + last_id: "0".to_string(), + age: "none".to_string(), + status: BucketStatus::Empty, + }, + ]; + let summary = summarize(&rows); + assert_eq!(summary.fresh, 3); + assert_eq!(summary.stale, 0); + assert_eq!(summary.dead, 1); + } + + #[test] + fn formats_age_like_legacy_full_check() { + assert_eq!(format_age(59), "0m"); + assert_eq!(format_age(3_600), "1h"); + assert_eq!(format_age(86_400), "1d"); + } +} diff --git a/adk-rust/crates/check-install-kit-vs-repo/Cargo.toml b/adk-rust/crates/check-install-kit-vs-repo/Cargo.toml new file mode 100644 index 0000000..39e6c80 --- /dev/null +++ b/adk-rust/crates/check-install-kit-vs-repo/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "check-install-kit-vs-repo" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/check-install-kit-vs-repo/src/main.rs b/adk-rust/crates/check-install-kit-vs-repo/src/main.rs new file mode 100644 index 0000000..93f016c --- /dev/null +++ b/adk-rust/crates/check-install-kit-vs-repo/src/main.rs @@ -0,0 +1,283 @@ +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const DEFAULT_KIT_DIR: &str = "install-kit-awindows-20260427-211240"; +const MANIFEST_NAME: &str = "MANIFEST.txt"; +const ALLOWED_KIT_ONLY_FILES: &[&str] = &["README-INSTALL-KIT.txt"]; +const ALLOWED_KIT_ONLY_PREFIXES: &[&str] = &["server-configs-"]; + +#[derive(Debug, Parser)] +#[command(about = "Compare ActivityWatch-Russian install-kit contents against the repository")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long, default_value = DEFAULT_KIT_DIR)] + kit_dir: PathBuf, + + #[arg(long, default_value_t = false)] + json: bool, +} + +#[derive(Debug, Default, Serialize)] +struct Report { + compared_files: usize, + missing_in_repo: Vec, + mismatches: Vec, + powershell_mismatches: Vec, +} + +impl Report { + fn is_ok(&self) -> bool { + self.missing_in_repo.is_empty() && self.mismatches.is_empty() + } +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let report = compare_install_kit(&cli.root, &cli.kit_dir)?; + print_report(&report, cli.json)?; + Ok(if report.is_ok() { 0 } else { 1 }) +} + +fn compare_install_kit(root: &Path, kit_dir_arg: &Path) -> Result { + let root = root + .canonicalize() + .with_context(|| format!("canonicalize root {}", root.display()))?; + let kit_dir = if kit_dir_arg.is_absolute() { + kit_dir_arg.to_path_buf() + } else { + root.join(kit_dir_arg) + }; + if !kit_dir.exists() { + bail!("Install kit directory not found: {}", kit_dir.display()); + } + if !kit_dir.is_dir() { + bail!("Install kit path is not a directory: {}", kit_dir.display()); + } + + let mut report = Report::default(); + for kit_file in collect_files(&kit_dir)? { + let rel = kit_file + .strip_prefix(&kit_dir) + .with_context(|| format!("strip kit prefix from {}", kit_file.display()))?; + let rel_str = slash_path(rel); + if rel.file_name().and_then(|name| name.to_str()) == Some(MANIFEST_NAME) { + continue; + } + + let repo_file = root.join(rel); + if !repo_file.exists() { + if is_allowed_kit_only(&rel_str) { + continue; + } + report.missing_in_repo.push(rel_str); + continue; + } + if !repo_file.is_file() { + report.missing_in_repo.push(rel_str); + continue; + } + + report.compared_files += 1; + if sha256_file(&kit_file)? != sha256_file(&repo_file)? { + report.mismatches.push(rel_str); + } + } + + report.missing_in_repo.sort(); + report.mismatches.sort(); + report.powershell_mismatches = report + .mismatches + .iter() + .filter(|path| is_powershell_path(path)) + .cloned() + .collect(); + Ok(report) +} + +fn print_report(report: &Report, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(report)?); + return Ok(()); + } + + println!("Compared files: {}", report.compared_files); + println!("Missing in repo: {}", report.missing_in_repo.len()); + println!("Mismatched content: {}", report.mismatches.len()); + if !report.missing_in_repo.is_empty() { + println!("--- Missing in repo ---"); + for path in &report.missing_in_repo { + println!("{path}"); + } + } + if !report.mismatches.is_empty() { + println!("--- Mismatches ---"); + for path in &report.mismatches { + println!("{path}"); + } + } + println!( + "PowerShell mismatches: {}", + report.powershell_mismatches.len() + ); + if !report.powershell_mismatches.is_empty() { + println!("--- PowerShell mismatches ---"); + for path in &report.powershell_mismatches { + println!("{path}"); + } + } + Ok(()) +} + +fn collect_files(root: &Path) -> Result> { + let mut out = Vec::new(); + collect_files_inner(root, &mut out)?; + out.sort(); + Ok(out) +} + +fn collect_files_inner(path: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("read file type {}", entry_path.display()))?; + if file_type.is_dir() { + collect_files_inner(&entry_path, out)?; + } else if (file_type.is_file() || file_type.is_symlink()) && entry_path.is_file() { + out.push(entry_path); + } + } + Ok(()) +} + +fn sha256_file(path: &Path) -> Result { + let file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buf = [0_u8; 64 * 1024]; + loop { + let read = reader + .read(&mut buf) + .with_context(|| format!("read {}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn is_allowed_kit_only(rel: &str) -> bool { + ALLOWED_KIT_ONLY_FILES.contains(&rel) + || ALLOWED_KIT_ONLY_PREFIXES + .iter() + .any(|prefix| rel.starts_with(prefix)) +} + +fn is_powershell_path(path: &str) -> bool { + path.starts_with("windows/") + && (path.ends_with(".ps1") || path.ends_with(".psm1") || path.ends_with(".psd1")) +} + +fn slash_path(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +#[allow(dead_code)] +fn sorted_set(paths: &[String]) -> BTreeSet { + paths.iter().cloned().collect() +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::{compare_install_kit, sorted_set}; + + #[test] + fn reports_clean_tree() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let kit = root.join("install-kit-awindows-20260427-211240"); + fs::create_dir_all(kit.join("windows")).unwrap(); + fs::write(root.join("common.txt"), "same").unwrap(); + fs::write(kit.join("common.txt"), "same").unwrap(); + fs::create_dir_all(root.join("windows")).unwrap(); + fs::write(root.join("windows/script.ps1"), "same").unwrap(); + fs::write(kit.join("windows/script.ps1"), "same").unwrap(); + fs::write(kit.join("README-INSTALL-KIT.txt"), "kit-only").unwrap(); + fs::write(kit.join("MANIFEST.txt"), "ignored").unwrap(); + + let report = compare_install_kit(root, &kit).unwrap(); + assert!(report.is_ok()); + assert_eq!(report.compared_files, 2); + } + + #[test] + fn reports_mismatches_and_powershell_subset() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let kit = root.join("install-kit-awindows-20260427-211240"); + fs::create_dir_all(root.join("windows")).unwrap(); + fs::create_dir_all(kit.join("windows")).unwrap(); + fs::write(root.join("windows/script.ps1"), "repo").unwrap(); + fs::write(kit.join("windows/script.ps1"), "kit").unwrap(); + fs::write(root.join("plain.txt"), "repo").unwrap(); + fs::write(kit.join("plain.txt"), "kit").unwrap(); + + let report = compare_install_kit(root, &kit).unwrap(); + assert!(!report.is_ok()); + assert_eq!( + sorted_set(&report.mismatches), + sorted_set(&["plain.txt".to_string(), "windows/script.ps1".to_string()]) + ); + assert_eq!( + report.powershell_mismatches, + vec!["windows/script.ps1".to_string()] + ); + } + + #[test] + fn reports_unexpected_kit_only_files() { + let tmp = tempdir().unwrap(); + let root = tmp.path(); + let kit = root.join("install-kit-awindows-20260427-211240"); + fs::create_dir_all(&kit).unwrap(); + fs::write(kit.join("unexpected.txt"), "kit").unwrap(); + fs::create_dir_all(kit.join("server-configs-192.168.100.18")).unwrap(); + fs::write( + kit.join("server-configs-192.168.100.18/config.deployment-config.json"), + "{}", + ) + .unwrap(); + + let report = compare_install_kit(root, &kit).unwrap(); + assert_eq!(report.missing_in_repo, vec!["unexpected.txt"]); + } +} diff --git a/adk-rust/crates/detmir-auto/Cargo.toml b/adk-rust/crates/detmir-auto/Cargo.toml new file mode 100644 index 0000000..2f847cd --- /dev/null +++ b/adk-rust/crates/detmir-auto/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "detmir-auto" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-core.workspace = true +detmir-state.workspace = true +fs2.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/detmir-auto/src/main.rs b/adk-rust/crates/detmir-auto/src/main.rs new file mode 100644 index 0000000..bb52ee9 --- /dev/null +++ b/adk-rust/crates/detmir-auto/src/main.rs @@ -0,0 +1,661 @@ +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::symlink; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::{Duration, SystemTime}; + +use anyhow::{Context, Result}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use detmir_core::exit_codes; +use detmir_state::write_json_atomic; +use fs2::FileExt; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_STATE_DIR: &str = "/var/lib/detmir-ai"; +const DEFAULT_CHECK_BIN: &str = "detmir-check"; +const DEFAULT_DLP_BIN: &str = "detmir-dlp"; +const DEFAULT_HEAL_BIN: &str = "detmir-heal-safe-rust"; +const DEFAULT_POLLI_BIN: &str = "polli-chat"; + +#[derive(Debug, Parser)] +#[command(about = "DetMir autonomous check/report orchestration.")] +struct Cli { + #[arg(long, default_value = DEFAULT_STATE_DIR)] + state_dir: PathBuf, + + #[arg(long)] + lock_dir: Option, + + #[arg(long, default_value = DEFAULT_CHECK_BIN)] + check_bin: String, + + #[arg(long, default_value = DEFAULT_DLP_BIN)] + dlp_bin: String, + + #[arg(long, default_value = DEFAULT_HEAL_BIN)] + heal_bin: String, + + #[arg(long, default_value = DEFAULT_POLLI_BIN)] + polli_bin: String, + + #[arg(long, default_value_t = 14)] + retain_days: u64, + + #[arg(long, default_value_t = 120)] + command_timeout_seconds: u64, + + #[arg(long, default_value_t = 120)] + report_timeout_seconds: u64, + + #[arg(long)] + no_report: bool, + + #[arg(long)] + enable_heal: bool, + + #[arg(long)] + no_heal: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + self.state_dir = env_path("DETMIR_AI_STATE_DIR").unwrap_or(self.state_dir); + self.lock_dir = env_path("DETMIR_AI_RUN_DIR").or(self.lock_dir); + self.check_bin = env_string("DETMIR_CHECK_BIN").unwrap_or(self.check_bin); + self.dlp_bin = env_string("DETMIR_DLP_BIN").unwrap_or(self.dlp_bin); + self.heal_bin = env_string("DETMIR_HEAL_BIN").unwrap_or(self.heal_bin); + self.polli_bin = env_string("DETMIR_POLLI_BIN").unwrap_or(self.polli_bin); + self.retain_days = env_string("DETMIR_AI_RETAIN_DAYS") + .and_then(|value| value.parse().ok()) + .unwrap_or(self.retain_days); + self.report_timeout_seconds = env_string("DETMIR_REPORT_TIMEOUT_SECONDS") + .and_then(|value| value.parse().ok()) + .unwrap_or(self.report_timeout_seconds); + if env_string("DETMIR_AUTO_HEAL").is_some_and(|value| value == "1") { + self.enable_heal = true; + } + if self.no_heal { + self.enable_heal = false; + } + self + } + + fn lock_dir(&self) -> PathBuf { + self.lock_dir + .clone() + .unwrap_or_else(|| self.state_dir.join("locks")) + } +} + +#[derive(Debug)] +struct RunPaths { + state_dir: PathBuf, + run_dir: PathBuf, + reports_dir: PathBuf, + check_file: PathBuf, + dlp_file: PathBuf, + check_rc_file: PathBuf, + dlp_rc_file: PathBuf, + heal_rc_file: PathBuf, + heal_log: PathBuf, + bundle_file: PathBuf, + report_file: PathBuf, + state_file: PathBuf, +} + +#[derive(Debug, Serialize)] +struct AutoSummary { + check_rc: i32, + dlp_rc: i32, + check_ok: bool, + dlp_ok: bool, + severity: String, + needs_heal: bool, + reasons: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + detmir_summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dlp_counts: Option, +} + +fn env_string(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn env_path(name: &str) -> Option { + env_string(name).map(PathBuf::from) +} + +fn utc_stamp() -> String { + Utc::now().format("%Y%m%d-%H%M%S").to_string() +} + +fn utc_iso() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn init_paths(state_dir: PathBuf) -> Result { + let stamp = utc_stamp(); + let run_dir = state_dir.join("runs").join(&stamp); + let reports_dir = state_dir.join("reports"); + fs::create_dir_all(&run_dir)?; + fs::create_dir_all(&reports_dir)?; + fs::create_dir_all(state_dir.join("logs"))?; + + Ok(RunPaths { + state_dir: state_dir.clone(), + run_dir: run_dir.clone(), + reports_dir: reports_dir.clone(), + check_file: run_dir.join("detmir-check.json"), + dlp_file: run_dir.join("detmir-dlp.json"), + check_rc_file: run_dir.join("check.rc"), + dlp_rc_file: run_dir.join("dlp.rc"), + heal_rc_file: run_dir.join("heal.rc"), + heal_log: run_dir.join("heal.log"), + bundle_file: run_dir.join("bundle.txt"), + report_file: reports_dir.join(format!("detmir-report-{stamp}.md")), + state_file: state_dir.join(format!("state-{stamp}.json")), + }) +} + +fn acquire_lock(lock_dir: &Path) -> Result> { + fs::create_dir_all(lock_dir)?; + let lock_path = lock_dir.join("detmir-auto.lock"); + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&lock_path) + .with_context(|| format!("failed to open lock {}", lock_path.display()))?; + match file.try_lock_exclusive() { + Ok(()) => Ok(Some(file)), + Err(err) => { + if err.kind() == std::io::ErrorKind::WouldBlock { + println!("detmir-auto: another run is active"); + Ok(None) + } else { + Err(err).with_context(|| format!("failed to lock {}", lock_path.display())) + } + } + } +} + +fn run_to_file( + command: &str, + args: &[&str], + output_path: &Path, + rc_path: &Path, + timeout: Duration, +) -> Result { + let stdout = File::create(output_path) + .with_context(|| format!("failed to create {}", output_path.display()))?; + let stderr_path = output_path.with_extension("stderr"); + let stderr = File::create(&stderr_path) + .with_context(|| format!("failed to create {}", stderr_path.display()))?; + + let mut child = Command::new(command) + .args(args) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .spawn() + .with_context(|| format!("failed to execute {command}"))?; + + let started = std::time::Instant::now(); + let rc = loop { + if let Some(status) = child.try_wait()? { + break status.code().unwrap_or(1); + } + if started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + let mut stderr = OpenOptions::new().append(true).open(&stderr_path)?; + writeln!( + stderr, + "{command} timed out after {} seconds", + timeout.as_secs() + )?; + break 124; + } + std::thread::sleep(Duration::from_millis(200)); + }; + + if fs::metadata(&stderr_path) + .map(|meta| meta.len()) + .unwrap_or(0) + == 0 + { + let _ = fs::remove_file(&stderr_path); + } + + fs::write(rc_path, format!("{rc}\n"))?; + Ok(rc) +} + +fn read_rc(path: &Path) -> i32 { + fs::read_to_string(path) + .ok() + .and_then(|text| text.trim().parse().ok()) + .unwrap_or(1) +} + +fn summarize(paths: &RunPaths) -> AutoSummary { + let check_rc = read_rc(&paths.check_rc_file); + let dlp_rc = read_rc(&paths.dlp_rc_file); + let mut summary = AutoSummary { + check_rc, + dlp_rc, + check_ok: false, + dlp_ok: false, + severity: if check_rc != 0 || dlp_rc != 0 { + "FAIL".to_string() + } else { + "OK".to_string() + }, + needs_heal: check_rc != 0 || dlp_rc != 0, + reasons: Vec::new(), + detmir_summary: None, + dlp_counts: None, + }; + + match read_json(&paths.check_file) { + Ok(check) => { + summary.check_ok = check.get("ok").and_then(Value::as_bool).unwrap_or(false); + let check_summary = check.get("summary").cloned().unwrap_or(Value::Null); + if check_summary.is_object() { + if !summary.check_ok + && (int_field(&check_summary, "bucket_dead") > 0 + || int_field(&check_summary, "bucket_stale") > 0 + || int_field(&check_summary, "service_failures") > 0) + { + summary.reasons.push( + "detmir-check has stale/dead bucket or required service failure" + .to_string(), + ); + } + summary.detmir_summary = Some(check_summary); + } + } + Err(err) => summary + .reasons + .push(format!("detmir-check parse failed: {err}")), + } + + match read_json(&paths.dlp_file) { + Ok(dlp) => { + summary.dlp_ok = dlp.get("ok").and_then(Value::as_bool).unwrap_or(false); + let counts = dlp.get("counts").cloned().unwrap_or(Value::Null); + if counts.is_object() { + if !summary.dlp_ok + && (int_field(&counts, "fail") > 0 || int_field(&counts, "warn") > 0) + { + summary + .reasons + .push("dlp-health-check has warn/fail".to_string()); + } + summary.dlp_counts = Some(counts); + } + } + Err(err) => summary + .reasons + .push(format!("detmir-dlp parse failed: {err}")), + } + + if summary.check_ok && summary.dlp_ok { + summary.severity = "OK".to_string(); + summary.needs_heal = false; + } else if summary.reasons.is_empty() { + summary.severity = "WARN".to_string(); + } else { + summary.severity = "FAIL".to_string(); + } + + summary +} + +fn read_json(path: &Path) -> Result { + let raw = fs::read_to_string(path)?; + Ok(serde_json::from_str(&raw)?) +} + +fn int_field(value: &Value, key: &str) -> i64 { + value.get(key).and_then(Value::as_i64).unwrap_or(0) +} + +fn write_bundle(paths: &RunPaths, summary_after: &AutoSummary) -> Result<()> { + let mut out = String::new(); + out.push_str( + "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет.\n", + ); + out.push_str("Структура ответа:\n"); + out.push_str("1. Состояние: OK/WARN/FAIL\n"); + out.push_str("2. Что важно\n"); + out.push_str("3. Что уже сделал автомат\n"); + out.push_str("4. Что сделать человеку, если нужно\n\n"); + out.push_str("Правила:\n"); + out.push_str("- Не предлагай рестарты, если факты чистые.\n"); + out.push_str("- Отличай event-driven bucket от dead/stale.\n"); + out.push_str("- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail.\n"); + out.push_str( + "- Auto-heal умеет только серверные systemd-сервисы AW/DLP; Windows/RDP не трогает.\n\n", + ); + out.push_str("=== summary-before ===\n"); + out.push_str( + &fs::read_to_string(paths.run_dir.join("summary-before.json")).unwrap_or_default(), + ); + out.push_str("\n\n=== summary-after ===\n"); + out.push_str(&serde_json::to_string_pretty(summary_after)?); + out.push_str("\n\n=== heal-log ===\n"); + out.push_str(&fs::read_to_string(&paths.heal_log).unwrap_or_default()); + out.push_str("\n\n=== detmir-check ===\n"); + out.push_str(&truncate_file(&paths.check_file, 1600)); + out.push_str("\n\n=== detmir-dlp ===\n"); + out.push_str(&truncate_file(&paths.dlp_file, 1600)); + fs::write(&paths.bundle_file, out)?; + Ok(()) +} + +fn truncate_file(path: &Path, max_lines: usize) -> String { + fs::read_to_string(path) + .unwrap_or_default() + .lines() + .take(max_lines) + .collect::>() + .join("\n") +} + +fn write_report( + paths: &RunPaths, + polli_bin: &str, + no_report: bool, + report_timeout: Duration, + summary: &AutoSummary, +) -> Result<()> { + let mut report = format!( + "# DetMir Autonomous Report\n\n- generated_at_utc: {}\n- run_dir: {}\n\n", + utc_iso(), + paths.run_dir.display() + ); + + if no_report { + report.push_str("Pollinations report skipped; raw summary follows.\n"); + report.push_str(&serde_json::to_string_pretty(summary)?); + } else { + let bundle = File::open(&paths.bundle_file)?; + match run_report_command(polli_bin, bundle, report_timeout) { + Ok(output) if output.status.success() => { + report.push_str(&String::from_utf8_lossy(&output.stdout)); + } + Ok(output) => { + report.push_str("Pollinations report failed; raw summary follows.\n"); + if !output.stderr.is_empty() { + report.push_str(&String::from_utf8_lossy(&output.stderr)); + report.push('\n'); + } + report.push_str(&serde_json::to_string_pretty(summary)?); + } + Err(err) => { + report.push_str("Pollinations report failed; raw summary follows.\n"); + report.push_str(&format!("{err}\n")); + report.push_str(&serde_json::to_string_pretty(summary)?); + } + } + } + + fs::write(&paths.report_file, report)?; + Ok(()) +} + +fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Result { + let mut child = Command::new(polli_bin) + .args(["--model", "text.daily", "--max-tokens", "900"]) + .stdin(Stdio::from(bundle)) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {polli_bin}"))?; + + let started = std::time::Instant::now(); + loop { + if child.try_wait()?.is_some() { + return child + .wait_with_output() + .with_context(|| format!("failed to collect {polli_bin} output")); + } + if started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + anyhow::bail!( + "Pollinations report timed out after {} seconds", + timeout.as_secs() + ); + } + std::thread::sleep(Duration::from_millis(200)); + } +} + +fn update_latest(paths: &RunPaths) -> Result<()> { + update_symlink(&paths.run_dir, &paths.state_dir.join("latest-run"))?; + update_symlink( + &paths.report_file, + &paths.state_dir.join("latest-report.md"), + )?; + update_symlink( + &paths.state_file, + &paths.state_dir.join("latest-state.json"), + )?; + Ok(()) +} + +fn update_symlink(target: &Path, link: &Path) -> Result<()> { + let tmp = link.with_extension(format!("tmp.{}", std::process::id())); + let _ = fs::remove_file(&tmp); + symlink(target, &tmp)?; + fs::rename(&tmp, link) + .with_context(|| format!("failed to update symlink {}", link.display()))?; + Ok(()) +} + +fn cleanup_retention(paths: &RunPaths, retain_days: u64) -> Result<()> { + let cutoff = SystemTime::now() + .checked_sub(Duration::from_secs(retain_days * 24 * 60 * 60)) + .unwrap_or(SystemTime::UNIX_EPOCH); + cleanup_old_dirs(&paths.state_dir.join("runs"), cutoff)?; + cleanup_old_files(&paths.reports_dir, "detmir-report-", Some(".md"), cutoff)?; + cleanup_old_files(&paths.state_dir, "state-", Some(".json"), cutoff)?; + Ok(()) +} + +fn cleanup_old_dirs(dir: &Path, cutoff: SystemTime) -> Result<()> { + if !dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() && is_old(&entry.path(), cutoff) { + fs::remove_dir_all(entry.path())?; + } + } + Ok(()) +} + +fn cleanup_old_files( + dir: &Path, + prefix: &str, + suffix: Option<&str>, + cutoff: SystemTime, +) -> Result<()> { + if !dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(dir)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + let suffix_ok = suffix.is_none_or(|suffix| name.ends_with(suffix)); + if entry.file_type()?.is_file() + && name.starts_with(prefix) + && suffix_ok + && is_old(&entry.path(), cutoff) + { + fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +fn is_old(path: &Path, cutoff: SystemTime) -> bool { + path.metadata() + .and_then(|meta| meta.modified()) + .map(|modified| modified < cutoff) + .unwrap_or(false) +} + +fn main() -> Result<()> { + let args = Cli::parse().apply_env(); + let Some(_lock) = acquire_lock(&args.lock_dir())? else { + return Ok(()); + }; + + let paths = init_paths(args.state_dir.clone())?; + let command_timeout = Duration::from_secs(args.command_timeout_seconds); + let report_timeout = Duration::from_secs(args.report_timeout_seconds); + run_to_file( + &args.check_bin, + &["--json"], + &paths.check_file, + &paths.check_rc_file, + command_timeout, + )?; + run_to_file( + &args.dlp_bin, + &[], + &paths.dlp_file, + &paths.dlp_rc_file, + command_timeout, + )?; + let summary_before = summarize(&paths); + write_json_atomic(paths.run_dir.join("summary-before.json"), &summary_before)?; + + if summary_before.needs_heal && args.enable_heal { + fs::write( + &paths.heal_log, + format!("auto-heal started at {}\n", utc_iso()), + )?; + let heal_rc = run_to_file( + &args.heal_bin, + &["--apply", "--json"], + &paths.heal_log, + &paths.heal_rc_file, + command_timeout, + )?; + let mut heal_log = OpenOptions::new().append(true).open(&paths.heal_log)?; + writeln!( + heal_log, + "\nauto-heal finished at {} rc={heal_rc}", + utc_iso() + )?; + std::thread::sleep(Duration::from_secs(10)); + run_to_file( + &args.check_bin, + &["--json"], + &paths.check_file, + &paths.check_rc_file, + command_timeout, + )?; + run_to_file( + &args.dlp_bin, + &[], + &paths.dlp_file, + &paths.dlp_rc_file, + command_timeout, + )?; + } else { + fs::write( + &paths.heal_log, + if summary_before.needs_heal { + "auto-heal skipped (disabled)\n" + } else { + "auto-heal skipped\n" + }, + )?; + fs::write(&paths.heal_rc_file, "0\n")?; + } + + let summary_after = summarize(&paths); + write_json_atomic(&paths.state_file, &summary_after)?; + write_bundle(&paths, &summary_after)?; + write_report( + &paths, + &args.polli_bin, + args.no_report, + report_timeout, + &summary_after, + )?; + update_latest(&paths)?; + cleanup_retention(&paths, args.retain_days)?; + + print!("{}", fs::read_to_string(&paths.report_file)?); + std::io::stdout().flush().ok(); + + let final_check_rc = read_rc(&paths.check_rc_file); + let final_dlp_rc = read_rc(&paths.dlp_rc_file); + std::process::exit( + if summary_after.severity == "OK" && final_check_rc == 0 && final_dlp_rc == 0 { + exit_codes::OK + } else { + exit_codes::CHECK_FAILED + }, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn summarizes_clean_inputs() { + let dir = tempfile::tempdir().unwrap(); + let paths = init_paths(dir.path().to_path_buf()).unwrap(); + fs::write( + &paths.check_file, + r#"{"ok": true, "summary": {"bucket_ok": 8, "bucket_stale": 0, "bucket_dead": 0, "service_failures": 0, "service_warnings": 0}}"#, + ) + .unwrap(); + fs::write( + &paths.dlp_file, + r#"{"ok": true, "counts": {"ok": 22, "warn": 0, "fail": 0}}"#, + ) + .unwrap(); + fs::write(&paths.check_rc_file, "0\n").unwrap(); + fs::write(&paths.dlp_rc_file, "0\n").unwrap(); + let summary = summarize(&paths); + assert_eq!(summary.severity, "OK"); + assert!(!summary.needs_heal); + assert!(summary.reasons.is_empty()); + } + + #[test] + fn does_not_keep_reasons_when_child_reports_ok() { + let dir = tempfile::tempdir().unwrap(); + let paths = init_paths(dir.path().to_path_buf()).unwrap(); + fs::write( + &paths.check_file, + r#"{"ok": true, "summary": {"bucket_ok": 8, "bucket_stale": 0, "bucket_dead": 0, "service_failures": 0, "service_warnings": 0}}"#, + ) + .unwrap(); + fs::write( + &paths.dlp_file, + r#"{"ok": true, "counts": {"ok": 21, "warn": 1, "fail": 0}}"#, + ) + .unwrap(); + fs::write(&paths.check_rc_file, "0\n").unwrap(); + fs::write(&paths.dlp_rc_file, "0\n").unwrap(); + let summary = summarize(&paths); + assert_eq!(summary.severity, "OK"); + assert!(!summary.needs_heal); + assert!(summary.reasons.is_empty()); + } +} diff --git a/adk-rust/crates/detmir-aw-client/Cargo.toml b/adk-rust/crates/detmir-aw-client/Cargo.toml new file mode 100644 index 0000000..0f9e41f --- /dev/null +++ b/adk-rust/crates/detmir-aw-client/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "detmir-aw-client" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +detmir-core.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/detmir-aw-client/src/lib.rs b/adk-rust/crates/detmir-aw-client/src/lib.rs new file mode 100644 index 0000000..cf4ca07 --- /dev/null +++ b/adk-rust/crates/detmir-aw-client/src/lib.rs @@ -0,0 +1,119 @@ +use std::time::Duration; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use detmir_core::parse_utc_rfc3339; +use reqwest::blocking::Client; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +const DEFAULT_GET_ATTEMPTS: usize = 6; + +#[derive(Debug, Clone)] +pub struct ActivityWatchClient { + api_base: String, + client: Client, +} + +impl ActivityWatchClient { + pub fn new(api_base: impl Into, timeout: Duration) -> Result { + let client = Client::builder() + .timeout(timeout) + .no_proxy() + .build() + .context("failed to build ActivityWatch HTTP client")?; + Ok(Self { + api_base: api_base.into().trim_end_matches('/').to_string(), + client, + }) + } + + pub fn get_json(&self, path: &str) -> Result { + let url = self.url(path); + let mut last_error = None; + for attempt in 0..DEFAULT_GET_ATTEMPTS { + let result = self + .client + .get(&url) + .send() + .with_context(|| format!("ActivityWatch request failed: {url}")) + .and_then(|response| { + response.error_for_status().with_context(|| { + format!("ActivityWatch returned non-success status: {url}") + }) + }) + .and_then(|response| { + response + .json() + .with_context(|| format!("failed to parse ActivityWatch JSON: {url}")) + }); + + match result { + Ok(value) => return Ok(value), + Err(err) => last_error = Some(err), + } + + if attempt + 1 < DEFAULT_GET_ATTEMPTS { + std::thread::sleep(Duration::from_millis(500 * (attempt as u64 + 1))); + } + } + + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("ActivityWatch request failed: {url}"))) + } + + pub fn latest_event(&self, bucket_id: &str) -> Result> { + let path = format!("/buckets/{bucket_id}/events?limit=1"); + let mut events: Vec = self.get_json(&path)?; + events.sort_by(|left, right| right.timestamp.cmp(&left.timestamp)); + Ok(events.into_iter().next()) + } + + fn url(&self, path: &str) -> String { + if path.starts_with('/') { + format!("{}{}", self.api_base, path) + } else { + format!("{}/{}", self.api_base, path) + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AwEvent { + pub timestamp: String, + #[serde(default)] + pub duration: f64, + #[serde(default)] + pub data: serde_json::Value, +} + +impl AwEvent { + pub fn timestamp_utc(&self) -> Result> { + parse_utc_rfc3339(&self.timestamp) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_paths_without_double_slashes() { + let client = + ActivityWatchClient::new("http://127.0.0.1:5600/api/0/", Duration::from_secs(1)) + .unwrap(); + assert_eq!( + client.url("/buckets/x/events?limit=1"), + "http://127.0.0.1:5600/api/0/buckets/x/events?limit=1" + ); + } + + #[test] + fn parses_event_timestamp() { + let event = AwEvent { + timestamp: "2026-05-31T10:20:30Z".to_string(), + duration: 0.0, + data: serde_json::json!({}), + }; + assert_eq!(event.timestamp_utc().unwrap().timestamp(), 1_780_222_830); + } +} diff --git a/adk-rust/crates/detmir-check/Cargo.toml b/adk-rust/crates/detmir-check/Cargo.toml new file mode 100644 index 0000000..ca6f5ef --- /dev/null +++ b/adk-rust/crates/detmir-check/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "detmir-check" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-aw-client.workspace = true +detmir-core.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/detmir-check/src/main.rs b/adk-rust/crates/detmir-check/src/main.rs new file mode 100644 index 0000000..da5c51f --- /dev/null +++ b/adk-rust/crates/detmir-check/src/main.rs @@ -0,0 +1,522 @@ +use std::net::{SocketAddr, TcpStream}; +use std::time::Duration; + +use anyhow::{Context, Result}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use detmir_aw_client::ActivityWatchClient; +use detmir_core::{exit_codes, now_utc_rfc3339}; +use reqwest::blocking::Client; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_AW_API: &str = "http://10.10.10.13:5600/api/0"; +const DEFAULT_WORKTIME_URL: &str = "http://10.10.10.13:5610"; +const DEFAULT_ONE_C_URL: &str = "http://10.10.10.2:8710"; +const DEFAULT_HOSTNAME: &str = "SHARKON2025"; +const DEFAULT_GATEWAY_HOST: &str = "dm.iri1968.dpdns.org"; + +#[derive(Debug, Parser)] +#[command(about = "Autonomous read-only DetMir contour check from Proxmox.")] +struct Cli { + #[arg(long)] + json: bool, + + #[arg(long, default_value = DEFAULT_AW_API)] + aw_api: String, + + #[arg(long, default_value = DEFAULT_WORKTIME_URL)] + worktime_url: String, + + #[arg(long, default_value = DEFAULT_ONE_C_URL)] + one_c_url: String, + + #[arg(long, default_value = DEFAULT_HOSTNAME)] + hostname: String, + + #[arg(long, default_value_t = 5)] + service_timeout_seconds: u64, + + #[arg(long, default_value_t = 8)] + bucket_timeout_seconds: u64, + + #[arg(long, default_value_t = 3.0)] + tcp_timeout_seconds: f64, +} + +#[derive(Debug, Clone, Copy)] +enum BucketMode { + Fresh, + InactiveOk, + EventDriven, +} + +impl BucketMode { + fn as_str(self) -> &'static str { + match self { + Self::Fresh => "fresh", + Self::InactiveOk => "inactive_ok", + Self::EventDriven => "event_driven", + } + } +} + +#[derive(Debug)] +struct BucketSpec { + label: &'static str, + bucket: String, + max_age_seconds: Option, + mode: BucketMode, +} + +#[derive(Debug, Serialize)] +struct CheckReport { + ok: bool, + #[serde(rename = "generatedAtUtc")] + generated_at_utc: String, + services: Vec, + buckets: Vec, + summary: CheckSummary, +} + +#[derive(Debug, Serialize)] +struct CheckSummary { + bucket_ok: usize, + bucket_stale: usize, + bucket_dead: usize, + service_failures: usize, + service_warnings: usize, +} + +#[derive(Debug, Serialize)] +struct ServiceCheck { + name: String, + required: bool, + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + payload: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +#[derive(Debug, Serialize)] +struct BucketCheck { + label: String, + bucket: String, + mode: String, + status: String, + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + event_count_sample: Option, + #[serde(skip_serializing_if = "Option::is_none")] + latest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + age_seconds: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +fn env_or_default(name: &str, default: &str) -> String { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_string()) +} + +fn bucket_specs(hostname: &str) -> Vec { + vec![ + BucketSpec { + label: "AFK watcher", + bucket: format!("aw-watcher-afk_{hostname}"), + max_age_seconds: Some(15 * 60), + mode: BucketMode::Fresh, + }, + BucketSpec { + label: "Window watcher", + bucket: format!("aw-watcher-window_{hostname}"), + max_age_seconds: Some(2 * 60 * 60), + mode: BucketMode::InactiveOk, + }, + BucketSpec { + label: "Worktime sessions", + bucket: format!("aw-worktime-sessions_{hostname}"), + max_age_seconds: Some(5 * 60), + mode: BucketMode::Fresh, + }, + BucketSpec { + label: "Session events", + bucket: format!("aw-session-events_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + BucketSpec { + label: "DLP signals", + bucket: format!("aw-dlp-endpoint-signals_{hostname}"), + max_age_seconds: Some(10 * 60), + mode: BucketMode::Fresh, + }, + BucketSpec { + label: "DLP incidents", + bucket: format!("aw-dlp-incidents_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + BucketSpec { + label: "DLP review", + bucket: format!("aw-dlp-review_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + BucketSpec { + label: "DLP rules", + bucket: format!("aw-dlp-rules_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + ] +} + +fn build_headers(items: &[(&str, &str)]) -> Result { + let mut headers = HeaderMap::new(); + for (name, value) in items { + headers.insert( + HeaderName::from_bytes(name.as_bytes())?, + HeaderValue::from_str(value)?, + ); + } + Ok(headers) +} + +fn fetch_text( + url: &str, + timeout: Duration, + insecure: bool, + headers: HeaderMap, + attempts: usize, +) -> Result { + let client = Client::builder() + .timeout(timeout) + .danger_accept_invalid_certs(insecure) + .no_proxy() + .build() + .context("failed to build HTTP client")?; + let mut last_error = None; + for attempt in 0..attempts.max(1) { + match client.get(url).headers(headers.clone()).send() { + Ok(response) => match response.error_for_status() { + Ok(response) => { + return response.text().context("failed to read HTTP response body"); + } + Err(err) => last_error = Some(err.into()), + }, + Err(err) => last_error = Some(anyhow::anyhow!("{err:#}")), + } + if attempt + 1 < attempts.max(1) { + std::thread::sleep(Duration::from_millis(500)); + } + } + Err(last_error.unwrap_or_else(|| anyhow::anyhow!("HTTP request failed"))) +} + +fn service_checks(args: &Cli) -> Vec { + let timeout = Duration::from_secs(args.service_timeout_seconds); + let one_c_url = args.one_c_url.trim_end_matches('/'); + let services = [ + ( + "aw-info", + format!("{}/info", args.aw_api.trim_end_matches('/')), + false, + false, + HeaderMap::new(), + ), + ( + "worktime-today", + format!( + "{}/reports/worktime/today", + args.worktime_url.trim_end_matches('/') + ), + false, + false, + HeaderMap::new(), + ), + ( + "1c-api-health", + format!("{one_c_url}/api/health"), + false, + false, + HeaderMap::new(), + ), + ( + "gateway-healthz", + "https://127.0.0.1/healthz".to_string(), + true, + true, + build_headers(&[("Host", DEFAULT_GATEWAY_HOST)]).unwrap_or_default(), + ), + ]; + + let mut checks = Vec::new(); + for (name, url, insecure, required, headers) in services { + match fetch_text(&url, timeout, insecure, headers, 2) { + Ok(raw) => { + let payload = serde_json::from_str::(&raw) + .unwrap_or_else(|_| Value::String(raw.trim().to_string())); + checks.push(ServiceCheck { + name: name.to_string(), + required, + ok: true, + url: Some(url), + payload: Some(payload), + error: None, + }); + } + Err(err) => checks.push(ServiceCheck { + name: name.to_string(), + required, + ok: false, + url: Some(url), + payload: None, + error: Some(err.to_string()), + }), + } + } + + checks.push(tcp_check( + "192.168.100.18", + 5985, + args.tcp_timeout_seconds, + true, + )); + checks.push(tcp_check( + "192.168.100.18", + 22, + args.tcp_timeout_seconds, + true, + )); + checks +} + +fn tcp_check(host: &str, port: u16, timeout_seconds: f64, required: bool) -> ServiceCheck { + let name = format!("tcp:{host}:{port}"); + let timeout = Duration::from_secs_f64(timeout_seconds); + let addr = format!("{host}:{port}"); + match addr.parse::() { + Ok(addr) => match TcpStream::connect_timeout(&addr, timeout) { + Ok(_) => ServiceCheck { + name, + required, + ok: true, + url: None, + payload: None, + error: None, + }, + Err(err) => ServiceCheck { + name, + required, + ok: false, + url: None, + payload: None, + error: Some(err.to_string()), + }, + }, + Err(err) => ServiceCheck { + name, + required, + ok: false, + url: None, + payload: None, + error: Some(err.to_string()), + }, + } +} + +fn bucket_health(args: &Cli) -> Result> { + let client = ActivityWatchClient::new( + args.aw_api.trim_end_matches('/'), + Duration::from_secs(args.bucket_timeout_seconds), + )?; + let now = Utc::now(); + let mut out = Vec::new(); + + for spec in bucket_specs(&args.hostname) { + if matches!(spec.mode, BucketMode::EventDriven) { + out.push(BucketCheck { + label: spec.label.to_string(), + bucket: spec.bucket, + mode: spec.mode.as_str().to_string(), + status: "EVENT-DRIVEN".to_string(), + ok: true, + event_count_sample: None, + latest: None, + age_seconds: None, + error: None, + }); + continue; + } + + match client.latest_event(&spec.bucket) { + Ok(Some(event)) => { + let ts = event.timestamp_utc()?; + let age = (now - ts).num_seconds(); + let status = match spec.mode { + BucketMode::InactiveOk => { + if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) { + "FRESH" + } else { + "INACTIVE" + } + } + BucketMode::Fresh => { + if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) { + "FRESH" + } else { + "STALE" + } + } + BucketMode::EventDriven => "EVENT-DRIVEN", + }; + let ok = match spec.mode { + BucketMode::InactiveOk => true, + BucketMode::Fresh => status == "FRESH", + BucketMode::EventDriven => true, + }; + out.push(BucketCheck { + label: spec.label.to_string(), + bucket: spec.bucket, + mode: spec.mode.as_str().to_string(), + status: status.to_string(), + ok, + event_count_sample: Some(1), + latest: Some(ts.to_rfc3339_opts(SecondsFormat::Secs, true)), + age_seconds: Some(age), + error: None, + }); + } + Ok(None) => out.push(BucketCheck { + label: spec.label.to_string(), + bucket: spec.bucket, + mode: spec.mode.as_str().to_string(), + status: "DEAD".to_string(), + ok: false, + event_count_sample: Some(0), + latest: None, + age_seconds: None, + error: None, + }), + Err(err) => out.push(BucketCheck { + label: spec.label.to_string(), + bucket: spec.bucket, + mode: spec.mode.as_str().to_string(), + status: "DEAD".to_string(), + ok: false, + event_count_sample: None, + latest: None, + age_seconds: None, + error: Some(err.to_string()), + }), + } + } + Ok(out) +} + +fn build_report(args: &Cli) -> Result { + let services = service_checks(args); + let buckets = bucket_health(args)?; + let bucket_ok = buckets.iter().filter(|bucket| bucket.ok).count(); + let bucket_stale = buckets + .iter() + .filter(|bucket| !bucket.ok && bucket.status == "STALE") + .count(); + let bucket_dead = buckets + .iter() + .filter(|bucket| !bucket.ok && bucket.status == "DEAD") + .count(); + let service_failures = services + .iter() + .filter(|service| service.required && !service.ok) + .count(); + let service_warnings = services + .iter() + .filter(|service| !service.required && service.error.is_some()) + .count(); + let ok = service_failures == 0 && bucket_stale == 0 && bucket_dead == 0; + + Ok(CheckReport { + ok, + generated_at_utc: now_utc_rfc3339(), + services, + buckets, + summary: CheckSummary { + bucket_ok, + bucket_stale, + bucket_dead, + service_failures, + service_warnings, + }, + }) +} + +fn render_text(report: &CheckReport) -> String { + let mut lines = vec![ + "=== DetMir Autonomous Check ===".to_string(), + format!("OK: {}", if report.ok { "True" } else { "False" }), + String::new(), + "Services:".to_string(), + ]; + for service in &report.services { + let mark = if service.ok && service.error.is_none() { + "OK" + } else if service.required { + "FAIL" + } else { + "WARN" + }; + lines.push(format!(" {:<18} {}", service.name, mark)); + } + lines.push(String::new()); + lines.push(format!("{:<20} {:<13} {:>8}", "Bucket", "Status", "Age(s)")); + lines.push("-".repeat(44)); + for bucket in &report.buckets { + let age = bucket + .age_seconds + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()); + lines.push(format!( + "{:<20} {:<13} {:>8}", + bucket.label, bucket.status, age + )); + } + lines.push(String::new()); + lines.push(format!( + "Summary: OK={} STALE={} DEAD={} SERVICE_FAIL={} SERVICE_WARN={}", + report.summary.bucket_ok, + report.summary.bucket_stale, + report.summary.bucket_dead, + report.summary.service_failures, + report.summary.service_warnings, + )); + lines.join("\n") +} + +fn main() -> Result<()> { + let mut args = Cli::parse(); + args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api); + args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url); + args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url); + args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname); + + let report = build_report(&args)?; + if args.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + println!("{}", render_text(&report)); + } + std::process::exit(if report.ok { + exit_codes::OK + } else { + exit_codes::CHECK_FAILED + }); +} diff --git a/adk-rust/crates/detmir-core/Cargo.toml b/adk-rust/crates/detmir-core/Cargo.toml new file mode 100644 index 0000000..3ec2ef3 --- /dev/null +++ b/adk-rust/crates/detmir-core/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "detmir-core" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +serde.workspace = true diff --git a/adk-rust/crates/detmir-core/src/lib.rs b/adk-rust/crates/detmir-core/src/lib.rs new file mode 100644 index 0000000..a86c80d --- /dev/null +++ b/adk-rust/crates/detmir-core/src/lib.rs @@ -0,0 +1,88 @@ +use std::fmt; + +use anyhow::{Context, Result}; +use chrono::{DateTime, SecondsFormat, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum StatusLevel { + Ok, + Warn, + Fail, + Unknown, +} + +impl StatusLevel { + pub fn as_str(self) -> &'static str { + match self { + Self::Ok => "OK", + Self::Warn => "WARN", + Self::Fail => "FAIL", + Self::Unknown => "UNKNOWN", + } + } + + pub fn exit_code(self) -> i32 { + match self { + Self::Ok => exit_codes::OK, + Self::Warn | Self::Fail | Self::Unknown => exit_codes::CHECK_FAILED, + } + } +} + +impl fmt::Display for StatusLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl From<&str> for StatusLevel { + fn from(value: &str) -> Self { + match value.trim().to_ascii_uppercase().as_str() { + "OK" => Self::Ok, + "WARN" | "WARNING" => Self::Warn, + "FAIL" | "FAILED" | "ERROR" => Self::Fail, + _ => Self::Unknown, + } + } +} + +pub mod exit_codes { + pub const OK: i32 = 0; + pub const ERROR: i32 = 1; + pub const CHECK_FAILED: i32 = 2; + pub const POLICY_DENIED: i32 = 3; +} + +pub fn now_utc_rfc3339() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +pub fn parse_utc_rfc3339(value: &str) -> Result> { + DateTime::parse_from_rfc3339(value) + .with_context(|| format!("invalid RFC3339 timestamp: {value}")) + .map(|ts| ts.with_timezone(&Utc)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_level_maps_known_values() { + assert_eq!(StatusLevel::from("OK"), StatusLevel::Ok); + assert_eq!(StatusLevel::from("warn"), StatusLevel::Warn); + assert_eq!(StatusLevel::from("ERROR"), StatusLevel::Fail); + assert_eq!(StatusLevel::from("other"), StatusLevel::Unknown); + } + + #[test] + fn parses_zulu_timestamp() { + let ts = parse_utc_rfc3339("2026-05-31T10:20:30Z").unwrap(); + assert_eq!( + ts.to_rfc3339_opts(SecondsFormat::Secs, true), + "2026-05-31T10:20:30Z" + ); + } +} diff --git a/adk-rust/crates/detmir-dlp/Cargo.toml b/adk-rust/crates/detmir-dlp/Cargo.toml new file mode 100644 index 0000000..a8c03bf --- /dev/null +++ b/adk-rust/crates/detmir-dlp/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "detmir-dlp" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true diff --git a/adk-rust/crates/detmir-dlp/src/main.rs b/adk-rust/crates/detmir-dlp/src/main.rs new file mode 100644 index 0000000..6448e99 --- /dev/null +++ b/adk-rust/crates/detmir-dlp/src/main.rs @@ -0,0 +1,107 @@ +use std::io::{self, Write}; +use std::process::Command; + +use anyhow::{Context, Result}; +use clap::Parser; + +const DEFAULT_SSH_TARGET: &str = "igor@10.10.10.13"; +const DEFAULT_REMOTE_COMMAND: &str = "sudo -n /usr/local/bin/dlp-health-check --json"; + +#[derive(Debug, Parser)] +#[command(about = "Run the DetMir DLP health check on the AW server over SSH.")] +struct Cli { + #[arg(long, default_value = "ssh")] + ssh_bin: String, + + #[arg(long, default_value = DEFAULT_SSH_TARGET)] + ssh_target: String, + + #[arg(long, default_value_t = 10)] + connect_timeout_seconds: u64, + + #[arg(long, default_value = DEFAULT_REMOTE_COMMAND)] + remote_command: String, +} + +impl Cli { + fn apply_env(mut self) -> Self { + self.ssh_target = env_first( + &["DETMIR_DLP_SSH_TARGET", "DETMIR_AW_SSH_HOST"], + &self.ssh_target, + ); + self.remote_command = env_first(&["DETMIR_DLP_REMOTE_COMMAND"], &self.remote_command); + self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin); + self + } +} + +fn env_first(names: &[&str], fallback: &str) -> String { + names + .iter() + .find_map(|name| std::env::var(name).ok().filter(|value| !value.is_empty())) + .unwrap_or_else(|| fallback.to_string()) +} + +fn ssh_args(cli: &Cli) -> Vec { + vec![ + "-o".to_string(), + "BatchMode=yes".to_string(), + "-o".to_string(), + format!("ConnectTimeout={}", cli.connect_timeout_seconds), + "-o".to_string(), + "StrictHostKeyChecking=accept-new".to_string(), + cli.ssh_target.clone(), + cli.remote_command.clone(), + ] +} + +fn run(cli: Cli) -> Result { + let args = ssh_args(&cli); + let output = Command::new(&cli.ssh_bin) + .args(&args) + .output() + .with_context(|| format!("failed to execute {}", cli.ssh_bin))?; + + io::stdout() + .write_all(&output.stdout) + .context("failed to write DLP stdout")?; + io::stderr() + .write_all(&output.stderr) + .context("failed to write DLP stderr")?; + + Ok(output.status.code().unwrap_or(1)) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + let code = run(cli)?; + std::process::exit(code); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_legacy_ssh_args() { + let cli = Cli { + ssh_bin: "ssh".to_string(), + ssh_target: DEFAULT_SSH_TARGET.to_string(), + connect_timeout_seconds: 10, + remote_command: DEFAULT_REMOTE_COMMAND.to_string(), + }; + assert_eq!( + ssh_args(&cli), + vec![ + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=accept-new", + DEFAULT_SSH_TARGET, + DEFAULT_REMOTE_COMMAND, + ] + ); + } +} diff --git a/adk-rust/crates/detmir-heal-safe/Cargo.toml b/adk-rust/crates/detmir-heal-safe/Cargo.toml new file mode 100644 index 0000000..fc08c93 --- /dev/null +++ b/adk-rust/crates/detmir-heal-safe/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "detmir-heal-safe" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +detmir-core.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/detmir-heal-safe/src/main.rs b/adk-rust/crates/detmir-heal-safe/src/main.rs new file mode 100644 index 0000000..6fa072a --- /dev/null +++ b/adk-rust/crates/detmir-heal-safe/src/main.rs @@ -0,0 +1,321 @@ +use std::io::Write; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result}; +use clap::Parser; +use detmir_core::{exit_codes, now_utc_rfc3339}; +use serde::Serialize; + +const DEFAULT_SSH_TARGET: &str = "igor@10.10.10.13"; +const REQUIRED_UNITS: &[&str] = &[ + "activitywatch-server.service", + "aw-worktime-api.service", + "aw-worktime-ui-bridge.timer", +]; +const OPTIONAL_UNITS: &[&str] = &["activitywatch-dlp-aggregator.timer"]; + +#[derive(Debug, Parser)] +#[command(about = "Safely heal allowlisted DetMir server-side services over SSH.")] +struct Cli { + #[arg(long, default_value = "ssh")] + ssh_bin: String, + + #[arg(long, default_value = DEFAULT_SSH_TARGET)] + ssh_target: String, + + #[arg(long, default_value_t = 10)] + connect_timeout_seconds: u64, + + #[arg(long)] + json: bool, + + #[arg(long)] + apply: bool, + + #[arg(long)] + start_optional: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + self.ssh_target = env_first( + &["DETMIR_HEAL_SSH_TARGET", "DETMIR_AW_SSH_HOST"], + &self.ssh_target, + ); + self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin); + self + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +struct HealAction { + action: String, + unit: Option, + required: bool, + status: String, +} + +#[derive(Debug, Serialize)] +struct HealReport { + ok: bool, + dry_run: bool, + applied: bool, + generated_at_utc: String, + ssh_target: String, + actions: Vec, + raw_stdout: String, + raw_stderr: String, +} + +fn env_first(names: &[&str], fallback: &str) -> String { + names + .iter() + .find_map(|name| std::env::var(name).ok().filter(|value| !value.is_empty())) + .unwrap_or_else(|| fallback.to_string()) +} + +fn ssh_args(cli: &Cli) -> Vec { + vec![ + "-o".to_string(), + "BatchMode=yes".to_string(), + "-o".to_string(), + format!("ConnectTimeout={}", cli.connect_timeout_seconds), + "-o".to_string(), + "StrictHostKeyChecking=accept-new".to_string(), + cli.ssh_target.clone(), + "bash".to_string(), + "-s".to_string(), + "--".to_string(), + if cli.apply { "apply" } else { "dry-run" }.to_string(), + if cli.start_optional { + "start-optional" + } else { + "skip-optional" + } + .to_string(), + ] +} + +fn remote_script() -> String { + let required = REQUIRED_UNITS.join(" "); + let optional = OPTIONAL_UNITS.join(" "); + format!( + r#"set -euo pipefail +mode="${{1:-dry-run}}" +optional_mode="${{2:-skip-optional}}" +required_units="{required}" +optional_units="{optional}" + +emit() {{ + action="$1" + unit="${{2:-}}" + required="${{3:-false}}" + status="${{4:-ok}}" + printf '%s\t%s\t%s\t%s\n' "$action" "$unit" "$required" "$status" +}} + +if [ "$mode" = "apply" ]; then + sudo -n systemctl reset-failed $required_units $optional_units >/dev/null 2>&1 || true + emit reset-failed "" false applied +else + emit reset-failed "" false planned +fi + +for service in $required_units; do + if systemctl is-active --quiet "$service"; then + emit active "$service" true ok + elif [ "$mode" = "apply" ]; then + emit restart "$service" true started + sudo -n systemctl restart "$service" + emit active "$service" true ok + else + emit restart "$service" true planned + fi +done + +for service in $optional_units; do + if ! systemctl list-unit-files "$service" >/dev/null 2>&1; then + emit absent "$service" false skipped + elif systemctl is-active --quiet "$service"; then + emit active "$service" false ok + elif [ "$mode" = "apply" ]; then + if [ "$optional_mode" = "start-optional" ]; then + emit start "$service" false started + sudo -n systemctl start "$service" || true + else + emit inactive "$service" false skipped + fi + else + if [ "$optional_mode" = "start-optional" ]; then + emit start "$service" false planned + else + emit inactive "$service" false skipped + fi + fi +done + +if [ "$mode" = "apply" ]; then + sudo -n /usr/local/bin/dlp-health-check --json >/tmp/detmir-heal-dlp-health.json || true + emit dlp-health-check "/tmp/detmir-heal-dlp-health.json" false written +else + emit dlp-health-check "/tmp/detmir-heal-dlp-health.json" false planned +fi +"# + ) +} + +fn parse_actions(raw: &str) -> Vec { + raw.lines() + .filter_map(|line| { + let mut parts = line.split('\t'); + let action = parts.next()?; + let unit = parts.next().unwrap_or_default(); + let required = parts.next().unwrap_or("false") == "true"; + let status = parts.next().unwrap_or("unknown"); + Some(HealAction { + action: action.to_string(), + unit: if unit.is_empty() { + None + } else { + Some(unit.to_string()) + }, + required, + status: status.to_string(), + }) + }) + .collect() +} + +fn run(cli: Cli) -> Result { + let args = ssh_args(&cli); + let mut child = Command::new(&cli.ssh_bin) + .args(&args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {}", cli.ssh_bin))?; + + child + .stdin + .as_mut() + .context("failed to open SSH stdin")? + .write_all(remote_script().as_bytes()) + .context("failed to write remote heal script")?; + + let output = child + .wait_with_output() + .context("SSH heal command failed")?; + let raw_stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let raw_stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let code = output.status.code().unwrap_or(exit_codes::ERROR); + let actions = parse_actions(&raw_stdout); + let report = HealReport { + ok: code == exit_codes::OK, + dry_run: !cli.apply, + applied: cli.apply && code == exit_codes::OK, + generated_at_utc: now_utc_rfc3339(), + ssh_target: cli.ssh_target.clone(), + actions, + raw_stdout, + raw_stderr, + }; + + if cli.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + if report.dry_run { + println!("detmir-heal-safe: dry-run"); + } else { + println!("detmir-heal-safe: apply"); + } + for action in &report.actions { + match &action.unit { + Some(unit) => println!( + "{} {} required={} status={}", + action.action, unit, action.required, action.status + ), + None => println!("{} status={}", action.action, action.status), + } + } + if !report.raw_stderr.trim().is_empty() { + eprint!("{}", report.raw_stderr); + } + } + + Ok(code) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + let code = run(cli)?; + std::process::exit(code); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_dry_run_ssh_args() { + let cli = Cli { + ssh_bin: "ssh".to_string(), + ssh_target: DEFAULT_SSH_TARGET.to_string(), + connect_timeout_seconds: 10, + json: false, + apply: false, + start_optional: false, + }; + assert_eq!( + ssh_args(&cli), + vec![ + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + "-o", + "StrictHostKeyChecking=accept-new", + DEFAULT_SSH_TARGET, + "bash", + "-s", + "--", + "dry-run", + "skip-optional", + ] + ); + } + + #[test] + fn parses_remote_actions() { + let actions = parse_actions( + "reset-failed\t\tfalse\tplanned\nactive\tactivitywatch-server.service\ttrue\tok\n", + ); + assert_eq!( + actions, + vec![ + HealAction { + action: "reset-failed".to_string(), + unit: None, + required: false, + status: "planned".to_string(), + }, + HealAction { + action: "active".to_string(), + unit: Some("activitywatch-server.service".to_string()), + required: true, + status: "ok".to_string(), + }, + ] + ); + } + + #[test] + fn remote_script_uses_allowlisted_units() { + let script = remote_script(); + assert!(script.contains("activitywatch-server.service")); + assert!(script.contains("aw-worktime-api.service")); + assert!(script.contains("aw-worktime-ui-bridge.timer")); + assert!(script.contains("activitywatch-dlp-aggregator.timer")); + assert!(!script.contains("*")); + } +} diff --git a/adk-rust/crates/detmir-state/Cargo.toml b/adk-rust/crates/detmir-state/Cargo.toml new file mode 100644 index 0000000..373ab13 --- /dev/null +++ b/adk-rust/crates/detmir-state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "detmir-state" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +detmir-core.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/detmir-state/src/lib.rs b/adk-rust/crates/detmir-state/src/lib.rs new file mode 100644 index 0000000..e465b2f --- /dev/null +++ b/adk-rust/crates/detmir-state/src/lib.rs @@ -0,0 +1,209 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use detmir_core::StatusLevel; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const DEFAULT_STATE_FILE: &str = "/var/lib/detmir-ai/latest-state.json"; + +#[derive(Debug, Deserialize)] +pub struct DetmirState { + pub severity: Option, + pub check_ok: Option, + pub dlp_ok: Option, + pub needs_heal: Option, + pub reasons: Option>, + pub detmir_summary: Option, + pub dlp_counts: Option, + pub check: Option, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct DetmirSummary { + pub bucket_ok: Option, + pub bucket_stale: Option, + pub bucket_dead: Option, + pub service_failures: Option, + pub service_warnings: Option, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct DlpCounts { + pub ok: Option, + pub warn: Option, + pub fail: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct NormalizedStatus { + pub severity: String, + pub check_ok: bool, + pub dlp_ok: bool, + pub needs_heal: bool, + pub reasons: Vec, + pub detmir_summary: DetmirSummary, + pub dlp_counts: DlpCounts, + pub ok_for_operator: bool, +} + +impl NormalizedStatus { + pub fn level(&self) -> StatusLevel { + StatusLevel::from(self.severity.as_str()) + } + + pub fn exit_code(&self) -> i32 { + if self.ok_for_operator { + StatusLevel::Ok.exit_code() + } else { + StatusLevel::Fail.exit_code() + } + } +} + +impl DetmirState { + pub fn normalize(self) -> NormalizedStatus { + let summary = self.detmir_summary.or_else(|| { + self.check + .as_ref() + .and_then(|check| check.get("summary")) + .and_then(|summary| serde_json::from_value(summary.clone()).ok()) + }); + let summary = summary.unwrap_or_default(); + let dlp_counts = self.dlp_counts.unwrap_or_default(); + + let severity = self.severity.unwrap_or_else(|| "UNKNOWN".to_string()); + let check_ok = self + .check_ok + .or_else(|| { + self.check + .as_ref() + .and_then(|check| check.get("ok")) + .and_then(Value::as_bool) + }) + .unwrap_or(false); + let dlp_ok = self.dlp_ok.unwrap_or(false); + let needs_heal = self.needs_heal.unwrap_or(false); + let reasons = self.reasons.unwrap_or_default(); + + let ok_for_operator = severity == "OK" + && check_ok + && dlp_ok + && !needs_heal + && summary.bucket_stale.unwrap_or(0) == 0 + && summary.bucket_dead.unwrap_or(0) == 0 + && summary.service_failures.unwrap_or(0) == 0 + && dlp_counts.warn.unwrap_or(0) == 0 + && dlp_counts.fail.unwrap_or(0) == 0; + + NormalizedStatus { + severity, + check_ok, + dlp_ok, + needs_heal, + reasons, + detmir_summary: summary, + dlp_counts, + ok_for_operator, + } + } +} + +pub fn read_state(path: impl AsRef) -> Result { + let path = path.as_ref(); + let raw = fs::read_to_string(path) + .with_context(|| format!("failed to read DetMir state file {}", path.display()))?; + let state: DetmirState = serde_json::from_str(&raw) + .with_context(|| format!("failed to parse DetMir state JSON {}", path.display()))?; + Ok(state.normalize()) +} + +pub fn write_json_atomic(path: impl AsRef, value: &T) -> Result<()> { + let path = path.as_ref(); + let parent = path + .parent() + .with_context(|| format!("path has no parent: {}", path.display()))?; + fs::create_dir_all(parent) + .with_context(|| format!("failed to create state directory {}", parent.display()))?; + + let tmp_path = temp_path(path); + let mut payload = serde_json::to_vec_pretty(value)?; + payload.push(b'\n'); + fs::write(&tmp_path, payload) + .with_context(|| format!("failed to write temporary state {}", tmp_path.display()))?; + fs::rename(&tmp_path, path) + .with_context(|| format!("failed to atomically replace {}", path.display()))?; + Ok(()) +} + +fn temp_path(path: &Path) -> PathBuf { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("state.json"); + path.with_file_name(format!(".{file_name}.{}.tmp", std::process::id())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_current_state_shape() { + let raw = r#"{ + "severity": "OK", + "check_ok": true, + "dlp_ok": true, + "needs_heal": false, + "reasons": [], + "detmir_summary": { + "bucket_ok": 8, + "bucket_stale": 0, + "bucket_dead": 0, + "service_failures": 0, + "service_warnings": 0 + }, + "dlp_counts": {"ok": 22, "warn": 0, "fail": 0} + }"#; + let state: DetmirState = serde_json::from_str(raw).unwrap(); + let normalized = state.normalize(); + assert!(normalized.ok_for_operator); + assert_eq!(normalized.exit_code(), 0); + } + + #[test] + fn reads_legacy_check_summary_shape() { + let raw = r#"{ + "severity": "OK", + "dlp_ok": true, + "check": { + "ok": true, + "summary": { + "bucket_ok": 8, + "bucket_stale": 0, + "bucket_dead": 0, + "service_failures": 0, + "service_warnings": 0 + } + }, + "dlp_counts": {"ok": 1, "warn": 0, "fail": 0} + }"#; + let state: DetmirState = serde_json::from_str(raw).unwrap(); + assert!(state.normalize().ok_for_operator); + } + + #[test] + fn writes_json_atomically() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.json"); + let payload = DetmirSummary { + bucket_ok: Some(1), + ..Default::default() + }; + write_json_atomic(&path, &payload).unwrap(); + let stored: DetmirSummary = + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + assert_eq!(stored.bucket_ok, Some(1)); + } +} diff --git a/adk-rust/crates/detmir-status/.gitignore b/adk-rust/crates/detmir-status/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/adk-rust/crates/detmir-status/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/adk-rust/crates/detmir-status/Cargo.toml b/adk-rust/crates/detmir-status/Cargo.toml new file mode 100644 index 0000000..1cbf8cc --- /dev/null +++ b/adk-rust/crates/detmir-status/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "detmir-status" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "detmir-status" +path = "src/bin/detmir-status.rs" + +[[bin]] +name = "detmir-adk-status" +path = "src/bin/detmir-adk-status.rs" + +[dependencies] +adk-rust.workspace = true +anyhow.workspace = true +clap.workspace = true +detmir-state.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/detmir-status/README.md b/adk-rust/crates/detmir-status/README.md new file mode 100644 index 0000000..40142ea --- /dev/null +++ b/adk-rust/crates/detmir-status/README.md @@ -0,0 +1,19 @@ +# detmir-status + +Read-only DetMir status probe written in Rust as a small ADK-Rust adoption +spike. + +The command reads `/var/lib/detmir-ai/latest-state.json` and prints either a +compact operator summary, raw normalized JSON, or an ADK `Content` envelope that +can later be handed to an ADK runner/model/tool chain. + +It does not start recovery, modify services, or write state. + +```bash +cd adk-rust +cargo run -p detmir-status -- --json +cargo run -p detmir-status -- --adk-json +cargo run -p detmir-status -- status --json +``` + +The package also builds a compatibility binary named `detmir-adk-status`. diff --git a/adk-rust/crates/detmir-status/src/bin/detmir-adk-status.rs b/adk-rust/crates/detmir-status/src/bin/detmir-adk-status.rs new file mode 100644 index 0000000..f7ce994 --- /dev/null +++ b/adk-rust/crates/detmir-status/src/bin/detmir-adk-status.rs @@ -0,0 +1,3 @@ +fn main() -> anyhow::Result<()> { + detmir_status::main_entry() +} diff --git a/adk-rust/crates/detmir-status/src/bin/detmir-status.rs b/adk-rust/crates/detmir-status/src/bin/detmir-status.rs new file mode 100644 index 0000000..f7ce994 --- /dev/null +++ b/adk-rust/crates/detmir-status/src/bin/detmir-status.rs @@ -0,0 +1,3 @@ +fn main() -> anyhow::Result<()> { + detmir_status::main_entry() +} diff --git a/adk-rust/crates/detmir-status/src/lib.rs b/adk-rust/crates/detmir-status/src/lib.rs new file mode 100644 index 0000000..d70e26d --- /dev/null +++ b/adk-rust/crates/detmir-status/src/lib.rs @@ -0,0 +1,93 @@ +use std::path::PathBuf; + +use adk_rust::Content; +use anyhow::Result; +use clap::{Args, Parser, Subcommand}; +use detmir_state::{DEFAULT_STATE_FILE, NormalizedStatus, read_state}; +use serde_json::json; + +#[derive(Debug, Parser)] +#[command(about = "Read-only DetMir status probe with JSON and ADK output.")] +struct Cli { + #[command(flatten)] + status: StatusArgs, + + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Subcommand)] +enum Command { + Status(StatusArgs), +} + +#[derive(Debug, Args)] +struct StatusArgs { + #[arg(long, default_value = DEFAULT_STATE_FILE)] + state: PathBuf, + + #[arg(long)] + json: bool, + + #[arg(long)] + adk_json: bool, +} + +fn render_text(status: &NormalizedStatus) -> String { + let s = &status.detmir_summary; + let d = &status.dlp_counts; + format!( + "DetMir status: {severity}\n\ + check_ok={check_ok} dlp_ok={dlp_ok} needs_heal={needs_heal}\n\ + buckets: ok={bucket_ok} stale={bucket_stale} dead={bucket_dead}\n\ + services: fail={service_failures} warn={service_warnings}\n\ + dlp: ok={dlp_ok_count} warn={dlp_warn} fail={dlp_fail}\n\ + operator_ok={operator_ok}", + severity = status.severity, + check_ok = status.check_ok, + dlp_ok = status.dlp_ok, + needs_heal = status.needs_heal, + bucket_ok = s.bucket_ok.unwrap_or(0), + bucket_stale = s.bucket_stale.unwrap_or(0), + bucket_dead = s.bucket_dead.unwrap_or(0), + service_failures = s.service_failures.unwrap_or(0), + service_warnings = s.service_warnings.unwrap_or(0), + dlp_ok_count = d.ok.unwrap_or(0), + dlp_warn = d.warn.unwrap_or(0), + dlp_fail = d.fail.unwrap_or(0), + operator_ok = status.ok_for_operator, + ) +} + +fn render_adk_content_json(status: &NormalizedStatus) -> Result { + let text = render_text(status); + let content = Content::new("user").with_text(text); + let envelope = json!({ + "agent": "detmir-status-agent", + "mode": "read-only", + "adk_content": content, + "normalized": status, + }); + Ok(serde_json::to_string_pretty(&envelope)?) +} + +fn run_status(args: StatusArgs) -> Result { + let status = read_state(&args.state)?; + if args.adk_json { + println!("{}", render_adk_content_json(&status)?); + } else if args.json { + println!("{}", serde_json::to_string_pretty(&status)?); + } else { + println!("{}", render_text(&status)); + } + Ok(status.exit_code()) +} + +pub fn main_entry() -> Result<()> { + let cli = Cli::parse(); + let code = match cli.command { + Some(Command::Status(args)) => run_status(args)?, + None => run_status(cli.status)?, + }; + std::process::exit(code); +} diff --git a/adk-rust/crates/diag-and-manual-restart/Cargo.toml b/adk-rust/crates/diag-and-manual-restart/Cargo.toml new file mode 100644 index 0000000..dc39575 --- /dev/null +++ b/adk-rust/crates/diag-and-manual-restart/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "diag-and-manual-restart" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/diag-and-manual-restart/src/main.rs b/adk-rust/crates/diag-and-manual-restart/src/main.rs new file mode 100644 index 0000000..a814aec --- /dev/null +++ b/adk-rust/crates/diag-and-manual-restart/src/main.rs @@ -0,0 +1,250 @@ +use std::io::{self, Write}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use chrono::Utc; +use clap::Parser; + +const SERVER_UNITS: &[&str] = &[ + "activitywatch-server", + "aw-worktime-api", + "aw-worktime-ui-bridge.timer", + "aw-dlp-policy-engine.service", + "aw-dlp-aggregator.timer", + "activitywatch-dlp-aggregator.timer", +]; + +#[derive(Debug, Parser)] +#[command(about = "Run AW/DLP diagnostics and optionally perform manual restart recovery")] +struct Cli { + #[arg(long)] + with_windows: bool, + + #[arg(long)] + yes: bool, + + #[arg(long, default_value = "ansible/inventory.ini")] + inventory: String, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + require_command("ansible")?; + require_command("ansible-playbook")?; + if !std::path::Path::new(&cli.inventory).is_file() { + bail!("inventory not found: {}", cli.inventory); + } + + log("Running diagnostics on aw_server..."); + if run_health_check(&cli.inventory) { + log("Diagnostics: healthy. Restart not needed."); + return Ok(0); + } + + log("Diagnostics: FAILED."); + if !confirm_restart(cli.yes)? { + log("Restart declined."); + return Ok(1); + } + + restart_server_components(&cli.inventory); + if cli.with_windows { + restart_windows_collectors(&cli.inventory); + seed_windows_dlp_events(&cli.inventory); + } + seed_server_dlp_events(&cli.inventory)?; + + log("Waiting 15 seconds before re-check..."); + std::thread::sleep(std::time::Duration::from_secs(15)); + + log("Running post-restart diagnostics..."); + if run_health_check(&cli.inventory) { + log("Post-restart diagnostics: healthy."); + return Ok(0); + } + log("Post-restart diagnostics: still failing."); + Ok(1) +} + +fn log(message: &str) { + eprintln!("{} {}", Utc::now().format("%Y-%m-%d %H:%M:%S"), message); +} + +fn require_command(name: &str) -> Result<()> { + let status = Command::new("sh") + .arg("-c") + .arg(format!("command -v {name} >/dev/null 2>&1")) + .status() + .with_context(|| format!("check command {name}"))?; + if status.success() { + Ok(()) + } else { + bail!("{name} not found") + } +} + +fn run_health_check(inventory: &str) -> bool { + ansible_command( + inventory, + "aw_server", + "-b", + "ansible.builtin.command", + "/usr/local/bin/aw-health-check", + ) && ansible_command( + inventory, + "aw_server", + "-b", + "ansible.builtin.command", + "/usr/local/bin/dlp-health-check", + ) +} + +fn restart_server_components(inventory: &str) { + log("Restarting server components on aw_server..."); + for unit in SERVER_UNITS { + if ansible_command( + inventory, + "aw_server", + "-b", + "ansible.builtin.command", + &format!("systemctl status {unit}"), + ) { + let _ = ansible_command( + inventory, + "aw_server", + "-b", + "ansible.builtin.systemd", + &format!("name={unit} state=restarted enabled=true"), + ); + } + } +} + +fn seed_server_dlp_events(inventory: &str) -> Result<()> { + log("Seeding DLP freshness events on aw_server..."); + let ts = Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); + let script = format!( + r#"cat >/tmp/aw-endpoint-seed.json <<'JSON' +{{"timestamp":"{ts}","duration":0.0,"data":{{"hostname":"SHARKON2025","signalType":"self_test","source":"diag_and_manual_restart","username":"system","queueDepth":0,"eventsEnqueued":0,"eventsFlushed":0,"sendFailures":0}}}} +JSON +cat >/tmp/aw-fileops-seed-host.json <<'JSON' +{{"timestamp":"{ts}","duration":0.0,"data":{{"hostname":"SHARKON2025","operation":"self_test","source":"diag_and_manual_restart"}}}} +JSON +cat >/tmp/aw-fileops-seed-server.json <<'JSON' +{{"timestamp":"{ts}","duration":0.0,"data":{{"hostname":"10.10.10.13","operation":"self_test","source":"diag_and_manual_restart"}}}} +JSON +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_SHARKON2025' -H 'Content-Type: application/json' -d '{{"client":"aw-dlp-endpoint-signals","type":"aw.dlp.endpoint.signal","hostname":"SHARKON2025"}}' >/dev/null 2>&1 || true +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_SHARKON2025' -H 'Content-Type: application/json' -d '{{"client":"aw-file-operations","type":"aw.file.operation","hostname":"SHARKON2025"}}' >/dev/null 2>&1 || true +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_10.10.10.13' -H 'Content-Type: application/json' -d '{{"client":"aw-file-operations","type":"aw.file.operation","hostname":"10.10.10.13"}}' >/dev/null 2>&1 || true +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_SHARKON2025/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-endpoint-seed.json >/dev/null +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_SHARKON2025/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-host.json >/dev/null +curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_10.10.10.13/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-server.json >/dev/null +"# + ); + let _ = ansible_command( + inventory, + "aw_server", + "-b", + "ansible.builtin.shell", + &script, + ); + Ok(()) +} + +fn restart_windows_collectors(inventory: &str) { + log("Restarting Windows recovery/launch tasks on aw_windows..."); + let script = r#"powershell -NoProfile -ExecutionPolicy Bypass -Command "$ErrorActionPreference = 'Stop'; try { Start-ScheduledTask -TaskName 'ActivityWatch Recovery' -ErrorAction Stop | Out-Null } catch {}; Get-ScheduledTask | Where-Object TaskName -like 'ActivityWatch Launch *' | ForEach-Object { try { Start-ScheduledTask -TaskName $_.TaskName -ErrorAction Stop | Out-Null } catch {} }; Write-Output 'windows-tasks-restarted'""#; + let _ = ansible_windows_shell(inventory, script); +} + +fn seed_windows_dlp_events(inventory: &str) { + log("Seeding endpoint/file-ops events from aw_windows..."); + let script = r#"powershell -NoProfile -ExecutionPolicy Bypass -Command "$ErrorActionPreference = 'Stop'; $ts = (Get-Date).ToUniversalTime().ToString('o'); $api='http://10.10.10.13:5600/api/0'; $endpoint=@{timestamp=$ts;duration=0.0;data=@{hostname='SHARKON2025';signalType='self_test';source='diag_and_manual_restart';username=$env:USERNAME;queueDepth=0;eventsEnqueued=0;eventsFlushed=0;sendFailures=0}} | ConvertTo-Json -Depth 8 -Compress; $fileops=@{timestamp=$ts;duration=0.0;data=@{hostname='SHARKON2025';operation='self_test';source='diag_and_manual_restart';username=$env:USERNAME}} | ConvertTo-Json -Depth 8 -Compress; Invoke-RestMethod -Method Post -Uri $api'/buckets/aw-dlp-endpoint-signals_SHARKON2025' -ContentType 'application/json' -Body '{\"client\":\"aw-dlp-endpoint-signals\",\"type\":\"aw.dlp.endpoint.signal\",\"hostname\":\"SHARKON2025\"}' -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri $api'/buckets/aw-file-operations_SHARKON2025' -ContentType 'application/json' -Body '{\"client\":\"aw-file-operations\",\"type\":\"aw.file.operation\",\"hostname\":\"SHARKON2025\"}' -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri $api'/buckets/aw-dlp-endpoint-signals_SHARKON2025/heartbeat?pulsetime=30' -ContentType 'application/json' -Body $endpoint -TimeoutSec 15 -DisableKeepAlive | Out-Null; Invoke-RestMethod -Method Post -Uri $api'/buckets/aw-file-operations_SHARKON2025/heartbeat?pulsetime=30' -ContentType 'application/json' -Body $fileops -TimeoutSec 15 -DisableKeepAlive | Out-Null; Write-Output 'windows-dlp-seeded'""#; + let _ = ansible_windows_shell(inventory, script); +} + +fn confirm_restart(auto_yes: bool) -> Result { + if auto_yes { + return Ok(true); + } + eprint!("Diagnostics failed. Restart required components now? [y/N]: "); + io::stderr().flush().ok(); + let mut answer = String::new(); + io::stdin() + .read_line(&mut answer) + .context("read confirmation")?; + Ok(matches!(answer.trim(), "y" | "Y")) +} + +fn ansible_command( + inventory: &str, + group: &str, + become_flag: &str, + module: &str, + args: &str, +) -> bool { + run_command( + "ansible", + &[ + group, + "-i", + inventory, + become_flag, + "-m", + module, + "-a", + args, + ], + ) +} + +fn ansible_windows_shell(inventory: &str, args: &str) -> bool { + run_command( + "ansible", + &[ + "aw_windows", + "-i", + inventory, + "-m", + "ansible.windows.win_shell", + "-a", + args, + ], + ) +} + +fn run_command(cmd: &str, args: &[&str]) -> bool { + match Command::new(cmd).args(args).output() { + Ok(output) => { + print!("{}", String::from_utf8_lossy(&output.stdout)); + eprint!("{}", String::from_utf8_lossy(&output.stderr)); + output.status.success() + } + Err(err) => { + eprintln!("{cmd}: {err}"); + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn server_units_keep_legacy_order() { + assert_eq!(SERVER_UNITS[0], "activitywatch-server"); + assert!(SERVER_UNITS.contains(&"activitywatch-dlp-aggregator.timer")); + } +} diff --git a/adk-rust/crates/dlp-admin-cli/Cargo.toml b/adk-rust/crates/dlp-admin-cli/Cargo.toml new file mode 100644 index 0000000..c65990b --- /dev/null +++ b/adk-rust/crates/dlp-admin-cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dlp-admin-cli" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde_json.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-admin-cli/src/main.rs b/adk-rust/crates/dlp-admin-cli/src/main.rs new file mode 100644 index 0000000..349883f --- /dev/null +++ b/adk-rust/crates/dlp-admin-cli/src/main.rs @@ -0,0 +1,373 @@ +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::{DateTime, Utc}; +use clap::{Parser, Subcommand}; +use reqwest::blocking::Client; +use reqwest::header::{ACCEPT, CONTENT_TYPE}; +use serde_json::{Map, Value, json}; +use urlencoding::encode; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP admin CLI")] +struct Cli { + #[arg(long, default_value = "http://127.0.0.1:5601")] + policy_server: String, + + #[arg(long, default_value = "http://127.0.0.1:5602")] + case_server: String, + + #[arg(long, default_value = "http://127.0.0.1:5600")] + aw_server: String, + + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + Policies { + #[command(subcommand)] + command: PoliciesCommand, + }, + Incidents { + #[command(subcommand)] + command: IncidentsCommand, + }, + Cases { + #[command(subcommand)] + command: CasesCommand, + }, + Health { + #[command(subcommand)] + command: HealthCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum PoliciesCommand { + List, + Active, +} + +#[derive(Debug, Subcommand)] +enum IncidentsCommand { + List { + #[arg(long)] + host: Option, + + #[arg(long)] + severity: Option, + + #[arg(long, default_value_t = 100)] + limit: usize, + + #[arg(long, default_value_t = 24)] + since_hours: i64, + }, +} + +#[derive(Debug, Subcommand)] +enum CasesCommand { + List { + #[arg(long)] + host: Option, + + #[arg(long)] + status: Option, + + #[arg(long, default_value_t = 100)] + limit: usize, + }, + Create { + #[arg(long)] + incident_id: String, + + #[arg(long)] + title: String, + + #[arg(long)] + host: Option, + + #[arg(long, default_value = "medium")] + severity: String, + }, +} + +#[derive(Debug, Subcommand)] +enum HealthCommand { + Check, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .no_proxy() + .build() + .context("build HTTP client")?; + + let output = match cli.command { + Command::Policies { command } => match command { + PoliciesCommand::List => get_json( + &client, + &format!("{}/api/0/dlp/policies", trim_url(&cli.policy_server)), + )?, + PoliciesCommand::Active => get_json( + &client, + &format!("{}/api/0/dlp/policies/active", trim_url(&cli.policy_server)), + )?, + }, + Command::Incidents { command } => match command { + IncidentsCommand::List { + host, + severity, + limit, + since_hours, + } => list_incidents(&client, &cli.aw_server, host, severity, limit, since_hours)?, + }, + Command::Cases { command } => match command { + CasesCommand::List { + host, + status, + limit, + } => list_cases(&client, &cli.case_server, host, status, limit)?, + CasesCommand::Create { + incident_id, + title, + host, + severity, + } => create_case( + &client, + &cli.case_server, + incident_id, + title, + host, + severity, + )?, + }, + Command::Health { command } => match command { + HealthCommand::Check => health_check( + &client, + &cli.policy_server, + &cli.case_server, + &cli.aw_server, + ), + }, + }; + + println!( + "{}", + serde_json::to_string_pretty(&output).context("serialize output")? + ); + Ok(()) +} + +fn get_json(client: &Client, url: &str) -> Result { + client + .get(url) + .header(ACCEPT, "application/json") + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn send_json(client: &Client, url: &str, method: &str, payload: &Value) -> Result { + let request = match method { + "POST" => client.post(url), + "PUT" => client.put(url), + other => bail!("unsupported method: {other}"), + }; + let response = request + .header(ACCEPT, "application/json") + .header(CONTENT_TYPE, "application/json") + .json(payload) + .send() + .with_context(|| format!("{method} {url}"))? + .error_for_status() + .with_context(|| format!("{method} {url} status"))?; + let raw = response.text().context("read response body")?; + if raw.trim().is_empty() { + Ok(json!({})) + } else { + serde_json::from_str(&raw).with_context(|| format!("decode JSON from {url}")) + } +} + +fn list_incidents( + client: &Client, + aw_server: &str, + host: Option, + severity: Option, + limit: usize, + since_hours: i64, +) -> Result { + let aw_server = trim_url(aw_server); + let bucket_map = get_json(client, &format!("{aw_server}/api/0/buckets"))?; + let Some(buckets) = bucket_map.as_object() else { + return Ok(json!([])); + }; + + let mut bucket_ids: Vec = buckets + .keys() + .filter(|bucket_id| bucket_id.starts_with("aw-dlp-incidents_")) + .cloned() + .collect(); + if let Some(host) = host.as_deref() { + let suffix = format!("_{host}"); + bucket_ids.retain(|bucket_id| bucket_id.ends_with(&suffix)); + } + bucket_ids.sort(); + + let after = Utc::now() - chrono::Duration::hours(since_hours.max(1)); + let severity = severity.map(|value| value.to_lowercase()); + let mut rows = Vec::new(); + for bucket_id in bucket_ids { + let encoded = encode(&bucket_id); + let events_url = format!( + "{aw_server}/api/0/buckets/{encoded}/events?limit={}", + limit.max(1) + ); + let events = get_json(client, &events_url)?; + let Some(events) = events.as_array() else { + continue; + }; + for event in events { + let Some(ts) = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(parse_iso) + else { + continue; + }; + if ts < after { + continue; + } + if let Some(expected) = severity.as_deref() { + let actual = event + .get("data") + .and_then(|data| data.get("severity")) + .and_then(Value::as_str) + .unwrap_or("") + .to_lowercase(); + if actual != expected { + continue; + } + } + rows.push(event.clone()); + } + } + Ok(Value::Array(rows)) +} + +fn list_cases( + client: &Client, + case_server: &str, + host: Option, + status: Option, + limit: usize, +) -> Result { + let mut query = Vec::new(); + if let Some(host) = host { + query.push(format!("host={}", encode(&host))); + } + if let Some(status) = status { + query.push(format!("status={}", encode(&status))); + } + query.push(format!("limit={}", limit.max(1))); + get_json( + client, + &format!( + "{}/api/0/dlp/cases?{}", + trim_url(case_server), + query.join("&") + ), + ) +} + +fn create_case( + client: &Client, + case_server: &str, + incident_id: String, + title: String, + host: Option, + severity: String, +) -> Result { + let payload = json!({ + "incident_id": incident_id, + "title": title, + "host": host, + "severity": severity, + "evidence": { + "source": "dlp-admin-cli" + } + }); + send_json( + client, + &format!("{}/api/0/dlp/cases", trim_url(case_server)), + "POST", + &payload, + ) +} + +fn health_check(client: &Client, policy_server: &str, case_server: &str, aw_server: &str) -> Value { + let mut out = Map::new(); + out.insert( + "policy".to_string(), + get_json(client, &format!("{}/healthz", trim_url(policy_server))) + .unwrap_or_else(error_payload), + ); + out.insert( + "cases".to_string(), + get_json(client, &format!("{}/health", trim_url(case_server))) + .unwrap_or_else(error_payload), + ); + out.insert( + "aw".to_string(), + get_json(client, &format!("{}/api/0/info", trim_url(aw_server))) + .unwrap_or_else(error_payload), + ); + Value::Object(out) +} + +fn parse_iso(value: &str) -> Option> { + DateTime::parse_from_rfc3339(value) + .map(|ts| ts.with_timezone(&Utc)) + .ok() +} + +fn trim_url(url: &str) -> String { + url.trim_end_matches('/').to_string() +} + +fn error_payload(err: anyhow::Error) -> Value { + json!({ + "status": "error", + "error": err.to_string() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trims_urls() { + assert_eq!(trim_url("http://127.0.0.1:5600/"), "http://127.0.0.1:5600"); + } + + #[test] + fn parses_rfc3339_z_timestamp() { + assert!(parse_iso("2026-06-01T10:11:12Z").is_some()); + } +} diff --git a/adk-rust/crates/dlp-aggregator/Cargo.toml b/adk-rust/crates/dlp-aggregator/Cargo.toml new file mode 100644 index 0000000..c9789b5 --- /dev/null +++ b/adk-rust/crates/dlp-aggregator/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "dlp-aggregator" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-aw-client.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-aggregator/src/main.rs b/adk-rust/crates/dlp-aggregator/src/main.rs new file mode 100644 index 0000000..1c1b90e --- /dev/null +++ b/adk-rust/crates/dlp-aggregator/src/main.rs @@ -0,0 +1,618 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{DateTime, SecondsFormat, TimeDelta, Utc}; +use clap::Parser; +use detmir_aw_client::ActivityWatchClient; +use rusqlite::{Connection, params}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const DEFAULT_AW_URL: &str = "http://127.0.0.1:5600/api/0"; +const DEFAULT_SQLITE_PATH: &str = "data/dlp-events.sqlite3"; +const DEFAULT_STATE_PATH: &str = "data/dlp-aggregator-state.json"; +const DEFAULT_BUCKET_PREFIXES: &str = "aw-file-operations_,aw-dlp-incidents_"; + +#[derive(Debug, Parser)] +#[command(about = "Aggregate AWatch-rus DLP buckets into a local warehouse database.")] +struct Cli { + #[arg(long, default_value = DEFAULT_AW_URL)] + aw_url: String, + + #[arg(long)] + postgres_dsn: Option, + + #[arg(long, default_value = DEFAULT_SQLITE_PATH)] + sqlite_path: PathBuf, + + #[arg(long, default_value = DEFAULT_STATE_PATH)] + state_path: PathBuf, + + #[arg(long, default_value = DEFAULT_BUCKET_PREFIXES)] + bucket_prefixes: String, + + #[arg(long)] + since: Option, + + #[arg(long, default_value_t = 24)] + lookback_hours: i64, + + #[arg(long, default_value_t = 60)] + overlap_seconds: i64, + + #[arg(long, default_value_t = 10_000)] + limit: usize, + + #[arg(long, default_value_t = 15)] + timeout: u64, + + #[arg(long)] + dry_run: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + self.aw_url = env_value("AW_URL", &self.aw_url); + self.postgres_dsn = self + .postgres_dsn + .or_else(|| std::env::var("DLP_AGGREGATOR_POSTGRES_DSN").ok()) + .filter(|value| !value.is_empty()); + self.sqlite_path = env_path("DLP_AGGREGATOR_SQLITE_PATH", self.sqlite_path); + self.state_path = env_path("DLP_AGGREGATOR_STATE_PATH", self.state_path); + self + } +} + +#[derive(Debug, Clone, Deserialize)] +struct Bucket { + #[serde(default)] + r#type: String, + #[serde(default)] + hostname: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct RawAwEvent { + #[serde(default)] + id: Option, + timestamp: String, + #[serde(default)] + duration: f64, + #[serde(default)] + data: Value, +} + +#[derive(Debug, Clone)] +struct AwEvent { + bucket_id: String, + hostname: String, + stream_type: String, + event_id: String, + timestamp: String, + duration: f64, + data: Value, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +struct State { + #[serde(default)] + last_end: Option, +} + +#[derive(Debug, Serialize)] +struct DryRunSummary { + aw_url: String, + start: String, + end: String, + selected_buckets: Vec, + fetched_events: usize, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + aw_url: String, + target: String, + target_path: String, + state_path: String, + start: String, + end: String, + selected_buckets: usize, + fetched_events: usize, + inserted_events: usize, +} + +fn env_value(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_path(name: &str, fallback: PathBuf) -> PathBuf { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .unwrap_or(fallback) +} + +fn parse_timestamp(value: &str) -> Result> { + let normalized = value.replace('Z', "+00:00"); + Ok(DateTime::parse_from_rfc3339(&normalized) + .with_context(|| format!("parse timestamp {value}"))? + .with_timezone(&Utc)) +} + +fn format_aw_timestamp(value: DateTime) -> String { + value.to_rfc3339_opts(SecondsFormat::Micros, true) +} + +fn load_state(path: &Path) -> Result { + if !path.exists() { + return Ok(State::default()); + } + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parse {}", path.display())) +} + +fn save_state(path: &Path, state: &State) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temp file in {}", parent.display()))?; + serde_json::to_writer_pretty(&mut temp, state)?; + std::io::Write::write_all(&mut temp, b"\n")?; + temp.persist(path) + .map_err(|err| anyhow!("persist {}: {}", path.display(), err))?; + return Ok(()); + } + fs::write(path, serde_json::to_string_pretty(state)? + "\n") + .with_context(|| format!("write {}", path.display())) +} + +fn get_start_time(cli: &Cli, state: &State) -> Result> { + if let Some(since) = &cli.since { + return parse_timestamp(since); + } + if let Some(last_end) = &state.last_end { + return Ok(parse_timestamp(last_end)? - TimeDelta::seconds(cli.overlap_seconds)); + } + Ok(Utc::now() - TimeDelta::hours(cli.lookback_hours)) +} + +fn parse_prefixes(value: &str) -> Result> { + let prefixes: Vec = value + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToOwned::to_owned) + .collect(); + if prefixes.is_empty() { + bail!("at least one bucket prefix is required"); + } + Ok(prefixes) +} + +fn bucket_stream_type(bucket_id: &str, bucket: &Bucket) -> Option<&'static str> { + if bucket_id.starts_with("aw-file-operations_") || bucket.r#type == "aw.file.operation" { + return Some("file_operation"); + } + if bucket_id.starts_with("aw-dlp-incidents_") || bucket.r#type == "aw.dlp.incident" { + return Some("dlp_incident"); + } + None +} + +fn select_buckets( + buckets: &BTreeMap, + prefixes: &[String], +) -> Vec<(String, Bucket, &'static str)> { + buckets + .iter() + .filter_map(|(bucket_id, bucket)| { + let stream_type = bucket_stream_type(bucket_id, bucket)?; + if prefixes.iter().any(|prefix| bucket_id.starts_with(prefix)) { + Some((bucket_id.clone(), bucket.clone(), stream_type)) + } else { + None + } + }) + .collect() +} + +fn build_events_path( + bucket_id: &str, + start: DateTime, + end: DateTime, + limit: usize, +) -> String { + format!( + "/buckets/{}/events?start={}&end={}&limit={}", + urlencoding::encode(bucket_id), + urlencoding::encode(&format_aw_timestamp(start)), + urlencoding::encode(&format_aw_timestamp(end)), + limit + ) +} + +fn normalized_data(value: Value) -> Value { + if value.is_object() { + value + } else { + json!({ "raw": value }) + } +} + +fn event_key(bucket_id: &str, timestamp: &str, duration: f64, data: &Value) -> String { + let payload = serde_json::to_string(data).unwrap_or_else(|_| "{}".to_string()); + format!("{bucket_id}|{timestamp}|{duration}|{payload}") +} + +fn event_id(bucket_id: &str, event: &RawAwEvent, data: &Value) -> String { + match &event.id { + Some(Value::String(value)) if !value.is_empty() => value.clone(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + _ => event_key(bucket_id, &event.timestamp, event.duration, data), + } +} + +fn fetch_bucket_events( + client: &ActivityWatchClient, + bucket_id: &str, + bucket: &Bucket, + stream_type: &str, + start: DateTime, + end: DateTime, + limit: usize, +) -> Result> { + let path = build_events_path(bucket_id, start, end, limit); + let raw_events: Vec = client.get_json(&path)?; + Ok(raw_events + .into_iter() + .map(|raw| { + let data = normalized_data(raw.data.clone()); + let hostname = if bucket.hostname.is_empty() { + first_string(&data, &["hostname"]).unwrap_or_default() + } else { + bucket.hostname.clone() + }; + AwEvent { + bucket_id: bucket_id.to_string(), + hostname, + stream_type: stream_type.to_string(), + event_id: event_id(bucket_id, &raw, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + } + }) + .collect()) +} + +fn first_string(data: &Value, keys: &[&str]) -> Option { + let object = data.as_object()?; + for key in keys { + if let Some(value) = object.get(*key) { + match value { + Value::Null => {} + Value::String(s) if !s.is_empty() => return Some(s.clone()), + other => { + let rendered = other.to_string(); + if !rendered.is_empty() { + return Some(rendered); + } + } + } + } + } + None +} + +fn bool_as_int(value: Option<&Value>) -> i64 { + match value { + Some(Value::Bool(value)) => i64::from(*value), + Some(Value::String(value)) => i64::from(matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "y" + )), + Some(Value::Number(value)) => i64::from(value.as_i64().unwrap_or(0) != 0), + Some(Value::Null) | None => 0, + Some(_) => 1, + } +} + +fn ensure_schema(connection: &Connection) -> Result<()> { + connection.execute_batch( + r#" + create table if not exists dlp_events ( + id integer primary key autoincrement, + bucket_id text not null, + event_id text not null, + stream_type text not null, + hostname text not null, + username text, + event_ts text not null, + duration real not null default 0, + operation text, + file_path text, + old_file_path text, + extension text, + archive_hint integer not null default 0, + rule_id text, + action text, + severity text, + signal_type text, + message text, + source text, + screenshot_path text, + raw_json text not null, + ingested_at text not null, + unique (bucket_id, event_id) + ); + create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts); + create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts); + create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts); + create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts); + create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts); + + create view if not exists dlp_file_operations as + select * + from dlp_events + where stream_type = 'file_operation'; + + create view if not exists dlp_incidents as + select * + from dlp_events + where stream_type = 'dlp_incident'; + "#, + )?; + Ok(()) +} + +fn connect_sqlite(path: &Path) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + let connection = Connection::open(path).with_context(|| format!("open {}", path.display()))?; + connection.execute_batch( + r#" + pragma journal_mode=WAL; + pragma synchronous=NORMAL; + pragma foreign_keys=ON; + "#, + )?; + Ok(connection) +} + +fn insert_events(connection: &mut Connection, events: &[AwEvent]) -> Result { + let ingested_at = format_aw_timestamp(Utc::now()); + let tx = connection.transaction()?; + let mut inserted = 0; + { + let mut statement = tx.prepare( + r#" + insert or ignore into dlp_events ( + bucket_id, + event_id, + stream_type, + hostname, + username, + event_ts, + duration, + operation, + file_path, + old_file_path, + extension, + archive_hint, + rule_id, + action, + severity, + signal_type, + message, + source, + screenshot_path, + raw_json, + ingested_at + ) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "#, + )?; + for event in events { + let raw_json = serde_json::to_string(&event.data)?; + let archive_hint = bool_as_int(event.data.get("archiveHint")); + let row_count = statement.execute(params![ + event.bucket_id, + event.event_id, + event.stream_type, + event.hostname, + first_string(&event.data, &["username", "user"]), + event.timestamp, + event.duration, + first_string(&event.data, &["operation"]), + first_string(&event.data, &["path", "filePath"]), + first_string(&event.data, &["oldPath", "oldFilePath"]), + first_string(&event.data, &["extension"]), + archive_hint, + first_string(&event.data, &["ruleId", "rule"]), + first_string(&event.data, &["action"]), + first_string(&event.data, &["severity"]), + first_string(&event.data, &["signalType"]), + first_string(&event.data, &["message"]), + first_string(&event.data, &["source"]), + first_string( + &event.data, + &["screenshotPath", "capturePath", "artifactPath"] + ), + raw_json, + ingested_at, + ])?; + inserted += usize::from(row_count > 0); + } + } + tx.commit()?; + Ok(inserted) +} + +fn run(cli: &Cli) -> Result<()> { + if cli.postgres_dsn.is_some() { + bail!( + "PostgreSQL mode is not supported by the Rust DLP aggregator yet; use SQLite mode or the legacy Python script" + ); + } + let prefixes = parse_prefixes(&cli.bucket_prefixes)?; + let state = load_state(&cli.state_path)?; + let start = get_start_time(cli, &state)?; + let end = Utc::now(); + let client = ActivityWatchClient::new(&cli.aw_url, Duration::from_secs(cli.timeout))?; + let buckets: BTreeMap = client.get_json("/buckets")?; + let selected = select_buckets(&buckets, &prefixes); + let mut events = Vec::new(); + for (bucket_id, bucket, stream_type) in &selected { + events.extend(fetch_bucket_events( + &client, + bucket_id, + bucket, + stream_type, + start, + end, + cli.limit, + )?); + } + + if cli.dry_run { + let summary = DryRunSummary { + aw_url: cli.aw_url.clone(), + start: format_aw_timestamp(start), + end: format_aw_timestamp(end), + selected_buckets: selected + .iter() + .map(|(bucket_id, _bucket, _stream_type)| bucket_id.clone()) + .collect(), + fetched_events: events.len(), + }; + println!("{}", serde_json::to_string_pretty(&summary)?); + return Ok(()); + } + + let mut connection = connect_sqlite(&cli.sqlite_path)?; + ensure_schema(&connection)?; + let inserted = insert_events(&mut connection, &events)?; + save_state( + &cli.state_path, + &State { + last_end: Some(format_aw_timestamp(end)), + }, + )?; + let summary = RunSummary { + aw_url: cli.aw_url.clone(), + target: "sqlite".to_string(), + target_path: cli.sqlite_path.display().to_string(), + state_path: cli.state_path.display().to_string(), + start: format_aw_timestamp(start), + end: format_aw_timestamp(end), + selected_buckets: selected.len(), + fetched_events: events.len(), + inserted_events: inserted, + }; + println!("{}", serde_json::to_string_pretty(&summary)?); + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + run(&cli) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selects_expected_bucket_prefixes() { + let buckets = BTreeMap::from([ + ( + "aw-file-operations_HOST".to_string(), + Bucket { + r#type: "aw.file.operation".to_string(), + hostname: "HOST".to_string(), + }, + ), + ( + "aw-dlp-incidents_HOST".to_string(), + Bucket { + r#type: "aw.dlp.incident".to_string(), + hostname: "HOST".to_string(), + }, + ), + ( + "aw-watcher-window_HOST".to_string(), + Bucket { + r#type: "currentwindow".to_string(), + hostname: "HOST".to_string(), + }, + ), + ]); + let selected = select_buckets( + &buckets, + &parse_prefixes("aw-file-operations_,aw-dlp-incidents_").unwrap(), + ); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0].2, "dlp_incident"); + assert_eq!(selected[1].2, "file_operation"); + } + + #[test] + fn event_row_helpers_match_python_contract() { + let data = json!({ + "username": "igor", + "path": "C:/x.zip", + "oldFilePath": "C:/x.tmp", + "archiveHint": "true", + "rule": "archive", + "signalType": "file_operation" + }); + assert_eq!( + first_string(&data, &["username", "user"]).as_deref(), + Some("igor") + ); + assert_eq!( + first_string(&data, &["oldPath", "oldFilePath"]).as_deref(), + Some("C:/x.tmp") + ); + assert_eq!(bool_as_int(data.get("archiveHint")), 1); + assert_eq!( + first_string(&data, &["ruleId", "rule"]).as_deref(), + Some("archive") + ); + } + + #[test] + fn inserts_events_once() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("dlp.sqlite"); + let mut connection = connect_sqlite(&path).unwrap(); + ensure_schema(&connection).unwrap(); + let event = AwEvent { + bucket_id: "aw-file-operations_HOST".to_string(), + hostname: "HOST".to_string(), + stream_type: "file_operation".to_string(), + event_id: "1".to_string(), + timestamp: "2026-05-31T10:00:00Z".to_string(), + duration: 0.0, + data: json!({"username":"igor","archiveHint":true}), + }; + assert_eq!( + insert_events(&mut connection, std::slice::from_ref(&event)).unwrap(), + 1 + ); + assert_eq!(insert_events(&mut connection, &[event]).unwrap(), 0); + let count: i64 = connection + .query_row("select count(*) from dlp_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(count, 1); + } +} diff --git a/adk-rust/crates/dlp-case-management/Cargo.toml b/adk-rust/crates/dlp-case-management/Cargo.toml new file mode 100644 index 0000000..2f1c927 --- /dev/null +++ b/adk-rust/crates/dlp-case-management/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dlp-case-management" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +rusqlite.workspace = true +serde_json.workspace = true +sha2.workspace = true +tiny_http.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-case-management/src/main.rs b/adk-rust/crates/dlp-case-management/src/main.rs new file mode 100644 index 0000000..f9ed78d --- /dev/null +++ b/adk-rust/crates/dlp-case-management/src/main.rs @@ -0,0 +1,880 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use rusqlite::{Connection, OptionalExtension, params}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; +use urlencoding::decode; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP Case Management API")] +struct Cli { + #[arg(long, default_value = "127.0.0.1")] + bind_host: String, + + #[arg(long, default_value_t = 5602)] + port: u16, + + #[arg( + long, + default_value = "/opt/activitywatch/dlp-case-management/cases.db" + )] + db_path: PathBuf, +} + +#[derive(Clone)] +struct AppState { + db_path: PathBuf, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = apply_env(Cli::parse()); + let storage = CaseStorage::new(cli.db_path.clone())?; + storage.init_schema()?; + let state = AppState { + db_path: cli.db_path, + }; + let address = format!("{}:{}", cli.bind_host, cli.port); + let server = Server::http(&address).map_err(|err| anyhow!("bind {address}: {err}"))?; + eprintln!("aw-dlp-case-management 0.1.0 listening on {address}"); + for request in server.incoming_requests() { + if let Err(err) = handle_request(&state, request) { + eprintln!("request error: {err:#}"); + } + } + Ok(()) +} + +fn apply_env(mut cli: Cli) -> Cli { + if !cli_arg_present("--bind-host") { + if let Ok(value) = std::env::var("AW_DLP_CASE_BIND_HOST") { + if !value.is_empty() { + cli.bind_host = value; + } + } + } + if !cli_arg_present("--port") { + if let Ok(value) = std::env::var("AW_DLP_CASE_PORT") { + if let Ok(port) = value.parse() { + cli.port = port; + } + } + } + if !cli_arg_present("--db-path") { + if let Ok(value) = std::env::var("AW_DLP_CASE_DB_PATH") { + if !value.is_empty() { + cli.db_path = PathBuf::from(value); + } + } + } + cli +} + +fn cli_arg_present(flag: &str) -> bool { + std::env::args().any(|arg| arg == flag || arg.starts_with(&format!("{flag}="))) +} + +fn handle_request(state: &AppState, mut request: Request) -> Result<()> { + let method = request.method().clone(); + let url = request.url().to_string(); + let (path, query) = split_url(&url); + let segments = path_segments(&path); + let storage = CaseStorage::new(state.db_path.clone())?; + + let response = match (method, segments.as_slice()) { + (Method::Get, ["health"]) => json_response( + StatusCode(200), + json!({"ok": true, "db": state.db_path.to_string_lossy()}), + ), + (Method::Post, ["api", "0", "dlp", "cases"]) => { + let payload = read_json_body(&mut request)?; + if is_self_test_case( + payload.get("incident_id").and_then(Value::as_str), + payload.get("title").and_then(Value::as_str), + ) { + json_response( + StatusCode(422), + json!({"detail": "self_test cases are not allowed"}), + ) + } else { + match storage.create_case(&payload) { + Ok(item) => json_response(StatusCode(200), item), + Err(err) => json_response(StatusCode(422), json!({"detail": err.to_string()})), + } + } + } + (Method::Get, ["api", "0", "dlp", "cases"]) => { + let status = query.get("status").cloned(); + let host = query.get("host").cloned(); + let limit = query_limit(&query, 200, 2000); + json_response( + StatusCode(200), + Value::Array(storage.list_cases(status, host, limit)?), + ) + } + (Method::Get, ["api", "0", "dlp", "cases", case_id]) => { + let case_id = parse_id(case_id)?; + match storage.get_case_full(case_id)? { + Some(item) => json_response(StatusCode(200), item), + None => json_response(StatusCode(404), json!({"detail": "case not found"})), + } + } + (Method::Patch, ["api", "0", "dlp", "cases", case_id]) => { + let case_id = parse_id(case_id)?; + let payload = read_json_body(&mut request)?; + match storage.update_case(case_id, &payload) { + Ok(Some(item)) => json_response(StatusCode(200), item), + Ok(None) => json_response(StatusCode(404), json!({"detail": "case not found"})), + Err(err) => json_response(StatusCode(422), json!({"detail": err.to_string()})), + } + } + (Method::Post, ["api", "0", "dlp", "cases", case_id, "comments"]) => { + let case_id = parse_id(case_id)?; + let payload = read_json_body(&mut request)?; + if storage.get_case(case_id)?.is_none() { + json_response(StatusCode(404), json!({"detail": "case not found"})) + } else { + match storage.add_comment(case_id, &payload) { + Ok(item) => json_response(StatusCode(200), item), + Err(err) => json_response(StatusCode(422), json!({"detail": err.to_string()})), + } + } + } + (Method::Get, ["api", "0", "dlp", "cases", case_id, "comments"]) => { + let case_id = parse_id(case_id)?; + let limit = query_limit(&query, 200, 2000); + json_response( + StatusCode(200), + Value::Array(storage.list_comments(case_id, limit)?), + ) + } + (Method::Post, ["api", "0", "dlp", "cases", case_id, "forensics", "hayabusa"]) => { + let case_id = parse_id(case_id)?; + let payload = read_json_body(&mut request)?; + match storage.link_hayabusa(case_id, &payload) { + Ok(Some(item)) => json_response(StatusCode(200), item), + Ok(None) => json_response(StatusCode(404), json!({"detail": "case not found"})), + Err(CaseError::HostMismatch(text)) => { + json_response(StatusCode(409), json!({"detail": text})) + } + Err(CaseError::Other(err)) => { + json_response(StatusCode(422), json!({"detail": err.to_string()})) + } + } + } + _ => json_response(StatusCode(404), json!({"detail": "not found"})), + }; + + request.respond(response).context("send HTTP response") +} + +struct CaseStorage { + db_path: PathBuf, +} + +enum CaseError { + HostMismatch(String), + Other(anyhow::Error), +} + +impl From for CaseError { + fn from(value: anyhow::Error) -> Self { + Self::Other(value) + } +} + +impl From for CaseError { + fn from(value: rusqlite::Error) -> Self { + Self::Other(value.into()) + } +} + +impl From for CaseError { + fn from(value: serde_json::Error) -> Self { + Self::Other(value.into()) + } +} + +impl CaseStorage { + fn new(db_path: PathBuf) -> Result { + if let Some(parent) = db_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create db parent {}", parent.display()))?; + } + Ok(Self { db_path }) + } + + fn connect(&self) -> Result { + let conn = Connection::open(&self.db_path) + .with_context(|| format!("open sqlite {}", self.db_path.display()))?; + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + Ok(conn) + } + + fn init_schema(&self) -> Result<()> { + let conn = self.connect()?; + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS cases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + incident_id TEXT NOT NULL, + host TEXT, + title TEXT NOT NULL, + severity TEXT NOT NULL DEFAULT 'medium', + assignee TEXT, + status TEXT NOT NULL DEFAULT 'open', + source_bucket TEXT, + source_event_ts TEXT, + evidence_json TEXT, + forensics_json TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_cases_incident_id ON cases(incident_id); + CREATE INDEX IF NOT EXISTS idx_cases_status ON cases(status); + + CREATE TABLE IF NOT EXISTS case_comments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + case_id INTEGER NOT NULL, + comment TEXT NOT NULL, + author TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY(case_id) REFERENCES cases(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS case_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + case_id INTEGER NOT NULL, + action TEXT NOT NULL, + actor TEXT, + details_json TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY(case_id) REFERENCES cases(id) ON DELETE CASCADE + ); + "#, + )?; + ensure_column(&conn, "cases", "forensics_json", "TEXT")?; + Ok(()) + } + + fn create_case(&self, payload: &Value) -> Result { + let incident_id = required_string(payload.get("incident_id"), "incident_id")?; + validate_len("incident_id", &incident_id, 1, 256)?; + let title = required_string(payload.get("title"), "title")?; + validate_len("title", &title, 1, 512)?; + let host = optional_string(payload.get("host")); + validate_optional_len("host", host.as_deref(), 128)?; + let severity = + optional_string(payload.get("severity")).unwrap_or_else(|| "medium".to_string()); + validate_optional_len("severity", Some(&severity), 32)?; + let assignee = optional_string(payload.get("assignee")); + validate_optional_len("assignee", assignee.as_deref(), 128)?; + let source_bucket = optional_string(payload.get("source_bucket")); + validate_optional_len("source_bucket", source_bucket.as_deref(), 256)?; + let source_event_ts = optional_string(payload.get("source_event_ts")); + validate_optional_len("source_event_ts", source_event_ts.as_deref(), 64)?; + + let (normalized_evidence, evidence_digest) = match payload.get("evidence") { + Some(Value::Null) | None => (None, None), + Some(evidence) => { + let normalized = normalize_evidence_chain( + evidence, + source_bucket.as_deref(), + source_event_ts.as_deref(), + )?; + let digest = normalized + .get("latest_sha256") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .unwrap_or_else(|| evidence_sha256(evidence)); + (Some(normalized), Some(digest)) + } + }; + let now = utc_now(); + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + let existing = tx + .query_row( + r#" + SELECT * FROM cases + WHERE incident_id = ? AND COALESCE(host, '') = COALESCE(?, '') + ORDER BY id DESC + LIMIT 1 + "#, + params![incident_id, host], + case_from_row, + ) + .optional()?; + if let Some(existing) = existing { + tx.commit()?; + return Ok(existing); + } + tx.execute( + r#" + INSERT INTO cases ( + incident_id, host, title, severity, assignee, status, + source_bucket, source_event_ts, evidence_json, forensics_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, 'open', ?, ?, ?, ?, ?, ?) + "#, + params![ + incident_id, + host, + title, + severity, + assignee, + source_bucket, + source_event_ts, + normalized_evidence + .as_ref() + .map(serde_json::to_string) + .transpose()?, + Option::::None, + now, + now, + ], + )?; + let case_id = tx.last_insert_rowid(); + let mut fields = payload.as_object().cloned().unwrap_or_default(); + fields.remove("evidence"); + insert_audit( + &tx, + case_id, + "create", + Some("api"), + Some(json!({"fields": fields, "evidence_sha256": evidence_digest})), + )?; + tx.commit()?; + self.get_case(case_id)? + .ok_or_else(|| anyhow!("created case not found")) + } + + fn list_cases( + &self, + status: Option, + host: Option, + limit: i64, + ) -> Result> { + let conn = self.connect()?; + let mut query = "SELECT * FROM cases".to_string(); + let mut clauses = Vec::new(); + let mut values = Vec::new(); + if let Some(status) = status { + clauses.push("status = ?"); + values.push(status); + } + if let Some(host) = host { + clauses.push("host = ?"); + values.push(host); + } + if !clauses.is_empty() { + query.push_str(" WHERE "); + query.push_str(&clauses.join(" AND ")); + } + query.push_str(" ORDER BY id DESC LIMIT ?"); + let mut stmt = conn.prepare(&query)?; + let rows = match values.as_slice() { + [] => stmt.query_map([limit.to_string()], case_from_row)?, + [a] => stmt.query_map(params![a, limit], case_from_row)?, + [a, b] => stmt.query_map(params![a, b, limit], case_from_row)?, + _ => unreachable!(), + }; + rows.collect::, _>>() + .map_err(Into::into) + } + + fn get_case(&self, case_id: i64) -> Result> { + let conn = self.connect()?; + conn.query_row("SELECT * FROM cases WHERE id = ?", [case_id], case_from_row) + .optional() + .map_err(Into::into) + } + + fn get_case_full(&self, case_id: i64) -> Result> { + let Some(mut item) = self.get_case(case_id)? else { + return Ok(None); + }; + let comments = self.list_comments(case_id, 200)?; + let audit = self.list_audit(case_id, 200)?; + let object = item + .as_object_mut() + .ok_or_else(|| anyhow!("case row is not object"))?; + object.insert("comments".to_string(), Value::Array(comments)); + object.insert("audit".to_string(), Value::Array(audit)); + Ok(Some(item)) + } + + fn update_case(&self, case_id: i64, patch: &Value) -> Result> { + if self.get_case(case_id)?.is_none() { + return Ok(None); + } + let mut fields = Vec::new(); + let mut args = Vec::new(); + for key in ["status", "assignee", "title", "severity"] { + if let Some(value) = patch.get(key).filter(|value| !value.is_null()) { + if key == "status" { + let status = required_string(Some(value), "status")?; + if !matches!( + status.as_str(), + "open" | "investigating" | "resolved" | "closed" + ) { + bail!("invalid status"); + } + args.push(status); + } else { + args.push(required_string(Some(value), key)?); + } + fields.push(key); + } + } + if fields.is_empty() { + return self.get_case(case_id); + } + let now = utc_now(); + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + let assignments = fields + .iter() + .map(|field| format!("{field} = ?")) + .collect::>() + .join(", "); + let sql = format!("UPDATE cases SET {assignments}, updated_at = ? WHERE id = ?"); + let mut params_values: Vec<&dyn rusqlite::ToSql> = args + .iter() + .map(|value| value as &dyn rusqlite::ToSql) + .collect(); + params_values.push(&now); + params_values.push(&case_id); + tx.execute(&sql, params_values.as_slice())?; + insert_audit(&tx, case_id, "update", Some("api"), Some(patch.clone()))?; + tx.commit()?; + self.get_case(case_id) + } + + fn add_comment(&self, case_id: i64, payload: &Value) -> Result { + let comment = required_string(payload.get("comment"), "comment")?; + validate_len("comment", &comment, 1, 2000)?; + let author = optional_string(payload.get("author")); + validate_optional_len("author", author.as_deref(), 128)?; + let now = utc_now(); + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + tx.execute( + "INSERT INTO case_comments (case_id, comment, author, created_at) VALUES (?, ?, ?, ?)", + params![case_id, comment, author, now], + )?; + let comment_id = tx.last_insert_rowid(); + insert_audit( + &tx, + case_id, + "comment", + author.as_deref(), + Some(json!({"comment_id": comment_id})), + )?; + let row = tx.query_row( + "SELECT id, case_id, comment, author, created_at FROM case_comments WHERE id = ?", + [comment_id], + comment_from_row, + )?; + tx.commit()?; + Ok(row) + } + + fn list_comments(&self, case_id: i64, limit: i64) -> Result> { + let conn = self.connect()?; + let mut stmt = conn.prepare( + "SELECT id, case_id, comment, author, created_at FROM case_comments WHERE case_id = ? ORDER BY id DESC LIMIT ?", + )?; + let rows = stmt.query_map(params![case_id, limit], comment_from_row)?; + rows.collect::, _>>() + .map_err(Into::into) + } + + fn list_audit(&self, case_id: i64, limit: i64) -> Result> { + let conn = self.connect()?; + let mut stmt = conn.prepare( + "SELECT id, case_id, action, actor, details_json, created_at FROM case_audit WHERE case_id = ? ORDER BY id DESC LIMIT ?", + )?; + let rows = stmt.query_map(params![case_id, limit], audit_from_row)?; + rows.collect::, _>>() + .map_err(Into::into) + } + + fn link_hayabusa( + &self, + case_id: i64, + payload: &Value, + ) -> std::result::Result, CaseError> { + let existing = match self.get_case(case_id)? { + Some(item) => item, + None => return Ok(None), + }; + let case_host = normalize_host(existing.get("host").and_then(Value::as_str)); + let forensic_host = normalize_host(payload.get("host").and_then(Value::as_str)); + if !case_host.is_empty() && !forensic_host.is_empty() && case_host != forensic_host { + return Err(CaseError::HostMismatch(format!( + "hayabusa host mismatch: case host={} payload host={}", + existing.get("host").and_then(Value::as_str).unwrap_or(""), + payload.get("host").and_then(Value::as_str).unwrap_or("") + ))); + } + let host = required_string(payload.get("host"), "host")?; + validate_len("host", &host, 1, 128)?; + let mode = required_string(payload.get("mode"), "mode")?; + validate_len("mode", &mode, 1, 32)?; + let status = required_string(payload.get("status"), "status")?; + validate_len("status", &status, 1, 64)?; + let now = utc_now(); + let mut forensics = existing + .get("forensics") + .cloned() + .filter(|value| value.is_object()) + .unwrap_or_else(|| json!({})); + forensics.as_object_mut().expect("object").insert( + "hayabusa".to_string(), + json!({ + "tool": "hayabusa", + "host": host, + "mode": mode, + "status": status, + "intake_id": optional_json_string(payload.get("intake_id")), + "package_path": optional_json_string(payload.get("package_path")), + "sha256": optional_json_string(payload.get("sha256")), + "report_dir": optional_json_string(payload.get("report_dir")), + "summary_html": optional_json_string(payload.get("summary_html")), + "timeline_path": optional_json_string(payload.get("timeline_path")), + "manifest_path": optional_json_string(payload.get("manifest_path")), + "linked_at": optional_string(payload.get("linked_at")).unwrap_or_else(|| now.clone()), + "link_source": optional_string(payload.get("link_source")).unwrap_or_else(|| "api".to_string()), + }), + ); + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + tx.execute( + "UPDATE cases SET forensics_json = ?, updated_at = ? WHERE id = ?", + params![serde_json::to_string(&forensics)?, now, case_id], + )?; + insert_audit( + &tx, + case_id, + "link_hayabusa", + Some("api"), + Some(json!({ + "host": payload.get("host").cloned().unwrap_or(Value::Null), + "mode": payload.get("mode").cloned().unwrap_or(Value::Null), + "status": payload.get("status").cloned().unwrap_or(Value::Null), + "intake_id": payload.get("intake_id").cloned().unwrap_or(Value::Null), + "report_dir": payload.get("report_dir").cloned().unwrap_or(Value::Null), + })), + )?; + tx.commit()?; + Ok(self.get_case(case_id)?) + } +} + +fn ensure_column(conn: &Connection, table: &str, column: &str, definition: &str) -> Result<()> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; + let columns = stmt + .query_map([], |row| row.get::<_, String>(1))? + .collect::, _>>()?; + if !columns.iter().any(|item| item == column) { + conn.execute( + &format!("ALTER TABLE {table} ADD COLUMN {column} {definition}"), + [], + )?; + } + Ok(()) +} + +fn case_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let evidence = json_field(row.get::<_, Option>("evidence_json")?); + let forensics = json_field(row.get::<_, Option>("forensics_json")?); + Ok(json!({ + "id": row.get::<_, i64>("id")?, + "incident_id": row.get::<_, String>("incident_id")?, + "host": row.get::<_, Option>("host")?, + "title": row.get::<_, String>("title")?, + "severity": row.get::<_, String>("severity")?, + "assignee": row.get::<_, Option>("assignee")?, + "status": row.get::<_, String>("status")?, + "source_bucket": row.get::<_, Option>("source_bucket")?, + "source_event_ts": row.get::<_, Option>("source_event_ts")?, + "evidence": evidence, + "forensics": forensics, + "created_at": row.get::<_, String>("created_at")?, + "updated_at": row.get::<_, String>("updated_at")?, + })) +} + +fn comment_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(json!({ + "id": row.get::<_, i64>("id")?, + "case_id": row.get::<_, i64>("case_id")?, + "comment": row.get::<_, String>("comment")?, + "author": row.get::<_, Option>("author")?, + "created_at": row.get::<_, String>("created_at")?, + })) +} + +fn audit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(json!({ + "id": row.get::<_, i64>("id")?, + "case_id": row.get::<_, i64>("case_id")?, + "action": row.get::<_, String>("action")?, + "actor": row.get::<_, Option>("actor")?, + "details": json_field(row.get::<_, Option>("details_json")?), + "created_at": row.get::<_, String>("created_at")?, + })) +} + +fn insert_audit( + conn: &Connection, + case_id: i64, + action: &str, + actor: Option<&str>, + details: Option, +) -> Result<()> { + conn.execute( + "INSERT INTO case_audit (case_id, action, actor, details_json, created_at) VALUES (?, ?, ?, ?, ?)", + params![ + case_id, + action, + actor, + details.as_ref().map(serde_json::to_string).transpose()?, + utc_now(), + ], + )?; + Ok(()) +} + +fn normalize_evidence_chain( + payload: &Value, + source_bucket: Option<&str>, + source_event_ts: Option<&str>, +) -> Result { + if payload.get("items").and_then(Value::as_array).is_some() { + return Ok(payload.clone()); + } + let digest = evidence_sha256(payload); + let record = json!({ + "recorded_at": utc_now(), + "source_bucket": source_bucket, + "source_event_ts": source_event_ts, + "sha256": digest, + "payload": payload, + }); + Ok(json!({ + "items": [record], + "latest_sha256": digest, + "chain_length": 1, + })) +} + +fn evidence_sha256(payload: &Value) -> String { + let mut hasher = Sha256::new(); + hasher.update( + canonical_json(payload) + .expect("evidence JSON serializable") + .as_bytes(), + ); + format!("{:x}", hasher.finalize()) +} + +fn canonical_json(value: &Value) -> Result { + match value { + Value::Null => Ok("null".to_string()), + Value::Bool(value) => Ok(value.to_string()), + Value::Number(value) => Ok(value.to_string()), + Value::String(value) => serde_json::to_string(value).map_err(Into::into), + Value::Array(items) => { + let mut out = String::from("["); + for (idx, item) in items.iter().enumerate() { + if idx > 0 { + out.push(','); + } + out.push_str(&canonical_json(item)?); + } + out.push(']'); + Ok(out) + } + Value::Object(map) => { + let mut out = String::from("{"); + let mut sorted = BTreeMap::new(); + for (key, value) in map { + sorted.insert(key, value); + } + for (idx, (key, value)) in sorted.into_iter().enumerate() { + if idx > 0 { + out.push(','); + } + out.push_str(&serde_json::to_string(key)?); + out.push(':'); + out.push_str(&canonical_json(value)?); + } + out.push('}'); + Ok(out) + } + } +} + +fn is_self_test_case(incident_id: Option<&str>, title: Option<&str>) -> bool { + incident_id + .unwrap_or("") + .to_lowercase() + .contains("|self_test|") + || title + .unwrap_or("") + .to_lowercase() + .starts_with("dlp self_test") +} + +fn split_url(url: &str) -> (String, BTreeMap) { + let (path, query) = url.split_once('?').unwrap_or((url, "")); + let mut params = BTreeMap::new(); + for pair in query.split('&').filter(|item| !item.is_empty()) { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + params.insert(url_decode(key), url_decode(value)); + } + (path.to_string(), params) +} + +fn path_segments(path: &str) -> Vec<&str> { + path.trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect() +} + +fn query_limit(query: &BTreeMap, default: i64, max: i64) -> i64 { + query + .get("limit") + .and_then(|value| value.parse().ok()) + .unwrap_or(default) + .clamp(1, max) +} + +fn read_json_body(request: &mut Request) -> Result { + let mut body = String::new(); + request + .as_reader() + .read_to_string(&mut body) + .context("read request body")?; + if body.trim().is_empty() { + Ok(json!({})) + } else { + serde_json::from_str(&body).context("decode JSON body") + } +} + +fn json_response(status: StatusCode, payload: Value) -> Response>> { + let body = + serde_json::to_vec_pretty(&payload).unwrap_or_else(|_| b"{\"detail\":\"json\"}".to_vec()); + let mut response = Response::from_data(body).with_status_code(status); + for (key, value) in [ + ("Content-Type", "application/json; charset=utf-8"), + ("Access-Control-Allow-Origin", "*"), + ("Access-Control-Allow-Methods", "*"), + ("Access-Control-Allow-Headers", "*"), + ] { + response.add_header( + Header::from_bytes(key.as_bytes(), value.as_bytes()).expect("valid header"), + ); + } + response +} + +fn parse_id(value: &str) -> Result { + value + .parse::() + .with_context(|| format!("invalid case id: {value}")) +} + +fn required_string(value: Option<&Value>, field: &str) -> Result { + value + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .ok_or_else(|| anyhow!("{field} is required")) +} + +fn optional_string(value: Option<&Value>) -> Option { + match value { + Some(Value::String(text)) => Some(text.clone()), + _ => None, + } +} + +fn optional_json_string(value: Option<&Value>) -> Value { + optional_string(value) + .map(Value::String) + .unwrap_or(Value::Null) +} + +fn validate_len(field: &str, value: &str, min: usize, max: usize) -> Result<()> { + let len = value.chars().count(); + if len < min || len > max { + bail!("{field} length must be {min}..{max}"); + } + Ok(()) +} + +fn validate_optional_len(field: &str, value: Option<&str>, max: usize) -> Result<()> { + if let Some(value) = value { + if value.chars().count() > max { + bail!("{field} length must be <= {max}"); + } + } + Ok(()) +} + +fn json_field(raw: Option) -> Value { + raw.as_deref() + .and_then(|text| serde_json::from_str(text).ok()) + .unwrap_or(Value::Null) +} + +fn utc_now() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Micros, false) +} + +fn normalize_host(value: Option<&str>) -> String { + value.unwrap_or("").trim().to_lowercase() +} + +fn url_decode(value: &str) -> String { + decode(value) + .map(|value| value.into_owned()) + .unwrap_or_else(|_| value.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn self_test_detection_matches_python() { + assert!(is_self_test_case(Some("x|self_test|y"), Some("normal"))); + assert!(is_self_test_case(None, Some("DLP self_test probe"))); + assert!(!is_self_test_case(Some("incident"), Some("DLP incident"))); + } + + #[test] + fn evidence_checksum_matches_python_fixture() { + let value = json!({"b": 2, "a": "тест"}); + assert_eq!( + evidence_sha256(&value), + "350682d4ca349bdb90a337bf891720f43e85fad66e7811637a72f96b3f29984a" + ); + } +} diff --git a/adk-rust/crates/dlp-cef-exporter/Cargo.toml b/adk-rust/crates/dlp-cef-exporter/Cargo.toml new file mode 100644 index 0000000..db1b92e --- /dev/null +++ b/adk-rust/crates/dlp-cef-exporter/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "dlp-cef-exporter" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true diff --git a/adk-rust/crates/dlp-cef-exporter/src/main.rs b/adk-rust/crates/dlp-cef-exporter/src/main.rs new file mode 100644 index 0000000..6187d5d --- /dev/null +++ b/adk-rust/crates/dlp-cef-exporter/src/main.rs @@ -0,0 +1,505 @@ +use std::collections::BTreeMap; +use std::fs; +use std::io::Write; +use std::net::{TcpStream, UdpSocket}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Value, json}; + +const DEFAULT_CONFIG: &str = "/opt/activitywatch/dlp-integrations/cef-config.yaml"; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP CEF exporter")] +struct Cli { + #[arg(long, default_value = DEFAULT_CONFIG)] + config: PathBuf, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + state_path: PathBuf, + syslog_host: String, + syslog_port: u16, + syslog_proto: String, + per_bucket_limit: usize, + severity_mapping: BTreeMap, +} + +impl Default for Config { + fn default() -> Self { + Self { + aw_api_base: "http://127.0.0.1:5600/api/0".to_string(), + state_path: PathBuf::from("/var/lib/activitywatch/dlp-integrations/cef-state.json"), + syslog_host: "127.0.0.1".to_string(), + syslog_port: 514, + syslog_proto: "udp".to_string(), + per_bucket_limit: 300, + severity_mapping: BTreeMap::from([ + ("low".to_string(), 3), + ("medium".to_string(), 6), + ("high".to_string(), 10), + ]), + } + } +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + sent: usize, + buckets: usize, + dry_run: bool, + state_saved: bool, + state_path: String, + target: String, + error: Option, +} + +fn load_config(path: &Path) -> Config { + let mut config = Config::default(); + let Ok(text) = fs::read_to_string(path) else { + return config; + }; + let mut in_severity_mapping = false; + for raw_line in text.lines() { + let line_without_comment = raw_line.split('#').next().unwrap_or(""); + let line = line_without_comment.trim(); + if line.is_empty() { + continue; + } + if line == "severity_mapping:" { + in_severity_mapping = true; + config.severity_mapping.clear(); + continue; + } + if in_severity_mapping && (raw_line.starts_with(' ') || raw_line.starts_with('\t')) { + if let Some((key, value)) = line.split_once(':') + && let Ok(score) = clean_scalar(value).parse::() + { + config + .severity_mapping + .insert(key.trim().to_ascii_lowercase(), score); + } + continue; + } + in_severity_mapping = false; + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let key = key.trim(); + let value = clean_scalar(value); + match key { + "aw_api_base" if !value.is_empty() => config.aw_api_base = value, + "state_path" if !value.is_empty() => config.state_path = PathBuf::from(value), + "syslog_host" if !value.is_empty() => config.syslog_host = value, + "syslog_port" => config.syslog_port = value.parse().unwrap_or(config.syslog_port), + "syslog_proto" if !value.is_empty() => config.syslog_proto = value.to_ascii_lowercase(), + "per_bucket_limit" => { + config.per_bucket_limit = value.parse().unwrap_or(config.per_bucket_limit); + } + _ => {} + } + } + if config.severity_mapping.is_empty() { + config.severity_mapping = Config::default().severity_mapping; + } + config +} + +fn clean_scalar(value: &str) -> String { + value + .trim() + .trim_matches('"') + .trim_matches('\'') + .to_string() +} + +fn load_json(path: &Path) -> Value { + let Ok(text) = fs::read_to_string(path) else { + return json!({}); + }; + serde_json::from_str::(&text) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| json!({})) +} + +fn save_json(path: &Path, payload: &Value) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::write(path, serde_json::to_string_pretty(payload)?) + .with_context(|| format!("write {}", path.display())) +} + +fn http_json(client: &Client, url: &str) -> Result { + client + .get(url) + .timeout(Duration::from_secs(15)) + .send() + .and_then(|resp| resp.error_for_status()) + .with_context(|| format!("GET {url}"))? + .json::() + .with_context(|| format!("parse JSON from {url}")) +} + +fn int_value(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(s)) => s.parse::().unwrap_or(0), + _ => 0, + } +} + +fn value_str(value: Option<&Value>, default: &str) -> String { + value.and_then(Value::as_str).unwrap_or(default).to_string() +} + +fn incident_bucket_ids(buckets: &Value) -> Vec { + let mut ids: Vec = buckets + .as_object() + .map(|map| { + map.keys() + .filter(|bucket_id| bucket_id.starts_with("aw-dlp-incidents_")) + .cloned() + .collect() + }) + .unwrap_or_default(); + ids.sort(); + ids +} + +fn iter_new_incidents( + client: &Client, + aw_base: &str, + state: &Value, + per_bucket_limit: usize, +) -> Result<(Vec, BTreeMap)> { + let base = aw_base.trim_end_matches('/'); + let buckets = http_json(client, &format!("{base}/buckets/"))?; + let bucket_ids = incident_bucket_ids(&buckets); + let last_ids = state.get("last_ids").and_then(Value::as_object); + let mut max_ids = BTreeMap::new(); + let mut out = Vec::new(); + for bucket_id in bucket_ids { + let events = match http_json( + client, + &format!("{base}/buckets/{bucket_id}/events?limit={per_bucket_limit}"), + ) { + Ok(Value::Array(items)) => items, + Ok(_) => continue, + Err(err) => return Err(err), + }; + let prev = last_ids + .and_then(|ids| ids.get(&bucket_id)) + .and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }) + .unwrap_or(0); + let mut bucket_max = prev; + for event in events { + let event_id = int_value(event.get("id")); + if event_id <= prev { + continue; + } + if event_id > bucket_max { + bucket_max = event_id; + } + out.push(event); + } + max_ids.insert(bucket_id, bucket_max); + } + out.sort_by_key(|event| int_value(event.get("id"))); + Ok((out, max_ids)) +} + +fn escape_cef(value: Option<&Value>) -> String { + let text = match value { + Some(Value::String(s)) => s.clone(), + Some(Value::Number(n)) => n.to_string(), + Some(Value::Bool(v)) => v.to_string(), + Some(Value::Null) | None => String::new(), + Some(other) => other.to_string(), + }; + text.replace('\\', "\\\\") + .replace('|', "\\|") + .replace('=', "\\=") + .replace('\n', "\\n") + .replace('\r', "") +} + +fn map_severity(name: &str, mapping: &BTreeMap) -> i64 { + mapping + .get(&name.to_ascii_lowercase()) + .copied() + .unwrap_or(3) +} + +fn build_cef(event: &Value, mapping: &BTreeMap) -> String { + let data = event + .get("data") + .filter(|value| value.is_object()) + .unwrap_or(&Value::Null); + let sev_name = value_str(data.get("severity"), "low").to_ascii_lowercase(); + let sev_num = map_severity(&sev_name, mapping); + let rt = event + .get("timestamp") + .cloned() + .unwrap_or_else(|| json!(Utc::now().to_rfc3339_opts(SecondsFormat::Micros, false))); + let rule = data + .get("ruleId") + .cloned() + .unwrap_or_else(|| json!("dlp-incident")); + let msg = data + .get("message") + .cloned() + .unwrap_or_else(|| json!("AWatch DLP incident")); + let sig = data + .get("signalType") + .cloned() + .unwrap_or_else(|| json!("unknown")); + let host = data + .get("hostname") + .cloned() + .unwrap_or_else(|| json!("unknown")); + let user = data + .get("username") + .cloned() + .unwrap_or_else(|| json!("unknown")); + let action = data + .get("action") + .cloned() + .unwrap_or_else(|| json!("alert")); + let ext = format!( + "rt={} shost={} suser={} cs1Label=signalType cs1={} cs2Label=action cs2={} cs3Label=ruleId cs3={}", + escape_cef(Some(&rt)), + escape_cef(Some(&host)), + escape_cef(Some(&user)), + escape_cef(Some(&sig)), + escape_cef(Some(&action)), + escape_cef(Some(&rule)), + ); + format!( + "CEF:0|AWatch-rus|DLP|1.0|{}|{}|{}|{}", + escape_cef(Some(&rule)), + escape_cef(Some(&msg)), + sev_num, + ext + ) +} + +fn send_syslog(line: &str, host: &str, port: u16, proto: &str) -> Result<()> { + if proto.eq_ignore_ascii_case("tcp") { + let mut stream = TcpStream::connect((host, port)) + .with_context(|| format!("connect TCP syslog {host}:{port}"))?; + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .context("set TCP write timeout")?; + stream + .write_all(format!("{line}\n").as_bytes()) + .context("write TCP syslog")?; + return Ok(()); + } + let socket = UdpSocket::bind("0.0.0.0:0").context("bind UDP syslog socket")?; + socket + .send_to(line.as_bytes(), (host, port)) + .with_context(|| format!("send UDP syslog {host}:{port}"))?; + Ok(()) +} + +fn run(cli: &Cli, client: &Client) -> RunSummary { + let cfg = load_config(&cli.config); + let state = load_json(&cfg.state_path); + let (incidents, max_ids) = + match iter_new_incidents(client, &cfg.aw_api_base, &state, cfg.per_bucket_limit) { + Ok(result) => result, + Err(err) => { + eprintln!("skip CEF exporter run: AW API unavailable: {err}"); + return RunSummary { + ok: true, + sent: 0, + buckets: 0, + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + target: format!( + "{}:{}/{}", + cfg.syslog_host, cfg.syslog_port, cfg.syslog_proto + ), + error: Some(err.to_string()), + }; + } + }; + + let mut sent = 0; + for event in &incidents { + let line = build_cef(event, &cfg.severity_mapping); + if !cli.dry_run + && let Err(err) = + send_syslog(&line, &cfg.syslog_host, cfg.syslog_port, &cfg.syslog_proto) + { + return RunSummary { + ok: false, + sent, + buckets: max_ids.len(), + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + target: format!( + "{}:{}/{}", + cfg.syslog_host, cfg.syslog_port, cfg.syslog_proto + ), + error: Some(err.to_string()), + }; + } + sent += 1; + } + + let mut next_state = state; + next_state["last_ids"] = json!(max_ids); + next_state["updated_at"] = json!(Utc::now().to_rfc3339_opts(SecondsFormat::Micros, false)); + let mut state_saved = false; + if !cli.dry_run { + if let Err(err) = save_json(&cfg.state_path, &next_state) { + return RunSummary { + ok: false, + sent, + buckets: next_state["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: false, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + target: format!( + "{}:{}/{}", + cfg.syslog_host, cfg.syslog_port, cfg.syslog_proto + ), + error: Some(err.to_string()), + }; + } + state_saved = true; + } + + RunSummary { + ok: true, + sent, + buckets: next_state["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: cli.dry_run, + state_saved, + state_path: cfg.state_path.to_string_lossy().to_string(), + target: format!( + "{}:{}/{}", + cfg.syslog_host, cfg.syslog_port, cfg.syslog_proto + ), + error: None, + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let client = Client::builder() + .no_proxy() + .build() + .context("build HTTP client")?; + let summary = run(&cli, &client); + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if let Some(err) = &summary.error { + eprintln!("{err}"); + } else { + println!( + "CEF exporter done: sent={} buckets={} target={}", + summary.sent, summary.buckets, summary.target + ); + } + if summary.ok { + Ok(()) + } else { + Err(anyhow!( + summary + .error + .unwrap_or_else(|| "CEF exporter failed".to_string()) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_cef_fields_like_python_contract() { + assert_eq!( + escape_cef(Some(&json!("a\\b|c=d\nx\r"))), + "a\\\\b\\|c\\=d\\nx" + ); + assert_eq!(escape_cef(None), ""); + } + + #[test] + fn builds_cef_line() { + let event = json!({ + "timestamp": "2026-06-01T10:00:00Z", + "data": { + "severity": "high", + "ruleId": "usb|copy", + "message": "a=b", + "signalType": "dlp", + "hostname": "host", + "username": "user", + "action": "alert" + } + }); + let line = build_cef(&event, &Config::default().severity_mapping); + assert!(line.starts_with("CEF:0|AWatch-rus|DLP|1.0|usb\\|copy|a\\=b|10|")); + assert!(line.contains("shost=host")); + assert!(line.contains("cs3Label=ruleId cs3=usb\\|copy")); + } + + #[test] + fn parses_config_mapping() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cef-config.yaml"); + fs::write( + &path, + r#" +aw_api_base: "http://127.0.0.1:5600/api/0" +state_path: "/tmp/cef-state.json" +syslog_host: "127.0.0.1" +syslog_port: 5514 +syslog_proto: "tcp" +per_bucket_limit: 10 +severity_mapping: + low: 1 + medium: 5 + high: 9 +"#, + ) + .unwrap(); + let cfg = load_config(&path); + assert_eq!(cfg.syslog_port, 5514); + assert_eq!(cfg.syslog_proto, "tcp"); + assert_eq!(cfg.per_bucket_limit, 10); + assert_eq!(cfg.severity_mapping["high"], 9); + } +} diff --git a/adk-rust/crates/dlp-compliance/Cargo.toml b/adk-rust/crates/dlp-compliance/Cargo.toml new file mode 100644 index 0000000..e131a9a --- /dev/null +++ b/adk-rust/crates/dlp-compliance/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "dlp-compliance" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-compliance/src/main.rs b/adk-rust/crates/dlp-compliance/src/main.rs new file mode 100644 index 0000000..fef81a8 --- /dev/null +++ b/adk-rust/crates/dlp-compliance/src/main.rs @@ -0,0 +1,414 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{DateTime, Datelike, SecondsFormat, TimeZone, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use reqwest::header::ACCEPT; +use serde::Serialize; +use serde_json::{Value, json}; +use urlencoding::encode; + +#[derive(Debug, Parser)] +#[command(about = "Generate DLP compliance reports from AW DLP incidents")] +struct Cli { + #[arg(long)] + month: Option, + + #[arg(long)] + profile: Option, + + #[arg(long, default_value = "152-fz,pci-dss")] + profiles: String, + + #[arg(long)] + stdout_json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + output_dir: PathBuf, + base_dir: PathBuf, + template_152fz: PathBuf, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct ReportStats { + total_incidents: usize, + high: usize, + medium: usize, + low: usize, + by_host: BTreeMap, + channels: BTreeMap, +} + +#[derive(Debug, Serialize)] +struct Metadata { + profile: String, + period: String, + generated_at: String, + aw_api_base: String, + report_path: String, + stats: MetadataStats, +} + +#[derive(Debug, Serialize)] +struct MetadataStats { + total_incidents: usize, + high: usize, + medium: usize, + low: usize, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let config = Config::from_env()?; + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .no_proxy() + .build() + .context("build HTTP client")?; + let profiles = if let Some(profile) = cli.profile.as_deref() { + vec![profile.to_string()] + } else { + cli.profiles + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToOwned::to_owned) + .collect::>() + }; + let mut items = Vec::new(); + for profile in profiles { + items.push(generate_report( + &client, + &config, + cli.month.as_deref(), + &profile, + )?); + } + if cli.stdout_json { + if cli.profile.is_some() && items.len() == 1 { + println!("{}", serde_json::to_string(&items[0])?); + } else { + println!("{}", serde_json::to_string(&json!({"items": items}))?); + } + } + Ok(()) +} + +impl Config { + fn from_env() -> Result { + let aw_raw = env_string("AW_DLP_AW_API_BASE") + .or_else(|| env_string("AW_SERVER_URL")) + .unwrap_or_else(|| "http://127.0.0.1:5600".to_string()); + let aw_api_base = build_aw_api_base(&aw_raw); + let output_dir = env_path("AW_DLP_COMPLIANCE_REPORT_DIR") + .unwrap_or_else(|| PathBuf::from("/opt/activitywatch/dlp-compliance/reports")); + let base_dir = env_path("AW_DLP_COMPLIANCE_BASE_DIR") + .unwrap_or_else(|| PathBuf::from("/opt/activitywatch/dlp-compliance")); + let template_152fz = env_path("AW_DLP_COMPLIANCE_TEMPLATE") + .unwrap_or_else(|| base_dir.join("templates/152-fz-report.html")); + Ok(Self { + aw_api_base, + output_dir, + base_dir, + template_152fz, + }) + } +} + +fn generate_report( + client: &Client, + config: &Config, + month: Option<&str>, + profile: &str, +) -> Result { + let (start, end, period_label) = period_bounds(month)?; + let incidents = load_incidents(client, config, start, end)?; + let stats = build_stats(&incidents); + fs::create_dir_all(&config.output_dir) + .with_context(|| format!("create output dir {}", config.output_dir.display()))?; + let generated_at = Utc::now().to_rfc3339_opts(SecondsFormat::Micros, true); + let html_path = config + .output_dir + .join(format!("{profile}-{period_label}.html")); + let html = render_html(config, profile, &period_label, &stats, &generated_at)?; + fs::write(&html_path, html).with_context(|| format!("write {}", html_path.display()))?; + let metadata = Metadata { + profile: profile.to_string(), + period: period_label.clone(), + generated_at, + aw_api_base: config.aw_api_base.clone(), + report_path: html_path.to_string_lossy().to_string(), + stats: MetadataStats { + total_incidents: stats.total_incidents, + high: stats.high, + medium: stats.medium, + low: stats.low, + }, + }; + let json_path = config + .output_dir + .join(format!("{profile}-{period_label}.json")); + fs::write(&json_path, serde_json::to_string_pretty(&metadata)?) + .with_context(|| format!("write {}", json_path.display()))?; + Ok(metadata) +} + +fn build_aw_api_base(raw_url: &str) -> String { + let url = raw_url.trim().trim_end_matches('/').to_string(); + if url.ends_with("/api/0") { + url + } else { + format!("{url}/api/0") + } +} + +fn load_incidents( + client: &Client, + config: &Config, + start: DateTime, + end: DateTime, +) -> Result> { + let buckets = http_json(client, &format!("{}/buckets", config.aw_api_base))?; + let Some(map) = buckets.as_object() else { + return Ok(Vec::new()); + }; + let mut bucket_ids = map + .keys() + .filter(|bucket_id| bucket_id.starts_with("aw-dlp-incidents_")) + .cloned() + .collect::>(); + bucket_ids.sort(); + let mut incidents = Vec::new(); + for bucket_id in bucket_ids { + let encoded = encode(&bucket_id); + let url = format!("{}/buckets/{encoded}/events?limit=2000", config.aw_api_base); + let events = http_json(client, &url)?; + let Some(events) = events.as_array() else { + continue; + }; + for event in events { + let Some(ts) = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(parse_ts) + else { + continue; + }; + if ts < start || ts > end { + continue; + } + incidents.push(event.clone()); + } + } + Ok(incidents) +} + +fn http_json(client: &Client, url: &str) -> Result { + client + .get(url) + .header(ACCEPT, "application/json") + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn build_stats(incidents: &[Value]) -> ReportStats { + let mut by_host = BTreeMap::new(); + let mut channels = BTreeMap::new(); + let mut high = 0; + let mut medium = 0; + let mut low = 0; + for event in incidents { + let data = event.get("data").and_then(Value::as_object); + let host = data + .and_then(|item| item.get("hostname")) + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(); + *by_host.entry(host).or_insert(0) += 1; + + match data + .and_then(|item| item.get("severity")) + .and_then(Value::as_str) + .unwrap_or("low") + .to_lowercase() + .as_str() + { + "high" => high += 1, + "medium" => medium += 1, + _ => low += 1, + } + let channel = data + .and_then(|item| item.get("signalType").or_else(|| item.get("source"))) + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(); + *channels.entry(channel).or_insert(0) += 1; + } + ReportStats { + total_incidents: incidents.len(), + high, + medium, + low, + by_host: sort_counts_desc(by_host), + channels: sort_counts_desc(channels), + } +} + +fn sort_counts_desc(values: BTreeMap) -> BTreeMap { + let mut rows = values.into_iter().collect::>(); + rows.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + rows.into_iter().collect() +} + +fn render_html( + config: &Config, + profile: &str, + period_label: &str, + stats: &ReportStats, + generated_at: &str, +) -> Result { + let template_path = resolve_template_path(config, profile); + let template = fs::read_to_string(&template_path) + .with_context(|| format!("read template {}", template_path.display()))?; + Ok(template + .replace("{{PERIOD}}", period_label) + .replace("{{PROFILE}}", profile) + .replace("{{GENERATED_AT}}", generated_at) + .replace("{{TOTAL}}", &stats.total_incidents.to_string()) + .replace("{{HIGH}}", &stats.high.to_string()) + .replace("{{MEDIUM}}", &stats.medium.to_string()) + .replace("{{LOW}}", &stats.low.to_string()) + .replace( + "{{HOST_TABLE}}", + &render_table("Инциденты по хостам", &stats.by_host), + ) + .replace( + "{{CHANNEL_TABLE}}", + &render_table("Инциденты по каналам", &stats.channels), + )) +} + +fn resolve_template_path(config: &Config, profile: &str) -> PathBuf { + if profile == "152-fz" { + config.template_152fz.clone() + } else { + let specific = config + .base_dir + .join(format!("templates/{profile}-report.html")); + if specific.exists() { + specific + } else { + config.template_152fz.clone() + } + } +} + +fn render_table(title: &str, rows: &BTreeMap) -> String { + if rows.is_empty() { + return format!("

{title}

Нет данных

"); + } + let body = rows + .iter() + .map(|(name, count)| format!("{name}{count}")) + .collect::(); + format!( + "

{title}

{body}
ПараметрЗначение
" + ) +} + +fn period_bounds(month: Option<&str>) -> Result<(DateTime, DateTime, String)> { + let start = if let Some(month) = month { + let (year, month) = month + .split_once('-') + .ok_or_else(|| anyhow!("month must be YYYY-MM"))?; + Utc.with_ymd_and_hms(year.parse()?, month.parse()?, 1, 0, 0, 0) + .single() + .ok_or_else(|| anyhow!("invalid month"))? + } else { + let now = Utc::now(); + Utc.with_ymd_and_hms(now.year(), now.month(), 1, 0, 0, 0) + .single() + .ok_or_else(|| anyhow!("invalid current month"))? + }; + let end = if start.month() == 12 { + Utc.with_ymd_and_hms(start.year() + 1, 1, 1, 0, 0, 0) + .single() + .ok_or_else(|| anyhow!("invalid end month"))? + } else { + Utc.with_ymd_and_hms(start.year(), start.month() + 1, 1, 0, 0, 0) + .single() + .ok_or_else(|| anyhow!("invalid end month"))? + }; + Ok(( + start, + end, + format!("{:04}-{:02}", start.year(), start.month()), + )) +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .map(|ts| ts.with_timezone(&Utc)) + .ok() +} + +fn env_string(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn env_path(name: &str) -> Option { + env_string(name).map(PathBuf::from) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aw_api_base_matches_python_contract() { + assert_eq!(build_aw_api_base("http://x"), "http://x/api/0"); + assert_eq!(build_aw_api_base("http://x/api/0"), "http://x/api/0"); + } + + #[test] + fn stats_match_expected_counts() { + let incidents = vec![ + json!({"data":{"hostname":"a","severity":"high","signalType":"usb"}}), + json!({"data":{"hostname":"a","severity":"medium","source":"clipboard"}}), + json!({"data":{"hostname":"b","severity":"low"}}), + ]; + let stats = build_stats(&incidents); + assert_eq!(stats.total_incidents, 3); + assert_eq!(stats.high, 1); + assert_eq!(stats.medium, 1); + assert_eq!(stats.low, 1); + assert_eq!(stats.by_host.get("a"), Some(&2)); + assert_eq!(stats.channels.get("unknown"), Some(&1)); + } + + #[test] + fn month_bounds_are_utc_month() { + let (start, end, label) = period_bounds(Some("2026-06")).unwrap(); + assert_eq!(label, "2026-06"); + assert_eq!(start.to_rfc3339(), "2026-06-01T00:00:00+00:00"); + assert_eq!(end.to_rfc3339(), "2026-07-01T00:00:00+00:00"); + } +} diff --git a/adk-rust/crates/dlp-content-analyzer/Cargo.toml b/adk-rust/crates/dlp-content-analyzer/Cargo.toml new file mode 100644 index 0000000..0fb6187 --- /dev/null +++ b/adk-rust/crates/dlp-content-analyzer/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "dlp-content-analyzer" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/dlp-content-analyzer/src/main.rs b/adk-rust/crates/dlp-content-analyzer/src/main.rs new file mode 100644 index 0000000..73d8e26 --- /dev/null +++ b/adk-rust/crates/dlp-content-analyzer/src/main.rs @@ -0,0 +1,387 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use regex::Regex; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_BASE_DIR: &str = "/opt/activitywatch/dlp-content-analysis"; + +#[derive(Debug, Parser)] +#[command(about = "Analyze text or screenshot with DLP dictionaries/regex packs.")] +struct Cli { + #[arg(long)] + text: Option, + + #[arg(long)] + image: Option, + + #[arg(long)] + dictionary_pack: Option, + + #[arg(long)] + regex_pack: Option, + + #[arg(long, default_value = DEFAULT_BASE_DIR)] + base_dir: PathBuf, + + #[arg( + long, + default_value = "/opt/activitywatch/dlp-content-analysis/.venv/bin/python" + )] + legacy_python: PathBuf, + + #[arg( + long, + default_value = "/opt/activitywatch/dlp-content-analysis/content_analyzer.py" + )] + legacy_analyzer: PathBuf, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +struct MatchResult { + name: String, + description: String, + value: String, + start: usize, + end: usize, + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option, +} + +#[derive(Debug, Serialize)] +struct Analysis { + text: String, + dictionary_pack: Option, + regex_pack: Option, + dictionary_matches: Vec, + regex_matches: Vec, + source: String, +} + +fn digits_only(value: &str) -> String { + value.chars().filter(|ch| ch.is_ascii_digit()).collect() +} + +fn validate_inn(value: &str) -> bool { + let digits = digits_only(value); + let nums = digits + .bytes() + .map(|byte| (byte - b'0') as u32) + .collect::>(); + match nums.len() { + 10 => { + let coef = [2, 4, 10, 3, 5, 9, 4, 6, 8]; + let chk = nums + .iter() + .take(9) + .zip(coef) + .map(|(digit, coef)| digit * coef) + .sum::() + % 11 + % 10; + chk == nums[9] + } + 12 => { + let c11 = [7, 2, 4, 10, 3, 5, 9, 4, 6, 8]; + let c12 = [3, 7, 2, 4, 10, 3, 5, 9, 4, 6, 8]; + let chk11 = nums + .iter() + .take(10) + .zip(c11) + .map(|(digit, coef)| digit * coef) + .sum::() + % 11 + % 10; + let chk12 = nums + .iter() + .take(11) + .zip(c12) + .map(|(digit, coef)| digit * coef) + .sum::() + % 11 + % 10; + chk11 == nums[10] && chk12 == nums[11] + } + _ => false, + } +} + +fn validate_snils(value: &str) -> bool { + let digits = digits_only(value); + if digits.len() != 11 { + return false; + } + let nums = digits + .bytes() + .map(|byte| (byte - b'0') as u32) + .collect::>(); + let checksum = nums[9] * 10 + nums[10]; + let sum = nums + .iter() + .take(9) + .enumerate() + .map(|(idx, digit)| digit * (9 - idx as u32)) + .sum::(); + let expected = match sum { + 0..=99 => sum, + 100 | 101 => 0, + _ => { + let value = sum % 101; + if value == 100 { 0 } else { value } + } + }; + checksum == expected +} + +fn validate_passport(value: &str) -> bool { + let digits = digits_only(value); + digits.len() == 10 + && digits != "0000000000" + && digits.chars().any(|ch| ch != digits.as_bytes()[0] as char) +} + +fn validate_checksum(kind: &str, value: &str) -> bool { + match kind { + "inn" => validate_inn(value), + "snils" => validate_snils(value), + "passport" => validate_passport(value), + _ => true, + } +} + +fn char_offset(text: &str, byte_offset: usize) -> usize { + text[..byte_offset].chars().count() +} + +fn load_json(path: &Path) -> Result { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parse {}", path.display())) +} + +fn resolve_pack(base_dir: &Path, kind: &str, name: Option<&str>) -> Option { + let name = name?; + let path = base_dir.join(kind).join(format!("{name}.json")); + path.exists().then_some(path) +} + +fn match_dictionary(text: &str, path: Option<&Path>) -> Result> { + let Some(path) = path else { + return Ok(Vec::new()); + }; + let rules = load_json(path)?; + let Some(rules) = rules.as_object() else { + return Ok(Vec::new()); + }; + let mut results = Vec::new(); + for (name, rule) in rules { + let Some(regex) = rule.get("regex").and_then(Value::as_str) else { + continue; + }; + let regex = + Regex::new(regex).with_context(|| format!("compile dictionary regex {name}"))?; + let checksum = rule + .get("checksum") + .and_then(Value::as_str) + .unwrap_or("none"); + let description = rule + .get("description") + .and_then(Value::as_str) + .unwrap_or(name) + .to_string(); + for capture in regex.find_iter(text) { + let value = capture.as_str(); + if validate_checksum(checksum, value) { + results.push(MatchResult { + name: name.to_string(), + description: description.clone(), + value: value.to_string(), + start: char_offset(text, capture.start()), + end: char_offset(text, capture.end()), + severity: None, + }); + } + } + } + results.sort_by_key(|result| (result.start, result.end, result.name.clone())); + Ok(results) +} + +fn regex_entries(pack: &Value) -> Vec<(String, String, String, String)> { + let mut entries = Vec::new(); + if let Some(rules) = pack.get("rules").and_then(Value::as_array) { + for rule in rules.iter().filter_map(Value::as_object) { + let Some(regex) = rule.get("regex").and_then(Value::as_str) else { + continue; + }; + let id = rule + .get("id") + .or_else(|| rule.get("name")) + .and_then(Value::as_str) + .unwrap_or("regex-rule"); + let description = rule + .get("description") + .and_then(Value::as_str) + .unwrap_or(id); + let severity = rule + .get("severity") + .and_then(Value::as_str) + .unwrap_or("medium"); + entries.push(( + id.to_string(), + description.to_string(), + regex.to_string(), + severity.to_string(), + )); + } + } else if let Some(patterns) = pack.get("patterns").and_then(Value::as_object) { + for (id, rule) in patterns { + let Some(rule) = rule.as_object() else { + continue; + }; + let Some(regex) = rule.get("regex").and_then(Value::as_str) else { + continue; + }; + let description = rule + .get("description") + .and_then(Value::as_str) + .unwrap_or(id); + let severity = rule + .get("severity") + .and_then(Value::as_str) + .unwrap_or("medium"); + entries.push(( + id.to_string(), + description.to_string(), + regex.to_string(), + severity.to_string(), + )); + } + } + entries +} + +fn match_regex_pack(text: &str, path: Option<&Path>) -> Result> { + let Some(path) = path else { + return Ok(Vec::new()); + }; + let pack = load_json(path)?; + let mut results = Vec::new(); + for (name, description, regex, severity) in regex_entries(&pack) { + let regex = + Regex::new(®ex).with_context(|| format!("compile regex pack rule {name}"))?; + for capture in regex.find_iter(text) { + results.push(MatchResult { + name: name.clone(), + description: description.clone(), + value: capture.as_str().to_string(), + start: char_offset(text, capture.start()), + end: char_offset(text, capture.end()), + severity: Some(severity.clone()), + }); + } + } + Ok(results) +} + +fn analyze_text(cli: &Cli) -> Result { + let text = cli.text.clone().unwrap_or_default(); + let dictionary_path = resolve_pack( + &cli.base_dir, + "dictionaries", + cli.dictionary_pack.as_deref(), + ); + let regex_pack_path = resolve_pack(&cli.base_dir, "regex-packs", cli.regex_pack.as_deref()); + Ok(Analysis { + dictionary_matches: match_dictionary(&text, dictionary_path.as_deref())?, + regex_matches: match_regex_pack(&text, regex_pack_path.as_deref())?, + text, + dictionary_pack: cli.dictionary_pack.clone(), + regex_pack: cli.regex_pack.clone(), + source: "text".to_string(), + }) +} + +fn run_legacy_image(cli: &Cli, image: &Path) -> Result { + if !cli.legacy_python.exists() || !cli.legacy_analyzer.exists() { + bail!( + "image OCR requires legacy analyzer: {} {}", + cli.legacy_python.display(), + cli.legacy_analyzer.display() + ); + } + let mut command = Command::new(&cli.legacy_python); + command.arg(&cli.legacy_analyzer).arg("--image").arg(image); + if let Some(pack) = &cli.dictionary_pack { + command.arg("--dictionary-pack").arg(pack); + } + if let Some(pack) = &cli.regex_pack { + command.arg("--regex-pack").arg(pack); + } + let output = command.output().context("run legacy image analyzer")?; + if !output.status.success() { + bail!( + "legacy image analyzer failed rc={:?}: {}{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + serde_json::from_slice(&output.stdout).context("parse legacy image analyzer JSON") +} + +fn run(cli: &Cli) -> Result { + if let Some(image) = &cli.image { + return run_legacy_image(cli, image); + } + Ok(serde_json::to_value(analyze_text(cli)?)?) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let value = run(&cli)?; + println!("{}", serde_json::to_string_pretty(&value)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_inn_like_python() { + assert!(validate_inn("7707083893")); + assert!(validate_inn("500100732259")); + assert!(!validate_inn("7707083894")); + } + + #[test] + fn validates_snils_like_python() { + assert!(validate_snils("112-233-445 95")); + assert!(!validate_snils("112-233-445 96")); + } + + #[test] + fn validates_passport_like_python() { + assert!(validate_passport("1234 567890")); + assert!(!validate_passport("0000 000000")); + assert!(!validate_passport("1111 111111")); + } + + #[test] + fn regex_pack_patterns_shape_is_supported() { + let pack = serde_json::json!({"patterns": {"secret": {"regex": "token", "description": "Token", "severity": "high"}}}); + assert_eq!( + regex_entries(&pack), + vec![( + "secret".to_string(), + "Token".to_string(), + "token".to_string(), + "high".to_string() + )] + ); + } +} diff --git a/adk-rust/crates/dlp-health-check/Cargo.toml b/adk-rust/crates/dlp-health-check/Cargo.toml new file mode 100644 index 0000000..9b366b6 --- /dev/null +++ b/adk-rust/crates/dlp-health-check/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dlp-health-check" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-core.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true diff --git a/adk-rust/crates/dlp-health-check/src/main.rs b/adk-rust/crates/dlp-health-check/src/main.rs new file mode 100644 index 0000000..588dfd7 --- /dev/null +++ b/adk-rust/crates/dlp-health-check/src/main.rs @@ -0,0 +1,1523 @@ +use std::cmp::Reverse; +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::thread::sleep; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{DateTime, SecondsFormat, Utc}; +use clap::Parser; +use detmir_core::parse_utc_rfc3339; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Value, json}; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP health check")] +struct Cli { + #[arg(long, default_value = "http://127.0.0.1:5600")] + aw_server: String, + + #[arg(long, default_value = "http://127.0.0.1:5601")] + policy_server: String, + + #[arg(long, default_value = "http://127.0.0.1:5602")] + case_server: String, + + #[arg(long, default_value_t = 900)] + max_age_seconds: i64, + + #[arg(long)] + strict_fileops: bool, + + #[arg(long, default_value_t = 100)] + endpoint_queue_warn_depth: i64, + + #[arg(long, default_value_t = 1)] + endpoint_send_failure_warn_count: i64, + + #[arg(long, default_value_t = 20)] + fileops_sample_limit: i64, + + #[arg(long, default_value_t = 100)] + fileops_queue_warn_depth: i64, + + #[arg(long, default_value_t = 1)] + fileops_send_failure_warn_count: i64, + + #[arg(long, default_value_t = 0)] + incident_sample_limit: i64, + + #[arg(long, default_value = "/var/lib/activitywatch/health")] + state_dir: PathBuf, + + #[arg(long, default_value = "/opt/activitywatch/dlp-compliance/reports")] + report_dir: PathBuf, + + #[arg(long, default_value = "152-fz,pci-dss")] + profiles: String, + + #[arg(long)] + json: bool, +} + +impl Cli { + fn apply_env(mut self) -> Self { + if !cli_arg_present("--aw-server") { + self.aw_server = env_string("AW_HEALTH_AW_SERVER").unwrap_or(self.aw_server); + } + if !cli_arg_present("--policy-server") { + self.policy_server = + env_string("AW_HEALTH_POLICY_SERVER").unwrap_or(self.policy_server); + } + if !cli_arg_present("--case-server") { + self.case_server = env_string("AW_HEALTH_CASE_SERVER").unwrap_or(self.case_server); + } + if !cli_arg_present("--max-age-seconds") { + self.max_age_seconds = env_i64("AW_HEALTH_MAX_AGE_SECONDS", self.max_age_seconds); + } + if !cli_arg_present("--strict-fileops") { + self.strict_fileops = env_bool("AW_HEALTH_STRICT_FILEOPS"); + } + if !cli_arg_present("--endpoint-queue-warn-depth") { + self.endpoint_queue_warn_depth = env_i64( + "AW_DLP_HEALTH_ENDPOINT_QUEUE_WARN_DEPTH", + self.endpoint_queue_warn_depth, + ); + } + if !cli_arg_present("--endpoint-send-failure-warn-count") { + self.endpoint_send_failure_warn_count = env_i64( + "AW_DLP_HEALTH_ENDPOINT_SEND_FAILURE_WARN_COUNT", + self.endpoint_send_failure_warn_count, + ); + } + if !cli_arg_present("--fileops-sample-limit") { + self.fileops_sample_limit = env_i64( + "AW_DLP_HEALTH_FILEOPS_SAMPLE_LIMIT", + self.fileops_sample_limit, + ); + } + if !cli_arg_present("--fileops-queue-warn-depth") { + self.fileops_queue_warn_depth = env_i64( + "AW_DLP_HEALTH_FILEOPS_QUEUE_WARN_DEPTH", + self.fileops_queue_warn_depth, + ); + } + if !cli_arg_present("--fileops-send-failure-warn-count") { + self.fileops_send_failure_warn_count = env_i64( + "AW_DLP_HEALTH_FILEOPS_SEND_FAILURE_WARN_COUNT", + self.fileops_send_failure_warn_count, + ); + } + if !cli_arg_present("--incident-sample-limit") { + self.incident_sample_limit = env_i64( + "AW_DLP_HEALTH_INCIDENT_SAMPLE_LIMIT", + self.incident_sample_limit, + ); + } + if !cli_arg_present("--state-dir") { + self.state_dir = + env_path("AW_DLP_HEALTH_STATE_DIR").unwrap_or_else(|| self.state_dir.clone()); + } + if !cli_arg_present("--report-dir") { + self.report_dir = + env_path("AW_DLP_COMPLIANCE_REPORT_DIR").unwrap_or_else(|| self.report_dir.clone()); + } + if !cli_arg_present("--profiles") { + self.profiles = env_string("AW_DLP_COMPLIANCE_PROFILES").unwrap_or(self.profiles); + } + self + } +} + +#[derive(Debug, Clone, Serialize)] +struct CheckResult { + name: String, + status: String, + summary: String, + details: Value, +} + +#[derive(Debug, Serialize)] +struct Counts { + ok: usize, + warn: usize, + fail: usize, +} + +#[derive(Debug, Serialize)] +struct HealthPayload { + ok: bool, + counts: Counts, + results: Vec, +} + +#[derive(Default)] +struct HealthReport { + results: Vec, +} + +impl HealthReport { + fn add( + &mut self, + name: impl Into, + status: &str, + summary: impl Into, + details: Value, + ) { + self.results.push(CheckResult { + name: name.into(), + status: status.to_string(), + summary: summary.into(), + details, + }); + } + + fn ok(&self) -> bool { + !self.results.iter().any(|item| item.status == "fail") + } + + fn payload(&self) -> HealthPayload { + let mut counts = Counts { + ok: 0, + warn: 0, + fail: 0, + }; + for item in &self.results { + match item.status.as_str() { + "ok" => counts.ok += 1, + "warn" => counts.warn += 1, + "fail" => counts.fail += 1, + _ => {} + } + } + HealthPayload { + ok: self.ok(), + counts, + results: self.results.clone(), + } + } + + fn render_text(&self) -> String { + let mut lines = vec![ + "=== DLP Health Check ===".to_string(), + format!("Timestamp: {}", utc_iso()), + String::new(), + ]; + for item in &self.results { + let icon = match item.status.as_str() { + "ok" => "OK", + "warn" => "WARN", + "fail" => "FAIL", + other => other, + }; + lines.push(format!("[{icon}] {}: {}", item.name, item.summary)); + if !item.details.as_object().is_none_or(|v| v.is_empty()) { + lines.push(format!( + " details: {}", + serde_json::to_string(&sort_json_value(&item.details)) + .unwrap_or_else(|_| "{}".to_string()) + )); + } + } + lines.push(String::new()); + lines.push(format!( + "Overall: {}", + if self.ok() { "OK" } else { "FAIL" } + )); + lines.join("\n") + } +} + +fn utc_iso() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Micros, true) +} + +fn parse_ts(value: Option<&str>) -> Option> { + value.and_then(|text| parse_utc_rfc3339(text).ok()) +} + +fn age_seconds(ts: Option>, now: DateTime) -> Option { + ts.map(|ts| (now - ts).num_seconds().max(0)) +} + +fn int_or_zero(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(s)) => s.parse::().unwrap_or(0), + Some(Value::Bool(v)) => i64::from(*v), + _ => 0, + } +} + +fn value_str(value: Option<&Value>) -> String { + match value { + Some(Value::String(s)) => s.clone(), + Some(Value::Number(n)) => n.to_string(), + Some(Value::Bool(v)) => v.to_string(), + Some(Value::Null) | None => String::new(), + Some(other) => other.to_string(), + } +} + +fn path_tail(value: Option<&Value>, parts: usize) -> String { + let text = value_str(value).replace('\\', "/").trim().to_string(); + if text.is_empty() { + return String::new(); + } + let tokens: Vec<&str> = text.split('/').filter(|item| !item.is_empty()).collect(); + if tokens.is_empty() { + return text; + } + let start = tokens.len().saturating_sub(parts); + tokens[start..].join("/") +} + +fn text_excerpt(value: Option<&Value>, limit: usize) -> String { + let text = value_str(value) + .split_whitespace() + .collect::>() + .join(" "); + if text.chars().count() <= limit { + return text; + } + let prefix: String = text.chars().take(limit.saturating_sub(1)).collect(); + format!("{}…", prefix.trim_end()) +} + +fn http_json(client: &Client, url: &str, timeout_seconds: u64, attempts: usize) -> Result { + let mut last_error: Option = None; + for attempt in 0..attempts.max(1) { + match client + .get(url) + .timeout(Duration::from_secs(timeout_seconds)) + .send() + .and_then(|resp| resp.error_for_status()) + .with_context(|| format!("GET {url}")) + .and_then(|resp| resp.json::().context("parse JSON")) + { + Ok(value) => return Ok(value), + Err(err) => { + last_error = Some(err); + if attempt + 1 < attempts.max(1) { + sleep(Duration::from_millis(500 * (1 << attempt))); + } + } + } + } + Err(last_error.unwrap_or_else(|| anyhow!("HTTP request failed"))) +} + +fn check_http_endpoint(report: &mut HealthReport, client: &Client, name: &str, url: &str) { + match http_json(client, url, 15, 2) { + Ok(payload) => report.add( + name, + "ok", + "HTTP endpoint responded", + json!({"url": url, "payload": payload}), + ), + Err(err) => report.add( + name, + "fail", + format!("HTTP endpoint failed: {err}"), + json!({"url": url}), + ), + } +} + +fn run_systemctl(args: &[&str]) -> (i32, String) { + match Command::new("systemctl").args(args).output() { + Ok(output) => { + let mut text = String::from_utf8_lossy(&output.stdout).to_string(); + text.push_str(&String::from_utf8_lossy(&output.stderr)); + (output.status.code().unwrap_or(1), text.trim().to_string()) + } + Err(err) => (1, err.to_string()), + } +} + +fn check_systemd_unit(report: &mut HealthReport, unit: &str, kind: &str) { + let (active_rc, active_out) = run_systemctl(&["is-active", unit]); + let (enabled_rc, enabled_out) = run_systemctl(&["is-enabled", unit]); + let (exists_rc, _) = run_systemctl(&["status", unit]); + if exists_rc != 0 && active_rc != 0 && enabled_rc != 0 { + report.add( + format!("systemd:{unit}"), + "warn", + "unit not installed", + json!({"kind": kind}), + ); + return; + } + if active_rc == 0 && enabled_rc == 0 { + report.add( + format!("systemd:{unit}"), + "ok", + "active and enabled", + json!({"kind": kind}), + ); + return; + } + report.add( + format!("systemd:{unit}"), + "fail", + "unit is not active/enabled", + json!({ + "kind": kind, + "active": if active_out.is_empty() { active_rc.to_string() } else { active_out }, + "enabled": if enabled_out.is_empty() { enabled_rc.to_string() } else { enabled_out }, + }), + ); +} + +fn latest_bucket_ts( + client: &Client, + api_base: &str, + bucket_id: &str, + bucket_meta: &Value, +) -> Option> { + let meta_ts = bucket_meta + .get("metadata") + .and_then(|v| v.get("end")) + .and_then(Value::as_str); + if let Some(ts) = parse_ts(meta_ts) { + return Some(ts); + } + let url = format!("{api_base}/buckets/{bucket_id}/events?limit=1"); + let events = http_json(client, &url, 15, 2).ok()?; + events + .as_array() + .and_then(|items| items.first()) + .and_then(|event| event.get("timestamp")) + .and_then(Value::as_str) + .and_then(|text| parse_ts(Some(text))) +} + +fn bucket_suffix(bucket_id: &str, prefix: &str) -> String { + bucket_id + .strip_prefix(prefix) + .unwrap_or(bucket_id) + .to_string() +} + +fn bucket_keys_with_prefix(buckets: &BTreeMap, prefix: &str) -> Vec { + buckets + .keys() + .filter(|bucket_id| bucket_id.starts_with(prefix)) + .cloned() + .collect() +} + +fn check_incident_buckets( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + max_age_seconds: i64, +) { + let now = Utc::now(); + let prefix = "aw-dlp-incidents_"; + let matched = bucket_keys_with_prefix(buckets, prefix); + if matched.is_empty() { + report.add( + "buckets:incidents", + "ok", + "no incident buckets yet", + json!({"prefix": prefix, "bucket_count": 0}), + ); + return; + } + let mut ages = Vec::new(); + let mut unknown = Vec::new(); + let mut stale = Vec::new(); + for bucket_id in &matched { + let ts = latest_bucket_ts( + client, + api_base, + bucket_id, + buckets.get(bucket_id).unwrap_or(&Value::Null), + ); + match age_seconds(ts, now) { + Some(age) => { + ages.push(age); + if age > max_age_seconds { + stale.push(json!({"bucket": bucket_id, "age_seconds": age})); + } + } + None => unknown.push(bucket_id.clone()), + } + } + if !stale.is_empty() && unknown.is_empty() { + report.add( + "buckets:incidents", + "ok", + "no recent incidents", + json!({ + "prefix": prefix, + "bucket_count": matched.len(), + "max_age_seconds": max_age_seconds, + "max_observed_age_seconds": ages.iter().max().copied(), + "stale": stale, + "unknown": [], + }), + ); + return; + } + let (status, summary) = if unknown.is_empty() { + ("ok", "incident buckets healthy".to_string()) + } else { + ( + "warn", + format!("{} incident buckets without timestamp", unknown.len()), + ) + }; + report.add( + "buckets:incidents", + status, + summary, + json!({ + "prefix": prefix, + "bucket_count": matched.len(), + "max_age_seconds": max_age_seconds, + "max_observed_age_seconds": ages.iter().max().copied(), + "stale": stale, + "unknown": unknown, + }), + ); +} + +struct HostBucketCheck<'a> { + check_name: &'a str, + prefix: &'a str, + max_age_seconds: i64, + strict: bool, +} + +fn worktime_activity_map( + client: &Client, + api_base: &str, + buckets: &BTreeMap, + max_age_seconds: i64, +) -> BTreeMap { + let now = Utc::now(); + let prefix = "aw-worktime-sessions_"; + let mut activity = BTreeMap::new(); + for bucket_id in bucket_keys_with_prefix(buckets, prefix) { + let host = bucket_suffix(&bucket_id, prefix); + let mut latest_ts: Option> = None; + let mut latest_active = false; + let url = format!("{api_base}/buckets/{bucket_id}/events?limit=20"); + if let Ok(events) = http_json(client, &url, 15, 2) + && let Some(items) = events.as_array() + { + for event in items { + let ts = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(|text| parse_ts(Some(text))); + if let Some(ts) = ts + && latest_ts.is_none_or(|current| ts > current) + { + latest_ts = Some(ts); + latest_active = event + .get("data") + .and_then(|v| v.get("active")) + .and_then(Value::as_bool) + .unwrap_or(false); + } + } + } + let age = age_seconds(latest_ts, now); + activity.insert( + host, + json!({ + "active": latest_ts.is_some() && latest_active && age.unwrap_or(0) <= max_age_seconds, + "age_seconds": age, + "bucket": bucket_id, + }), + ); + } + activity +} + +fn check_host_bucket_freshness( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + config: HostBucketCheck<'_>, +) { + let now = Utc::now(); + let matched = bucket_keys_with_prefix(buckets, config.prefix); + let worktime = worktime_activity_map(client, api_base, buckets, config.max_age_seconds); + let active_hosts: Vec = worktime + .iter() + .filter(|(_, meta)| meta.get("active").and_then(Value::as_bool).unwrap_or(false)) + .map(|(host, _)| host.clone()) + .collect(); + let matched_by_host: BTreeMap = matched + .iter() + .map(|bucket_id| (bucket_suffix(bucket_id, config.prefix), bucket_id.clone())) + .collect(); + let mut ignored_unmanaged = Vec::new(); + let mut ignored_inactive = Vec::new(); + let mut missing_active = Vec::new(); + let mut stale = Vec::new(); + let mut unknown = Vec::new(); + let mut fresh = Vec::new(); + + for (host, bucket_id) in &matched_by_host { + if !worktime.contains_key(host) { + ignored_unmanaged.push(bucket_id.clone()); + continue; + } + if !active_hosts.contains(host) { + ignored_inactive.push(bucket_id.clone()); + continue; + } + let ts = latest_bucket_ts( + client, + api_base, + bucket_id, + buckets.get(bucket_id).unwrap_or(&Value::Null), + ); + match age_seconds(ts, now) { + Some(age) if age > config.max_age_seconds => { + stale.push(json!({"bucket": bucket_id, "age_seconds": age})); + } + Some(_) => fresh.push(bucket_id.clone()), + None => unknown.push(bucket_id.clone()), + } + } + + for host in &active_hosts { + if !matched_by_host.contains_key(host) { + missing_active.push(host.clone()); + } + } + + if active_hosts.is_empty() { + report.add( + config.check_name, + "ok", + format!( + "no active managed hosts require {} freshness", + config.check_name.trim_start_matches("buckets:") + ), + json!({ + "active_hosts": [], + "ignored_unmanaged": ignored_unmanaged, + "ignored_inactive": ignored_inactive, + "worktime_hosts": worktime.keys().cloned().collect::>(), + }), + ); + return; + } + + let mut status = "ok"; + let mut summary = format!("{} active host buckets fresh", fresh.len()); + if !missing_active.is_empty() { + status = if config.strict { "fail" } else { "warn" }; + summary = format!( + "{} active hosts missing {} buckets", + missing_active.len(), + config.check_name.trim_start_matches("buckets:") + ); + } else if !stale.is_empty() { + status = if config.strict { "fail" } else { "warn" }; + summary = format!("{} active host buckets stale", stale.len()); + } else if !unknown.is_empty() { + status = "warn"; + summary = format!("{} active host buckets without timestamp", unknown.len()); + } + + report.add( + config.check_name, + status, + summary, + json!({ + "active_hosts": active_hosts, + "fresh": fresh, + "stale": stale, + "missing_active": missing_active, + "unknown": unknown, + "ignored_unmanaged": ignored_unmanaged, + "ignored_inactive": ignored_inactive, + }), + ); +} + +fn check_endpoint_signal_buckets( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + max_age_seconds: i64, +) { + check_host_bucket_freshness( + report, + client, + api_base, + buckets, + HostBucketCheck { + check_name: "buckets:endpoint-signals", + prefix: "aw-dlp-endpoint-signals_", + max_age_seconds, + strict: true, + }, + ); +} + +fn check_file_operations_buckets( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + max_age_seconds: i64, + strict: bool, +) { + check_host_bucket_freshness( + report, + client, + api_base, + buckets, + HostBucketCheck { + check_name: "buckets:file-operations", + prefix: "aw-file-operations_", + max_age_seconds, + strict, + }, + ); +} + +fn load_counter_state(path: &Path) -> Value { + let Ok(text) = fs::read_to_string(path) else { + return json!({"counters": {}}); + }; + let Ok(mut payload) = serde_json::from_str::(&text) else { + return json!({"counters": {}}); + }; + if !payload.is_object() { + return json!({"counters": {}}); + } + if !payload.get("counters").is_some_and(Value::is_object) { + payload["counters"] = json!({}); + } + payload +} + +fn save_counter_state(path: &Path, state: &Value) -> Option { + let content = match serde_json::to_string_pretty(&sort_json_value(state)) { + Ok(text) => text + "\n", + Err(err) => return Some(err.to_string()), + }; + if let Err(err) = write_atomic(path, &content) { + match fs::write(path, content) { + Ok(()) => None, + Err(_) => Some(err.to_string()), + } + } else { + None + } +} + +fn counter_delta( + counter_state: Option<&mut Value>, + key: &str, + current_value: i64, +) -> (Option, i64) { + let Some(state) = counter_state else { + return (None, current_value); + }; + if !state.get("counters").is_some_and(Value::is_object) { + state["counters"] = json!({}); + } + let counters = state + .get_mut("counters") + .and_then(Value::as_object_mut) + .unwrap(); + let previous = counters.get(key).and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }); + counters.insert(key.to_string(), json!(current_value)); + if previous.is_none() || current_value < previous.unwrap_or(0) { + (previous, 0) + } else { + (previous, current_value - previous.unwrap_or(0)) + } +} + +#[derive(Clone, Copy)] +struct RuntimeThresholds { + sample_limit: i64, + queue_warn_depth: i64, + send_failure_warn_count: i64, +} + +fn check_file_operations_runtime( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + thresholds: RuntimeThresholds, + mut counter_state: Option<&mut Value>, +) { + let now = Utc::now(); + let prefix = "aw-file-operations_"; + let matched = bucket_keys_with_prefix(buckets, prefix); + if matched.is_empty() { + report.add( + "file-operations-runtime", + "warn", + "no file-operations buckets to sample", + json!({"bucket_count": 0}), + ); + return; + } + + let mut sampled = Vec::new(); + let mut latest_operations = Vec::new(); + let mut latest_health = Vec::new(); + let mut warnings = Vec::new(); + let mut read_failed = Vec::new(); + + for bucket_id in &matched { + let url = format!( + "{api_base}/buckets/{bucket_id}/events?limit={}", + thresholds.sample_limit + ); + let events = match http_json(client, &url, 15, 2) { + Ok(Value::Array(items)) => items, + Ok(_) => { + read_failed + .push(json!({"bucket": bucket_id, "error": "events response is not a list"})); + continue; + } + Err(err) => { + read_failed.push(json!({"bucket": bucket_id, "error": err.to_string()})); + continue; + } + }; + let mut operation_counts: BTreeMap = BTreeMap::new(); + let mut latest_health_event: Option = None; + let mut latest_health_ts: Option> = None; + for event in &events { + let data = event + .get("data") + .filter(|v| v.is_object()) + .unwrap_or(&Value::Null); + let ts = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(|text| parse_ts(Some(text))); + let signal_type = value_str(data.get("signalType")); + let operation = value_str(data.get("operation")); + if signal_type == "collector_health" { + if latest_health_event.is_none() + || ts.is_some_and(|ts| latest_health_ts.is_none_or(|current| ts > current)) + { + latest_health_event = Some(event.clone()); + latest_health_ts = ts; + } + continue; + } + if !operation.is_empty() { + *operation_counts.entry(operation.clone()).or_insert(0) += 1; + latest_operations.push(json!({ + "bucket": bucket_id, + "timestamp": event.get("timestamp").cloned().unwrap_or(Value::Null), + "age_seconds": age_seconds(ts, now), + "operation": operation, + "username": value_str(data.get("username")), + "hostname": value_str(data.get("hostname")), + "extension": value_str(data.get("extension")), + "archiveHint": data.get("archiveHint").and_then(Value::as_bool).unwrap_or(false), + "path_tail": path_tail(data.get("path"), 2), + "size": int_or_zero(data.get("size")), + })); + } + } + sampled.push(json!({ + "bucket": bucket_id, + "sampled_events": events.len(), + "operation_counts": operation_counts, + })); + let Some(health_event) = latest_health_event else { + warnings.push(json!({"bucket": bucket_id, "metric": "collector_health", "value": "missing_in_sample"})); + continue; + }; + let health_data = health_event.get("data").unwrap_or(&Value::Null); + let send_failures = int_or_zero(health_data.get("sendFailures")); + let (previous, delta) = counter_delta( + counter_state.as_deref_mut(), + &format!("file-operations:{bucket_id}:sendFailures"), + send_failures, + ); + let health_item = json!({ + "bucket": bucket_id, + "timestamp": health_event.get("timestamp").cloned().unwrap_or(Value::Null), + "age_seconds": age_seconds(latest_health_ts, now), + "queueDepth": int_or_zero(health_data.get("queueDepth")), + "eventsEnqueued": int_or_zero(health_data.get("eventsEnqueued")), + "eventsFlushed": int_or_zero(health_data.get("eventsFlushed")), + "sendFailures": send_failures, + "sendFailuresPrevious": previous, + "sendFailuresDelta": delta, + "username": value_str(health_data.get("username")), + "hostname": value_str(health_data.get("hostname")), + "sessionId": int_or_zero(health_data.get("sessionId")), + }); + if health_item["queueDepth"].as_i64().unwrap_or(0) > thresholds.queue_warn_depth { + warnings.push(json!({"bucket": bucket_id, "metric": "queueDepth", "value": health_item["queueDepth"], "threshold": thresholds.queue_warn_depth})); + } + if thresholds.send_failure_warn_count > 0 && delta >= thresholds.send_failure_warn_count { + warnings.push(json!({ + "bucket": bucket_id, + "metric": "sendFailuresDelta", + "value": delta, + "current": send_failures, + "previous": previous, + "threshold": thresholds.send_failure_warn_count, + })); + } + latest_health.push(health_item); + } + + latest_operations.sort_by_key(|item| Reverse(value_str(item.get("timestamp")))); + let mut status = "ok"; + let mut summary = format!("{} file-operations buckets sampled", matched.len()); + if !read_failed.is_empty() { + status = "warn"; + summary = format!( + "{} file-operations buckets failed to sample", + read_failed.len() + ); + } else if !warnings.is_empty() { + status = "warn"; + summary = "file-operations runtime counters outside expectations".to_string(); + } + + report.add( + "file-operations-runtime", + status, + summary, + json!({ + "bucket_count": matched.len(), + "sample_limit": thresholds.sample_limit, + "sampled": sampled, + "latest_health": latest_health, + "latest_operations": latest_operations.into_iter().take(5).collect::>(), + "warnings": warnings, + "read_failed": read_failed, + "thresholds": {"queueDepth": thresholds.queue_warn_depth, "sendFailures": thresholds.send_failure_warn_count}, + }), + ); +} + +fn check_endpoint_self_test_metrics( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + queue_warn_depth: i64, + send_failure_warn_count: i64, + mut counter_state: Option<&mut Value>, +) { + let now = Utc::now(); + let mut missing = Vec::new(); + let mut latest_self_tests = Vec::new(); + let mut warnings = Vec::new(); + for bucket_id in bucket_keys_with_prefix(buckets, "aw-dlp-endpoint-signals_") { + let url = format!("{api_base}/buckets/{bucket_id}/events?limit=20"); + let events = match http_json(client, &url, 15, 2) { + Ok(Value::Array(items)) => items, + Ok(_) => { + report.add( + format!("endpoint-self-test:{bucket_id}"), + "warn", + "failed to read events: events response is not a list", + json!({"bucket": bucket_id}), + ); + continue; + } + Err(err) => { + report.add( + format!("endpoint-self-test:{bucket_id}"), + "warn", + format!("failed to read events: {err}"), + json!({"bucket": bucket_id}), + ); + continue; + } + }; + let mut latest_event: Option = None; + let mut latest_ts: Option> = None; + for event in &events { + let data = event + .get("data") + .filter(|v| v.is_object()) + .unwrap_or(&Value::Null); + let is_self_test = data.get("signalType").and_then(Value::as_str) == Some("self_test"); + let has_expected = [ + "queueDepth", + "eventsEnqueued", + "eventsFlushed", + "sendFailures", + ] + .iter() + .all(|key| data.get(*key).is_some()); + if !is_self_test || !has_expected { + continue; + } + let ts = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(|text| parse_ts(Some(text))); + if latest_event.is_none() + || ts.is_some_and(|ts| latest_ts.is_none_or(|current| ts > current)) + { + latest_event = Some(event.clone()); + latest_ts = ts; + } + } + let Some(latest_event) = latest_event else { + missing.push(bucket_id); + continue; + }; + let data = latest_event.get("data").unwrap_or(&Value::Null); + let send_failures = int_or_zero(data.get("sendFailures")); + let (previous, delta) = counter_delta( + counter_state.as_deref_mut(), + &format!("endpoint-self-test:{bucket_id}:sendFailures"), + send_failures, + ); + let item = json!({ + "bucket": bucket_id, + "timestamp": latest_event.get("timestamp").cloned().unwrap_or(Value::Null), + "age_seconds": age_seconds(latest_ts, now), + "queueDepth": int_or_zero(data.get("queueDepth")), + "eventsEnqueued": int_or_zero(data.get("eventsEnqueued")), + "eventsFlushed": int_or_zero(data.get("eventsFlushed")), + "sendFailures": send_failures, + "sendFailuresPrevious": previous, + "sendFailuresDelta": delta, + }); + if item["queueDepth"].as_i64().unwrap_or(0) > queue_warn_depth { + warnings.push(json!({"bucket": item["bucket"], "metric": "queueDepth", "value": item["queueDepth"], "threshold": queue_warn_depth})); + } + if send_failure_warn_count > 0 && delta >= send_failure_warn_count { + warnings.push(json!({ + "bucket": item["bucket"], + "metric": "sendFailuresDelta", + "value": delta, + "current": send_failures, + "previous": previous, + "threshold": send_failure_warn_count, + })); + } + latest_self_tests.push(item); + } + let thresholds = + json!({"queueDepth": queue_warn_depth, "sendFailures": send_failure_warn_count}); + if !missing.is_empty() { + report.add( + "endpoint-self-test-metrics", + "warn", + "missing transport metrics in sampled self_test events", + json!({"buckets": missing, "latest_self_tests": latest_self_tests, "thresholds": thresholds}), + ); + } else if !warnings.is_empty() { + report.add( + "endpoint-self-test-metrics", + "warn", + "endpoint transport counters outside thresholds", + json!({"latest_self_tests": latest_self_tests, "warnings": warnings, "thresholds": thresholds}), + ); + } else { + report.add( + "endpoint-self-test-metrics", + "ok", + "self_test transport metrics present", + json!({"latest_self_tests": latest_self_tests, "thresholds": thresholds}), + ); + } +} + +fn check_incident_runtime( + report: &mut HealthReport, + client: &Client, + api_base: &str, + buckets: &BTreeMap, + sample_limit: i64, +) { + let now = Utc::now(); + let prefix = "aw-dlp-incidents_"; + let matched = bucket_keys_with_prefix(buckets, prefix); + if matched.is_empty() { + report.add( + "incident-runtime", + "ok", + "no incident buckets to sample", + json!({"bucket_count": 0}), + ); + return; + } + if sample_limit <= 0 { + let metadata: Vec = matched + .iter() + .map(|bucket_id| { + let ts = latest_bucket_ts( + client, + api_base, + bucket_id, + buckets.get(bucket_id).unwrap_or(&Value::Null), + ); + json!({ + "bucket": bucket_id, + "end": ts.map(|ts| ts.to_rfc3339_opts(SecondsFormat::Secs, true)), + "age_seconds": age_seconds(ts, now), + }) + }) + .collect(); + report.add( + "incident-runtime", + "ok", + "incident event sampling disabled", + json!({"bucket_count": matched.len(), "sample_limit": sample_limit, "metadata": metadata}), + ); + return; + } + + let mut sampled = Vec::new(); + let mut latest_incidents = Vec::new(); + let mut read_failed = Vec::new(); + let mut totals = BTreeMap::from([ + ("sampled_events".to_string(), 0_i64), + ("real_incidents".to_string(), 0_i64), + ("self_tests".to_string(), 0_i64), + ]); + let mut severity_counts: BTreeMap = BTreeMap::new(); + let mut action_counts: BTreeMap = BTreeMap::new(); + let mut rule_counts: HashMap = HashMap::new(); + + for bucket_id in &matched { + let url = format!("{api_base}/buckets/{bucket_id}/events?limit={sample_limit}"); + let events = match http_json(client, &url, 5, 1) { + Ok(Value::Array(items)) => items, + Ok(_) => { + read_failed + .push(json!({"bucket": bucket_id, "error": "events response is not a list"})); + continue; + } + Err(err) => { + read_failed.push(json!({"bucket": bucket_id, "error": err.to_string()})); + continue; + } + }; + let mut bucket_summary = BTreeMap::from([ + ("bucket".to_string(), json!(bucket_id)), + ("sampled_events".to_string(), json!(events.len())), + ("real_incidents".to_string(), json!(0)), + ("self_tests".to_string(), json!(0)), + ]); + *totals.get_mut("sampled_events").unwrap() += events.len() as i64; + for event in &events { + let data = event + .get("data") + .filter(|v| v.is_object()) + .unwrap_or(&Value::Null); + let signal_type = value_str(data.get("signalType")); + let source = value_str(data.get("source")); + let rule_id = value_str(data.get("ruleId")).trim().to_string(); + let rule_id = if rule_id.is_empty() { + value_str(data.get("rule_id")).trim().to_string() + } else { + rule_id + }; + let is_self_test = signal_type == "self_test" + || source == "self-test" + || rule_id.starts_with("selftest-"); + if is_self_test { + *bucket_summary.get_mut("self_tests").unwrap() = + json!(bucket_summary["self_tests"].as_i64().unwrap_or(0) + 1); + *totals.get_mut("self_tests").unwrap() += 1; + continue; + } + let ts = event + .get("timestamp") + .and_then(Value::as_str) + .and_then(|text| parse_ts(Some(text))); + let severity = nonempty_lower(data.get("severity"), "unknown"); + let action = nonempty_lower(data.get("action"), "unknown"); + let rule_key = if rule_id.is_empty() { + "unknown".to_string() + } else { + rule_id.clone() + }; + *severity_counts.entry(severity.clone()).or_insert(0) += 1; + *action_counts.entry(action.clone()).or_insert(0) += 1; + *rule_counts.entry(rule_key).or_insert(0) += 1; + *bucket_summary.get_mut("real_incidents").unwrap() = + json!(bucket_summary["real_incidents"].as_i64().unwrap_or(0) + 1); + *totals.get_mut("real_incidents").unwrap() += 1; + latest_incidents.push(json!({ + "bucket": bucket_id, + "timestamp": event.get("timestamp").cloned().unwrap_or(Value::Null), + "age_seconds": age_seconds(ts, now), + "ruleId": rule_id, + "severity": severity, + "action": action, + "username": value_str(data.get("username")), + "hostname": value_str(data.get("hostname")), + "source": source, + "message_excerpt": text_excerpt(data.get("message"), 120), + })); + } + sampled.push(json!(bucket_summary)); + } + + latest_incidents.sort_by_key(|item| Reverse(value_str(item.get("timestamp")))); + let mut sorted_rules: Vec<(String, i64)> = rule_counts.into_iter().collect(); + sorted_rules.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + let rule_counts_top: BTreeMap = sorted_rules.into_iter().take(10).collect(); + let mut status = "ok"; + let mut summary = format!( + "{} real incidents in sampled events", + totals["real_incidents"] + ); + if !read_failed.is_empty() { + status = "warn"; + summary = format!("{} incident buckets failed to sample", read_failed.len()); + } else if totals["real_incidents"] == 0 { + summary = "no real incidents in sampled events".to_string(); + } + report.add( + "incident-runtime", + status, + summary, + json!({ + "bucket_count": matched.len(), + "sample_limit": sample_limit, + "totals": totals, + "sampled": sampled, + "severity_counts": severity_counts, + "action_counts": action_counts, + "rule_counts": rule_counts_top, + "latest_incidents": latest_incidents.into_iter().take(5).collect::>(), + "read_failed": read_failed, + }), + ); +} + +fn nonempty_lower(value: Option<&Value>, default: &str) -> String { + let text = value_str(value).trim().to_ascii_lowercase(); + if text.is_empty() { + default.to_string() + } else { + text + } +} + +fn check_compliance_reports( + report: &mut HealthReport, + report_dir: &Path, + profiles: &[String], + month: &str, +) { + let mut missing = Vec::new(); + let mut present = Vec::new(); + for profile in profiles { + for suffix in ["html", "json"] { + let path = report_dir.join(format!("{profile}-{month}.{suffix}")); + if path.exists() { + present.push(path.to_string_lossy().to_string()); + } else { + missing.push(path.to_string_lossy().to_string()); + } + } + } + if missing.is_empty() { + report.add( + "compliance-reports", + "ok", + "all expected compliance artifacts exist", + json!({"present": present}), + ); + } else { + report.add( + "compliance-reports", + "fail", + "missing expected compliance report artifacts", + json!({"present": present, "missing": missing}), + ); + } +} + +fn build_report(cli: &Cli, client: &Client) -> HealthReport { + let mut report = HealthReport::default(); + let aw_api_base = format!("{}/api/0", cli.aw_server.trim_end_matches('/')); + let counter_state_path = cli.state_dir.join("dlp-health-check-counters.json"); + let mut counter_state = load_counter_state(&counter_state_path); + + check_http_endpoint( + &mut report, + client, + "http:aw", + &format!("{aw_api_base}/info"), + ); + check_http_endpoint( + &mut report, + client, + "http:policy", + &format!("{}/healthz", cli.policy_server.trim_end_matches('/')), + ); + check_http_endpoint( + &mut report, + client, + "http:cases", + &format!("{}/health", cli.case_server.trim_end_matches('/')), + ); + + for unit in [ + "activitywatch-server", + "aw-dlp-policy-engine.service", + "aw-dlp-case-management.service", + "aw-worktime-api.service", + ] { + check_systemd_unit(&mut report, unit, "service"); + } + for unit in [ + "aw-dlp-report-scheduler.timer", + "aw-dlp-syslog-forwarder.timer", + "aw-dlp-webhook-sender.timer", + "aw-dlp-cef-exporter.timer", + "activitywatch-dlp-aggregator.timer", + "aw-dlp-ioc-refresh.timer", + "aw-worktime-ui-bridge.timer", + ] { + check_systemd_unit(&mut report, unit, "timer"); + } + + match http_json(client, &format!("{aw_api_base}/buckets"), 15, 2) { + Ok(Value::Object(map)) => { + let buckets: BTreeMap = map.into_iter().collect(); + report.add( + "aw:buckets-index", + "ok", + "bucket index loaded", + json!({"total": buckets.len()}), + ); + check_endpoint_signal_buckets( + &mut report, + client, + &aw_api_base, + &buckets, + cli.max_age_seconds, + ); + check_file_operations_buckets( + &mut report, + client, + &aw_api_base, + &buckets, + cli.max_age_seconds, + cli.strict_fileops, + ); + check_file_operations_runtime( + &mut report, + client, + &aw_api_base, + &buckets, + RuntimeThresholds { + sample_limit: cli.fileops_sample_limit, + queue_warn_depth: cli.fileops_queue_warn_depth, + send_failure_warn_count: cli.fileops_send_failure_warn_count, + }, + Some(&mut counter_state), + ); + check_incident_buckets( + &mut report, + client, + &aw_api_base, + &buckets, + cli.max_age_seconds * 24, + ); + check_incident_runtime( + &mut report, + client, + &aw_api_base, + &buckets, + cli.incident_sample_limit, + ); + check_endpoint_self_test_metrics( + &mut report, + client, + &aw_api_base, + &buckets, + cli.endpoint_queue_warn_depth, + cli.endpoint_send_failure_warn_count, + Some(&mut counter_state), + ); + } + Ok(_) => report.add( + "aw:buckets-index", + "fail", + "failed to inspect bucket index: bucket list is not a dict", + json!({}), + ), + Err(err) => report.add( + "aw:buckets-index", + "fail", + format!("failed to inspect bucket index: {err}"), + json!({}), + ), + } + + if let Some(state_error) = save_counter_state(&counter_state_path, &counter_state) { + report.add( + "state:counters", + "warn", + format!("failed to save counter baseline: {state_error}"), + json!({"path": counter_state_path}), + ); + } + let month = Utc::now().format("%Y-%m").to_string(); + let profiles: Vec = cli + .profiles + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(str::to_string) + .collect(); + check_compliance_reports(&mut report, &cli.report_dir, &profiles, &month); + report +} + +fn write_atomic(path: &Path, content: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + let mut tmp = tempfile::NamedTempFile::new_in(path.parent().unwrap_or_else(|| Path::new(".")))?; + tmp.write_all(content.as_bytes())?; + tmp.flush()?; + tmp.persist(path) + .map(|_| ()) + .map_err(|err| anyhow!(err.error)) + .with_context(|| format!("persist {}", path.display())) +} + +fn sort_json_value(value: &Value) -> Value { + match value { + Value::Object(map) => { + let sorted: serde_json::Map = map + .iter() + .map(|(key, value)| (key.clone(), sort_json_value(value))) + .collect(); + Value::Object(sorted) + } + Value::Array(items) => Value::Array(items.iter().map(sort_json_value).collect()), + other => other.clone(), + } +} + +fn cli_arg_present(name: &str) -> bool { + std::env::args_os().skip(1).any(|arg| { + let Some(value) = arg.to_str() else { + return false; + }; + value == name + || value + .strip_prefix(name) + .is_some_and(|rest| rest.starts_with('=')) + }) +} + +fn env_string(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn env_path(name: &str) -> Option { + env_string(name).map(PathBuf::from) +} + +fn env_i64(name: &str, default: i64) -> i64 { + env_string(name) + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + +fn env_bool(name: &str) -> bool { + env_string(name) + .map(|value| { + matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) + .unwrap_or(false) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + let client = Client::builder() + .no_proxy() + .build() + .context("build HTTP client")?; + let report = build_report(&cli, &client); + let payload = report.payload(); + if cli.json { + println!("{}", serde_json::to_string_pretty(&payload)?); + } else { + println!("{}", report.render_text()); + } + std::process::exit(if payload.ok { 0 } else { 1 }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn path_tail_keeps_last_two_parts() { + assert_eq!( + path_tail(Some(&json!("C:\\Users\\USER1\\Downloads\\report.zip")), 2), + "Downloads/report.zip" + ); + } + + #[test] + fn counter_delta_uses_baseline() { + let mut state = json!({"counters": {}}); + assert_eq!(counter_delta(Some(&mut state), "k", 12), (None, 0)); + assert_eq!(counter_delta(Some(&mut state), "k", 12), (Some(12), 0)); + assert_eq!(counter_delta(Some(&mut state), "k", 13), (Some(12), 1)); + assert_eq!(counter_delta(Some(&mut state), "k", 1), (Some(13), 0)); + } + + #[test] + fn text_excerpt_truncates_like_python() { + let text = json!("one two three"); + assert_eq!(text_excerpt(Some(&text), 20), "one two three"); + let long = json!("abcdef"); + assert_eq!(text_excerpt(Some(&long), 4), "abc…"); + } + + #[test] + fn report_ok_ignores_warnings() { + let mut report = HealthReport::default(); + report.add("a", "ok", "ok", json!({})); + report.add("b", "warn", "warn", json!({})); + let payload = report.payload(); + assert!(payload.ok); + assert_eq!(payload.counts.ok, 1); + assert_eq!(payload.counts.warn, 1); + assert_eq!(payload.counts.fail, 0); + } +} diff --git a/adk-rust/crates/dlp-influx-exporter/Cargo.toml b/adk-rust/crates/dlp-influx-exporter/Cargo.toml new file mode 100644 index 0000000..8f2d9c9 --- /dev/null +++ b/adk-rust/crates/dlp-influx-exporter/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "dlp-influx-exporter" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-influx-exporter/src/main.rs b/adk-rust/crates/dlp-influx-exporter/src/main.rs new file mode 100644 index 0000000..aacefc1 --- /dev/null +++ b/adk-rust/crates/dlp-influx-exporter/src/main.rs @@ -0,0 +1,1096 @@ +use std::{thread, time::Duration}; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{DateTime, TimeDelta, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_AW_API_BASE: &str = "http://127.0.0.1:5600/api/0"; +const DEFAULT_CASE_API_BASE: &str = "http://127.0.0.1:5602/api/0/dlp/cases"; +const DEFAULT_INFLUX_ORG: &str = "proxmox"; +const DEFAULT_INFLUX_BUCKET: &str = "aw_metrics"; +const DEFAULT_HOSTS: &str = "SHARKON2025"; + +#[derive(Debug, Parser)] +#[command(about = "AW DLP InfluxDB exporter")] +struct Cli { + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, + + #[arg(long, default_value_t = 30)] + timeout_seconds: u64, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + case_api_base: String, + influx_url: String, + influx_org: String, + influx_bucket: String, + influx_token: String, + influx_enabled: bool, + hosts: Vec, + lookback_days: i64, + event_limit: usize, + case_limit: usize, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + enabled: bool, + dry_run: bool, + hosts: Vec, + lookback_days: i64, + lines: usize, + written: usize, + bucket: String, + error: Option, +} + +#[derive(Debug, Clone)] +enum FieldValue { + Bool(bool), + Int(i64), + String(String), +} + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_bool(name: &str, fallback: bool) -> bool { + match std::env::var(name) + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + { + Some(value) if !value.is_empty() => matches!(value.as_str(), "1" | "true" | "yes" | "on"), + _ => fallback, + } +} + +fn env_int(name: &str, fallback: i64) -> i64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn env_usize(name: &str, fallback: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn load_config() -> Config { + Config { + aw_api_base: env("AW_DLP_AW_API_BASE", DEFAULT_AW_API_BASE) + .trim_end_matches('/') + .to_string(), + case_api_base: env("AW_DLP_CASE_API_BASE", DEFAULT_CASE_API_BASE) + .trim_end_matches('/') + .to_string(), + influx_url: env("AW_DLP_INFLUX_URL", "") + .trim_end_matches('/') + .to_string(), + influx_org: env("AW_DLP_INFLUX_ORG", DEFAULT_INFLUX_ORG), + influx_bucket: env("AW_DLP_INFLUX_BUCKET", DEFAULT_INFLUX_BUCKET), + influx_token: env("AW_DLP_INFLUX_TOKEN", ""), + influx_enabled: env_bool("AW_DLP_INFLUX_ENABLED", false), + hosts: env("AW_DLP_INFLUX_HOSTS", DEFAULT_HOSTS) + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToOwned::to_owned) + .collect(), + lookback_days: env_int("AW_DLP_INFLUX_LOOKBACK_DAYS", 30), + event_limit: env_usize("AW_DLP_INFLUX_EVENT_LIMIT", 2000), + case_limit: env_usize("AW_DLP_CASE_LIMIT", 500), + } +} + +fn utc_now() -> DateTime { + Utc::now() +} + +fn pts(value: Option<&str>) -> DateTime { + let Some(value) = value.filter(|value| !value.is_empty()) else { + return utc_now(); + }; + let normalized = value.replace('Z', "+00:00"); + DateTime::parse_from_rfc3339(&normalized) + .map(|parsed| parsed.with_timezone(&Utc)) + .unwrap_or_else(|_| utc_now()) +} + +fn escape_tag(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace(' ', "\\ ") + .replace(',', "\\,") + .replace('=', "\\=") +} + +fn timestamp_ns(dt: DateTime) -> i64 { + dt.timestamp_nanos_opt() + .unwrap_or_else(|| dt.timestamp() * 1_000_000_000) +} + +fn line( + measurement: &str, + tags: Vec<(&str, String)>, + fields: Vec<(&str, FieldValue)>, + timestamp_ns: i64, +) -> Option { + let mut tag_items: Vec<(&str, String)> = tags + .into_iter() + .filter(|(_key, value)| !value.is_empty()) + .collect(); + tag_items.sort_by(|left, right| left.0.cmp(right.0)); + let tag_part = tag_items + .iter() + .map(|(key, value)| format!("{key}={}", escape_tag(value))) + .collect::>() + .join(","); + + let field_parts = fields + .into_iter() + .map(|(key, value)| match value { + FieldValue::Bool(value) => format!("{key}={}", if value { "true" } else { "false" }), + FieldValue::Int(value) => format!("{key}={value}i"), + FieldValue::String(value) => { + let text = value.replace('\\', "\\\\").replace('"', "\\\""); + format!("{key}=\"{text}\"") + } + }) + .collect::>(); + if field_parts.is_empty() { + return None; + } + if tag_part.is_empty() { + Some(format!( + "{measurement} {} {timestamp_ns}", + field_parts.join(",") + )) + } else { + Some(format!( + "{measurement},{tag_part} {} {timestamp_ns}", + field_parts.join(",") + )) + } +} + +fn get_json(client: &Client, url: &str) -> Result { + let mut last_error = None; + for attempt in 1..=6 { + match client + .get(url) + .header("Accept", "application/json") + .send() + .and_then(|resp| resp.error_for_status()) + .and_then(|resp| resp.json()) + { + Ok(payload) => return Ok(payload), + Err(err) => { + last_error = Some(err); + if attempt < 6 { + thread::sleep(Duration::from_millis(250 * attempt)); + } + } + } + } + Err(anyhow!( + "GET {url}: {}", + last_error + .map(|err| err.to_string()) + .unwrap_or_else(|| "unknown error".to_string()) + )) +} + +fn format_aw_time(value: DateTime) -> String { + value.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +fn fetch_bucket_events( + client: &Client, + config: &Config, + bucket_id: &str, + start: DateTime, + end: DateTime, + limit: usize, +) -> Result> { + let query = format!( + "start={}&end={}&limit={}", + urlencoding::encode(&format_aw_time(start)), + urlencoding::encode(&format_aw_time(end)), + limit + ); + let url = format!( + "{}/buckets/{}/events?{}", + config.aw_api_base, + urlencoding::encode(bucket_id), + query + ); + let payload = get_json(client, &url)?; + Ok(payload.as_array().cloned().unwrap_or_default()) +} + +fn fetch_cases(client: &Client, config: &Config, host: &str) -> Result> { + let url = format!( + "{}?host={}&limit={}", + config.case_api_base, + urlencoding::encode(host), + config.case_limit + ); + let payload = get_json(client, &url)?; + Ok(payload.as_array().cloned().unwrap_or_default()) +} + +fn data_object(item: &Value) -> &serde_json::Map { + match item.get("data").and_then(Value::as_object) { + Some(data) => data, + None => empty_object(), + } +} + +fn empty_object() -> &'static serde_json::Map { + static EMPTY: std::sync::OnceLock> = std::sync::OnceLock::new(); + EMPTY.get_or_init(serde_json::Map::new) +} + +fn s(value: Option<&Value>) -> String { + match value { + Some(Value::Null) | None => String::new(), + Some(Value::String(value)) => value.trim().to_string(), + Some(Value::Bool(value)) => value.to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(other) => other.to_string(), + } +} + +fn first_nonempty(values: &[Option<&Value>], default: &str) -> String { + values + .iter() + .map(|value| s(*value)) + .find(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_string()) +} + +fn int_value(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(value)) => value + .as_i64() + .or_else(|| value.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(value)) => value.parse().unwrap_or(0), + Some(Value::Bool(value)) => i64::from(*value), + _ => 0, + } +} + +fn bool_value(value: Option<&Value>) -> bool { + match value { + Some(Value::Bool(value)) => *value, + Some(Value::Number(value)) => value.as_i64().unwrap_or(0) != 0, + Some(Value::String(value)) => matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ), + _ => false, + } +} + +fn event_id(item: &Value, fallback: String) -> String { + first_nonempty(&[item.get("id")], &fallback) +} + +#[derive(Debug)] +struct NormalizedIncident { + host: String, + signal_type: String, + username: String, + severity: String, + action: String, + message: String, + rule_id: String, + source: String, + document_name: String, + printer_name: String, + incident_status: String, + incident_verdict: String, + regex_matches: usize, + dictionary_matches: usize, + ocr_requested: bool, +} + +fn normalize_incident(event: &Value, default_host: &str) -> NormalizedIncident { + let empty = empty_object(); + let data = event + .get("data") + .and_then(Value::as_object) + .unwrap_or(empty); + let source_event = data + .get("sourceEvent") + .and_then(Value::as_object) + .unwrap_or(empty); + let source_data = source_event + .get("data") + .and_then(Value::as_object) + .unwrap_or(empty); + let nested = data + .get("incident") + .and_then(Value::as_object) + .unwrap_or(empty); + + NormalizedIncident { + host: first_nonempty( + &[ + data.get("hostname"), + data.get("host"), + source_data.get("hostname"), + ], + default_host, + ), + signal_type: first_nonempty( + &[data.get("signalType"), source_data.get("signalType")], + "unknown", + ), + username: first_nonempty( + &[ + data.get("username"), + source_data.get("username"), + source_data.get("owner"), + source_data.get("host"), + ], + "unknown", + ), + severity: first_nonempty(&[data.get("severity"), nested.get("severity")], "unknown"), + action: first_nonempty(&[data.get("action"), nested.get("verdict")], "incident"), + message: first_nonempty( + &[ + data.get("message"), + source_data.get("documentName"), + source_data.get("documentNameOriginal"), + nested.get("comment"), + ], + "", + ), + rule_id: first_nonempty(&[data.get("ruleId")], ""), + source: first_nonempty( + &[ + data.get("source"), + source_data.get("source"), + data.get("sourceBucket"), + ], + "", + ), + document_name: first_nonempty( + &[ + source_data.get("documentName"), + source_data.get("documentNameOriginal"), + ], + "", + ), + printer_name: first_nonempty(&[source_data.get("printerName")], ""), + incident_status: first_nonempty(&[nested.get("status")], ""), + incident_verdict: first_nonempty(&[nested.get("verdict")], ""), + regex_matches: data + .get("regexMatches") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0), + dictionary_matches: data + .get("dictionaryMatches") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0), + ocr_requested: bool_value(data.get("ocrRequested")), + } +} + +fn build_endpoint_lines(host: &str, events: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in events { + let data = data_object(item); + let signal_type = first_nonempty(&[data.get("signalType")], "unknown"); + let ts = timestamp_ns(pts(item.get("timestamp").and_then(Value::as_str))); + let id = event_id(item, format!("{signal_type}-{ts}")); + let username = first_nonempty(&[data.get("username"), data.get("owner")], "unknown"); + if signal_type == "self_test" { + if let Some(line) = line( + "aw_dlp_endpoint_self_test", + vec![ + ("host", first_nonempty(&[data.get("hostname")], host)), + ("event_id", id), + ("username", username), + ( + "policy_mode", + first_nonempty(&[data.get("policyMode")], "unknown"), + ), + ( + "policy_source", + first_nonempty(&[data.get("policySource")], "unknown"), + ), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "queue_depth", + FieldValue::Int(int_value(data.get("queueDepth"))), + ), + ( + "events_enqueued", + FieldValue::Int(int_value(data.get("eventsEnqueued"))), + ), + ( + "events_flushed", + FieldValue::Int(int_value(data.get("eventsFlushed"))), + ), + ( + "send_failures", + FieldValue::Int(int_value(data.get("sendFailures"))), + ), + ( + "policy_enabled", + FieldValue::Bool(bool_value(data.get("policyEnabled"))), + ), + ], + ts, + ) { + lines.push(line); + } + continue; + } + if let Some(line) = line( + "aw_dlp_signal", + vec![ + ("host", first_nonempty(&[data.get("hostname")], host)), + ("event_id", id), + ("signal_type", signal_type), + ("username", username), + ("source", first_nonempty(&[data.get("source")], "unknown")), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "document_name", + FieldValue::String(first_nonempty( + &[data.get("documentName"), data.get("documentNameOriginal")], + "", + )), + ), + ( + "printer_name", + FieldValue::String(first_nonempty(&[data.get("printerName")], "")), + ), + ( + "owner", + FieldValue::String(first_nonempty(&[data.get("owner")], "")), + ), + ( + "session_id", + FieldValue::Int(int_value(data.get("sessionId"))), + ), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_incident_lines(host: &str, events: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in events { + let normalized = normalize_incident(item, host); + let ts = timestamp_ns(pts(item.get("timestamp").and_then(Value::as_str))); + let id = event_id(item, format!("incident-{ts}")); + if let Some(line) = line( + "aw_dlp_incident", + vec![ + ("host", normalized.host), + ("event_id", id), + ("signal_type", normalized.signal_type), + ("severity", normalized.severity), + ("action", normalized.action), + ("username", normalized.username), + ("source", normalized.source), + ], + vec![ + ("count", FieldValue::Int(1)), + ("message", FieldValue::String(normalized.message)), + ("rule_id", FieldValue::String(normalized.rule_id)), + ( + "document_name", + FieldValue::String(normalized.document_name), + ), + ("printer_name", FieldValue::String(normalized.printer_name)), + ( + "incident_status", + FieldValue::String(normalized.incident_status), + ), + ( + "incident_verdict", + FieldValue::String(normalized.incident_verdict), + ), + ( + "regex_matches", + FieldValue::Int(normalized.regex_matches as i64), + ), + ( + "dictionary_matches", + FieldValue::Int(normalized.dictionary_matches as i64), + ), + ("ocr_requested", FieldValue::Bool(normalized.ocr_requested)), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_review_lines(host: &str, events: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in events { + let empty = empty_object(); + let data = item.get("data").and_then(Value::as_object).unwrap_or(empty); + let review = data + .get("review") + .and_then(Value::as_object) + .unwrap_or(empty); + let source_data = data + .get("sourceEvent") + .and_then(Value::as_object) + .and_then(|value| value.get("data")) + .and_then(Value::as_object) + .unwrap_or(empty); + let ts = timestamp_ns(pts(item.get("timestamp").and_then(Value::as_str))); + let review_id = first_nonempty(&[review.get("reviewId")], &format!("review-{ts}")); + if let Some(line) = line( + "aw_dlp_review", + vec![ + ( + "host", + first_nonempty(&[data.get("host"), source_data.get("hostname")], host), + ), + ("review_id", review_id), + ( + "verdict", + first_nonempty(&[review.get("verdict")], "unknown"), + ), + ( + "signal_type", + first_nonempty(&[source_data.get("signalType")], "unknown"), + ), + ( + "username", + first_nonempty( + &[source_data.get("username"), source_data.get("owner")], + "unknown", + ), + ), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "archived", + FieldValue::Bool(bool_value(review.get("archived"))), + ), + ( + "comment", + FieldValue::String(first_nonempty(&[review.get("comment")], "")), + ), + ( + "category", + FieldValue::String(first_nonempty(&[review.get("category")], "")), + ), + ( + "document_name", + FieldValue::String(first_nonempty( + &[ + source_data.get("documentName"), + source_data.get("documentNameOriginal"), + ], + "", + )), + ), + ( + "printer_name", + FieldValue::String(first_nonempty(&[source_data.get("printerName")], "")), + ), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_rule_lines(host: &str, events: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in events { + let empty = empty_object(); + let data = item.get("data").and_then(Value::as_object).unwrap_or(empty); + let match_data = data + .get("match") + .and_then(Value::as_object) + .unwrap_or(empty); + let ts = timestamp_ns(pts(item.get("timestamp").and_then(Value::as_str))); + let rule_id = first_nonempty(&[data.get("ruleId")], &format!("rule-{ts}")); + let enabled = data + .get("enabled") + .is_none_or(|value| bool_value(Some(value))); + if let Some(line) = line( + "aw_dlp_rule", + vec![ + ( + "host", + first_nonempty(&[data.get("host"), match_data.get("hostname")], host), + ), + ("rule_id", rule_id), + ("action", first_nonempty(&[data.get("action")], "unknown")), + ( + "signal_type", + first_nonempty(&[match_data.get("signalType")], "unknown"), + ), + ( + "username", + first_nonempty( + &[match_data.get("username"), match_data.get("owner")], + "unknown", + ), + ), + ( + "enabled", + if enabled { + "true".to_string() + } else { + "false".to_string() + }, + ), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "category", + FieldValue::String(first_nonempty(&[data.get("category")], "")), + ), + ( + "comment", + FieldValue::String(first_nonempty(&[data.get("comment")], "")), + ), + ( + "document_name", + FieldValue::String(first_nonempty(&[match_data.get("documentName")], "")), + ), + ( + "printer_name", + FieldValue::String(first_nonempty(&[match_data.get("printerName")], "")), + ), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_fileops_lines(host: &str, events: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in events { + let data = data_object(item); + let signal_type = first_nonempty(&[data.get("signalType")], "unknown"); + if signal_type != "collector_health" { + continue; + } + let ts = timestamp_ns(pts(item.get("timestamp").and_then(Value::as_str))); + let id = event_id(item, format!("fileops-{ts}")); + if let Some(line) = line( + "aw_dlp_fileops_health", + vec![ + ("host", first_nonempty(&[data.get("hostname")], host)), + ("event_id", id), + ( + "username", + first_nonempty(&[data.get("username")], "unknown"), + ), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "queue_depth", + FieldValue::Int(int_value(data.get("queueDepth"))), + ), + ( + "events_enqueued", + FieldValue::Int(int_value(data.get("eventsEnqueued"))), + ), + ( + "events_flushed", + FieldValue::Int(int_value(data.get("eventsFlushed"))), + ), + ( + "send_failures", + FieldValue::Int(int_value(data.get("sendFailures"))), + ), + ( + "session_id", + FieldValue::Int(int_value(data.get("sessionId"))), + ), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_case_lines(host: &str, cases: &[Value]) -> Vec { + let mut lines = Vec::new(); + for item in cases { + let empty = empty_object(); + let object = item.as_object().unwrap_or(empty); + let evidence = object + .get("evidence") + .and_then(Value::as_object) + .unwrap_or(empty); + let evidence_items = evidence + .get("items") + .and_then(Value::as_array) + .map(Vec::len) + .unwrap_or(0); + let ts = timestamp_ns(pts(object + .get("updated_at") + .or_else(|| object.get("created_at")) + .and_then(Value::as_str))); + if let Some(line) = line( + "aw_dlp_case", + vec![ + ("host", first_nonempty(&[object.get("host")], host)), + ("case_id", first_nonempty(&[object.get("id")], "")), + ("status", first_nonempty(&[object.get("status")], "unknown")), + ( + "severity", + first_nonempty(&[object.get("severity")], "unknown"), + ), + ( + "assignee", + first_nonempty(&[object.get("assignee")], "unassigned"), + ), + ], + vec![ + ("count", FieldValue::Int(1)), + ( + "title", + FieldValue::String(first_nonempty(&[object.get("title")], "")), + ), + ( + "incident_id", + FieldValue::String(first_nonempty(&[object.get("incident_id")], "")), + ), + ( + "has_forensics", + FieldValue::Bool(!matches!(object.get("forensics"), None | Some(Value::Null))), + ), + ("evidence_items", FieldValue::Int(evidence_items as i64)), + ( + "chain_length", + FieldValue::Int(int_value(evidence.get("chain_length"))), + ), + ], + ts, + ) { + lines.push(line); + } + } + lines +} + +fn build_lines_for_host( + client: &Client, + config: &Config, + host: &str, + start: DateTime, + end: DateTime, +) -> Result> { + let mut lines = Vec::new(); + lines.extend(build_endpoint_lines( + host, + &fetch_bucket_events( + client, + config, + &format!("aw-dlp-endpoint-signals_{host}"), + start, + end, + config.event_limit, + )?, + )); + lines.extend(build_incident_lines( + host, + &fetch_bucket_events( + client, + config, + &format!("aw-dlp-incidents_{host}"), + start, + end, + config.event_limit, + )?, + )); + lines.extend(build_review_lines( + host, + &fetch_bucket_events( + client, + config, + &format!("aw-dlp-review_{host}"), + start, + end, + config.event_limit, + )?, + )); + lines.extend(build_rule_lines( + host, + &fetch_bucket_events( + client, + config, + &format!("aw-dlp-rules_{host}"), + start, + end, + config.event_limit, + )?, + )); + lines.extend(build_fileops_lines( + host, + &fetch_bucket_events( + client, + config, + &format!("aw-file-operations_{host}"), + start, + end, + config.event_limit, + )?, + )); + lines.extend(build_case_lines(host, &fetch_cases(client, config, host)?)); + Ok(lines) +} + +fn write_lines(client: &Client, config: &Config, lines: &[String]) -> Result { + if lines.is_empty() { + return Ok(0); + } + if config.influx_url.is_empty() || config.influx_token.is_empty() { + bail!("InfluxDB destination is not configured"); + } + let url = format!( + "{}/api/v2/write?org={}&bucket={}&precision=ns", + config.influx_url, + urlencoding::encode(&config.influx_org), + urlencoding::encode(&config.influx_bucket) + ); + let payload = format!("{}\n", lines.join("\n")); + client + .post(url) + .header("Authorization", format!("Token {}", config.influx_token)) + .header("Content-Type", "text/plain; charset=utf-8") + .body(payload) + .send() + .and_then(|resp| resp.error_for_status()) + .context("InfluxDB write failed")?; + Ok(lines.len()) +} + +fn run(cli: &Cli) -> Result { + let config = load_config(); + if !config.influx_enabled && !cli.dry_run { + return Ok(RunSummary { + ok: true, + enabled: false, + dry_run: false, + hosts: config.hosts, + lookback_days: config.lookback_days, + lines: 0, + written: 0, + bucket: config.influx_bucket, + error: None, + }); + } + let client = Client::builder() + .timeout(Duration::from_secs(cli.timeout_seconds)) + .no_proxy() + .build() + .context("build HTTP client")?; + let end = utc_now(); + let start = end - TimeDelta::days(config.lookback_days); + let mut lines = Vec::new(); + for host in &config.hosts { + lines.extend(build_lines_for_host(&client, &config, host, start, end)?); + } + let written = if cli.dry_run { + 0 + } else { + write_lines(&client, &config, &lines)? + }; + Ok(RunSummary { + ok: true, + enabled: config.influx_enabled, + dry_run: cli.dry_run, + hosts: config.hosts, + lookback_days: config.lookback_days, + lines: lines.len(), + written, + bucket: config.influx_bucket, + error: None, + }) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + match run(&cli) { + Ok(summary) => { + if cli.json || cli.dry_run { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if !summary.enabled { + eprintln!("[aw-dlp-influx-exporter] disabled by AW_DLP_INFLUX_ENABLED"); + } else { + eprintln!( + "[aw-dlp-influx-exporter] wrote {} points to {}", + summary.written, summary.bucket + ); + } + Ok(()) + } + Err(err) => { + if cli.json { + println!( + "{}", + serde_json::to_string_pretty(&RunSummary { + ok: false, + enabled: false, + dry_run: cli.dry_run, + hosts: Vec::new(), + lookback_days: 0, + lines: 0, + written: 0, + bucket: String::new(), + error: Some(err.to_string()), + })? + ); + } + Err(err) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn endpoint_lines_emit_self_test_and_signal() { + let events = vec![ + json!({ + "id": 10, + "timestamp": "2026-05-15T10:00:00Z", + "data": { + "hostname": "SHARKON2025", + "username": "Администратор", + "signalType": "self_test", + "policyMode": "server", + "policySource": "local-fallback", + "queueDepth": 2, + "eventsEnqueued": 100, + "eventsFlushed": 99, + "sendFailures": 1, + "policyEnabled": true + } + }), + json!({ + "id": 11, + "timestamp": "2026-05-15T10:01:00Z", + "data": { + "hostname": "SHARKON2025", + "username": "Администратор", + "signalType": "print_job", + "source": "endpoint-signals-phase2", + "documentName": "Документ.docx", + "printerName": "HP LaserJet" + } + }), + ]; + let lines = build_endpoint_lines("SHARKON2025", &events); + assert!( + lines + .iter() + .any(|line| line.starts_with("aw_dlp_endpoint_self_test,")) + ); + assert!(lines.iter().any(|line| line.starts_with("aw_dlp_signal,"))); + } + + #[test] + fn normalize_incident_handles_nested_source_event() { + let item = json!({ + "timestamp": "2026-05-15T10:02:00Z", + "data": { + "host": "SHARKON2025", + "incident": {"status": "open", "verdict": "incident"}, + "sourceBucket": "aw-dlp-endpoint-signals_SHARKON2025", + "sourceEvent": { + "data": { + "signalType": "print_job", + "hostname": "SHARKON2025", + "username": "Администратор", + "documentName": "Письмо", + "printerName": "HP", + "source": "endpoint-signals-phase2" + } + } + } + }); + let normalized = normalize_incident(&item, "SHARKON2025"); + assert_eq!(normalized.signal_type, "print_job"); + assert_eq!(normalized.username, "Администратор"); + assert_eq!(normalized.action, "incident"); + assert_eq!(normalized.incident_status, "open"); + } + + #[test] + fn case_lines_emit_case_state() { + let cases = vec![json!({ + "id": 28, + "host": "SHARKON2025", + "status": "open", + "severity": "medium", + "assignee": null, + "title": "DLP print_job · Администратор", + "incident_id": "case-1", + "evidence": {"items": [1], "chain_length": 1}, + "forensics": null, + "updated_at": "2026-05-15T10:03:00+00:00" + })]; + let lines = build_case_lines("SHARKON2025", &cases); + assert_eq!(lines.len(), 1); + assert!(lines[0].starts_with("aw_dlp_case,")); + } + + #[test] + fn timestamp_parser_accepts_zulu() { + assert_eq!(pts(Some("2026-05-15T10:00:00Z")).timestamp(), 1_778_839_200); + } +} diff --git a/adk-rust/crates/dlp-policy-engine/Cargo.toml b/adk-rust/crates/dlp-policy-engine/Cargo.toml new file mode 100644 index 0000000..e3241ed --- /dev/null +++ b/adk-rust/crates/dlp-policy-engine/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "dlp-policy-engine" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +rusqlite.workspace = true +serde_json.workspace = true +sha2.workspace = true +tiny_http.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/dlp-policy-engine/src/main.rs b/adk-rust/crates/dlp-policy-engine/src/main.rs new file mode 100644 index 0000000..c442b88 --- /dev/null +++ b/adk-rust/crates/dlp-policy-engine/src/main.rs @@ -0,0 +1,1089 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use rusqlite::{Connection, OptionalExtension, params}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; +use urlencoding::decode; + +const APP_NAME: &str = "aw-dlp-policy-engine"; +const APP_VERSION: &str = "0.1.0"; + +#[derive(Debug, Parser)] +#[command(about = "AW DLP Policy Engine")] +struct Cli { + #[arg(long, default_value = "127.0.0.1")] + bind_host: String, + + #[arg(long, default_value_t = 5601)] + port: u16, + + #[arg( + long, + default_value = "/var/lib/activitywatch/dlp-policy-engine.sqlite" + )] + db_path: PathBuf, +} + +#[derive(Clone)] +struct AppState { + db_path: PathBuf, + agents: Arc>>, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = apply_env(Cli::parse()); + let storage = PolicyStorage::new(cli.db_path.clone())?; + storage.init_schema()?; + let state = AppState { + db_path: cli.db_path, + agents: Arc::new(Mutex::new(HashMap::new())), + }; + let address = format!("{}:{}", cli.bind_host, cli.port); + let server = Server::http(&address).map_err(|err| anyhow!("bind {address}: {err}"))?; + eprintln!("{APP_NAME} {APP_VERSION} listening on {address}"); + for request in server.incoming_requests() { + if let Err(err) = handle_request(&state, request) { + eprintln!("request error: {err:#}"); + } + } + Ok(()) +} + +fn apply_env(mut cli: Cli) -> Cli { + if !cli_arg_present("--bind-host") { + if let Ok(value) = std::env::var("AW_DLP_POLICY_ENGINE_BIND_HOST") { + if !value.is_empty() { + cli.bind_host = value; + } + } + } + if !cli_arg_present("--port") { + if let Ok(value) = std::env::var("AW_DLP_POLICY_ENGINE_PORT") { + if let Ok(port) = value.parse() { + cli.port = port; + } + } + } + if !cli_arg_present("--db-path") { + if let Ok(value) = std::env::var("AW_DLP_POLICY_ENGINE_DB_PATH") { + if !value.is_empty() { + cli.db_path = PathBuf::from(value); + } + } + } + cli +} + +fn cli_arg_present(flag: &str) -> bool { + std::env::args().any(|arg| arg == flag || arg.starts_with(&format!("{flag}="))) +} + +fn handle_request(state: &AppState, mut request: Request) -> Result<()> { + let method = request.method().clone(); + let url = request.url().to_string(); + let (path, query) = split_url(&url); + let segments = path_segments(&path); + let storage = PolicyStorage::new(state.db_path.clone())?; + + let response = match (method, segments.as_slice()) { + (Method::Get, ["healthz"]) => json_response( + StatusCode(200), + json!({ + "status": "ok", + "service": APP_NAME, + "db_path": state.db_path.to_string_lossy(), + "db_exists": state.db_path.exists().to_string(), + }), + ), + (Method::Get, ["api", "0", "dlp", "policies"]) => { + json_response(StatusCode(200), json!({"items": storage.list_policies()?})) + } + (Method::Post, ["api", "0", "dlp", "policies"]) => { + let payload = read_json_body(&mut request)?; + match storage.create_policy(&payload) { + Ok(item) => json_response(StatusCode(201), json!({"item": item})), + Err(err) => json_response(StatusCode(400), json!({"detail": err.to_string()})), + } + } + (Method::Get, ["api", "0", "dlp", "policies", "active"]) => { + match storage.get_active_policy()? { + Some(item) => json_response(StatusCode(200), build_policy_bundle(&item)), + None => json_response( + StatusCode(404), + json!({"detail": "no active policy configured"}), + ), + } + } + (Method::Get, ["api", "0", "dlp", "policies", "active", "version"]) => { + match storage.get_active_policy()? { + Some(item) => json_response( + StatusCode(200), + json!({ + "active": true, + "policyId": item.get("id").cloned().unwrap_or(Value::Null), + "version": item.get("current_version").cloned().unwrap_or(Value::Null), + "checksum": item.get("checksum").cloned().unwrap_or(Value::Null), + "updatedAtUtc": item.get("updated_at").cloned().unwrap_or(Value::Null), + }), + ), + None => json_response( + StatusCode(404), + json!({"detail": "no active policy configured"}), + ), + } + } + ( + Method::Post, + [ + "api", + "0", + "dlp", + "policies", + "agents", + agent_id, + "heartbeat", + ], + ) => { + let payload = read_json_body(&mut request)?; + let agent = agent_heartbeat(agent_id, &payload); + state + .agents + .lock() + .map_err(|_| anyhow!("agent state lock poisoned"))? + .insert(agent_id.to_string(), agent.clone()); + json_response(StatusCode(200), json!({"ok": true, "agent": agent})) + } + (Method::Get, ["api", "0", "dlp", "policies", "agents", agent_id, "desired"]) => { + match storage.get_active_policy()? { + Some(item) => { + let current = state + .agents + .lock() + .map_err(|_| anyhow!("agent state lock poisoned"))? + .get(*agent_id) + .cloned() + .unwrap_or_else(|| json!({})); + json_response(StatusCode(200), desired_policy(agent_id, ¤t, &item)) + } + None => json_response( + StatusCode(404), + json!({"detail": "no active policy configured"}), + ), + } + } + (Method::Post, ["api", "0", "dlp", "policies", "rollback"]) => { + let payload = read_json_body(&mut request)?; + match storage.rollback_active_policy(actor_from(&payload))? { + Some(item) => json_response(StatusCode(200), json!({"item": item})), + None => json_response( + StatusCode(404), + json!({"detail": "no active policy configured"}), + ), + } + } + (Method::Get, ["api", "0", "dlp", "policies", "audit"]) => { + let limit = query_limit(&query); + json_response( + StatusCode(200), + json!({"items": storage.list_audit(None, limit)?}), + ) + } + (Method::Get, ["api", "0", "dlp", "policies", policy_id]) => { + match parse_id(policy_id).and_then(|id| storage.get_policy(id))? { + Some(item) => json_response(StatusCode(200), json!({"item": item})), + None => json_response(StatusCode(404), json!({"detail": "policy not found"})), + } + } + (Method::Put, ["api", "0", "dlp", "policies", policy_id]) => { + let payload = read_json_body(&mut request)?; + let policy_id = parse_id(policy_id)?; + match storage.update_policy(policy_id, &payload) { + Ok(Some(item)) => json_response(StatusCode(200), json!({"item": item})), + Ok(None) => json_response(StatusCode(404), json!({"detail": "policy not found"})), + Err(err) => json_response(StatusCode(400), json!({"detail": err.to_string()})), + } + } + (Method::Post, ["api", "0", "dlp", "policies", policy_id, "activate"]) => { + let payload = read_json_body(&mut request)?; + let policy_id = parse_id(policy_id)?; + match storage.activate_policy(policy_id, actor_from(&payload)) { + Ok(Some(item)) => json_response(StatusCode(200), json!({"item": item})), + Ok(None) => json_response(StatusCode(404), json!({"detail": "policy not found"})), + Err(err) => json_response(StatusCode(400), json!({"detail": err.to_string()})), + } + } + (Method::Post, ["api", "0", "dlp", "policies", policy_id, action]) + if matches!(*action, "submit" | "approve" | "draft") => + { + let payload = read_json_body(&mut request)?; + let policy_id = parse_id(policy_id)?; + let status = match *action { + "submit" => "pending_approval", + "approve" => "approved", + "draft" => "draft", + _ => unreachable!(), + }; + match storage.set_policy_status( + policy_id, + status, + actor_from(&payload), + optional_string(payload.get("comment")), + ) { + Ok(Some(item)) => json_response(StatusCode(200), json!({"item": item})), + Ok(None) => json_response(StatusCode(404), json!({"detail": "policy not found"})), + Err(err) => json_response(StatusCode(400), json!({"detail": err.to_string()})), + } + } + (Method::Delete, ["api", "0", "dlp", "policies", policy_id]) => { + let policy_id = parse_id(policy_id)?; + match storage.delete_policy(policy_id) { + Ok(true) => json_response(StatusCode(200), json!({"deleted": true})), + Ok(false) => json_response(StatusCode(404), json!({"detail": "policy not found"})), + Err(err) => json_response(StatusCode(409), json!({"detail": err.to_string()})), + } + } + (Method::Get, ["api", "0", "dlp", "policies", policy_id, "audit"]) => { + let policy_id = parse_id(policy_id)?; + if storage.get_policy(policy_id)?.is_none() { + json_response(StatusCode(404), json!({"detail": "policy not found"})) + } else { + let limit = query_limit(&query); + json_response( + StatusCode(200), + json!({"items": storage.list_audit(Some(policy_id), limit)?}), + ) + } + } + _ => json_response(StatusCode(404), json!({"detail": "not found"})), + }; + request.respond(response).context("send HTTP response") +} + +struct PolicyStorage { + db_path: PathBuf, +} + +impl PolicyStorage { + fn new(db_path: PathBuf) -> Result { + if let Some(parent) = db_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create db parent {}", parent.display()))?; + } + Ok(Self { db_path }) + } + + fn connect(&self) -> Result { + let conn = Connection::open(&self.db_path) + .with_context(|| format!("open sqlite {}", self.db_path.display()))?; + conn.pragma_update(None, "journal_mode", "WAL")?; + Ok(conn) + } + + fn init_schema(&self) -> Result<()> { + let conn = self.connect()?; + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS policies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT, + status TEXT NOT NULL DEFAULT 'draft', + is_active INTEGER NOT NULL DEFAULT 0, + current_version INTEGER NOT NULL DEFAULT 1, + checksum TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS policy_versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + policy_id INTEGER NOT NULL, + version INTEGER NOT NULL, + policy_json TEXT NOT NULL, + checksum TEXT NOT NULL, + created_at TEXT NOT NULL, + created_by TEXT, + rollback_of_version INTEGER, + FOREIGN KEY(policy_id) REFERENCES policies(id), + UNIQUE(policy_id, version) + ); + + CREATE TABLE IF NOT EXISTS policy_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + policy_id INTEGER, + action TEXT NOT NULL, + actor TEXT, + comment TEXT, + details_json TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY(policy_id) REFERENCES policies(id) + ); + + CREATE INDEX IF NOT EXISTS idx_policies_active ON policies(is_active); + CREATE INDEX IF NOT EXISTS idx_policy_versions_policy ON policy_versions(policy_id, version DESC); + CREATE INDEX IF NOT EXISTS idx_policy_audit_policy ON policy_audit(policy_id, id DESC); + "#, + )?; + let has_status = { + let mut stmt = conn.prepare("PRAGMA table_info(policies)")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(1))?; + let mut found = false; + for row in rows { + if row? == "status" { + found = true; + break; + } + } + found + }; + if !has_status { + conn.execute( + "ALTER TABLE policies ADD COLUMN status TEXT NOT NULL DEFAULT 'draft'", + [], + )?; + } + Ok(()) + } + + fn list_policies(&self) -> Result> { + let conn = self.connect()?; + let mut stmt = conn.prepare( + r#" + SELECT id, name, description, status, is_active, current_version, checksum, created_at, updated_at + FROM policies + ORDER BY is_active DESC, updated_at DESC, id DESC + "#, + )?; + let rows = stmt.query_map([], policy_summary_from_row)?; + rows.collect::, _>>() + .map_err(Into::into) + } + + fn get_policy(&self, policy_id: i64) -> Result> { + let conn = self.connect()?; + let policy = conn + .query_row( + r#" + SELECT id, name, description, status, is_active, current_version, checksum, created_at, updated_at + FROM policies + WHERE id = ? + "#, + [policy_id], + policy_summary_from_row, + ) + .optional()?; + let Some(mut policy) = policy else { + return Ok(None); + }; + let current_version = policy + .get("current_version") + .and_then(Value::as_i64) + .ok_or_else(|| anyhow!("policy current_version missing"))?; + let version = conn + .query_row( + r#" + SELECT version, policy_json, checksum, created_at, created_by + FROM policy_versions + WHERE policy_id = ? AND version = ? + "#, + params![policy_id, current_version], + |row| { + Ok(( + row.get::<_, String>(1)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + )) + }, + ) + .optional()?; + let Some((policy_json, version_created_at, version_created_by)) = version else { + return Ok(None); + }; + let policy_doc: Value = serde_json::from_str(&policy_json).context("decode policy_json")?; + let obj = policy + .as_object_mut() + .ok_or_else(|| anyhow!("policy row is not object"))?; + obj.insert("policy".to_string(), policy_doc); + obj.insert( + "version_created_at".to_string(), + Value::String(version_created_at), + ); + obj.insert( + "version_created_by".to_string(), + version_created_by.map(Value::String).unwrap_or(Value::Null), + ); + Ok(Some(policy)) + } + + fn get_active_policy(&self) -> Result> { + let conn = self.connect()?; + let id = conn + .query_row( + "SELECT id FROM policies WHERE is_active = 1 ORDER BY updated_at DESC LIMIT 1", + [], + |row| row.get::<_, i64>(0), + ) + .optional()?; + match id { + Some(id) => self.get_policy(id), + None => Ok(None), + } + } + + fn create_policy(&self, payload: &Value) -> Result { + let name = required_string(payload.get("name"), "name")?; + validate_name(&name)?; + let description = optional_string(payload.get("description")); + validate_description(description.as_deref())?; + let policy = payload + .get("policy") + .cloned() + .ok_or_else(|| anyhow!("policy is required"))?; + let activate = payload + .get("activate") + .and_then(Value::as_bool) + .unwrap_or(false); + let actor = actor_from(payload); + let checksum = checksum_policy(&policy); + let policy_json = canonical_json(&policy)?; + let now = utc_now(); + let status = if activate { "deployed" } else { "draft" }; + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + if activate { + tx.execute("UPDATE policies SET is_active = 0", [])?; + } + tx.execute( + r#" + INSERT INTO policies(name, description, status, is_active, current_version, checksum, created_at, updated_at) + VALUES(?, ?, ?, ?, 1, ?, ?, ?) + "#, + params![ + name, + description, + status, + if activate { 1 } else { 0 }, + checksum, + now, + now + ], + )?; + let policy_id = tx.last_insert_rowid(); + tx.execute( + r#" + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, 1, ?, ?, ?, ?, NULL) + "#, + params![policy_id, policy_json, checksum, now, actor], + )?; + audit( + &tx, + Some(policy_id), + "create", + actor.as_deref(), + None, + Some(json!({"activate": activate, "status": status})), + )?; + tx.commit()?; + self.get_policy(policy_id)? + .ok_or_else(|| anyhow!("created policy not found")) + } + + fn update_policy(&self, policy_id: i64, payload: &Value) -> Result> { + let current = match self.get_policy(policy_id)? { + Some(item) => item, + None => return Ok(None), + }; + let name = match payload.get("name") { + Some(Value::Null) | None => current + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + Some(value) => { + let name = required_string(Some(value), "name")?; + validate_name(&name)?; + name + } + }; + let description = match payload.get("description") { + Some(value) => optional_string(Some(value)), + None => optional_string(current.get("description")), + }; + validate_description(description.as_deref())?; + let activate = payload + .get("activate") + .and_then(Value::as_bool) + .unwrap_or(false); + let actor = actor_from(payload); + let mut new_version = current + .get("current_version") + .and_then(Value::as_i64) + .ok_or_else(|| anyhow!("current_version missing"))?; + let mut new_checksum = required_string(current.get("checksum"), "checksum")?; + let mut new_status = current + .get("status") + .and_then(Value::as_str) + .unwrap_or("draft") + .to_string(); + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + if let Some(policy) = payload.get("policy").filter(|value| !value.is_null()) { + new_version += 1; + new_checksum = checksum_policy(policy); + new_status = "draft".to_string(); + tx.execute( + r#" + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, ?, ?, ?, ?, ?, NULL) + "#, + params![ + policy_id, + new_version, + canonical_json(policy)?, + new_checksum, + utc_now(), + actor + ], + )?; + } + if activate { + tx.execute("UPDATE policies SET is_active = 0", [])?; + new_status = "deployed".to_string(); + } + tx.execute( + r#" + UPDATE policies + SET name = ?, description = ?, status = ?, is_active = ?, current_version = ?, checksum = ?, updated_at = ? + WHERE id = ? + "#, + params![ + name, + description, + new_status, + if activate { + 1 + } else { + current + .get("is_active") + .and_then(Value::as_i64) + .unwrap_or(0) + }, + new_version, + new_checksum, + utc_now(), + policy_id, + ], + )?; + audit( + &tx, + Some(policy_id), + "update", + actor.as_deref(), + None, + Some(json!({"activate": activate, "status": new_status})), + )?; + tx.commit()?; + self.get_policy(policy_id) + } + + fn activate_policy(&self, policy_id: i64, actor: Option) -> Result> { + let current = match self.get_policy(policy_id)? { + Some(item) => item, + None => return Ok(None), + }; + if current.get("status").and_then(Value::as_str) != Some("approved") { + bail!("policy must be approved before deploy"); + } + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + tx.execute("UPDATE policies SET is_active = 0", [])?; + tx.execute( + "UPDATE policies SET status = 'deployed', is_active = 1, updated_at = ? WHERE id = ?", + params![utc_now(), policy_id], + )?; + audit(&tx, Some(policy_id), "deploy", actor.as_deref(), None, None)?; + tx.commit()?; + self.get_policy(policy_id) + } + + fn rollback_active_policy(&self, actor: Option) -> Result> { + let active = match self.get_active_policy()? { + Some(item) => item, + None => return Ok(None), + }; + let policy_id = active + .get("id") + .and_then(Value::as_i64) + .ok_or_else(|| anyhow!("active policy id missing"))?; + let current_version = active + .get("current_version") + .and_then(Value::as_i64) + .ok_or_else(|| anyhow!("active current_version missing"))?; + let conn = self.connect()?; + let rows = { + let mut stmt = conn.prepare( + r#" + SELECT version, policy_json + FROM policy_versions + WHERE policy_id = ? + ORDER BY version DESC + LIMIT 2 + "#, + )?; + stmt.query_map([policy_id], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()? + }; + if rows.len() < 2 { + return Ok(Some(active)); + } + let previous_version = rows[1].0; + let previous_policy: Value = serde_json::from_str(&rows[1].1)?; + let rollback_version = current_version + 1; + let rollback_checksum = checksum_policy(&previous_policy); + let now = utc_now(); + let tx = conn.unchecked_transaction()?; + tx.execute( + r#" + INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) + VALUES(?, ?, ?, ?, ?, ?, ?) + "#, + params![ + policy_id, + rollback_version, + canonical_json(&previous_policy)?, + rollback_checksum, + now, + actor, + previous_version + ], + )?; + tx.execute( + r#" + UPDATE policies + SET status = 'draft', current_version = ?, checksum = ?, updated_at = ? + WHERE id = ? + "#, + params![rollback_version, rollback_checksum, now, policy_id], + )?; + audit( + &tx, + Some(policy_id), + "rollback", + actor.as_deref(), + None, + Some(json!({"rollback_to": previous_version})), + )?; + tx.commit()?; + self.get_policy(policy_id) + } + + fn delete_policy(&self, policy_id: i64) -> Result { + let current = match self.get_policy(policy_id)? { + Some(item) => item, + None => return Ok(false), + }; + if current + .get("is_active") + .and_then(Value::as_i64) + .unwrap_or(0) + != 0 + { + bail!("cannot delete active policy"); + } + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + audit(&tx, Some(policy_id), "delete", None, None, None)?; + tx.execute( + "DELETE FROM policy_versions WHERE policy_id = ?", + [policy_id], + )?; + tx.execute("DELETE FROM policies WHERE id = ?", [policy_id])?; + tx.commit()?; + Ok(true) + } + + fn set_policy_status( + &self, + policy_id: i64, + status: &str, + actor: Option, + comment: Option, + ) -> Result> { + if !matches!( + status, + "draft" | "pending_approval" | "approved" | "deployed" + ) { + bail!("unsupported status: {status}"); + } + if self.get_policy(policy_id)?.is_none() { + return Ok(None); + } + validate_description(comment.as_deref())?; + let conn = self.connect()?; + let tx = conn.unchecked_transaction()?; + tx.execute( + "UPDATE policies SET status = ?, updated_at = ? WHERE id = ?", + params![status, utc_now(), policy_id], + )?; + audit( + &tx, + Some(policy_id), + "status_change", + actor.as_deref(), + comment.as_deref(), + Some(json!({"status": status})), + )?; + tx.commit()?; + self.get_policy(policy_id) + } + + fn list_audit(&self, policy_id: Option, limit: i64) -> Result> { + let conn = self.connect()?; + let limit = limit.clamp(1, 1000); + let mut items = Vec::new(); + if let Some(policy_id) = policy_id { + let mut stmt = conn.prepare( + r#" + SELECT id, policy_id, action, actor, comment, details_json, created_at + FROM policy_audit + WHERE policy_id = ? + ORDER BY id DESC + LIMIT ? + "#, + )?; + let rows = stmt.query_map(params![policy_id, limit], audit_from_row)?; + for row in rows { + items.push(row?); + } + } else { + let mut stmt = conn.prepare( + r#" + SELECT id, policy_id, action, actor, comment, details_json, created_at + FROM policy_audit + ORDER BY id DESC + LIMIT ? + "#, + )?; + let rows = stmt.query_map([limit], audit_from_row)?; + for row in rows { + items.push(row?); + } + } + Ok(items) + } +} + +fn audit( + conn: &Connection, + policy_id: Option, + action: &str, + actor: Option<&str>, + comment: Option<&str>, + details: Option, +) -> Result<()> { + let details_json = details + .as_ref() + .map(canonical_json) + .transpose() + .context("serialize audit details")?; + conn.execute( + r#" + INSERT INTO policy_audit(policy_id, action, actor, comment, details_json, created_at) + VALUES(?, ?, ?, ?, ?, ?) + "#, + params![policy_id, action, actor, comment, details_json, utc_now()], + )?; + Ok(()) +} + +fn policy_summary_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(json!({ + "id": row.get::<_, i64>(0)?, + "name": row.get::<_, String>(1)?, + "description": row.get::<_, Option>(2)?, + "status": row.get::<_, String>(3)?, + "is_active": row.get::<_, i64>(4)?, + "current_version": row.get::<_, i64>(5)?, + "checksum": row.get::<_, String>(6)?, + "created_at": row.get::<_, String>(7)?, + "updated_at": row.get::<_, String>(8)?, + })) +} + +fn audit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let details_json: Option = row.get(5)?; + let details = details_json + .as_deref() + .and_then(|text| serde_json::from_str(text).ok()) + .unwrap_or(Value::Null); + Ok(json!({ + "id": row.get::<_, i64>(0)?, + "policy_id": row.get::<_, Option>(1)?, + "action": row.get::<_, String>(2)?, + "actor": row.get::<_, Option>(3)?, + "comment": row.get::<_, Option>(4)?, + "created_at": row.get::<_, String>(6)?, + "details": details, + })) +} + +fn build_policy_bundle(record: &Value) -> Value { + json!({ + "active": true, + "policyId": record.get("id").cloned().unwrap_or(Value::Null), + "name": record.get("name").cloned().unwrap_or(Value::Null), + "version": record.get("current_version").cloned().unwrap_or(Value::Null), + "checksum": record.get("checksum").cloned().unwrap_or(Value::Null), + "updatedAtUtc": record.get("updated_at").cloned().unwrap_or(Value::Null), + "policy": record.get("policy").cloned().unwrap_or(Value::Null), + }) +} + +fn desired_policy(agent_id: &str, current: &Value, active: &Value) -> Value { + let current_version = current.get("version").cloned().unwrap_or(Value::Null); + let current_checksum = current.get("checksum").cloned().unwrap_or(Value::Null); + let desired_version = active + .get("current_version") + .cloned() + .unwrap_or(Value::Null); + let desired_checksum = active.get("checksum").cloned().unwrap_or(Value::Null); + let refresh_now = value_to_string(¤t_version) != value_to_string(&desired_version) + || value_to_string(¤t_checksum) != value_to_string(&desired_checksum); + json!({ + "agentId": agent_id, + "refreshNow": refresh_now, + "reason": if refresh_now { "mismatch" } else { "up-to-date" }, + "current": { + "version": current_version, + "checksum": current_checksum, + }, + "desired": { + "policyId": active.get("id").cloned().unwrap_or(Value::Null), + "version": desired_version, + "checksum": desired_checksum, + "updatedAtUtc": active.get("updated_at").cloned().unwrap_or(Value::Null), + }, + }) +} + +fn agent_heartbeat(agent_id: &str, payload: &Value) -> Value { + json!({ + "agentId": agent_id, + "hostname": payload.get("hostname").and_then(Value::as_str).unwrap_or(agent_id), + "version": payload.get("version").cloned().unwrap_or(Value::Null), + "checksum": payload.get("checksum").cloned().unwrap_or(Value::Null), + "updatedAtUtc": payload.get("updatedAtUtc").cloned().unwrap_or(Value::Null), + }) +} + +fn split_url(url: &str) -> (String, BTreeMap) { + let (path, query) = url.split_once('?').unwrap_or((url, "")); + let mut params = BTreeMap::new(); + for pair in query.split('&').filter(|item| !item.is_empty()) { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + params.insert(url_decode(key), url_decode(value)); + } + (path.to_string(), params) +} + +fn path_segments(path: &str) -> Vec<&str> { + path.trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect() +} + +fn query_limit(query: &BTreeMap) -> i64 { + query + .get("limit") + .and_then(|value| value.parse().ok()) + .unwrap_or(200) + .clamp(1, 1000) +} + +fn read_json_body(request: &mut Request) -> Result { + let mut body = String::new(); + request + .as_reader() + .read_to_string(&mut body) + .context("read request body")?; + if body.trim().is_empty() { + Ok(json!({})) + } else { + serde_json::from_str(&body).context("decode JSON body") + } +} + +fn json_response(status: StatusCode, payload: Value) -> Response>> { + let body = + serde_json::to_vec_pretty(&payload).unwrap_or_else(|_| b"{\"detail\":\"json\"}".to_vec()); + let mut response = Response::from_data(body).with_status_code(status); + response.add_header( + Header::from_bytes( + &b"Content-Type"[..], + &b"application/json; charset=utf-8"[..], + ) + .expect("valid header"), + ); + response +} + +fn parse_id(value: &str) -> Result { + value + .parse::() + .with_context(|| format!("invalid policy id: {value}")) +} + +fn actor_from(payload: &Value) -> Option { + match payload.get("actor") { + None => Some("api".to_string()), + Some(Value::Null) => None, + Some(value) => value.as_str().map(ToOwned::to_owned), + } +} + +fn required_string(value: Option<&Value>, field: &str) -> Result { + value + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .ok_or_else(|| anyhow!("{field} is required")) +} + +fn optional_string(value: Option<&Value>) -> Option { + match value { + Some(Value::String(text)) => Some(text.clone()), + _ => None, + } +} + +fn validate_name(value: &str) -> Result<()> { + if value.is_empty() || value.chars().count() > 128 { + bail!("name length must be 1..128"); + } + Ok(()) +} + +fn validate_description(value: Option<&str>) -> Result<()> { + if value.is_some_and(|text| text.chars().count() > 2048) { + bail!("description/comment length must be <= 2048"); + } + Ok(()) +} + +fn utc_now() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn checksum_policy(policy: &Value) -> String { + let mut hasher = Sha256::new(); + hasher.update( + canonical_json(policy) + .expect("policy JSON serializable") + .as_bytes(), + ); + format!("{:x}", hasher.finalize()) +} + +fn canonical_json(value: &Value) -> Result { + match value { + Value::Null => Ok("null".to_string()), + Value::Bool(value) => Ok(value.to_string()), + Value::Number(value) => Ok(value.to_string()), + Value::String(value) => serde_json::to_string(value).map_err(Into::into), + Value::Array(items) => { + let mut out = String::from("["); + for (idx, item) in items.iter().enumerate() { + if idx > 0 { + out.push(','); + } + out.push_str(&canonical_json(item)?); + } + out.push(']'); + Ok(out) + } + Value::Object(map) => { + let mut out = String::from("{"); + let mut first = true; + let mut sorted = BTreeMap::new(); + for (key, value) in map { + sorted.insert(key, value); + } + for (key, value) in sorted { + if !first { + out.push(','); + } + first = false; + out.push_str(&serde_json::to_string(key)?); + out.push(':'); + out.push_str(&canonical_json(value)?); + } + out.push('}'); + Ok(out) + } + } +} + +fn value_to_string(value: &Value) -> String { + match value { + Value::Null => "None".to_string(), + Value::String(text) => text.clone(), + other => other.to_string(), + } +} + +fn url_decode(value: &str) -> String { + decode(value) + .map(|value| value.into_owned()) + .unwrap_or_else(|_| value.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_json_sorts_keys_without_spaces() { + let value = json!({"b": 2, "a": {"z": true, "m": "тест"}}); + assert_eq!( + canonical_json(&value).unwrap(), + r#"{"a":{"m":"тест","z":true},"b":2}"# + ); + } + + #[test] + fn checksum_matches_python_contract_fixture() { + let value = json!({ + "version": 1, + "defaults": {"enabled": true, "action": "alert"}, + "endpoint": {"clipboard": []} + }); + assert_eq!( + checksum_policy(&value), + "88d9966b1517b45b009dbf8aca7260cf67b64201ea91bc85f743ab8ba88da0cf" + ); + } + + #[test] + fn desired_policy_detects_string_equivalent_version() { + let current = json!({"version": "6", "checksum": "abc"}); + let active = json!({"id": 1, "current_version": 6, "checksum": "abc", "updated_at": "now"}); + assert_eq!(desired_policy("a", ¤t, &active)["refreshNow"], false); + } +} diff --git a/adk-rust/crates/dlp-syslog-forwarder/Cargo.toml b/adk-rust/crates/dlp-syslog-forwarder/Cargo.toml new file mode 100644 index 0000000..3581c67 --- /dev/null +++ b/adk-rust/crates/dlp-syslog-forwarder/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dlp-syslog-forwarder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/dlp-syslog-forwarder/src/main.rs b/adk-rust/crates/dlp-syslog-forwarder/src/main.rs new file mode 100644 index 0000000..297e6ee --- /dev/null +++ b/adk-rust/crates/dlp-syslog-forwarder/src/main.rs @@ -0,0 +1,446 @@ +use std::collections::BTreeMap; +use std::fs; +use std::io::Write; +use std::net::{TcpStream, UdpSocket}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Value, json}; + +const DEFAULT_CONFIG: &str = "/opt/activitywatch/dlp-integrations/syslog-forwarder-config.yaml"; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP syslog forwarder")] +struct Cli { + #[arg(long, default_value = DEFAULT_CONFIG)] + config: PathBuf, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + state_path: PathBuf, + syslog_host: String, + syslog_port: u16, + syslog_proto: String, + facility: i64, + app_name: String, + per_bucket_limit: usize, +} + +impl Default for Config { + fn default() -> Self { + Self { + aw_api_base: default_aw_api_base(), + state_path: default_state_path(), + syslog_host: default_syslog_host(), + syslog_port: default_syslog_port(), + syslog_proto: default_syslog_proto(), + facility: default_facility(), + app_name: default_app_name(), + per_bucket_limit: default_per_bucket_limit(), + } + } +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + sent: usize, + buckets: usize, + dry_run: bool, + state_saved: bool, + state_path: String, + error: Option, +} + +fn default_aw_api_base() -> String { + "http://127.0.0.1:5600/api/0".to_string() +} + +fn default_state_path() -> PathBuf { + PathBuf::from("/var/lib/activitywatch/dlp-integrations/syslog-forwarder-state.json") +} + +fn default_syslog_host() -> String { + "127.0.0.1".to_string() +} + +fn default_syslog_port() -> u16 { + 514 +} + +fn default_syslog_proto() -> String { + "udp".to_string() +} + +fn default_facility() -> i64 { + 16 +} + +fn default_app_name() -> String { + "aw-dlp".to_string() +} + +fn default_per_bucket_limit() -> usize { + 300 +} + +fn load_config(path: &Path) -> Config { + if !path.exists() { + return Config::default(); + } + let Ok(text) = fs::read_to_string(path) else { + return Config::default(); + }; + let mut config = Config::default(); + for raw_line in text.lines() { + let line = raw_line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let key = key.trim(); + let value = value.trim().trim_matches('"').trim_matches('\''); + match key { + "aw_api_base" if !value.is_empty() => config.aw_api_base = value.to_string(), + "state_path" if !value.is_empty() => config.state_path = PathBuf::from(value), + "syslog_host" if !value.is_empty() => config.syslog_host = value.to_string(), + "syslog_port" => { + if let Ok(port) = value.parse::() { + config.syslog_port = port; + } + } + "syslog_proto" if !value.is_empty() => config.syslog_proto = value.to_string(), + "facility" => { + if let Ok(facility) = value.parse::() { + config.facility = facility; + } + } + "app_name" if !value.is_empty() => config.app_name = value.to_string(), + "per_bucket_limit" => { + if let Ok(limit) = value.parse::() { + config.per_bucket_limit = limit; + } + } + _ => {} + } + } + config +} + +fn load_json(path: &Path) -> Value { + let Ok(text) = fs::read_to_string(path) else { + return json!({}); + }; + serde_json::from_str::(&text) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| json!({})) +} + +fn save_json(path: &Path, payload: &Value) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::write(path, serde_json::to_string_pretty(payload)?) + .with_context(|| format!("write {}", path.display())) +} + +fn http_json(client: &Client, url: &str) -> Result { + client + .get(url) + .timeout(Duration::from_secs(15)) + .send() + .and_then(|resp| resp.error_for_status()) + .with_context(|| format!("GET {url}"))? + .json::() + .with_context(|| format!("parse JSON from {url}")) +} + +fn int_value(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(s)) => s.parse::().unwrap_or(0), + _ => 0, + } +} + +fn incident_bucket_ids(buckets: &Value) -> Vec { + let mut ids: Vec = buckets + .as_object() + .map(|map| { + map.keys() + .filter(|bucket_id| bucket_id.starts_with("aw-dlp-incidents_")) + .cloned() + .collect() + }) + .unwrap_or_default(); + ids.sort(); + ids +} + +fn iter_new_incidents( + client: &Client, + aw_base: &str, + state: &Value, + per_bucket_limit: usize, +) -> Result<(Vec, BTreeMap)> { + let buckets = http_json( + client, + &format!("{}/buckets/", aw_base.trim_end_matches('/')), + )?; + let bucket_ids = incident_bucket_ids(&buckets); + let last_ids = state.get("last_ids").and_then(Value::as_object); + let mut max_ids = BTreeMap::new(); + let mut out = Vec::new(); + for bucket_id in bucket_ids { + let url = format!( + "{}/buckets/{bucket_id}/events?limit={}", + aw_base.trim_end_matches('/'), + per_bucket_limit + ); + let events = match http_json(client, &url) { + Ok(Value::Array(items)) => items, + Ok(_) => { + eprintln!("skip bucket {bucket_id}: events response is not a list"); + continue; + } + Err(err) => { + eprintln!("skip bucket {bucket_id}: {err}"); + continue; + } + }; + let prev = last_ids + .and_then(|ids| ids.get(&bucket_id)) + .and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }) + .unwrap_or(0); + let mut bucket_max = prev; + for event in events { + let event_id = int_value(event.get("id")); + if event_id <= prev { + continue; + } + if event_id > bucket_max { + bucket_max = event_id; + } + out.push(event); + } + max_ids.insert(bucket_id, bucket_max); + } + out.sort_by_key(|event| int_value(event.get("id"))); + Ok((out, max_ids)) +} + +fn build_message(event: &Value, app_name: &str, facility: i64) -> String { + let pri = facility * 8 + 6; + let ts = Utc::now().to_rfc3339_opts(SecondsFormat::Micros, true); + let data = event + .get("data") + .filter(|value| value.is_object()) + .unwrap_or(&Value::Null); + let host = data + .get("hostname") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(); + let payload = json!({ + "event_id": event.get("id").cloned().unwrap_or(Value::Null), + "timestamp": event.get("timestamp").cloned().unwrap_or(Value::Null), + "host": host, + "severity": data.get("severity").cloned().unwrap_or(Value::Null), + "signalType": data.get("signalType").cloned().unwrap_or(Value::Null), + "username": data.get("username").cloned().unwrap_or(Value::Null), + "action": data.get("action").cloned().unwrap_or(Value::Null), + "message": data.get("message").cloned().unwrap_or(Value::Null), + "data": data, + }); + let payload_text = serde_json::to_string(&payload).unwrap_or_else(|_| "{}".to_string()); + format!("<{pri}>1 {ts} {host} {app_name} - - - {payload_text}") +} + +fn send_syslog(line: &str, host: &str, port: u16, proto: &str) -> Result<()> { + if proto.eq_ignore_ascii_case("tcp") { + let mut stream = TcpStream::connect((host, port)) + .with_context(|| format!("connect TCP syslog {host}:{port}"))?; + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .context("set TCP write timeout")?; + stream + .write_all(format!("{line}\n").as_bytes()) + .context("write TCP syslog")?; + return Ok(()); + } + let socket = UdpSocket::bind("0.0.0.0:0").context("bind UDP syslog socket")?; + socket + .send_to(line.as_bytes(), (host, port)) + .with_context(|| format!("send UDP syslog {host}:{port}"))?; + Ok(()) +} + +fn run(cli: &Cli, client: &Client) -> RunSummary { + let cfg = load_config(&cli.config); + let state = load_json(&cfg.state_path); + let (incidents, max_ids) = + match iter_new_incidents(client, &cfg.aw_api_base, &state, cfg.per_bucket_limit) { + Ok(result) => result, + Err(err) => { + eprintln!("skip syslog forwarder run: AW API unavailable: {err}"); + return RunSummary { + ok: true, + sent: 0, + buckets: 0, + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: Some(err.to_string()), + }; + } + }; + + let mut sent = 0; + for event in &incidents { + let line = build_message(event, &cfg.app_name, cfg.facility); + if !cli.dry_run + && let Err(err) = + send_syslog(&line, &cfg.syslog_host, cfg.syslog_port, &cfg.syslog_proto) + { + return RunSummary { + ok: false, + sent, + buckets: max_ids.len(), + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: Some(err.to_string()), + }; + } + sent += 1; + } + + let mut payload = json!({"last_ids": max_ids}); + let mut state_saved = false; + if !cli.dry_run { + if let Err(err) = save_json(&cfg.state_path, &payload) { + return RunSummary { + ok: false, + sent, + buckets: payload["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: Some(err.to_string()), + }; + } + state_saved = true; + } else { + payload["dry_run"] = json!(true); + } + + RunSummary { + ok: true, + sent, + buckets: payload["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: cli.dry_run, + state_saved, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: None, + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let client = Client::builder() + .no_proxy() + .build() + .context("build HTTP client")?; + let summary = run(&cli, &client); + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if let Some(err) = &summary.error { + eprintln!("{err}"); + } else { + println!( + "syslog forwarder sent={} buckets={}", + summary.sent, summary.buckets + ); + } + if summary.ok { + Ok(()) + } else { + Err(anyhow!( + summary + .error + .unwrap_or_else(|| "syslog forwarder failed".to_string()) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selects_only_new_incidents_by_id() { + let state = json!({"last_ids": {"aw-dlp-incidents_HOST": 10}}); + let last_ids = state.get("last_ids").and_then(Value::as_object).unwrap(); + assert_eq!(last_ids["aw-dlp-incidents_HOST"], 10); + } + + #[test] + fn builds_rfc5424_like_message() { + let event = json!({ + "id": 42, + "timestamp": "2026-05-31T12:00:00Z", + "data": { + "hostname": "HOST1", + "severity": "high", + "signalType": "dlp_incident", + "username": "user", + "action": "alert", + "message": "test" + } + }); + let msg = build_message(&event, "aw-dlp", 16); + assert!(msg.starts_with("<134>1 ")); + assert!(msg.contains(" HOST1 aw-dlp - - - ")); + assert!(msg.contains("\"event_id\":42")); + } + + #[test] + fn incident_bucket_filter_is_sorted() { + let buckets = json!({ + "other": {}, + "aw-dlp-incidents_B": {}, + "aw-dlp-incidents_A": {} + }); + assert_eq!( + incident_bucket_ids(&buckets), + vec!["aw-dlp-incidents_A", "aw-dlp-incidents_B"] + ); + } +} diff --git a/adk-rust/crates/dlp-webhook-sender/Cargo.toml b/adk-rust/crates/dlp-webhook-sender/Cargo.toml new file mode 100644 index 0000000..cca5751 --- /dev/null +++ b/adk-rust/crates/dlp-webhook-sender/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "dlp-webhook-sender" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true diff --git a/adk-rust/crates/dlp-webhook-sender/src/main.rs b/adk-rust/crates/dlp-webhook-sender/src/main.rs new file mode 100644 index 0000000..869835b --- /dev/null +++ b/adk-rust/crates/dlp-webhook-sender/src/main.rs @@ -0,0 +1,504 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::thread::sleep; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use reqwest::header::CONTENT_TYPE; +use serde::Serialize; +use serde_json::{Value, json}; + +const DEFAULT_CONFIG: &str = "/opt/activitywatch/dlp-integrations/webhook-config.yaml"; + +#[derive(Debug, Parser)] +#[command(about = "AWatch DLP webhook sender")] +struct Cli { + #[arg(long, default_value = DEFAULT_CONFIG)] + config: PathBuf, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct HookConfig { + url: String, + severity: Vec, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + state_path: PathBuf, + retries: usize, + timeout_sec: u64, + backoff_base: f64, + per_bucket_limit: usize, + critical_webhooks: Vec, +} + +impl Default for Config { + fn default() -> Self { + Self { + aw_api_base: "http://127.0.0.1:5600/api/0".to_string(), + state_path: PathBuf::from("/var/lib/activitywatch/dlp-integrations/webhook-state.json"), + retries: 4, + timeout_sec: 15, + backoff_base: 2.0, + per_bucket_limit: 300, + critical_webhooks: Vec::new(), + } + } +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + delivered: usize, + incidents_seen: usize, + buckets: usize, + dry_run: bool, + state_saved: bool, + state_path: String, + error: Option, +} + +fn load_config(path: &Path) -> Config { + let mut config = Config::default(); + let Ok(text) = fs::read_to_string(path) else { + return config; + }; + let mut current_hook: Option = None; + for raw_line in text.lines() { + let line_without_comment = raw_line.split('#').next().unwrap_or(""); + let line = line_without_comment.trim(); + if line.is_empty() || line == "critical_webhooks:" { + continue; + } + if line.starts_with("- ") { + if let Some(hook) = current_hook.take() { + config.critical_webhooks.push(hook); + } + let mut hook = HookConfig { + url: String::new(), + severity: vec!["high".to_string()], + }; + parse_hook_field(line.trim_start_matches("- ").trim(), &mut hook); + current_hook = Some(hook); + continue; + } + if raw_line.starts_with(' ') || raw_line.starts_with('\t') { + if let Some(hook) = current_hook.as_mut() { + parse_hook_field(line, hook); + } + continue; + } + if let Some(hook) = current_hook.take() { + config.critical_webhooks.push(hook); + } + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let key = key.trim(); + let value = clean_scalar(value); + match key { + "aw_api_base" if !value.is_empty() => config.aw_api_base = value, + "state_path" if !value.is_empty() => config.state_path = PathBuf::from(value), + "retries" => config.retries = value.parse().unwrap_or(config.retries), + "timeout_sec" => config.timeout_sec = value.parse().unwrap_or(config.timeout_sec), + "backoff_base" => config.backoff_base = value.parse().unwrap_or(config.backoff_base), + "per_bucket_limit" => { + config.per_bucket_limit = value.parse().unwrap_or(config.per_bucket_limit); + } + _ => {} + } + } + if let Some(hook) = current_hook { + config.critical_webhooks.push(hook); + } + config +} + +fn parse_hook_field(line: &str, hook: &mut HookConfig) { + let Some((key, value)) = line.split_once(':') else { + return; + }; + let key = key.trim(); + let value = clean_scalar(value); + match key { + "url" => hook.url = value, + "severity" => hook.severity = parse_list_or_scalar(&value), + _ => {} + } +} + +fn clean_scalar(value: &str) -> String { + value + .trim() + .trim_matches('"') + .trim_matches('\'') + .to_string() +} + +fn parse_list_or_scalar(value: &str) -> Vec { + let text = value.trim(); + if text.starts_with('[') && text.ends_with(']') { + return text + .trim_start_matches('[') + .trim_end_matches(']') + .split(',') + .map(clean_scalar) + .filter(|item| !item.is_empty()) + .collect(); + } + if text.is_empty() { + Vec::new() + } else { + vec![clean_scalar(text)] + } +} + +fn load_json(path: &Path) -> Value { + let Ok(text) = fs::read_to_string(path) else { + return json!({}); + }; + serde_json::from_str::(&text) + .ok() + .filter(Value::is_object) + .unwrap_or_else(|| json!({})) +} + +fn save_json(path: &Path, payload: &Value) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::write(path, serde_json::to_string_pretty(payload)?) + .with_context(|| format!("write {}", path.display())) +} + +fn http_json(client: &Client, url: &str, timeout_sec: u64) -> Result { + client + .get(url) + .timeout(Duration::from_secs(timeout_sec)) + .send() + .and_then(|resp| resp.error_for_status()) + .with_context(|| format!("GET {url}"))? + .json::() + .with_context(|| format!("parse JSON from {url}")) +} + +fn int_value(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(s)) => s.parse::().unwrap_or(0), + _ => 0, + } +} + +fn incident_bucket_ids(buckets: &Value) -> Vec { + let mut ids: Vec = buckets + .as_object() + .map(|map| { + map.keys() + .filter(|bucket_id| bucket_id.starts_with("aw-dlp-incidents_")) + .cloned() + .collect() + }) + .unwrap_or_default(); + ids.sort(); + ids +} + +fn iter_new_incidents( + client: &Client, + aw_base: &str, + state: &Value, + per_bucket_limit: usize, + timeout_sec: u64, +) -> Result<(Vec, BTreeMap)> { + let base = aw_base.trim_end_matches('/'); + let buckets = http_json(client, &format!("{base}/buckets/"), timeout_sec)?; + let bucket_ids = incident_bucket_ids(&buckets); + let last_ids = state.get("last_ids").and_then(Value::as_object); + let mut max_ids = BTreeMap::new(); + let mut out = Vec::new(); + for bucket_id in bucket_ids { + let events = match http_json( + client, + &format!("{base}/buckets/{bucket_id}/events?limit={per_bucket_limit}"), + timeout_sec, + ) { + Ok(Value::Array(items)) => items, + Ok(_) => continue, + Err(err) => return Err(err), + }; + let prev = last_ids + .and_then(|ids| ids.get(&bucket_id)) + .and_then(|v| match v { + Value::Number(n) => n.as_i64(), + Value::String(s) => s.parse().ok(), + _ => None, + }) + .unwrap_or(0); + let mut bucket_max = prev; + for event in events { + let event_id = int_value(event.get("id")); + if event_id <= prev { + continue; + } + if event_id > bucket_max { + bucket_max = event_id; + } + out.push(event); + } + max_ids.insert(bucket_id, bucket_max); + } + out.sort_by_key(|event| int_value(event.get("id"))); + Ok((out, max_ids)) +} + +fn should_send(severity: &str, allowed: &[String]) -> bool { + let severity = severity.to_ascii_lowercase(); + allowed + .iter() + .any(|item| item.to_ascii_lowercase() == severity) +} + +fn value_str(value: Option<&Value>, default: &str) -> String { + value.and_then(Value::as_str).unwrap_or(default).to_string() +} + +fn build_payload(event: &Value) -> Value { + let data = event + .get("data") + .filter(|value| value.is_object()) + .unwrap_or(&Value::Null); + let severity = value_str(data.get("severity"), "low"); + json!({ + "source": "AWatch-rus DLP", + "timestamp": event.get("timestamp").cloned().unwrap_or(Value::Null), + "event_id": event.get("id").cloned().unwrap_or(Value::Null), + "severity": severity, + "message": data.get("message").cloned().unwrap_or(Value::Null), + "ruleId": data.get("ruleId").cloned().unwrap_or(Value::Null), + "signalType": data.get("signalType").cloned().unwrap_or(Value::Null), + "hostname": data.get("hostname").cloned().unwrap_or(Value::Null), + "username": data.get("username").cloned().unwrap_or(Value::Null), + "action": data.get("action").cloned().unwrap_or(Value::Null), + "raw": data, + }) +} + +fn post_with_retry(client: &Client, url: &str, payload: &Value, cfg: &Config) -> bool { + let body = match serde_json::to_vec(payload) { + Ok(body) => body, + Err(err) => { + eprintln!("webhook payload serialization error url={url}: {err}"); + return false; + } + }; + for attempt in 1..=cfg.retries.max(1) { + let result = client + .post(url) + .timeout(Duration::from_secs(cfg.timeout_sec)) + .header(CONTENT_TYPE, "application/json; charset=utf-8") + .body(body.clone()) + .send(); + match result { + Ok(resp) if resp.status().is_success() => return true, + Ok(resp) => eprintln!( + "webhook http error url={url} code={} attempt={attempt}/{}", + resp.status(), + cfg.retries.max(1) + ), + Err(err) => eprintln!( + "webhook transport error url={url} err={err} attempt={attempt}/{}", + cfg.retries.max(1) + ), + } + if attempt < cfg.retries.max(1) { + sleep(Duration::from_secs_f64( + cfg.backoff_base.powi((attempt - 1) as i32), + )); + } + } + false +} + +fn run(cli: &Cli, client: &Client) -> RunSummary { + let cfg = load_config(&cli.config); + let state = load_json(&cfg.state_path); + let (incidents, max_ids) = match iter_new_incidents( + client, + &cfg.aw_api_base, + &state, + cfg.per_bucket_limit, + cfg.timeout_sec, + ) { + Ok(result) => result, + Err(err) => { + eprintln!("skip webhook sender run: AW API unavailable: {err}"); + return RunSummary { + ok: true, + delivered: 0, + incidents_seen: 0, + buckets: 0, + dry_run: cli.dry_run, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: Some(err.to_string()), + }; + } + }; + + let mut delivered = 0; + for event in &incidents { + let data = event + .get("data") + .filter(|value| value.is_object()) + .unwrap_or(&Value::Null); + let severity = value_str(data.get("severity"), "low"); + for hook in &cfg.critical_webhooks { + if hook.url.trim().is_empty() || !should_send(&severity, &hook.severity) { + continue; + } + if cli.dry_run || post_with_retry(client, &hook.url, &build_payload(event), &cfg) { + delivered += 1; + } + } + } + + let mut next_state = state; + next_state["last_ids"] = json!(max_ids); + next_state["updated_at"] = json!(Utc::now().to_rfc3339_opts(SecondsFormat::Micros, false)); + let mut state_saved = false; + if !cli.dry_run { + if let Err(err) = save_json(&cfg.state_path, &next_state) { + return RunSummary { + ok: false, + delivered, + incidents_seen: incidents.len(), + buckets: next_state["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: false, + state_saved: false, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: Some(err.to_string()), + }; + } + state_saved = true; + } + + RunSummary { + ok: true, + delivered, + incidents_seen: incidents.len(), + buckets: next_state["last_ids"] + .as_object() + .map_or(0, serde_json::Map::len), + dry_run: cli.dry_run, + state_saved, + state_path: cfg.state_path.to_string_lossy().to_string(), + error: None, + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let client = Client::builder() + .no_proxy() + .build() + .context("build HTTP client")?; + let summary = run(&cli, &client); + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if let Some(err) = &summary.error { + eprintln!("{err}"); + } else { + println!( + "Webhook sender done: delivered={} incidents_seen={}", + summary.delivered, summary.incidents_seen + ); + } + if summary.ok { + Ok(()) + } else { + Err(anyhow!( + summary + .error + .unwrap_or_else(|| "webhook sender failed".to_string()) + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_simple_config_with_hook() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("webhook-config.yaml"); + fs::write( + &path, + r#" +aw_api_base: "http://127.0.0.1:5600/api/0" +state_path: "/tmp/webhook-state.json" +retries: 2 +timeout_sec: 3 +backoff_base: 1.5 +per_bucket_limit: 10 +critical_webhooks: + - url: "https://example.invalid/hook" + severity: ["high", "medium"] +"#, + ) + .unwrap(); + let cfg = load_config(&path); + assert_eq!(cfg.retries, 2); + assert_eq!(cfg.timeout_sec, 3); + assert_eq!(cfg.per_bucket_limit, 10); + assert_eq!(cfg.critical_webhooks.len(), 1); + assert_eq!(cfg.critical_webhooks[0].url, "https://example.invalid/hook"); + assert_eq!(cfg.critical_webhooks[0].severity, ["high", "medium"]); + } + + #[test] + fn severity_matching_is_case_insensitive() { + assert!(should_send("HIGH", &[String::from("high")])); + assert!(!should_send("low", &[String::from("high")])); + } + + #[test] + fn payload_shape_matches_python_contract() { + let event = json!({ + "id": 7, + "timestamp": "2026-06-01T10:00:00Z", + "data": { + "severity": "high", + "message": "m", + "ruleId": "r", + "signalType": "dlp_incident", + "hostname": "h", + "username": "u", + "action": "alert" + } + }); + let payload = build_payload(&event); + assert_eq!(payload["source"], "AWatch-rus DLP"); + assert_eq!(payload["event_id"], 7); + assert_eq!(payload["severity"], "high"); + assert_eq!(payload["raw"]["ruleId"], "r"); + } +} diff --git a/adk-rust/crates/extract-ioc-from-sigma/Cargo.toml b/adk-rust/crates/extract-ioc-from-sigma/Cargo.toml new file mode 100644 index 0000000..d0b8e5a --- /dev/null +++ b/adk-rust/crates/extract-ioc-from-sigma/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "extract-ioc-from-sigma" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_yaml.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/extract-ioc-from-sigma/src/main.rs b/adk-rust/crates/extract-ioc-from-sigma/src/main.rs new file mode 100644 index 0000000..17afdb1 --- /dev/null +++ b/adk-rust/crates/extract-ioc-from-sigma/src/main.rs @@ -0,0 +1,499 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use regex::Regex; +use serde::Serialize; +use serde_yaml::Value; + +const OUTPUT_JSON: &str = "ioc_blacklist.json"; +const OUTPUT_CSV: &str = "ioc_blacklist.csv"; +const OUTPUT_SQL: &str = "ioc_blacklist.sql"; +const CSV_FIELDS: &[&str] = &[ + "ioc_type", + "ioc_value", + "field", + "rule_id", + "rule_title", + "source_file", +]; + +#[derive(Debug, Parser)] +#[command(about = "Extract IOC-like Sigma values for DLP preload")] +struct Cli { + #[arg(long, default_value = "rules")] + rules_root: PathBuf, + + #[arg(long, default_value = "ioc_export")] + out_dir: PathBuf, + + #[arg(long, default_value = "dlp_blacklist_ioc")] + table_name: String, +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +struct IocRow { + ioc_type: String, + ioc_value: String, + field: String, + rule_id: String, + rule_title: String, + source_file: String, +} + +fn main() { + let code = match run() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + extract_to_outputs(&cli.rules_root, &cli.out_dir, &cli.table_name) +} + +fn extract_to_outputs(rules_root: &Path, out_dir: &Path, table_name: &str) -> Result<()> { + if !rules_root.exists() { + bail!("rules root not found: {}", rules_root.display()); + } + if !rules_root.is_dir() { + bail!("rules root is not a directory: {}", rules_root.display()); + } + + let yaml_files = collect_yaml_files(rules_root)?; + let mut rows = Vec::new(); + for path in &yaml_files { + rows.extend(extract_from_yaml(path)?); + } + let rows = dedupe_and_sort(rows); + + fs::create_dir_all(out_dir).with_context(|| format!("create {}", out_dir.display()))?; + write_json(&out_dir.join(OUTPUT_JSON), &rows)?; + write_csv(&out_dir.join(OUTPUT_CSV), &rows)?; + write_sql(&out_dir.join(OUTPUT_SQL), &rows, table_name)?; + + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for row in &rows { + *counts.entry(row.ioc_type.as_str()).or_default() += 1; + } + + println!("rules_scanned={}", yaml_files.len()); + println!("iocs_extracted={}", rows.len()); + for (kind, count) in counts { + println!("{kind}={count}"); + } + println!("json={}", out_dir.join(OUTPUT_JSON).display()); + println!("csv={}", out_dir.join(OUTPUT_CSV).display()); + println!("sql={}", out_dir.join(OUTPUT_SQL).display()); + Ok(()) +} + +fn collect_yaml_files(root: &Path) -> Result> { + let mut out = Vec::new(); + collect_yaml_files_inner(root, &mut out)?; + out.sort(); + Ok(out) +} + +fn collect_yaml_files_inner(path: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("read file type {}", path.display()))?; + if file_type.is_dir() { + collect_yaml_files_inner(&path, out)?; + } else if file_type.is_file() && is_yaml_path(&path) { + out.push(path); + } + } + Ok(()) +} + +fn is_yaml_path(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "yml" | "yaml")) +} + +fn extract_from_yaml(path: &Path) -> Result> { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let doc: Value = match serde_yaml::from_str(&text) { + Ok(doc) => doc, + Err(_) => return Ok(Vec::new()), + }; + let Some(mapping) = doc.as_mapping() else { + return Ok(Vec::new()); + }; + let Some(detection) = mapping_get(mapping, "detection") else { + return Ok(Vec::new()); + }; + + let rule_id = mapping_get(mapping, "id") + .map(scalar_to_string) + .unwrap_or_default(); + let rule_title = mapping_get(mapping, "title") + .map(scalar_to_string) + .unwrap_or_default(); + let mut rows = Vec::new(); + let sha256_re = Regex::new(r"\b[a-fA-F0-9]{64}\b").expect("valid sha256 regex"); + walk( + detection, + &rule_id, + &rule_title, + &path.display().to_string(), + &sha256_re, + &mut rows, + ); + Ok(rows) +} + +fn mapping_get<'a>(mapping: &'a serde_yaml::Mapping, key: &str) -> Option<&'a Value> { + mapping.get(Value::String(key.to_string())) +} + +fn split_key(key: &str) -> (String, Vec) { + let parts: Vec<_> = key + .split('|') + .map(str::trim) + .filter(|part| !part.is_empty()) + .collect(); + if parts.is_empty() { + return (String::new(), Vec::new()); + } + ( + parts[0].to_ascii_lowercase(), + parts[1..] + .iter() + .map(|part| part.to_ascii_lowercase()) + .collect(), + ) +} + +fn walk( + node: &Value, + rule_id: &str, + rule_title: &str, + source_file: &str, + sha256_re: &Regex, + out: &mut Vec, +) { + match node { + Value::Mapping(mapping) => { + for (key, value) in mapping { + let key_string = scalar_to_string(key); + let (base, ops) = split_key(&key_string); + if is_base_field(&base) { + for raw in flatten_values(value) { + let Some(ioc_type) = detect_ioc_type(&base, &ops, &raw, sha256_re) else { + continue; + }; + if ioc_type == "sha256" { + for hash in parse_sha256(&raw, sha256_re) { + out.push(IocRow { + ioc_type: "sha256".to_string(), + ioc_value: hash, + field: key_string.clone(), + rule_id: rule_id.to_string(), + rule_title: rule_title.to_string(), + source_file: source_file.to_string(), + }); + } + } else { + out.push(IocRow { + ioc_type: ioc_type.to_string(), + ioc_value: raw, + field: key_string.clone(), + rule_id: rule_id.to_string(), + rule_title: rule_title.to_string(), + source_file: source_file.to_string(), + }); + } + } + } + walk(value, rule_id, rule_title, source_file, sha256_re, out); + } + } + Value::Sequence(items) => { + for item in items { + walk(item, rule_id, rule_title, source_file, sha256_re, out); + } + } + _ => {} + } +} + +fn is_base_field(base: &str) -> bool { + matches!( + base, + "image" | "commandline" | "originalfilename" | "hashes" + ) +} + +fn flatten_values(value: &Value) -> Vec { + match value { + Value::Null => Vec::new(), + Value::Bool(v) => vec![py_bool_string(*v).to_string()], + Value::Number(v) => vec![v.to_string()], + Value::String(v) => { + let trimmed = v.trim(); + if trimmed.is_empty() { + Vec::new() + } else { + vec![pyyaml_scalar_string(trimmed)] + } + } + Value::Sequence(items) => items.iter().flat_map(flatten_values).collect(), + Value::Mapping(mapping) => { + let mut out = Vec::new(); + for (key, value) in mapping { + let key = scalar_to_string(key); + for item in flatten_values(value) { + out.push(format!("{key}:{item}")); + } + } + out + } + Value::Tagged(tagged) => flatten_values(&tagged.value), + } +} + +fn scalar_to_string(value: &Value) -> String { + match value { + Value::Null => String::new(), + Value::Bool(v) => py_bool_string(*v).to_string(), + Value::Number(v) => v.to_string(), + Value::String(v) => v.clone(), + Value::Sequence(_) | Value::Mapping(_) | Value::Tagged(_) => serde_yaml::to_string(value) + .unwrap_or_default() + .trim() + .to_string(), + } +} + +fn pyyaml_scalar_string(value: &str) -> String { + match value.to_ascii_lowercase().as_str() { + "yes" | "true" | "on" => "True".to_string(), + "no" | "false" | "off" => "False".to_string(), + _ => value.to_string(), + } +} + +fn py_bool_string(value: bool) -> &'static str { + if value { "True" } else { "False" } +} + +fn detect_ioc_type<'a>( + base: &str, + ops: &[String], + raw: &str, + sha256_re: &Regex, +) -> Option<&'a str> { + if base == "image" && ops.iter().any(|op| op == "endswith") { + return Some("process_image_endswith"); + } + if base == "commandline" && ops.iter().any(|op| op == "contains") { + return Some("commandline_contains"); + } + if base == "originalfilename" { + return Some("original_filename"); + } + if base == "hashes" && (ops.iter().any(|op| op == "sha256") || sha256_re.is_match(raw)) { + return Some("sha256"); + } + None +} + +fn parse_sha256(raw: &str, sha256_re: &Regex) -> Vec { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for hit in sha256_re.find_iter(raw) { + let value = hit.as_str().to_ascii_lowercase(); + if seen.insert(value.clone()) { + out.push(value); + } + } + out +} + +fn dedupe_and_sort(rows: Vec) -> Vec { + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for row in rows { + let key = ( + row.ioc_type.clone(), + row.ioc_value.to_lowercase(), + row.field.clone(), + ); + if seen.insert(key) { + out.push(row); + } + } + out.sort_by(|a, b| { + ( + a.ioc_type.as_str(), + a.ioc_value.to_lowercase(), + a.field.as_str(), + ) + .cmp(&( + b.ioc_type.as_str(), + b.ioc_value.to_lowercase(), + b.field.as_str(), + )) + }); + out +} + +fn write_json(path: &Path, rows: &[IocRow]) -> Result<()> { + fs::write(path, serde_json::to_string_pretty(rows)?) + .with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn write_csv(path: &Path, rows: &[IocRow]) -> Result<()> { + let mut out = String::new(); + out.push_str(&CSV_FIELDS.join(",")); + out.push('\n'); + for row in rows { + out.push_str(&csv_escape(&row.ioc_type)); + out.push(','); + out.push_str(&csv_escape(&row.ioc_value)); + out.push(','); + out.push_str(&csv_escape(&row.field)); + out.push(','); + out.push_str(&csv_escape(&row.rule_id)); + out.push(','); + out.push_str(&csv_escape(&row.rule_title)); + out.push(','); + out.push_str(&csv_escape(&row.source_file)); + out.push('\n'); + } + fs::write(path, out).with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn csv_escape(value: &str) -> String { + if value.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } +} + +fn write_sql(path: &Path, rows: &[IocRow], table_name: &str) -> Result<()> { + let mut out = format!( + "CREATE TABLE IF NOT EXISTS {table_name} (\n\ + \x20 id INTEGER PRIMARY KEY AUTOINCREMENT,\n\ + \x20 ioc_type TEXT NOT NULL,\n\ + \x20 ioc_value TEXT NOT NULL,\n\ + \x20 field TEXT,\n\ + \x20 rule_id TEXT,\n\ + \x20 rule_title TEXT,\n\ + \x20 source_file TEXT\n\ + );\n\n" + ); + for row in rows { + out.push_str(&format!( + "INSERT INTO {table_name} (ioc_type, ioc_value, field, rule_id, rule_title, source_file) VALUES \ + ('{}','{}','{}','{}','{}','{}');\n", + sql_escape(&row.ioc_type), + sql_escape(&row.ioc_value), + sql_escape(&row.field), + sql_escape(&row.rule_id), + sql_escape(&row.rule_title), + sql_escape(&row.source_file), + )); + } + fs::write(path, out).with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn sql_escape(value: &str) -> String { + value.replace('\'', "''") +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn extracts_supported_iocs_and_dedupes() { + let tmp = tempdir().unwrap(); + let rule = tmp.path().join("rule.yml"); + fs::write( + &rule, + r#" +id: test-rule +title: Test Rule +detection: + selection: + Image|endswith: + - '\bad.exe' + - '\bad.exe' + CommandLine|contains: ['--dump', 'sekret'] + OriginalFileName: evil.exe + Hashes|SHA256: + - 'SHA256=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + condition: selection +"#, + ) + .unwrap(); + + let rows = dedupe_and_sort(extract_from_yaml(&rule).unwrap()); + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].ioc_type, "commandline_contains"); + assert!(rows.iter().any(|row| row.ioc_type == "sha256")); + assert_eq!( + rows.iter() + .filter(|row| row.ioc_type == "process_image_endswith") + .count(), + 1 + ); + } + + #[test] + fn writes_expected_outputs() { + let tmp = tempdir().unwrap(); + let rules = tmp.path().join("rules"); + let out = tmp.path().join("out"); + fs::create_dir_all(&rules).unwrap(); + fs::write( + rules.join("rule.yml"), + r#" +id: write-rule +title: "CSV, SQL Rule" +detection: + selection: + CommandLine|contains: "a,b" + condition: selection +"#, + ) + .unwrap(); + + extract_to_outputs(&rules, &out, "dlp_blacklist_ioc").unwrap(); + let json = fs::read_to_string(out.join(OUTPUT_JSON)).unwrap(); + let csv = fs::read_to_string(out.join(OUTPUT_CSV)).unwrap(); + let sql = fs::read_to_string(out.join(OUTPUT_SQL)).unwrap(); + assert!(json.contains("commandline_contains")); + assert!(csv.contains("\"a,b\"")); + assert!(sql.contains("CREATE TABLE IF NOT EXISTS dlp_blacklist_ioc")); + } + + #[test] + fn skips_invalid_yaml() { + let tmp = tempdir().unwrap(); + let rule = tmp.path().join("bad.yml"); + fs::write(&rule, ": not yaml: :").unwrap(); + assert!(extract_from_yaml(&rule).unwrap().is_empty()); + } +} diff --git a/adk-rust/crates/hayabusa-tools/Cargo.toml b/adk-rust/crates/hayabusa-tools/Cargo.toml new file mode 100644 index 0000000..3e527cc --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "hayabusa-tools" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "aw-hayabusa-case-alert-rust" +path = "src/bin/case_alert.rs" + +[[bin]] +name = "aw-hayabusa-link-case-rust" +path = "src/bin/link_case.rs" + +[[bin]] +name = "aw-hayabusa-from-windows-rust" +path = "src/bin/from_windows.rs" + +[[bin]] +name = "aw-hayabusa-autoprocess-rust" +path = "src/bin/autoprocess.rs" + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +fs2.workspace = true +regex.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs b/adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs new file mode 100644 index 0000000..015083b --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs @@ -0,0 +1,300 @@ +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use fs2::FileExt; +use hayabusa_tools::{guess_host_from_filename, read_json_file}; +use serde_json::{Value, json}; + +const LOCK_PATH: &str = "/opt/hayabusa/state/aw-hayabusa-autoprocess.lock"; +const WRAPPER: &str = "/usr/local/bin/aw-hayabusa"; +const LINKER: &str = "/usr/local/bin/aw-hayabusa-link-case"; +const CASE_ALERT: &str = "/usr/local/bin/aw-hayabusa-case-alert"; +const LATEST_INTAKE: &str = "/opt/hayabusa/state/latest-intake.json"; + +#[derive(Debug, Parser)] +#[command(about = "Auto-process Hayabusa zip packages dropped onto aw-rus server")] +struct Cli { + #[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/drop")] + drop_dir: PathBuf, + + #[arg(long, default_value_t = true)] + once: bool, +} + +#[derive(Debug)] +struct Sidecars { + case_id: Option, + host: Option, + mode: String, + link_source: String, + caseid_path: PathBuf, + meta_path: PathBuf, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let _once = cli.once; + fs::create_dir_all(&cli.drop_dir) + .with_context(|| format!("create {}", cli.drop_dir.display()))?; + let lock_path = Path::new(LOCK_PATH); + if let Some(parent) = lock_path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + let lock = File::create(lock_path).with_context(|| format!("open {}", lock_path.display()))?; + if lock.try_lock_exclusive().is_err() { + eprintln!("autoprocess already running"); + return Ok(0); + } + + let zips = list_zips(&cli.drop_dir)?; + if zips.is_empty() { + println!("no zip packages in drop dir"); + return Ok(0); + } + for zip_path in zips { + let result = process_one(&zip_path)?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "processed": zip_path.display().to_string(), + "latest_intake": result.latest_intake, + "case_alert": result.case_alert, + }))? + ); + } + Ok(0) +} + +struct ProcessResult { + latest_intake: Value, + case_alert: Option, +} + +fn list_zips(drop_dir: &Path) -> Result> { + let mut zips = Vec::new(); + for entry in fs::read_dir(drop_dir).with_context(|| format!("read {}", drop_dir.display()))? { + let path = entry?.path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("zip") { + zips.push(path); + } + } + zips.sort(); + Ok(zips) +} + +fn process_one(zip_path: &Path) -> Result { + let sidecars = load_sidecars(zip_path)?; + let host = guess_host(zip_path, &sidecars); + let mode = if sidecars.mode.is_empty() { + "incident".to_string() + } else { + sidecars.mode.clone() + }; + + let mut accept_cmd = vec![ + "accept".to_string(), + "--package".to_string(), + zip_path.display().to_string(), + ]; + if let Some(host) = host { + accept_cmd.extend(["--host".to_string(), host]); + } + run_checked(Path::new(WRAPPER), &accept_cmd)?; + run_checked( + Path::new(WRAPPER), + &[ + "process-inbox".to_string(), + "--mode".to_string(), + mode.clone(), + "--limit".to_string(), + "1".to_string(), + ], + )?; + let latest = read_json_file(Path::new(LATEST_INTAKE))?; + let report_dir = PathBuf::from( + latest + .get("report_dir") + .and_then(Value::as_str) + .context("latest intake report_dir missing")?, + ); + let mut case_alert = None; + if Path::new(CASE_ALERT).is_file() { + let mut alert_cmd = vec![ + "--mode".to_string(), + mode.clone(), + "--link-source".to_string(), + sidecars.link_source.clone(), + ]; + if let Some(case_id) = sidecars.case_id { + alert_cmd.extend(["--case-id".to_string(), case_id.to_string()]); + } + let output = run_capture(Path::new(CASE_ALERT), &alert_cmd)?; + case_alert = Some(json!({ + "returncode": output.returncode, + "stdout": output.stdout.trim(), + "stderr": output.stderr.trim(), + })); + } + archive_sidecars(&report_dir, &sidecars)?; + archive_drop_package(&report_dir, zip_path)?; + if sidecars.case_id.is_some() && !Path::new(CASE_ALERT).is_file() { + run_checked( + Path::new(LINKER), + &[ + "--case-id".to_string(), + sidecars.case_id.unwrap().to_string(), + "--mode".to_string(), + mode, + "--link-source".to_string(), + sidecars.link_source, + ], + )?; + } + Ok(ProcessResult { + latest_intake: latest, + case_alert, + }) +} + +fn load_sidecars(zip_path: &Path) -> Result { + let base = zip_path.with_extension(""); + let caseid_path = base.with_extension("caseid"); + let meta_path = base.with_extension("meta.json"); + let meta = if meta_path.is_file() { + read_json_file(&meta_path)? + } else { + json!({}) + }; + let mut case_id = meta.get("case_id").and_then(Value::as_i64); + if case_id.is_none() && caseid_path.is_file() { + let raw = fs::read_to_string(&caseid_path) + .with_context(|| format!("read {}", caseid_path.display()))?; + let trimmed = raw.trim(); + if !trimmed.is_empty() { + case_id = Some( + trimmed + .parse::() + .with_context(|| format!("parse {}", caseid_path.display()))?, + ); + } + } + Ok(Sidecars { + case_id, + host: meta.get("host").and_then(Value::as_str).map(str::to_string), + mode: meta + .get("mode") + .and_then(Value::as_str) + .unwrap_or("incident") + .to_string(), + link_source: meta + .get("link_source") + .and_then(Value::as_str) + .unwrap_or("aw-rus-drop-autoprocess") + .to_string(), + caseid_path, + meta_path, + }) +} + +fn archive_sidecars(report_dir: &Path, sidecars: &Sidecars) -> Result<()> { + let target_dir = report_dir.join("input-sidecars"); + fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?; + for path in [&sidecars.caseid_path, &sidecars.meta_path] { + if path.is_file() { + fs::rename( + path, + target_dir.join(path.file_name().context("sidecar file name")?), + ) + .with_context(|| format!("move {}", path.display()))?; + } + } + Ok(()) +} + +fn archive_drop_package(report_dir: &Path, zip_path: &Path) -> Result<()> { + let target_dir = report_dir.join("input-drop"); + fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?; + let target_path = target_dir.join(zip_path.file_name().context("zip file name")?); + if target_path.exists() { + fs::remove_file(&target_path) + .with_context(|| format!("remove {}", target_path.display()))?; + } + fs::rename(zip_path, &target_path).with_context(|| format!("move {}", zip_path.display()))?; + Ok(()) +} + +fn guess_host(zip_path: &Path, sidecars: &Sidecars) -> Option { + if let Some(host) = &sidecars.host { + if !host.is_empty() { + return Some(host.clone()); + } + } + let name = zip_path + .file_stem() + .and_then(|name| name.to_str()) + .map(guess_host_from_filename)?; + (!name.is_empty()).then_some(name) +} + +fn run_checked(program: &Path, args: &[String]) -> Result<()> { + println!( + "RUN {} {}", + program.display(), + args.iter() + .map(String::as_str) + .collect::>() + .join(" ") + ); + let status = Command::new(program) + .args(args) + .status() + .with_context(|| format!("run {}", program.display()))?; + if !status.success() { + bail!( + "{} failed with status {}", + program.display(), + status.code().unwrap_or(1) + ); + } + Ok(()) +} + +struct Captured { + returncode: i32, + stdout: String, + stderr: String, +} + +fn run_capture(program: &Path, args: &[String]) -> Result { + println!( + "RUN {} {}", + program.display(), + args.iter() + .map(String::as_str) + .collect::>() + .join(" ") + ); + let output = Command::new(program) + .args(args) + .output() + .with_context(|| format!("run {}", program.display()))?; + Ok(Captured { + returncode: output.status.code().unwrap_or(1), + stdout: String::from_utf8_lossy(&output.stdout).to_string(), + stderr: String::from_utf8_lossy(&output.stderr).to_string(), + }) +} diff --git a/adk-rust/crates/hayabusa-tools/src/bin/case_alert.rs b/adk-rust/crates/hayabusa-tools/src/bin/case_alert.rs new file mode 100644 index 0000000..8115eec --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/src/bin/case_alert.rs @@ -0,0 +1,184 @@ +use std::path::PathBuf; + +use anyhow::Result; +use clap::Parser; +use hayabusa_tools::{ + analyze_report, build_case_payload, build_comment, build_hayabusa_payload, build_telegram_text, + env_bool, env_string, http_client, normalize_case_api_base, patch_json, post_json, + read_json_file, required_str, severity_meets, +}; +use reqwest::blocking::Client; +use serde_json::{Value, json}; + +#[derive(Debug, Parser)] +#[command( + about = "Auto-create/update AW-rus case, compute Hayabusa severity, and send Telegram alerts" +)] +struct Cli { + #[arg(long)] + case_id: Option, + + #[arg(long, default_value = "/opt/hayabusa/state/latest-intake.json")] + intake_json: PathBuf, + + #[arg(long)] + case_api_base: Option, + + #[arg(long, default_value = "incident")] + mode: String, + + #[arg(long, default_value = "aw-rus-drop-autoprocess")] + link_source: String, + + #[arg(long, default_value_t = false)] + auto_create: bool, + + #[arg(long)] + auto_create_min_severity: Option, + + #[arg(long, default_value_t = false)] + telegram_enabled: bool, + + #[arg(long)] + telegram_min_severity: Option, + + #[arg(long)] + telegram_bot_token: Option, + + #[arg(long)] + telegram_chat_ids: Option, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let intake = read_json_file(&cli.intake_json)?; + let summary = analyze_report(PathBuf::from(required_str(&intake, "report_dir")?).as_path())?; + let case_api_base = normalize_case_api_base(cli.case_api_base.as_deref().unwrap_or( + &env_string("AW_HAYABUSA_CASE_API_BASE", "http://127.0.0.1:5602"), + )); + let auto_create = cli.auto_create || env_bool("AW_HAYABUSA_AUTO_CASE_ENABLED", true); + let auto_create_min_severity = cli + .auto_create_min_severity + .unwrap_or_else(|| env_string("AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY", "medium")); + let telegram_enabled = cli.telegram_enabled || env_bool("AW_HAYABUSA_TELEGRAM_ENABLED", false); + let telegram_min_severity = cli + .telegram_min_severity + .unwrap_or_else(|| env_string("AW_HAYABUSA_TELEGRAM_MIN_SEVERITY", "high")); + let telegram_bot_token = cli + .telegram_bot_token + .unwrap_or_else(|| env_string("AW_HAYABUSA_TELEGRAM_BOT_TOKEN", "")); + let telegram_chat_ids = cli + .telegram_chat_ids + .unwrap_or_else(|| env_string("AW_HAYABUSA_TELEGRAM_CHAT_IDS", "")); + let client = http_client()?; + + let mut case_id = cli.case_id; + let mut created_case = Value::Null; + let mut case_error: Option = None; + let mut linked = false; + let mut comment_added = false; + + if let Err(err) = (|| -> Result<()> { + if case_id.is_none() + && auto_create + && severity_meets(&summary.severity, &auto_create_min_severity) + { + created_case = post_json( + &client, + &format!("{case_api_base}/api/0/dlp/cases"), + &build_case_payload(&intake, &summary)?, + )?; + case_id = created_case.get("id").and_then(Value::as_i64); + } + if let Some(id) = case_id { + patch_json( + &client, + &format!("{case_api_base}/api/0/dlp/cases/{id}"), + &json!({"severity": summary.severity}), + )?; + post_json( + &client, + &format!("{case_api_base}/api/0/dlp/cases/{id}/forensics/hayabusa"), + &build_hayabusa_payload(&intake, &cli.mode, &cli.link_source)?, + )?; + linked = true; + post_json( + &client, + &format!("{case_api_base}/api/0/dlp/cases/{id}/comments"), + &json!({"comment": build_comment(&summary, &intake)?, "author": "aw-hayabusa-auto"}), + )?; + comment_added = true; + } + Ok(()) + })() { + case_error = Some(err.to_string()); + } + + let telegram_results = if telegram_enabled + && !telegram_bot_token.is_empty() + && severity_meets(&summary.severity, &telegram_min_severity) + { + let chat_ids = telegram_chat_ids + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .collect::>(); + send_telegram( + &client, + &telegram_bot_token, + &chat_ids, + &build_telegram_text(case_id, &intake, &summary)?, + ) + } else { + Vec::new() + }; + + let result = json!({ + "summary": summary, + "case_id": case_id, + "case_created": if created_case.is_null() { Value::Null } else { created_case }, + "case_linked": linked, + "case_comment_added": comment_added, + "case_error": case_error, + "telegram_results": telegram_results, + }); + println!("{}", serde_json::to_string_pretty(&result)?); + Ok(if result.get("case_error").is_some_and(|v| !v.is_null()) { + 1 + } else { + 0 + }) +} + +fn send_telegram(client: &Client, bot_token: &str, chat_ids: &[&str], text: &str) -> Vec { + let mut results = Vec::new(); + for chat_id in chat_ids { + let url = format!("https://api.telegram.org/bot{bot_token}/sendMessage"); + let response = client + .post(&url) + .form(&[("chat_id", *chat_id), ("text", text)]) + .send(); + match response { + Ok(resp) => match resp.json::() { + Ok(body) => results.push(json!({"chat_id": chat_id, "ok": true, "response": body})), + Err(err) => { + results.push(json!({"chat_id": chat_id, "ok": false, "error": err.to_string()})) + } + }, + Err(_) => results + .push(json!({"chat_id": chat_id, "ok": false, "error": "telegram request failed"})), + } + } + results +} diff --git a/adk-rust/crates/hayabusa-tools/src/bin/from_windows.rs b/adk-rust/crates/hayabusa-tools/src/bin/from_windows.rs new file mode 100644 index 0000000..38abf75 --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/src/bin/from_windows.rs @@ -0,0 +1,243 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use hayabusa_tools::{guess_host_from_filename, windows_filename}; +use regex::Regex; +use serde_json::Value; + +const WINDOWS_EXPORT_CMD: &str = r"powershell.exe -ExecutionPolicy Bypass -File C:\ProgramData\AWatch-rus\export-evtx-for-hayabusa.ps1 -DaysBack {days_back} | ConvertTo-Json -Depth 8 -Compress"; +const WINDOWS_LATEST_ZIP_CMD: &str = r"Get-ChildItem 'C:\ProgramData\AWatch-rus\forensics\evtx-exports' -File -Filter '*.zip' | Sort-Object LastWriteTime -Descending | Select-Object -First 1 FullName,Length,LastWriteTime | ConvertTo-Json -Compress"; + +#[derive(Debug, Parser)] +#[command( + about = "Run Windows EVTX export and Hayabusa intake directly from aw-server, without the laptop" +)] +struct Cli { + #[arg( + long, + default_value = "/opt/activitywatch/aw-rus-ops/ansible/inventory.ini" + )] + inventory: PathBuf, + + #[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/venv/bin/ansible")] + ansible_bin: PathBuf, + + #[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/drop")] + drop_dir: PathBuf, + + #[arg(long, default_value_t = 1)] + days_back: i64, + + #[arg(long, default_value = "incident", value_parser = ["quick", "incident", "full"])] + mode: String, + + #[arg(long)] + case_id: Option, + + #[arg(long, default_value = "aw-rus-ops-from-windows")] + link_source: String, + + #[arg(long, default_value = "aw_windows")] + windows_group: String, + + #[arg(long, default_value = "/usr/local/bin/aw-hayabusa")] + wrapper: PathBuf, + + #[arg(long, default_value = "/usr/local/bin/aw-hayabusa-link-case")] + linker: PathBuf, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + ensure_file(&cli.inventory, "inventory")?; + ensure_file(&cli.ansible_bin, "ansible binary")?; + ensure_file(&cli.wrapper, "wrapper")?; + fs::create_dir_all(&cli.drop_dir) + .with_context(|| format!("create {}", cli.drop_dir.display()))?; + + let export_arg = WINDOWS_EXPORT_CMD.replace("{days_back}", &cli.days_back.to_string()); + let export_cmd = vec![ + cli.windows_group.as_str(), + "-i", + path_str(&cli.inventory)?, + "-m", + "win_shell", + "-a", + export_arg.as_str(), + ]; + print_run("RUN_EXPORT", &cli.ansible_bin, &export_cmd); + let export_out = run_capture(&cli.ansible_bin, &export_cmd)?; + let export_json = extract_json_blob(&export_out)?; + + let list_cmd = vec![ + cli.windows_group.as_str(), + "-i", + path_str(&cli.inventory)?, + "-m", + "win_shell", + "-a", + WINDOWS_LATEST_ZIP_CMD, + ]; + print_run("RUN_LIST", &cli.ansible_bin, &list_cmd); + let latest_out = run_capture(&cli.ansible_bin, &list_cmd)?; + let latest = normalize_latest_zip(extract_json_blob(&latest_out)?)?; + let remote_zip = latest + .get("FullName") + .and_then(Value::as_str) + .context("latest zip FullName missing")?; + let filename = windows_filename(remote_zip); + let local_zip = cli.drop_dir.join(&filename); + let remote_zip_posix = remote_zip.replace('\\', "/"); + let fetch_arg = format!( + "src={} dest={}/ flat=yes", + remote_zip_posix, + cli.drop_dir.display() + ); + let fetch_cmd = vec![ + cli.windows_group.as_str(), + "-i", + path_str(&cli.inventory)?, + "-m", + "fetch", + "-a", + fetch_arg.as_str(), + ]; + print_run("RUN_FETCH", &cli.ansible_bin, &fetch_cmd); + run_checked(&cli.ansible_bin, &fetch_cmd)?; + if !local_zip.is_file() { + bail!("fetched zip not found: {}", local_zip.display()); + } + + let host = export_json + .get("hostname") + .and_then(Value::as_str) + .map(str::to_string) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| guess_host_from_filename(&filename)); + let mut accept_args = vec![ + "accept".to_string(), + "--package".to_string(), + local_zip.display().to_string(), + ]; + if !host.is_empty() { + accept_args.extend(["--host".to_string(), host]); + } + print_run_owned("RUN_ACCEPT", &cli.wrapper, &accept_args); + run_checked_owned(&cli.wrapper, &accept_args)?; + + let process_args = vec![ + "process-inbox".to_string(), + "--mode".to_string(), + cli.mode.clone(), + "--limit".to_string(), + "1".to_string(), + ]; + print_run_owned("RUN_PROCESS", &cli.wrapper, &process_args); + run_checked_owned(&cli.wrapper, &process_args)?; + + if let Some(case_id) = cli.case_id { + ensure_file(&cli.linker, "linker")?; + let link_args = vec![ + "--case-id".to_string(), + case_id.to_string(), + "--mode".to_string(), + cli.mode, + "--link-source".to_string(), + cli.link_source, + ]; + print_run_owned("RUN_LINK", &cli.linker, &link_args); + run_checked_owned(&cli.linker, &link_args)?; + } + + let latest_intake = Path::new("/opt/hayabusa/state/latest-intake.json"); + println!("LATEST_INTAKE"); + println!( + "{}", + fs::read_to_string(latest_intake).context("read latest intake")? + ); + Ok(()) +} + +fn ensure_file(path: &Path, label: &str) -> Result<()> { + if !path.is_file() { + bail!("{label} not found: {}", path.display()); + } + Ok(()) +} + +fn path_str(path: &Path) -> Result<&str> { + path.to_str() + .with_context(|| format!("path is not UTF-8: {}", path.display())) +} + +fn run_capture(program: &Path, args: &[&str]) -> Result { + let output = Command::new(program) + .args(args) + .output() + .with_context(|| format!("run {}", program.display()))?; + if !output.status.success() { + print!("{}", String::from_utf8_lossy(&output.stdout)); + eprint!("{}", String::from_utf8_lossy(&output.stderr)); + std::process::exit(output.status.code().unwrap_or(1)); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +fn run_checked(program: &Path, args: &[&str]) -> Result<()> { + let status = Command::new(program) + .args(args) + .status() + .with_context(|| format!("run {}", program.display()))?; + if !status.success() { + std::process::exit(status.code().unwrap_or(1)); + } + Ok(()) +} + +fn run_checked_owned(program: &Path, args: &[String]) -> Result<()> { + let status = Command::new(program) + .args(args) + .status() + .with_context(|| format!("run {}", program.display()))?; + if !status.success() { + std::process::exit(status.code().unwrap_or(1)); + } + Ok(()) +} + +fn print_run(label: &str, program: &Path, args: &[&str]) { + println!("{label} {} {}", program.display(), args.join(" ")); +} + +fn print_run_owned(label: &str, program: &Path, args: &[String]) { + println!("{label} {} {}", program.display(), args.join(" ")); +} + +fn extract_json_blob(text: &str) -> Result { + let re = Regex::new(r"(?s)(\{.*\}|\[.*\])").context("compile JSON extractor")?; + let matches = re.find_iter(text).collect::>(); + for candidate in matches.iter().rev().map(|item| item.as_str()) { + if let Ok(value) = serde_json::from_str::(candidate) { + return Ok(value); + } + } + bail!("cannot parse JSON from ansible output:\n{text}") +} + +fn normalize_latest_zip(value: Value) -> Result { + if let Some(items) = value.as_array() { + items.first().cloned().context("latest zip list is empty") + } else { + Ok(value) + } +} diff --git a/adk-rust/crates/hayabusa-tools/src/bin/link_case.rs b/adk-rust/crates/hayabusa-tools/src/bin/link_case.rs new file mode 100644 index 0000000..b9e7c65 --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/src/bin/link_case.rs @@ -0,0 +1,55 @@ +use anyhow::Result; +use clap::Parser; +use hayabusa_tools::{build_hayabusa_payload, http_client, link_hayabusa_to_case, read_json_file}; +use serde_json::json; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +#[command(about = "Link Hayabusa intake metadata to AW-rus case management")] +struct Cli { + #[arg(long)] + case_id: i64, + + #[arg(long, default_value = "/opt/hayabusa/state/latest-intake.json")] + intake_json: PathBuf, + + #[arg(long, default_value = "http://127.0.0.1:5602")] + case_api_base: String, + + #[arg(long, default_value = "incident")] + mode: String, + + #[arg(long, default_value = "aw-rus-ops")] + link_source: String, +} + +fn main() { + if let Err(err) = run() { + eprintln!("{err:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let intake = read_json_file(&cli.intake_json)?; + let client = http_client()?; + let case = link_hayabusa_to_case( + &client, + &cli.case_api_base, + cli.case_id, + &intake, + &cli.mode, + &cli.link_source, + )?; + let _payload = build_hayabusa_payload(&intake, &cli.mode, &cli.link_source)?; + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "case_id": cli.case_id, + "intake": intake, + "forensics": case.get("forensics").cloned().unwrap_or(serde_json::Value::Null), + }))? + ); + Ok(()) +} diff --git a/adk-rust/crates/hayabusa-tools/src/lib.rs b/adk-rust/crates/hayabusa-tools/src/lib.rs new file mode 100644 index 0000000..8dc3618 --- /dev/null +++ b/adk-rust/crates/hayabusa-tools/src/lib.rs @@ -0,0 +1,526 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{DateTime, Utc}; +use reqwest::blocking::Client; +use reqwest::header::{ACCEPT, CONTENT_TYPE}; +use serde::Serialize; +use serde_json::{Value, json}; + +pub const SEVERITY_ORDER: &[(&str, i64)] = + &[("low", 1), ("medium", 2), ("high", 3), ("critical", 4)]; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Summary { + pub severity: String, + pub score: i64, + pub events_total: usize, + pub level_counts: BTreeMap, + pub top_rules: Vec, + pub first_timestamp: Option, + pub last_timestamp: Option, + pub failed_logon_rows: usize, + pub successful_logon_rows: usize, + pub suspicious_pwsh: usize, + pub credential_events: usize, + pub timestomp_events: usize, + pub logon_failure_events: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct TopRule { + pub title: String, + pub count: usize, +} + +pub fn http_client() -> Result { + Client::builder() + .timeout(Duration::from_secs(30)) + .no_proxy() + .build() + .context("build HTTP client") +} + +pub fn get_json(client: &Client, url: &str) -> Result { + client + .get(url) + .header(ACCEPT, "application/json") + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))? + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +pub fn post_json(client: &Client, url: &str, payload: &Value) -> Result { + let response = client + .post(url) + .header(ACCEPT, "application/json") + .header(CONTENT_TYPE, "application/json") + .json(payload) + .send() + .with_context(|| format!("POST {url}"))? + .error_for_status() + .with_context(|| format!("POST {url} status"))?; + decode_optional_json(response.text().context("read POST response")?) +} + +pub fn patch_json(client: &Client, url: &str, payload: &Value) -> Result { + let response = client + .patch(url) + .header(ACCEPT, "application/json") + .header(CONTENT_TYPE, "application/json") + .json(payload) + .send() + .with_context(|| format!("PATCH {url}"))? + .error_for_status() + .with_context(|| format!("PATCH {url} status"))?; + decode_optional_json(response.text().context("read PATCH response")?) +} + +fn decode_optional_json(body: String) -> Result { + if body.trim().is_empty() { + Ok(json!({})) + } else { + serde_json::from_str(&body).context("decode JSON response") + } +} + +pub fn read_json_file(path: &Path) -> Result { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("decode {}", path.display())) +} + +pub fn write_json_pretty(value: &Value) -> Result { + serde_json::to_string_pretty(value).context("encode JSON") +} + +pub fn normalize_host(value: Option<&str>) -> String { + value.unwrap_or("").trim().to_lowercase() +} + +pub fn normalize_case_api_base(raw: &str) -> String { + let mut base = raw.trim().trim_end_matches('/').to_string(); + if base.ends_with("/api/0/dlp/cases") { + let cut = base.len() - "/api/0/dlp/cases".len(); + base.truncate(cut); + } + base +} + +pub fn build_hayabusa_payload(intake: &Value, mode: &str, link_source: &str) -> Result { + let report_dir = required_str(intake, "report_dir")?; + let report_dir_path = PathBuf::from(report_dir); + Ok(json!({ + "tool": "hayabusa", + "host": required_str(intake, "host")?, + "mode": mode, + "status": required_str(intake, "status")?, + "intake_id": required_str(intake, "intake_id")?, + "package_path": required_str(intake, "package_path")?, + "sha256": required_str(intake, "sha256")?, + "report_dir": report_dir, + "summary_html": report_dir_path.join("summary.html").to_string_lossy(), + "timeline_path": report_dir_path.join("timeline.jsonl").to_string_lossy(), + "manifest_path": report_dir_path.join("manifest.json").to_string_lossy(), + "link_source": link_source, + })) +} + +pub fn link_hayabusa_to_case( + client: &Client, + case_api_base: &str, + case_id: i64, + intake: &Value, + mode: &str, + link_source: &str, +) -> Result { + let case_api_base = normalize_case_api_base(case_api_base); + let case = get_json( + client, + &format!("{case_api_base}/api/0/dlp/cases/{case_id}"), + )?; + let case_host = normalize_host(case.get("host").and_then(Value::as_str)); + let intake_host = normalize_host(intake.get("host").and_then(Value::as_str)); + if !case_host.is_empty() && !intake_host.is_empty() && case_host != intake_host { + bail!( + "hayabusa host mismatch: case host={} intake host={}", + case.get("host").and_then(Value::as_str).unwrap_or(""), + intake.get("host").and_then(Value::as_str).unwrap_or("") + ); + } + let payload = build_hayabusa_payload(intake, mode, link_source)?; + post_json( + client, + &format!("{case_api_base}/api/0/dlp/cases/{case_id}/forensics/hayabusa"), + &payload, + )?; + get_json( + client, + &format!("{case_api_base}/api/0/dlp/cases/{case_id}"), + ) +} + +pub fn analyze_report(report_dir: &Path) -> Result { + let timeline_path = report_dir.join("timeline.jsonl"); + let mut level_counts: HashMap = HashMap::new(); + let mut title_counts: HashMap = HashMap::new(); + let mut first_ts: Option> = None; + let mut last_ts: Option> = None; + let mut total_events = 0usize; + + if timeline_path.is_file() { + let text = fs::read_to_string(&timeline_path) + .with_context(|| format!("read {}", timeline_path.display()))?; + for raw_line in text.lines() { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + let Ok(event) = serde_json::from_str::(line) else { + continue; + }; + total_events += 1; + let level = normalize_level(event.get("Level").and_then(Value::as_str)); + *level_counts.entry(level).or_insert(0) += 1; + let title = event + .get("RuleTitle") + .and_then(Value::as_str) + .map(str::trim) + .filter(|title| !title.is_empty()) + .unwrap_or("Unknown rule") + .to_string(); + let next_index = title_counts.len(); + let entry = title_counts.entry(title).or_insert((0, next_index)); + entry.0 += 1; + if let Some(ts) = event + .get("Timestamp") + .and_then(Value::as_str) + .and_then(parse_ts) + { + first_ts = Some(first_ts.map_or(ts, |old| old.min(ts))); + last_ts = Some(last_ts.map_or(ts, |old| old.max(ts))); + } + } + } + + let failed_logons = read_csv_rows(&report_dir.join("logon-summary-failed.csv"))?; + let successful_logons = read_csv_rows(&report_dir.join("logon-summary-successful.csv"))?; + let suspicious_pwsh = title_counts + .iter() + .filter(|(title, _)| { + let text = title.to_lowercase(); + text.contains("pwsh") || text.contains("powershell") || text.contains("obfuscation") + }) + .map(|(_, (count, _))| *count) + .sum(); + let credential_events = title_counts + .iter() + .filter(|(title, _)| title.to_lowercase().contains("credential")) + .map(|(_, (count, _))| *count) + .sum(); + let timestomp_events = title_counts + .iter() + .filter(|(title, _)| title.to_lowercase().contains("timestomp")) + .map(|(_, (count, _))| *count) + .sum(); + let logon_failure_events = title_counts + .iter() + .filter(|(title, _)| title.to_lowercase().contains("logon failure")) + .map(|(_, (count, _))| *count) + .sum(); + let score = level_counts + .iter() + .map(|(level, count)| level_weight(level) * (*count as i64)) + .sum::() + + (failed_logons.min(200) as i64) * 2 + + (suspicious_pwsh as i64) * 6 + + (credential_events as i64) * 8 + + (timestomp_events as i64) * 12 + + (logon_failure_events as i64) * 2; + + let crit_count = *level_counts.get("crit").unwrap_or(&0); + let high_count = *level_counts.get("high").unwrap_or(&0); + let med_count = *level_counts.get("med").unwrap_or(&0); + let severity = + if crit_count >= 1 || score >= 240 || (high_count >= 4 && suspicious_pwsh >= 4) { + "critical" + } else if high_count >= 1 || score >= 120 || suspicious_pwsh >= 8 || credential_events >= 5 + { + "high" + } else if med_count >= 1 || score >= 40 || failed_logons >= 10 { + "medium" + } else { + "low" + } + .to_string(); + + Ok(Summary { + severity, + score, + events_total: total_events, + level_counts: sort_map(level_counts), + top_rules: top_rules(title_counts), + first_timestamp: first_ts.map(format_ts), + last_timestamp: last_ts.map(format_ts), + failed_logon_rows: failed_logons, + successful_logon_rows: successful_logons, + suspicious_pwsh, + credential_events, + timestomp_events, + logon_failure_events, + }) +} + +pub fn build_case_payload(intake: &Value, summary: &Summary) -> Result { + Ok(json!({ + "incident_id": format!("hayabusa:{}:{}", required_str(intake, "host")?, required_str(intake, "intake_id")?), + "host": required_str(intake, "host")?, + "title": build_case_title(required_str(intake, "host")?, summary), + "severity": summary.severity, + "evidence": { + "hayabusa": { + "intake_id": required_str(intake, "intake_id")?, + "package_path": required_str(intake, "package_path")?, + "sha256": required_str(intake, "sha256")?, + "report_dir": required_str(intake, "report_dir")?, + "summary": serde_json::to_value(summary)?, + } + } + })) +} + +pub fn build_case_title(host: &str, summary: &Summary) -> String { + let suffix = summary + .top_rules + .first() + .map(|item| item.title.as_str()) + .unwrap_or("No dominant rule"); + format!( + "Hayabusa {} · {host} · {suffix}", + summary.severity.to_uppercase() + ) +} + +pub fn build_comment(summary: &Summary, intake: &Value) -> Result { + let top = summary + .top_rules + .iter() + .take(3) + .map(|item| format!("{} ({})", item.title, item.count)) + .collect::>() + .join(", "); + Ok(format!( + "Hayabusa auto-summary\nSeverity: {} (score={})\nHost: {}\nIntake: {}\nEvents: {}, failed_logons={}, suspicious_pwsh={}, credential_events={}\nTop rules: {}\nReport: {}", + summary.severity, + summary.score, + required_str(intake, "host")?, + required_str(intake, "intake_id")?, + summary.events_total, + summary.failed_logon_rows, + summary.suspicious_pwsh, + summary.credential_events, + if top.is_empty() { + "n/a".to_string() + } else { + top + }, + required_str(intake, "report_dir")?, + )) +} + +pub fn build_telegram_text( + case_id: Option, + intake: &Value, + summary: &Summary, +) -> Result { + let severity_label = match summary.severity.as_str() { + "critical" => "критичное событие", + "high" => "опасное событие", + "medium" => "подозрительное событие", + "low" => "слабый сигнал", + other => other, + }; + let top_rule = summary + .top_rules + .first() + .map(|item| item.title.as_str()) + .unwrap_or("нет явного доминирующего правила"); + let top_count = summary + .top_rules + .first() + .map(|item| item.count) + .unwrap_or(0); + let mut lines = vec![ + format!("Hayabusa: {severity_label}"), + String::new(), + format!("Хост: {}", required_str(intake, "host")?), + ]; + if let Some(case_id) = case_id { + lines.push(format!("Кейс: {case_id}")); + } + lines.extend([ + format!("Уровень: {}", summary.severity), + String::new(), + "Что найдено:".to_string(), + format!("- {top_rule}: {top_count}"), + format!("- подозрительный PowerShell: {}", summary.suspicious_pwsh), + format!("- ошибок входа: {}", summary.logon_failure_events), + format!("- событий по учётным данным: {}", summary.credential_events), + ]); + if summary.timestomp_events > 0 { + lines.push(format!( + "- timestomp-подобных событий: {}", + summary.timestomp_events + )); + } + lines.extend([ + String::new(), + "Главный риск:".to_string(), + "возможная активность вокруг учётных данных и PowerShell".to_string(), + String::new(), + "Отчёт:".to_string(), + required_str(intake, "report_dir")?.to_string(), + ]); + Ok(lines.join("\n")) +} + +pub fn severity_meets(actual: &str, threshold: &str) -> bool { + severity_order(actual) >= severity_order(threshold) +} + +pub fn env_bool(name: &str, default: bool) -> bool { + match std::env::var(name) { + Ok(value) => matches!( + value.trim().to_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ), + Err(_) => default, + } +} + +pub fn env_string(name: &str, default: &str) -> String { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| default.to_string()) +} + +pub fn required_str<'a>(value: &'a Value, key: &str) -> Result<&'a str> { + value + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| anyhow!("{key} missing")) +} + +pub fn windows_filename(path: &str) -> String { + path.rsplit(['\\', '/']) + .next() + .filter(|name| !name.is_empty()) + .unwrap_or(path) + .to_string() +} + +pub fn guess_host_from_filename(filename: &str) -> String { + let stem = filename.strip_suffix(".zip").unwrap_or(filename); + stem.split_once('-') + .map(|(host, _)| host) + .unwrap_or(stem) + .to_string() +} + +fn read_csv_rows(path: &Path) -> Result { + if !path.is_file() { + return Ok(0); + } + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let rows = text.lines().count(); + Ok(rows.saturating_sub(1)) +} + +fn normalize_level(level: Option<&str>) -> String { + match level.unwrap_or("").trim().to_lowercase().as_str() { + "" => "info", + "informational" | "info" => "info", + "low" => "low", + "med" | "medium" => "med", + "high" => "high", + "crit" | "critical" => "crit", + other => other, + } + .to_string() +} + +fn level_weight(level: &str) -> i64 { + match level { + "info" => 1, + "low" => 4, + "med" => 12, + "high" => 40, + "crit" => 100, + _ => 0, + } +} + +fn severity_order(value: &str) -> i64 { + SEVERITY_ORDER + .iter() + .find_map(|(name, order)| (*name == value).then_some(*order)) + .unwrap_or(0) +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .map(|ts| ts.with_timezone(&Utc)) + .ok() +} + +fn format_ts(value: DateTime) -> String { + value.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true) +} + +fn sort_map(values: HashMap) -> BTreeMap { + values.into_iter().collect() +} + +fn top_rules(values: HashMap) -> Vec { + let mut rows = values.into_iter().collect::>(); + rows.sort_by(|a, b| b.1.0.cmp(&a.1.0).then_with(|| a.1.1.cmp(&b.1.1))); + rows.into_iter() + .take(5) + .map(|(title, (count, _))| TopRule { title, count }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn severity_rules_match_expected_thresholds() { + assert!(severity_meets("high", "medium")); + assert!(!severity_meets("low", "medium")); + } + + #[test] + fn analyzes_timeline_and_csv_counts() { + let dir = tempdir().unwrap(); + fs::write( + dir.path().join("timeline.jsonl"), + r#"{"Level":"high","RuleTitle":"PowerShell Credential Dump","Timestamp":"2026-06-01T00:00:00Z"}"#, + ) + .unwrap(); + fs::write(dir.path().join("logon-summary-failed.csv"), "h\n1\n2\n").unwrap(); + let summary = analyze_report(dir.path()).unwrap(); + assert_eq!(summary.severity, "high"); + assert_eq!(summary.events_total, 1); + assert_eq!(summary.failed_logon_rows, 2); + assert_eq!(summary.credential_events, 1); + assert_eq!(summary.suspicious_pwsh, 1); + } +} diff --git a/adk-rust/crates/merge-aw-server-dbs/Cargo.toml b/adk-rust/crates/merge-aw-server-dbs/Cargo.toml new file mode 100644 index 0000000..a0cfe70 --- /dev/null +++ b/adk-rust/crates/merge-aw-server-dbs/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "merge-aw-server-dbs" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +rusqlite = { workspace = true, features = ["backup"] } +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/merge-aw-server-dbs/src/main.rs b/adk-rust/crates/merge-aw-server-dbs/src/main.rs new file mode 100644 index 0000000..1a13c3a --- /dev/null +++ b/adk-rust/crates/merge-aw-server-dbs/src/main.rs @@ -0,0 +1,376 @@ +use std::collections::{HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use rusqlite::{Connection, DatabaseName, OptionalExtension, Row, params}; +use serde_json::json; + +#[derive(Debug, Parser)] +#[command(about = "Merge ActivityWatch aw-server-rust SQLite databases")] +struct Cli { + #[arg(long)] + base: PathBuf, + + #[arg(long)] + output: PathBuf, + + #[arg(long)] + overlay: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct BucketKey { + name: String, + type_: String, + client: String, + hostname: String, +} + +#[derive(Debug, Clone)] +struct Bucket { + rowid: i64, + id: Option, + key: BucketKey, + created: i64, + data_deprecated: Option, + data: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct EventKey { + starttime: i64, + endtime: i64, + data: String, +} + +#[derive(Debug, Default)] +struct MergeStats { + inserted_buckets: usize, + inserted_events: usize, +} + +fn main() { + let code = match run() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + if !cli.base.is_file() { + bail!("Base DB not found: {}", cli.base.display()); + } + ensure_parent(&cli.output)?; + let tmp_output = tmp_output_path(&cli.output); + if tmp_output.exists() { + fs::remove_file(&tmp_output) + .with_context(|| format!("remove stale tmp output {}", tmp_output.display()))?; + } + copy_sqlite_via_backup(&cli.base, &tmp_output)?; + + let mut stats = MergeStats::default(); + if let Some(overlay) = &cli.overlay { + if overlay.exists() { + stats = merge_overlay(&tmp_output, overlay)?; + } + } + + fs::rename(&tmp_output, &cli.output) + .with_context(|| format!("move {} to {}", tmp_output.display(), cli.output.display()))?; + println!( + "{}", + json!({ + "base": cli.base.to_string_lossy(), + "overlay": cli.overlay.as_ref().map(|p| p.to_string_lossy().to_string()), + "output": cli.output.to_string_lossy(), + "inserted_buckets": stats.inserted_buckets, + "inserted_events": stats.inserted_events, + }) + ); + Ok(()) +} + +fn copy_sqlite_via_backup(src: &Path, dst: &Path) -> Result<()> { + ensure_parent(dst)?; + let source = + Connection::open(src).with_context(|| format!("open base DB {}", src.display()))?; + source + .backup(DatabaseName::Main, dst, None) + .with_context(|| format!("backup {} to {}", src.display(), dst.display())) +} + +fn merge_overlay(dest_path: &Path, overlay_path: &Path) -> Result { + let dest = + connect(dest_path).with_context(|| format!("open output DB {}", dest_path.display()))?; + let source = connect(overlay_path) + .with_context(|| format!("open overlay DB {}", overlay_path.display()))?; + + let mut stats = MergeStats::default(); + let source_buckets = load_buckets(&source)?; + let mut dest_bucket_map = load_dest_bucket_key_map(&dest)?; + let mut dest_id_map = load_dest_bucket_id_map(&dest)?; + + for src_bucket in source_buckets { + let dest_rowid = if let Some(src_id) = &src_bucket.id { + dest_id_map.get(src_id).copied() + } else { + None + } + .or_else(|| dest_bucket_map.get(&src_bucket.key).copied()) + .or_else(|| { + find_bucket_by_name(&dest, &src_bucket.key.name) + .ok() + .flatten() + }); + + let dest_rowid = match dest_rowid { + Some(rowid) => { + update_bucket_by_rowid(&dest, rowid, &src_bucket)?; + rowid + } + None => { + stats.inserted_buckets += 1; + insert_bucket(&dest, &src_bucket)? + } + }; + + dest_bucket_map.insert(src_bucket.key.clone(), dest_rowid); + if let Some(id) = &src_bucket.id { + dest_id_map.insert(id.clone(), dest_rowid); + } + + let mut existing_events = load_existing_events(&dest, dest_rowid)?; + let source_events = load_source_events(&source, src_bucket.rowid)?; + for event in source_events { + if existing_events.contains(&event) { + continue; + } + dest.execute( + "insert into events (bucketrow, starttime, endtime, data) values (?1, ?2, ?3, ?4)", + params![dest_rowid, event.starttime, event.endtime, event.data], + ) + .with_context(|| format!("insert event into bucketrow {dest_rowid}"))?; + existing_events.insert(event); + stats.inserted_events += 1; + } + } + dest.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .context("checkpoint output DB")?; + Ok(stats) +} + +fn connect(path: &Path) -> Result { + let connection = Connection::open(path)?; + connection.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;")?; + Ok(connection) +} + +fn load_buckets(connection: &Connection) -> Result> { + let mut stmt = connection.prepare( + "select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid", + )?; + let rows = stmt.query_map([], bucket_from_row)?; + rows.collect::>>() + .context("load buckets") +} + +fn bucket_from_row(row: &Row<'_>) -> rusqlite::Result { + Ok(Bucket { + rowid: row.get("bucketrow")?, + id: row.get("id")?, + key: BucketKey { + name: row.get("name")?, + type_: row.get("type")?, + client: row.get("client")?, + hostname: row.get("hostname")?, + }, + created: row.get("created")?, + data_deprecated: row.get("data_deprecated")?, + data: row.get("data")?, + }) +} + +fn load_dest_bucket_key_map(connection: &Connection) -> Result> { + Ok(load_buckets(connection)? + .into_iter() + .map(|bucket| (bucket.key, bucket.rowid)) + .collect()) +} + +fn load_dest_bucket_id_map(connection: &Connection) -> Result> { + Ok(load_buckets(connection)? + .into_iter() + .filter_map(|bucket| bucket.id.map(|id| (id, bucket.rowid))) + .collect()) +} + +fn find_bucket_by_name(connection: &Connection, name: &str) -> Result> { + connection + .query_row( + "select rowid as bucketrow from buckets where name = ?1 order by rowid limit 1", + [name], + |row| row.get::<_, i64>("bucketrow"), + ) + .optional() + .context("find bucket by name") +} + +fn insert_bucket(connection: &Connection, bucket: &Bucket) -> Result { + connection.execute( + "insert into buckets (id, name, type, client, hostname, created, data_deprecated, data) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + bucket.id, + bucket.key.name, + bucket.key.type_, + bucket.key.client, + bucket.key.hostname, + bucket.created, + bucket.data_deprecated, + bucket.data + ], + )?; + Ok(connection.last_insert_rowid()) +} + +fn update_bucket_by_rowid(connection: &Connection, rowid: i64, bucket: &Bucket) -> Result<()> { + connection.execute( + "update buckets set type = ?1, client = ?2, hostname = ?3, created = ?4, data_deprecated = ?5, data = ?6 where rowid = ?7", + params![ + bucket.key.type_, + bucket.key.client, + bucket.key.hostname, + bucket.created, + bucket.data_deprecated, + bucket.data, + rowid + ], + )?; + Ok(()) +} + +fn load_existing_events(connection: &Connection, bucketrow: i64) -> Result> { + let mut stmt = + connection.prepare("select starttime, endtime, data from events where bucketrow = ?1")?; + let rows = stmt.query_map([bucketrow], |row| { + Ok(EventKey { + starttime: row.get(0)?, + endtime: row.get(1)?, + data: row.get(2)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +fn load_source_events(connection: &Connection, bucketrow: i64) -> Result> { + let mut stmt = connection + .prepare("select starttime, endtime, data from events where bucketrow = ?1 order by id")?; + let rows = stmt.query_map([bucketrow], |row| { + Ok(EventKey { + starttime: row.get(0)?, + endtime: row.get(1)?, + data: row.get(2)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +fn ensure_parent(path: &Path) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create parent directory {}", parent.display()))?; + } + Ok(()) +} + +fn tmp_output_path(output: &Path) -> PathBuf { + let mut os = output.as_os_str().to_os_string(); + os.push(".tmp"); + PathBuf::from(os) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tmp_output_appends_tmp_like_python_suffix() { + assert_eq!( + tmp_output_path(Path::new("/tmp/sqlite.db")), + PathBuf::from("/tmp/sqlite.db.tmp") + ); + } + + #[test] + fn merge_inserts_new_bucket_and_skips_duplicate_event() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().join("base.db"); + let overlay = dir.path().join("overlay.db"); + let output = dir.path().join("out.db"); + create_fixture_db(&base, &[("bucket-a", "id-a")], &[(1, 10, 20, r#"{"a":1}"#)]); + create_fixture_db( + &overlay, + &[("bucket-a", "id-a"), ("bucket-b", "id-b")], + &[(1, 10, 20, r#"{"a":1}"#), (2, 30, 40, r#"{"b":1}"#)], + ); + + copy_sqlite_via_backup(&base, &output).unwrap(); + let stats = merge_overlay(&output, &overlay).unwrap(); + assert_eq!(stats.inserted_buckets, 1); + assert_eq!(stats.inserted_events, 1); + + let conn = Connection::open(&output).unwrap(); + let bucket_count: i64 = conn + .query_row("select count(*) from buckets", [], |row| row.get(0)) + .unwrap(); + let event_count: i64 = conn + .query_row("select count(*) from events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(bucket_count, 2); + assert_eq!(event_count, 2); + } + + fn create_fixture_db(path: &Path, buckets: &[(&str, &str)], events: &[(i64, i64, i64, &str)]) { + let conn = Connection::open(path).unwrap(); + conn.execute_batch( + "create table buckets ( + id text, + name text unique, + type text, + client text, + hostname text, + created integer, + data_deprecated text, + data text + ); + create table events ( + id integer primary key autoincrement, + bucketrow integer, + starttime integer, + endtime integer, + data text + );", + ) + .unwrap(); + for (name, id) in buckets { + conn.execute( + "insert into buckets (id, name, type, client, hostname, created, data_deprecated, data) values (?1, ?2, 'type', 'client', 'host', 1, null, '{}')", + params![id, name], + ) + .unwrap(); + } + for (bucketrow, starttime, endtime, data) in events { + conn.execute( + "insert into events (bucketrow, starttime, endtime, data) values (?1, ?2, ?3, ?4)", + params![bucketrow, starttime, endtime, data], + ) + .unwrap(); + } + } +} diff --git a/adk-rust/crates/prod-backup-restore/Cargo.toml b/adk-rust/crates/prod-backup-restore/Cargo.toml new file mode 100644 index 0000000..2d08c0e --- /dev/null +++ b/adk-rust/crates/prod-backup-restore/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "prod-backup-restore" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/prod-backup-restore/src/main.rs b/adk-rust/crates/prod-backup-restore/src/main.rs new file mode 100644 index 0000000..1d4a00f --- /dev/null +++ b/adk-rust/crates/prod-backup-restore/src/main.rs @@ -0,0 +1,476 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use chrono::Local; +use clap::Parser; +use serde::Serialize; + +const DEFAULT_SERVER_HOST: &str = "10.10.10.13"; +const DEFAULT_SERVER_USER: &str = "igor"; +const DEFAULT_LEGACY_DB: &str = "/root/.local/share/activitywatch/aw-server-rust/sqlite.db"; +const DEFAULT_TARGET_DB: &str = + "/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db"; +const DEFAULT_REMOTE_MERGE_SCRIPT: &str = "/tmp/merge_aw_server_dbs.py"; + +#[derive(Debug, Parser)] +#[command(about = "Safe planner/checker for the destructive prod backup restore flow")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long)] + check_inputs: bool, + + #[arg(long)] + json: bool, + + #[arg(long)] + timestamp: Option, + + #[arg(long)] + server_host: Option, + + #[arg(long)] + server_user: Option, + + #[arg(long, default_value = "ansible/inventory.ini")] + inventory: PathBuf, + + #[arg(long, default_value = DEFAULT_LEGACY_DB)] + legacy_db: String, + + #[arg(long, default_value = DEFAULT_TARGET_DB)] + target_db: String, + + #[arg(long, default_value = DEFAULT_REMOTE_MERGE_SCRIPT)] + remote_merge_script: String, + + #[arg(long)] + apply: bool, +} + +#[derive(Debug, Serialize)] +struct Plan { + mode: &'static str, + root: String, + env_file: String, + server_host: String, + server_user: String, + timestamp: String, + remote_backup_dir: String, + legacy_db: String, + target_db: String, + remote_merge_script: String, + required_env: Vec, + required_commands: Vec, + required_files: Vec, + steps: Vec, + missing_count: usize, +} + +#[derive(Debug, Serialize)] +struct Requirement { + name: String, + ok: bool, + detail: String, +} + +#[derive(Debug, Serialize)] +struct Step { + order: usize, + kind: &'static str, + command: String, + destructive: bool, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + if cli.apply { + bail!( + "Rust apply is intentionally disabled for this stage; use scripts/prod_backup_restore.sh --apply-legacy only for explicit legacy restore" + ); + } + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + let env_file = root.join("secrets/runtime.env"); + let env_values = read_env_file(&env_file).unwrap_or_default(); + let timestamp = cli + .timestamp + .clone() + .unwrap_or_else(|| Local::now().format("%Y%m%d-%H%M%S").to_string()); + let server_host = cli + .server_host + .or_else(|| env_value("AW_SERVER_HOST", &env_values)) + .unwrap_or_else(|| DEFAULT_SERVER_HOST.to_string()); + let server_user = cli + .server_user + .or_else(|| env_value("AW_SERVER_USER", &env_values)) + .unwrap_or_else(|| DEFAULT_SERVER_USER.to_string()); + let remote_backup_dir = format!("/var/lib/activitywatch/backups/prod-restore-{timestamp}"); + let inventory = if cli.inventory.is_absolute() { + cli.inventory.clone() + } else { + root.join(&cli.inventory) + }; + let merge_script = root.join("scripts/merge_aw_server_dbs.py"); + let plan = build_plan( + &root, + &env_file, + &env_values, + &server_host, + &server_user, + ×tamp, + &remote_backup_dir, + &cli.legacy_db, + &cli.target_db, + &cli.remote_merge_script, + &inventory, + &merge_script, + ); + + if cli.json { + println!("{}", serde_json::to_string_pretty(&plan)?); + } else { + print_plan(&plan); + } + + if cli.check_inputs && plan.missing_count > 0 { + Ok(2) + } else { + Ok(0) + } +} + +#[allow(clippy::too_many_arguments)] +fn build_plan( + root: &Path, + env_file: &Path, + env_values: &HashMap, + server_host: &str, + server_user: &str, + timestamp: &str, + remote_backup_dir: &str, + legacy_db: &str, + target_db: &str, + remote_merge_script: &str, + inventory: &Path, + merge_script: &Path, +) -> Plan { + let required_env = ["AW_SSH_PASSWORD", "AW_WINRM_PASSWORD"] + .into_iter() + .map(|name| { + let ok = env_value(name, env_values).is_some(); + Requirement { + name: name.to_string(), + ok, + detail: if ok { + "present (value hidden)".to_string() + } else { + "missing".to_string() + }, + } + }) + .collect::>(); + let required_commands = ["sshpass", "ansible-playbook"] + .into_iter() + .map(|name| Requirement { + name: name.to_string(), + ok: command_exists(name), + detail: if command_exists(name) { + "found in PATH".to_string() + } else { + "missing in PATH".to_string() + }, + }) + .collect::>(); + let required_files = [ + ("inventory", inventory), + ("merge_aw_server_dbs.py", merge_script), + ] + .into_iter() + .map(|(name, path)| Requirement { + name: name.to_string(), + ok: path.is_file(), + detail: path.display().to_string(), + }) + .collect::>(); + + let mut steps = Vec::new(); + push_step( + &mut steps, + "scp", + format!( + "sshpass scp scripts/merge_aw_server_dbs.py {server_user}@{server_host}:{remote_merge_script}" + ), + false, + ); + push_step( + &mut steps, + "ssh", + format!( + "sudo mkdir -p '{remote_backup_dir}' && sudo chown root:root '{remote_backup_dir}'" + ), + false, + ); + push_step( + &mut steps, + "ssh", + format!("sudo test -f '{legacy_db}'"), + false, + ); + push_step( + &mut steps, + "ssh", + format!("sudo test -f '{target_db}'"), + false, + ); + push_step( + &mut steps, + "ssh", + format!( + "sudo cp -a '{legacy_db}' '{remote_backup_dir}/legacy-root-sqlite.db' && sudo cp -a '{target_db}' '{remote_backup_dir}/target-before-merge-sqlite.db'" + ), + false, + ); + push_step( + &mut steps, + "ssh", + "sudo systemctl stop activitywatch-server.service || true".to_string(), + true, + ); + push_step( + &mut steps, + "ssh", + format!( + "sudo python3 '{remote_merge_script}' --base '{legacy_db}' --overlay '{target_db}' --output '{remote_backup_dir}/sqlite.merged.db'" + ), + true, + ); + push_step( + &mut steps, + "ssh", + format!( + "sudo install -o activitywatch -g activitywatch -m 0644 '{remote_backup_dir}/sqlite.merged.db' '{target_db}'" + ), + true, + ); + for playbook in [ + "ansible/deploy_aw_server.yml", + "ansible/deploy_aw_windows.yml", + "ansible/post_validate_aw_windows.yml", + ] { + push_step( + &mut steps, + "ansible-playbook", + format!("ansible-playbook -i {} {playbook}", inventory.display()), + true, + ); + } + push_step( + &mut steps, + "validate", + "query AW historical window data for 2026-04-29 and settings always_active_pattern" + .to_string(), + false, + ); + + let missing_count = required_env + .iter() + .chain(required_commands.iter()) + .chain(required_files.iter()) + .filter(|item| !item.ok) + .count(); + + Plan { + mode: "plan-only", + root: root.display().to_string(), + env_file: env_file.display().to_string(), + server_host: server_host.to_string(), + server_user: server_user.to_string(), + timestamp: timestamp.to_string(), + remote_backup_dir: remote_backup_dir.to_string(), + legacy_db: legacy_db.to_string(), + target_db: target_db.to_string(), + remote_merge_script: remote_merge_script.to_string(), + required_env, + required_commands, + required_files, + steps, + missing_count, + } +} + +fn push_step(steps: &mut Vec, kind: &'static str, command: String, destructive: bool) { + steps.push(Step { + order: steps.len() + 1, + kind, + command, + destructive, + }); +} + +fn print_plan(plan: &Plan) { + println!("prod-backup-restore: {}", plan.mode); + println!("root: {}", plan.root); + println!("server: {}@{}", plan.server_user, plan.server_host); + println!("remote_backup_dir: {}", plan.remote_backup_dir); + println!("missing_inputs: {}", plan.missing_count); + println!(); + println!("Required env:"); + for item in &plan.required_env { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!("Required commands:"); + for item in &plan.required_commands { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!("Required files:"); + for item in &plan.required_files { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!(); + println!("Planned steps, not executed:"); + for step in &plan.steps { + let risk = if step.destructive { + "MUTATION" + } else { + "read/prepare" + }; + println!( + " {:02}. {:<16} {:<12} {}", + step.order, step.kind, risk, step.command + ); + } +} + +fn ok_mark(ok: bool) -> &'static str { + if ok { "OK" } else { "MISS" } +} + +fn env_value(name: &str, env_values: &HashMap) -> Option { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| env_values.get(name).cloned()) + .filter(|value| !value.trim().is_empty()) +} + +fn read_env_file(path: &Path) -> Result> { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let mut values = HashMap::new(); + for raw_line in text.lines() { + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') || !line.contains('=') { + continue; + } + let mut parts = line.splitn(2, '='); + let Some(key) = parts.next().map(str::trim) else { + continue; + }; + let Some(value) = parts.next().map(str::trim) else { + continue; + }; + if key.is_empty() || key.starts_with("export ") { + continue; + } + values.insert(key.to_string(), strip_shell_quotes(value).to_string()); + } + Ok(values) +} + +fn strip_shell_quotes(value: &str) -> &str { + if value.len() >= 2 + && ((value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\''))) + { + &value[1..value.len() - 1] + } else { + value + } +} + +fn command_exists(name: &str) -> bool { + Command::new("sh") + .arg("-c") + .arg(format!("command -v {name} >/dev/null 2>&1")) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_simple_shell_quotes() { + assert_eq!(strip_shell_quotes("'secret'"), "secret"); + assert_eq!(strip_shell_quotes("\"secret\""), "secret"); + assert_eq!(strip_shell_quotes("plain"), "plain"); + } + + #[test] + fn reads_simple_env_file_without_exposing_values() { + let dir = tempfile::tempdir().unwrap(); + let env = dir.path().join("runtime.env"); + fs::write( + &env, + "AW_SSH_PASSWORD='one'\nAW_WINRM_PASSWORD=\"two\"\n# ignored\n", + ) + .unwrap(); + let values = read_env_file(&env).unwrap(); + assert_eq!(values.get("AW_SSH_PASSWORD").unwrap(), "one"); + assert_eq!(values.get("AW_WINRM_PASSWORD").unwrap(), "two"); + } + + #[test] + fn plan_marks_destructive_steps() { + let dir = tempfile::tempdir().unwrap(); + let inventory = dir.path().join("ansible/inventory.ini"); + let merge = dir.path().join("scripts/merge_aw_server_dbs.py"); + fs::create_dir_all(inventory.parent().unwrap()).unwrap(); + fs::create_dir_all(merge.parent().unwrap()).unwrap(); + fs::write(&inventory, "").unwrap(); + fs::write(&merge, "").unwrap(); + let mut env = HashMap::new(); + env.insert("AW_SSH_PASSWORD".to_string(), "hidden".to_string()); + env.insert("AW_WINRM_PASSWORD".to_string(), "hidden".to_string()); + let plan = build_plan( + dir.path(), + &dir.path().join("secrets/runtime.env"), + &env, + "10.10.10.13", + "igor", + "20260602-000000", + "/var/lib/activitywatch/backups/prod-restore-20260602-000000", + DEFAULT_LEGACY_DB, + DEFAULT_TARGET_DB, + DEFAULT_REMOTE_MERGE_SCRIPT, + &inventory, + &merge, + ); + assert!(plan.steps.iter().any(|step| step.destructive)); + assert!( + plan.steps + .iter() + .any(|step| step.command.contains("systemctl stop activitywatch-server")) + ); + assert_eq!(plan.required_env.iter().filter(|item| item.ok).count(), 2); + } +} diff --git a/adk-rust/crates/prod-rollout/Cargo.toml b/adk-rust/crates/prod-rollout/Cargo.toml new file mode 100644 index 0000000..1ead670 --- /dev/null +++ b/adk-rust/crates/prod-rollout/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "prod-rollout" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/prod-rollout/src/main.rs b/adk-rust/crates/prod-rollout/src/main.rs new file mode 100644 index 0000000..eb1844e --- /dev/null +++ b/adk-rust/crates/prod-rollout/src/main.rs @@ -0,0 +1,662 @@ +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use chrono::Local; +use clap::Parser; +use serde::Serialize; + +#[derive(Debug, Parser)] +#[command(about = "Safe planner/orchestrator for the production ActivityWatch rollout")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long, default_value = "ansible/inventory.ini")] + inventory: PathBuf, + + #[arg(long)] + check_inputs: bool, + + #[arg(long)] + json: bool, + + #[arg(long)] + apply: bool, + + #[arg(long)] + skip_quality_gate: bool, + + #[arg(long)] + timestamp: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct Requirement { + name: String, + ok: bool, + detail: String, +} + +#[derive(Debug, Clone, Serialize)] +struct Step { + order: usize, + name: String, + command: Vec, + log_file: String, + mutation: bool, +} + +#[derive(Debug, Serialize)] +struct Plan { + mode: &'static str, + root: String, + branch: String, + env_file: String, + inventory: String, + log_dir: String, + required_env: Vec, + required_commands: Vec, + required_files: Vec, + steps: Vec, + missing_count: usize, +} + +#[derive(Debug, Serialize)] +struct StepResult { + order: usize, + name: String, + status: &'static str, + exit_code: Option, + log_file: String, +} + +#[derive(Debug, Serialize)] +struct ApplyReport { + plan: Plan, + results: Vec, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + let timestamp = cli + .timestamp + .clone() + .unwrap_or_else(|| Local::now().format("%Y%m%d-%H%M%S").to_string()); + let env_file = root.join("secrets/runtime.env"); + let env_values = read_env_file(&env_file).unwrap_or_default(); + let inventory = absolute_path(&root, &cli.inventory); + let log_dir = root.join(".rollout-logs").join(×tamp); + let plan = build_plan( + &root, + &env_file, + &env_values, + &inventory, + &log_dir, + cli.apply, + cli.skip_quality_gate, + ); + + if !cli.apply { + if cli.json { + println!("{}", serde_json::to_string_pretty(&plan)?); + } else { + print_plan(&plan); + } + return if cli.check_inputs && plan.missing_count > 0 { + Ok(2) + } else { + Ok(0) + }; + } + + if plan.missing_count > 0 { + if cli.json { + println!("{}", serde_json::to_string_pretty(&plan)?); + } else { + print_plan(&plan); + } + bail!("refusing --apply because required inputs are missing"); + } + + let report = apply_plan(plan, env_values)?; + if cli.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print_apply_report(&report); + } + Ok(if report.results.iter().all(|item| item.status == "ok") { + 0 + } else { + 1 + }) +} + +fn absolute_path(root: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_path_buf() + } else { + root.join(path) + } +} + +fn build_plan( + root: &Path, + env_file: &Path, + env_values: &HashMap, + inventory: &Path, + log_dir: &Path, + apply: bool, + skip_quality_gate: bool, +) -> Plan { + let required_env = ["AW_SSH_PASSWORD", "AW_WINRM_PASSWORD"] + .into_iter() + .map(|name| { + let ok = env_value(name, env_values).is_some(); + Requirement { + name: name.to_string(), + ok, + detail: if ok { + "present (value hidden)".to_string() + } else { + "missing".to_string() + }, + } + }) + .collect::>(); + + let required_commands = ["git", "ansible", "ansible-playbook"] + .into_iter() + .map(|name| { + let ok = command_exists(name); + Requirement { + name: name.to_string(), + ok, + detail: if ok { + "found in PATH".to_string() + } else { + "missing in PATH".to_string() + }, + } + }) + .collect::>(); + + let quality_gate = root.join("scripts/quality-gate.sh"); + let deploy_aw_server = root.join("ansible/deploy_aw_server.yml"); + let deploy_aw_windows = root.join("ansible/deploy_aw_windows.yml"); + let post_validate_aw_windows = root.join("ansible/post_validate_aw_windows.yml"); + let required_files = [ + ("inventory", inventory), + ("scripts/quality-gate.sh", &quality_gate), + ("ansible/deploy_aw_server.yml", &deploy_aw_server), + ("ansible/deploy_aw_windows.yml", &deploy_aw_windows), + ( + "ansible/post_validate_aw_windows.yml", + &post_validate_aw_windows, + ), + ] + .into_iter() + .map(|(name, path)| Requirement { + name: name.to_string(), + ok: path.is_file(), + detail: path.display().to_string(), + }) + .collect::>(); + + let mut steps = Vec::new(); + if !skip_quality_gate { + push_step( + &mut steps, + "quality-gate", + vec!["./scripts/quality-gate.sh".to_string()], + "quality-gate.log", + false, + ); + } + push_step( + &mut steps, + "ping-aw-server", + vec![ + "ansible".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "aw_server".to_string(), + "-m".to_string(), + "ping".to_string(), + ], + "ping_aw_server.log", + false, + ); + push_step( + &mut steps, + "ping-aw-windows", + vec![ + "ansible".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "aw_windows".to_string(), + "-m".to_string(), + "win_ping".to_string(), + ], + "ping_aw_windows.log", + false, + ); + push_step( + &mut steps, + "check-aw-server", + vec![ + "ansible-playbook".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "ansible/deploy_aw_server.yml".to_string(), + "--check".to_string(), + "--diff".to_string(), + ], + "check_aw_server.log", + false, + ); + push_step( + &mut steps, + "deploy-aw-server", + vec![ + "ansible-playbook".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "ansible/deploy_aw_server.yml".to_string(), + ], + "deploy_aw_server.log", + true, + ); + push_step( + &mut steps, + "check-aw-windows", + vec![ + "ansible-playbook".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "ansible/deploy_aw_windows.yml".to_string(), + "--check".to_string(), + "--diff".to_string(), + ], + "check_aw_windows.log", + false, + ); + push_step( + &mut steps, + "deploy-aw-windows", + vec![ + "ansible-playbook".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "ansible/deploy_aw_windows.yml".to_string(), + ], + "deploy_aw_windows.log", + true, + ); + push_step( + &mut steps, + "post-validate-aw-windows", + vec![ + "ansible-playbook".to_string(), + "-i".to_string(), + "ansible/inventory.ini".to_string(), + "ansible/post_validate_aw_windows.yml".to_string(), + ], + "post_validate_aw_windows.log", + false, + ); + + let missing_count = required_env + .iter() + .chain(required_commands.iter()) + .chain(required_files.iter()) + .filter(|item| !item.ok) + .count(); + + Plan { + mode: if apply { "apply" } else { "plan-only" }, + root: root.display().to_string(), + branch: git_branch(root).unwrap_or_else(|| "unknown".to_string()), + env_file: env_file.display().to_string(), + inventory: inventory.display().to_string(), + log_dir: log_dir.display().to_string(), + required_env, + required_commands, + required_files, + steps, + missing_count, + } +} + +fn push_step( + steps: &mut Vec, + name: &str, + command: Vec, + log_file: &str, + mutation: bool, +) { + steps.push(Step { + order: steps.len() + 1, + name: name.to_string(), + command, + log_file: log_file.to_string(), + mutation, + }); +} + +fn apply_plan(plan: Plan, env_values: HashMap) -> Result { + let log_dir = PathBuf::from(&plan.log_dir); + fs::create_dir_all(&log_dir).with_context(|| format!("create {}", log_dir.display()))?; + let mut results = Vec::new(); + for step in &plan.steps { + log_line(&log_dir, &format!("START {} {}", step.order, step.name))?; + let result = run_step(&plan, step, &env_values)?; + log_line( + &log_dir, + &format!( + "END {} {} status={} exit={:?}", + step.order, step.name, result.status, result.exit_code + ), + )?; + let ok = result.status == "ok"; + results.push(result); + if !ok { + break; + } + } + Ok(ApplyReport { plan, results }) +} + +fn run_step(plan: &Plan, step: &Step, env_values: &HashMap) -> Result { + let Some(program) = step.command.first() else { + bail!("empty command for step {}", step.name); + }; + let args = &step.command[1..]; + let output = Command::new(program) + .args(args) + .current_dir(&plan.root) + .envs(env_values) + .output() + .with_context(|| format!("run {}", shell_join(&step.command)))?; + let log_file = PathBuf::from(&plan.log_dir).join(&step.log_file); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&log_file) + .with_context(|| format!("open {}", log_file.display()))?; + file.write_all(&output.stdout)?; + file.write_all(&output.stderr)?; + std::io::stdout().write_all(&output.stdout)?; + std::io::stderr().write_all(&output.stderr)?; + Ok(StepResult { + order: step.order, + name: step.name.clone(), + status: if output.status.success() { + "ok" + } else { + "failed" + }, + exit_code: output.status.code(), + log_file: log_file.display().to_string(), + }) +} + +fn print_plan(plan: &Plan) { + println!("prod-rollout: {}", plan.mode); + println!("root: {}", plan.root); + println!("branch: {}", plan.branch); + println!("inventory: {}", plan.inventory); + println!("log_dir: {}", plan.log_dir); + println!("missing_inputs: {}", plan.missing_count); + println!(); + println!("Required env:"); + for item in &plan.required_env { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!("Required commands:"); + for item in &plan.required_commands { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!("Required files:"); + for item in &plan.required_files { + println!(" [{}] {} - {}", ok_mark(item.ok), item.name, item.detail); + } + println!(); + println!("Planned steps:"); + for step in &plan.steps { + let risk = if step.mutation { "MUTATION" } else { "check" }; + println!( + " {:02}. {:<24} {:<8} {}", + step.order, + step.name, + risk, + shell_join(&step.command) + ); + } + if plan.mode == "plan-only" { + println!(); + println!("No steps executed. Use --apply for explicit rollout execution."); + } +} + +fn print_apply_report(report: &ApplyReport) { + println!("prod-rollout apply result:"); + println!("log_dir: {}", report.plan.log_dir); + for item in &report.results { + println!( + " {:02}. {:<24} {:<6} exit={:?} log={}", + item.order, item.name, item.status, item.exit_code, item.log_file + ); + } +} + +fn log_line(log_dir: &Path, line: &str) -> Result<()> { + fs::create_dir_all(log_dir).with_context(|| format!("create {}", log_dir.display()))?; + let path = log_dir.join("rollout.log"); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .with_context(|| format!("open {}", path.display()))?; + writeln!( + file, + "{} {}", + Local::now().format("%Y-%m-%d %H:%M:%S"), + line + )?; + Ok(()) +} + +fn ok_mark(ok: bool) -> &'static str { + if ok { "OK" } else { "MISS" } +} + +fn env_value(name: &str, env_values: &HashMap) -> Option { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| env_values.get(name).cloned()) + .filter(|value| !value.trim().is_empty()) +} + +fn read_env_file(path: &Path) -> Result> { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + let mut values = HashMap::new(); + for raw_line in text.lines() { + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') || !line.contains('=') { + continue; + } + let mut parts = line.splitn(2, '='); + let Some(key) = parts.next().map(str::trim) else { + continue; + }; + let Some(value) = parts.next().map(str::trim) else { + continue; + }; + if key.is_empty() { + continue; + } + let key = key.strip_prefix("export ").unwrap_or(key).trim(); + if key.is_empty() { + continue; + } + values.insert(key.to_string(), strip_shell_quotes(value).to_string()); + } + Ok(values) +} + +fn strip_shell_quotes(value: &str) -> &str { + if value.len() >= 2 + && ((value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\''))) + { + &value[1..value.len() - 1] + } else { + value + } +} + +fn command_exists(name: &str) -> bool { + Command::new("sh") + .arg("-c") + .arg(format!("command -v {name} >/dev/null 2>&1")) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn git_branch(root: &Path) -> Option { + let output = Command::new("git") + .args(["branch", "--show-current"]) + .current_dir(root) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let branch = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if branch.is_empty() { + None + } else { + Some(branch) + } +} + +fn shell_join(command: &[String]) -> String { + command + .iter() + .map(|part| { + if part + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || "-_./:=,".contains(ch)) + { + part.clone() + } else { + format!("'{}'", part.replace('\'', "'\\''")) + } + }) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_exported_env_and_strips_quotes() { + let dir = tempfile::tempdir().unwrap(); + let env = dir.path().join("runtime.env"); + fs::write( + &env, + "export AW_SSH_PASSWORD='one'\nAW_WINRM_PASSWORD=\"two\"\n", + ) + .unwrap(); + let values = read_env_file(&env).unwrap(); + assert_eq!(values.get("AW_SSH_PASSWORD").unwrap(), "one"); + assert_eq!(values.get("AW_WINRM_PASSWORD").unwrap(), "two"); + } + + #[test] + fn plan_marks_rollout_mutations() { + let dir = tempfile::tempdir().unwrap(); + create_file(dir.path().join("scripts/quality-gate.sh")); + create_file(dir.path().join("ansible/inventory.ini")); + create_file(dir.path().join("ansible/deploy_aw_server.yml")); + create_file(dir.path().join("ansible/deploy_aw_windows.yml")); + create_file(dir.path().join("ansible/post_validate_aw_windows.yml")); + let mut env = HashMap::new(); + env.insert("AW_SSH_PASSWORD".to_string(), "hidden".to_string()); + env.insert("AW_WINRM_PASSWORD".to_string(), "hidden".to_string()); + let plan = build_plan( + dir.path(), + &dir.path().join("secrets/runtime.env"), + &env, + &dir.path().join("ansible/inventory.ini"), + &dir.path().join(".rollout-logs/test"), + false, + false, + ); + assert!( + plan.steps + .iter() + .any(|step| step.name == "deploy-aw-server") + ); + assert!( + plan.steps + .iter() + .any(|step| step.name == "deploy-aw-windows") + ); + assert!( + plan.steps + .iter() + .filter(|step| step.mutation) + .all(|step| step.name.starts_with("deploy-")) + ); + assert_eq!(plan.required_env.iter().filter(|item| item.ok).count(), 2); + } + + #[test] + fn skip_quality_gate_removes_first_step() { + let dir = tempfile::tempdir().unwrap(); + create_file(dir.path().join("scripts/quality-gate.sh")); + create_file(dir.path().join("ansible/inventory.ini")); + create_file(dir.path().join("ansible/deploy_aw_server.yml")); + create_file(dir.path().join("ansible/deploy_aw_windows.yml")); + create_file(dir.path().join("ansible/post_validate_aw_windows.yml")); + let plan = build_plan( + dir.path(), + &dir.path().join("secrets/runtime.env"), + &HashMap::new(), + &dir.path().join("ansible/inventory.ini"), + &dir.path().join(".rollout-logs/test"), + false, + true, + ); + assert_ne!(plan.steps.first().unwrap().name, "quality-gate"); + } + + fn create_file(path: PathBuf) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "").unwrap(); + } +} diff --git a/adk-rust/crates/quality-gate/Cargo.toml b/adk-rust/crates/quality-gate/Cargo.toml new file mode 100644 index 0000000..0ddf0ba --- /dev/null +++ b/adk-rust/crates/quality-gate/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "quality-gate" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/quality-gate/src/main.rs b/adk-rust/crates/quality-gate/src/main.rs new file mode 100644 index 0000000..f7d0fc1 --- /dev/null +++ b/adk-rust/crates/quality-gate/src/main.rs @@ -0,0 +1,236 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; + +#[derive(Debug, Parser)] +#[command(about = "Run ActivityWatch-Russian repository quality gates")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, +} + +fn main() { + let code = match run() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + quality_gate(&root) +} + +fn quality_gate(root: &Path) -> Result<()> { + println!("[1/5] Bash syntax check"); + for file in collect_by_extension(root, &["aw-server", "proxmox", "scripts"], "sh")? { + run_status(Command::new("bash").arg("-n").arg(&file), false)?; + } + + println!("[2/5] Shellcheck (if available)"); + if command_exists("shellcheck") { + let mut files = collect_by_extension(root, &["aw-server", "proxmox"], "sh")?; + files.push(root.join("scripts/aw-webui-browser-smoke.sh")); + let mut command = Command::new("shellcheck"); + command.arg("-e").arg("SC1007,SC1090,SC2016"); + for file in files.into_iter().filter(|path| path.is_file()) { + command.arg(file); + } + run_status(&mut command, false)?; + } else { + println!("shellcheck not found, skipping."); + } + + println!("[3/5] Node syntax check (if node available)"); + if command_exists("node") { + run_status( + Command::new("node") + .arg("--check") + .arg(root.join("scripts/aw-webui-browser-smoke.mjs")) + .stdout(Stdio::null()), + false, + )?; + } else { + println!("node not found, skipping."); + } + + println!("[4/5] PowerShell parse check (if pwsh available)"); + if command_exists("pwsh") { + let ps = r#" +$ErrorActionPreference = "Stop" +Get-ChildItem windows -Filter *.ps1 | ForEach-Object { + $tokens = $null + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseFile($_.FullName,[ref]$tokens,[ref]$errors) + if ($errors.Count) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 } +} +foreach ($path in @("windows/ActivityWatch.Windows.Common.psm1", "windows/ActivityWatch.Windows.Common.psd1")) { + $tokens = $null + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path $path),[ref]$tokens,[ref]$errors) + if ($errors.Count) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 } +} +"#; + run_status( + Command::new("pwsh") + .arg("-NoLogo") + .arg("-NoProfile") + .arg("-Command") + .arg(ps) + .current_dir(root), + false, + )?; + run_status( + Command::new("pwsh") + .arg("-NoLogo") + .arg("-NoProfile") + .arg("-File") + .arg("windows/aw-collector-guard.ps1") + .arg("-SelfTest") + .current_dir(root) + .stdout(Stdio::null()), + false, + )?; + } else { + println!("pwsh not found, skipping."); + } + + println!("[5/5] Ansible syntax check (if ansible-playbook available)"); + if command_exists("ansible-playbook") { + for playbook in collect_top_level_yml(&root.join("ansible"))? { + run_status( + Command::new("ansible-playbook") + .arg("--syntax-check") + .arg(&playbook) + .arg("-i") + .arg(root.join("ansible/inventory.example.ini")) + .stdout(Stdio::null()), + false, + )?; + } + } else { + println!("ansible-playbook not found, skipping."); + } + + println!("quality-gate: OK"); + Ok(()) +} + +fn run_status(command: &mut Command, quiet: bool) -> Result<()> { + let program = command.get_program().to_string_lossy().to_string(); + if quiet { + command.stdout(Stdio::null()).stderr(Stdio::null()); + } + let status = command.status().with_context(|| format!("run {program}"))?; + if !status.success() { + bail!("{program} failed with status {status}"); + } + Ok(()) +} + +fn command_exists(name: &str) -> bool { + let Some(path) = env::var_os("PATH") else { + return false; + }; + env::split_paths(&path).any(|dir| dir.join(name).is_file()) +} + +fn collect_by_extension(root: &Path, dirs: &[&str], extension: &str) -> Result> { + let mut out = Vec::new(); + for dir in dirs { + let path = root.join(dir); + if path.is_dir() { + collect_by_extension_inner(&path, extension, &mut out)?; + } + } + out.sort(); + Ok(out) +} + +fn collect_by_extension_inner(path: &Path, extension: &str, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("read file type {}", entry_path.display()))?; + if file_type.is_dir() { + collect_by_extension_inner(&entry_path, extension, out)?; + } else if entry_path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext == extension) + { + out.push(entry_path); + } + } + Ok(()) +} + +fn collect_top_level_yml(dir: &Path) -> Result> { + let mut out = Vec::new(); + if !dir.is_dir() { + return Ok(out); + } + for entry in fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", dir.display()))?; + let path = entry.path(); + if path.is_file() + && path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext == "yml") + { + out.push(path); + } + } + out.sort(); + Ok(out) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::{collect_by_extension, collect_top_level_yml}; + + #[test] + fn collects_recursive_shell_files_sorted() { + let tmp = tempdir().unwrap(); + fs::create_dir_all(tmp.path().join("scripts/nested")).unwrap(); + fs::write(tmp.path().join("scripts/b.sh"), "").unwrap(); + fs::write(tmp.path().join("scripts/nested/a.sh"), "").unwrap(); + fs::write(tmp.path().join("scripts/skip.py"), "").unwrap(); + let files = collect_by_extension(tmp.path(), &["scripts"], "sh").unwrap(); + let names: Vec<_> = files + .iter() + .map(|path| path.file_name().unwrap().to_string_lossy().to_string()) + .collect(); + assert_eq!(names, vec!["b.sh", "a.sh"]); + } + + #[test] + fn collects_top_level_yml_only() { + let tmp = tempdir().unwrap(); + fs::create_dir_all(tmp.path().join("ansible/nested")).unwrap(); + fs::write(tmp.path().join("ansible/a.yml"), "").unwrap(); + fs::write(tmp.path().join("ansible/nested/b.yml"), "").unwrap(); + let files = collect_top_level_yml(&tmp.path().join("ansible")).unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].file_name().unwrap(), "a.yml"); + } +} diff --git a/adk-rust/crates/rdp-worktime-report/Cargo.toml b/adk-rust/crates/rdp-worktime-report/Cargo.toml new file mode 100644 index 0000000..00a387f --- /dev/null +++ b/adk-rust/crates/rdp-worktime-report/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rdp-worktime-report" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/rdp-worktime-report/src/main.rs b/adk-rust/crates/rdp-worktime-report/src/main.rs new file mode 100644 index 0000000..3649e6a --- /dev/null +++ b/adk-rust/crates/rdp-worktime-report/src/main.rs @@ -0,0 +1,615 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::{DateTime, Days, Local, NaiveDate, TimeDelta, TimeZone, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +const DEFAULT_AW_BASE_URL: &str = "http://10.10.10.13:5600/api/0"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const DEFAULT_SAMPLE_SECONDS: f64 = 30.0; +const DEFAULT_MAX_SAMPLE_SECONDS: f64 = 300.0; +const DEFAULT_OUT_DIR: &str = "reports"; + +#[derive(Debug, Parser)] +#[command(about = "Build per-user RDP worktime CSV/JSON report from AW session samples")] +struct Cli { + #[arg(long)] + day: Option, + + #[arg(long)] + from: Option, + + #[arg(long)] + to: Option, +} + +#[derive(Debug, Clone)] +struct Config { + aw_base_url: String, + host: String, + default_sample_seconds: f64, + max_sample_seconds: f64, + out_dir: PathBuf, +} + +#[derive(Debug, Clone, Deserialize)] +struct AwEvent { + timestamp: Option, + #[serde(default)] + duration: Option, + #[serde(default)] + data: Value, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +struct ReportRow { + user: String, + user_id: String, + active_seconds: i64, + active_hhmm: String, + first_activity: String, + last_activity: String, + idle_seconds: i64, + sessions_count: usize, + samples_count: i64, + active_samples: i64, +} + +#[derive(Debug, Serialize)] +struct JsonReport { + host: String, + bucket_id: String, + from: String, + to: String, + generated_at_utc: String, + rows: Vec, +} + +#[derive(Debug, Default)] +struct UserAggregate { + user: String, + user_id: String, + sessions: BTreeSet, + samples_count: i64, + active_samples: i64, + intervals: Vec<(DateTime, DateTime)>, +} + +#[derive(Debug, Clone)] +struct Sample { + ts: DateTime, + duration: Option, + data: Value, +} + +fn main() { + let code = match run() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let (from, to) = resolve_range(&cli)?; + let config = load_config(); + fs::create_dir_all(&config.out_dir) + .with_context(|| format!("create {}", config.out_dir.display()))?; + + let bucket_id = format!("aw-worktime-sessions_{}", config.host); + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(30)) + .build() + .context("build HTTP client")?; + + get_json::( + &client, + &format!("{}/buckets/{bucket_id}", config.aw_base_url), + ) + .with_context(|| format!("Bucket not found: {bucket_id}"))?; + let events: Vec = get_json( + &client, + &format!( + "{}/buckets/{bucket_id}/events?limit=50000", + config.aw_base_url + ), + )?; + + let rows = build_rows( + events, + &config.host, + from, + to, + config.default_sample_seconds, + config.max_sample_seconds, + )?; + let csv_out = config + .out_dir + .join(format!("rdp-worktime-{}_{}.csv", from, to)); + let json_out = config + .out_dir + .join(format!("rdp-worktime-{}_{}.json", from, to)); + write_csv(&csv_out, &rows)?; + write_json(&json_out, &config.host, &bucket_id, from, to, &rows)?; + + println!("{}", csv_out.display()); + println!("{}", json_out.display()); + println!("CSV: {}", csv_out.display()); + println!("JSON: {}", json_out.display()); + Ok(()) +} + +fn load_config() -> Config { + let default_sample_seconds = + env_f64("AW_WORKTIME_DEFAULT_SAMPLE_SECONDS", DEFAULT_SAMPLE_SECONDS).max(1.0); + let max_sample_seconds = env_f64("AW_WORKTIME_MAX_SAMPLE_SECONDS", DEFAULT_MAX_SAMPLE_SECONDS) + .max(default_sample_seconds); + Config { + aw_base_url: normalize_aw_base(&env_string("AW_BASE_URL", DEFAULT_AW_BASE_URL)), + host: env_string("AW_WORKTIME_HOST", DEFAULT_HOST), + default_sample_seconds, + max_sample_seconds, + out_dir: PathBuf::from(env_string("OUT_DIR", DEFAULT_OUT_DIR)), + } +} + +fn env_string(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn normalize_aw_base(raw: &str) -> String { + let base = raw.trim().trim_end_matches('/'); + if base.ends_with("/api/0") { + base.to_string() + } else { + format!("{base}/api/0") + } +} + +fn resolve_range(cli: &Cli) -> Result<(NaiveDate, NaiveDate)> { + if let Some(day) = &cli.day { + let today = Local::now().date_naive(); + return match day.as_str() { + "today" => Ok((today, today)), + "yesterday" => { + let yesterday = today + .checked_sub_days(Days::new(1)) + .context("calculate yesterday")?; + Ok((yesterday, yesterday)) + } + _ => bail!("Invalid --day: {day}"), + }; + } + + let Some(from) = cli.from else { + bail!( + "Usage: rdp-worktime-report --day today|yesterday OR --from YYYY-MM-DD --to YYYY-MM-DD" + ); + }; + let Some(to) = cli.to else { + bail!( + "Usage: rdp-worktime-report --day today|yesterday OR --from YYYY-MM-DD --to YYYY-MM-DD" + ); + }; + Ok((from, to)) +} + +fn get_json(client: &Client, url: &str) -> Result { + let response = client + .get(url) + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} status"))?; + response + .json() + .with_context(|| format!("decode JSON from {url}")) +} + +fn build_rows( + events: Vec, + host: &str, + from: NaiveDate, + to: NaiveDate, + default_sample_seconds: f64, + max_sample_seconds: f64, +) -> Result> { + let start = day_start_utc(from)?; + let end = day_end_utc(to)?; + let mut by_identity: BTreeMap<(String, String), Vec> = BTreeMap::new(); + + for event in events { + let Some(ts) = event.timestamp.as_deref().and_then(parse_ts) else { + continue; + }; + if ts < start || ts > end { + continue; + } + let user = value_string(&event.data, "username").trim().to_string(); + if user.is_empty() { + continue; + } + let session_id = value_string(&event.data, "sessionId"); + let session_id = if session_id.trim().is_empty() { + "unknown".to_string() + } else { + session_id.trim().to_string() + }; + by_identity + .entry((user, session_id)) + .or_default() + .push(Sample { + ts, + duration: event.duration, + data: event.data, + }); + } + + let full_range = (end - start).num_seconds() + 1; + let mut by_user: BTreeMap = BTreeMap::new(); + for ((user, session_id), mut samples) in by_identity { + samples.sort_by_key(|sample| sample.ts); + for idx in 0..samples.len() { + let sample = &samples[idx]; + let rec = by_user + .entry(user.clone()) + .or_insert_with(|| UserAggregate { + user: user.clone(), + user_id: normalize_user_id(&sample.data, host, &user), + ..UserAggregate::default() + }); + rec.sessions.insert(session_id.clone()); + rec.samples_count += 1; + if !is_active(&sample.data) { + continue; + } + rec.active_samples += 1; + let sample_seconds = sample_seconds( + sample, + samples.get(idx + 1).map(|next| next.ts), + default_sample_seconds, + max_sample_seconds, + ); + let interval_end = std::cmp::min( + sample.ts + TimeDelta::milliseconds((sample_seconds * 1000.0).round() as i64), + end + TimeDelta::seconds(1), + ); + if interval_end > sample.ts { + rec.intervals.push((sample.ts, interval_end)); + } + } + } + + let mut rows = Vec::new(); + for (_, rec) in by_user { + let merged = merge_intervals(rec.intervals); + let mut active: i64 = merged + .iter() + .map(|(begin, finish)| (*finish - *begin).num_seconds()) + .sum(); + active = active.min(full_range); + let idle_seconds = (full_range - active).max(0); + rows.push(ReportRow { + user: rec.user, + user_id: rec.user_id, + active_seconds: active, + active_hhmm: format!("{:02}:{:02}", active / 3600, (active % 3600) / 60), + first_activity: merged + .first() + .map(|(begin, _)| format_ts(*begin)) + .unwrap_or_default(), + last_activity: merged + .last() + .map(|(_, finish)| format_ts(*finish)) + .unwrap_or_default(), + idle_seconds, + sessions_count: rec.sessions.len(), + samples_count: rec.samples_count, + active_samples: rec.active_samples, + }); + } + Ok(rows) +} + +fn day_start_utc(day: NaiveDate) -> Result> { + Utc.from_local_datetime( + &day.and_hms_opt(0, 0, 0) + .with_context(|| format!("invalid start date {day}"))?, + ) + .single() + .context("resolve UTC start") +} + +fn day_end_utc(day: NaiveDate) -> Result> { + Utc.from_local_datetime( + &day.and_hms_opt(23, 59, 59) + .with_context(|| format!("invalid end date {day}"))?, + ) + .single() + .context("resolve UTC end") +} + +fn parse_ts(value: &str) -> Option> { + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +fn value_string(data: &Value, key: &str) -> String { + match data.get(key) { + Some(Value::String(value)) => value.clone(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + _ => String::new(), + } +} + +fn is_active(data: &Value) -> bool { + if data.get("active").and_then(Value::as_bool) == Some(true) { + return true; + } + let state = value_string(data, "state").trim().to_lowercase(); + state.contains("актив") || state == "active" +} + +fn normalize_user_id(data: &Value, host: &str, username: &str) -> String { + let raw = value_string(data, "userId").trim().to_string(); + if let Some((_, right)) = raw.split_once('\\') { + return format!("{host}\\{right}"); + } + if !raw.is_empty() { + return raw; + } + format!("{host}\\{username}") +} + +fn sample_seconds( + sample: &Sample, + next_ts: Option>, + default_sample_seconds: f64, + max_sample_seconds: f64, +) -> f64 { + for key in ["sampleSeconds", "pollSeconds"] { + if let Some(value) = value_f64(&sample.data, key).filter(|value| *value > 0.0) { + return clamp_seconds(value, default_sample_seconds, max_sample_seconds); + } + } + if let Some(duration) = sample.duration.filter(|value| *value > 0.0) { + return clamp_seconds(duration, default_sample_seconds, max_sample_seconds); + } + if let Some(next_ts) = next_ts { + return clamp_seconds( + (next_ts - sample.ts).num_milliseconds() as f64 / 1000.0, + default_sample_seconds, + max_sample_seconds, + ); + } + clamp_seconds( + default_sample_seconds, + default_sample_seconds, + max_sample_seconds, + ) +} + +fn value_f64(data: &Value, key: &str) -> Option { + match data.get(key) { + Some(Value::Number(value)) => value.as_f64(), + Some(Value::String(value)) => value.parse().ok(), + _ => None, + } +} + +fn clamp_seconds(value: f64, fallback: f64, max_sample_seconds: f64) -> f64 { + let seconds = if value <= 0.0 { fallback } else { value }; + seconds.min(max_sample_seconds) +} + +fn merge_intervals( + mut intervals: Vec<(DateTime, DateTime)>, +) -> Vec<(DateTime, DateTime)> { + if intervals.is_empty() { + return Vec::new(); + } + intervals.sort_by_key(|(start, _)| *start); + let mut merged = vec![intervals[0]]; + for (start, end) in intervals.into_iter().skip(1) { + let last = merged.last_mut().expect("merged non-empty"); + if start <= last.1 { + if end > last.1 { + last.1 = end; + } + } else { + merged.push((start, end)); + } + } + merged +} + +fn write_csv(path: &Path, rows: &[ReportRow]) -> Result<()> { + let fields = [ + "user", + "user_id", + "active_seconds", + "active_hhmm", + "first_activity", + "last_activity", + "idle_seconds", + "sessions_count", + "samples_count", + "active_samples", + ]; + let mut out = String::new(); + out.push_str(&fields.join(",")); + out.push('\n'); + for row in rows { + out.push_str(&csv_escape(&row.user)); + out.push(','); + out.push_str(&csv_escape(&row.user_id)); + out.push(','); + out.push_str(&row.active_seconds.to_string()); + out.push(','); + out.push_str(&csv_escape(&row.active_hhmm)); + out.push(','); + out.push_str(&csv_escape(&row.first_activity)); + out.push(','); + out.push_str(&csv_escape(&row.last_activity)); + out.push(','); + out.push_str(&row.idle_seconds.to_string()); + out.push(','); + out.push_str(&row.sessions_count.to_string()); + out.push(','); + out.push_str(&row.samples_count.to_string()); + out.push(','); + out.push_str(&row.active_samples.to_string()); + out.push('\n'); + } + fs::write(path, out).with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn csv_escape(value: &str) -> String { + if value.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } +} + +fn write_json( + path: &Path, + host: &str, + bucket_id: &str, + from: NaiveDate, + to: NaiveDate, + rows: &[ReportRow], +) -> Result<()> { + let report = JsonReport { + host: host.to_string(), + bucket_id: bucket_id.to_string(), + from: from.to_string(), + to: to.to_string(), + generated_at_utc: format_ts(Utc::now()), + rows: rows.to_vec(), + }; + fs::write(path, serde_json::to_string_pretty(&report)?) + .with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn format_ts(value: DateTime) -> String { + value.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} + +#[cfg(test)] +mod tests { + use chrono::NaiveDate; + use serde_json::json; + + use super::*; + + #[test] + fn computes_active_intervals_and_merges_overlap() { + let events = vec![ + event( + "2026-06-01T10:00:00Z", + None, + json!({"username":"user5","userId":"HOST\\user5","sessionId":2,"active":true,"sampleSeconds":60}), + ), + event( + "2026-06-01T10:00:30Z", + None, + json!({"username":"user5","userId":"HOST\\user5","sessionId":2,"state":"Активно","sampleSeconds":60}), + ), + event( + "2026-06-01T11:00:00Z", + None, + json!({"username":"user5","sessionId":3,"state":"Disc","sampleSeconds":60}), + ), + ]; + let rows = build_rows( + events, + "SHARKON2025", + NaiveDate::from_ymd_opt(2026, 6, 1).unwrap(), + NaiveDate::from_ymd_opt(2026, 6, 1).unwrap(), + 30.0, + 300.0, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].active_seconds, 90); + assert_eq!(rows[0].active_hhmm, "00:01"); + assert_eq!(rows[0].sessions_count, 2); + assert_eq!(rows[0].samples_count, 3); + assert_eq!(rows[0].active_samples, 2); + assert_eq!(rows[0].user_id, "SHARKON2025\\user5"); + } + + #[test] + fn uses_next_timestamp_when_duration_missing() { + let events = vec![ + event( + "2026-06-01T10:00:00Z", + None, + json!({"username":"admin","sessionId":1,"state":"active"}), + ), + event( + "2026-06-01T10:02:00Z", + None, + json!({"username":"admin","sessionId":1,"state":"active"}), + ), + ]; + let rows = build_rows( + events, + "HOST", + NaiveDate::from_ymd_opt(2026, 6, 1).unwrap(), + NaiveDate::from_ymd_opt(2026, 6, 1).unwrap(), + 30.0, + 300.0, + ) + .unwrap(); + assert_eq!(rows[0].active_seconds, 150); + assert_eq!(rows[0].active_samples, 2); + } + + #[test] + fn normalizes_base_url() { + assert_eq!( + normalize_aw_base("http://127.0.0.1:5600"), + "http://127.0.0.1:5600/api/0" + ); + assert_eq!( + normalize_aw_base("http://127.0.0.1:5600/api/0/"), + "http://127.0.0.1:5600/api/0" + ); + } + + fn event(timestamp: &str, duration: Option, data: Value) -> AwEvent { + AwEvent { + timestamp: Some(timestamp.to_string()), + duration, + data, + } + } +} diff --git a/adk-rust/crates/rebuild-install-kit/Cargo.toml b/adk-rust/crates/rebuild-install-kit/Cargo.toml new file mode 100644 index 0000000..412ba63 --- /dev/null +++ b/adk-rust/crates/rebuild-install-kit/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rebuild-install-kit" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +flate2 = "1" +sha2.workspace = true +tar = "0.4" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/rebuild-install-kit/src/main.rs b/adk-rust/crates/rebuild-install-kit/src/main.rs new file mode 100644 index 0000000..31c34db --- /dev/null +++ b/adk-rust/crates/rebuild-install-kit/src/main.rs @@ -0,0 +1,379 @@ +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use clap::Parser; +use flate2::Compression; +use flate2::write::GzEncoder; +use sha2::{Digest, Sha256}; +use zip::write::SimpleFileOptions; + +const KIT_DIR: &str = "install-kit-awindows-20260427-211240"; +const SERVER_CONFIG_DIR: &str = "server-configs-192.168.100.18"; +const OLD_SERVER_CONFIG_DIR: &str = "server-configs-192.168.100.21"; +const README: &str = r#"ActivityWatch DetMir Windows Install Kit + +Includes: +- windows/* (deploy scripts, collectors, common module, configs/examples) +- ansible/* (Windows and AW server playbooks, examples, inventory, tasks) +- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) +- scripts/* (install-kit rebuild/validation, quality gates, browser/web smoke checks) +- server-configs-192.168.100.18/* (working Windows/RDP config snapshots) + +Source: +- Local project snapshot at build time. +"#; + +const ANSIBLE_FILES: &[&str] = &[ + "ansible/README.md", + "ansible/deploy_aw_pfsense_poller.yml", + "ansible/deploy_aw_server.yml", + "ansible/deploy_aw_windows.yml", + "ansible/group_vars/all.example.yml", + "ansible/group_vars/pfsense-poller.example.yml", + "ansible/group_vars/proxmox-matrix.example.yml", + "ansible/group_vars/proxmox.example.yml", + "ansible/group_vars/windows.example.yml", + "ansible/install_full_stack.yml", + "ansible/inventory.example.ini", + "ansible/provision_proxmox_ct_and_deploy_aw.yml", + "ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml", + "ansible/tasks/provision_ct_and_deploy_aw.yml", +]; + +const AW_SERVER_FILES: &[&str] = &[ + "aw-server/activitywatch-server.service", + "aw-server/apply_webui_ru_patch.sh", + "aw-server/aw-host-groups.json", + "aw-server/aw-ru-patch.js", + "aw-server/aw-rus-healthd.py", + "aw-server/aw-rus-healthd.service", + "aw-server/aw-rus-healthd.timer", + "aw-server/aw-browser-smoke.service", + "aw-server/aw-browser-smoke.timer", + "aw-server/aw-slo-monitor.py", + "aw-server/aw-slo-monitor.service", + "aw-server/aw-slo-monitor.timer", + "aw-server/aw-server.env.example", + "aw-server/aw-sw-cleanup.js", + "aw-server/aw-worktime-api.py", + "aw-server/aw-worktime-api.service", + "aw-server/aw-worktime-prewarm.sh", + "aw-server/aw-worktime-prewarm.service", + "aw-server/aw-worktime-prewarm.timer", + "aw-server/aw-worktime-panel.js", + "aw-server/health-check.sh", + "aw-server/install_aw_server.sh", + "aw-server/settings/classes-worktime.json", + "aw-server/settings/views-default.json", +]; + +const WINDOWS_FILES: &[&str] = &[ + "windows/ActivityWatch.Windows.Common.psd1", + "windows/ActivityWatch.Windows.Common.psm1", + "windows/browser-domains-native-collector.ps1", + "windows/deploy-domain-users.ps1", + "windows/deploy-ensemble.ps1", + "windows/deploy-single-user.ps1", + "windows/AWatchRusCollectorGuardService.cs", + "windows/aw-collector-guard.ps1", + "windows/install-collector-guard-service.ps1", + "windows/dlp-endpoint-signals-collector.ps1", + "windows/dlp-policy.example.json", + "windows/dlp-policy.native-cross-os.example.json", + "windows/email-outbound-collector.ps1", + "windows/hardening-recovery.ps1", + "windows/migrate-awatch-rus-paths.ps1", + "windows/validate-deployment.ps1", + "windows/web-category-rules.example.json", + "windows/worktime-session-collector.ps1", +]; + +const SCRIPTS_FILES: &[&str] = &[ + "scripts/aw-webui-browser-smoke.mjs", + "scripts/aw-webui-browser-smoke.sh", + "scripts/check_install_kit_vs_repo.sh", + "scripts/quality-gate.sh", + "scripts/rebuild_install_kit.sh", + "scripts/validate_install_kit.sh", + "scripts/verify_innosetup_installer.sh", +]; + +#[derive(Debug, Parser)] +#[command(about = "Rebuild ActivityWatch-Russian Windows install-kit directory and archives")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, +} + +fn main() { + let code = match run() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result<()> { + let cli = Cli::parse(); + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + rebuild(&root) +} + +fn rebuild(root: &Path) -> Result<()> { + let kit = root.join(KIT_DIR); + let preserved_configs = preserve_server_configs(&kit)?; + for name in ["ansible", "aw-server", "windows", "scripts"] { + remove_if_exists(&kit.join(name))?; + } + remove_server_config_dirs(&kit)?; + + for rel in ANSIBLE_FILES + .iter() + .chain(AW_SERVER_FILES) + .chain(WINDOWS_FILES) + .chain(SCRIPTS_FILES) + { + copy_file(root, &kit, rel)?; + } + + let server_config_dir = kit.join(SERVER_CONFIG_DIR); + fs::create_dir_all(&server_config_dir) + .with_context(|| format!("create {}", server_config_dir.display()))?; + for (name, bytes) in preserved_configs { + let path = server_config_dir.join(name); + write_file_replace(&path, &bytes).with_context(|| { + format!("restore server config into {}", server_config_dir.display()) + })?; + } + + write_file_replace(&kit.join("README-INSTALL-KIT.txt"), README.as_bytes()) + .with_context(|| format!("write {}", kit.join("README-INSTALL-KIT.txt").display()))?; + write_manifest(root, &kit)?; + write_zip(root, &kit)?; + write_tar(root, &kit)?; + Ok(()) +} + +fn preserve_server_configs(kit: &Path) -> Result)>> { + let mut out = std::collections::BTreeMap::new(); + for dir in [OLD_SERVER_CONFIG_DIR, SERVER_CONFIG_DIR] { + let path = kit.join(dir); + if !path.is_dir() { + continue; + } + for entry in fs::read_dir(&path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let entry_path = entry.path(); + if entry_path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".deployment-config.json")) + && entry_path.is_file() + { + let name = entry.file_name().to_string_lossy().to_string(); + let bytes = fs::read(&entry_path) + .with_context(|| format!("read {}", entry_path.display()))?; + out.insert(name, bytes); + } + } + } + Ok(out.into_iter().collect()) +} + +fn remove_server_config_dirs(kit: &Path) -> Result<()> { + if !kit.is_dir() { + return Ok(()); + } + for entry in fs::read_dir(kit).with_context(|| format!("read dir {}", kit.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", kit.display()))?; + let path = entry.path(); + if path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("server-configs-")) + { + remove_if_exists(&path)?; + } + } + Ok(()) +} + +fn copy_file(root: &Path, kit: &Path, rel: &str) -> Result<()> { + let src = root.join(rel); + let dest = kit.join(rel); + fs::create_dir_all(dest.parent().context("destination parent")?) + .with_context(|| format!("create parent for {}", dest.display()))?; + let bytes = fs::read(&src).with_context(|| format!("read {}", src.display()))?; + write_file_replace(&dest, &bytes) + .with_context(|| format!("copy {} to {}", src.display(), dest.display()))?; + Ok(()) +} + +fn write_file_replace(path: &Path, bytes: &[u8]) -> Result<()> { + remove_if_exists(path)?; + fs::write(path, bytes).with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn remove_if_exists(path: &Path) -> Result<()> { + if !path.exists() { + return Ok(()); + } + let meta = fs::symlink_metadata(path).with_context(|| format!("stat {}", path.display()))?; + if meta.is_dir() { + fs::remove_dir_all(path).with_context(|| format!("remove dir {}", path.display()))?; + } else { + fs::remove_file(path).with_context(|| format!("remove file {}", path.display()))?; + } + Ok(()) +} + +fn write_manifest(root: &Path, kit: &Path) -> Result<()> { + let mut files = Vec::new(); + collect_files(kit, &mut files)?; + files.retain(|path| path.file_name().and_then(|name| name.to_str()) != Some("MANIFEST.txt")); + files.sort(); + let manifest = kit.join("MANIFEST.txt"); + let mut out = String::new(); + for path in files { + let digest = sha256_file(&path)?; + let rel = path + .strip_prefix(root) + .with_context(|| format!("strip root prefix from {}", path.display()))?; + out.push_str(&format!("{digest} {}\n", slash_path(rel))); + } + write_file_replace(&manifest, out.as_bytes())?; + Ok(()) +} + +fn write_zip(root: &Path, kit: &Path) -> Result<()> { + let archive_path = root.join(format!("{KIT_DIR}.zip")); + remove_if_exists(&archive_path)?; + let file = File::create(&archive_path) + .with_context(|| format!("create {}", archive_path.display()))?; + let mut zip = zip::ZipWriter::new(file); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let mut files = Vec::new(); + collect_files(kit, &mut files)?; + files.sort(); + for path in files { + let rel = path + .strip_prefix(root) + .with_context(|| format!("strip root prefix from {}", path.display()))?; + let name = slash_path(rel); + zip.start_file(name, options).context("start zip file")?; + let mut input = File::open(&path).with_context(|| format!("open {}", path.display()))?; + std::io::copy(&mut input, &mut zip) + .with_context(|| format!("write zip {}", path.display()))?; + } + zip.finish().context("finish zip")?; + Ok(()) +} + +fn write_tar(root: &Path, kit: &Path) -> Result<()> { + let archive_path = root.join(format!("{KIT_DIR}.tar.gz")); + remove_if_exists(&archive_path)?; + let file = File::create(&archive_path) + .with_context(|| format!("create {}", archive_path.display()))?; + let encoder = GzEncoder::new(file, Compression::default()); + let mut builder = tar::Builder::new(encoder); + let mut files = Vec::new(); + collect_files(kit, &mut files)?; + files.sort(); + for path in files { + builder + .append_path_with_name( + &path, + slash_path( + path.strip_prefix(root) + .with_context(|| format!("strip root prefix from {}", path.display()))?, + ), + ) + .with_context(|| format!("append tar {}", path.display()))?; + } + builder.finish().context("finish tar")?; + Ok(()) +} + +fn collect_files(path: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("read file type {}", entry_path.display()))?; + if file_type.is_dir() { + collect_files(&entry_path, out)?; + } else if (file_type.is_file() || file_type.is_symlink()) && entry_path.is_file() { + out.push(entry_path); + } + } + Ok(()) +} + +fn sha256_file(path: &Path) -> Result { + let mut file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut hasher = Sha256::new(); + let mut buf = [0_u8; 64 * 1024]; + loop { + let read = file + .read(&mut buf) + .with_context(|| format!("read {}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn slash_path(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::{preserve_server_configs, remove_server_config_dirs}; + + #[test] + fn preserves_new_server_configs_over_old_duplicates() { + let tmp = tempdir().unwrap(); + let kit = tmp.path(); + let old = kit.join("server-configs-192.168.100.21"); + let new = kit.join("server-configs-192.168.100.18"); + fs::create_dir_all(&old).unwrap(); + fs::create_dir_all(&new).unwrap(); + fs::write(old.join("a.deployment-config.json"), "old").unwrap(); + fs::write(new.join("a.deployment-config.json"), "new").unwrap(); + let configs = preserve_server_configs(kit).unwrap(); + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].0, "a.deployment-config.json"); + assert_eq!(configs[0].1, b"new"); + } + + #[test] + fn removes_server_config_dirs_only() { + let tmp = tempdir().unwrap(); + let kit = tmp.path(); + fs::create_dir_all(kit.join("server-configs-x")).unwrap(); + fs::create_dir_all(kit.join("windows")).unwrap(); + remove_server_config_dirs(kit).unwrap(); + assert!(!kit.join("server-configs-x").exists()); + assert!(kit.join("windows").exists()); + } +} diff --git a/adk-rust/crates/tsj-guardian-status/Cargo.toml b/adk-rust/crates/tsj-guardian-status/Cargo.toml new file mode 100644 index 0000000..3d494ff --- /dev/null +++ b/adk-rust/crates/tsj-guardian-status/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "tsj-guardian-status" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[[bin]] +name = "tsj-guardian-status" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-state.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/tsj-guardian-status/src/main.rs b/adk-rust/crates/tsj-guardian-status/src/main.rs new file mode 100644 index 0000000..31f3fd0 --- /dev/null +++ b/adk-rust/crates/tsj-guardian-status/src/main.rs @@ -0,0 +1,2110 @@ +use std::{ + fs, + io::{self, Read}, + path::{Path, PathBuf}, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, SecondsFormat, Utc}; +use clap::Parser; +use detmir_state::{DEFAULT_STATE_FILE, NormalizedStatus, read_state}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Debug, Parser)] +#[command(about = "Read-only backend helpers for TSJ Guardian Telegram status.")] +struct Cli { + #[arg(long, default_value = DEFAULT_STATE_FILE)] + state: PathBuf, + + #[arg(long)] + json: bool, + + #[arg(long)] + aw_slo_status_line: bool, + + #[arg(long)] + status_text: bool, + + #[arg(long)] + incident_suggestions: bool, + + #[arg(long)] + incident_defer_decision: bool, + + #[arg(long)] + escalation_decision: bool, + + #[arg(long)] + operator_action_decision: bool, + + #[arg(long)] + dlp_policy_decision: bool, + + #[arg(long)] + confirmation_decision: bool, + + #[arg(long)] + autoheal_plan_decision: bool, + + #[arg(long, default_value_t = 1)] + incident_failure_quorum_checks: i64, + + #[arg(long, default_value_t = 900)] + operator_timeout_seconds: i64, + + #[arg(long, default_value_t = 900)] + confirmation_ttl_seconds: i64, + + #[arg(long)] + aw_slo_summary_command: Option, + + #[arg(long, default_value = "24h")] + aw_slo_alert_window: String, + + #[arg(long)] + bot_state: Option, + + #[arg(long)] + rollback_file: Option, + + #[arg(long)] + pfsense_status_command: Option, + + #[arg(long)] + now_epoch: Option, +} + +fn detmir_auto_line(status: &NormalizedStatus) -> String { + let summary = &status.detmir_summary; + let dlp = &status.dlp_counts; + format!( + "- detmir_auto: {severity} check_ok={check_ok} dlp_ok={dlp_ok} \ + bucket_stale={bucket_stale} bucket_dead={bucket_dead} \ + service_fail={service_failures} service_warn={service_warnings} \ + dlp_warn={dlp_warn} dlp_fail={dlp_fail}", + severity = status.severity, + check_ok = status.check_ok, + dlp_ok = status.dlp_ok, + bucket_stale = summary.bucket_stale.unwrap_or(0), + bucket_dead = summary.bucket_dead.unwrap_or(0), + service_failures = summary.service_failures.unwrap_or(0), + service_warnings = summary.service_warnings.unwrap_or(0), + dlp_warn = dlp.warn.unwrap_or(0), + dlp_fail = dlp.fail.unwrap_or(0), + ) +} + +fn run_shell_json(command: &str) -> Result { + let stdout = run_shell_text(command)?; + serde_json::from_str(&stdout).context("parse AW SLO summary JSON") +} + +fn run_shell_text(command: &str) -> Result { + let output = Command::new("sh") + .arg("-lc") + .arg(command) + .output() + .with_context(|| format!("run command: {command}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("command failed rc={:?}: {}", output.status.code(), stderr); + } + String::from_utf8(output.stdout).context("decode command stdout as UTF-8") +} + +fn value_i64(value: Option<&Value>) -> Option { + match value { + Some(Value::Number(n)) => n.as_i64().or_else(|| n.as_f64().map(|v| v as i64)), + Some(Value::String(s)) => s.parse::().ok(), + _ => None, + } +} + +fn value_f64(value: Option<&Value>) -> Option { + match value { + Some(Value::Number(n)) => n.as_f64(), + Some(Value::String(s)) => s.parse::().ok(), + _ => None, + } +} + +fn value_string(value: Option<&Value>, fallback: &str) -> String { + match value { + Some(Value::String(s)) => s.clone(), + Some(Value::Number(n)) => n.to_string(), + Some(Value::Bool(v)) => v.to_string(), + Some(Value::Null) | None => fallback.to_string(), + Some(v) => v.to_string(), + } +} + +fn aw_slo_status_line(summary: Option<&Value>, alert_window: &str) -> String { + let Some(summary) = summary else { + return "- aw_rus_slo: unavailable".to_string(); + }; + let windows = summary.get("windows").and_then(Value::as_object); + let window_name = if windows + .and_then(|w| w.get(alert_window)) + .and_then(Value::as_object) + .is_some() + { + alert_window + } else { + "24h" + }; + let window = windows + .and_then(|w| w.get(window_name)) + .and_then(Value::as_object); + let availability = window.and_then(|w| value_f64(w.get("availability_percent"))); + let remaining_raw = window.and_then(|w| w.get("budget_remaining_seconds")); + let remaining_value = value_i64(remaining_raw).unwrap_or(0); + let remaining_text = value_string(remaining_raw, "None"); + let samples_text = value_string(window.and_then(|w| w.get("samples")), "None"); + let status = value_string(window.and_then(|w| w.get("status")), "unknown"); + let current = summary.get("current_sample").and_then(Value::as_object); + let current_ok = current + .and_then(|c| c.get("ok")) + .and_then(Value::as_bool) + .unwrap_or(false); + let display_status = if remaining_value < 0 && current_ok { + "recovered".to_string() + } else if remaining_value < 0 && !current_ok { + "fail".to_string() + } else { + status + }; + let availability_text = availability + .map(|v| format!("{v:.5}%")) + .unwrap_or_else(|| "n/a".to_string()); + format!( + "- aw_rus_slo: {display_status} {window_name} current_sample={} availability={availability_text} samples={samples_text} budget_remaining_seconds={remaining_text}", + if current_ok { "OK" } else { "FAIL" } + ) +} + +fn parse_generated_age_seconds(summary: &Value) -> Option { + let generated = summary.get("generated_at_utc")?.as_str()?; + let ts = DateTime::parse_from_rfc3339(&generated.replace('Z', "+00:00")) + .ok()? + .with_timezone(&Utc); + Some((Utc::now() - ts).num_seconds()) +} + +#[derive(Debug, Serialize)] +struct StatusTextPayload { + status_text: String, + pfsense_status: String, + aw_rus_slo_line: String, + detmir_auto_line: String, + rollback_pending_items: usize, +} + +#[derive(Debug, Deserialize)] +struct IncidentInput { + #[serde(default)] + failures: Vec, + #[serde(default)] + state: Value, +} + +#[derive(Debug, Serialize)] +struct SuggestionsPayload { + suggestions: Vec, +} + +#[derive(Debug, Serialize)] +struct DeferDecisionPayload { + defer: bool, + signature: String, + failure_streak_signature: String, + failure_streak_count: i64, + failure_streak_first_ts: i64, + reset_failure_streak: bool, + log_line: Option, +} + +#[derive(Debug, Serialize)] +struct EscalationDecisionPayload { + should_escalate: bool, + should_fallback: bool, + timed_out: bool, + operator_acked: bool, + age_seconds: i64, + reason: String, +} + +#[derive(Debug, Deserialize)] +struct OperatorActionInput { + #[serde(default)] + action: String, + #[serde(default)] + state: Value, +} + +#[derive(Debug, Serialize)] +struct OperatorActionDecisionPayload { + requested_action: String, + canonical_action: String, + handler: String, + allowed: bool, + requires_confirmation: bool, + risk_level: String, + reason: String, + message: Option, + state_update_hints: Vec, +} + +#[derive(Debug, Deserialize)] +struct DlpPolicyInput { + #[serde(default)] + policy: Value, + #[serde(default)] + target_mode: String, +} + +#[derive(Debug, Serialize)] +struct DlpRuleGroupSummary { + name: String, + total: usize, + blocked: usize, +} + +#[derive(Debug, Serialize)] +struct DlpPolicyDecisionPayload { + current_mode: String, + target_mode: Option, + changed_count: usize, + changed_rules: Vec, + groups: Vec, + updated_policy: Option, + reason: String, +} + +#[derive(Debug, Deserialize)] +struct ConfirmationInput { + #[serde(default)] + kind: String, + #[serde(default)] + action: String, + #[serde(default)] + code: String, + #[serde(default)] + state: Value, +} + +#[derive(Debug, Serialize)] +struct ConfirmationDecisionPayload { + kind: String, + action: String, + present: bool, + expired: bool, + allowed: bool, + clear_pending: bool, + next_stage: Option, + first_confirmed_ts: Option, + reason: String, + message: Option, +} + +#[derive(Debug, Deserialize)] +struct AutohealPlanInput { + #[serde(default)] + failures: Vec, + #[serde(default)] + slo_stale: bool, +} + +#[derive(Debug, Serialize)] +struct AutohealPlanPayload { + failures: Vec, + slo_only: bool, + slo_stale: bool, + include_watchers: bool, + include_worktime: bool, + include_windows_dlp: bool, + server_dlp_failures: Vec, + run_windows_heal: bool, + run_server_dlp_heal: bool, + run_worktime_heal: bool, + sleep_after_seconds: i64, + report_triggers: Vec, + direct_autoheal_target: bool, +} + +fn read_json_file(path: &Path) -> Option { + let content = fs::read_to_string(path).ok()?; + serde_json::from_str(&content).ok() +} + +fn read_stdin_value() -> Result { + let mut input = String::new(); + io::stdin() + .read_to_string(&mut input) + .context("read stdin JSON")?; + serde_json::from_str(&input).context("parse stdin JSON") +} + +fn read_stdin_json() -> Result { + serde_json::from_value(read_stdin_value()?).context("parse incident stdin JSON") +} + +fn read_operator_action_input() -> Result { + serde_json::from_value(read_stdin_value()?).context("parse operator action stdin JSON") +} + +fn read_dlp_policy_input() -> Result { + serde_json::from_value(read_stdin_value()?).context("parse DLP policy stdin JSON") +} + +fn read_confirmation_input() -> Result { + serde_json::from_value(read_stdin_value()?).context("parse confirmation stdin JSON") +} + +fn read_autoheal_plan_input() -> Result { + serde_json::from_value(read_stdin_value()?).context("parse autoheal plan stdin JSON") +} + +fn failure_signature(failures: &[String]) -> String { + let mut unique = failures.to_vec(); + unique.sort(); + unique.dedup(); + unique.join("\n") +} + +fn has_filesystem_critical(failures: &[String]) -> bool { + failures + .iter() + .any(|line| line.to_ascii_lowercase().contains("filesystem_usage")) +} + +fn suggestions_from_failures(failures: &[String]) -> Vec { + let text = failures.join("\n").to_ascii_lowercase(); + let mut suggestions = Vec::new(); + if text.contains("proxmox_api") { + suggestions.push("Перезапустить pveproxy/pvedaemon/pve-cluster и проверить порт 8006."); + } + if text.contains("pfsense_web") { + suggestions + .push("Проверить доступность pfSense 10.10.10.1:8443, перезапустить WebGUI/nginx."); + } + if text.contains("pfsense_mcp") { + suggestions.push("Проверить локальный pfsense-mcp-server.service, bearer token и endpoint 127.0.0.1:3010/mcp."); + } + if text.contains("influxdb") { + suggestions.push("Проверить контейнер InfluxDB и restart сервиса influxdb."); + } + if text.contains("grafana") { + suggestions.push("Проверить grafana-server и NO_PROXY для 10.10.10.0/24."); + } + if text.contains("loki") || text.contains("alloy") { + suggestions.push("Проверить LXC логов и restart сервисов loki/alloy."); + } + if text.contains("aw-rus:watcher-") || text.contains("aw-rus:worktime:") { + suggestions.push("Проверить Windows collector recovery: worktime-session-collector, ActivityWatch Recovery и Launch tasks на 192.168.100.18."); + suggestions.push("После Windows recovery проверить server-side aw-worktime-autoheal/ui-bridge для пересборки afk/window bucket'ов."); + } + if text.contains("aw-rus:dlp-") { + suggestions.push( + "Проверить DLP endpoint/fileops collectors и server-side DLP transport на 10.10.10.13.", + ); + } + if text.contains("filesystem_usage") { + suggestions.push("Проверить самые большие каталоги: du -x /var /srv /home, журналы в /var/log и apt cache."); + suggestions.push("Проверить давление по снапшотам/хранилищу Proxmox и решить: очистка, ротация или расширение диска."); + } + if suggestions.is_empty() { + suggestions.push("Запустить расширенную диагностику: /run check"); + } + suggestions.into_iter().map(ToOwned::to_owned).collect() +} + +fn incident_defer_decision( + failures: &[String], + state: &Value, + threshold: i64, + now_epoch: i64, +) -> DeferDecisionPayload { + let signature = failure_signature(failures); + if threshold <= 1 || has_filesystem_critical(failures) { + return DeferDecisionPayload { + defer: false, + signature, + failure_streak_signature: value_string(state.get("failure_streak_signature"), ""), + failure_streak_count: value_i64(state.get("failure_streak_count")).unwrap_or(0), + failure_streak_first_ts: value_i64(state.get("failure_streak_first_ts")).unwrap_or(0), + reset_failure_streak: false, + log_line: None, + }; + } + + let current_signature = value_string(state.get("failure_streak_signature"), ""); + let mut count = value_i64(state.get("failure_streak_count")).unwrap_or(0); + let first_ts; + if signature == current_signature { + count += 1; + first_ts = value_i64(state.get("failure_streak_first_ts")).unwrap_or(now_epoch); + } else { + count = 1; + first_ts = now_epoch; + } + let defer = count < threshold; + DeferDecisionPayload { + defer, + signature: signature.clone(), + failure_streak_signature: signature, + failure_streak_count: count, + failure_streak_first_ts: first_ts, + reset_failure_streak: !defer, + log_line: defer.then(|| { + format!( + "Suppressing transient incident streak={count}/{threshold} failures={}", + failures.len() + ) + }), + } +} + +fn escalation_decision( + state: &Value, + operator_timeout_seconds: i64, + now_epoch: i64, +) -> EscalationDecisionPayload { + let Some(pi) = state + .get("pending_incident") + .filter(|value| value.is_object()) + else { + return EscalationDecisionPayload { + should_escalate: false, + should_fallback: false, + timed_out: false, + operator_acked: false, + age_seconds: 0, + reason: "no_pending_incident".to_string(), + }; + }; + let operator_acked = pi + .get("operator_acked") + .and_then(Value::as_bool) + .unwrap_or(false); + let created_ts = value_i64(pi.get("created_ts")).unwrap_or(now_epoch); + let age_seconds = (now_epoch - created_ts).max(0); + if operator_acked { + return EscalationDecisionPayload { + should_escalate: false, + should_fallback: false, + timed_out: false, + operator_acked, + age_seconds, + reason: "operator_acked".to_string(), + }; + } + if age_seconds < operator_timeout_seconds { + return EscalationDecisionPayload { + should_escalate: false, + should_fallback: false, + timed_out: false, + operator_acked, + age_seconds, + reason: "timeout_not_reached".to_string(), + }; + } + let escalated_to_ai = pi + .get("escalated_to_ai") + .and_then(Value::as_bool) + .unwrap_or(false); + let fallback_executed = pi + .get("fallback_executed") + .and_then(Value::as_bool) + .unwrap_or(false); + EscalationDecisionPayload { + should_escalate: !escalated_to_ai, + should_fallback: !fallback_executed, + timed_out: true, + operator_acked, + age_seconds, + reason: "operator_timeout_reached".to_string(), + } +} + +fn strip_run_prefix(action: &str) -> String { + let trimmed = action.trim(); + if let Some(rest) = trimmed.strip_prefix("/run ") { + rest.trim().to_string() + } else { + trimmed.to_string() + } +} + +fn canonical_operator_action(action: &str) -> Option<&'static str> { + match strip_run_prefix(action).to_ascii_lowercase().as_str() { + "check" => Some("check"), + "aw-dlp-check" | "awrus-dlp-check" => Some("aw-dlp-check"), + "dlp-mode" | "dlp-policy-mode" => Some("dlp-mode"), + "dlp-mode-toggle" | "dlp-policy-toggle" => Some("dlp-mode-toggle"), + "heal" => Some("heal"), + "ai" | "support" | "techsupport" | "техподдержка" | "тех.поддержка" => { + Some("support") + } + "fallback" => Some("fallback"), + "updates-check" => Some("updates-check"), + "updates-install-request" => Some("updates-install-request"), + "updates-install-confirm" => Some("updates-install-confirm"), + "updates-rollback-confirm" => Some("updates-rollback-confirm"), + _ => None, + } +} + +fn operator_action_risk(action: &str) -> &'static str { + match action { + "check" | "dlp-mode" => "low", + "aw-dlp-check" | "updates-check" | "support" => "medium", + "heal" + | "fallback" + | "dlp-mode-toggle" + | "updates-install-request" + | "updates-install-confirm" => "high", + "updates-rollback-confirm" => "critical", + _ => "unknown", + } +} + +fn operator_action_handler(action: &str) -> &'static str { + match action { + "check" => "check_script", + "aw-dlp-check" => "aw_rus_dlp_check_and_heal", + "dlp-mode" => "aw_dlp_policy_mode", + "dlp-mode-toggle" => "aw_dlp_policy_toggle", + "heal" => "autoheal", + "support" => "ai_escalation", + "fallback" => "server_fallback", + "updates-check" => "updates_check", + "updates-install-request" => "updates_install_request", + "updates-install-confirm" => "updates_install_apply", + "updates-rollback-confirm" => "updates_rollback", + _ => "unknown", + } +} + +fn operator_action_requires_confirmation(action: &str) -> bool { + matches!( + action, + "updates-install-confirm" | "updates-rollback-confirm" + ) +} + +fn state_bool(state: &Value, key: &str) -> bool { + state.get(key).and_then(Value::as_bool).unwrap_or(false) +} + +fn operator_action_decision(input: &OperatorActionInput) -> OperatorActionDecisionPayload { + let requested_action = input.action.trim().to_string(); + let Some(canonical) = canonical_operator_action(&requested_action) else { + return OperatorActionDecisionPayload { + requested_action, + canonical_action: String::new(), + handler: "unknown".to_string(), + allowed: false, + requires_confirmation: false, + risk_level: "unknown".to_string(), + reason: "unknown_action".to_string(), + message: Some("Неизвестное действие.".to_string()), + state_update_hints: Vec::new(), + }; + }; + + if canonical == "updates-install-confirm" + && !state_bool(&input.state, "pending_update_install_confirm") + { + return OperatorActionDecisionPayload { + requested_action, + canonical_action: canonical.to_string(), + handler: operator_action_handler(canonical).to_string(), + allowed: false, + requires_confirmation: true, + risk_level: operator_action_risk(canonical).to_string(), + reason: "missing_update_install_confirmation".to_string(), + message: Some( + "Нет ожидающего запроса на установку. Сначала нажмите \"Установить критичные и важные обновления\"." + .to_string(), + ), + state_update_hints: Vec::new(), + }; + } + + if canonical == "updates-rollback-confirm" + && !state_bool(&input.state, "pending_rollback_confirm") + { + return OperatorActionDecisionPayload { + requested_action, + canonical_action: canonical.to_string(), + handler: operator_action_handler(canonical).to_string(), + allowed: false, + requires_confirmation: true, + risk_level: operator_action_risk(canonical).to_string(), + reason: "missing_update_rollback_confirmation".to_string(), + message: Some( + "Нет ожидающего отката. Откат доступен только после неуспешного ручного обновления." + .to_string(), + ), + state_update_hints: Vec::new(), + }; + } + + let state_update_hints = match canonical { + "updates-install-request" => vec!["set_pending_update_install_confirm".to_string()], + "updates-install-confirm" => vec![ + "clear_pending_update_install_confirm".to_string(), + "set_pending_rollback_confirm_if_rollback_items_exist".to_string(), + ], + "updates-rollback-confirm" => vec!["clear_pending_rollback_confirm".to_string()], + _ => Vec::new(), + }; + + OperatorActionDecisionPayload { + requested_action, + canonical_action: canonical.to_string(), + handler: operator_action_handler(canonical).to_string(), + allowed: true, + requires_confirmation: operator_action_requires_confirmation(canonical), + risk_level: operator_action_risk(canonical).to_string(), + reason: "allowed".to_string(), + message: None, + state_update_hints, + } +} + +fn dlp_toggle_group_names() -> [&'static str; 4] { + ["clipboard", "usb", "print", "email"] +} + +fn dlp_group_summaries(policy: &Value) -> Vec { + let Some(endpoint) = policy.get("endpoint").and_then(Value::as_object) else { + return Vec::new(); + }; + dlp_toggle_group_names() + .into_iter() + .filter_map(|key| { + let rules = endpoint.get(key)?.as_array()?; + let mut total = 0; + let mut blocked = 0; + for rule in rules { + let Some(rule) = rule.as_object() else { + continue; + }; + if rule.get("enabled").and_then(Value::as_bool) == Some(false) { + continue; + } + let action = value_string(rule.get("action"), "") + .trim() + .to_ascii_lowercase(); + if action.is_empty() { + continue; + } + total += 1; + if action == "block" { + blocked += 1; + } + } + Some(DlpRuleGroupSummary { + name: format!("endpoint.{key}"), + total, + blocked, + }) + }) + .collect() +} + +fn dlp_mode_from_policy(policy: &Value) -> String { + if let Some(mode) = policy + .get("_tsj_meta") + .and_then(Value::as_object) + .and_then(|meta| meta.get("dlp_mode")) + .and_then(Value::as_str) + .map(|mode| mode.trim().to_ascii_lowercase()) + .filter(|mode| mode == "monitor" || mode == "enforce") + { + return mode; + } + + let groups = dlp_group_summaries(policy); + let total: usize = groups.iter().map(|group| group.total).sum(); + let blocked: usize = groups.iter().map(|group| group.blocked).sum(); + if blocked == 0 { + "monitor".to_string() + } else if blocked == total { + "enforce".to_string() + } else { + "mixed".to_string() + } +} + +fn dlp_updated_at_utc(now_epoch: i64) -> String { + DateTime::::from_timestamp(now_epoch, 0) + .unwrap_or_else(Utc::now) + .to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn dlp_policy_for_mode( + policy: &Value, + target_mode: &str, + now_epoch: i64, +) -> Result<(Value, usize, Vec)> { + if !matches!(target_mode, "monitor" | "enforce") { + anyhow::bail!("unsupported DLP mode: {target_mode}"); + } + + let mut updated = policy.clone(); + let mut changed = 0; + let mut changed_rules = Vec::new(); + if let Some(endpoint) = updated.get_mut("endpoint").and_then(Value::as_object_mut) { + for key in dlp_toggle_group_names() { + let Some(rules) = endpoint.get_mut(key).and_then(Value::as_array_mut) else { + continue; + }; + for rule in rules { + let Some(rule_obj) = rule.as_object_mut() else { + continue; + }; + if rule_obj.get("enabled").and_then(Value::as_bool) == Some(false) { + continue; + } + let old_action = value_string(rule_obj.get("action"), "alert") + .trim() + .to_ascii_lowercase(); + let new_action = if target_mode == "monitor" { + if old_action == "block" { + "alert" + } else { + old_action.as_str() + } + } else if matches!(old_action.as_str(), "log" | "alert") { + "block" + } else { + old_action.as_str() + }; + if new_action == old_action { + continue; + } + rule_obj.insert("action".to_string(), Value::String(new_action.to_string())); + changed += 1; + if changed_rules.len() < 10 { + let rule_id = value_string(rule_obj.get("id"), "?"); + changed_rules.push(format!( + "endpoint.{key}:{rule_id} {old_action}->{new_action}" + )); + } + } + } + } + + let root = updated + .as_object_mut() + .context("DLP policy root must be a JSON object")?; + let meta = root.entry("_tsj_meta").or_insert_with(|| json!({})); + if !meta.is_object() { + *meta = json!({}); + } + let meta_obj = meta.as_object_mut().expect("meta object created above"); + meta_obj.insert( + "dlp_mode".to_string(), + Value::String(target_mode.to_string()), + ); + meta_obj.insert( + "updated_by".to_string(), + Value::String("tsj-guardian-bot".to_string()), + ); + meta_obj.insert( + "updated_at_utc".to_string(), + Value::String(dlp_updated_at_utc(now_epoch)), + ); + + Ok((updated, changed, changed_rules)) +} + +fn dlp_policy_decision(input: &DlpPolicyInput, now_epoch: i64) -> Result { + let current_mode = dlp_mode_from_policy(&input.policy); + let requested = input.target_mode.trim().to_ascii_lowercase(); + let target_mode = match requested.as_str() { + "" => None, + "toggle" => Some(if current_mode == "enforce" || current_mode == "mixed" { + "monitor".to_string() + } else { + "enforce".to_string() + }), + "monitor" | "enforce" => Some(requested), + other => anyhow::bail!("unsupported DLP target mode: {other}"), + }; + + let groups = dlp_group_summaries(&input.policy); + let Some(target_mode) = target_mode else { + return Ok(DlpPolicyDecisionPayload { + current_mode, + target_mode: None, + changed_count: 0, + changed_rules: Vec::new(), + groups, + updated_policy: None, + reason: "mode_only".to_string(), + }); + }; + + let (updated_policy, changed_count, changed_rules) = + dlp_policy_for_mode(&input.policy, &target_mode, now_epoch)?; + let reason = if changed_count == 0 { + "no_toggleable_changes" + } else { + "policy_plan_ready" + }; + Ok(DlpPolicyDecisionPayload { + current_mode, + target_mode: Some(target_mode), + changed_count, + changed_rules, + groups, + updated_policy: Some(updated_policy), + reason: reason.to_string(), + }) +} + +fn confirmation_no_pending_message(kind: &str) -> &'static str { + match kind { + "pfsense" => "Нет ожидающего изменения pfSense.", + "openvpn" => "Нет ожидающего запроса на OpenVPN конфиг.", + "proxmox_restore" => "Нет ожидающего восстановления Proxmox.", + "proxmox_selection" => "Нет ожидающего выбора узла Proxmox.", + _ => "Нет ожидающего подтверждения.", + } +} + +fn confirmation_wrong_stage_message(kind: &str) -> &'static str { + match kind { + "pfsense" => { + "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения." + } + "openvpn" => { + "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения." + } + _ => "Подтверждение пока недоступно.", + } +} + +fn confirmation_wrong_code_message(kind: &str) -> &'static str { + match kind { + "pfsense" => "Неверный код второго подтверждения pfSense.", + "openvpn" => "Неверный код второго подтверждения OpenVPN-конфига.", + "proxmox_restore" => "Неверный код подтверждения восстановления Proxmox.", + _ => "Неверный код подтверждения.", + } +} + +fn confirmation_cancel_message(kind: &str) -> &'static str { + match kind { + "pfsense" => "Ожидающее изменение pfSense отменено.", + "openvpn" => "Ожидающий запрос на OpenVPN конфиг отменён.", + "proxmox_restore" => "Ожидающее восстановление Proxmox отменено.", + "proxmox_selection" => "Выбор узла Proxmox отменён.", + _ => "Ожидающее подтверждение отменено.", + } +} + +fn confirmation_first_already_message(kind: &str, code: &str) -> String { + match kind { + "pfsense" => format!( + "Первое подтверждение уже принято.\nДля второго подтверждения отправьте `/pfsense_apply {code}`." + ), + "openvpn" => format!( + "Первое подтверждение уже принято.\nДля второго подтверждения отправьте `/openvpn_config_apply {code}`." + ), + _ => "Первое подтверждение уже принято.".to_string(), + } +} + +fn confirmation_state_object(state: &Value) -> Option<&serde_json::Map> { + state.as_object().filter(|object| !object.is_empty()) +} + +fn confirmation_decision( + input: &ConfirmationInput, + ttl_seconds: i64, + now_epoch: i64, +) -> ConfirmationDecisionPayload { + let kind = input.kind.trim().to_ascii_lowercase(); + let action = input.action.trim().to_ascii_lowercase(); + let state = confirmation_state_object(&input.state); + let present = state.is_some(); + let created_ts = state + .and_then(|object| value_i64(object.get("created_ts"))) + .unwrap_or(now_epoch); + let expired = present && ttl_seconds >= 0 && now_epoch - created_ts > ttl_seconds; + let stage = state + .and_then(|object| object.get("stage")) + .map(|value| value_string(Some(value), "")) + .unwrap_or_default(); + let confirm_code = state + .and_then(|object| object.get("confirm_code")) + .map(|value| value_string(Some(value), "")) + .unwrap_or_default(); + + let base = |allowed: bool, + clear_pending: bool, + next_stage: Option, + first_confirmed_ts: Option, + reason: &str, + message: Option| ConfirmationDecisionPayload { + kind: kind.clone(), + action: action.clone(), + present, + expired, + allowed, + clear_pending, + next_stage, + first_confirmed_ts, + reason: reason.to_string(), + message, + }; + + if action == "cancel" { + return base( + true, + true, + None, + None, + "cancelled", + Some(confirmation_cancel_message(&kind).to_string()), + ); + } + + if !present { + return base( + false, + false, + None, + None, + "no_pending", + Some(confirmation_no_pending_message(&kind).to_string()), + ); + } + + if expired { + return base( + false, + true, + None, + None, + "expired", + Some(confirmation_no_pending_message(&kind).to_string()), + ); + } + + match action.as_str() { + "expire" => base(false, false, None, None, "active", None), + "first_confirm" => { + if stage != "awaiting_first_confirm" { + return base( + false, + false, + None, + None, + "already_first_confirmed", + Some(confirmation_first_already_message(&kind, &confirm_code)), + ); + } + base( + true, + false, + Some("awaiting_second_confirm".to_string()), + Some(now_epoch), + "first_confirmed", + None, + ) + } + "apply" => { + if kind != "proxmox_restore" && stage != "awaiting_second_confirm" { + return base( + false, + false, + None, + None, + "wrong_stage", + Some(confirmation_wrong_stage_message(&kind).to_string()), + ); + } + if input.code.trim() != confirm_code { + return base( + false, + false, + None, + None, + "wrong_code", + Some(confirmation_wrong_code_message(&kind).to_string()), + ); + } + base(true, false, None, None, "apply_allowed", None) + } + _ => base( + false, + false, + None, + None, + "unknown_action", + Some("Неизвестное действие подтверждения.".to_string()), + ), + } +} + +fn autoheal_plan_decision(input: &AutohealPlanInput) -> AutohealPlanPayload { + let mut failures: Vec = input + .failures + .iter() + .map(|item| item.trim().to_string()) + .filter(|item| !item.is_empty()) + .collect(); + failures.sort(); + failures.dedup(); + + let slo_only = failures.iter().any(|item| item == "slo"); + if slo_only { + let trigger = if input.slo_stale { + "- heal trigger: SLO summary stale, check aw-slo-monitor.timer/service" + } else { + "- heal trigger: SLO error budget exhausted, no direct autoheal target" + }; + return AutohealPlanPayload { + failures, + slo_only: true, + slo_stale: input.slo_stale, + include_watchers: false, + include_worktime: false, + include_windows_dlp: false, + server_dlp_failures: Vec::new(), + run_windows_heal: false, + run_server_dlp_heal: false, + run_worktime_heal: false, + sleep_after_seconds: 0, + report_triggers: vec![trigger.to_string()], + direct_autoheal_target: false, + }; + } + + let watcher_failures: Vec = failures + .iter() + .filter(|item| item.starts_with("watcher-")) + .cloned() + .collect(); + let dlp_failures: Vec = failures + .iter() + .filter(|item| item.starts_with("dlp-")) + .cloned() + .collect(); + let include_worktime = failures.iter().any(|item| item == "worktime"); + let windows_dlp_failures: Vec = dlp_failures + .iter() + .filter(|item| matches!(item.as_str(), "dlp-endpoint" | "dlp-fileops-host")) + .cloned() + .collect(); + let server_dlp_failures: Vec = dlp_failures + .iter() + .filter(|item| !matches!(item.as_str(), "dlp-endpoint" | "dlp-fileops-host")) + .cloned() + .collect(); + + let include_watchers = !watcher_failures.is_empty(); + let include_windows_dlp = !windows_dlp_failures.is_empty(); + let run_windows_heal = include_watchers || include_worktime || include_windows_dlp; + let run_server_dlp_heal = !server_dlp_failures.is_empty(); + let run_worktime_heal = include_watchers || include_worktime; + let sleep_after_seconds = if include_watchers || include_windows_dlp { + 30 + } else { + 5 + }; + let mut report_triggers = Vec::new(); + if run_windows_heal { + report_triggers.push( + "- heal trigger: Windows session collectors degraded, starting remediation".to_string(), + ); + } + if run_server_dlp_heal { + report_triggers + .push("- heal trigger: server-side DLP degraded, starting remediation".to_string()); + } + if run_worktime_heal { + report_triggers.push( + "- heal trigger: worktime/watchers degraded, rebuilding server-side worktime views" + .to_string(), + ); + } + + AutohealPlanPayload { + failures, + slo_only: false, + slo_stale: false, + include_watchers, + include_worktime, + include_windows_dlp, + server_dlp_failures, + run_windows_heal, + run_server_dlp_heal, + run_worktime_heal, + sleep_after_seconds, + report_triggers, + direct_autoheal_target: run_windows_heal || run_server_dlp_heal || run_worktime_heal, + } +} + +fn pfsense_status_lines(command: Option<&str>) -> String { + let Some(command) = command.map(str::trim).filter(|value| !value.is_empty()) else { + return "- pfsense_security: status unavailable (command not configured)".to_string(); + }; + match run_shell_text(command) { + Ok(output) => { + let lines: Vec = output + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect(); + if lines.is_empty() { + "- pfsense_security: status unavailable (empty output)".to_string() + } else { + lines.join("\n") + } + } + Err(error) => { + let text = error.to_string(); + let tail = text + .chars() + .rev() + .take(500) + .collect::>() + .into_iter() + .rev() + .collect::(); + format!("- pfsense_security: status unavailable ({tail})") + } + } +} + +fn rollback_pending_count(path: Option<&PathBuf>) -> usize { + let Some(path) = path else { + return 0; + }; + read_json_file(path) + .and_then(|payload| { + payload + .get("pending_rollback") + .and_then(Value::as_array) + .map(Vec::len) + }) + .unwrap_or(0) +} + +fn pending_line(state: &Value, key: &str, none: &str) -> String { + let Some(value) = state.get(key).filter(|value| value.is_object()) else { + return none.to_string(); + }; + match key { + "pending_pfsense_change" => format!( + "- pending_pfsense_change: {} stage={}", + value_string(value.get("request_id"), "unknown"), + value_string(value.get("stage"), "unknown") + ), + "pending_proxmox_selection" => format!( + "- pending_proxmox_selection: mode={}", + value_string(value.get("mode"), "unknown") + ), + "pending_proxmox_restore" => format!( + "- pending_proxmox_restore: {}:{} snapshot={}", + value_string(value.get("kind"), "unknown"), + value_string(value.get("guest_id"), "unknown"), + value_string(value.get("snapshot"), "unknown") + ), + "pending_openvpn_config" => format!( + "- pending_openvpn_config: {} cn={} stage={}", + value_string(value.get("request_id"), "unknown"), + value_string(value.get("common_name"), "unknown"), + value_string(value.get("stage"), "unknown") + ), + _ => none.to_string(), + } +} + +fn status_bool_line(state: &Value, key: &str) -> String { + format!( + "- {key}: {}", + state.get(key).and_then(Value::as_bool).unwrap_or(false) + ) +} + +fn status_text_payload( + detmir_state: &Path, + bot_state_path: Option<&PathBuf>, + rollback_path: Option<&PathBuf>, + pfsense_command: Option<&str>, + slo_command: Option<&str>, + slo_alert_window: &str, + now_epoch: i64, +) -> StatusTextPayload { + let detmir_auto_line = match read_state(detmir_state) { + Ok(status) => detmir_auto_line(&status), + Err(error) => format!("- detmir_auto: unavailable ({error})"), + }; + let summary = match slo_command.map(str::trim) { + Some(command) if !command.is_empty() => run_shell_json(command).ok(), + _ => None, + }; + let aw_rus_slo_line = aw_slo_status_line(summary.as_ref(), slo_alert_window); + let pfsense_status = pfsense_status_lines(pfsense_command); + let bot_state = bot_state_path + .and_then(|path| read_json_file(path)) + .unwrap_or_else(|| json!({})); + let rollback_pending_items = rollback_pending_count(rollback_path); + let pending_incident = bot_state + .get("pending_incident") + .filter(|value| value.is_object()); + + let ppc_line = pending_line( + &bot_state, + "pending_pfsense_change", + "- pending_pfsense_change: none", + ); + let ovpn_warn_line = if bot_state + .get("last_openvpn_expiry_signature") + .and_then(Value::as_str) + .unwrap_or("") + .is_empty() + { + "- openvpn_expiry_warning_signature: none".to_string() + } else { + "- openvpn_expiry_warning_signature: set".to_string() + }; + let pps_line = pending_line( + &bot_state, + "pending_proxmox_selection", + "- pending_proxmox_selection: none", + ); + let ppr_line = pending_line( + &bot_state, + "pending_proxmox_restore", + "- pending_proxmox_restore: none", + ); + let povpn_line = pending_line( + &bot_state, + "pending_openvpn_config", + "- pending_openvpn_config: none", + ); + let update_line = status_bool_line(&bot_state, "pending_update_install_confirm"); + let rollback_line = status_bool_line(&bot_state, "pending_rollback_confirm"); + + let status_text = if let Some(pi) = pending_incident { + let created_ts = value_i64(pi.get("created_ts")).unwrap_or(now_epoch); + let age = (now_epoch - created_ts).max(0); + let failures = pi + .get("failures") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .map(|item| value_string(Some(item), "")) + .filter(|item| !item.is_empty()) + .collect::>() + .join(" | ") + }) + .unwrap_or_default(); + [ + format!( + "Статус: активный инцидент {}", + value_string(pi.get("incident_id"), "unknown") + ), + pfsense_status.clone(), + aw_rus_slo_line.clone(), + detmir_auto_line.clone(), + format!("- возраст: {age}s"), + format!( + "- autoheal attempts: {}", + value_string(pi.get("autoheal_attempts"), "0") + ), + format!( + "- operator_acked: {}", + value_string(pi.get("operator_acked"), "false") + ), + format!( + "- escalated_to_ai: {}", + value_string(pi.get("escalated_to_ai"), "false") + ), + format!( + "- fallback_executed: {}", + value_string(pi.get("fallback_executed"), "false") + ), + format!("- failures: {failures}"), + ppc_line, + ovpn_warn_line, + pps_line, + ppr_line, + povpn_line, + update_line, + rollback_line, + format!("- rollback_pending_items: {rollback_pending_items}"), + ] + .join("\n") + } else { + [ + "Статус: инцидентов нет.".to_string(), + pfsense_status.clone(), + aw_rus_slo_line.clone(), + detmir_auto_line.clone(), + ppc_line, + ovpn_warn_line, + pps_line, + ppr_line, + povpn_line, + update_line, + rollback_line, + format!("- rollback_pending_items: {rollback_pending_items}"), + ] + .join("\n") + }; + + StatusTextPayload { + status_text, + pfsense_status, + aw_rus_slo_line, + detmir_auto_line, + rollback_pending_items, + } +} + +fn now_epoch() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + if cli.incident_suggestions { + let input = read_stdin_json()?; + println!( + "{}", + serde_json::to_string_pretty(&SuggestionsPayload { + suggestions: suggestions_from_failures(&input.failures), + })? + ); + return Ok(()); + } + if cli.incident_defer_decision { + let input = read_stdin_json()?; + let payload = incident_defer_decision( + &input.failures, + &input.state, + cli.incident_failure_quorum_checks, + cli.now_epoch.unwrap_or_else(now_epoch), + ); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.escalation_decision { + let input = read_stdin_json()?; + let payload = escalation_decision( + &input.state, + cli.operator_timeout_seconds, + cli.now_epoch.unwrap_or_else(now_epoch), + ); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.operator_action_decision { + let input = read_operator_action_input()?; + let payload = operator_action_decision(&input); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.dlp_policy_decision { + let input = read_dlp_policy_input()?; + let payload = dlp_policy_decision(&input, cli.now_epoch.unwrap_or_else(now_epoch))?; + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.confirmation_decision { + let input = read_confirmation_input()?; + let payload = confirmation_decision( + &input, + cli.confirmation_ttl_seconds, + cli.now_epoch.unwrap_or_else(now_epoch), + ); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.autoheal_plan_decision { + let input = read_autoheal_plan_input()?; + let payload = autoheal_plan_decision(&input); + println!("{}", serde_json::to_string_pretty(&payload)?); + return Ok(()); + } + if cli.status_text { + let payload = status_text_payload( + &cli.state, + cli.bot_state.as_ref(), + cli.rollback_file.as_ref(), + cli.pfsense_status_command.as_deref(), + cli.aw_slo_summary_command.as_deref(), + &cli.aw_slo_alert_window, + cli.now_epoch.unwrap_or_else(now_epoch), + ); + if cli.json { + println!("{}", serde_json::to_string_pretty(&payload)?); + } else { + println!("{}", payload.status_text); + } + return Ok(()); + } + if cli.aw_slo_status_line { + let summary = match cli.aw_slo_summary_command.as_deref().map(str::trim) { + Some(command) if !command.is_empty() => run_shell_json(command).ok(), + _ => None, + }; + let line = aw_slo_status_line(summary.as_ref(), &cli.aw_slo_alert_window); + if cli.json { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "aw_rus_slo_line": line, + "generated_age_seconds": summary.as_ref().and_then(parse_generated_age_seconds), + "summary": summary, + }))? + ); + } else { + println!("{line}"); + } + return Ok(()); + } + let status = read_state(&cli.state)?; + let line = detmir_auto_line(&status); + if cli.json { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "detmir_auto_line": line, + "normalized": status, + }))? + ); + } else { + println!("{line}"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn renders_python_compatible_detmir_line() { + let raw = r#"{ + "severity": "OK", + "check_ok": true, + "dlp_ok": true, + "needs_heal": false, + "detmir_summary": { + "bucket_ok": 8, + "bucket_stale": 0, + "bucket_dead": 0, + "service_failures": 0, + "service_warnings": 0 + }, + "dlp_counts": {"ok": 22, "warn": 0, "fail": 0} + }"#; + let state: detmir_state::DetmirState = serde_json::from_str(raw).unwrap(); + let line = detmir_auto_line(&state.normalize()); + assert_eq!( + line, + "- detmir_auto: OK check_ok=true dlp_ok=true bucket_stale=0 bucket_dead=0 service_fail=0 service_warn=0 dlp_warn=0 dlp_fail=0" + ); + } + + #[test] + fn json_mode_includes_line_and_normalized_status() { + let dir = tempfile::tempdir().unwrap(); + let state_path = dir.path().join("state.json"); + std::fs::write( + &state_path, + r#"{ + "severity": "FAIL", + "check_ok": false, + "dlp_ok": true, + "needs_heal": true, + "detmir_summary": {"bucket_stale": 1, "service_failures": 1}, + "dlp_counts": {"warn": 2, "fail": 0} + }"#, + ) + .unwrap(); + let status = read_state(&state_path).unwrap(); + let payload = json!({ + "detmir_auto_line": detmir_auto_line(&status), + "normalized": status, + }); + assert_eq!(payload["normalized"]["severity"], "FAIL"); + assert!( + payload["detmir_auto_line"] + .as_str() + .unwrap() + .contains("bucket_stale=1") + ); + } + + #[test] + fn renders_aw_slo_recovered_status_line() { + let summary = json!({ + "generated_at_utc": "2026-06-01T00:00:00Z", + "windows": { + "24h": { + "status": "fail", + "availability_percent": 98.123456, + "samples": 42, + "budget_remaining_seconds": -12 + } + }, + "current_sample": {"ok": true} + }); + assert_eq!( + aw_slo_status_line(Some(&summary), "24h"), + "- aw_rus_slo: recovered 24h current_sample=OK availability=98.12346% samples=42 budget_remaining_seconds=-12" + ); + } + + #[test] + fn renders_aw_slo_unavailable_without_summary() { + assert_eq!(aw_slo_status_line(None, "24h"), "- aw_rus_slo: unavailable"); + } + + #[test] + fn renders_full_status_without_incident() { + let dir = tempfile::tempdir().unwrap(); + let detmir_path = dir.path().join("detmir.json"); + let bot_state_path = dir.path().join("bot.json"); + let rollback_path = dir.path().join("rollback.json"); + std::fs::write( + &detmir_path, + r#"{ + "severity": "OK", + "check_ok": true, + "dlp_ok": true, + "needs_heal": false, + "detmir_summary": {"bucket_stale": 0, "bucket_dead": 0, "service_failures": 0, "service_warnings": 0}, + "dlp_counts": {"ok": 22, "warn": 0, "fail": 0} + }"#, + ) + .unwrap(); + std::fs::write( + &bot_state_path, + r#"{ + "pending_pfsense_change": null, + "pending_openvpn_config": {"request_id": "ovpn-1", "common_name": "user1", "stage": "created"}, + "pending_update_install_confirm": true, + "pending_rollback_confirm": false, + "last_openvpn_expiry_signature": "sig" + }"#, + ) + .unwrap(); + std::fs::write(&rollback_path, r#"{"pending_rollback":[{"id":"101"}]}"#).unwrap(); + + let payload = status_text_payload( + &detmir_path, + Some(&bot_state_path), + Some(&rollback_path), + None, + None, + "24h", + 1000, + ); + + assert!(payload.status_text.starts_with("Статус: инцидентов нет.")); + assert!(payload.status_text.contains("- aw_rus_slo: unavailable")); + assert!(payload.status_text.contains("- detmir_auto: OK")); + assert!( + payload + .status_text + .contains("- pending_openvpn_config: ovpn-1 cn=user1 stage=created") + ); + assert!( + payload + .status_text + .contains("- pending_update_install_confirm: true") + ); + assert!(payload.status_text.contains("- rollback_pending_items: 1")); + } + + #[test] + fn renders_full_status_with_incident() { + let dir = tempfile::tempdir().unwrap(); + let detmir_path = dir.path().join("detmir.json"); + let bot_state_path = dir.path().join("bot.json"); + std::fs::write( + &detmir_path, + r#"{ + "severity": "FAIL", + "check_ok": false, + "dlp_ok": true, + "needs_heal": true, + "detmir_summary": {"bucket_stale": 2}, + "dlp_counts": {"warn": 1, "fail": 0} + }"#, + ) + .unwrap(); + std::fs::write( + &bot_state_path, + r#"{ + "pending_incident": { + "incident_id": "inc-1", + "created_ts": 900, + "failures": ["f1", "f2"], + "suggestions": [], + "last_autoheal_ts": 0, + "autoheal_attempts": 2, + "operator_acked": true, + "escalated_to_ai": false, + "fallback_executed": true + } + }"#, + ) + .unwrap(); + + let payload = status_text_payload( + &detmir_path, + Some(&bot_state_path), + None, + None, + None, + "24h", + 1000, + ); + + assert!( + payload + .status_text + .contains("Статус: активный инцидент inc-1") + ); + assert!(payload.status_text.contains("- возраст: 100s")); + assert!(payload.status_text.contains("- autoheal attempts: 2")); + assert!(payload.status_text.contains("- operator_acked: true")); + assert!(payload.status_text.contains("- fallback_executed: true")); + assert!(payload.status_text.contains("- failures: f1 | f2")); + } + + #[test] + fn incident_suggestions_match_filesystem_and_aw_paths() { + let suggestions = suggestions_from_failures(&[ + "[FAIL] filesystem_usage: /var 96%".to_string(), + "[FAIL] aw-rus:watcher-window: stale".to_string(), + ]); + + assert!( + suggestions + .iter() + .any(|line| line.contains("самые большие каталоги")) + ); + assert!( + suggestions + .iter() + .any(|line| line.contains("Windows collector recovery")) + ); + } + + #[test] + fn incident_defer_decision_suppresses_until_quorum() { + let state = json!({ + "failure_streak_signature": "", + "failure_streak_count": 0, + "failure_streak_first_ts": 0 + }); + let failures = vec!["[FAIL] node_13: unavailable".to_string()]; + + let first = incident_defer_decision(&failures, &state, 2, 1000); + assert!(first.defer); + assert_eq!(first.failure_streak_count, 1); + assert_eq!(first.failure_streak_first_ts, 1000); + + let state = json!({ + "failure_streak_signature": first.failure_streak_signature, + "failure_streak_count": first.failure_streak_count, + "failure_streak_first_ts": first.failure_streak_first_ts + }); + let second = incident_defer_decision(&failures, &state, 2, 1060); + assert!(!second.defer); + assert_eq!(second.failure_streak_count, 2); + assert!(second.reset_failure_streak); + } + + #[test] + fn incident_defer_decision_never_suppresses_filesystem_critical() { + let decision = incident_defer_decision( + &["[FAIL] filesystem_usage: /var 96%".to_string()], + &json!({}), + 3, + 1000, + ); + + assert!(!decision.defer); + assert!(!decision.reset_failure_streak); + } + + #[test] + fn escalation_decision_waits_for_timeout_and_ack() { + let state = json!({ + "pending_incident": { + "created_ts": 1000, + "operator_acked": false, + "escalated_to_ai": false, + "fallback_executed": false + } + }); + let early = escalation_decision(&state, 900, 1200); + assert!(!early.should_escalate); + assert_eq!(early.reason, "timeout_not_reached"); + + let timed_out = escalation_decision(&state, 900, 2000); + assert!(timed_out.timed_out); + assert!(timed_out.should_escalate); + assert!(timed_out.should_fallback); + + let acked = escalation_decision( + &json!({"pending_incident": {"created_ts": 1000, "operator_acked": true}}), + 900, + 2000, + ); + assert!(!acked.should_escalate); + assert_eq!(acked.reason, "operator_acked"); + } + + #[test] + fn operator_action_decision_normalizes_aliases() { + let decision = operator_action_decision(&OperatorActionInput { + action: "techsupport".to_string(), + state: json!({}), + }); + + assert!(decision.allowed); + assert_eq!(decision.canonical_action, "support"); + assert_eq!(decision.handler, "ai_escalation"); + assert_eq!(decision.risk_level, "medium"); + } + + #[test] + fn operator_action_decision_blocks_unknown_action() { + let decision = operator_action_decision(&OperatorActionInput { + action: "format-disk".to_string(), + state: json!({}), + }); + + assert!(!decision.allowed); + assert_eq!(decision.reason, "unknown_action"); + assert_eq!(decision.message.as_deref(), Some("Неизвестное действие.")); + } + + #[test] + fn operator_action_decision_guards_update_install_confirmation() { + let blocked = operator_action_decision(&OperatorActionInput { + action: "updates-install-confirm".to_string(), + state: json!({"pending_update_install_confirm": false}), + }); + assert!(!blocked.allowed); + assert!(blocked.requires_confirmation); + assert_eq!(blocked.reason, "missing_update_install_confirmation"); + + let allowed = operator_action_decision(&OperatorActionInput { + action: "updates-install-confirm".to_string(), + state: json!({"pending_update_install_confirm": true}), + }); + assert!(allowed.allowed); + assert_eq!(allowed.canonical_action, "updates-install-confirm"); + assert!( + allowed + .state_update_hints + .contains(&"clear_pending_update_install_confirm".to_string()) + ); + } + + #[test] + fn operator_action_decision_guards_update_rollback_confirmation() { + let blocked = operator_action_decision(&OperatorActionInput { + action: "/run updates-rollback-confirm".to_string(), + state: json!({"pending_rollback_confirm": false}), + }); + assert!(!blocked.allowed); + assert_eq!(blocked.reason, "missing_update_rollback_confirmation"); + + let allowed = operator_action_decision(&OperatorActionInput { + action: "/run updates-rollback-confirm".to_string(), + state: json!({"pending_rollback_confirm": true}), + }); + assert!(allowed.allowed); + assert_eq!(allowed.canonical_action, "updates-rollback-confirm"); + assert_eq!(allowed.risk_level, "critical"); + } + + #[test] + fn dlp_policy_decision_detects_monitor_and_group_counts() { + let policy = json!({ + "endpoint": { + "clipboard": [{"id": "c1", "enabled": true, "action": "alert"}], + "usb": [{"id": "u1", "enabled": true, "action": "log"}], + "print": [{"id": "p1", "enabled": false, "action": "block"}] + } + }); + + let decision = dlp_policy_decision( + &DlpPolicyInput { + policy, + target_mode: String::new(), + }, + 1000, + ) + .unwrap(); + + assert_eq!(decision.current_mode, "monitor"); + assert_eq!(decision.reason, "mode_only"); + assert_eq!(decision.groups.len(), 3); + assert_eq!(decision.groups[0].name, "endpoint.clipboard"); + assert_eq!(decision.groups[0].blocked, 0); + assert!(decision.updated_policy.is_none()); + } + + #[test] + fn dlp_policy_decision_toggle_promotes_endpoint_rules_only() { + let policy = json!({ + "rules": [{"id": "web1", "enabled": true, "action": "alert"}], + "endpoint": { + "clipboard": [{"id": "c1", "enabled": true, "action": "alert"}], + "usb": [{"id": "u1", "enabled": true, "action": "log"}], + "print": [{"id": "p1", "enabled": true, "action": "block"}], + "email": [{"id": "e1", "enabled": false, "action": "alert"}] + } + }); + + let decision = dlp_policy_decision( + &DlpPolicyInput { + policy, + target_mode: "toggle".to_string(), + }, + 1000, + ) + .unwrap(); + + let updated = decision.updated_policy.as_ref().unwrap(); + assert_eq!(decision.current_mode, "mixed"); + assert_eq!(decision.target_mode.as_deref(), Some("monitor")); + assert_eq!(decision.changed_count, 1); + assert_eq!( + updated["endpoint"]["print"][0]["action"].as_str(), + Some("alert") + ); + assert_eq!(updated["rules"][0]["action"].as_str(), Some("alert")); + assert_eq!( + updated["_tsj_meta"]["updated_at_utc"].as_str(), + Some("1970-01-01T00:16:40Z") + ); + } + + #[test] + fn dlp_policy_decision_enforce_sets_alert_and_log_to_block() { + let policy = json!({ + "endpoint": { + "clipboard": [{"id": "c1", "enabled": true, "action": "alert"}], + "usb": [{"id": "u1", "enabled": true, "action": "log"}], + "print": [{"id": "p1", "enabled": true, "action": "block"}] + } + }); + + let decision = dlp_policy_decision( + &DlpPolicyInput { + policy, + target_mode: "enforce".to_string(), + }, + 1000, + ) + .unwrap(); + + let updated = decision.updated_policy.as_ref().unwrap(); + assert_eq!(decision.target_mode.as_deref(), Some("enforce")); + assert_eq!(decision.changed_count, 2); + assert_eq!( + updated["endpoint"]["clipboard"][0]["action"].as_str(), + Some("block") + ); + assert_eq!( + updated["endpoint"]["usb"][0]["action"].as_str(), + Some("block") + ); + assert!( + decision + .changed_rules + .contains(&"endpoint.clipboard:c1 alert->block".to_string()) + ); + } + + #[test] + fn confirmation_decision_expires_pending_request() { + let decision = confirmation_decision( + &ConfirmationInput { + kind: "pfsense".to_string(), + action: "expire".to_string(), + code: String::new(), + state: json!({"created_ts": 1000, "stage": "awaiting_first_confirm", "confirm_code": "123456"}), + }, + 900, + 2000, + ); + + assert!(decision.present); + assert!(decision.expired); + assert!(decision.clear_pending); + assert!(!decision.allowed); + assert_eq!(decision.reason, "expired"); + } + + #[test] + fn confirmation_decision_first_confirm_advances_stage() { + let decision = confirmation_decision( + &ConfirmationInput { + kind: "openvpn".to_string(), + action: "first_confirm".to_string(), + code: String::new(), + state: json!({"created_ts": 1000, "stage": "awaiting_first_confirm", "confirm_code": "654321"}), + }, + 900, + 1200, + ); + + assert!(decision.allowed); + assert_eq!( + decision.next_stage.as_deref(), + Some("awaiting_second_confirm") + ); + assert_eq!(decision.first_confirmed_ts, Some(1200)); + assert_eq!(decision.reason, "first_confirmed"); + } + + #[test] + fn confirmation_decision_apply_guards_stage_and_code() { + let wrong_stage = confirmation_decision( + &ConfirmationInput { + kind: "pfsense".to_string(), + action: "apply".to_string(), + code: "123456".to_string(), + state: json!({"created_ts": 1000, "stage": "awaiting_first_confirm", "confirm_code": "123456"}), + }, + 900, + 1200, + ); + assert!(!wrong_stage.allowed); + assert_eq!(wrong_stage.reason, "wrong_stage"); + + let wrong_code = confirmation_decision( + &ConfirmationInput { + kind: "pfsense".to_string(), + action: "apply".to_string(), + code: "000000".to_string(), + state: json!({"created_ts": 1000, "stage": "awaiting_second_confirm", "confirm_code": "123456"}), + }, + 900, + 1200, + ); + assert!(!wrong_code.allowed); + assert_eq!(wrong_code.reason, "wrong_code"); + + let allowed = confirmation_decision( + &ConfirmationInput { + kind: "pfsense".to_string(), + action: "apply".to_string(), + code: "123456".to_string(), + state: json!({"created_ts": 1000, "stage": "awaiting_second_confirm", "confirm_code": "123456"}), + }, + 900, + 1200, + ); + assert!(allowed.allowed); + assert_eq!(allowed.reason, "apply_allowed"); + } + + #[test] + fn confirmation_decision_proxmox_restore_apply_uses_code_only() { + let allowed = confirmation_decision( + &ConfirmationInput { + kind: "proxmox_restore".to_string(), + action: "apply".to_string(), + code: "222333".to_string(), + state: json!({"created_ts": 1000, "confirm_code": "222333"}), + }, + 900, + 1200, + ); + + assert!(allowed.allowed); + assert_eq!(allowed.reason, "apply_allowed"); + } + + #[test] + fn autoheal_plan_routes_endpoint_failure_to_windows_heal_only() { + let plan = autoheal_plan_decision(&AutohealPlanInput { + failures: vec!["dlp-endpoint".to_string()], + slo_stale: false, + }); + + assert!(plan.run_windows_heal); + assert!(plan.include_windows_dlp); + assert!(!plan.include_watchers); + assert!(!plan.include_worktime); + assert!(!plan.run_server_dlp_heal); + assert!(!plan.run_worktime_heal); + assert_eq!(plan.sleep_after_seconds, 30); + assert!(plan.direct_autoheal_target); + } + + #[test] + fn autoheal_plan_routes_server_dlp_failure_to_server_dlp_heal() { + let plan = autoheal_plan_decision(&AutohealPlanInput { + failures: vec!["dlp-fileops-server".to_string()], + slo_stale: false, + }); + + assert!(!plan.run_windows_heal); + assert!(plan.run_server_dlp_heal); + assert_eq!(plan.server_dlp_failures, vec!["dlp-fileops-server"]); + assert_eq!(plan.sleep_after_seconds, 5); + } + + #[test] + fn autoheal_plan_routes_watcher_and_worktime_to_windows_and_worktime() { + let plan = autoheal_plan_decision(&AutohealPlanInput { + failures: vec!["watcher-window".to_string(), "worktime".to_string()], + slo_stale: false, + }); + + assert!(plan.run_windows_heal); + assert!(plan.include_watchers); + assert!(plan.include_worktime); + assert!(plan.run_worktime_heal); + assert_eq!(plan.sleep_after_seconds, 30); + assert_eq!(plan.report_triggers.len(), 2); + } + + #[test] + fn autoheal_plan_slo_has_no_direct_target() { + let plan = autoheal_plan_decision(&AutohealPlanInput { + failures: vec!["slo".to_string()], + slo_stale: true, + }); + + assert!(plan.slo_only); + assert!(plan.slo_stale); + assert!(!plan.direct_autoheal_target); + assert_eq!( + plan.report_triggers[0], + "- heal trigger: SLO summary stale, check aw-slo-monitor.timer/service" + ); + } +} diff --git a/adk-rust/crates/tsj-guardian-watchdog/Cargo.toml b/adk-rust/crates/tsj-guardian-watchdog/Cargo.toml new file mode 100644 index 0000000..9443465 --- /dev/null +++ b/adk-rust/crates/tsj-guardian-watchdog/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tsj-guardian-watchdog" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +detmir-core.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/tsj-guardian-watchdog/src/main.rs b/adk-rust/crates/tsj-guardian-watchdog/src/main.rs new file mode 100644 index 0000000..3d4bd20 --- /dev/null +++ b/adk-rust/crates/tsj-guardian-watchdog/src/main.rs @@ -0,0 +1,345 @@ +use std::fs; +use std::process::Command; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result}; +use clap::Parser; +use detmir_core::{exit_codes, now_utc_rfc3339}; +use serde::Serialize; + +const DEFAULT_HEARTBEAT_FILE: &str = "/opt/infra-admin/.state/tsj_guardian_heartbeat"; +const DEFAULT_SERVICE_NAME: &str = "tsj-guardian-bot.service"; +const DEFAULT_GOST_SERVICE_NAME: &str = "gost-tg.service"; +const DEFAULT_GOST_PATTERN: &str = "/usr/local/bin/gost -L http+socks5://127.0.0.1:11090 -F socks5+wss://gw.iri1968.dpdns.org:4443"; + +#[derive(Debug, Parser)] +#[command(about = "TSJ Guardian bot heartbeat watchdog and gost duplicate guard.")] +struct Cli { + #[arg(long, default_value = DEFAULT_HEARTBEAT_FILE)] + heartbeat_file: String, + + #[arg(long, default_value_t = 180)] + max_age_seconds: i64, + + #[arg(long, default_value = DEFAULT_SERVICE_NAME)] + service_name: String, + + #[arg(long, default_value = DEFAULT_GOST_SERVICE_NAME)] + gost_service_name: String, + + #[arg(long, default_value = DEFAULT_GOST_PATTERN)] + gost_dup_pattern: String, + + #[arg(long)] + apply: bool, + + #[arg(long)] + json: bool, + + #[arg(long)] + loop_forever: bool, + + #[arg(long, default_value_t = 60)] + interval_seconds: u64, +} + +impl Cli { + fn apply_env(mut self) -> Self { + self.heartbeat_file = env_string("HEARTBEAT_FILE").unwrap_or(self.heartbeat_file); + self.max_age_seconds = env_string("MAX_AGE_SEC") + .and_then(|value| value.parse().ok()) + .unwrap_or(self.max_age_seconds); + self.service_name = env_string("SERVICE_NAME").unwrap_or(self.service_name); + self.gost_service_name = env_string("GOST_SERVICE_NAME").unwrap_or(self.gost_service_name); + self.gost_dup_pattern = env_string("GOST_DUP_PATTERN").unwrap_or(self.gost_dup_pattern); + self + } +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +struct WatchdogAction { + action: String, + target: String, + status: String, +} + +#[derive(Debug, Serialize)] +struct WatchdogReport { + ok: bool, + dry_run: bool, + generated_at_utc: String, + heartbeat_file: String, + heartbeat_age_seconds: Option, + heartbeat_status: String, + gost_main_pid: Option, + gost_pids: Vec, + actions: Vec, +} + +fn env_string(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn main_pid(service_name: &str) -> Option { + let output = Command::new("systemctl") + .args(["show", "-p", "MainPID", "--value", service_name]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .ok() + .filter(|pid| *pid > 1) +} + +fn matching_pids(pattern: &str) -> Vec { + let output = match Command::new("ps").args(["-eo", "pid=,args="]).output() { + Ok(output) => output, + Err(_) => return Vec::new(), + }; + if !output.status.success() { + return Vec::new(); + } + let self_pid = std::process::id(); + let mut pids = String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| { + let line = line.trim_start(); + let (pid, args) = line.split_once(char::is_whitespace)?; + let pid = pid.parse::().ok()?; + if pid != self_pid && args.contains(pattern) { + Some(pid) + } else { + None + } + }) + .collect::>(); + pids.sort_unstable(); + pids.dedup(); + pids +} + +fn choose_keep_pid(main_pid: Option, pids: &[u32]) -> Option { + if pids.is_empty() { + return None; + } + match main_pid { + Some(pid) if pids.contains(&pid) => Some(pid), + _ => pids.first().copied(), + } +} + +fn signal_pid(pid: u32, signal: &str) -> Result<()> { + let status = Command::new("kill") + .args([format!("-{signal}"), pid.to_string()]) + .status() + .with_context(|| format!("failed to signal pid {pid}"))?; + if status.success() { + Ok(()) + } else { + Err(anyhow::anyhow!("kill -{signal} {pid} exited with {status}")) + } +} + +fn pid_alive(pid: u32) -> bool { + Command::new("kill") + .args(["-0", &pid.to_string()]) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn restart_service(service_name: &str) -> Result<()> { + let status = Command::new("systemctl") + .args(["restart", service_name]) + .status() + .with_context(|| format!("failed to restart {service_name}"))?; + if status.success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "systemctl restart {service_name} exited with {status}" + )) + } +} + +fn heartbeat_status(path: &str, max_age_seconds: i64, now: i64) -> (String, Option) { + let raw = match fs::read_to_string(path) { + Ok(raw) => raw, + Err(_) => return ("missing".to_string(), None), + }; + let heartbeat = match raw.trim().parse::() { + Ok(value) => value, + Err(_) => return ("invalid".to_string(), None), + }; + let age = now.saturating_sub(heartbeat); + if age > max_age_seconds { + ("stale".to_string(), Some(age)) + } else { + ("fresh".to_string(), Some(age)) + } +} + +fn run_once(cli: &Cli) -> Result { + let mut actions = Vec::new(); + let gost_main_pid = main_pid(&cli.gost_service_name); + let gost_pids = matching_pids(&cli.gost_dup_pattern); + if let Some(keep_pid) = choose_keep_pid(gost_main_pid, &gost_pids) { + for pid in gost_pids.iter().copied().filter(|pid| *pid != keep_pid) { + if cli.apply { + let status = signal_pid(pid, "TERM") + .map(|_| "terminated") + .unwrap_or("term-failed"); + actions.push(WatchdogAction { + action: "kill-term".to_string(), + target: pid.to_string(), + status: status.to_string(), + }); + } else { + actions.push(WatchdogAction { + action: "kill-term".to_string(), + target: pid.to_string(), + status: "planned".to_string(), + }); + } + } + if cli.apply && gost_pids.len() > 1 { + thread::sleep(Duration::from_secs(2)); + for pid in gost_pids.iter().copied().filter(|pid| *pid != keep_pid) { + if pid_alive(pid) { + let status = signal_pid(pid, "KILL") + .map(|_| "killed") + .unwrap_or("kill-failed"); + actions.push(WatchdogAction { + action: "kill-kill".to_string(), + target: pid.to_string(), + status: status.to_string(), + }); + } + } + } + } + + let now = chrono_like_now_epoch(); + let (hb_status, hb_age) = heartbeat_status(&cli.heartbeat_file, cli.max_age_seconds, now); + if hb_status != "fresh" { + if cli.apply { + let status = restart_service(&cli.service_name) + .map(|_| "restarted") + .unwrap_or("restart-failed"); + actions.push(WatchdogAction { + action: "restart-service".to_string(), + target: cli.service_name.clone(), + status: status.to_string(), + }); + } else { + actions.push(WatchdogAction { + action: "restart-service".to_string(), + target: cli.service_name.clone(), + status: "planned".to_string(), + }); + } + } + + Ok(WatchdogReport { + ok: true, + dry_run: !cli.apply, + generated_at_utc: now_utc_rfc3339(), + heartbeat_file: cli.heartbeat_file.clone(), + heartbeat_age_seconds: hb_age, + heartbeat_status: hb_status, + gost_main_pid, + gost_pids, + actions, + }) +} + +fn chrono_like_now_epoch() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or(0) +} + +fn print_report(report: &WatchdogReport, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(report)?); + } else { + println!( + "tsj-guardian-watchdog: heartbeat={} age={:?} dry_run={}", + report.heartbeat_status, report.heartbeat_age_seconds, report.dry_run + ); + for action in &report.actions { + println!("{} {} {}", action.action, action.target, action.status); + } + } + Ok(()) +} + +fn run(cli: Cli) -> Result<()> { + if cli.loop_forever { + loop { + let report = run_once(&cli)?; + print_report(&report, cli.json)?; + thread::sleep(Duration::from_secs(cli.interval_seconds)); + } + } + let report = run_once(&cli)?; + print_report(&report, cli.json)?; + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse().apply_env(); + match run(cli) { + Ok(()) => Ok(()), + Err(err) => { + eprintln!("{err:#}"); + std::process::exit(exit_codes::ERROR); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_main_pid_when_present() { + assert_eq!(choose_keep_pid(Some(20), &[10, 20, 30]), Some(20)); + } + + #[test] + fn keeps_first_pid_when_main_missing() { + assert_eq!(choose_keep_pid(Some(99), &[10, 20, 30]), Some(10)); + } + + #[test] + fn detects_heartbeat_states() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("heartbeat"); + assert_eq!( + heartbeat_status(path.to_str().unwrap(), 180, 1_000), + ("missing".to_string(), None) + ); + fs::write(&path, "bad\n").unwrap(); + assert_eq!( + heartbeat_status(path.to_str().unwrap(), 180, 1_000), + ("invalid".to_string(), None) + ); + fs::write(&path, "900\n").unwrap(); + assert_eq!( + heartbeat_status(path.to_str().unwrap(), 180, 1_000), + ("fresh".to_string(), Some(100)) + ); + fs::write(&path, "700\n").unwrap(); + assert_eq!( + heartbeat_status(path.to_str().unwrap(), 180, 1_000), + ("stale".to_string(), Some(300)) + ); + } +} diff --git a/adk-rust/crates/validate-install-kit/Cargo.toml b/adk-rust/crates/validate-install-kit/Cargo.toml new file mode 100644 index 0000000..a9c0eb8 --- /dev/null +++ b/adk-rust/crates/validate-install-kit/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "validate-install-kit" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +flate2 = "1" +tar = "0.4" +zip = { version = "2", default-features = false, features = ["deflate"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/validate-install-kit/src/main.rs b/adk-rust/crates/validate-install-kit/src/main.rs new file mode 100644 index 0000000..8398da7 --- /dev/null +++ b/adk-rust/crates/validate-install-kit/src/main.rs @@ -0,0 +1,611 @@ +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use flate2::read::GzDecoder; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const DEFAULT_KIT_DIR: &str = "install-kit-awindows-20260427-211240"; +const DEFAULT_ZIP_ARCHIVE: &str = "install-kit-awindows-20260427-211240.zip"; +const DEFAULT_TAR_ARCHIVE: &str = "install-kit-awindows-20260427-211240.tar.gz"; +const MANIFEST_NAME: &str = "MANIFEST.txt"; +const REQUIRED_RELATIVE_FILES: &[&str] = &[ + "MANIFEST.txt", + "README-INSTALL-KIT.txt", + "windows/deploy-ensemble.ps1", + "windows/validate-deployment.ps1", + "ansible/deploy_aw_windows.yml", + "aw-server/install_aw_server.sh", + "scripts/rebuild_install_kit.sh", + "scripts/validate_install_kit.sh", + "scripts/check_install_kit_vs_repo.sh", + "scripts/quality-gate.sh", +]; + +#[derive(Debug, Parser)] +#[command(about = "Validate ActivityWatch-Russian Windows install-kit artifacts")] +struct Cli { + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long, default_value = DEFAULT_KIT_DIR)] + kit_dir: PathBuf, + + #[arg(long, default_value = DEFAULT_ZIP_ARCHIVE)] + zip_archive: PathBuf, + + #[arg(long, default_value = DEFAULT_TAR_ARCHIVE)] + tar_archive: PathBuf, + + #[arg(long, default_value_t = false)] + json: bool, +} + +#[derive(Debug, Serialize)] +struct ValidationReport { + ok: bool, + required_files: StageReport, + manifest_checksums: StageReport, + manifest_completeness: CompletenessReport, + archive_composition: ArchiveReport, +} + +#[derive(Debug, Default, Serialize)] +struct StageReport { + ok: bool, + checked: usize, + errors: Vec, +} + +#[derive(Debug, Default, Serialize)] +struct CompletenessReport { + ok: bool, + tracked_files: usize, + missing_listed: Vec, + unlisted_files: Vec, + errors: Vec, +} + +#[derive(Debug, Default, Serialize)] +struct ArchiveReport { + ok: bool, + files: usize, + errors: Vec, +} + +#[derive(Debug, Clone)] +struct ManifestEntry { + digest: String, + path: String, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + let cfg = Config::new(root, cli.kit_dir, cli.zip_archive, cli.tar_archive); + let report = validate(&cfg); + print_report(&report, cli.json)?; + Ok(if report.ok { 0 } else { 1 }) +} + +#[derive(Debug)] +struct Config { + root: PathBuf, + kit_dir: PathBuf, + zip_archive: PathBuf, + tar_archive: PathBuf, +} + +impl Config { + fn new(root: PathBuf, kit_dir: PathBuf, zip_archive: PathBuf, tar_archive: PathBuf) -> Self { + Self { + kit_dir: resolve_path(&root, kit_dir), + zip_archive: resolve_path(&root, zip_archive), + tar_archive: resolve_path(&root, tar_archive), + root, + } + } + + fn manifest(&self) -> PathBuf { + self.kit_dir.join(MANIFEST_NAME) + } +} + +fn validate(cfg: &Config) -> ValidationReport { + let required_files = check_required_files(cfg); + let manifest_entries = read_manifest(&cfg.manifest()); + let manifest_checksums = match &manifest_entries { + Ok(entries) => check_manifest_checksums(cfg, entries), + Err(err) => StageReport { + ok: false, + checked: 0, + errors: vec![err.to_string()], + }, + }; + let manifest_completeness = match &manifest_entries { + Ok(entries) => check_manifest_completeness(cfg, entries), + Err(err) => CompletenessReport { + ok: false, + errors: vec![err.to_string()], + ..CompletenessReport::default() + }, + }; + let archive_composition = check_archive_composition(cfg); + let ok = required_files.ok + && manifest_checksums.ok + && manifest_completeness.ok + && archive_composition.ok; + ValidationReport { + ok, + required_files, + manifest_checksums, + manifest_completeness, + archive_composition, + } +} + +fn check_required_files(cfg: &Config) -> StageReport { + let mut errors = Vec::new(); + for rel in REQUIRED_RELATIVE_FILES { + let path = cfg.kit_dir.join(rel); + if !path.is_file() { + errors.push(format!("Missing required file: {}", path.display())); + } + } + StageReport { + ok: errors.is_empty(), + checked: REQUIRED_RELATIVE_FILES.len(), + errors, + } +} + +fn read_manifest(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open manifest {}", path.display()))?; + let reader = BufReader::new(file); + let mut entries = Vec::new(); + for (idx, line) in reader.lines().enumerate() { + let line = line.with_context(|| format!("read manifest line {}", idx + 1))?; + let line = line.trim(); + if line.is_empty() { + continue; + } + let Some((digest, path)) = line.split_once(" ") else { + bail!("Invalid MANIFEST line {}: {}", idx + 1, line); + }; + if digest.len() != 64 || !digest.chars().all(|ch| ch.is_ascii_hexdigit()) { + bail!("Invalid MANIFEST digest on line {}: {}", idx + 1, digest); + } + entries.push(ManifestEntry { + digest: digest.to_ascii_lowercase(), + path: path.to_string(), + }); + } + Ok(entries) +} + +fn check_manifest_checksums(cfg: &Config, entries: &[ManifestEntry]) -> StageReport { + let mut errors = Vec::new(); + for entry in entries { + let path = cfg.root.join(&entry.path); + if !path.is_file() { + errors.push(format!("Manifest file missing: {}", entry.path)); + continue; + } + match sha256_file(&path) { + Ok(actual) if actual == entry.digest => {} + Ok(actual) => errors.push(format!( + "Checksum mismatch: {} expected {} got {}", + entry.path, entry.digest, actual + )), + Err(err) => errors.push(err.to_string()), + } + } + StageReport { + ok: errors.is_empty(), + checked: entries.len(), + errors, + } +} + +fn check_manifest_completeness(cfg: &Config, entries: &[ManifestEntry]) -> CompletenessReport { + let listed: BTreeSet = entries.iter().map(|entry| entry.path.clone()).collect(); + let actual = match collect_kit_manifest_paths(&cfg.kit_dir) { + Ok(paths) => paths, + Err(err) => { + return CompletenessReport { + ok: false, + errors: vec![err.to_string()], + ..CompletenessReport::default() + }; + } + }; + let missing_listed: Vec = listed.difference(&actual).cloned().collect(); + let unlisted_files: Vec = actual.difference(&listed).cloned().collect(); + CompletenessReport { + ok: missing_listed.is_empty() && unlisted_files.is_empty(), + tracked_files: actual.len(), + missing_listed, + unlisted_files, + errors: Vec::new(), + } +} + +fn collect_kit_manifest_paths(kit_dir: &Path) -> Result> { + let mut files = Vec::new(); + collect_files(kit_dir, &mut files)?; + let mut out = BTreeSet::new(); + for file in files { + let rel = file + .strip_prefix(kit_dir) + .with_context(|| format!("strip kit prefix from {}", file.display()))?; + if rel.file_name().and_then(|name| name.to_str()) == Some(MANIFEST_NAME) { + continue; + } + out.insert(format!("{}/{}", DEFAULT_KIT_DIR, slash_path(rel))); + } + Ok(out) +} + +fn check_archive_composition(cfg: &Config) -> ArchiveReport { + let zip_files = match read_zip_files(&cfg.zip_archive) { + Ok(files) => files, + Err(err) => { + return ArchiveReport { + ok: false, + errors: vec![err.to_string()], + ..ArchiveReport::default() + }; + } + }; + let tar_files = match read_tar_files(&cfg.tar_archive) { + Ok(files) => files, + Err(err) => { + return ArchiveReport { + ok: false, + errors: vec![err.to_string()], + ..ArchiveReport::default() + }; + } + }; + + let mut errors = Vec::new(); + if zip_files != tar_files { + errors.push("ZIP and TAR contents differ".to_string()); + } + let expected_prefix = format!("{DEFAULT_KIT_DIR}/"); + if !zip_files + .iter() + .all(|path| path.starts_with(&expected_prefix)) + { + errors.push("Unexpected archive prefix layout".to_string()); + } + ArchiveReport { + ok: errors.is_empty(), + files: zip_files.len(), + errors, + } +} + +fn read_zip_files(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open zip {}", path.display()))?; + let mut archive = + zip::ZipArchive::new(file).with_context(|| format!("read zip {}", path.display()))?; + let mut files = Vec::new(); + for idx in 0..archive.len() { + let file = archive + .by_index(idx) + .with_context(|| format!("read zip entry {idx}"))?; + let name = normalize_archive_path(file.name()); + if !name.ends_with('/') { + files.push(name); + } + } + files.sort(); + Ok(files) +} + +fn read_tar_files(path: &Path) -> Result> { + let file = File::open(path).with_context(|| format!("open tar {}", path.display()))?; + let decoder = GzDecoder::new(file); + let mut archive = tar::Archive::new(decoder); + let mut files = Vec::new(); + for entry in archive + .entries() + .with_context(|| format!("read tar {}", path.display()))? + { + let entry = entry.with_context(|| format!("read tar entry {}", path.display()))?; + if entry.header().entry_type().is_file() { + let path = entry.path().context("read tar entry path")?; + files.push(normalize_archive_path(&path.to_string_lossy())); + } + } + files.sort(); + Ok(files) +} + +fn collect_files(path: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(path).with_context(|| format!("read dir {}", path.display()))? { + let entry = entry.with_context(|| format!("read dir entry {}", path.display()))?; + let entry_path = entry.path(); + let file_type = entry + .file_type() + .with_context(|| format!("read file type {}", entry_path.display()))?; + if file_type.is_dir() { + collect_files(&entry_path, out)?; + } else if (file_type.is_file() || file_type.is_symlink()) && entry_path.is_file() { + out.push(entry_path); + } + } + Ok(()) +} + +fn sha256_file(path: &Path) -> Result { + let file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buf = [0_u8; 64 * 1024]; + loop { + let read = reader + .read(&mut buf) + .with_context(|| format!("read {}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn print_report(report: &ValidationReport, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(report)?); + return Ok(()); + } + + println!("[1/4] Required files presence"); + for err in &report.required_files.errors { + println!("{err}"); + } + println!("[2/4] Manifest checksum verification"); + for err in &report.manifest_checksums.errors { + println!("{err}"); + } + println!("[3/4] Manifest completeness"); + if report.manifest_completeness.ok { + println!( + "MANIFEST complete: {} files tracked", + report.manifest_completeness.tracked_files + ); + } else { + println!( + "Missing files listed in MANIFEST: {:?}", + report.manifest_completeness.missing_listed + ); + println!( + "Files not listed in MANIFEST: {:?}", + report.manifest_completeness.unlisted_files + ); + for err in &report.manifest_completeness.errors { + println!("{err}"); + } + } + println!("[4/4] Archive composition check"); + if report.archive_composition.ok { + println!("Archives match: {} files", report.archive_composition.files); + } else { + for err in &report.archive_composition.errors { + println!("{err}"); + } + } + if report.ok { + println!("validate_install_kit: OK"); + } + Ok(()) +} + +fn resolve_path(root: &Path, path: PathBuf) -> PathBuf { + if path.is_absolute() { + path + } else { + root.join(path) + } +} + +fn normalize_archive_path(path: &str) -> String { + path.replace('\\', "/").trim_start_matches("./").to_string() +} + +fn slash_path(path: &Path) -> String { + path.components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::io::Write; + + use flate2::Compression; + use flate2::write::GzEncoder; + use tempfile::tempdir; + use zip::write::SimpleFileOptions; + + use super::{Config, DEFAULT_KIT_DIR, check_archive_composition, read_manifest, validate}; + + #[test] + fn manifest_parser_rejects_bad_line() { + let tmp = tempdir().unwrap(); + let manifest = tmp.path().join("MANIFEST.txt"); + fs::write(&manifest, "not-a-manifest-line\n").unwrap(); + let err = read_manifest(&manifest).unwrap_err(); + assert!(err.to_string().contains("Invalid MANIFEST line")); + } + + #[test] + fn validates_clean_fixture() { + let tmp = tempdir().unwrap(); + create_fixture(tmp.path(), true); + let cfg = Config::new( + tmp.path().to_path_buf(), + DEFAULT_KIT_DIR.into(), + format!("{DEFAULT_KIT_DIR}.zip").into(), + format!("{DEFAULT_KIT_DIR}.tar.gz").into(), + ); + let report = validate(&cfg); + assert!(report.ok, "{report:#?}"); + assert_eq!(report.manifest_completeness.tracked_files, 9); + assert_eq!(report.archive_composition.files, 10); + } + + #[test] + fn detects_manifest_extra_file() { + let tmp = tempdir().unwrap(); + create_fixture(tmp.path(), true); + fs::write( + tmp.path().join(DEFAULT_KIT_DIR).join("unlisted.txt"), + "extra", + ) + .unwrap(); + let cfg = Config::new( + tmp.path().to_path_buf(), + DEFAULT_KIT_DIR.into(), + format!("{DEFAULT_KIT_DIR}.zip").into(), + format!("{DEFAULT_KIT_DIR}.tar.gz").into(), + ); + let report = validate(&cfg); + assert!(!report.ok); + assert!( + report + .manifest_completeness + .unlisted_files + .contains(&format!("{DEFAULT_KIT_DIR}/unlisted.txt")) + ); + } + + #[test] + fn detects_archive_mismatch() { + let tmp = tempdir().unwrap(); + create_fixture(tmp.path(), false); + let cfg = Config::new( + tmp.path().to_path_buf(), + DEFAULT_KIT_DIR.into(), + format!("{DEFAULT_KIT_DIR}.zip").into(), + format!("{DEFAULT_KIT_DIR}.tar.gz").into(), + ); + let report = check_archive_composition(&cfg); + assert!(!report.ok); + assert!(report.errors.iter().any(|err| err.contains("differ"))); + } + + fn create_fixture(root: &std::path::Path, matching_archives: bool) { + let kit = root.join(DEFAULT_KIT_DIR); + for rel in [ + "windows/deploy-ensemble.ps1", + "windows/validate-deployment.ps1", + "ansible/deploy_aw_windows.yml", + "aw-server/install_aw_server.sh", + "scripts/rebuild_install_kit.sh", + "scripts/validate_install_kit.sh", + "scripts/check_install_kit_vs_repo.sh", + "scripts/quality-gate.sh", + ] { + let path = kit.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, rel).unwrap(); + } + fs::write(kit.join("README-INSTALL-KIT.txt"), "readme").unwrap(); + let files = [ + "README-INSTALL-KIT.txt", + "windows/deploy-ensemble.ps1", + "windows/validate-deployment.ps1", + "ansible/deploy_aw_windows.yml", + "aw-server/install_aw_server.sh", + "scripts/rebuild_install_kit.sh", + "scripts/validate_install_kit.sh", + "scripts/check_install_kit_vs_repo.sh", + "scripts/quality-gate.sh", + ]; + let mut manifest = String::new(); + for rel in files { + let path = kit.join(rel); + let digest = super::sha256_file(&path).unwrap(); + manifest.push_str(&format!("{digest} {DEFAULT_KIT_DIR}/{rel}\n")); + } + fs::write(kit.join("MANIFEST.txt"), manifest).unwrap(); + write_zip(root, matching_archives); + write_tar(root, matching_archives); + } + + fn archive_files(matching: bool) -> Vec<(&'static str, &'static [u8])> { + let mut files = vec![ + ("README-INSTALL-KIT.txt", b"readme".as_slice()), + ("MANIFEST.txt", b"manifest".as_slice()), + ("windows/deploy-ensemble.ps1", b"deploy".as_slice()), + ("windows/validate-deployment.ps1", b"validate".as_slice()), + ("ansible/deploy_aw_windows.yml", b"ansible".as_slice()), + ("aw-server/install_aw_server.sh", b"server".as_slice()), + ("scripts/rebuild_install_kit.sh", b"rebuild".as_slice()), + ( + "scripts/validate_install_kit.sh", + b"validate-kit".as_slice(), + ), + ("scripts/check_install_kit_vs_repo.sh", b"check".as_slice()), + ("scripts/quality-gate.sh", b"quality".as_slice()), + ]; + if !matching { + files.pop(); + files.push(("scripts/other.sh", b"other".as_slice())); + } + files + } + + fn write_zip(root: &std::path::Path, _matching: bool) { + let path = root.join(format!("{DEFAULT_KIT_DIR}.zip")); + let file = fs::File::create(path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let options = SimpleFileOptions::default(); + for (rel, data) in archive_files(true) { + zip.start_file(format!("{DEFAULT_KIT_DIR}/{rel}"), options) + .unwrap(); + zip.write_all(data).unwrap(); + } + zip.finish().unwrap(); + } + + fn write_tar(root: &std::path::Path, matching: bool) { + let path = root.join(format!("{DEFAULT_KIT_DIR}.tar.gz")); + let file = fs::File::create(path).unwrap(); + let encoder = GzEncoder::new(file, Compression::default()); + let mut builder = tar::Builder::new(encoder); + for (rel, data) in archive_files(matching) { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, format!("{DEFAULT_KIT_DIR}/{rel}"), data) + .unwrap(); + } + builder.finish().unwrap(); + } +} diff --git a/adk-rust/crates/verify-innosetup-installer/Cargo.toml b/adk-rust/crates/verify-innosetup-installer/Cargo.toml new file mode 100644 index 0000000..4503724 --- /dev/null +++ b/adk-rust/crates/verify-innosetup-installer/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "verify-innosetup-installer" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/verify-innosetup-installer/src/main.rs b/adk-rust/crates/verify-innosetup-installer/src/main.rs new file mode 100644 index 0000000..8e43c8b --- /dev/null +++ b/adk-rust/crates/verify-innosetup-installer/src/main.rs @@ -0,0 +1,263 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde::Serialize; + +const DEFAULT_INSTALLER: &str = "windows/installkit/innosetup/AWatch-rus-InstallKit.exe"; +const DEFAULT_WINEPREFIX: &str = "/tmp/aw-inno-verify-wineprefix"; +const INSTALL_DIR_WIN: &str = r"C:\AWatchRusExtract"; +const REQUIRED_FILES: &[&str] = &[ + "windows/AWatchRusCollectorGuardService.cs", + "windows/aw-collector-guard.ps1", + "windows/install-collector-guard-service.ps1", + "windows/dlp-policy.native-cross-os.example.json", +]; +const GUARD_MARKER: &str = "collector guard self-test OK"; + +#[derive(Debug, Parser)] +#[command(about = "Verify AWatch-rus InnoSetup installer payload through Wine")] +struct Cli { + #[arg(default_value = DEFAULT_INSTALLER)] + installer: PathBuf, + + #[arg(long, default_value = ".")] + root: PathBuf, + + #[arg(long, env = "WINEPREFIX_VERIFY", default_value = DEFAULT_WINEPREFIX)] + wineprefix: PathBuf, + + #[arg(long, default_value_t = false)] + json: bool, +} + +#[derive(Debug, Serialize)] +struct Report { + ok: bool, + installer: PathBuf, + wineprefix: PathBuf, + checked_files: Vec, + errors: Vec, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let root = cli + .root + .canonicalize() + .with_context(|| format!("canonicalize root {}", cli.root.display()))?; + let installer = resolve_path(&root, cli.installer); + let report = verify(&root, &installer, &cli.wineprefix); + print_report(&report, cli.json)?; + Ok(if report.ok { 0 } else { 1 }) +} + +fn verify(root: &Path, installer: &Path, wineprefix: &Path) -> Report { + let mut errors = Vec::new(); + if !installer.is_file() { + errors.push(format!("Installer not found: {}", installer.display())); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if command_path("wine").is_none() { + errors.push("wine not found".to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if command_path("wineboot").is_none() { + errors.push("wineboot not found".to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if command_path("wineserver").is_none() { + errors.push("wineserver not found".to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + + if wineprefix.exists() + && let Err(err) = fs::remove_dir_all(wineprefix) + { + errors.push(format!( + "failed to remove WINEPREFIX {}: {err}", + wineprefix.display() + )); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if let Err(err) = fs::create_dir_all(wineprefix) { + errors.push(format!( + "failed to create WINEPREFIX {}: {err}", + wineprefix.display() + )); + return report(false, installer, wineprefix, Vec::new(), errors); + } + + if let Err(err) = run_quiet( + Command::new("wineboot") + .arg("-u") + .env("WINEPREFIX", wineprefix) + .env( + "WINEDEBUG", + env::var("WINEDEBUG").unwrap_or_else(|_| "-all".to_string()), + ), + ) { + errors.push(err.to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if let Err(err) = run_quiet( + Command::new("wine") + .arg(installer) + .arg("/VERYSILENT") + .arg("/SUPPRESSMSGBOXES") + .arg("/NORESTART") + .arg("/SP-") + .arg("/TASKS=") + .arg(format!(r#"/DIR={INSTALL_DIR_WIN}"#)) + .env("WINEPREFIX", wineprefix) + .env( + "WINEDEBUG", + env::var("WINEDEBUG").unwrap_or_else(|_| "-all".to_string()), + ), + ) { + errors.push(err.to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + if let Err(err) = run_quiet( + Command::new("wineserver") + .arg("-w") + .env("WINEPREFIX", wineprefix), + ) { + errors.push(err.to_string()); + return report(false, installer, wineprefix, Vec::new(), errors); + } + + let install_dir = wineprefix.join("drive_c/AWatchRusExtract"); + let mut checked = Vec::new(); + for rel in REQUIRED_FILES { + let extracted = install_dir.join(rel); + let repo = root.join(rel); + if !extracted.is_file() { + errors.push(format!("Missing extracted file: {rel}")); + continue; + } + match files_equal(&repo, &extracted) { + Ok(true) => checked.push((*rel).to_string()), + Ok(false) => errors.push(format!("Extracted file differs from repo: {rel}")), + Err(err) => errors.push(err.to_string()), + } + } + let guard = install_dir.join("windows/aw-collector-guard.ps1"); + match fs::read_to_string(&guard) { + Ok(text) if text.contains(GUARD_MARKER) => {} + Ok(_) => { + errors.push("Guard self-test marker missing in extracted installer payload".to_string()) + } + Err(err) => errors.push(format!( + "read extracted guard script {}: {err}", + guard.display() + )), + } + + report(errors.is_empty(), installer, wineprefix, checked, errors) +} + +fn run_quiet(command: &mut Command) -> Result<()> { + let program = command.get_program().to_string_lossy().to_string(); + let status = command + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .with_context(|| format!("run {program}"))?; + if !status.success() { + bail!("{program} failed with status {status}"); + } + Ok(()) +} + +fn files_equal(left: &Path, right: &Path) -> Result { + let left = fs::read(left).with_context(|| format!("read {}", left.display()))?; + let right = fs::read(right).with_context(|| format!("read {}", right.display()))?; + Ok(left == right) +} + +fn print_report(report: &Report, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(report)?); + return Ok(()); + } + for err in &report.errors { + eprintln!("{err}"); + } + if report.ok { + println!("verify_innosetup_installer: OK"); + } + Ok(()) +} + +fn report( + ok: bool, + installer: &Path, + wineprefix: &Path, + checked_files: Vec, + errors: Vec, +) -> Report { + Report { + ok, + installer: installer.to_path_buf(), + wineprefix: wineprefix.to_path_buf(), + checked_files, + errors, + } +} + +fn command_path(name: &str) -> Option { + let path = env::var_os("PATH")?; + env::split_paths(&path) + .map(|dir| dir.join(name)) + .find(|candidate| candidate.is_file()) +} + +fn resolve_path(root: &Path, path: PathBuf) -> PathBuf { + if path.is_absolute() { + path + } else { + root.join(path) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::{files_equal, resolve_path}; + + #[test] + fn compares_files_by_bytes() { + let tmp = tempdir().unwrap(); + let a = tmp.path().join("a"); + let b = tmp.path().join("b"); + fs::write(&a, b"same").unwrap(); + fs::write(&b, b"same").unwrap(); + assert!(files_equal(&a, &b).unwrap()); + fs::write(&b, b"different").unwrap(); + assert!(!files_equal(&a, &b).unwrap()); + } + + #[test] + fn resolves_relative_path_against_root() { + let root = std::path::Path::new("/tmp/root"); + assert_eq!(resolve_path(root, "a/b".into()), root.join("a/b")); + } +} diff --git a/adk-rust/crates/worktime-api/Cargo.toml b/adk-rust/crates/worktime-api/Cargo.toml new file mode 100644 index 0000000..c206f08 --- /dev/null +++ b/adk-rust/crates/worktime-api/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "worktime-api" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tiny_http.workspace = true +url.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/worktime-api/src/main.rs b/adk-rust/crates/worktime-api/src/main.rs new file mode 100644 index 0000000..1530572 --- /dev/null +++ b/adk-rust/crates/worktime-api/src/main.rs @@ -0,0 +1,2585 @@ +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + fs, + io::Cursor, + path::PathBuf, + sync::{Arc, Mutex}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, anyhow}; +use chrono::{ + DateTime, Datelike, FixedOffset, Local, NaiveDate, SecondsFormat, TimeDelta, TimeZone, Utc, +}; +use clap::Parser; +use reqwest::{ + blocking::Client, + header::{CONNECTION, HeaderMap, HeaderValue}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; +use url::form_urlencoded; + +const DEFAULT_AW_URL: &str = "http://127.0.0.1:5600"; +const DEFAULT_HOST: &str = "SHARKON2025"; + +#[derive(Debug, Parser)] +#[command(about = "AW Worktime Report API")] +struct Cli { + #[arg(long)] + once: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + ioc_dir: PathBuf, + default_host: String, + default_sample_seconds: f64, + max_sample_seconds: f64, + listen_host: String, + listen_port: u16, + workday_start_hour: u32, + workday_end_hour: u32, + manager_target_coverage_pct: i64, + manager_low_coverage_pct: i64, + manager_late_start_grace_minutes: i64, + manager_early_finish_grace_minutes: i64, + manager_critical_source_max_age_seconds: i64, + manager_web_source_max_age_seconds: i64, + manager_session_source_max_age_seconds: i64, + manager_infra_source_max_age_seconds: i64, + manager_aliases_json: PathBuf, + manager_exclude_users: BTreeSet, + events_cache_ttl_seconds: i64, + worktime_events_limit: usize, + aw_http_timeout_seconds: f64, + source_http_timeout_seconds: f64, + report_cache_ttl_seconds: i64, + report_stale_ttl_seconds: i64, + report_disk_cache_dir: PathBuf, + report_disk_stale_ttl_seconds: i64, + true_active_evidence_window_seconds: i64, + true_active_max_event_seconds: i64, + offset: FixedOffset, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct AwEvent { + timestamp: Option, + #[serde(default)] + duration: Option, + #[serde(default)] + data: Value, +} + +#[derive(Debug, Clone)] +struct UserBucket { + user: String, + user_id: String, + samples_count: i64, + active_samples: i64, + session_ids: BTreeSet, + intervals: Vec<(DateTime, DateTime)>, +} + +#[derive(Debug, Clone)] +struct AliasProfile { + display_name: String, + manager_owner: String, + department: String, + role: String, + notes: String, + canonical_user_id: String, + exclude: bool, +} + +#[derive(Debug, Clone)] +struct CachedResponse { + stored: Instant, + data: Vec, + content_type: String, +} + +type EventsCache = Arc)>>>; +type ReportCache = Arc>>; +type IdentitySamples = BTreeMap<(String, String), Vec<(DateTime, AwEvent)>>; +type DateBounds = (DateTime, DateTime); +type EventsForDate = (DateBounds, Vec); + +#[derive(Clone)] +struct App { + config: Arc, + aw: Client, + events_cache: EventsCache, + report_cache: ReportCache, +} + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .filter(|value| *value > 0.0) + .unwrap_or(fallback) +} + +fn env_i64(name: &str, fallback: i64) -> i64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn env_usize(name: &str, fallback: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(fallback) +} + +fn build_aw_api_base(raw_url: &str) -> String { + let url = raw_url.trim().trim_end_matches('/'); + if url.ends_with("/api/0") { + url.to_string() + } else { + format!("{url}/api/0") + } +} + +fn load_config() -> Config { + let default_sample_seconds = env_f64("AW_WORKTIME_DEFAULT_SAMPLE_SECONDS", 30.0).max(1.0); + let max_sample_seconds = + env_f64("AW_WORKTIME_MAX_SAMPLE_SECONDS", 300.0).max(default_sample_seconds); + let manager_target_coverage_pct = + env_i64("AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT", 75).clamp(1, 100); + let manager_low_coverage_pct = + env_i64("AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT", 35).clamp(1, 100); + let report_cache_ttl_seconds = env_i64("AW_WORKTIME_REPORT_CACHE_TTL_SECONDS", 60).max(0); + let report_stale_ttl_seconds = + env_i64("AW_WORKTIME_REPORT_STALE_TTL_SECONDS", 900).max(report_cache_ttl_seconds); + Config { + aw_api_base: build_aw_api_base(&env("AW_SERVER_URL", DEFAULT_AW_URL)), + ioc_dir: PathBuf::from(env("AW_DLP_IOC_DIR", "/opt/activitywatch/dlp-ioc/output")), + default_host: env("AW_WORKTIME_HOST", DEFAULT_HOST), + default_sample_seconds, + max_sample_seconds, + listen_host: env("AW_WORKTIME_LISTEN_HOST", "0.0.0.0"), + listen_port: env("AW_WORKTIME_PORT", "5610").parse().unwrap_or(5610), + workday_start_hour: env_i64("AW_WORKTIME_MANAGER_START_HOUR", 9).clamp(0, 23) as u32, + workday_end_hour: env_i64("AW_WORKTIME_MANAGER_END_HOUR", 18).clamp(0, 23) as u32, + manager_target_coverage_pct, + manager_low_coverage_pct, + manager_late_start_grace_minutes: env_i64( + "AW_WORKTIME_MANAGER_LATE_START_GRACE_MINUTES", + 60, + ) + .max(0), + manager_early_finish_grace_minutes: env_i64( + "AW_WORKTIME_MANAGER_EARLY_FINISH_GRACE_MINUTES", + 90, + ) + .max(0), + manager_critical_source_max_age_seconds: env_i64( + "AW_WORKTIME_MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS", + 900, + ) + .max(60), + manager_web_source_max_age_seconds: env_i64( + "AW_WORKTIME_MANAGER_WEB_SOURCE_MAX_AGE_SECONDS", + 259200, + ) + .max(3600), + manager_session_source_max_age_seconds: env_i64( + "AW_WORKTIME_MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS", + 604800, + ) + .max(3600), + manager_infra_source_max_age_seconds: env_i64( + "AW_WORKTIME_MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS", + 172800, + ) + .max(3600), + manager_aliases_json: PathBuf::from(env( + "AW_WORKTIME_MANAGER_ALIASES_JSON", + "/etc/activitywatch/worktime-manager-aliases.json", + )), + manager_exclude_users: env("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "") + .split(',') + .map(|item| item.trim().to_lowercase()) + .filter(|item| !item.is_empty()) + .collect(), + events_cache_ttl_seconds: env_i64("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", 30).max(0), + worktime_events_limit: env_usize("AW_WORKTIME_EVENTS_LIMIT", 50_000).max(1000), + aw_http_timeout_seconds: env_f64("AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS", 8.0).max(0.5), + source_http_timeout_seconds: env_f64("AW_WORKTIME_SOURCE_HTTP_TIMEOUT_SECONDS", 1.5) + .max(0.25), + report_cache_ttl_seconds, + report_stale_ttl_seconds, + report_disk_cache_dir: PathBuf::from(env( + "AW_WORKTIME_REPORT_DISK_CACHE_DIR", + "/var/lib/activitywatch/worktime-report-cache", + )), + report_disk_stale_ttl_seconds: env_i64("AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS", 86400) + .max(report_stale_ttl_seconds), + true_active_evidence_window_seconds: env_i64( + "AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", + 180, + ) + .max(30), + true_active_max_event_seconds: env_i64("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", 600) + .max(30), + offset: FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"), + } +} + +impl App { + fn new(config: Config) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(CONNECTION, HeaderValue::from_static("close")); + let aw = Client::builder() + .timeout(Duration::from_secs_f64(config.aw_http_timeout_seconds)) + .no_proxy() + .pool_max_idle_per_host(0) + .default_headers(headers) + .build() + .context("build AW HTTP client")?; + Ok(Self { + config: Arc::new(config), + aw, + events_cache: Arc::new(Mutex::new(HashMap::new())), + report_cache: Arc::new(Mutex::new(HashMap::new())), + }) + } + + fn aw_get_json_once(&self, path: &str) -> Result { + self.aw_get_json_with(path, 1, None) + } + + fn aw_get_json_with( + &self, + path: &str, + attempts: usize, + timeout: Option, + ) -> Result { + let url = format!("{}{}", self.config.aw_api_base, path); + let mut last_error = None; + for attempt in 1..=attempts.max(1) { + let mut request = self.aw.get(&url); + if let Some(timeout) = timeout { + request = request.timeout(timeout); + } + match request.send() { + Ok(response) => { + if !response.status().is_success() { + return Err(anyhow!("AW {path} returned HTTP {}", response.status())); + } + return response + .json() + .with_context(|| format!("decode AW JSON {path}")); + } + Err(error) => { + let is_timeout = error.is_timeout(); + last_error = Some(error); + if is_timeout { + break; + } + if attempt < attempts { + std::thread::sleep(Duration::from_millis(250)); + } + } + } + } + Err(last_error.expect("request failed")).with_context(|| format!("request AW {path}")) + } + + fn fetch_bucket_events( + &self, + bucket_id: &str, + start: Option>, + end: Option>, + ) -> Vec { + match self.fetch_bucket_events_result(bucket_id, start, end) { + Ok(events) => events, + Err(error) => { + eprintln!( + "[aw-worktime-api-rust] events fetch failed bucket={bucket_id}: {error:#}" + ); + Vec::new() + } + } + } + + fn fetch_bucket_events_result( + &self, + bucket_id: &str, + start: Option>, + end: Option>, + ) -> Result> { + let mut key = bucket_id.to_string(); + let query_limit = self.config.worktime_events_limit.min(5_000); + let params = [format!("limit={query_limit}")]; + if let (Some(start), Some(end)) = (start, end) { + let s = to_iso_utc(start); + let e = to_iso_utc(end); + key = format!("{bucket_id}|{s}|{e}"); + } + if self.config.events_cache_ttl_seconds > 0 { + if let Some((stored, events)) = self + .events_cache + .lock() + .ok() + .and_then(|cache| cache.get(&key).cloned()) + { + if stored.elapsed().as_secs() <= self.config.events_cache_ttl_seconds as u64 { + return Ok(events); + } + } + } + let path = format!("/buckets/{bucket_id}/events?{}", params.join("&")); + let mut events: Vec = serde_json::from_value(self.aw_get_json_once(&path)?) + .with_context(|| format!("decode AW events for {bucket_id}"))?; + if let (Some(start), Some(end)) = (start, end) { + events.retain(|event| { + event + .timestamp + .as_deref() + .and_then(parse_iso_utc) + .is_some_and(|ts| ts >= start && ts <= end) + }); + } + if self.config.events_cache_ttl_seconds > 0 { + if let Ok(mut cache) = self.events_cache.lock() { + cache.insert(key, (Instant::now(), events.clone())); + } + } + Ok(events) + } + + fn latest_bucket_event(&self, bucket_id: &str) -> Option { + let bucket = self + .aw_get_json_with( + &format!("/buckets/{bucket_id}"), + 1, + Some(Duration::from_secs_f64( + self.config.source_http_timeout_seconds, + )), + ) + .ok()?; + let end = bucket.pointer("/metadata/end").and_then(Value::as_str)?; + Some(AwEvent { + timestamp: Some(end.to_string()), + duration: Some(0.0), + data: json!({"source": "bucket_metadata"}), + }) + } + + fn report_response( + &self, + path: &str, + params: &Params, + accept: &str, + ) -> (Vec, String, Vec<(String, String)>) { + let fmt = resolve_report_format(params, accept); + let host = resolve_host(&self.config, params.first("host").as_deref()); + let report_date = resolve_report_date( + &self.config, + params.first("day").as_deref(), + params.first("date").as_deref(), + ); + let owner = normalize_filter(params.first("owner").as_deref().unwrap_or("")); + let department = normalize_filter(params.first("department").as_deref().unwrap_or("")); + let cache_key = make_report_cache_key( + path, + &fmt, + &host, + report_date, + params.first("day").as_deref().unwrap_or(""), + &owner, + &department, + ); + if let Some(cached) = self.get_report_cache(&cache_key, false) { + return ( + cached.data, + cached.content_type, + vec![ + ("X-AW-Worktime-Cache".into(), "fresh".into()), + ("X-AW-Worktime-Cache-Reason".into(), "ttl".into()), + ], + ); + } + let built = + self.build_report_response(path, params, &fmt, &host, report_date, &owner, &department); + match built { + Ok((data, content_type)) => { + self.save_report_cache(cache_key, data.clone(), content_type.clone()); + (data, content_type, Vec::new()) + } + Err(error) => { + eprintln!("[aw-worktime-api-rust] report build failed path={path}: {error:#}"); + if let Some(cached) = self.get_report_cache(&cache_key, true) { + return ( + cached.data, + cached.content_type, + vec![ + ("X-AW-Worktime-Cache".into(), "stale".into()), + ("X-AW-Worktime-Cache-Reason".into(), "build-error".into()), + ], + ); + } + let data = serde_json::to_vec_pretty(&json!({ + "ok": false, + "error": "report_unavailable", + "message": "report build failed and no cached response is available", + "generated_at_utc": to_iso_utc(Utc::now()), + })) + .unwrap_or_default(); + ( + data, + "application/json; charset=utf-8".to_string(), + Vec::new(), + ) + } + } + } + + fn get_report_cache(&self, key: &str, allow_stale: bool) -> Option { + let cached = self.report_cache.lock().ok()?.get(key).cloned(); + if let Some(cached) = cached { + let max_age = if allow_stale { + self.config.report_stale_ttl_seconds + } else { + self.config.report_cache_ttl_seconds + }; + if max_age > 0 && cached.stored.elapsed().as_secs() <= max_age as u64 { + return Some(cached); + } + } + if allow_stale { + return self.load_disk_cache(key); + } + None + } + + fn save_report_cache(&self, key: String, data: Vec, content_type: String) { + if self.config.report_stale_ttl_seconds <= 0 { + return; + } + if let Ok(mut cache) = self.report_cache.lock() { + cache.insert( + key.clone(), + CachedResponse { + stored: Instant::now(), + data: data.clone(), + content_type: content_type.clone(), + }, + ); + } + self.save_disk_cache(&key, &data, &content_type); + } + + fn disk_cache_path(&self, key: &str) -> PathBuf { + let mut h: u64 = 1469598103934665603; + for b in key.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(1099511628211); + } + self.config + .report_disk_cache_dir + .join(format!("{h:016x}.json")) + } + + fn save_disk_cache(&self, key: &str, data: &[u8], content_type: &str) { + if self.config.report_disk_stale_ttl_seconds <= 0 { + return; + } + let path = self.disk_cache_path(key); + let _ = fs::create_dir_all(&self.config.report_disk_cache_dir); + let payload = json!({ + "stored_epoch": SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs_f64()).unwrap_or(0.0), + "content_type": content_type, + "data": String::from_utf8_lossy(data), + }); + let _ = fs::write(path, serde_json::to_vec(&payload).unwrap_or_default()); + } + + fn load_disk_cache(&self, key: &str) -> Option { + let path = self.disk_cache_path(key); + let payload: Value = serde_json::from_slice(&fs::read(path).ok()?).ok()?; + let stored_epoch = payload.get("stored_epoch")?.as_f64()?; + let now_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok()? + .as_secs_f64(); + if now_epoch - stored_epoch > self.config.report_disk_stale_ttl_seconds as f64 { + return None; + } + Some(CachedResponse { + stored: Instant::now() - Duration::from_secs_f64((now_epoch - stored_epoch).max(0.0)), + data: payload.get("data")?.as_str()?.as_bytes().to_vec(), + content_type: payload.get("content_type")?.as_str()?.to_string(), + }) + } + + #[allow(clippy::too_many_arguments)] + fn build_report_response( + &self, + path: &str, + params: &Params, + fmt: &str, + host: &str, + report_date: NaiveDate, + owner: &str, + department: &str, + ) -> Result<(Vec, String)> { + let is_management = path == "/reports/worktime/management"; + if is_management { + let payload = self.management_report_for_date(host, report_date, owner, department)?; + return Ok(match fmt { + "csv" => ( + management_csv(&payload).into_bytes(), + "text/csv; charset=utf-8".to_string(), + ), + "html" => ( + render_management_html(&payload).into_bytes(), + "text/html; charset=utf-8".to_string(), + ), + _ => ( + serde_json::to_vec_pretty(&payload)?, + "application/json; charset=utf-8".to_string(), + ), + }); + } + + let (bounds, events) = self.fetch_events_for_date(host, report_date)?; + let rows = aggregate_rows(&self.config, &events, bounds.0, bounds.1, host, false); + let total_active_seconds = rows + .iter() + .map(|row| { + row.get("active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0) + }) + .sum::(); + let true_active_apps = if total_active_seconds > 0 { + self.build_true_active_apps(host, report_date) + } else { + Vec::new() + }; + let day = params.first("day"); + Ok(match fmt { + "csv" => ( + today_csv(&rows).into_bytes(), + "text/csv; charset=utf-8".to_string(), + ), + "html" => ( + render_today_html( + &self.config, + &rows, + host, + report_date, + day.as_deref(), + &true_active_apps, + ) + .into_bytes(), + "text/html; charset=utf-8".to_string(), + ), + _ => { + let payload = json!({ + "generated_at_utc": to_iso_utc(Utc::now()), + "report_timezone": "Europe/Moscow", + "host": host, + "report_date": report_date.to_string(), + "bucket_id": sessions_bucket(host), + "rows": rows, + "true_active_apps": true_active_apps, + }); + ( + serde_json::to_vec_pretty(&payload)?, + "application/json; charset=utf-8".to_string(), + ) + } + }) + } + + fn fetch_events_for_date(&self, host: &str, report_date: NaiveDate) -> Result { + let bounds = report_bounds(&self.config, report_date); + let events = self.fetch_bucket_events_result( + &sessions_bucket(host), + Some(bounds.0), + Some(bounds.1), + )?; + Ok((bounds, events)) + } + + fn management_report_for_date( + &self, + host: &str, + report_date: NaiveDate, + owner: &str, + department: &str, + ) -> Result { + let (bounds, events) = self.fetch_events_for_date(host, report_date)?; + let rows = aggregate_rows(&self.config, &events, bounds.0, bounds.1, host, true); + Ok(self.build_management_payload(rows, host, report_date, owner, department)) + } + + fn build_true_active_apps(&self, host: &str, report_date: NaiveDate) -> Vec { + let bounds = report_bounds(&self.config, report_date); + let window_events = { + let primary = self.fetch_bucket_events( + &format!("aw-watcher-window_{host}"), + Some(bounds.0), + Some(bounds.1), + ); + if primary.is_empty() { + self.fetch_bucket_events( + &format!("aw-rdp-window_{host}"), + Some(bounds.0), + Some(bounds.1), + ) + } else { + primary + } + }; + let afk_events = { + let primary = self.fetch_bucket_events( + &format!("aw-watcher-afk_{host}"), + Some(bounds.0), + Some(bounds.1), + ); + if primary.is_empty() { + self.fetch_bucket_events( + &format!("aw-rdp-afk_{host}"), + Some(bounds.0), + Some(bounds.1), + ) + } else { + primary + } + }; + let mut evidence = HashMap::new(); + for bucket in [ + format!("aw-file-operations_{host}"), + format!("aw-dlp-endpoint-signals_{host}"), + format!("aw-watcher-web-chrome_{host}"), + format!("aw-watcher-web-edge_{host}"), + format!("aw-detmir-web-category_{host}"), + ] { + evidence.insert( + bucket.clone(), + self.fetch_bucket_events(&bucket, Some(bounds.0), Some(bounds.1)), + ); + } + build_true_active_apps_from_events( + &self.config, + &window_events, + &afk_events, + &evidence, + bounds.0, + bounds.1, + ) + } +} + +#[derive(Debug, Clone, Default)] +struct Params(HashMap>); + +impl Params { + fn parse(query: &str) -> Self { + let mut map: HashMap> = HashMap::new(); + for (k, v) in form_urlencoded::parse(query.as_bytes()) { + if !v.is_empty() { + map.entry(k.into_owned()).or_default().push(v.into_owned()); + } + } + Self(map) + } + fn first(&self, name: &str) -> Option { + self.0.get(name).and_then(|v| v.first()).cloned() + } +} + +fn parse_iso_utc(value: &str) -> Option> { + DateTime::parse_from_rfc3339(value) + .map(|dt| dt.with_timezone(&Utc)) + .ok() +} + +fn to_iso_utc(dt: DateTime) -> String { + dt.to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn hhmm(seconds: i64) -> String { + let seconds = seconds.max(0); + format!("{:02}:{:02}", seconds / 3600, (seconds % 3600) / 60) +} + +fn human_duration_ru(seconds: i64) -> String { + let seconds = seconds.max(0); + let h = seconds / 3600; + let m = (seconds % 3600) / 60; + if h > 0 && m > 0 { + format!("{h} ч {m} мин") + } else if h > 0 { + format!("{h} ч") + } else if m > 0 { + format!("{m} мин") + } else { + format!("{seconds} сек") + } +} + +fn resolve_host(config: &Config, host: Option<&str>) -> String { + let h = host.unwrap_or(&config.default_host).trim(); + if h.is_empty() { + config.default_host.clone() + } else { + h.to_string() + } +} + +fn sessions_bucket(host: &str) -> String { + format!("aw-worktime-sessions_{host}") +} + +fn resolve_report_date(config: &Config, day: Option<&str>, date: Option<&str>) -> NaiveDate { + if let Some(date) = date.and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok()) { + return date; + } + let today = Utc::now().with_timezone(&config.offset).date_naive(); + if day == Some("yesterday") { + today - TimeDelta::days(1) + } else { + today + } +} + +fn report_bounds(config: &Config, report_date: NaiveDate) -> (DateTime, DateTime) { + let start_local = config + .offset + .with_ymd_and_hms( + report_date.year(), + report_date.month(), + report_date.day(), + 0, + 0, + 0, + ) + .single() + .unwrap(); + let start = start_local.with_timezone(&Utc); + let end = (start_local + TimeDelta::days(1) - TimeDelta::seconds(1)).with_timezone(&Utc); + (start, end) +} + +fn workday_bounds( + config: &Config, + report_date: NaiveDate, +) -> (DateTime, DateTime, i64) { + let start = config + .offset + .with_ymd_and_hms( + report_date.year(), + report_date.month(), + report_date.day(), + config.workday_start_hour, + 0, + 0, + ) + .single() + .unwrap(); + let mut end = config + .offset + .with_ymd_and_hms( + report_date.year(), + report_date.month(), + report_date.day(), + config.workday_end_hour, + 0, + 0, + ) + .single() + .unwrap(); + if end <= start { + end = start + TimeDelta::hours(8); + } + let duration = (end - start).num_seconds(); + (start, end, duration) +} + +fn value_string(data: &Value, key: &str) -> String { + match data.get(key) { + Some(Value::String(value)) => value.trim().to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + Some(Value::Null) | None => String::new(), + Some(value) => value.to_string().trim_matches('"').trim().to_string(), + } +} + +fn is_machine_user(user: &str) -> bool { + let u = user.trim().to_lowercase(); + u.ends_with('$') || matches!(u.as_str(), "system" | "localservice" | "networkservice") +} + +fn is_active_sample(data: &Value) -> bool { + if data.get("active").and_then(Value::as_bool).unwrap_or(false) { + return true; + } + let state = value_string(data, "state").to_lowercase(); + if state.contains("актив") || state == "active" { + return true; + } + if state == "unknown" { + let sid = value_string(data, "sessionId").parse::().unwrap_or(-1); + let user = value_string(data, "username"); + let session_name = value_string(data, "sessionName").to_lowercase(); + return sid > 0 + && !user.is_empty() + && !is_machine_user(&user) + && (session_name.starts_with("rdp-") || session_name == "console"); + } + false +} + +fn normalize_user_id(data: &Value, host: &str, username: &str) -> String { + let user_id = value_string(data, "userId"); + if !user_id.is_empty() { + if let Some((_, right)) = user_id.split_once('\\') { + return format!("{host}\\{right}"); + } + return user_id; + } + format!("{host}\\{username}") +} + +fn clamp_seconds(config: &Config, value: f64, fallback: f64) -> f64 { + let seconds = if value > 0.0 { value } else { fallback }; + seconds.min(config.max_sample_seconds).max(1.0) +} + +fn event_sample_seconds( + config: &Config, + event: &AwEvent, + ts: DateTime, + next_ts: Option>, +) -> f64 { + for key in ["sampleSeconds", "pollSeconds"] { + let value = event.data.get(key).and_then(|v| { + if let Some(n) = v.as_f64() { + Some(n) + } else { + v.as_str().and_then(|s| s.parse().ok()) + } + }); + if let Some(value) = value.filter(|v| *v > 0.0) { + return clamp_seconds(config, value, config.default_sample_seconds); + } + } + if let Some(duration) = event.duration.filter(|v| *v > 0.0) { + return clamp_seconds(config, duration, config.default_sample_seconds); + } + if let Some(next_ts) = next_ts { + let delta = (next_ts - ts).num_milliseconds() as f64 / 1000.0; + if delta > 0.0 { + return clamp_seconds(config, delta, config.default_sample_seconds); + } + } + clamp_seconds( + config, + config.default_sample_seconds, + config.default_sample_seconds, + ) +} + +fn merge_intervals( + mut intervals: Vec<(DateTime, DateTime)>, +) -> Vec<(DateTime, DateTime)> { + intervals.sort_by_key(|i| i.0); + let mut merged: Vec<(DateTime, DateTime)> = Vec::new(); + for (start, end) in intervals { + if let Some(last) = merged.last_mut() { + if start <= last.1 { + if end > last.1 { + last.1 = end; + } + continue; + } + } + merged.push((start, end)); + } + merged +} + +fn aggregate_rows( + config: &Config, + events: &[AwEvent], + start: DateTime, + end: DateTime, + host: &str, + intervals: bool, +) -> Vec { + let mut by_identity: IdentitySamples = BTreeMap::new(); + for event in events { + let Some(ts) = event.timestamp.as_deref().and_then(parse_iso_utc) else { + continue; + }; + if ts < start || ts > end { + continue; + } + let username = value_string(&event.data, "username"); + if username.is_empty() { + continue; + } + let session_id = value_string(&event.data, "sessionId"); + by_identity + .entry(( + username, + if session_id.is_empty() { + "unknown".into() + } else { + session_id + }, + )) + .or_default() + .push((ts, event.clone())); + } + let end_exclusive = end + TimeDelta::seconds(1); + let mut by_user: BTreeMap = BTreeMap::new(); + for ((username, session_id), mut samples) in by_identity { + samples.sort_by_key(|item| item.0); + for idx in 0..samples.len() { + let (ts, event) = &samples[idx]; + let next_ts = samples.get(idx + 1).map(|item| item.0); + let active = is_active_sample(&event.data); + let row = by_user + .entry(username.clone()) + .or_insert_with(|| UserBucket { + user: username.clone(), + user_id: normalize_user_id(&event.data, host, &username), + samples_count: 0, + active_samples: 0, + session_ids: BTreeSet::new(), + intervals: Vec::new(), + }); + row.samples_count += 1; + row.session_ids.insert(session_id.clone()); + if active { + row.active_samples += 1; + let sample_seconds = event_sample_seconds(config, event, *ts, next_ts); + let interval_end = (*ts + + TimeDelta::milliseconds((sample_seconds * 1000.0) as i64)) + .min(end_exclusive); + if interval_end > *ts { + row.intervals.push((*ts, interval_end)); + } + } + } + } + let full_range = (end - start).num_seconds() + 1; + let mut rows = Vec::new(); + for (_, row) in by_user { + let merged = merge_intervals(row.intervals); + let active_seconds = merged + .iter() + .map(|(s, e)| (*e - *s).num_seconds()) + .sum::() + .min(full_range); + let first = merged.first().map(|i| to_iso_utc(i.0)).unwrap_or_default(); + let last = merged.last().map(|i| to_iso_utc(i.1)).unwrap_or_default(); + let mut obj = Map::new(); + obj.insert("user".into(), json!(row.user)); + obj.insert("user_id".into(), json!(row.user_id)); + obj.insert("active_seconds".into(), json!(active_seconds)); + obj.insert("active_hhmm".into(), json!(hhmm(active_seconds))); + obj.insert("first_activity".into(), json!(first)); + obj.insert("last_activity".into(), json!(last)); + obj.insert( + "idle_seconds".into(), + json!((full_range - active_seconds).max(0)), + ); + obj.insert("sessions_count".into(), json!(row.session_ids.len())); + obj.insert("samples_count".into(), json!(row.samples_count)); + obj.insert("active_samples".into(), json!(row.active_samples)); + if intervals { + obj.insert( + "_intervals".into(), + json!( + merged + .iter() + .map(|(s, e)| json!([to_iso_utc(*s), to_iso_utc(*e)])) + .collect::>() + ), + ); + } + rows.push(Value::Object(obj)); + } + rows +} + +fn build_report_summary(rows: &[Value]) -> Value { + let total_active = rows + .iter() + .map(|r| r.get("active_seconds").and_then(Value::as_i64).unwrap_or(0)) + .sum::(); + let first = rows + .iter() + .filter_map(|r| r.get("first_activity").and_then(Value::as_str)) + .filter(|s| !s.is_empty()) + .min() + .unwrap_or(""); + let last = rows + .iter() + .filter_map(|r| r.get("last_activity").and_then(Value::as_str)) + .filter(|s| !s.is_empty()) + .max() + .unwrap_or(""); + let top = rows + .iter() + .max_by_key(|r| r.get("active_seconds").and_then(Value::as_i64).unwrap_or(0)); + json!({ + "users_count": rows.len(), + "total_active_seconds": total_active, + "total_active_hhmm": hhmm(total_active), + "first_activity": first, + "last_activity": last, + "top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""), + "top_user_active_hhmm": top.and_then(|r| r.get("active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"), + }) +} + +fn normalize_key(value: &str) -> String { + value.trim().to_lowercase() +} + +fn default_display_name(user: &str, user_id: &str) -> String { + let mut base = if !user_id.trim().is_empty() { + user_id.trim().to_string() + } else { + user.trim().to_string() + }; + if let Some((_, right)) = base.split_once('\\') { + base = right.to_string(); + } + if base.is_ascii() && base.to_lowercase() == base && base.chars().any(|c| c.is_alphabetic()) { + base.to_uppercase() + } else { + base + } +} + +fn load_aliases(config: &Config) -> Value { + fs::read_to_string(&config.manager_aliases_json) + .ok() + .and_then(|raw| serde_json::from_str(&raw).ok()) + .unwrap_or_else(|| json!({})) +} + +fn resolve_user_alias(config: &Config, user: &str, user_id: &str, host: &str) -> AliasProfile { + let raw = load_aliases(config); + let users = raw.get("users").unwrap_or(&raw); + let candidates = [ + normalize_key(user_id), + normalize_key(&format!("{host}\\{user}")), + normalize_key(user), + ]; + let mut alias = json!({}); + if let Some(map) = users.as_object() { + for candidate in candidates { + if let Some(value) = map.get(&candidate) { + alias = if value.is_string() { + json!({"display_name": value.as_str().unwrap_or("")}) + } else { + value.clone() + }; + break; + } + } + } + let display_name = value_string(&alias, "display_name") + .if_empty(value_string(&alias, "name")) + .if_empty(default_display_name(user, user_id)); + let manager_owner = value_string(&alias, "manager") + .if_empty(value_string(&alias, "owner")) + .if_empty(display_name.clone()); + let exclude = alias + .get("exclude") + .and_then(Value::as_bool) + .unwrap_or(false) + || config.manager_exclude_users.contains(&normalize_key(user)) + || config + .manager_exclude_users + .contains(&normalize_key(&display_name)); + AliasProfile { + display_name, + manager_owner, + department: value_string(&alias, "department"), + role: value_string(&alias, "role"), + notes: value_string(&alias, "notes"), + canonical_user_id: value_string(&alias, "canonical_user_id").if_empty(user_id.to_string()), + exclude, + } +} + +trait IfEmpty { + fn if_empty(self, fallback: String) -> String; +} +impl IfEmpty for String { + fn if_empty(self, fallback: String) -> String { + if self.trim().is_empty() { + fallback + } else { + self + } + } +} + +fn normalize_filter(value: &str) -> String { + value.split_whitespace().collect::>().join(" ") +} + +fn filter_key(value: &str) -> String { + normalize_filter(value).to_lowercase() +} + +fn interval_overlap_seconds( + intervals: &[Value], + start: DateTime, + end: DateTime, +) -> (i64, Option>, Option>) { + let mut total = 0; + let mut first = None; + let mut last = None; + for item in intervals { + let Some(arr) = item.as_array() else { + continue; + }; + if arr.len() != 2 { + continue; + } + let Some(s) = arr[0].as_str().and_then(parse_iso_utc) else { + continue; + }; + let Some(e) = arr[1].as_str().and_then(parse_iso_utc) else { + continue; + }; + let os = s.max(start); + let oe = e.min(end); + if oe <= os { + continue; + } + total += (oe - os).num_seconds(); + if first.is_none_or(|f| os < f) { + first = Some(os); + } + if last.is_none_or(|l| oe > l) { + last = Some(oe); + } + } + (total, first, last) +} + +impl App { + fn build_management_payload( + &self, + rows: Vec, + host: &str, + report_date: NaiveDate, + owner_filter: &str, + department_filter: &str, + ) -> Value { + let (day_start, day_end) = report_bounds(&self.config, report_date); + let (work_start_local, work_end_local, work_duration) = + workday_bounds(&self.config, report_date); + let now_local = Utc::now().with_timezone(&self.config.offset); + let is_today = report_date == now_local.date_naive(); + let effective_end_local = if is_today && now_local < work_end_local { + now_local + } else { + work_end_local + }; + let expected_seconds = if is_today { + (effective_end_local - work_start_local) + .num_seconds() + .max(0) + .min(work_duration) + } else { + work_duration + }; + let target_seconds = expected_seconds * self.config.manager_target_coverage_pct / 100; + let low_seconds = expected_seconds * self.config.manager_low_coverage_pct / 100; + let late_start = + work_start_local + TimeDelta::minutes(self.config.manager_late_start_grace_minutes); + let early_finish = + work_end_local - TimeDelta::minutes(self.config.manager_early_finish_grace_minutes); + let owner_filter = normalize_filter(owner_filter); + let department_filter = normalize_filter(department_filter); + + let mut roster = Vec::new(); + let mut actions = Vec::new(); + for row in rows { + let user = row.get("user").and_then(Value::as_str).unwrap_or(""); + let user_id = row.get("user_id").and_then(Value::as_str).unwrap_or(""); + let alias = resolve_user_alias(&self.config, user, user_id, host); + if alias.exclude { + continue; + } + if !owner_filter.is_empty() + && filter_key(&alias.manager_owner) != filter_key(&owner_filter) + { + continue; + } + if !department_filter.is_empty() + && filter_key(&alias.department) != filter_key(&department_filter) + { + continue; + } + let intervals = row + .get("_intervals") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let (work_secs, work_first, work_last) = interval_overlap_seconds( + &intervals, + work_start_local.with_timezone(&Utc), + effective_end_local.with_timezone(&Utc), + ); + let calendar_secs = row + .get("active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0); + let coverage = if expected_seconds > 0 { + clamp_pct(work_secs as f64 / expected_seconds as f64 * 100.0) + } else { + 0.0 + }; + let status = if work_secs <= 0 { + "inactive" + } else if work_secs < target_seconds { + "below_target" + } else { + "ok" + }; + let first_local = row + .get("first_activity") + .and_then(Value::as_str) + .and_then(parse_iso_utc) + .map(|dt| dt.with_timezone(&self.config.offset).to_rfc3339()) + .unwrap_or_default(); + let last_local = row + .get("last_activity") + .and_then(Value::as_str) + .and_then(parse_iso_utc) + .map(|dt| dt.with_timezone(&self.config.offset).to_rfc3339()) + .unwrap_or_default(); + let work_first_local = work_first + .map(|dt| dt.with_timezone(&self.config.offset)) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_default(); + let work_last_local = work_last + .map(|dt| dt.with_timezone(&self.config.offset)) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_default(); + let mut public = row.as_object().cloned().unwrap_or_default(); + public.remove("_intervals"); + public.insert("user".into(), json!(alias.display_name)); + public.insert("user_original".into(), json!(user)); + public.insert("manager_owner".into(), json!(alias.manager_owner)); + public.insert("department".into(), json!(alias.department)); + public.insert("role".into(), json!(alias.role)); + public.insert("notes".into(), json!(alias.notes)); + public.insert("canonical_user_id".into(), json!(alias.canonical_user_id)); + public.insert("calendar_active_seconds".into(), json!(calendar_secs)); + public.insert("calendar_active_hhmm".into(), json!(hhmm(calendar_secs))); + public.insert("workday_active_seconds".into(), json!(work_secs)); + public.insert("workday_active_hhmm".into(), json!(hhmm(work_secs))); + public.insert("coverage_pct".into(), json!(coverage)); + public.insert("status".into(), json!(status)); + public.insert("first_activity_local".into(), json!(first_local)); + public.insert("last_activity_local".into(), json!(last_local)); + public.insert( + "workday_first_activity_local".into(), + json!(work_first_local), + ); + public.insert("workday_last_activity_local".into(), json!(work_last_local)); + let public_value = Value::Object(public.clone()); + let evidence = json!({ + "calendar_active_hhmm": hhmm(calendar_secs), + "workday_active_hhmm": hhmm(work_secs), + "coverage_pct": coverage, + "first_activity": row.get("first_activity").cloned().unwrap_or(json!("")), + "last_activity": row.get("last_activity").cloned().unwrap_or(json!("")), + "sessions_count": row.get("sessions_count").cloned().unwrap_or(json!(0)), + "manager_owner": public.get("manager_owner").cloned().unwrap_or(json!("")), + "department": public.get("department").cloned().unwrap_or(json!("")), + "role": public.get("role").cloned().unwrap_or(json!("")), + }); + let owner = public + .get("manager_owner") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let display = public + .get("user") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let canonical = public + .get("canonical_user_id") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + if work_secs <= 0 { + actions.push(action("missing_activity", "critical", &owner, "today", &format!("За {} у сотрудника {} нет подтверждённой активности в рабочем окне RDP.", report_date, display), &format!("Проверить сотрудника {display}: работал ли он в рабочее время, была ли потеря сбора данных или отсутствие входа в систему."), &canonical, evidence)); + } else { + if expected_seconds > 0 && work_secs < low_seconds { + actions.push(action("low_activity_review", "high", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже {}% от ожидаемого окна.", hhmm(work_secs), self.config.manager_low_coverage_pct), &format!("Проверить загрузку сотрудника {display}, задачи и фактическое присутствие в рабочем процессе."), &canonical, evidence.clone())); + } else if expected_seconds > 0 && work_secs < target_seconds { + actions.push(action("target_gap_review", "medium", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже управленческого целевого порога {}%.", hhmm(work_secs), self.config.manager_target_coverage_pct), &format!("Уточнить причину отклонения по сотруднику {display} и подтвердить план работ."), &canonical, evidence.clone())); + } + if work_first.is_some_and(|dt| dt.with_timezone(&self.config.offset) > late_start) { + actions.push(action("late_start_review", "medium", &owner, "24h", &format!("У сотрудника {display} первая активность в рабочем окне зафиксирована поздно."), &format!("Проверить причину позднего старта сотрудника {display} и подтвердить, что это не проблема доступа или дисциплины."), &canonical, evidence.clone())); + } + if !is_today + && work_last + .is_some_and(|dt| dt.with_timezone(&self.config.offset) < early_finish) + { + actions.push(action("early_finish_review", "medium", &owner, "24h", &format!("У сотрудника {display} последняя активность в рабочем окне завершилась рано."), &format!("Проверить, было ли досрочное завершение рабочего дня сотрудника {display} согласовано и чем оно объясняется."), &canonical, evidence)); + } + } + roster.push(public_value); + } + actions.sort_by_key(|a| { + ( + priority_rank(a.get("priority").and_then(Value::as_str).unwrap_or("")), + a.get("owner") + .and_then(Value::as_str) + .unwrap_or("") + .to_lowercase(), + a.get("action_id") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + ) + }); + let sources_actions = self.build_source_freshness( + host, + roster + .iter() + .any(|r| r.get("status").and_then(Value::as_str) != Some("inactive")), + ); + let sources = sources_actions.0; + actions.extend(sources_actions.1); + actions.sort_by_key(|a| { + ( + priority_rank(a.get("priority").and_then(Value::as_str).unwrap_or("")), + a.get("owner") + .and_then(Value::as_str) + .unwrap_or("") + .to_lowercase(), + a.get("action_id") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + ) + }); + let summary = summarize_management_rows(&roster, &actions, expected_seconds); + let owner_rollups = build_rollups(&roster, &actions, "manager_owner"); + let department_rollups = build_rollups(&roster, &actions, "department"); + let owner_roster = owner_rollups.clone(); + let executive = build_executive_summary(&summary, &actions, &sources); + let trend = vec![json!({ + "report_date": report_date.to_string(), + "users_count": summary["users_count"], + "active_users": summary["active_users"], + "inactive_users": summary["inactive_users"], + "workday_total_active_seconds": summary["workday_total_active_seconds"], + "workday_total_active_hhmm": summary["workday_total_active_hhmm"], + "portfolio_coverage_pct": summary["portfolio_coverage_pct"], + "actions_count": summary["actions_count"], + "critical_actions_count": summary["critical_actions_count"], + })]; + json!({ + "generated_at_utc": to_iso_utc(Utc::now()), + "host": host, + "report_date": report_date.to_string(), + "report_timezone": "Europe/Moscow", + "filters": {"owner": owner_filter, "department": department_filter}, + "workday": { + "start_local": work_start_local.to_rfc3339(), + "end_local": work_end_local.to_rfc3339(), + "expected_seconds_per_user": expected_seconds, + "expected_hhmm_per_user": hhmm(expected_seconds), + "target_coverage_pct": self.config.manager_target_coverage_pct, + "low_coverage_pct": self.config.manager_low_coverage_pct, + }, + "summary": summary, + "actions": actions, + "rows": roster, + "sources": sources, + "trend": trend, + "trend_scope": "portfolio", + "executive": executive, + "owner_rollups": owner_rollups, + "department_rollups": department_rollups, + "owner_roster": owner_roster, + "bucket_id": sessions_bucket(host), + "report_bounds": {"start_utc": to_iso_utc(day_start), "end_utc": to_iso_utc(day_end)}, + }) + } + + fn build_source_freshness( + &self, + host: &str, + interactive_required: bool, + ) -> (Vec, Vec) { + let specs = vec![ + ( + "worktime_sessions", + "RDP worktime sessions", + sessions_bucket(host), + self.config.manager_critical_source_max_age_seconds, + true, + false, + ), + ( + "rdp_window", + "RDP current window", + format!("aw-rdp-window_{host}"), + self.config.manager_critical_source_max_age_seconds, + true, + false, + ), + ( + "rdp_afk", + "RDP AFK", + format!("aw-rdp-afk_{host}"), + self.config.manager_critical_source_max_age_seconds, + true, + false, + ), + ( + "watcher_window", + "Local watcher window", + format!("aw-watcher-window_{host}"), + self.config.manager_critical_source_max_age_seconds, + true, + true, + ), + ( + "watcher_afk", + "Local watcher AFK", + format!("aw-watcher-afk_{host}"), + self.config.manager_critical_source_max_age_seconds, + true, + false, + ), + ( + "file_operations", + "File operations collector", + format!("aw-file-operations_{host}"), + self.config.manager_critical_source_max_age_seconds, + true, + true, + ), + ( + "web_categories", + "Browser/web categories", + format!("aw-detmir-web-category_{host}"), + self.config.manager_web_source_max_age_seconds, + false, + false, + ), + ( + "session_events", + "Windows session events", + format!("aw-session-events_{host}"), + self.config.manager_session_source_max_age_seconds, + false, + false, + ), + ( + "pve_tasks", + "PVE task feed", + "aw-pve-task-events_pve-detmir".to_string(), + self.config.manager_infra_source_max_age_seconds, + false, + false, + ), + ]; + let now = Utc::now(); + let mut sources = Vec::new(); + let mut actions = Vec::new(); + for (source_id, label, bucket, max_age, required, interactive_only) in specs { + if interactive_only && !interactive_required { + sources.push(json!({ + "source_id": source_id, + "label": label, + "status": "inactive", + "status_label": "inactive", + "bucket_id": bucket, + "timestamp": "", + "age_seconds": null, + "required": required, + "interactive_only": interactive_only, + "interactive_required": interactive_required, + "max_age_seconds": max_age, + "summary": "inactive: no active interactive users", + "event_summary": "", + })); + continue; + } + let event = self.latest_bucket_event(&bucket); + let ts = event + .as_ref() + .and_then(|e| e.timestamp.as_deref()) + .and_then(parse_iso_utc); + let age = ts.map(|dt| (now - dt).num_seconds().max(0)); + let mut status = if event.is_none() { + if required { "fail" } else { "warn" } + } else if age.is_none() { + "warn" + } else if age.unwrap() > max_age { + if required { "fail" } else { "warn" } + } else { + "ok" + }; + let mut summary = if event.is_none() { + "bucket missing or empty".to_string() + } else if age.is_none() { + "timestamp parse failed".to_string() + } else if status == "ok" { + format!("fresh ({}s)", age.unwrap()) + } else { + format!("stale ({}s)", age.unwrap()) + }; + if interactive_only && !interactive_required && status != "ok" { + status = "inactive"; + summary = "inactive: no active interactive users".to_string(); + } + let source = json!({ + "source_id": source_id, + "label": label, + "status": status, + "status_label": source_status_label(status), + "bucket_id": bucket, + "timestamp": event.as_ref().and_then(|e| e.timestamp.clone()).unwrap_or_default(), + "age_seconds": age, + "required": required, + "interactive_only": interactive_only, + "interactive_required": interactive_required, + "max_age_seconds": max_age, + "summary": summary, + "event_summary": event.as_ref().map(source_summary).unwrap_or_default(), + }); + if !matches!(status, "ok" | "inactive") { + actions.push(action( + "source_freshness_review", + if required { "critical" } else { "medium" }, + "ops", + if required { "today" } else { "3d" }, + &format!("Источник '{label}' в состоянии {}: {}.", source_status_label(status), summary), + "Проверить collector/service, причину отставания и подтвердить, что управленческие выводы по данным ещё надёжны.", + "", + json!({"source_id": source_id, "bucket_id": bucket, "age_seconds": age, "required": required}), + )); + } + sources.push(source); + } + (sources, actions) + } +} + +#[allow(clippy::too_many_arguments)] +fn action( + id: &str, + priority: &str, + owner: &str, + deadline: &str, + reason: &str, + recommended: &str, + user_id: &str, + evidence: Value, +) -> Value { + json!({ + "action_id": id, + "priority": priority, + "owner": owner, + "user_id": user_id, + "deadline_hint": deadline, + "reason": reason, + "recommended_action": recommended, + "evidence": evidence, + }) +} + +fn priority_rank(priority: &str) -> i32 { + match priority { + "critical" => 0, + "high" => 1, + "medium" => 2, + "low" => 3, + _ => 9, + } +} + +fn clamp_pct(value: f64) -> f64 { + (value.clamp(0.0, 100.0) * 100.0).round() / 100.0 +} + +fn source_status_label(status: &str) -> &str { + match status { + "ok" => "fresh", + "warn" => "stale", + "fail" => "missing", + "inactive" => "inactive", + _ => status, + } +} + +fn source_summary(event: &AwEvent) -> String { + let d = &event.data; + let signal = value_string(d, "signalType"); + if signal == "collector_health" { + return format!( + "queue={} failures={} flushed={}", + d.get("queueDepth").unwrap_or(&json!(0)), + d.get("sendFailures").unwrap_or(&json!(0)), + d.get("eventsFlushed").unwrap_or(&json!(0)) + ); + } + for key in ["domain", "eventType", "action", "title", "status", "app"] { + let value = value_string(d, key); + if !value.is_empty() { + return value.chars().take(120).collect(); + } + } + String::new() +} + +fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds: i64) -> Value { + let users_count = rows.len() as i64; + let active_users = rows + .iter() + .filter(|r| r.get("status").and_then(Value::as_str) != Some("inactive")) + .count() as i64; + let inactive_users = rows + .iter() + .filter(|r| r.get("status").and_then(Value::as_str) == Some("inactive")) + .count() as i64; + let below_target_users = rows + .iter() + .filter(|r| r.get("status").and_then(Value::as_str) == Some("below_target")) + .count() as i64; + let on_target_users = rows + .iter() + .filter(|r| r.get("status").and_then(Value::as_str) == Some("ok")) + .count() as i64; + let work_total = rows + .iter() + .map(|r| { + r.get("workday_active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0) + }) + .sum::(); + let cal_total = rows + .iter() + .map(|r| { + r.get("calendar_active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0) + }) + .sum::(); + let coverage = if users_count > 0 && expected_seconds > 0 { + clamp_pct(work_total as f64 / (expected_seconds * users_count) as f64 * 100.0) + } else { + 0.0 + }; + let top = rows.iter().max_by_key(|r| { + r.get("workday_active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0) + }); + json!({ + "users_count": users_count, + "active_users": active_users, + "inactive_users": inactive_users, + "on_target_users": on_target_users, + "below_target_users": below_target_users, + "portfolio_coverage_pct": coverage, + "actions_count": actions.len(), + "critical_actions_count": actions.iter().filter(|a| a.get("priority").and_then(Value::as_str) == Some("critical")).count(), + "high_actions_count": actions.iter().filter(|a| a.get("priority").and_then(Value::as_str) == Some("high")).count(), + "calendar_total_active_seconds": cal_total, + "calendar_total_active_hhmm": hhmm(cal_total), + "workday_total_active_seconds": work_total, + "workday_total_active_hhmm": hhmm(work_total), + "total_active_seconds": work_total, + "total_active_hhmm": hhmm(work_total), + "first_activity": rows.iter().filter_map(|r| r.get("workday_first_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).min().unwrap_or(""), + "last_activity": rows.iter().filter_map(|r| r.get("workday_last_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).max().unwrap_or(""), + "top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""), + "top_user_active_hhmm": top.and_then(|r| r.get("workday_active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"), + }) +} + +fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec { + let mut groups: BTreeMap> = BTreeMap::new(); + for row in rows { + let name = row + .get(field) + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .unwrap_or("Без подразделения") + .to_string(); + let group = groups.entry(name.clone()).or_insert_with(|| { + let mut g = Map::new(); + g.insert("name".into(), json!(name)); + for key in [ + "users_count", + "active_users", + "inactive_users", + "below_target_users", + "workday_total_active_seconds", + "actions_count", + "critical_actions_count", + "high_actions_count", + "medium_actions_count", + "low_actions_count", + ] { + g.insert(key.into(), json!(0)); + } + g.insert("users".into(), json!([])); + g + }); + inc(group, "users_count", 1); + if row.get("status").and_then(Value::as_str) == Some("inactive") { + inc(group, "inactive_users", 1); + } else { + inc(group, "active_users", 1); + } + if row.get("status").and_then(Value::as_str) == Some("below_target") { + inc(group, "below_target_users", 1); + } + inc( + group, + "workday_total_active_seconds", + row.get("workday_active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0), + ); + if let Some(arr) = group.get_mut("users").and_then(Value::as_array_mut) { + arr.push(row.get("user").cloned().unwrap_or(json!(""))); + } + } + for action in actions { + let name = action + .get("owner") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .unwrap_or("unassigned") + .to_string(); + let group = groups.entry(name.clone()).or_insert_with(|| { + let mut g = Map::new(); + g.insert("name".into(), json!(name)); + for key in [ + "users_count", + "active_users", + "inactive_users", + "below_target_users", + "workday_total_active_seconds", + "actions_count", + "critical_actions_count", + "high_actions_count", + "medium_actions_count", + "low_actions_count", + ] { + g.insert(key.into(), json!(0)); + } + g.insert("users".into(), json!([])); + g + }); + inc(group, "actions_count", 1); + let key = match action.get("priority").and_then(Value::as_str).unwrap_or("") { + "critical" => "critical_actions_count", + "high" => "high_actions_count", + "medium" => "medium_actions_count", + "low" => "low_actions_count", + _ => "low_actions_count", + }; + inc(group, key, 1); + } + groups + .into_values() + .map(|mut g| { + let secs = g + .get("workday_total_active_seconds") + .and_then(Value::as_i64) + .unwrap_or(0); + let users = g.get("users_count").and_then(Value::as_i64).unwrap_or(0); + g.insert("workday_total_active_hhmm".into(), json!(hhmm(secs))); + g.insert( + "portfolio_coverage_pct".into(), + json!(if users > 0 { + clamp_pct(secs as f64 / (users * 9 * 3600) as f64 * 100.0) + } else { + 0.0 + }), + ); + Value::Object(g) + }) + .collect() +} + +fn inc(map: &mut Map, key: &str, amount: i64) { + let old = map.get(key).and_then(Value::as_i64).unwrap_or(0); + map.insert(key.into(), json!(old + amount)); +} + +fn build_executive_summary(summary: &Value, actions: &[Value], sources: &[Value]) -> Value { + let critical = actions + .iter() + .filter(|a| a.get("priority").and_then(Value::as_str) == Some("critical")) + .count(); + let high = actions + .iter() + .filter(|a| a.get("priority").and_then(Value::as_str) == Some("high")) + .count(); + let portfolio_state = if critical > 0 { + "critical" + } else if high > 0 { + "attention" + } else { + "stable" + }; + let headline = if critical > 0 { + format!("Есть {critical} критичных вопроса, требующих решения сегодня.") + } else if high > 0 { + format!("Критичных провалов нет, но есть {high} вопроса повышенного внимания.") + } else { + "Критичных отклонений не найдено, рабочий день идёт в пределах нормы.".to_string() + }; + let stale = sources + .iter() + .filter(|s| s.get("status").and_then(Value::as_str) != Some("ok")) + .count(); + json!({ + "portfolio_state": portfolio_state, + "headline": headline, + "message": format!("Активны {} из {} сотрудников. Покрытие рабочего окна {}%.", summary["active_users"], summary["users_count"], summary["portfolio_coverage_pct"]), + "focus_items": actions.iter().take(5).map(|a| json!({"priority": a["priority"], "owner": a["owner"], "title": a["action_id"], "reason": a["reason"], "recommended_action": a["recommended_action"]})).collect::>(), + "stale_sources": sources.iter().filter(|s| s.get("status").and_then(Value::as_str) != Some("ok")).take(3).cloned().collect::>(), + "stale_sources_count": stale, + }) +} + +fn normalize_app_name(app: &str, title: &str) -> String { + let app_l = app.trim().to_lowercase(); + let title_l = title.to_lowercase(); + if app_l.starts_with("1cv8") || app_l.starts_with("1cestart") { + "1С".into() + } else if app_l == "chrome.exe" || title_l.contains("google chrome") { + "Chrome".into() + } else if app_l == "msedge.exe" || title_l.contains("microsoft edge") { + "Edge".into() + } else if app_l == "browser.exe" || title_l.contains("яндекс") { + "Яндекс Браузер".into() + } else if app_l == "excel.exe" { + "Excel".into() + } else if app_l == "winword.exe" { + "Word".into() + } else if app_l == "explorer.exe" { + "Проводник".into() + } else if app_l == "totalcmd.exe" || app_l == "totalcmd64.exe" { + "Total Commander".into() + } else if !app.trim().is_empty() { + app.trim().trim_end_matches(".exe").to_string() + } else if !title.trim().is_empty() { + title.trim().to_string() + } else { + "Неизвестное приложение".into() + } +} + +fn event_duration(config: &Config, event: &AwEvent, fallback: f64) -> f64 { + event + .duration + .unwrap_or(0.0) + .max(fallback) + .min(config.true_active_max_event_seconds as f64) + .max(1.0) +} + +fn event_context(event: &AwEvent) -> String { + for key in [ + "title", + "url", + "path", + "filePath", + "targetPath", + "windowTitle", + "foregroundTitle", + "signalType", + ] { + let value = value_string(&event.data, key); + if !value.is_empty() { + return value; + } + } + "активность".into() +} + +fn is_real_evidence(event: &AwEvent) -> bool { + let signal = value_string(&event.data, "signalType").to_lowercase(); + if matches!( + signal.as_str(), + "collector_health" | "self_test" | "heartbeat" | "health" + ) { + return false; + } + ["url", "title", "path", "filePath", "targetPath"] + .iter() + .any(|key| !value_string(&event.data, key).is_empty()) + || !signal.is_empty() +} + +fn build_true_active_apps_from_events( + config: &Config, + window_events: &[AwEvent], + afk_events: &[AwEvent], + evidence_by_bucket: &HashMap>, + start: DateTime, + end: DateTime, +) -> Vec { + let mut windows = Vec::new(); + let mut prev_key = None; + for event in sorted_events(window_events, start, end) { + let app = value_string(&event.data, "app") + .if_empty(value_string(&event.data, "process")) + .if_empty(value_string(&event.data, "processName")); + let title = + value_string(&event.data, "title").if_empty(value_string(&event.data, "windowTitle")); + if app.is_empty() && title.is_empty() { + continue; + } + let ts = event.timestamp.as_deref().and_then(parse_iso_utc).unwrap(); + let dur = event_duration(config, event, config.default_sample_seconds); + let interval = ( + ts.max(start), + (ts + TimeDelta::milliseconds((dur * 1000.0) as i64)).min(end + TimeDelta::seconds(1)), + ); + if interval.1 <= interval.0 { + continue; + } + let app_name = normalize_app_name(&app, &title); + let key = (app_name.clone(), title.clone()); + let changed = prev_key.as_ref().is_some_and(|p| p != &key); + prev_key = Some(key); + windows.push((app_name, app, title, interval.0, interval.1, changed, ts)); + } + let mut afk_intervals = Vec::new(); + for event in sorted_events(afk_events, start, end) { + let status = value_string(&event.data, "status").to_lowercase(); + if !matches!( + status.as_str(), + "not-afk" | "not_afk" | "active" | "активно" + ) { + continue; + } + let ts = event.timestamp.as_deref().and_then(parse_iso_utc).unwrap(); + let dur = event_duration(config, event, 5.0); + let interval = ( + ts.max(start), + (ts + TimeDelta::milliseconds((dur * 1000.0) as i64)).min(end + TimeDelta::seconds(1)), + ); + if interval.1 > interval.0 { + afk_intervals.push(interval); + } + } + afk_intervals = merge_intervals(afk_intervals); + let mut evidence: HashMap, String)>> = HashMap::new(); + for (app_name, _raw, title, _s, _e, changed, ts) in &windows { + if *changed { + evidence + .entry(app_name.clone()) + .or_default() + .push((*ts, title.clone())); + } + } + for events in evidence_by_bucket.values() { + for event in sorted_events(events, start, end) { + if !is_real_evidence(event) { + continue; + } + let ts = event.timestamp.as_deref().and_then(parse_iso_utc).unwrap(); + if let Some((app_name, _, _, _, _, _, _)) = windows + .iter() + .find(|(_, _, _, s, e, _, _)| *s <= ts && ts < *e) + { + evidence + .entry(app_name.clone()) + .or_default() + .push((ts, event_context(event))); + } + } + } + let mut rows = Vec::new(); + let delta = TimeDelta::seconds(config.true_active_evidence_window_seconds); + let apps: BTreeSet = windows + .iter() + .map(|w| w.0.clone()) + .chain(evidence.keys().cloned()) + .collect(); + for app in apps { + let app_evidence = evidence.get(&app).cloned().unwrap_or_default(); + if app_evidence.is_empty() { + continue; + } + let evidence_windows = merge_intervals( + app_evidence + .iter() + .map(|(ts, _)| (*ts - delta, *ts + delta)) + .collect(), + ); + let mut proved = Vec::new(); + for window in windows.iter().filter(|w| w.0 == app) { + for afk in &afk_intervals { + if let Some(active) = overlap((window.3, window.4), *afk) { + for ev in &evidence_windows { + if let Some(p) = overlap(active, *ev) { + proved.push(p); + } + } + } + } + } + let proved = merge_intervals(proved); + let seconds = proved + .iter() + .map(|(s, e)| (*e - *s).num_seconds()) + .sum::(); + if seconds <= 0 { + continue; + } + let (last_ts, last_context) = app_evidence.last().cloned().unwrap(); + rows.push(json!({ + "application": app, + "proved_work_seconds": seconds, + "proved_work_hhmm": hhmm(seconds), + "proved_work_human": human_duration_ru(seconds), + "last_action_utc": to_iso_utc(last_ts), + "last_action_local": last_ts.with_timezone(&config.offset).format("%H:%M").to_string(), + "last_action": last_context, + "evidence_events": app_evidence.len(), + })); + } + rows.sort_by_key(|r| { + ( + -r.get("proved_work_seconds") + .and_then(Value::as_i64) + .unwrap_or(0), + r.get("application") + .and_then(Value::as_str) + .unwrap_or("") + .to_lowercase(), + ) + }); + rows +} + +fn sorted_events(events: &[AwEvent], start: DateTime, end: DateTime) -> Vec<&AwEvent> { + let mut out: Vec<&AwEvent> = events + .iter() + .filter(|e| { + e.timestamp + .as_deref() + .and_then(parse_iso_utc) + .is_some_and(|ts| ts >= start && ts <= end) + }) + .collect(); + out.sort_by_key(|e| e.timestamp.clone()); + out +} + +fn overlap( + a: (DateTime, DateTime), + b: (DateTime, DateTime), +) -> Option<(DateTime, DateTime)> { + let s = a.0.max(b.0); + let e = a.1.min(b.1); + if e > s { Some((s, e)) } else { None } +} + +fn resolve_report_format(params: &Params, accept: &str) -> String { + let requested = params.first("format").unwrap_or_default().to_lowercase(); + if matches!(requested.as_str(), "csv" | "html" | "json") { + return requested; + } + let accept = accept.to_lowercase(); + if accept.contains("text/html") && !accept.contains("application/json") { + "html".into() + } else { + "json".into() + } +} + +fn make_report_cache_key( + path: &str, + fmt: &str, + host: &str, + report_date: NaiveDate, + day: &str, + owner: &str, + department: &str, +) -> String { + format!("{path}|{fmt}|{host}|{report_date}|{day}|{owner}|{department}") +} + +fn today_csv(rows: &[Value]) -> String { + let mut out = "user,user_id,active_seconds,active_hhmm,first_activity,last_activity,idle_seconds,sessions_count,samples_count,active_samples\n".to_string(); + for row in rows { + out.push_str(&format!( + "{},{},{},{},{},{},{},{},{},{}\n", + csv_cell(row["user"].as_str().unwrap_or("")), + csv_cell(row["user_id"].as_str().unwrap_or("")), + row["active_seconds"].as_i64().unwrap_or(0), + row["active_hhmm"].as_str().unwrap_or(""), + row["first_activity"].as_str().unwrap_or(""), + row["last_activity"].as_str().unwrap_or(""), + row["idle_seconds"].as_i64().unwrap_or(0), + row["sessions_count"].as_i64().unwrap_or(0), + row["samples_count"].as_i64().unwrap_or(0), + row["active_samples"].as_i64().unwrap_or(0) + )); + } + out +} + +fn management_csv(payload: &Value) -> String { + let mut out = + "priority,owner,user_id,action_id,deadline_hint,reason,recommended_action\n".to_string(); + for action in payload + .get("actions") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + out.push_str(&format!( + "{},{},{},{},{},{},{}\n", + csv_cell(action["priority"].as_str().unwrap_or("")), + csv_cell(action["owner"].as_str().unwrap_or("")), + csv_cell(action["user_id"].as_str().unwrap_or("")), + csv_cell(action["action_id"].as_str().unwrap_or("")), + csv_cell(action["deadline_hint"].as_str().unwrap_or("")), + csv_cell(action["reason"].as_str().unwrap_or("")), + csv_cell(action["recommended_action"].as_str().unwrap_or("")) + )); + } + out +} + +fn csv_cell(value: &str) -> String { + if value.contains([',', '"', '\n']) { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } +} + +fn esc(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +fn render_today_html( + _config: &Config, + rows: &[Value], + host: &str, + report_date: NaiveDate, + selected_day: Option<&str>, + true_apps: &[Value], +) -> String { + let summary = build_report_summary(rows); + let cards = format!( + "
Пользователи{}
Активное время{}
Лидер дня{}
", + summary["users_count"], + summary["total_active_hhmm"].as_str().unwrap_or("00:00"), + esc(summary["top_user"].as_str().unwrap_or("")) + ); + let app_rows = if true_apps.is_empty() { + "Пока нет доказанной активной работы по приложениям за выбранную дату.".to_string() + } else { + true_apps + .iter() + .map(|r| { + format!( + "{}{}{} · {}", + esc(r["application"].as_str().unwrap_or("-")), + esc(r["proved_work_human"].as_str().unwrap_or("0 сек")), + esc(r["last_action_local"].as_str().unwrap_or("-")), + esc(r["last_action"].as_str().unwrap_or("-")) + ) + }) + .collect::() + }; + let user_rows = if rows.is_empty() { + "За выбранную дату данных пока нет.".to_string() + } else { + rows.iter().map(|r| format!("{}{}{}{}{}{}{}{}{}", esc(r["user"].as_str().unwrap_or("")), esc(r["user_id"].as_str().unwrap_or("")), esc(r["active_hhmm"].as_str().unwrap_or("")), r["active_seconds"], esc(r["first_activity"].as_str().unwrap_or("")), esc(r["last_activity"].as_str().unwrap_or("")), r["idle_seconds"], r["sessions_count"], r["samples_count"])).collect::() + }; + let day_query = selected_day + .filter(|d| matches!(*d, "today" | "yesterday")) + .map(|d| format!("day={d}")) + .unwrap_or_else(|| format!("date={report_date}")); + format!( + r#"AW-rus Отчёт по работе в RDP

Отчёт по работе в RDP

Хост: {} · Дата: {} · Часовой пояс: Europe/Moscow · Сформировано: {}

{}

Доказанная работа по приложениям

{}
ПриложениеДоказанная работаПоследнее действие

Таблица по пользователям

{}
ПользовательУчётная записьАктивноАктивно, секНачалоКонецПростойСессииСэмплы
"#, + base_css(), + esc(host), + report_date, + Local::now().format("%F %T"), + esc(host), + esc(host), + esc(host), + day_query, + esc(host), + day_query, + esc(host), + day_query, + cards, + app_rows, + user_rows + ) +} + +fn render_management_html(payload: &Value) -> String { + let rows = payload["rows"].as_array().cloned().unwrap_or_default(); + let actions = payload["actions"].as_array().cloned().unwrap_or_default(); + let sources = payload["sources"].as_array().cloned().unwrap_or_default(); + let action_rows = if actions.is_empty() { + "Нет действий.".to_string() + } else { + actions + .iter() + .map(|a| { + format!( + "{}{}{}{}{}{}", + esc(a["priority"].as_str().unwrap_or("")), + esc(a["owner"].as_str().unwrap_or("")), + esc(a["action_id"].as_str().unwrap_or("")), + esc(a["deadline_hint"].as_str().unwrap_or("")), + esc(a["reason"].as_str().unwrap_or("")), + esc(a["recommended_action"].as_str().unwrap_or("")) + ) + }) + .collect() + }; + let user_rows = if rows.is_empty() { + "Нет сотрудников в выборке.".to_string() + } else { + rows.iter().map(|r| format!("{}{}{}{}{}{}%{}{}", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["workday_first_activity_local"].as_str().unwrap_or("")), esc(r["workday_last_activity_local"].as_str().unwrap_or("")))).collect() + }; + let source_rows = sources + .iter() + .map(|s| { + format!( + "{}{}{}{}{}{}", + esc(s["label"].as_str().unwrap_or("")), + esc(s["status_label"].as_str().unwrap_or("")), + esc(s["bucket_id"].as_str().unwrap_or("")), + esc(s["timestamp"].as_str().unwrap_or("")), + s["age_seconds"], + esc(s["summary"].as_str().unwrap_or("")) + ) + }) + .collect::(); + format!( + r#"AW-rus Управленческий отчёт по работе в RDP

AW-rus Управленческий отчёт по работе в RDP

{} · {} · {}

Что делать сегодня

{}

Очередь действий руководителя

{}

Сотрудники

{}

Тренд за период

Тренд за {} дней

По ответственным

{}

Ответственные и эскалация

{}

По подразделениям

{}

Свежесть источников данных

{}

Фильтр: {}

"#, + base_css(), + esc(payload["host"].as_str().unwrap_or("")), + esc(payload["report_date"].as_str().unwrap_or("")), + esc(payload["report_timezone"].as_str().unwrap_or("")), + esc(payload + .pointer("/executive/headline") + .and_then(Value::as_str) + .unwrap_or("")), + esc(payload["host"].as_str().unwrap_or("")), + esc(payload["host"].as_str().unwrap_or("")), + esc(payload["host"].as_str().unwrap_or("")), + esc(payload["host"].as_str().unwrap_or("")), + action_rows, + user_rows, + payload["trend"].as_array().map(|a| a.len()).unwrap_or(0), + esc(&payload["owner_rollups"].to_string()), + esc(&payload["owner_roster"].to_string()), + esc(&payload["department_rollups"].to_string()), + source_rows, + esc(&payload["filters"].to_string()) + ) +} + +fn base_css() -> &'static str { + "body{margin:0;background:#f5f7fb;color:#172033;font:14px/1.45 'Segoe UI',Arial,sans-serif}main{max-width:1360px;margin:0 auto;padding:20px}.hero{background:#12324a;color:white;border-radius:8px;padding:18px 20px;margin-bottom:16px}.hero a{color:white;margin-right:12px}.grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px}.card{background:rgba(255,255,255,.12);border:1px solid rgba(255,255,255,.18);border-radius:8px;padding:12px}.card b{display:block;color:#dbeafe}.card strong{font-size:22px}section{background:white;border:1px solid #dbe3ee;border-radius:8px;margin:14px 0;padding:14px;overflow:auto}table{border-collapse:collapse;width:100%}th,td{border-bottom:1px solid #e5eaf2;padding:9px 10px;text-align:left;vertical-align:top}.good{color:#0f766e;font-weight:700}@media(max-width:800px){main{padding:10px}.grid{grid-template-columns:1fr}table{min-width:900px}}" +} + +fn header(name: &str, value: &str) -> Header { + Header::from_bytes(name.as_bytes(), value.as_bytes()).expect("valid header") +} + +fn respond( + request: Request, + status: u16, + data: Vec, + content_type: &str, + extra_headers: Vec<(String, String)>, +) { + let mut response = Response::new( + StatusCode(status), + vec![ + header("Content-Type", content_type), + header("Content-Length", &data.len().to_string()), + ], + Cursor::new(data), + None, + None, + ); + for (key, value) in extra_headers { + response.add_header(header(&key, &value)); + } + let _ = request.respond(response); +} + +fn handle(app: &App, request: Request) { + if request.method() != &Method::Get { + respond( + request, + 405, + Vec::new(), + "text/plain; charset=utf-8", + Vec::new(), + ); + return; + } + let url = request.url().to_string(); + let (path, query) = url.split_once('?').unwrap_or((&url, "")); + if path == "/health" || path == "/api/health" { + let payload = json!({ + "ok": true, + "generated_at_utc": to_iso_utc(Utc::now()), + "report_timezone": "Europe/Moscow", + "default_host": app.config.default_host, + "aw_api_base": app.config.aw_api_base, + }); + respond( + request, + 200, + serde_json::to_vec_pretty(&payload).unwrap(), + "application/json; charset=utf-8", + Vec::new(), + ); + return; + } + if path.starts_with("/dlp-ioc/") { + let name = path.rsplit('/').next().unwrap_or(""); + if !matches!( + name, + "ioc_blacklist.json" | "ioc_blacklist.csv" | "ioc_blacklist.sql" + ) { + respond( + request, + 404, + Vec::new(), + "text/plain; charset=utf-8", + Vec::new(), + ); + return; + } + let file = app.config.ioc_dir.join(name); + match fs::read(file) { + Ok(data) => { + let ctype = if name.ends_with(".json") { + "application/json; charset=utf-8" + } else if name.ends_with(".csv") { + "text/csv; charset=utf-8" + } else { + "text/plain; charset=utf-8" + }; + respond(request, 200, data, ctype, Vec::new()); + } + Err(_) => respond( + request, + 404, + Vec::new(), + "text/plain; charset=utf-8", + Vec::new(), + ), + } + return; + } + if path != "/reports/worktime/today" && path != "/reports/worktime/management" { + respond( + request, + 404, + Vec::new(), + "text/plain; charset=utf-8", + Vec::new(), + ); + return; + } + let params = Params::parse(query); + let accept = request + .headers() + .iter() + .find(|h| h.field.equiv("Accept")) + .map(|h| h.value.as_str()) + .unwrap_or(""); + let (data, content_type, headers) = app.report_response(path, ¶ms, accept); + let status = if content_type.starts_with("application/json") + && serde_json::from_slice::(&data) + .ok() + .and_then(|v| v.get("error").cloned()) + .is_some() + { + 503 + } else { + 200 + }; + respond(request, status, data, &content_type, headers); +} + +fn main() -> Result<()> { + let _cli = Cli::parse(); + let config = load_config(); + let addr = format!("{}:{}", config.listen_host, config.listen_port); + let app = App::new(config)?; + let server = Server::http(&addr).map_err(|error| anyhow!("bind {addr}: {error}"))?; + println!("{} listening on {}", Local::now().format("%F %T"), addr); + for request in server.incoming_requests() { + let app = app.clone(); + std::thread::spawn(move || handle(&app, request)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> Config { + load_config() + } + + fn event(ts: &str, username: &str, sid: i64, active: bool, extra: Value) -> AwEvent { + let mut data = Map::new(); + data.insert("username".into(), json!(username)); + data.insert("userId".into(), json!(format!("WORKGROUP\\{username}"))); + data.insert("sessionId".into(), json!(sid)); + data.insert( + "state".into(), + json!(if active { "Активно" } else { "Диск" }), + ); + data.insert("active".into(), json!(active)); + if let Some(extra) = extra.as_object() { + for (k, v) in extra { + data.insert(k.clone(), v.clone()); + } + } + AwEvent { + timestamp: Some(ts.into()), + duration: Some(0.0), + data: Value::Object(data), + } + } + + #[test] + fn aggregate_rows_merges_overlap() { + let cfg = test_config(); + let start = parse_iso_utc("2026-05-14T06:00:00Z").unwrap(); + let end = parse_iso_utc("2026-05-14T06:59:59Z").unwrap(); + let rows = aggregate_rows( + &cfg, + &[ + event( + "2026-05-14T06:00:00Z", + "user5", + 4, + true, + json!({"sampleSeconds":30}), + ), + event( + "2026-05-14T06:00:30Z", + "user5", + 4, + true, + json!({"sampleSeconds":30}), + ), + event( + "2026-05-14T06:00:15Z", + "user5", + 5, + true, + json!({"sampleSeconds":30}), + ), + ], + start, + end, + "SHARKON2025", + false, + ); + assert_eq!(rows[0]["active_seconds"], 60); + assert_eq!(rows[0]["sessions_count"], 2); + } + + #[test] + fn report_format_prefers_html_for_browser() { + assert_eq!( + resolve_report_format(&Params::default(), "text/html,application/xhtml+xml"), + "html" + ); + assert_eq!( + resolve_report_format(&Params::parse("format=json"), "text/html"), + "json" + ); + } + + #[test] + fn true_active_apps_require_evidence() { + let cfg = test_config(); + let start = parse_iso_utc("2026-05-14T06:00:00Z").unwrap(); + let end = parse_iso_utc("2026-05-14T07:59:59Z").unwrap(); + let windows = vec![ + AwEvent { + timestamp: Some("2026-05-14T06:00:00Z".into()), + duration: Some(120.0), + data: json!({"app":"1cv8.exe","title":"ИНФОВЕСТ"}), + }, + AwEvent { + timestamp: Some("2026-05-14T06:02:00Z".into()), + duration: Some(180.0), + data: json!({"app":"1cv8.exe","title":"Счета учета: Материалы"}), + }, + ]; + let afk = vec![AwEvent { + timestamp: Some("2026-05-14T06:00:00Z".into()), + duration: Some(600.0), + data: json!({"status":"not-afk"}), + }]; + let rows = + build_true_active_apps_from_events(&cfg, &windows, &afk, &HashMap::new(), start, end); + assert_eq!(rows[0]["application"], "1С"); + assert_eq!(rows[0]["proved_work_seconds"], 300); + } +} diff --git a/adk-rust/crates/worktime-autoheal/Cargo.toml b/adk-rust/crates/worktime-autoheal/Cargo.toml new file mode 100644 index 0000000..47e7438 --- /dev/null +++ b/adk-rust/crates/worktime-autoheal/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "worktime-autoheal" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/worktime-autoheal/src/main.rs b/adk-rust/crates/worktime-autoheal/src/main.rs new file mode 100644 index 0000000..e47f7a9 --- /dev/null +++ b/adk-rust/crates/worktime-autoheal/src/main.rs @@ -0,0 +1,815 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + process::Command, + thread, + time::Duration, +}; + +use anyhow::{Context, Result, anyhow}; +use chrono::{DateTime, Datelike, FixedOffset, SecondsFormat, TimeZone, Utc}; +use clap::Parser; +use reqwest::header::{CONNECTION, HeaderMap, HeaderValue}; +use reqwest::{ + Method, StatusCode, + blocking::{Client, Response}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const DEFAULT_AW_URL: &str = "http://127.0.0.1:5600"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const AUTOHEAL_SOURCE: &str = "aw-worktime-autoheal"; + +#[derive(Debug, Parser)] +#[command(about = "AW Worktime autoheal")] +struct Cli { + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_url: String, + host: String, + worktime_health_url: String, + worktime_report_timeout_seconds: f64, + management_warm_enabled: bool, + management_warm_url: String, + management_warm_timeout_seconds: f64, + today_probe_enabled: bool, + today_probe_url: String, + today_probe_timeout_seconds: f64, + session_freshness_seconds: f64, + aw_timeout_seconds: f64, + aw_post_chunk_size: usize, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct AwEvent { + timestamp: Option, + #[serde(default)] + duration: Option, + #[serde(default)] + data: Value, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + dry_run: bool, + host: String, + worktime_ok: bool, + api_restarted: bool, + management_warm_ok: Option, + need_heal: bool, + ui_bridge_started: bool, + reset_buckets: bool, + backfill_afk: usize, + backfill_window: usize, + reason: String, +} + +struct AwClient { + base_url: String, + client: Client, +} + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_bool_legacy(name: &str, fallback: bool) -> bool { + match std::env::var(name) + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + { + Some(value) if !value.is_empty() => matches!(value.as_str(), "1" | "true" | "yes" | "on"), + _ => fallback, + } +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .filter(|value| *value > 0.0) + .unwrap_or(fallback) +} + +fn env_usize(name: &str, fallback: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(fallback) +} + +fn load_config() -> Config { + Config { + aw_url: env("AW_URL", DEFAULT_AW_URL) + .trim_end_matches('/') + .to_string(), + host: env("AW_WORKTIME_HOST", DEFAULT_HOST), + worktime_health_url: env("WORKTIME_HEALTH_URL", "http://127.0.0.1:5610/health"), + worktime_report_timeout_seconds: env_f64("WORKTIME_REPORT_TIMEOUT_SECONDS", 20.0), + management_warm_enabled: env_bool_legacy("WORKTIME_MANAGEMENT_WARM_ENABLED", true), + management_warm_url: env( + "WORKTIME_MANAGEMENT_WARM_URL", + "http://127.0.0.1:5610/reports/worktime/management?day=today&format=json", + ), + management_warm_timeout_seconds: env_f64("WORKTIME_MANAGEMENT_WARM_TIMEOUT_SECONDS", 60.0), + today_probe_enabled: env_bool_legacy("WORKTIME_TODAY_PROBE_ENABLED", true), + today_probe_url: env( + "WORKTIME_TODAY_PROBE_URL", + "http://127.0.0.1:5610/reports/worktime/today?day=today&format=json", + ), + today_probe_timeout_seconds: env_f64("WORKTIME_TODAY_PROBE_TIMEOUT_SECONDS", 20.0), + session_freshness_seconds: env_f64("WORKTIME_SESSION_FRESHNESS_SECONDS", 600.0), + aw_timeout_seconds: env_f64("WORKTIME_AUTOHEAL_AW_TIMEOUT_SECONDS", 30.0), + aw_post_chunk_size: env_usize("WORKTIME_AUTOHEAL_AW_POST_CHUNK_SIZE", 500), + } +} + +impl Config { + fn window_bucket(&self) -> String { + format!("aw-rdp-window_{}", self.host) + } + + fn afk_bucket(&self) -> String { + format!("aw-rdp-afk_{}", self.host) + } + + fn session_bucket(&self) -> String { + format!("aw-worktime-sessions_{}", self.host) + } +} + +impl AwClient { + fn new(config: &Config) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(CONNECTION, HeaderValue::from_static("close")); + let client = Client::builder() + .timeout(Duration::from_secs_f64( + config.aw_timeout_seconds.max(0.001), + )) + .no_proxy() + .pool_max_idle_per_host(0) + .default_headers(headers) + .build() + .context("build ActivityWatch HTTP client")?; + Ok(Self { + base_url: config.aw_url.clone(), + client, + }) + } + + fn url(&self, path: &str) -> String { + format!("{}{}", self.base_url, path) + } + + fn send_retry(&self, method: Method, path: &str, payload: Option) -> Result { + let mut last_error = None; + for attempt in 1..=3 { + let mut request = self.client.request(method.clone(), self.url(path)); + if let Some(payload) = payload.clone() { + request = request.json(&payload); + } + match request.send() { + Ok(response) => return Ok(response), + Err(error) => { + last_error = Some(error); + if attempt < 3 { + log(&format!("warn aw request={path} retry_after_send_error")); + thread::sleep(Duration::from_millis(750)); + } + } + } + } + Err(last_error.expect("error exists after failed request")) + .with_context(|| format!("request ActivityWatch {path}")) + } + + fn request_json( + &self, + method: Method, + path: &str, + payload: Option, + ignore_not_found: bool, + ) -> Result> { + let response = self.send_retry(method, path, payload)?; + let status = response.status(); + if ignore_not_found && status == StatusCode::NOT_FOUND { + return Ok(None); + } + if !status.is_success() { + return Err(anyhow!("ActivityWatch {path} returned HTTP {status}")); + } + let bytes = response.bytes().context("read ActivityWatch response")?; + if bytes.is_empty() { + Ok(None) + } else { + serde_json::from_slice(&bytes) + .map(Some) + .with_context(|| format!("decode ActivityWatch JSON from {path}")) + } + } + + fn get_events(&self, bucket_id: &str, limit: usize) -> Result> { + let path = format!("/api/0/buckets/{bucket_id}/events?limit={limit}"); + match self.request_json(Method::GET, &path, None, true)? { + Some(value) => serde_json::from_value(value).context("decode AW events"), + None => Ok(Vec::new()), + } + } + + fn delete_bucket(&self, bucket_id: &str) -> Result<()> { + let path = format!("/api/0/buckets/{bucket_id}"); + let response = self.send_retry(Method::DELETE, &path, None)?; + if response.status().is_success() || response.status() == StatusCode::NOT_FOUND { + Ok(()) + } else { + Err(anyhow!( + "delete bucket {bucket_id} returned HTTP {}", + response.status() + )) + } + } + + fn ensure_bucket( + &self, + bucket_id: &str, + event_type: &str, + client_name: &str, + host: &str, + ) -> Result<()> { + let path = format!("/api/0/buckets/{bucket_id}"); + let payload = json!({ + "client": client_name, + "type": event_type, + "hostname": host, + }); + let response = self.send_retry(Method::POST, &path, Some(payload))?; + let status = response.status(); + if status.is_success() + || status == StatusCode::NOT_MODIFIED + || status == StatusCode::CONFLICT + { + Ok(()) + } else { + Err(anyhow!("ensure bucket {bucket_id} returned HTTP {status}")) + } + } + + fn post_events_chunked( + &self, + bucket_id: &str, + events: &[AwEvent], + chunk_size: usize, + ) -> Result<()> { + for chunk in events.chunks(chunk_size.max(1)) { + let path = format!("/api/0/buckets/{bucket_id}/events"); + self.request_json(Method::POST, &path, Some(json!(chunk)), false)?; + } + Ok(()) + } +} + +fn log(message: &str) { + println!("{} {}", chrono::Local::now().format("%F %T"), message); +} + +fn build_probe_client(timeout_seconds: f64) -> Result { + Client::builder() + .timeout(Duration::from_secs_f64(timeout_seconds.max(0.001))) + .no_proxy() + .build() + .context("build probe HTTP client") +} + +fn probe_url(url: &str, timeout_seconds: f64) -> bool { + let Ok(client) = build_probe_client(timeout_seconds) else { + return false; + }; + client + .get(url) + .send() + .map(|response| response.status().is_success()) + .unwrap_or(false) +} + +fn probe_reports(config: &Config) -> bool { + if !probe_url( + &config.worktime_health_url, + config.worktime_report_timeout_seconds, + ) { + return false; + } + if config.today_probe_enabled + && !probe_url(&config.today_probe_url, config.today_probe_timeout_seconds) + { + return false; + } + true +} + +fn run_systemctl(args: &[&str], dry_run: bool) -> bool { + if dry_run { + log(&format!("dry-run systemctl {}", args.join(" "))); + return true; + } + Command::new("systemctl") + .args(args) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn parse_iso_utc(ts: &str) -> Result> { + DateTime::parse_from_rfc3339(ts) + .map(|parsed| parsed.with_timezone(&Utc)) + .with_context(|| format!("parse timestamp {ts}")) +} + +fn to_iso_utc(dt: DateTime) -> String { + dt.to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn moscow_day_start_utc() -> DateTime { + let msk = FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"); + let today = Utc::now().with_timezone(&msk); + msk.with_ymd_and_hms(today.year(), today.month(), today.day(), 0, 0, 0) + .single() + .expect("valid day start") + .with_timezone(&Utc) +} + +fn value_string(data: &Value, key: &str) -> String { + match data.get(key) { + Some(Value::String(value)) => value.trim().to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + Some(Value::Null) | None => String::new(), + Some(value) => value.to_string().trim_matches('"').trim().to_string(), + } +} + +fn value_i64(data: &Value, key: &str) -> Option { + match data.get(key) { + Some(Value::Number(value)) => value.as_i64(), + Some(Value::String(value)) => value.trim().parse().ok(), + _ => None, + } +} + +fn is_session_active(data: &Value) -> bool { + if data.get("active").and_then(Value::as_bool).unwrap_or(false) { + return true; + } + let state = value_string(data, "state").to_lowercase(); + if state == "active" || state == "активно" { + return true; + } + if state == "unknown" { + let sid = value_i64(data, "sessionId").unwrap_or(-1); + let user = value_string(data, "username").to_lowercase(); + let session_name = value_string(data, "sessionName").to_lowercase(); + return sid > 0 + && !user.is_empty() + && !user.ends_with('$') + && (session_name.starts_with("rdp-") || session_name == "console"); + } + false +} + +fn active_users_at_latest_session(events: &[AwEvent]) -> (Option>, BTreeSet) { + let mut latest_ts = None; + let mut active_users = BTreeSet::new(); + for event in events { + let Some(ts) = event.timestamp.as_deref() else { + continue; + }; + let Ok(cur) = parse_iso_utc(ts) else { + continue; + }; + if latest_ts.is_none_or(|latest| cur > latest) { + latest_ts = Some(cur); + active_users.clear(); + } + if latest_ts == Some(cur) { + let user = value_string(&event.data, "username"); + if !user.is_empty() && is_session_active(&event.data) { + active_users.insert(user); + } + } + } + (latest_ts, active_users) +} + +fn active_window_duration_today(window_events: &[AwEvent], start: DateTime) -> f64 { + window_events + .iter() + .filter(|event| { + event + .timestamp + .as_deref() + .and_then(|ts| parse_iso_utc(ts).ok()) + .is_some_and(|ts| ts >= start) + }) + .filter(|event| { + value_string(&event.data, "title") + .to_lowercase() + .contains("rdp active") + }) + .map(|event| event.duration.unwrap_or(0.0).max(0.0)) + .sum() +} + +fn should_heal( + aw: &AwClient, + config: &Config, + now: DateTime, + start: DateTime, +) -> Result<(bool, String)> { + let window_events = match aw.get_events(&config.window_bucket(), 12_000) { + Ok(events) => events, + Err(error) => { + return Ok((true, format!("window_read_failed: {error:#}"))); + } + }; + let session_events = match aw.get_events(&config.session_bucket(), 12_000) { + Ok(events) => events, + Err(error) => { + return Ok((true, format!("session_read_failed: {error:#}"))); + } + }; + let (latest_ts, active_users) = active_users_at_latest_session(&session_events); + let Some(latest_ts) = latest_ts else { + return Ok((false, "no_sessions".to_string())); + }; + let age = (now - latest_ts).num_milliseconds() as f64 / 1000.0; + if age > config.session_freshness_seconds { + return Ok((false, format!("sessions_stale age_seconds={age:.0}"))); + } + if active_users.is_empty() { + return Ok((false, "no_active_users".to_string())); + } + let active = active_window_duration_today(&window_events, start); + if active <= 0.0 { + Ok(( + true, + format!("zero_activity active_users={}", active_users.len()), + )) + } else { + Ok(( + false, + format!("activity_present active_seconds={active:.0}"), + )) + } +} + +fn build_window_title(users: &[String]) -> String { + if users.is_empty() { + "RDP idle".to_string() + } else { + format!("RDP active ({}): {}", users.len(), users.join(", ")) + } +} + +fn build_backfill( + session_events: Vec, + start: DateTime, +) -> (Vec, Vec) { + let mut grouped: BTreeMap, Vec> = BTreeMap::new(); + for event in session_events { + let Some(ts) = event.timestamp.as_deref() else { + continue; + }; + let Ok(parsed) = parse_iso_utc(ts) else { + continue; + }; + if parsed >= start { + grouped.entry(parsed).or_default().push(event); + } + } + let keys: Vec> = grouped.keys().copied().collect(); + let mut out_afk = Vec::new(); + let mut out_win = Vec::new(); + for (idx, ts) in keys.iter().enumerate() { + let rows = &grouped[ts]; + let mut duration = keys + .get(idx + 1) + .map(|next| ((*next - *ts).num_milliseconds() as f64 / 1000.0).max(0.0)) + .unwrap_or(10.0); + if duration <= 0.0 { + duration = 10.0; + } + duration = duration.min(30.0); + let users: Vec = rows + .iter() + .filter_map(|event| { + let user = value_string(&event.data, "username"); + if !user.is_empty() && is_session_active(&event.data) { + Some(user) + } else { + None + } + }) + .collect::>() + .into_iter() + .collect(); + let active = !users.is_empty(); + let ts = to_iso_utc(*ts); + out_afk.push(AwEvent { + timestamp: Some(ts.clone()), + duration: Some(duration), + data: json!({ + "status": if active { "not-afk" } else { "afk" }, + "source": AUTOHEAL_SOURCE, + }), + }); + out_win.push(AwEvent { + timestamp: Some(ts), + duration: Some(duration), + data: json!({ + "app": "RDP", + "title": build_window_title(&users), + "source": AUTOHEAL_SOURCE, + }), + }); + } + (out_afk, out_win) +} + +fn reset_bucket( + aw: &AwClient, + bucket_id: &str, + event_type: &str, + client_name: &str, + host: &str, +) -> Result<()> { + if let Err(error) = aw.delete_bucket(bucket_id) { + log(&format!( + "warn delete bucket {bucket_id} ignored: {error:#}" + )); + } + aw.ensure_bucket(bucket_id, event_type, client_name, host) +} + +fn run(cli: &Cli) -> Result { + let config = load_config(); + let aw = AwClient::new(&config)?; + let mut api_restarted = false; + let mut worktime_ok = true; + + if !probe_reports(&config) { + log("worktime API probe failed, restarting aw-worktime-api.service"); + api_restarted = true; + run_systemctl(&["restart", "aw-worktime-api.service"], cli.dry_run); + thread::sleep(Duration::from_secs(2)); + if !probe_reports(&config) { + log("worktime API still degraded after restart"); + worktime_ok = false; + } else { + log("worktime API recovered after restart"); + } + } + + if !worktime_ok { + log("skip warm/heal because worktime API is still unavailable"); + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + worktime_ok, + api_restarted, + management_warm_ok: None, + need_heal: false, + ui_bridge_started: false, + reset_buckets: false, + backfill_afk: 0, + backfill_window: 0, + reason: "worktime_unavailable".to_string(), + }); + } + + let management_warm_ok = if config.management_warm_enabled { + let ok = probe_url( + &config.management_warm_url, + config.management_warm_timeout_seconds, + ); + if ok { + log("management cache warm ok"); + } else { + log("management cache warm failed"); + } + Some(ok) + } else { + None + }; + + let start = moscow_day_start_utc(); + let now = Utc::now(); + let (need_heal, reason) = should_heal(&aw, &config, now, start)?; + if !need_heal { + log(&format!( + "health ok: {reason} for {}, no action", + config.host + )); + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + worktime_ok, + api_restarted, + management_warm_ok, + need_heal: false, + ui_bridge_started: false, + reset_buckets: false, + backfill_afk: 0, + backfill_window: 0, + reason, + }); + } + + log(&format!( + "detected zero activity for {}, running heal", + config.host + )); + let mut ui_bridge_started = + run_systemctl(&["restart", "aw-worktime-ui-bridge.timer"], cli.dry_run); + ui_bridge_started &= run_systemctl(&["start", "aw-worktime-ui-bridge.service"], cli.dry_run); + + let session_events = aw.get_events(&config.session_bucket(), 12_000)?; + let (afk_events, window_events) = build_backfill(session_events, start); + if afk_events.is_empty() || window_events.is_empty() { + log(&format!( + "heal skipped for {}, no source sessions", + config.host + )); + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + worktime_ok, + api_restarted, + management_warm_ok, + need_heal: true, + ui_bridge_started, + reset_buckets: false, + backfill_afk: 0, + backfill_window: 0, + reason: "no_source_sessions".to_string(), + }); + } + + if !cli.dry_run { + reset_bucket( + &aw, + &config.afk_bucket(), + "afkstatus", + "aw-worktime-ui-bridge", + &config.host, + )?; + reset_bucket( + &aw, + &config.window_bucket(), + "currentwindow", + "aw-worktime-ui-bridge", + &config.host, + )?; + aw.post_events_chunked(&config.afk_bucket(), &afk_events, config.aw_post_chunk_size)?; + aw.post_events_chunked( + &config.window_bucket(), + &window_events, + config.aw_post_chunk_size, + )?; + } + println!( + "autoheal backfill posted afk={} win={}", + afk_events.len(), + window_events.len() + ); + log(&format!("heal completed for {}", config.host)); + Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + worktime_ok, + api_restarted, + management_warm_ok, + need_heal: true, + ui_bridge_started, + reset_buckets: !cli.dry_run, + backfill_afk: afk_events.len(), + backfill_window: window_events.len(), + reason, + }) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let summary = run(&cli)?; + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(ts: &str, duration: f64, data: Value) -> AwEvent { + AwEvent { + timestamp: Some(ts.to_string()), + duration: Some(duration), + data, + } + } + + #[test] + fn unknown_rdp_session_counts_as_active() { + assert!(is_session_active(&json!({ + "sessionId": 5, + "state": "Unknown", + "username": "user5", + "sessionName": "rdp-tcp#0" + }))); + } + + #[test] + fn active_users_use_latest_sample_only() { + let events = vec![ + event( + "2026-05-27T07:59:25Z", + 0.0, + json!({"sessionId": 2, "state": "Активно", "username": "admin", "sessionName": "console"}), + ), + event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"sessionId": 3, "state": "Диск", "username": "old", "sessionName": ""}), + ), + event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"sessionId": 4, "state": "Активно", "username": "user5", "sessionName": "rdp-tcp#0"}), + ), + ]; + let (latest, users) = active_users_at_latest_session(&events); + assert_eq!( + latest.map(to_iso_utc).as_deref(), + Some("2026-05-27T07:59:30Z") + ); + assert_eq!(users, BTreeSet::from(["user5".to_string()])); + } + + #[test] + fn active_window_duration_counts_today_rdp_active_titles() { + let start = parse_iso_utc("2026-06-01T00:00:00Z").unwrap(); + let events = vec![ + event( + "2026-05-31T23:59:59Z", + 30.0, + json!({"title": "RDP active (1): user"}), + ), + event( + "2026-06-01T00:00:00Z", + 10.0, + json!({"title": "RDP active (1): user"}), + ), + event("2026-06-01T00:01:00Z", 20.0, json!({"title": "RDP idle"})), + ]; + assert_eq!(active_window_duration_today(&events, start), 10.0); + } + + #[test] + fn build_backfill_caps_duration_and_marks_active() { + let start = parse_iso_utc("2026-06-01T00:00:00Z").unwrap(); + let events = vec![ + event( + "2026-06-01T00:00:00Z", + 0.0, + json!({"sessionId": 3, "state": "Активно", "username": "user5", "sessionName": "rdp-tcp#0"}), + ), + event( + "2026-06-01T00:00:45Z", + 0.0, + json!({"sessionId": 3, "state": "Диск", "username": "user5", "sessionName": "rdp-tcp#0"}), + ), + ]; + let (afk, win) = build_backfill(events, start); + assert_eq!(afk.len(), 2); + assert_eq!(win[0].duration, Some(30.0)); + assert_eq!(value_string(&afk[0].data, "status"), "not-afk"); + assert_eq!(value_string(&win[0].data, "title"), "RDP active (1): user5"); + assert_eq!(value_string(&afk[1].data, "status"), "afk"); + } +} diff --git a/adk-rust/crates/worktime-influx-exporter/Cargo.toml b/adk-rust/crates/worktime-influx-exporter/Cargo.toml new file mode 100644 index 0000000..94a7849 --- /dev/null +++ b/adk-rust/crates/worktime-influx-exporter/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "worktime-influx-exporter" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/worktime-influx-exporter/src/main.rs b/adk-rust/crates/worktime-influx-exporter/src/main.rs new file mode 100644 index 0000000..bb2ac7a --- /dev/null +++ b/adk-rust/crates/worktime-influx-exporter/src/main.rs @@ -0,0 +1,1025 @@ +use std::{collections::HashMap, thread, time::Duration}; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{DateTime, Datelike, FixedOffset, NaiveDate, TimeDelta, TimeZone, Timelike, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::Value; + +const DEFAULT_AW_BASE: &str = "http://127.0.0.1:5600"; +const DEFAULT_WORKTIME_REPORT_BASE: &str = "http://127.0.0.1:5610"; +const DEFAULT_INFLUX_ORG: &str = "proxmox"; +const DEFAULT_INFLUX_BUCKET: &str = "aw_metrics"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const DEFAULT_DAYS: &str = "today,yesterday"; + +#[derive(Debug, Parser)] +#[command(about = "AW Worktime InfluxDB exporter")] +struct Cli { + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, + + #[arg(long, default_value_t = 30)] + timeout_seconds: u64, +} + +#[derive(Debug, Clone)] +struct Config { + aw_api_base: String, + report_base: String, + influx_url: String, + influx_org: String, + influx_bucket: String, + influx_token: String, + influx_enabled: bool, + hosts: Vec, + days: Vec, + report_offset: FixedOffset, + default_sample_seconds: f64, + max_sample_seconds: f64, + events_limit: usize, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + enabled: bool, + dry_run: bool, + hosts: Vec, + days: Vec, + lines: usize, + written: usize, + bucket: String, + error: Option, +} + +#[derive(Debug, Clone)] +enum FieldValue { + Int(i64), + String(String), +} + +#[derive(Debug, Clone)] +struct Interval { + start: DateTime, + end: DateTime, +} + +#[derive(Debug, Clone)] +struct UserAccumulator { + user: String, + user_id: String, + samples_count: i64, + active_samples: i64, + session_ids: Vec, + intervals: Vec, +} + +#[derive(Debug, Clone)] +struct DailyRow { + user: String, + user_id: String, + active_seconds: i64, + idle_seconds: i64, + sessions_count: i64, + samples_count: i64, + active_samples: i64, +} + +#[derive(Debug, Clone)] +struct HourlyRow { + user: String, + user_id: String, + bucket_start_utc: DateTime, + report_date: String, + hour_local: String, + active_seconds: i64, +} + +type IdentitySamples = HashMap<(String, String), Vec<(DateTime, Value)>>; + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_bool(name: &str, fallback: bool) -> bool { + match std::env::var(name) + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + { + Some(value) if !value.is_empty() => matches!(value.as_str(), "1" | "true" | "yes" | "on"), + _ => fallback, + } +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn env_usize(name: &str, fallback: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn split_csv(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn build_aw_api_base(raw: &str) -> String { + let base = raw.trim().trim_end_matches('/'); + if base.ends_with("/api/0") { + base.to_string() + } else { + format!("{base}/api/0") + } +} + +fn report_offset_from_env() -> FixedOffset { + let tz = env("AW_WORKTIME_TZ", "Europe/Moscow"); + match tz.as_str() { + "UTC" | "Etc/UTC" => FixedOffset::east_opt(0).expect("valid UTC offset"), + "Europe/Moscow" => FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"), + value if value.starts_with('+') || value.starts_with('-') => parse_offset(value) + .unwrap_or_else(|| FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset")), + _ => FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"), + } +} + +fn parse_offset(value: &str) -> Option { + let sign = if value.starts_with('-') { -1 } else { 1 }; + let raw = value.trim_start_matches(['+', '-']); + let mut parts = raw.split(':'); + let hours: i32 = parts.next()?.parse().ok()?; + let minutes: i32 = parts.next().unwrap_or("0").parse().ok()?; + FixedOffset::east_opt(sign * (hours * 3600 + minutes * 60)) +} + +fn load_config() -> Config { + let aw_base = env( + "AW_WORKTIME_AW_API_BASE", + &env("AW_SERVER_URL", DEFAULT_AW_BASE), + ); + let default_sample_seconds = env_f64("AW_WORKTIME_DEFAULT_SAMPLE_SECONDS", 30.0).max(1.0); + let max_sample_seconds = + env_f64("AW_WORKTIME_MAX_SAMPLE_SECONDS", 300.0).max(default_sample_seconds); + Config { + aw_api_base: build_aw_api_base(&aw_base), + report_base: env("AW_WORKTIME_REPORT_BASE", DEFAULT_WORKTIME_REPORT_BASE) + .trim_end_matches('/') + .to_string(), + influx_url: env("AW_WORKTIME_INFLUX_URL", "") + .trim_end_matches('/') + .to_string(), + influx_org: env("AW_WORKTIME_INFLUX_ORG", DEFAULT_INFLUX_ORG), + influx_bucket: env("AW_WORKTIME_INFLUX_BUCKET", DEFAULT_INFLUX_BUCKET), + influx_token: env("AW_WORKTIME_INFLUX_TOKEN", ""), + influx_enabled: env_bool("AW_WORKTIME_INFLUX_ENABLED", false), + hosts: split_csv(&env( + "AW_WORKTIME_INFLUX_HOSTS", + &env("AW_WORKTIME_HOST", DEFAULT_HOST), + )), + days: split_csv(&env("AW_WORKTIME_INFLUX_DAYS", DEFAULT_DAYS)), + report_offset: report_offset_from_env(), + default_sample_seconds, + max_sample_seconds, + events_limit: env_usize("AW_WORKTIME_EVENTS_LIMIT", 50_000).max(1000), + } +} + +fn utc_now() -> DateTime { + Utc::now() +} + +fn pts(value: Option<&str>) -> Option> { + let value = value?.trim(); + if value.is_empty() { + return None; + } + DateTime::parse_from_rfc3339(&value.replace('Z', "+00:00")) + .map(|parsed| parsed.with_timezone(&Utc)) + .ok() +} + +fn resolve_report_date(day: &str, offset: FixedOffset) -> Result { + if let Ok(date) = NaiveDate::parse_from_str(day, "%Y-%m-%d") { + return Ok(date); + } + let today = utc_now().with_timezone(&offset).date_naive(); + if day == "yesterday" { + Ok(today - TimeDelta::days(1)) + } else { + Ok(today) + } +} + +fn report_bounds( + report_date: NaiveDate, + offset: FixedOffset, +) -> Result<(DateTime, DateTime, DateTime)> { + let start_local = offset + .with_ymd_and_hms( + report_date.year(), + report_date.month(), + report_date.day(), + 0, + 0, + 0, + ) + .single() + .ok_or_else(|| anyhow!("invalid local report date"))?; + let start = start_local.with_timezone(&Utc); + let end_exclusive = (start_local + TimeDelta::days(1)).with_timezone(&Utc); + let end_inclusive = end_exclusive - TimeDelta::seconds(1); + Ok((start, end_inclusive, end_exclusive)) +} + +fn escape_tag(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace(' ', "\\ ") + .replace(',', "\\,") + .replace('=', "\\=") +} + +fn timestamp_ns(dt: DateTime) -> i64 { + dt.timestamp_nanos_opt() + .unwrap_or_else(|| dt.timestamp() * 1_000_000_000) +} + +fn line( + measurement: &str, + tags: Vec<(&str, String)>, + fields: Vec<(&str, FieldValue)>, + timestamp_ns: i64, +) -> Option { + let mut tag_items: Vec<(&str, String)> = tags + .into_iter() + .filter(|(_key, value)| !value.is_empty()) + .collect(); + tag_items.sort_by(|left, right| left.0.cmp(right.0)); + let tag_part = tag_items + .iter() + .map(|(key, value)| format!("{key}={}", escape_tag(value))) + .collect::>() + .join(","); + + let field_parts = fields + .into_iter() + .map(|(key, value)| match value { + FieldValue::Int(value) => format!("{key}={value}i"), + FieldValue::String(value) => { + let text = value.replace('\\', "\\\\").replace('"', "\\\""); + format!("{key}=\"{text}\"") + } + }) + .collect::>(); + if field_parts.is_empty() { + return None; + } + if tag_part.is_empty() { + Some(format!( + "{measurement} {} {timestamp_ns}", + field_parts.join(",") + )) + } else { + Some(format!( + "{measurement},{tag_part} {} {timestamp_ns}", + field_parts.join(",") + )) + } +} + +fn get_json(client: &Client, url: &str) -> Result { + let mut last_error = None; + for attempt in 1..=6 { + match client + .get(url) + .header("Accept", "application/json") + .send() + .and_then(|resp| resp.error_for_status()) + .and_then(|resp| resp.json()) + { + Ok(payload) => return Ok(payload), + Err(err) => { + last_error = Some(err); + if attempt < 6 { + thread::sleep(Duration::from_millis(250 * attempt)); + } + } + } + } + Err(anyhow!( + "GET {url}: {}", + last_error + .map(|err| err.to_string()) + .unwrap_or_else(|| "unknown error".to_string()) + )) +} + +fn format_aw_time(value: DateTime) -> String { + value.to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +fn fetch_events_for_date( + client: &Client, + config: &Config, + host: &str, + start: DateTime, + end: DateTime, +) -> Vec { + let bucket_id = format!("aw-worktime-sessions_{host}"); + let bucket_url = format!( + "{}/buckets/{}", + config.aw_api_base, + urlencoding::encode(&bucket_id) + ); + if let Err(err) = get_json(client, &bucket_url) { + eprintln!( + "[aw-worktime-influx-exporter] bucket lookup failed for host={host} bucket={bucket_id}: {err}" + ); + return Vec::new(); + } + let query = format!( + "start={}&end={}&limit={}", + urlencoding::encode(&format_aw_time(start)), + urlencoding::encode(&format_aw_time(end)), + config.events_limit + ); + let events_url = format!( + "{}/buckets/{}/events?{}", + config.aw_api_base, + urlencoding::encode(&bucket_id), + query + ); + match get_json(client, &events_url) { + Ok(payload) => payload.as_array().cloned().unwrap_or_default(), + Err(err) => { + eprintln!( + "[aw-worktime-influx-exporter] events fetch failed for host={host} bucket={bucket_id}: {err}" + ); + Vec::new() + } + } +} + +fn s(value: Option<&Value>) -> String { + match value { + Some(Value::Null) | None => String::new(), + Some(Value::String(value)) => value.trim().to_string(), + Some(Value::Bool(value)) => value.to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(other) => other.to_string(), + } +} + +fn int_value(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(value)) => value + .as_i64() + .or_else(|| value.as_u64().map(|v| v as i64)) + .unwrap_or(0), + Some(Value::String(value)) => value.parse().unwrap_or(0), + Some(Value::Bool(value)) => i64::from(*value), + _ => 0, + } +} + +fn bool_value(value: Option<&Value>) -> bool { + match value { + Some(Value::Bool(value)) => *value, + Some(Value::Number(value)) => value.as_i64().unwrap_or(0) != 0, + Some(Value::String(value)) => matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ), + _ => false, + } +} + +fn is_machine_user(user: &str) -> bool { + let user = user.trim().to_ascii_lowercase(); + user.ends_with('$') || matches!(user.as_str(), "system" | "localservice" | "networkservice") +} + +fn is_active_sample(data: &serde_json::Map) -> bool { + let state = s(data.get("state")).to_ascii_lowercase(); + if bool_value(data.get("active")) { + return true; + } + if state.contains("актив") || state == "active" { + return true; + } + if state == "unknown" { + let session_id = int_value(data.get("sessionId")); + let user = s(data.get("username")); + let session_name = s(data.get("sessionName")).to_ascii_lowercase(); + return session_id > 0 + && !user.is_empty() + && !is_machine_user(&user) + && (session_name.starts_with("rdp-") || session_name == "console"); + } + false +} + +fn normalize_user_id(data: &serde_json::Map, host: &str, username: &str) -> String { + let user_id = s(data.get("userId")); + if !user_id.is_empty() { + if let Some((_left, right)) = user_id.split_once('\\') { + if !right.is_empty() { + return format!("{host}\\{right}"); + } + } + return user_id; + } + format!("{host}\\{username}") +} + +fn clamp_seconds(value: f64, config: &Config) -> f64 { + let mut seconds = value; + if seconds <= 0.0 || !seconds.is_finite() { + seconds = config.default_sample_seconds; + } + seconds.min(config.max_sample_seconds) +} + +fn event_sample_seconds( + event: &Value, + next_same_session_ts: Option>, + event_ts: DateTime, + data: &serde_json::Map, + config: &Config, +) -> f64 { + for key in ["sampleSeconds", "pollSeconds"] { + let value = data.get(key); + let parsed = match value { + Some(Value::Number(number)) => number.as_f64(), + Some(Value::String(text)) => text.parse().ok(), + _ => None, + }; + if let Some(seconds) = parsed.filter(|seconds| *seconds > 0.0) { + return clamp_seconds(seconds, config); + } + } + let duration = match event.get("duration") { + Some(Value::Number(number)) => number.as_f64().unwrap_or(0.0), + Some(Value::String(text)) => text.parse().unwrap_or(0.0), + _ => 0.0, + }; + if duration > 0.0 { + return clamp_seconds(duration, config); + } + if let Some(next_ts) = next_same_session_ts { + let delta = (next_ts - event_ts).num_milliseconds() as f64 / 1000.0; + if delta > 0.0 { + return clamp_seconds(delta, config); + } + } + clamp_seconds(config.default_sample_seconds, config) +} + +fn merge_intervals(intervals: &[Interval]) -> Vec { + if intervals.is_empty() { + return Vec::new(); + } + let mut ordered = intervals.to_vec(); + ordered.sort_by_key(|item| item.start); + let mut merged = vec![ordered[0].clone()]; + for interval in ordered.into_iter().skip(1) { + let last = merged.last_mut().expect("merged interval exists"); + if interval.start <= last.end { + if interval.end > last.end { + last.end = interval.end; + } + } else { + merged.push(interval); + } + } + merged +} + +fn collect_user_rows( + events: &[Value], + end_exclusive: DateTime, + host: &str, + config: &Config, +) -> HashMap { + let mut by_identity: IdentitySamples = HashMap::new(); + for event in events { + let Some(ts) = pts(event.get("timestamp").and_then(Value::as_str)) else { + continue; + }; + let Some(data) = event.get("data").and_then(Value::as_object) else { + continue; + }; + let username = s(data.get("username")); + if username.is_empty() { + continue; + } + let session_id = s(data.get("sessionId")); + by_identity + .entry(( + username, + if session_id.is_empty() { + "unknown".to_string() + } else { + session_id + }, + )) + .or_default() + .push((ts, event.clone())); + } + + let mut by_user: HashMap = HashMap::new(); + for ((username, session_id), mut samples) in by_identity { + samples.sort_by_key(|item| item.0); + for idx in 0..samples.len() { + let (event_ts, event) = &samples[idx]; + let Some(data) = event.get("data").and_then(Value::as_object) else { + continue; + }; + let next_ts = samples.get(idx + 1).map(|item| item.0); + let active = is_active_sample(data); + let sample_seconds = event_sample_seconds(event, next_ts, *event_ts, data, config); + let row = by_user + .entry(username.clone()) + .or_insert_with(|| UserAccumulator { + user: username.clone(), + user_id: normalize_user_id(data, host, &username), + samples_count: 0, + active_samples: 0, + session_ids: Vec::new(), + intervals: Vec::new(), + }); + row.samples_count += 1; + if !row.session_ids.contains(&session_id) { + row.session_ids.push(session_id.clone()); + } + if active { + row.active_samples += 1; + let interval_start = *event_ts; + let interval_end = (*event_ts + + TimeDelta::milliseconds((sample_seconds * 1000.0) as i64)) + .min(end_exclusive); + if interval_end > interval_start { + row.intervals.push(Interval { + start: interval_start, + end: interval_end, + }); + } + } + } + } + by_user +} + +fn aggregate_daily_rows( + events: &[Value], + start: DateTime, + end_exclusive: DateTime, + host: &str, + config: &Config, +) -> Vec { + let by_user = collect_user_rows(events, end_exclusive, host, config); + let mut users: Vec<_> = by_user.keys().cloned().collect(); + users.sort(); + let full_range = (end_exclusive - start).num_seconds(); + let mut rows = Vec::new(); + for username in users { + let row = &by_user[&username]; + let active_seconds: i64 = merge_intervals(&row.intervals) + .iter() + .map(|interval| (interval.end - interval.start).num_seconds()) + .sum::() + .min(full_range); + rows.push(DailyRow { + user: row.user.clone(), + user_id: row.user_id.clone(), + active_seconds, + idle_seconds: (full_range - active_seconds).max(0), + sessions_count: row.session_ids.len() as i64, + samples_count: row.samples_count, + active_samples: row.active_samples, + }); + } + rows +} + +fn aggregate_hourly_rows( + events: &[Value], + end_exclusive: DateTime, + host: &str, + config: &Config, +) -> Vec { + let by_user = collect_user_rows(events, end_exclusive, host, config); + let mut users: Vec<_> = by_user.keys().cloned().collect(); + users.sort(); + let mut rows = Vec::new(); + for username in users { + let row = &by_user[&username]; + let mut per_bucket: HashMap, i64> = HashMap::new(); + for interval in merge_intervals(&row.intervals) { + let mut cursor = interval.start; + while cursor < interval.end { + let local = cursor.with_timezone(&config.report_offset); + let bucket_local = config + .report_offset + .with_ymd_and_hms(local.year(), local.month(), local.day(), local.hour(), 0, 0) + .single() + .expect("valid hourly bucket"); + let bucket_start = bucket_local.with_timezone(&Utc); + let bucket_end = (bucket_local + TimeDelta::hours(1)).with_timezone(&Utc); + let overlap_start = interval.start.max(bucket_start); + let overlap_end = interval.end.min(bucket_end); + if overlap_end > overlap_start { + *per_bucket.entry(bucket_start).or_default() += + (overlap_end - overlap_start).num_seconds(); + } + cursor = bucket_end; + } + } + let mut buckets: Vec<_> = per_bucket.into_iter().collect(); + buckets.sort_by_key(|item| item.0); + for (bucket_start_utc, active_seconds) in buckets { + if active_seconds <= 0 { + continue; + } + let bucket_local = bucket_start_utc.with_timezone(&config.report_offset); + rows.push(HourlyRow { + user: row.user.clone(), + user_id: row.user_id.clone(), + bucket_start_utc, + report_date: bucket_local.date_naive().to_string(), + hour_local: format!("{:02}:00", bucket_local.hour()), + active_seconds, + }); + } + } + rows +} + +fn fetch_true_active_apps( + client: &Client, + config: &Config, + host: &str, + day: &str, + report_date: NaiveDate, +) -> Vec { + let date_param = if matches!(day, "today" | "yesterday") { + format!("day={}", urlencoding::encode(day)) + } else { + format!("date={report_date}") + }; + let url = format!( + "{}/reports/worktime/today?host={}&{}&allow_stale=1", + config.report_base, + urlencoding::encode(host), + date_param + ); + match get_json(client, &url) { + Ok(payload) => payload + .get("true_active_apps") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(), + Err(err) => { + eprintln!( + "[aw-worktime-influx-exporter] true-active fetch failed for host={host} day={day}: {err}" + ); + Vec::new() + } + } +} + +fn build_report_summary(rows: &[DailyRow]) -> (i64, i64, String) { + if rows.is_empty() { + return (0, 0, String::new()); + } + let total_active_seconds = rows.iter().map(|row| row.active_seconds).sum(); + let top_user = rows + .iter() + .max_by_key(|row| row.active_seconds) + .map(|row| row.user.clone()) + .unwrap_or_default(); + (rows.len() as i64, total_active_seconds, top_user) +} + +fn build_lines_for_day( + client: &Client, + config: &Config, + host: &str, + day: &str, +) -> Result> { + let report_date = resolve_report_date(day, config.report_offset)?; + let (start, end_inclusive, end_exclusive) = report_bounds(report_date, config.report_offset)?; + let daily_ts = timestamp_ns(start); + let events = fetch_events_for_date(client, config, host, start, end_inclusive); + let daily_rows = aggregate_daily_rows(&events, start, end_exclusive, host, config); + let hourly_rows = aggregate_hourly_rows(&events, end_exclusive, host, config); + let true_active_apps = fetch_true_active_apps(client, config, host, day, report_date); + let mut lines = Vec::new(); + + for row in &daily_rows { + if let Some(line) = line( + "aw_rdp_worktime_daily", + vec![ + ("host", host.to_string()), + ("user", row.user.clone()), + ("user_id", row.user_id.clone()), + ("report_date", report_date.to_string()), + ], + vec![ + ("active_seconds", FieldValue::Int(row.active_seconds)), + ("idle_seconds", FieldValue::Int(row.idle_seconds)), + ("sessions_count", FieldValue::Int(row.sessions_count)), + ("samples_count", FieldValue::Int(row.samples_count)), + ("active_samples", FieldValue::Int(row.active_samples)), + ], + daily_ts, + ) { + lines.push(line); + } + } + + for row in &hourly_rows { + if let Some(line) = line( + "aw_rdp_worktime_hourly", + vec![ + ("host", host.to_string()), + ("user", row.user.clone()), + ("user_id", row.user_id.clone()), + ("report_date", row.report_date.clone()), + ("hour_local", row.hour_local.clone()), + ], + vec![("active_seconds", FieldValue::Int(row.active_seconds))], + timestamp_ns(row.bucket_start_utc), + ) { + lines.push(line); + } + } + + let (users_count, total_active_seconds, top_user) = build_report_summary(&daily_rows); + if let Some(line) = line( + "aw_rdp_worktime_summary_daily", + vec![ + ("host", host.to_string()), + ("report_date", report_date.to_string()), + ], + vec![ + ("users_count", FieldValue::Int(users_count)), + ( + "total_active_seconds", + FieldValue::Int(total_active_seconds), + ), + ("top_user", FieldValue::String(top_user)), + ], + daily_ts, + ) { + lines.push(line); + } + + for app in &true_active_apps { + if let Some(line) = line( + "aw_true_active_app_daily", + vec![ + ("host", host.to_string()), + ("application", s(app.get("application"))), + ("report_date", report_date.to_string()), + ], + vec![ + ( + "proved_work_seconds", + FieldValue::Int(int_value(app.get("proved_work_seconds"))), + ), + ( + "evidence_events", + FieldValue::Int(int_value(app.get("evidence_events"))), + ), + ( + "proved_work_human", + FieldValue::String(s(app.get("proved_work_human"))), + ), + ( + "proved_work_hhmm", + FieldValue::String(s(app.get("proved_work_hhmm"))), + ), + ("last_action", FieldValue::String(s(app.get("last_action")))), + ( + "last_action_local", + FieldValue::String(s(app.get("last_action_local"))), + ), + ( + "last_action_utc", + FieldValue::String(s(app.get("last_action_utc"))), + ), + ], + daily_ts, + ) { + lines.push(line); + } + } + + Ok(lines) +} + +fn write_lines(client: &Client, config: &Config, lines: &[String]) -> Result { + if lines.is_empty() { + return Ok(0); + } + if config.influx_url.is_empty() || config.influx_token.is_empty() { + bail!("InfluxDB destination is not configured"); + } + let url = format!( + "{}/api/v2/write?org={}&bucket={}&precision=ns", + config.influx_url, + urlencoding::encode(&config.influx_org), + urlencoding::encode(&config.influx_bucket) + ); + let payload = format!("{}\n", lines.join("\n")); + client + .post(url) + .header("Authorization", format!("Token {}", config.influx_token)) + .header("Content-Type", "text/plain; charset=utf-8") + .body(payload) + .send() + .and_then(|resp| resp.error_for_status()) + .context("InfluxDB write failed")?; + Ok(lines.len()) +} + +fn run(cli: &Cli) -> Result { + let config = load_config(); + if !config.influx_enabled && !cli.dry_run { + return Ok(RunSummary { + ok: true, + enabled: false, + dry_run: false, + hosts: config.hosts, + days: config.days, + lines: 0, + written: 0, + bucket: config.influx_bucket, + error: None, + }); + } + let client = Client::builder() + .timeout(Duration::from_secs(cli.timeout_seconds)) + .no_proxy() + .build() + .context("build HTTP client")?; + let mut lines = Vec::new(); + for host in &config.hosts { + for day in &config.days { + lines.extend(build_lines_for_day(&client, &config, host, day)?); + } + } + let written = if cli.dry_run { + 0 + } else { + write_lines(&client, &config, &lines)? + }; + Ok(RunSummary { + ok: true, + enabled: config.influx_enabled, + dry_run: cli.dry_run, + hosts: config.hosts, + days: config.days, + lines: lines.len(), + written, + bucket: config.influx_bucket, + error: None, + }) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + match run(&cli) { + Ok(summary) => { + if cli.json || cli.dry_run { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if !summary.enabled { + eprintln!("[aw-worktime-influx-exporter] disabled by AW_WORKTIME_INFLUX_ENABLED"); + } else { + eprintln!( + "[aw-worktime-influx-exporter] wrote {} points to {}", + summary.written, summary.bucket + ); + } + Ok(()) + } + Err(err) => { + if cli.json { + println!( + "{}", + serde_json::to_string_pretty(&RunSummary { + ok: false, + enabled: false, + dry_run: cli.dry_run, + hosts: Vec::new(), + days: Vec::new(), + lines: 0, + written: 0, + bucket: String::new(), + error: Some(err.to_string()), + })? + ); + } + Err(err) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn test_config() -> Config { + Config { + aw_api_base: "http://127.0.0.1:5600/api/0".to_string(), + report_base: "http://127.0.0.1:5610".to_string(), + influx_url: String::new(), + influx_org: DEFAULT_INFLUX_ORG.to_string(), + influx_bucket: DEFAULT_INFLUX_BUCKET.to_string(), + influx_token: String::new(), + influx_enabled: false, + hosts: vec![DEFAULT_HOST.to_string()], + days: vec!["today".to_string()], + report_offset: FixedOffset::east_opt(3 * 3600).unwrap(), + default_sample_seconds: 30.0, + max_sample_seconds: 300.0, + events_limit: 50_000, + } + } + + #[test] + fn line_escapes_tags_and_string_fields() { + let out = line( + "m", + vec![("host", "A B,C=D".to_string())], + vec![("text", FieldValue::String("a\"b".to_string()))], + 10, + ) + .unwrap(); + assert_eq!(out, "m,host=A\\ B\\,C\\=D text=\"a\\\"b\" 10"); + } + + #[test] + fn aggregates_daily_and_hourly_active_samples() { + let config = test_config(); + let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap(); + let (start, _end_inclusive, end_exclusive) = + report_bounds(report_date, config.report_offset).unwrap(); + let events = vec![json!({ + "timestamp": "2026-05-14T06:00:00Z", + "duration": 0.0, + "data": { + "username": "user5", + "userId": "WORKGROUP\\user5", + "sessionId": 4, + "state": "Активно", + "active": true, + "sampleSeconds": 1800 + } + })]; + + let daily = aggregate_daily_rows(&events, start, end_exclusive, "SHARKON2025", &config); + let hourly = aggregate_hourly_rows(&events, end_exclusive, "SHARKON2025", &config); + + assert_eq!(daily.len(), 1); + assert_eq!(daily[0].user_id, "SHARKON2025\\user5"); + assert_eq!(daily[0].active_seconds, 300); + assert_eq!(daily[0].active_samples, 1); + assert_eq!(hourly.len(), 1); + assert_eq!(hourly[0].report_date, "2026-05-14"); + assert_eq!(hourly[0].hour_local, "09:00"); + assert_eq!(hourly[0].active_seconds, 300); + } + + #[test] + fn resolves_moscow_report_bounds() { + let offset = FixedOffset::east_opt(3 * 3600).unwrap(); + let report_date = NaiveDate::from_ymd_opt(2026, 6, 1).unwrap(); + let (start, end_inclusive, end_exclusive) = report_bounds(report_date, offset).unwrap(); + assert_eq!(format_aw_time(start), "2026-05-31T21:00:00Z"); + assert_eq!(format_aw_time(end_inclusive), "2026-06-01T20:59:59Z"); + assert_eq!(format_aw_time(end_exclusive), "2026-06-01T21:00:00Z"); + } +} diff --git a/adk-rust/crates/worktime-prewarm/Cargo.toml b/adk-rust/crates/worktime-prewarm/Cargo.toml new file mode 100644 index 0000000..7012bba --- /dev/null +++ b/adk-rust/crates/worktime-prewarm/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "worktime-prewarm" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/worktime-prewarm/src/main.rs b/adk-rust/crates/worktime-prewarm/src/main.rs new file mode 100644 index 0000000..f57f7a8 --- /dev/null +++ b/adk-rust/crates/worktime-prewarm/src/main.rs @@ -0,0 +1,348 @@ +use std::{thread, time::Duration}; + +use anyhow::{Context, Result}; +use chrono::Local; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; + +const DEFAULT_BASE_URL: &str = "http://127.0.0.1:5610"; +const DEFAULT_HOST: &str = "SHARKON2025"; + +#[derive(Debug, Parser)] +#[command(about = "AW Worktime report cache prewarm")] +struct Cli { + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + base_url: String, + timeout_seconds: f64, + health_timeout_seconds: f64, + ready_timeout_seconds: f64, + ready_interval_seconds: f64, + host: String, + profile: String, +} + +#[derive(Debug, Serialize)] +struct ProbeResult { + url: String, + ok: bool, + code: Option, + cache: Option, + reason: Option, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + dry_run: bool, + profile: String, + host: String, + urls: Vec, + failures: usize, + readiness_ok: bool, + probes: Vec, +} + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .filter(|value| *value > 0.0) + .unwrap_or(fallback) +} + +fn load_config() -> Config { + Config { + base_url: env("WORKTIME_BASE_URL", DEFAULT_BASE_URL) + .trim_end_matches('/') + .to_string(), + timeout_seconds: env_f64("WORKTIME_PREWARM_TIMEOUT_SECONDS", 45.0), + health_timeout_seconds: env_f64("WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS", 10.0), + ready_timeout_seconds: env_f64("WORKTIME_PREWARM_READY_TIMEOUT_SECONDS", 60.0), + ready_interval_seconds: env_f64("WORKTIME_PREWARM_READY_INTERVAL_SECONDS", 2.0), + host: env( + "WORKTIME_PREWARM_HOST", + &env("AW_WORKTIME_HOST", DEFAULT_HOST), + ), + profile: env("WORKTIME_PREWARM_PROFILE", "full"), + } +} + +fn log(message: &str) { + println!("{} {}", Local::now().format("%F %T"), message); +} + +fn duration(seconds: f64) -> Duration { + Duration::from_secs_f64(seconds.max(0.001)) +} + +fn profile_urls(config: &Config) -> Option> { + let host = urlencoding::encode(&config.host); + let base = &config.base_url; + let full = vec![ + format!("{base}/reports/worktime/today?day=today&format=csv&host={host}"), + format!("{base}/reports/worktime/today?day=today&format=json&host={host}"), + format!("{base}/reports/worktime/today?day=today&format=html&host={host}"), + format!("{base}/reports/worktime/management?day=today&format=csv&host={host}"), + format!("{base}/reports/worktime/management?day=today&format=json&host={host}"), + format!("{base}/reports/worktime/management?day=today&format=html&host={host}"), + ]; + let startup = vec![ + format!("{base}/reports/worktime/today?day=today&format=csv&host={host}"), + format!("{base}/reports/worktime/today?day=today&format=json&host={host}"), + format!("{base}/reports/worktime/management?day=today&format=json&host={host}"), + format!("{base}/reports/worktime/management?day=today&format=csv&host={host}"), + ]; + match config.profile.as_str() { + "full" => Some(full), + "startup" => Some(startup), + _ => None, + } +} + +fn build_client(timeout_seconds: f64) -> Result { + Client::builder() + .timeout(duration(timeout_seconds)) + .no_proxy() + .build() + .context("build HTTP client") +} + +fn probe(url: &str, timeout_seconds: f64) -> ProbeResult { + let client = match build_client(timeout_seconds) { + Ok(client) => client, + Err(_) => { + return ProbeResult { + url: url.to_string(), + ok: false, + code: None, + cache: None, + reason: None, + }; + } + }; + match client.get(url).send() { + Ok(resp) => { + let code = resp.status().as_u16(); + let cache = resp + .headers() + .get("x-aw-worktime-cache") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned); + let reason = resp + .headers() + .get("x-aw-worktime-cache-reason") + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned); + ProbeResult { + url: url.to_string(), + ok: (200..300).contains(&code), + code: Some(code), + cache, + reason, + } + } + Err(_) => ProbeResult { + url: url.to_string(), + ok: false, + code: None, + cache: None, + reason: None, + }, + } +} + +fn log_probe(result: &ProbeResult) { + if result.ok { + log(&format!( + "ok code={} cache={} reason={} url={}", + result.code.unwrap_or(0), + result.cache.as_deref().unwrap_or("none"), + result.reason.as_deref().unwrap_or("none"), + result.url + )); + } else { + log(&format!( + "warn code={} url={}", + result + .code + .map(|code| code.to_string()) + .unwrap_or_else(|| "000".to_string()), + result.url + )); + } +} + +fn wait_until_ready(config: &Config) -> bool { + let deadline = std::time::Instant::now() + duration(config.ready_timeout_seconds); + loop { + let result = probe( + &format!("{}/health", config.base_url), + config.health_timeout_seconds, + ); + log_probe(&result); + if result.ok { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + thread::sleep(duration(config.ready_interval_seconds)); + } +} + +fn run(cli: &Cli) -> RunSummary { + let config = load_config(); + let Some(urls) = profile_urls(&config) else { + log(&format!("unknown profile={}", config.profile)); + return RunSummary { + ok: true, + dry_run: cli.dry_run, + profile: config.profile, + host: config.host, + urls: Vec::new(), + failures: 0, + readiness_ok: false, + probes: Vec::new(), + }; + }; + + if cli.dry_run { + return RunSummary { + ok: true, + dry_run: true, + profile: config.profile, + host: config.host, + urls, + failures: 0, + readiness_ok: false, + probes: Vec::new(), + }; + } + + if !wait_until_ready(&config) { + log("health readiness timed out; skip prewarm"); + return RunSummary { + ok: true, + dry_run: false, + profile: config.profile, + host: config.host, + urls, + failures: 0, + readiness_ok: false, + probes: Vec::new(), + }; + } + + let mut probes = Vec::new(); + let mut failures = 0; + for url in &urls { + let result = probe(url, config.timeout_seconds); + if !result.ok { + failures += 1; + } + log_probe(&result); + probes.push(result); + } + + if failures > 0 { + log(&format!( + "completed profile={} with failures={failures}", + config.profile + )); + } else { + log(&format!( + "completed profile={} successfully", + config.profile + )); + } + + RunSummary { + ok: true, + dry_run: false, + profile: config.profile, + host: config.host, + urls, + failures, + readiness_ok: true, + probes, + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let summary = run(&cli); + if cli.json || cli.dry_run { + println!("{}", serde_json::to_string_pretty(&summary)?); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn full_profile_has_legacy_urls() { + let config = Config { + base_url: DEFAULT_BASE_URL.to_string(), + timeout_seconds: 45.0, + health_timeout_seconds: 10.0, + ready_timeout_seconds: 60.0, + ready_interval_seconds: 2.0, + host: DEFAULT_HOST.to_string(), + profile: "full".to_string(), + }; + let urls = profile_urls(&config).unwrap(); + assert_eq!(urls.len(), 6); + assert!(urls[0].contains("format=csv")); + assert!(urls[5].contains("/reports/worktime/management")); + } + + #[test] + fn startup_profile_has_legacy_urls() { + let config = Config { + base_url: DEFAULT_BASE_URL.to_string(), + timeout_seconds: 45.0, + health_timeout_seconds: 10.0, + ready_timeout_seconds: 60.0, + ready_interval_seconds: 2.0, + host: DEFAULT_HOST.to_string(), + profile: "startup".to_string(), + }; + let urls = profile_urls(&config).unwrap(); + assert_eq!(urls.len(), 4); + assert!(urls[2].contains("format=json")); + assert!(urls[3].contains("format=csv")); + } + + #[test] + fn unknown_profile_is_ignored_like_legacy() { + let config = Config { + base_url: DEFAULT_BASE_URL.to_string(), + timeout_seconds: 45.0, + health_timeout_seconds: 10.0, + ready_timeout_seconds: 60.0, + ready_interval_seconds: 2.0, + host: DEFAULT_HOST.to_string(), + profile: "bad".to_string(), + }; + assert!(profile_urls(&config).is_none()); + } +} diff --git a/adk-rust/crates/worktime-ui-bridge/Cargo.toml b/adk-rust/crates/worktime-ui-bridge/Cargo.toml new file mode 100644 index 0000000..bca0586 --- /dev/null +++ b/adk-rust/crates/worktime-ui-bridge/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "worktime-ui-bridge" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/worktime-ui-bridge/src/main.rs b/adk-rust/crates/worktime-ui-bridge/src/main.rs new file mode 100644 index 0000000..f4cd608 --- /dev/null +++ b/adk-rust/crates/worktime-ui-bridge/src/main.rs @@ -0,0 +1,997 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, + thread, + time::Duration, +}; + +use anyhow::{Context, Result, anyhow}; +use chrono::{DateTime, SecondsFormat, TimeDelta, Utc}; +use clap::Parser; +use reqwest::header::{CONNECTION, HeaderMap, HeaderValue}; +use reqwest::{Method, StatusCode, blocking::Client}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const DEFAULT_AW_URL: &str = "http://127.0.0.1:5600"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const DEFAULT_STATE_PATH: &str = "/var/lib/activitywatch/aw-worktime-ui-bridge-state.json"; +const BRIDGE_SOURCE: &str = "aw-worktime-ui-bridge"; + +#[derive(Debug, Parser)] +#[command(about = "AW Worktime UI bridge (sessions -> afk/window)")] +struct Cli { + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone)] +struct Config { + aw_url: String, + host: String, + state_path: PathBuf, + timeout_seconds: f64, + watcher_fallback_enabled: bool, + watcher_fallback_stale_seconds: f64, + collector_health_max_age_seconds: f64, + collector_health_query_limit: usize, + foreground_context_cache_seconds: f64, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct AwEvent { + timestamp: Option, + #[serde(default)] + duration: Option, + #[serde(default)] + data: Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct State { + last_ts: String, + #[serde(skip_serializing_if = "Option::is_none")] + last_foreground_context: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct ForegroundContext { + app: String, + title: String, + timestamp: Option, +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + dry_run: bool, + host: String, + input_events: usize, + posted_afk: usize, + posted_win: usize, + watcher_afk_posted: usize, + watcher_win_posted: usize, + last_ts: Option, + state_saved: bool, +} + +struct AwClient { + base_url: String, + client: Client, +} + +fn env(name: &str, fallback: &str) -> String { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| fallback.to_string()) +} + +fn env_bool(name: &str, fallback: bool) -> bool { + match std::env::var(name) + .ok() + .map(|value| value.trim().to_ascii_lowercase()) + { + Some(value) if !value.is_empty() => matches!(value.as_str(), "1" | "true" | "yes" | "on"), + _ => fallback, + } +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn env_usize(name: &str, fallback: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(fallback) +} + +fn load_config() -> Config { + Config { + aw_url: env("AW_SERVER_URL", DEFAULT_AW_URL) + .trim_end_matches('/') + .to_string(), + host: env("AW_WORKTIME_HOST", DEFAULT_HOST), + state_path: PathBuf::from(env("AW_WORKTIME_UI_BRIDGE_STATE", DEFAULT_STATE_PATH)), + timeout_seconds: env_f64("AW_WORKTIME_UI_BRIDGE_TIMEOUT", 60.0), + watcher_fallback_enabled: env_bool("AW_WORKTIME_UI_BRIDGE_WATCHER_FALLBACK", true), + watcher_fallback_stale_seconds: env_f64( + "AW_WORKTIME_UI_BRIDGE_WATCHER_STALE_SECONDS", + 600.0, + ), + collector_health_max_age_seconds: env_f64( + "AW_WORKTIME_UI_BRIDGE_COLLECTOR_HEALTH_MAX_AGE_SECONDS", + 300.0, + ), + collector_health_query_limit: env_usize( + "AW_WORKTIME_UI_BRIDGE_COLLECTOR_HEALTH_QUERY_LIMIT", + 200, + ), + foreground_context_cache_seconds: env_f64( + "AW_WORKTIME_UI_BRIDGE_FOREGROUND_CACHE_SECONDS", + 900.0, + ), + } +} + +impl Config { + fn sessions_bucket(&self) -> String { + format!("aw-worktime-sessions_{}", self.host) + } + fn afk_bucket(&self) -> String { + format!("aw-rdp-afk_{}", self.host) + } + fn window_bucket(&self) -> String { + format!("aw-rdp-window_{}", self.host) + } + fn watcher_afk_bucket(&self) -> String { + format!("aw-watcher-afk_{}", self.host) + } + fn watcher_window_bucket(&self) -> String { + format!("aw-watcher-window_{}", self.host) + } + fn web_category_bucket(&self) -> String { + format!("aw-detmir-web-category_{}", self.host) + } +} + +impl AwClient { + fn new(config: &Config) -> Result { + let mut headers = HeaderMap::new(); + headers.insert(CONNECTION, HeaderValue::from_static("close")); + let client = Client::builder() + .timeout(Duration::from_secs_f64(config.timeout_seconds.max(0.001))) + .no_proxy() + .pool_max_idle_per_host(0) + .default_headers(headers) + .build() + .context("build HTTP client")?; + Ok(Self { + base_url: config.aw_url.clone(), + client, + }) + } + + fn url(&self, path: &str) -> String { + format!("{}{}", self.base_url, path) + } + + fn request_json( + &self, + method: Method, + path: &str, + payload: Option, + ) -> Result> { + let mut last_error = None; + let mut response = None; + for attempt in 1..=3 { + let mut request = self.client.request(method.clone(), self.url(path)); + if let Some(payload) = payload.clone() { + request = request.json(&payload); + } + match request.send() { + Ok(ok_response) => { + response = Some(ok_response); + break; + } + Err(error) => { + last_error = Some(error); + if attempt < 3 { + eprintln!("warn request={path} retry_after_send_error"); + thread::sleep(Duration::from_millis(750)); + } + } + } + } + let response = match response { + Some(response) => response, + None => { + let error = last_error.expect("error exists after failed request"); + return Err(error).with_context(|| format!("request ActivityWatch {path}")); + } + }; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + return Err(anyhow!("not found: {path}")); + } + if !status.is_success() { + return Err(anyhow!("ActivityWatch {path} returned HTTP {status}")); + } + let bytes = response.bytes().context("read ActivityWatch response")?; + if bytes.is_empty() { + Ok(None) + } else { + serde_json::from_slice(&bytes) + .map(Some) + .with_context(|| format!("decode ActivityWatch JSON from {path}")) + } + } + + fn get_events(&self, bucket_id: &str, limit: usize) -> Result> { + let path = format!("/api/0/buckets/{bucket_id}/events?limit={limit}"); + match self.request_json(Method::GET, &path, None) { + Ok(Some(value)) => serde_json::from_value(value).context("decode AW events"), + Ok(None) => Ok(Vec::new()), + Err(error) if error.to_string().starts_with("not found:") => Ok(Vec::new()), + Err(error) => Err(error), + } + } + + fn get_latest_bucket_event(&self, bucket_id: &str) -> Result> { + Ok(self.get_events(bucket_id, 1)?.into_iter().next()) + } + + fn ensure_bucket( + &self, + bucket_id: &str, + event_type: &str, + client_name: &str, + host: &str, + ) -> Result<()> { + let payload = json!({ + "client": client_name, + "type": event_type, + "hostname": host, + }); + let path = format!("/api/0/buckets/{bucket_id}"); + let response = match self.client.post(self.url(&path)).json(&payload).send() { + Ok(response) => response, + Err(error) => { + eprintln!("warn ensure_bucket={bucket_id} skipped: {error}"); + return Ok(()); + } + }; + let status = response.status(); + if status.is_success() + || status == StatusCode::NOT_MODIFIED + || status == StatusCode::CONFLICT + { + Ok(()) + } else { + eprintln!("warn ensure_bucket={bucket_id} returned HTTP {status}; continuing"); + Ok(()) + } + } + + fn post_events(&self, bucket_id: &str, events: &[AwEvent]) -> Result<()> { + let path = format!("/api/0/buckets/{bucket_id}/events"); + self.request_json(Method::POST, &path, Some(json!(events))) + .map(|_| ()) + } +} + +fn load_state(path: &Path) -> State { + let fallback = State { + last_ts: "1970-01-01T00:00:00Z".to_string(), + last_foreground_context: None, + }; + let Ok(raw) = fs::read_to_string(path) else { + return fallback; + }; + let Ok(state) = serde_json::from_str::(&raw) else { + return fallback; + }; + if state.last_ts.trim().is_empty() { + fallback + } else { + state + } +} + +fn save_state(path: &Path, state: &State) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + let tmp = PathBuf::from(format!("{}.tmp", path.display())); + let raw = serde_json::to_vec(state).context("encode state")?; + fs::write(&tmp, raw).with_context(|| format!("write {}", tmp.display()))?; + fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?; + Ok(()) +} + +fn parse_iso_utc(ts: &str) -> Result> { + DateTime::parse_from_rfc3339(ts) + .map(|parsed| parsed.with_timezone(&Utc)) + .with_context(|| format!("parse timestamp {ts}")) +} + +fn to_iso_utc(dt: DateTime) -> String { + dt.to_rfc3339_opts(SecondsFormat::Secs, true) +} + +fn normalize_event_ts(ts: &str) -> String { + parse_iso_utc(ts) + .map(|dt| to_iso_utc(dt - TimeDelta::nanoseconds(dt.timestamp_subsec_nanos().into()))) + .unwrap_or_else(|_| ts.to_string()) +} + +fn value_string(data: &Value, key: &str) -> String { + match data.get(key) { + Some(Value::String(value)) => value.trim().to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + Some(Value::Null) | None => String::new(), + Some(value) => value.to_string().trim_matches('"').trim().to_string(), + } +} + +fn value_i64(data: &Value, key: &str) -> Option { + match data.get(key) { + Some(Value::Number(value)) => value.as_i64(), + Some(Value::String(value)) => value.trim().parse().ok(), + _ => None, + } +} + +fn is_session_active(data: &Value) -> bool { + if data.get("active").and_then(Value::as_bool).unwrap_or(false) { + return true; + } + let state = value_string(data, "state").to_lowercase(); + if state == "active" || state == "активно" { + return true; + } + if state == "unknown" { + let sid = value_i64(data, "sessionId").unwrap_or(-1); + let user = value_string(data, "username").to_lowercase(); + let session_name = value_string(data, "sessionName").to_lowercase(); + return sid > 0 + && !user.is_empty() + && !user.ends_with('$') + && (session_name.starts_with("rdp-") || session_name == "console"); + } + false +} + +fn build_window_title(users: &[String], active_count: usize) -> String { + if users.is_empty() { + "RDP idle".to_string() + } else { + format!("RDP active ({active_count}): {}", users.join(", ")) + } +} + +fn get_latest_active_session_ids(events: &[AwEvent]) -> BTreeSet { + let mut grouped: BTreeMap, Vec<&AwEvent>> = BTreeMap::new(); + for event in events { + let Some(ts) = event.timestamp.as_deref() else { + continue; + }; + let Ok(parsed) = parse_iso_utc(ts) else { + continue; + }; + grouped.entry(parsed).or_default().push(event); + } + let Some((_, latest_events)) = grouped.iter().next_back() else { + return BTreeSet::new(); + }; + latest_events + .iter() + .filter(|event| is_session_active(&event.data)) + .filter_map(|event| value_i64(&event.data, "sessionId")) + .collect() +} + +fn normalize_foreground_context(data: &Value) -> Option { + let foreground_process = value_string(data, "foregroundProcess"); + let foreground_title = value_string(data, "foregroundTitle"); + if foreground_process.is_empty() && foreground_title.is_empty() { + return None; + } + let app = if foreground_process.ends_with(".exe") { + foreground_process.clone() + } else { + format!("{foreground_process}.exe") + }; + Some(ForegroundContext { + app, + title: if foreground_title.is_empty() { + foreground_process + } else { + foreground_title + }, + timestamp: None, + }) +} + +fn get_latest_foreground_context( + aw: &AwClient, + config: &Config, + now_utc: DateTime, + active_session_ids: &BTreeSet, + state: &State, +) -> Result> { + let mut recent_candidates: Vec<(Option, DateTime, ForegroundContext)> = Vec::new(); + for event in aw + .get_events( + &config.web_category_bucket(), + config.collector_health_query_limit, + )? + .into_iter() + .rev() + { + let data = &event.data; + if value_string(data, "signalType").to_lowercase() != "collector_health" { + continue; + } + let Some(ts) = event.timestamp.as_deref() else { + continue; + }; + let Ok(event_dt) = parse_iso_utc(ts) else { + continue; + }; + if (now_utc - event_dt).num_milliseconds() as f64 / 1000.0 + > config.collector_health_max_age_seconds + { + continue; + } + let Some(normalized) = normalize_foreground_context(data) else { + continue; + }; + recent_candidates.push((value_i64(data, "sessionId"), event_dt, normalized)); + } + + for (session_id, event_dt, mut normalized) in recent_candidates.iter().cloned() { + if !active_session_ids.is_empty() + && session_id.is_some_and(|sid| active_session_ids.contains(&sid)) + { + normalized.timestamp = Some(to_iso_utc(event_dt)); + return Ok(Some(normalized)); + } + } + + if let Some((_, event_dt, mut normalized)) = recent_candidates.into_iter().next() { + normalized.timestamp = Some(to_iso_utc(event_dt)); + return Ok(Some(normalized)); + } + + if let Some(cached) = &state.last_foreground_context { + if let Some(cached_ts) = cached + .timestamp + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + if let Ok(cached_dt) = parse_iso_utc(cached_ts) { + let age = (now_utc - cached_dt).num_milliseconds() as f64 / 1000.0; + if age <= config.foreground_context_cache_seconds + && (!cached.app.trim().is_empty() || !cached.title.trim().is_empty()) + { + return Ok(Some(ForegroundContext { + app: if cached.app.trim().is_empty() { + "RDP".to_string() + } else { + cached.app.trim().to_string() + }, + title: if cached.title.trim().is_empty() { + cached.app.trim().to_string() + } else { + cached.title.trim().to_string() + }, + timestamp: Some(cached_ts.to_string()), + })); + } + } + } + } + Ok(None) +} + +fn transform( + events: &[AwEvent], + foreground_context: Option<&ForegroundContext>, +) -> (Vec, Vec, Option) { + let mut out_afk = Vec::new(); + let mut out_win = Vec::new(); + let mut last_ts = None; + let mut grouped: BTreeMap> = BTreeMap::new(); + for event in events { + let Some(ts) = event.timestamp.as_deref() else { + continue; + }; + grouped + .entry(normalize_event_ts(ts)) + .or_default() + .push(event); + } + let ordered_ts: Vec = grouped.keys().cloned().collect(); + let parsed_ts: BTreeMap>> = ordered_ts + .iter() + .map(|ts| (ts.clone(), parse_iso_utc(ts).ok())) + .collect(); + + for (idx, ts) in ordered_ts.iter().enumerate() { + let rows = &grouped[ts]; + let src_duration = rows + .iter() + .map(|event| event.duration.unwrap_or(0.0)) + .fold(0.0, f64::max); + let mut duration = src_duration; + let cur_dt = parsed_ts.get(ts).and_then(|dt| *dt); + let next_dt = ordered_ts + .get(idx + 1) + .and_then(|next_ts| parsed_ts.get(next_ts)) + .and_then(|dt| *dt); + let next_gap = match (cur_dt, next_dt) { + (Some(cur_dt), Some(next_dt)) => { + Some(((next_dt - cur_dt).num_milliseconds() as f64 / 1000.0).max(0.0)) + } + _ => None, + }; + if duration <= 0.0 { + if let Some(next_gap) = next_gap { + duration = next_gap; + } + if duration <= 0.0 { + duration = 10.0; + } + } else if let Some(next_gap) = next_gap.filter(|value| *value > 0.0) { + duration = duration.min(next_gap); + } + duration = duration.min(30.0); + + let mut active_users: BTreeSet = BTreeSet::new(); + for row in rows { + let user = value_string(&row.data, "username"); + if !user.is_empty() && is_session_active(&row.data) { + active_users.insert(user); + } + } + let active_users: Vec = active_users.into_iter().collect(); + let active_count = active_users.len(); + let is_active = active_count > 0; + + out_afk.push(AwEvent { + timestamp: Some(ts.clone()), + duration: Some(duration), + data: json!({ + "status": if is_active { "not-afk" } else { "afk" }, + "source": BRIDGE_SOURCE, + }), + }); + + let win_data = if is_active { + if let Some(context) = foreground_context { + let mut title = context.title.trim().to_string(); + if active_count > 1 { + let rdp_title = build_window_title(&active_users, active_count); + title = if title.is_empty() { + rdp_title + } else { + format!("{title} | {rdp_title}") + }; + } + json!({ + "app": if context.app.trim().is_empty() { "RDP" } else { context.app.trim() }, + "title": if title.is_empty() { build_window_title(&active_users, active_count) } else { title }, + "source": BRIDGE_SOURCE, + }) + } else { + json!({ + "app": "RDP", + "title": build_window_title(&active_users, active_count), + "source": BRIDGE_SOURCE, + }) + } + } else { + json!({ + "app": "RDP", + "title": build_window_title(&active_users, active_count), + "source": BRIDGE_SOURCE, + }) + }; + out_win.push(AwEvent { + timestamp: Some(ts.clone()), + duration: Some(duration), + data: win_data, + }); + last_ts = Some(ts.clone()); + } + (out_afk, out_win, last_ts) +} + +fn normalize_watcher_window_events(win_events: &[AwEvent]) -> Vec { + let mut normalized = Vec::new(); + for event in win_events { + let app = value_string(&event.data, "app"); + if app.is_empty() || app.to_uppercase() == "RDP" { + continue; + } + let mut data = event.data.clone(); + let title = value_string(&data, "title"); + if let Some((prefix, _)) = title.split_once(" | RDP active (") { + if let Some(map) = data.as_object_mut() { + map.insert( + "title".to_string(), + Value::String(prefix.trim().to_string()), + ); + } + } + let mut cloned = event.clone(); + cloned.data = data; + normalized.push(cloned); + } + normalized +} + +fn get_latest_bucket_event_ts(aw: &AwClient, bucket_id: &str) -> Result>> { + let Some(event) = aw.get_latest_bucket_event(bucket_id)? else { + return Ok(None); + }; + let Some(ts) = event.timestamp.as_deref() else { + return Ok(None); + }; + Ok(parse_iso_utc(ts).ok()) +} + +fn bucket_needs_fallback( + aw: &AwClient, + bucket_id: &str, + now_utc: DateTime, + stale_after_seconds: f64, +) -> Result { + let Some(latest_dt) = get_latest_bucket_event_ts(aw, bucket_id)? else { + return Ok(true); + }; + Ok((now_utc - latest_dt).num_milliseconds() as f64 / 1000.0 >= stale_after_seconds) +} + +fn watcher_window_needs_bridge_sync( + aw: &AwClient, + config: &Config, + now_utc: DateTime, +) -> Result { + let bucket_id = config.watcher_window_bucket(); + let Some(latest_event) = aw.get_latest_bucket_event(&bucket_id)? else { + return Ok(true); + }; + let Some(latest_dt) = get_latest_bucket_event_ts(aw, &bucket_id)? else { + return Ok(true); + }; + if (now_utc - latest_dt).num_milliseconds() as f64 / 1000.0 + >= config.watcher_fallback_stale_seconds + { + return Ok(true); + } + + let source = value_string(&latest_event.data, "source").to_lowercase(); + Ok(source == BRIDGE_SOURCE) +} + +fn run(cli: &Cli) -> Result { + let config = load_config(); + let aw = AwClient::new(&config)?; + let state = load_state(&config.state_path); + let last_dt = parse_iso_utc(&state.last_ts) + .unwrap_or_else(|_| DateTime::from_timestamp(0, 0).expect("valid unix epoch")); + + if !cli.dry_run { + aw.ensure_bucket( + &config.afk_bucket(), + "afkstatus", + BRIDGE_SOURCE, + &config.host, + )?; + aw.ensure_bucket( + &config.window_bucket(), + "currentwindow", + BRIDGE_SOURCE, + &config.host, + )?; + } + + let now_utc = Utc::now(); + let recent = aw.get_events(&config.sessions_bucket(), 5000)?; + if recent.is_empty() { + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + input_events: 0, + posted_afk: 0, + posted_win: 0, + watcher_afk_posted: 0, + watcher_win_posted: 0, + last_ts: None, + state_saved: false, + }); + } + + let events: Vec = recent + .into_iter() + .filter(|event| { + event + .timestamp + .as_deref() + .and_then(|ts| parse_iso_utc(ts).ok()) + .is_some_and(|ts| ts > last_dt) + }) + .collect(); + if events.is_empty() { + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + input_events: 0, + posted_afk: 0, + posted_win: 0, + watcher_afk_posted: 0, + watcher_win_posted: 0, + last_ts: None, + state_saved: false, + }); + } + + let active_session_ids = get_latest_active_session_ids(&events); + let foreground_context = + get_latest_foreground_context(&aw, &config, now_utc, &active_session_ids, &state)?; + let (afk_events, win_events, new_last_ts) = transform(&events, foreground_context.as_ref()); + let Some(new_last_ts) = new_last_ts else { + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + input_events: events.len(), + posted_afk: 0, + posted_win: 0, + watcher_afk_posted: 0, + watcher_win_posted: 0, + last_ts: None, + state_saved: false, + }); + }; + if afk_events.is_empty() || win_events.is_empty() { + return Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + input_events: events.len(), + posted_afk: 0, + posted_win: 0, + watcher_afk_posted: 0, + watcher_win_posted: 0, + last_ts: Some(new_last_ts), + state_saved: false, + }); + } + + let watcher_win_events = normalize_watcher_window_events(&win_events); + let mut watcher_afk_posted = 0; + let mut watcher_win_posted = 0; + + if !cli.dry_run { + aw.post_events(&config.afk_bucket(), &afk_events)?; + aw.post_events(&config.window_bucket(), &win_events)?; + } + + if config.watcher_fallback_enabled { + if bucket_needs_fallback( + &aw, + &config.watcher_afk_bucket(), + now_utc, + config.watcher_fallback_stale_seconds, + )? { + watcher_afk_posted = afk_events.len(); + if !cli.dry_run { + aw.ensure_bucket( + &config.watcher_afk_bucket(), + "afkstatus", + "aw-watcher-afk", + &config.host, + )?; + aw.post_events(&config.watcher_afk_bucket(), &afk_events)?; + } + } + if !watcher_win_events.is_empty() + && watcher_window_needs_bridge_sync(&aw, &config, now_utc)? + { + watcher_win_posted = watcher_win_events.len(); + if !cli.dry_run { + aw.ensure_bucket( + &config.watcher_window_bucket(), + "currentwindow", + "aw-watcher-window", + &config.host, + )?; + aw.post_events(&config.watcher_window_bucket(), &watcher_win_events)?; + } + } + } + + let next_context = foreground_context.or(state.last_foreground_context); + let mut next_state = State { + last_ts: new_last_ts.clone(), + last_foreground_context: None, + }; + if let Some(context) = next_context { + next_state.last_foreground_context = Some(ForegroundContext { + app: context.app, + title: context.title, + timestamp: context.timestamp.or_else(|| Some(to_iso_utc(now_utc))), + }); + } + if !cli.dry_run { + save_state(&config.state_path, &next_state)?; + } + + Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + host: config.host, + input_events: events.len(), + posted_afk: afk_events.len(), + posted_win: win_events.len(), + watcher_afk_posted, + watcher_win_posted, + last_ts: Some(new_last_ts), + state_saved: !cli.dry_run, + }) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let summary = run(&cli)?; + if cli.json { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else if summary.posted_afk > 0 || summary.posted_win > 0 { + println!( + "posted_afk={} posted_win={} last_ts={}", + summary.posted_afk, + summary.posted_win, + summary.last_ts.as_deref().unwrap_or("") + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(ts: &str, duration: f64, data: Value) -> AwEvent { + AwEvent { + timestamp: Some(ts.to_string()), + duration: Some(duration), + data, + } + } + + #[test] + fn latest_active_session_ids_uses_latest_timestamp_group() { + let events = vec![ + event( + "2026-05-27T07:59:25Z", + 0.0, + json!({"sessionId": 2, "state": "Активно", "username": "администратор", "sessionName": "console"}), + ), + event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"sessionId": 2, "state": "Активно", "username": "администратор", "sessionName": "console"}), + ), + event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"sessionId": 3, "state": "Активно", "username": "user5", "sessionName": "rdp-tcp#0"}), + ), + event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"sessionId": 4, "state": "Диск", "username": "user1", "sessionName": ""}), + ), + ]; + assert_eq!( + get_latest_active_session_ids(&events), + BTreeSet::from([2, 3]) + ); + } + + #[test] + fn transform_uses_foreground_context_for_active_sessions() { + let events = vec![event( + "2026-05-27T07:59:30Z", + 0.0, + json!({"username": "user5", "state": "Активно", "sessionId": 3, "sessionName": "rdp-tcp#0"}), + )]; + let ctx = ForegroundContext { + app: "totalcmd.exe".to_string(), + title: "Total Commander 6.01 - HARVEST".to_string(), + timestamp: None, + }; + let (afk_events, win_events, last_ts) = transform(&events, Some(&ctx)); + assert_eq!(value_string(&afk_events[0].data, "status"), "not-afk"); + assert_eq!(value_string(&win_events[0].data, "app"), "totalcmd.exe"); + assert_eq!( + value_string(&win_events[0].data, "title"), + "Total Commander 6.01 - HARVEST" + ); + assert_eq!(last_ts.as_deref(), Some("2026-05-27T07:59:30Z")); + } + + #[test] + fn transform_caps_duration_at_next_timestamp_gap() { + let events = vec![ + event( + "2026-05-27T07:59:30Z", + 5.0, + json!({"username": "user5", "state": "Активно", "sessionId": 3, "sessionName": "rdp-tcp#0"}), + ), + event( + "2026-05-27T07:59:31Z", + 5.0, + json!({"username": "user5", "state": "Активно", "sessionId": 3, "sessionName": "rdp-tcp#0"}), + ), + ]; + let ctx = ForegroundContext { + app: "totalcmd.exe".to_string(), + title: "Total Commander 6.01 - HARVEST".to_string(), + timestamp: None, + }; + let (_, win_events, _) = transform(&events, Some(&ctx)); + assert_eq!(win_events[0].duration, Some(1.0)); + assert_eq!(win_events[1].duration, Some(5.0)); + } + + #[test] + fn normalize_watcher_window_events_strips_rdp_suffix_for_real_apps() { + let events = vec![ + event( + "2026-05-27T08:10:00Z", + 5.0, + json!({"app": "totalcmd.exe", "title": "Total Commander 6.01 - HARVEST | RDP active (2): user5, администратор", "source": BRIDGE_SOURCE}), + ), + event( + "2026-05-27T08:10:05Z", + 5.0, + json!({"app": "RDP", "title": "RDP active (2): user5, администратор", "source": BRIDGE_SOURCE}), + ), + ]; + let normalized = normalize_watcher_window_events(&events); + assert_eq!( + value_string(&normalized[0].data, "title"), + "Total Commander 6.01 - HARVEST" + ); + assert_eq!(normalized.len(), 1); + } + + #[test] + fn unknown_rdp_session_counts_as_active() { + let data = json!({"sessionId": 5, "state": "Unknown", "username": "user5", "sessionName": "rdp-tcp#0"}); + assert!(is_session_active(&data)); + } + + #[test] + fn machine_unknown_session_is_not_active() { + let data = json!({"sessionId": 5, "state": "Unknown", "username": "HOST$", "sessionName": "rdp-tcp#0"}); + assert!(!is_session_active(&data)); + } +} diff --git a/ansible/README.md b/ansible/README.md index 7a1acbb..d251432 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -122,8 +122,9 @@ Playbook: - выгружает полный `windows/*` toolkit на целевой хост в InnoSetup-compatible каталог `C:\Program Files\AWatch-rus\windows`, включая DLP и `worktime-session-collector.ps1`; - если найден legacy config `C:\ProgramData\ActivityWatch-Phase2\deployment-config.json`, выполняет безопасную миграцию через `migrate-awatch-rus-paths.ps1`: backup, остановка задач, перенос данных, переписывание путей, пересоздание scheduled tasks и validation; - выполняет `deploy-ensemble.ps1` (deploy + hardening/recovery) с policy/rules из AWatch-rus toolkit; -- после deploy принудительно запускает `ActivityWatch Recovery` и все `ActivityWatch Launch *` задачи; -- включает (`Enable-ScheduledTask`) `ActivityWatch Recovery` и все `ActivityWatch Launch *` задачи перед запуском (иначе WebUI может показывать `Active time: 0s`); +- после deploy принудительно запускает `ActivityWatch Recovery` и managed `ActivityWatch Launch *` задачи; +- включает (`Enable-ScheduledTask`) `ActivityWatch Recovery` и managed `ActivityWatch Launch *` задачи перед запуском (иначе WebUI может показывать `Active time: 0s`); +- оставляет `ActivityWatch Recovery` включённым даже при активном `AWatchRusCollectorGuard`: guard является основным контроллером, recovery остаётся fallback/bootstrap path; - выполняет API smoke-check bucket `aw-watcher-afk_` и ожидает свежие события; - выполняет API smoke-check bucket `aw-watcher-window_` и ожидает свежие события (по умолчанию включено); - запускает `validate-deployment.ps1`; diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index 371dac5..4cab6f9 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -14,6 +14,7 @@ aw_sw_cleanup_cache_bust: "{{ lookup('file', aw_repo_root + '/aw-server/aw-sw-cleanup.js') | hash('sha1') | truncate(12, true, '') }}" aw_worktime_classes: "{{ lookup('file', aw_repo_root + '/aw-server/settings/classes-worktime.json') | from_json }}" aw_default_views: "{{ lookup('file', aw_repo_root + '/aw-server/settings/views-default.json') | from_json }}" + aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}" tasks: - name: Установить базовые пакеты @@ -29,6 +30,15 @@ state: present update_cache: true + - name: Установить пакеты для browser smoke проверки + ansible.builtin.apt: + name: + - chromium + - nodejs + - node-playwright + state: present + when: aw_browser_smoke_enabled | default(true) | bool + - name: Создать системную группу сервиса ansible.builtin.group: name: "{{ aw_server_group }}" @@ -63,10 +73,13 @@ - "{{ aw_server_data_dir }}/.config/activitywatch" - "{{ aw_server_data_dir }}/.config/activitywatch/aw-server-rust" - "{{ aw_server_data_dir }}/backups" + - "{{ aw_server_data_dir }}/slo" + - "{{ aw_server_data_dir }}/browser-smoke" - "{{ aw_rus_health_state_dir }}" - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" - /etc/activitywatch + - /usr/local/lib/aw-rus-browser-smoke - "{{ aw_bootstrap_dir }}" - name: Настроить каталоги ActivityWatch с владельцем сервиса @@ -90,6 +103,8 @@ - "{{ aw_server_data_dir }}/.config/activitywatch" - "{{ aw_server_data_dir }}/.config/activitywatch/aw-server-rust" - "{{ aw_server_data_dir }}/backups" + - "{{ aw_server_data_dir }}/slo" + - "{{ aw_server_data_dir }}/browser-smoke" - "{{ aw_rus_health_state_dir }}" - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" @@ -102,6 +117,30 @@ group: root mode: "0755" + - name: Положить legacy prune script в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/legacy/aw-prune-local-state.sh" + dest: /opt/activitywatch/aw-rus-ops/aw-prune-local-state.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust prune local state + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-prune-local-state" + delegate_to: localhost + register: aw_prune_local_state_rust_binary + become: false + + - name: Установить Rust prune local state + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-prune-local-state" + dest: /usr/local/bin/aw-prune-local-state-rust + owner: root + group: root + mode: "0755" + when: aw_prune_local_state_rust_binary.stat.exists | default(false) + - name: Ограничить рост journald на aw-server ansible.builtin.copy: dest: /etc/systemd/journald.conf.d/aw-rus-retention.conf @@ -376,20 +415,138 @@ dest: /opt/activitywatch/aw-server/apply_webui_ru_patch.sh mode: "0755" + - name: Создать server-side ops bundle для ранних health helpers + ansible.builtin.file: + path: /opt/activitywatch/aw-rus-ops + state: directory + owner: root + group: root + mode: "0755" + + - name: Установить wrapper aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-health-check-wrapper.sh" + dest: /usr/local/bin/aw-health-check + owner: root + group: root + mode: "0755" + + - name: Положить legacy aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/health-check.sh" + dest: /opt/activitywatch/aw-rus-ops/health-check.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust aw-health-check до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-health-check" + delegate_to: localhost + register: aw_health_check_rust_binary_early + become: false + + - name: Установить Rust aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-health-check" + dest: /usr/local/bin/aw-health-check-rust + owner: root + group: root + mode: "0755" + when: aw_health_check_rust_binary_early.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-health до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-health" + delegate_to: localhost + register: aw_db_health_rust_binary_early + become: false + + - name: Установить Rust aw-db-health до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-health" + dest: /usr/local/bin/aw-db-health + owner: root + group: root + mode: "0755" + when: aw_db_health_rust_binary_early.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-maintenance до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-maintenance" + delegate_to: localhost + register: aw_db_maintenance_rust_binary_early + become: false + + - name: Установить Rust aw-db-maintenance до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-maintenance" + dest: /usr/local/bin/aw-db-maintenance + owner: root + group: root + mode: "0755" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Установить aw-db-maintenance service до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-db-maintenance.service" + dest: /etc/systemd/system/aw-db-maintenance.service + owner: root + group: root + mode: "0644" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Установить aw-db-maintenance timer до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-db-maintenance.timer" + dest: /etc/systemd/system/aw-db-maintenance.timer + owner: root + group: root + mode: "0644" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Включить weekly aw-db-maintenance timer до Influx проверок + ansible.builtin.systemd: + name: aw-db-maintenance.timer + enabled: true + state: started + daemon_reload: true + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Прочитать текущий aw-server.env для сохранения Influx token + ansible.builtin.slurp: + path: /etc/activitywatch/aw-server.env + register: aw_existing_server_env_raw + failed_when: false + no_log: true + + - name: Подготовить effective Influx tokens без вывода секретов + ansible.builtin.set_fact: + aw_existing_server_env_text: "{{ aw_existing_server_env_raw.content | default('') | b64decode }}" + aw_existing_worktime_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_WORKTIME_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_WORKTIME_INFLUX_TOKEN=', '')) }}" + aw_existing_dlp_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_DLP_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_DLP_INFLUX_TOKEN=', '')) }}" + no_log: true + + - name: Выбрать effective Influx tokens + ansible.builtin.set_fact: + aw_effective_worktime_influx_token: "{{ (aw_worktime_influx_token | default('') | string) if ((aw_worktime_influx_token | default('') | string | length) > 0) else aw_existing_worktime_influx_token }}" + aw_effective_dlp_influx_token: "{{ (aw_dlp_influx_token | default('') | string) if ((aw_dlp_influx_token | default('') | string | length) > 0) else aw_existing_dlp_influx_token }}" + no_log: true + - name: Проверить Influx token для AW worktime exporter ansible.builtin.assert: that: - - aw_worktime_influx_token is defined - - aw_worktime_influx_token | length > 0 - fail_msg: "aw_worktime_influx_enabled=true, но aw_worktime_influx_token пуст. Exporter будет падать и Grafana не получит worktime-ряды." + - aw_effective_worktime_influx_token is defined + - aw_effective_worktime_influx_token | length > 0 + fail_msg: "aw_worktime_influx_enabled=true, но token пуст и в локальном env, и в текущем /etc/activitywatch/aw-server.env. Exporter будет падать и Grafana не получит worktime-ряды." when: aw_worktime_influx_enabled | default(false) | bool - name: Проверить Influx token для AW DLP exporter ansible.builtin.assert: that: - - aw_dlp_influx_token is defined - - aw_dlp_influx_token | length > 0 - fail_msg: "aw_dlp_influx_enabled=true, но aw_dlp_influx_token пуст. Exporter будет падать и Grafana не получит DLP-ряды." + - aw_effective_dlp_influx_token is defined + - aw_effective_dlp_influx_token | length > 0 + fail_msg: "aw_dlp_influx_enabled=true, но token пуст и в локальном env, и в текущем /etc/activitywatch/aw-server.env. Exporter будет падать и Grafana не получит DLP-ряды." when: aw_dlp_influx_enabled | default(false) | bool - name: Записать /etc/activitywatch/aw-server.env перед хотфиксами @@ -427,7 +584,7 @@ AW_WORKTIME_INFLUX_BUCKET={{ aw_worktime_influx_bucket | default('aw_metrics') }} AW_WORKTIME_INFLUX_HOSTS={{ aw_worktime_influx_hosts | default('SHARKON2025') }} AW_WORKTIME_INFLUX_DAYS={{ aw_worktime_influx_days | default('today,yesterday') }} - AW_WORKTIME_INFLUX_TOKEN={{ aw_worktime_influx_token | default('') }} + AW_WORKTIME_INFLUX_TOKEN={{ aw_effective_worktime_influx_token | default('') }} AW_DLP_INFLUX_ENABLED={{ 'true' if (aw_dlp_influx_enabled | default(false) | bool) else 'false' }} AW_DLP_INFLUX_URL={{ aw_dlp_influx_url | default('') }} AW_DLP_INFLUX_ORG={{ aw_dlp_influx_org | default('proxmox') }} @@ -435,7 +592,7 @@ AW_DLP_INFLUX_HOSTS={{ aw_dlp_influx_hosts | default('SHARKON2025') }} AW_DLP_INFLUX_LOOKBACK_DAYS={{ aw_dlp_influx_lookback_days | default(30) }} AW_DLP_INFLUX_EVENT_LIMIT={{ aw_dlp_influx_event_limit | default(2000) }} - AW_DLP_INFLUX_TOKEN={{ aw_dlp_influx_token | default('') }} + AW_DLP_INFLUX_TOKEN={{ aw_effective_dlp_influx_token | default('') }} AW_DLP_AW_API_BASE=http://127.0.0.1:5600/api/0 AW_DLP_CASE_API_BASE=http://127.0.0.1:5602/api/0/dlp/cases AW_EXPECT_START_OF_DAY={{ aw_worktime_start_of_day | default('') }} @@ -448,6 +605,19 @@ AW_RUS_HEALTH_STATE_DIR={{ aw_rus_health_state_dir }} AW_RUS_HEALTH_VALIDATION_DIR={{ aw_rus_health_validation_dir }} AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS={{ aw_rus_health_session_events_max_age_seconds | default(86400) }} + AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS={{ aw_rus_health_guard_max_age_seconds | default(300) }} + AW_RUS_HEALTH_GUARD_REQUIRED={{ 1 if (aw_rus_health_guard_required | default(true) | bool) else 0 }} + AW_RUS_SLO_STATE_DIR={{ aw_server_data_dir }}/slo + AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600 + AW_RUS_SLO_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }} + AW_RUS_SLO_TARGET_PERCENT={{ aw_rus_slo_target_percent | default('99.97') }} + AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600 + AW_BROWSER_SMOKE_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }} + AW_BROWSER_SMOKE_HOST={{ aw_monitored_windows_hostname }} + AW_BROWSER_SMOKE_OUTPUT_DIR={{ aw_server_data_dir }}/browser-smoke + AW_BROWSER_SMOKE_ENGINE={{ aw_browser_smoke_engine | default('chromium-cli') }} + AW_BROWSER_SMOKE_TIMEOUT_MS={{ aw_browser_smoke_timeout_ms | default(20000) }} + AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS={{ aw_browser_smoke_render_timeout_ms | default(15000) }} AW_HAYABUSA_AUTO_CASE_ENABLED={{ 'true' if (aw_hayabusa_auto_case_enabled | default(true) | bool) else 'false' }} AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY={{ aw_hayabusa_auto_case_min_severity | default('medium') }} AW_HAYABUSA_TELEGRAM_ENABLED={{ 'true' if (aw_hayabusa_telegram_enabled | default(false) | bool) else 'false' }} @@ -501,6 +671,50 @@ mode: "0644" when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Проверить локальный Rust DLP policy engine + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-policy-engine" + delegate_to: localhost + register: aw_dlp_policy_engine_rust_binary + become: false + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Установить Rust DLP policy engine + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-policy-engine" + dest: /usr/local/bin/aw-dlp-policy-engine-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP policy engine + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-policy-engine.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + + - name: Переключить DLP policy engine на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-policy-engine.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-policy-engine-rust + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + - name: Создать каталог DLP content analysis ansible.builtin.file: path: /opt/activitywatch/dlp-content-analysis @@ -528,6 +742,25 @@ mode: "0755" when: aw_dlp_content_analysis_enabled | default(true) | bool + - name: Проверить локальный Rust DLP content analyzer + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-content-analyzer" + delegate_to: localhost + register: dlp_content_analyzer_rust_binary + become: false + when: aw_dlp_content_analysis_enabled | default(true) | bool + + - name: Установить Rust DLP content analyzer + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-content-analyzer" + dest: /usr/local/bin/aw-dlp-content-analyzer-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_content_analysis_enabled | default(true) | bool + - dlp_content_analyzer_rust_binary.stat.exists | default(false) + - name: Создать virtualenv DLP content analysis ansible.builtin.command: cmd: python3 -m venv /opt/activitywatch/dlp-content-analysis/.venv @@ -619,6 +852,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust CEF exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-cef-exporter" + delegate_to: localhost + register: dlp_cef_exporter_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust CEF exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-cef-exporter" + dest: /usr/local/bin/dlp-cef-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог CEF exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-cef-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить CEF exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-cef-exporter-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd unit syslog forwarder ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/syslog-forwarder.service" @@ -637,6 +914,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust syslog forwarder + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" + delegate_to: localhost + register: dlp_syslog_forwarder_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust syslog forwarder + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" + dest: /usr/local/bin/dlp-syslog-forwarder-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог syslog forwarder + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + + - name: Переключить syslog forwarder на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + - name: Установить systemd unit webhook sender ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/webhook-sender.service" @@ -655,6 +976,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust webhook sender + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-webhook-sender" + delegate_to: localhost + register: dlp_webhook_sender_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust webhook sender + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-webhook-sender" + dest: /usr/local/bin/dlp-webhook-sender-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог webhook sender + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-webhook-sender.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + + - name: Переключить webhook sender на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-webhook-sender-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + - name: Создать каталог DLP case management ansible.builtin.file: path: /opt/activitywatch/dlp-case-management @@ -710,6 +1075,50 @@ mode: "0644" when: aw_dlp_case_management_enabled | default(true) | bool + - name: Проверить локальный Rust DLP case management + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-case-management" + delegate_to: localhost + register: aw_dlp_case_management_rust_binary + become: false + when: aw_dlp_case_management_enabled | default(true) | bool + + - name: Установить Rust DLP case management + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-case-management" + dest: /usr/local/bin/aw-dlp-case-management-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP case management + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-case-management.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + + - name: Переключить DLP case management на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-case-management.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-case-management-rust + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + - name: Создать каталоги DLP compliance ansible.builtin.file: path: "{{ item }}" @@ -753,6 +1162,50 @@ virtualenv: /opt/activitywatch/dlp-compliance/.venv when: aw_dlp_compliance_enabled | default(true) | bool + - name: Проверить локальный Rust DLP compliance + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-compliance" + delegate_to: localhost + register: aw_dlp_compliance_rust_binary + become: false + when: aw_dlp_compliance_enabled | default(true) | bool + + - name: Установить Rust DLP compliance + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-compliance" + dest: /usr/local/bin/aw-dlp-compliance-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP compliance report scheduler + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-report-scheduler.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + + - name: Переключить DLP compliance report scheduler на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-report-scheduler.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-compliance-rust + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + - name: Установить dlp-admin-cli ansible.builtin.copy: src: "{{ aw_repo_root }}/scripts/dlp-admin-cli.py" @@ -761,6 +1214,21 @@ group: root mode: "0755" + - name: Проверить локальный Rust dlp-admin-cli + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-admin-cli" + delegate_to: localhost + register: aw_dlp_admin_cli_rust_binary + + - name: Установить Rust dlp-admin-cli + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-admin-cli" + dest: /usr/local/bin/dlp-admin-cli + owner: root + group: root + mode: "0755" + when: aw_dlp_admin_cli_rust_binary.stat.exists + - name: Установить скрипт AW worktime API ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" @@ -769,6 +1237,22 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime API + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-api" + delegate_to: localhost + register: aw_worktime_api_rust_binary + become: false + + - name: Установить Rust AW worktime API + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-api" + dest: /usr/local/bin/aw-worktime-api-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_api_rust_binary.stat.exists | default(false) + - name: Установить systemd unit AW worktime API ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.service" @@ -777,6 +1261,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime API + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-api.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_api_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime API на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-api.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-api-rust + when: aw_worktime_api_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.py" @@ -785,6 +1290,22 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime UI bridge + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-ui-bridge" + delegate_to: localhost + register: aw_worktime_ui_bridge_rust_binary + become: false + + - name: Установить Rust AW worktime UI bridge + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-ui-bridge" + dest: /usr/local/bin/aw-worktime-ui-bridge-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.sh" @@ -793,6 +1314,46 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime autoheal + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-autoheal" + delegate_to: localhost + register: aw_worktime_autoheal_rust_binary + become: false + + - name: Установить Rust AW worktime autoheal + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-autoheal" + dest: /usr/local/bin/aw-worktime-autoheal-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + + - name: Установить скрипт AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.sh" + dest: /usr/local/bin/aw-worktime-prewarm.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust AW worktime prewarm + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-prewarm" + delegate_to: localhost + register: aw_worktime_prewarm_rust_binary + become: false + + - name: Установить Rust AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-prewarm" + dest: /usr/local/bin/aw-worktime-prewarm-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.py" @@ -811,21 +1372,118 @@ mode: "0755" when: aw_dlp_influx_enabled | default(false) | bool + - name: Создать server-side ops bundle для legacy aw-health-check + ansible.builtin.file: + path: /opt/activitywatch/aw-rus-ops + state: directory + owner: root + group: root + mode: "0755" + - name: Установить скрипт aw-health-check ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" + src: "{{ aw_repo_root }}/aw-server/aw-health-check-wrapper.sh" dest: /usr/local/bin/aw-health-check owner: root group: root mode: "0755" - - name: Установить скрипт dlp-health-check + - name: Положить legacy aw-health-check в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/health-check.sh" + dest: /opt/activitywatch/aw-rus-ops/health-check.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust aw-health-check + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-health-check" + delegate_to: localhost + register: aw_health_check_rust_binary + become: false + + - name: Установить Rust aw-health-check + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-health-check" + dest: /usr/local/bin/aw-health-check-rust + owner: root + group: root + mode: "0755" + when: aw_health_check_rust_binary.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-health + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-health" + delegate_to: localhost + register: aw_db_health_rust_binary + become: false + + - name: Установить Rust aw-db-health + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-health" + dest: /usr/local/bin/aw-db-health + owner: root + group: root + mode: "0755" + when: aw_db_health_rust_binary.stat.exists | default(false) + + - name: Установить скрипт check-aw-data + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/check-aw-data-wrapper.sh" + dest: /usr/local/bin/check-aw-data + owner: root + group: root + mode: "0755" + + - name: Положить legacy check-aw-data в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/legacy/check-aw-data.sh" + dest: /opt/activitywatch/aw-rus-ops/check-aw-data.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust check-aw-data + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/check-aw-data" + delegate_to: localhost + register: check_aw_data_rust_binary + become: false + + - name: Установить Rust check-aw-data + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/check-aw-data" + dest: /usr/local/bin/check-aw-data-rust + owner: root + group: root + mode: "0755" + when: check_aw_data_rust_binary.stat.exists | default(false) + + - name: Проверить локальный Rust dlp-health-check + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-health-check" + delegate_to: localhost + register: dlp_health_check_rust_binary + become: false + + - name: Установить Rust dlp-health-check + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-health-check" + dest: /usr/local/bin/dlp-health-check + owner: root + group: root + mode: "0755" + when: dlp_health_check_rust_binary.stat.exists | default(false) + + - name: Установить legacy Python dlp-health-check ansible.builtin.copy: src: "{{ aw_repo_root }}/scripts/dlp-health-check.py" dest: /usr/local/bin/dlp-health-check owner: root group: root mode: "0755" + when: not (dlp_health_check_rust_binary.stat.exists | default(false)) - name: Установить скрипт aw-rus-healthd ansible.builtin.copy: @@ -835,6 +1493,39 @@ group: root mode: "0755" + - name: Установить скрипт aw-slo-monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.py" + dest: /usr/local/bin/aw-slo-monitor.py + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust AW-RUS SLO monitor + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-slo-monitor" + delegate_to: localhost + register: aw_slo_monitor_rust_binary + become: false + + - name: Установить Rust AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-slo-monitor" + dest: /usr/local/bin/aw-slo-monitor-rust + owner: root + group: root + mode: "0755" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Установить browser smoke скрипт + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/aw-webui-browser-smoke.mjs" + dest: /usr/local/lib/aw-rus-browser-smoke/aw-webui-browser-smoke.mjs + owner: root + group: root + mode: "0755" + when: aw_browser_smoke_enabled | default(true) | bool + - name: Установить systemd unit AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.service" @@ -843,6 +1534,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime UI bridge + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-ui-bridge.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime UI bridge на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-ui-bridge.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-ui-bridge-rust + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.timer" @@ -859,6 +1571,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime autoheal + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-autoheal.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime autoheal на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-autoheal.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-autoheal-rust + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.timer" @@ -867,6 +1600,43 @@ group: root mode: "0644" + - name: Установить systemd unit AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.service" + dest: /etc/systemd/system/aw-worktime-prewarm.service + owner: root + group: root + mode: "0644" + + - name: Создать drop-in каталог AW worktime prewarm + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-prewarm.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime prewarm на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-prewarm.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-prewarm-rust + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + + - name: Установить systemd timer AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.timer" + dest: /etc/systemd/system/aw-worktime-prewarm.timer + owner: root + group: root + mode: "0644" + - name: Установить systemd unit AW-RUS healthd ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-rus-healthd.service" @@ -883,6 +1653,58 @@ group: root mode: "0644" + - name: Установить systemd unit AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.service" + dest: /etc/systemd/system/aw-slo-monitor.service + owner: root + group: root + mode: "0644" + + - name: Создать drop-in каталог AW-RUS SLO monitor + ansible.builtin.file: + path: /etc/systemd/system/aw-slo-monitor.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Переключить AW-RUS SLO monitor на Rust + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor-rust-switch.conf" + dest: /etc/systemd/system/aw-slo-monitor.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Установить systemd timer AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.timer" + dest: /etc/systemd/system/aw-slo-monitor.timer + owner: root + group: root + mode: "0644" + + - name: Установить systemd unit AW-RUS browser smoke + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-browser-smoke.service" + dest: /etc/systemd/system/aw-browser-smoke.service + owner: root + group: root + mode: "0644" + when: aw_browser_smoke_enabled | default(true) | bool + + - name: Установить systemd timer AW-RUS browser smoke + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-browser-smoke.timer" + dest: /etc/systemd/system/aw-browser-smoke.timer + owner: root + group: root + mode: "0644" + when: aw_browser_smoke_enabled | default(true) | bool + - name: Установить systemd unit AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.service" @@ -892,6 +1714,50 @@ mode: "0644" when: aw_worktime_influx_enabled | default(false) | bool + - name: Проверить локальный Rust AW worktime Influx exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-influx-exporter" + delegate_to: localhost + register: aw_worktime_influx_exporter_rust_binary + become: false + when: aw_worktime_influx_enabled | default(false) | bool + + - name: Установить Rust AW worktime Influx exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-influx-exporter" + dest: /usr/local/bin/aw-worktime-influx-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог AW worktime Influx exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-influx-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime Influx exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-influx-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-influx-exporter-rust + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.timer" @@ -910,6 +1776,50 @@ mode: "0644" when: aw_dlp_influx_enabled | default(false) | bool + - name: Проверить локальный Rust AW DLP Influx exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-influx-exporter" + delegate_to: localhost + register: aw_dlp_influx_exporter_rust_binary + become: false + when: aw_dlp_influx_enabled | default(false) | bool + + - name: Установить Rust AW DLP Influx exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-influx-exporter" + dest: /usr/local/bin/aw-dlp-influx-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог AW DLP Influx exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-influx-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить AW DLP Influx exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-influx-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-influx-exporter-rust + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.timer" @@ -929,6 +1839,19 @@ enabled: true state: started + - name: Включить таймер AW-RUS SLO monitor + ansible.builtin.systemd: + name: aw-slo-monitor.timer + enabled: true + state: started + + - name: Включить таймер AW-RUS browser smoke + ansible.builtin.systemd: + name: aw-browser-smoke.timer + enabled: true + state: started + when: aw_browser_smoke_enabled | default(true) | bool + - name: Включить и перезапустить DLP policy engine ansible.builtin.systemd: name: aw-dlp-policy-engine.service @@ -1015,6 +1938,18 @@ state: started failed_when: false + - name: Включить и перезапустить AW worktime prewarm timer + ansible.builtin.systemd: + name: aw-worktime-prewarm.timer + enabled: true + state: restarted + + - name: Выполнить разовый прогон AW worktime prewarm + ansible.builtin.systemd: + name: aw-worktime-prewarm.service + state: started + failed_when: false + - name: Включить и перезапустить AW worktime Influx exporter timer ansible.builtin.systemd: name: aw-worktime-influx-exporter.timer @@ -1101,6 +2036,22 @@ group: root mode: "0755" + - name: Check Rust merge AW DB binary artifact + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" + register: aw_merge_db_rust_binary + delegate_to: localhost + become: false + + - name: Install Rust merge AW DB binary + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" + dest: /usr/local/bin/merge-aw-server-dbs + owner: root + group: root + mode: "0755" + when: aw_merge_db_rust_binary.stat.exists | default(false) + - name: Проверить наличие legacy root DB ansible.builtin.stat: path: /root/.local/share/activitywatch/aw-server-rust/sqlite.db @@ -1115,7 +2066,9 @@ ansible.builtin.systemd: name: activitywatch-server.service state: stopped - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Создать backup каталоги server DB ansible.builtin.file: @@ -1124,7 +2077,9 @@ owner: "{{ aw_server_user }}" group: "{{ aw_server_group }}" mode: "0755" - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Backup target DB перед merge ansible.builtin.copy: @@ -1135,6 +2090,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1146,7 +2102,9 @@ owner: "{{ aw_server_user }}" group: "{{ aw_server_group }}" mode: "0644" - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Merge legacy root DB в target DB ansible.builtin.command: @@ -1162,6 +2120,7 @@ register: aw_merge_result failed_when: false when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1169,6 +2128,7 @@ ansible.builtin.debug: msg: "{{ aw_merge_result.stdout | default(aw_merge_result.stderr | default('merge not executed')) }}" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1181,6 +2141,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) - aw_merge_result is defined @@ -1195,6 +2156,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - not (aw_target_db.stat.exists | default(false)) @@ -1306,6 +2268,46 @@ [Install] WantedBy=multi-user.target + - name: Проверить локальный Rust DLP aggregator + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-aggregator" + delegate_to: localhost + register: dlp_aggregator_rust_binary + become: false + + - name: Установить Rust DLP aggregator + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-aggregator" + dest: /usr/local/bin/dlp-aggregator-rust + owner: root + group: root + mode: "0755" + when: dlp_aggregator_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP aggregator + ansible.builtin.file: + path: /etc/systemd/system/activitywatch-dlp-aggregator.service.d + state: directory + owner: root + group: root + mode: "0755" + when: dlp_aggregator_rust_binary.stat.exists | default(false) + + - name: Переключить DLP aggregator на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/activitywatch-dlp-aggregator.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-aggregator-rust \ + --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ + --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ + --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json + when: dlp_aggregator_rust_binary.stat.exists | default(false) + - name: Установить systemd timer для агрегатора ansible.builtin.copy: dest: /etc/systemd/system/activitywatch-dlp-aggregator.timer @@ -1314,9 +2316,10 @@ Description=Run ActivityWatch DLP Aggregator every 5 minutes [Timer] - OnBootSec=1min - OnUnitActiveSec=5min - AccuracySec=1s + OnCalendar=*:3/10:10 + AccuracySec=30s + RandomizedDelaySec=30s + Persistent=false [Install] WantedBy=timers.target @@ -1681,9 +2684,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper link-case для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + delegate_to: localhost + register: aw_hayabusa_link_case_rust_binary + become: false + + - name: Установить Rust helper link-case для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + dest: /usr/local/bin/aw-hayabusa-link-case-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + - name: Установить helper link-case для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-link-case.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-link-case-rust ]]; then + exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py "$@" dest: /usr/local/bin/aw-hayabusa-link-case owner: root group: root @@ -1697,9 +2722,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper from-windows для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + delegate_to: localhost + register: aw_hayabusa_from_windows_rust_binary + become: false + + - name: Установить Rust helper from-windows для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + dest: /usr/local/bin/aw-hayabusa-from-windows-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + - name: Установить helper from-windows для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-from-windows.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-from-windows-rust ]]; then + exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py "$@" dest: /usr/local/bin/aw-hayabusa-from-windows owner: root group: root @@ -1721,9 +2768,31 @@ group: root mode: "0644" + - name: Проверить локальный Rust helper autoprocess для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + delegate_to: localhost + register: aw_hayabusa_autoprocess_rust_binary + become: false + + - name: Установить Rust helper autoprocess для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + dest: /usr/local/bin/aw-hayabusa-autoprocess-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + - name: Установить helper autoprocess для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-autoprocess.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-autoprocess-rust ]]; then + exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py "$@" dest: /usr/local/bin/aw-hayabusa-autoprocess owner: root group: root @@ -1737,9 +2806,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper case-alert для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + delegate_to: localhost + register: aw_hayabusa_case_alert_rust_binary + become: false + + - name: Установить Rust helper case-alert для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + dest: /usr/local/bin/aw-hayabusa-case-alert-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + - name: Установить helper case-alert для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-case-alert.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-case-alert-rust ]]; then + exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py "$@" dest: /usr/local/bin/aw-hayabusa-case-alert owner: root group: root diff --git a/ansible/deploy_aw_windows.yml b/ansible/deploy_aw_windows.yml index eec5eeb..e71a3a0 100644 --- a/ansible/deploy_aw_windows.yml +++ b/ansible/deploy_aw_windows.yml @@ -50,7 +50,7 @@ aw_windows_incident_artifacts_root: "{{ aw_windows_state_root }}\\incident-artifacts" aw_windows_forensics_root: "{{ aw_windows_state_root }}\\forensics\\evtx-exports" aw_windows_logon_marker_enabled: true - aw_windows_process_events_enabled: true + aw_windows_process_events_enabled: false aw_windows_skip_hardening: false aw_windows_rules_path: "{{ aw_windows_deploy_root }}\\windows\\web-category-rules.example.json" aw_windows_policy_path: "{{ aw_windows_deploy_root }}\\windows\\dlp-policy.example.json" @@ -58,6 +58,10 @@ aw_windows_validation_local_dir: "/tmp/aw-rus-validation-{{ lookup('env','USER') | default('ansible', true) }}" aw_windows_launch_task_pattern: "ActivityWatch Launch *" aw_windows_recovery_task_name: "ActivityWatch Recovery" + aw_windows_collector_guard_enabled: true + aw_windows_collector_guard_mode: "enforce" + aw_windows_collector_guard_service_name: "AWatchRusCollectorGuard" + aw_windows_collector_guard_loop_seconds: 60 aw_windows_force_task_restart: true aw_windows_api_smoke_check_enabled: true aw_windows_api_smoke_check_bucket: "" @@ -181,6 +185,9 @@ - deploy-domain-users.ps1 - deploy-ensemble.ps1 - hardening-recovery.ps1 + - AWatchRusCollectorGuardService.cs + - aw-collector-guard.ps1 + - install-collector-guard-service.ps1 - rebuild-worktime-tasks.ps1 - audit-cryptopro.ps1 - validate-deployment.ps1 @@ -371,27 +378,50 @@ Start-Sleep -Seconds 2 + $modulePath = "{{ aw_windows_deploy_root }}\windows\ActivityWatch.Windows.Common.psm1" + Import-Module $modulePath -Force + # Ensure tasks are enabled (some environments keep them disabled, causing "0s" in WebUI). try { Enable-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" -ErrorAction SilentlyContinue | Out-Null } catch {} $config = Get-Content -Raw -LiteralPath "{{ aw_windows_state_root }}\deployment-config.json" | ConvertFrom-Json - foreach ($taskDef in @($config.userTasks)) { - try { Enable-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue | Out-Null } catch {} + $configPaths = Get-ActivityWatchRecoveryConfigPaths -PrimaryConfigPath "{{ aw_windows_state_root }}\deployment-config.json" + $taskDefs = @(Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths) + $sessionRecords = @(Get-ActivityWatchSessionRecords) + + foreach ($taskDef in @($taskDefs)) { + try { Enable-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue | Out-Null } catch {} } $recoveryTask = Get-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" -ErrorAction SilentlyContinue if ($recoveryTask -and $recoveryTask.State -notin @('Running', 'Queued')) { Start-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" } - foreach ($taskDef in @($config.userTasks)) { - $launchTask = Get-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue + foreach ($taskDef in @($taskDefs)) { + if (-not (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected)) { + continue + } + $launchTask = Get-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue if ($launchTask -and $launchTask.State -notin @('Running', 'Queued')) { - Start-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue + Start-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue } } + - name: Установить session-aware AWatch-rus Collector Guard service + when: aw_windows_collector_guard_enabled | bool + ansible.windows.win_powershell: + script: | + $ErrorActionPreference = 'Stop' + $guardParams = @{ + ConfigPath = "{{ aw_windows_state_root }}\deployment-config.json" + Mode = "{{ aw_windows_collector_guard_mode }}" + ServiceName = "{{ aw_windows_collector_guard_service_name }}" + LoopSeconds = {{ aw_windows_collector_guard_loop_seconds | int }} + } + & "{{ aw_windows_deploy_root }}\windows\install-collector-guard-service.ps1" @guardParams + - name: Получить Windows hostname для AW smoke-check bucket when: - aw_windows_api_smoke_check_enabled | bool diff --git a/ansible/deploy_proxmox_web_gateway.yml b/ansible/deploy_proxmox_web_gateway.yml index a912393..744a752 100644 --- a/ansible/deploy_proxmox_web_gateway.yml +++ b/ansible/deploy_proxmox_web_gateway.yml @@ -14,67 +14,94 @@ proxmox_web_gateway_tls_cert_path: "{{ proxmox_web_gateway_tls_dir }}/fullchain.pem" proxmox_web_gateway_tls_key_path: "{{ proxmox_web_gateway_tls_dir }}/privkey.pem" proxmox_web_gateway_health_path: "/healthz" + proxmox_web_gateway_auth_realm: "DetMir operations gateway" + proxmox_web_gateway_auth_user: "detmir" + proxmox_web_gateway_auth_file: "/etc/nginx/proxmox-web-gateway.htpasswd" + proxmox_web_gateway_credentials_file: "/root/proxmox-web-gateway.credentials" proxmox_web_gateway_routes: - slug: "proxmox-gui" title: "Proxmox VE" category: "Host" description: "Основная панель управления Proxmox VE." target_url: "https://10.10.10.2:8006/" + external_enabled: false - slug: "file1c-brief" title: "1C Executive Brief" category: "Management" description: "Сводка по предприятиям и рискам 1С." target_url: "http://10.10.10.2:8710/manager/brief" + proxy_path: "/r/file1c/brief" + proxy_target_url: "http://10.10.10.2:8710/manager/brief" + external_enabled: true - slug: "file1c-actions" title: "1C Management Actions" category: "Management" description: "Очередь действий по предприятиям в 1С." target_url: "http://10.10.10.2:8710/manager/actions" + proxy_path: "/r/file1c/actions" + proxy_target_url: "http://10.10.10.2:8710/manager/actions" + external_enabled: true - slug: "file1c-finance" title: "1C Financial Reporting" category: "Management" description: "Первый financial board по файловой 1С с разделением ledger/proxy." target_url: "http://10.10.10.11:3000/d/1c-file-finance/1c-file-financial-reporting?orgId=1" + proxy_path: "/d/1c-file-finance/1c-file-financial-reporting?orgId=1" + external_enabled: true - slug: "file1c-telemetry" title: "1C Telemetry Board" category: "Dashboards" description: "Read-only telemetry экран по состоянию файловых баз, reglog и host." target_url: "http://10.10.10.11:3000/d/1c-file-telemetry/1c-file-telemetry-board?orgId=1" + proxy_path: "/d/1c-file-telemetry/1c-file-telemetry-board?orgId=1" + external_enabled: true - slug: "grafana-1c" title: "Grafana 1C" category: "Dashboards" description: "Рабочий file-1c dashboard contour в внешней Grafana." target_url: "http://10.10.10.11:3000/d/1c-file-mgmt/1c-file-management-board?orgId=1" + proxy_path: "/d/1c-file-mgmt/1c-file-management-board?orgId=1" + external_enabled: true - slug: "clickhouse-http" title: "ClickHouse HTTP" category: "Data" description: "HTTP endpoint ClickHouse для file-1C analytics." target_url: "http://10.10.10.2:8123/" + external_enabled: false - slug: "influxdb" title: "InfluxDB" category: "Data" description: "InfluxDB LXC на CT 200." target_url: "http://10.10.10.10:8086/" + external_enabled: false - slug: "grafana-core" title: "Grafana Core" category: "Dashboards" description: "Отдельный Grafana CT 201." target_url: "http://10.10.10.11:3000/" + proxy_path: "/dashboards" + external_enabled: true - slug: "loki-alloy" title: "Grafana Alloy" category: "Logs" description: "Web UI Alloy на CT 202." target_url: "http://10.10.10.12:12345/" + external_enabled: false - slug: "aw-ui" title: "AW-rus UI" category: "Operations" description: "Основной ActivityWatch-Russian Web UI на CT 203." target_url: "http://10.10.10.13:5600/" + proxy_path: "/r/aw/" + external_enabled: true - slug: "aw-worktime" title: "AW-rus Management Report" category: "Operations" description: "Управленческий worktime/report API на CT 203." target_url: "http://10.10.10.13:5610/reports/worktime/management?day=today" + proxy_path: "/r/aw-worktime" + proxy_target_url: "http://10.10.10.13:5610/reports/worktime/management?day=today" + external_enabled: true tasks: - name: Установить nginx @@ -123,6 +150,50 @@ args: creates: "{{ proxmox_web_gateway_tls_cert_path }}" + - name: Проверить наличие gateway Basic Auth файла + ansible.builtin.stat: + path: "{{ proxmox_web_gateway_auth_file }}" + register: proxmox_web_gateway_auth_stat + + - name: Создать gateway Basic Auth credential при первом запуске + ansible.builtin.shell: | + set -eu + umask 077 + password="$(openssl rand -base64 24 | tr -d '\n')" + hash="$(openssl passwd -apr1 "$password")" + printf '%s:%s\n' '{{ proxmox_web_gateway_auth_user }}' "$hash" > '{{ proxmox_web_gateway_auth_file }}' + printf 'url=https://{{ proxmox_web_gateway_public_hostname }}/\nuser={{ proxmox_web_gateway_auth_user }}\npassword=%s\n' "$password" > '{{ proxmox_web_gateway_credentials_file }}' + args: + executable: /bin/sh + creates: "{{ proxmox_web_gateway_auth_file }}" + no_log: true + when: not proxmox_web_gateway_auth_stat.stat.exists + + - name: Зафиксировать права gateway Basic Auth файла + ansible.builtin.file: + path: "{{ proxmox_web_gateway_auth_file }}" + owner: root + group: www-data + mode: "0640" + + - name: Зафиксировать права файла с gateway credential + ansible.builtin.file: + path: "{{ proxmox_web_gateway_credentials_file }}" + owner: root + group: root + mode: "0600" + + - name: Прочитать gateway credential для локальной проверки + ansible.builtin.slurp: + src: "{{ proxmox_web_gateway_credentials_file }}" + register: proxmox_web_gateway_credentials_slurp + no_log: true + + - name: Подготовить gateway auth password для проверок + ansible.builtin.set_fact: + proxmox_web_gateway_auth_password: "{{ (proxmox_web_gateway_credentials_slurp.content | b64decode).split('password=')[1].split('\n')[0] }}" + no_log: true + - name: Развернуть index.html gateway ansible.builtin.template: src: "templates/proxmox-web-gateway-index.html.j2" @@ -180,25 +251,36 @@ - proxmox_web_gateway_http_redirect.location != "https://{{ proxmox_web_gateway_public_hostname }}/" changed_when: false - - name: Проверить redirect на Proxmox GUI + - name: Проверить старый /go путь без auth закрыт ansible.builtin.uri: url: "https://127.0.0.1/go/proxmox-gui" headers: Host: "{{ proxmox_web_gateway_public_hostname }}" validate_certs: false follow_redirects: none - status_code: 302 + status_code: 401 register: proxmox_web_gateway_redirect failed_when: - - proxmox_web_gateway_redirect.status != 302 - - proxmox_web_gateway_redirect.location != "https://10.10.10.2:8006/" + - proxmox_web_gateway_redirect.status != 401 changed_when: false - - name: Проверить index по public hostname + - name: Проверить index без auth закрыт ansible.builtin.uri: url: "https://127.0.0.1/" headers: Host: "{{ proxmox_web_gateway_public_hostname }}" + validate_certs: false + status_code: 401 + changed_when: false + + - name: Проверить index по public hostname с auth + ansible.builtin.uri: + url: "https://127.0.0.1/" + headers: + Host: "{{ proxmox_web_gateway_public_hostname }}" + url_username: "{{ proxmox_web_gateway_auth_user }}" + url_password: "{{ proxmox_web_gateway_auth_password }}" + force_basic_auth: true return_content: true validate_certs: false register: proxmox_web_gateway_named_index @@ -207,6 +289,69 @@ - proxmox_web_gateway_public_hostname not in proxmox_web_gateway_named_index.content changed_when: false + - name: Проверить reverse proxy к 1C brief с auth + ansible.builtin.uri: + url: "https://127.0.0.1/r/file1c/brief" + headers: + Host: "{{ proxmox_web_gateway_public_hostname }}" + url_username: "{{ proxmox_web_gateway_auth_user }}" + url_password: "{{ proxmox_web_gateway_auth_password }}" + force_basic_auth: true + validate_certs: false + status_code: 200 + changed_when: false + + - name: Проверить reverse proxy к Grafana health с auth + ansible.builtin.uri: + url: "https://127.0.0.1/r/grafana/api/health" + headers: + Host: "{{ proxmox_web_gateway_public_hostname }}" + url_username: "{{ proxmox_web_gateway_auth_user }}" + url_password: "{{ proxmox_web_gateway_auth_password }}" + force_basic_auth: true + validate_certs: false + status_code: 200 + changed_when: false + + - name: Проверить reverse proxy к AW info с auth + ansible.builtin.uri: + url: "https://127.0.0.1/r/aw/api/0/info" + headers: + Host: "{{ proxmox_web_gateway_public_hostname }}" + url_username: "{{ proxmox_web_gateway_auth_user }}" + url_password: "{{ proxmox_web_gateway_auth_password }}" + force_basic_auth: true + validate_certs: false + status_code: 200 + changed_when: false + + - name: Проверить browser-origin AW API query без 403 + ansible.builtin.uri: + url: "https://127.0.0.1/api/0/query/" + method: POST + headers: + Host: "{{ proxmox_web_gateway_public_hostname }}" + Origin: "https://{{ proxmox_web_gateway_public_hostname }}" + Referer: "https://{{ proxmox_web_gateway_public_hostname }}/r/aw/" + Content-Type: "application/json" + body_format: json + body: + timeperiods: + - "1970-01-01T00:00:00+00:00/1970-01-01T00:01:00+00:00" + query: + - "RETURN = [];" + url_username: "{{ proxmox_web_gateway_auth_user }}" + url_password: "{{ proxmox_web_gateway_auth_password }}" + force_basic_auth: true + validate_certs: false + status_code: 200 + return_content: true + register: proxmox_web_gateway_aw_query + failed_when: + - proxmox_web_gateway_aw_query.status != 200 + - proxmox_web_gateway_aw_query.content != "[[]]" + changed_when: false + - name: Проверить локальный HTTPS health endpoint gateway ansible.builtin.uri: url: "https://127.0.0.1{{ proxmox_web_gateway_health_path }}" diff --git a/ansible/deploy_tsj_guardian_bot_proxmox.yml b/ansible/deploy_tsj_guardian_bot_proxmox.yml index aaaedca..ed3e24e 100644 --- a/ansible/deploy_tsj_guardian_bot_proxmox.yml +++ b/ansible/deploy_tsj_guardian_bot_proxmox.yml @@ -12,6 +12,10 @@ tsj_bot_script_name: "tsj_guardian_bot.py" tsj_bot_script_dest: "{{ tsj_bot_root }}/{{ tsj_bot_script_name }}" tsj_bot_source_local_path: "{{ aw_repo_root }}/proxmox/tsj_guardian_bot.py" + tsj_bot_watchdog_name: "tsj_guardian_watchdog.sh" + tsj_bot_watchdog_dest: "{{ tsj_bot_root }}/{{ tsj_bot_watchdog_name }}" + tsj_bot_watchdog_source_local_path: "{{ aw_repo_root }}/proxmox/{{ tsj_bot_watchdog_name }}" + tsj_bot_watchdog_service_name: "tsj-guardian-watchdog.service" tsj_bot_openvpn_helper_name: "pfsense_openvpn_client_export.php" tsj_bot_openvpn_helper_dest: "{{ tsj_bot_root }}/{{ tsj_bot_openvpn_helper_name }}" tsj_bot_openvpn_helper_source_local_path: "{{ aw_repo_root }}/proxmox/{{ tsj_bot_openvpn_helper_name }}" @@ -20,6 +24,16 @@ tsj_bot_state_dir: "{{ tsj_bot_runtime_root }}/.state" tsj_bot_logs_dir: "{{ tsj_bot_runtime_root }}/logs" tsj_bot_default_chat_id: "{{ telegram_default_chat_id | default(telegram_allowed_chat_ids.split(',')[0]) }}" + aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}" + tsj_guardian_status_required_flags: + - "--status-text" + - "--incident-suggestions" + - "--incident-defer-decision" + - "--escalation-decision" + - "--operator-action-decision" + - "--dlp-policy-decision" + - "--confirmation-decision" + - "--autoheal-plan-decision" pre_tasks: - name: Проверить наличие существующего .env бота на хосте @@ -62,6 +76,42 @@ delegate_to: localhost become: false + - name: Проверить наличие watchdog скрипта на контроллере + ansible.builtin.stat: + path: "{{ tsj_bot_watchdog_source_local_path }}" + register: tsj_bot_watchdog_stat + delegate_to: localhost + become: false + + - name: Проверить локальный Rust TSJ guardian status helper + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/tsj-guardian-status" + register: tsj_guardian_status_rust_binary + delegate_to: localhost + become: false + + - name: Остановить выполнение если Rust TSJ guardian status helper не найден + ansible.builtin.assert: + that: + - tsj_guardian_status_rust_binary.stat.exists + - tsj_guardian_status_rust_binary.stat.isreg + fail_msg: "Rust helper не найден: {{ aw_rust_release_dir }}/tsj-guardian-status. Соберите binary и проверьте CARGO_TARGET_DIR." + + - name: Проверить контракт локального Rust TSJ guardian status helper + ansible.builtin.command: + cmd: "{{ aw_rust_release_dir }}/tsj-guardian-status --help" + register: tsj_guardian_status_help + changed_when: false + delegate_to: localhost + become: false + + - name: Остановить выполнение если Rust TSJ guardian status helper устарел + ansible.builtin.assert: + that: + - item in tsj_guardian_status_help.stdout + fail_msg: "Rust helper {{ aw_rust_release_dir }}/tsj-guardian-status не поддерживает {{ item }}. Проверьте CARGO_TARGET_DIR и пересоберите helper." + loop: "{{ tsj_guardian_status_required_flags }}" + - name: Остановить выполнение если helper для OpenVPN не найден ansible.builtin.assert: that: @@ -69,6 +119,13 @@ - tsj_bot_openvpn_helper_stat.stat.isreg fail_msg: "Файл helper для OpenVPN не найден: {{ tsj_bot_openvpn_helper_source_local_path }}" + - name: Остановить выполнение если watchdog не найден + ansible.builtin.assert: + that: + - tsj_bot_watchdog_stat.stat.exists + - tsj_bot_watchdog_stat.stat.isreg + fail_msg: "Файл watchdog не найден: {{ tsj_bot_watchdog_source_local_path }}" + tasks: - name: Установить зависимости Python для бота ansible.builtin.package: @@ -99,6 +156,16 @@ mode: "0750" notify: Restart tsj bot + - name: Развернуть Rust helper статуса TSJ guardian + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/tsj-guardian-status" + dest: /usr/local/bin/tsj-guardian-status + owner: root + group: root + mode: "0755" + when: tsj_guardian_status_rust_binary.stat.exists | default(false) + notify: Restart tsj bot + - name: Развернуть helper для OpenVPN экспорта ansible.builtin.copy: src: "{{ tsj_bot_openvpn_helper_source_local_path }}" @@ -108,12 +175,21 @@ mode: "0640" notify: Restart tsj bot + - name: Развернуть watchdog скрипт бота + ansible.builtin.copy: + src: "{{ tsj_bot_watchdog_source_local_path }}" + dest: "{{ tsj_bot_watchdog_dest }}" + owner: root + group: root + mode: "0755" + - name: Сгенерировать полный .env бота when: - telegram_bot_token is defined - (telegram_bot_token | string | length) > 20 - telegram_allowed_chat_ids is defined - (telegram_allowed_chat_ids | string | length) > 0 + no_log: true ansible.builtin.copy: dest: "{{ tsj_bot_env_path }}" owner: "{{ tsj_bot_user }}" @@ -142,6 +218,7 @@ LOG_FILE={{ tsj_bot_log_file | default(tsj_bot_runtime_root + '/logs/tsj_guardian_bot.log') }} HEARTBEAT_FILE={{ tsj_bot_heartbeat_file | default(tsj_bot_runtime_root + '/.state/tsj_guardian_heartbeat') }} CHECK_INTERVAL_SEC={{ tsj_bot_check_interval_sec | default(60) }} + INCIDENT_FAILURE_QUORUM_CHECKS={{ tsj_bot_incident_failure_quorum_checks | default(2) }} OPERATOR_TIMEOUT_SEC={{ tsj_bot_operator_timeout_sec | default(900) }} RETRY_AUTORECOVERY_EVERY_SEC={{ tsj_bot_retry_autorecovery_every_sec | default(300) }} EXIT_ON_AUTORECOVERY_SUCCESS={{ tsj_bot_exit_on_autorecovery_success | default('true') }} @@ -175,18 +252,25 @@ SERVER_FALLBACK_COMMANDS={{ tsj_bot_server_fallback_commands | default(tsj_bot_runtime_root + '/scripts/system_self_support.sh --heal') }} UPDATES_SCRIPT={{ tsj_bot_updates_script | default('/usr/bin/python3 ' + tsj_bot_runtime_root + '/scripts/proxmox_lxc_critical_updates.py') }} UPDATE_TARGETS={{ tsj_bot_update_targets | default('auto') }} + DETMIR_AI_STATE_FILE={{ tsj_bot_detmir_ai_state_file | default('/var/lib/detmir-ai/latest-state.json') }} + TSJ_GUARDIAN_STATUS_BIN={{ tsj_bot_guardian_status_bin | default('/usr/local/bin/tsj-guardian-status') }} AW_RUS_API_BASE={{ tsj_bot_aw_rus_api_base | default('http://10.10.10.13:5600/api/0') }} AW_RUS_WORKTIME_BASE={{ tsj_bot_aw_rus_worktime_base | default('http://10.10.10.13:5610') }} AW_DLP_POLICY_API_BASE={{ tsj_bot_aw_dlp_policy_api_base | default('http://10.10.10.13:5601/api/0') }} AW_DLP_POLICY_ACTOR={{ tsj_bot_aw_dlp_policy_actor | default('tsj-guardian-bot') }} - AW_RUS_WORKTIME_HEAL_CMD={{ tsj_bot_aw_rus_worktime_heal_cmd | default("sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S /usr/local/bin/aw-worktime-autoheal.sh && sudo -S systemctl reset-failed aw-worktime-ui-bridge.service && sudo -S systemctl start aw-worktime-ui-bridge.service'") }} - AW_RUS_DLP_HEAL_CMD={{ tsj_bot_aw_rus_dlp_heal_cmd | default("sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S systemctl restart activitywatch-server.service && sudo -S systemctl start activitywatch-dlp-aggregator.service || true && sudo -S /usr/local/bin/aw-health-check && sudo -S /usr/local/bin/dlp-health-check'") }} + AW_RUS_WORKTIME_HEAL_CMD={{ tsj_bot_aw_rus_worktime_heal_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal.sh && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'") }} + AW_RUS_DLP_HEAL_CMD={{ tsj_bot_aw_rus_dlp_heal_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n systemctl restart activitywatch-server.service && (sudo -n systemctl start activitywatch-dlp-aggregator.service || true) && sudo -n /usr/local/bin/aw-health-check && sudo -n /usr/local/bin/dlp-health-check'") }} AW_RUS_CASE_API_BASE={{ tsj_bot_aw_rus_case_api_base | default('http://10.10.10.13:5602') }} AW_RUS_HAYABUSA_ENABLED={{ tsj_bot_aw_rus_hayabusa_enabled | default('true') }} - AW_RUS_HAYABUSA_SSH_CMD={{ tsj_bot_aw_rus_hayabusa_ssh_cmd | default("sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13") }} + AW_RUS_HAYABUSA_SSH_CMD={{ tsj_bot_aw_rus_hayabusa_ssh_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13") }} AW_RUS_HOST={{ tsj_bot_aw_rus_host | default('SHARKON2025') }} AW_RUS_PRIMARY_USER={{ tsj_bot_aw_rus_primary_user | default('USER1') }} AW_RUS_STALE_SEC={{ tsj_bot_aw_rus_stale_sec | default(900) }} + AW_RUS_SLO_ENABLED={{ tsj_bot_aw_rus_slo_enabled | default('true') }} + AW_RUS_SLO_ALERT_WINDOW={{ tsj_bot_aw_rus_slo_alert_window | default('24h') }} + AW_RUS_SLO_MIN_SAMPLES={{ tsj_bot_aw_rus_slo_min_samples | default(4) }} + AW_RUS_SLO_MAX_AGE_SEC={{ tsj_bot_aw_rus_slo_max_age_sec | default(90) }} + AW_RUS_SLO_SUMMARY_CMD={{ tsj_bot_aw_rus_slo_summary_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'cat /var/lib/activitywatch/slo/aw-slo-summary.json'") }} AW_RUS_WINDOWS_HOST={{ tsj_bot_aw_rus_windows_host | default(hostvars[(groups['aw_windows'] | first)].ansible_host | default('192.168.100.18')) }} AW_RUS_WINDOWS_SSH_USER={{ tsj_bot_aw_rus_windows_ssh_user | default(hostvars[(groups['aw_windows'] | first)].ansible_user | default('Администратор')) }} AW_RUS_WINDOWS_SSH_PASSWORD={{ tsj_bot_aw_rus_windows_ssh_password | default(hostvars[(groups['aw_windows'] | first)].ansible_password | default('')) }} @@ -207,6 +291,7 @@ (telegram_allowed_chat_ids | string | length) > 0 ) - tsj_bot_existing_env.stat.exists | default(false) + no_log: true ansible.builtin.lineinfile: path: "{{ tsj_bot_env_path }}" regexp: "^{{ item.key }}=" @@ -220,14 +305,19 @@ - { key: "AW_RUS_WORKTIME_BASE", value: "{{ tsj_bot_aw_rus_worktime_base | default('http://10.10.10.13:5610') }}" } - { key: "AW_DLP_POLICY_API_BASE", value: "{{ tsj_bot_aw_dlp_policy_api_base | default('http://10.10.10.13:5601/api/0') }}" } - { key: "AW_DLP_POLICY_ACTOR", value: "{{ tsj_bot_aw_dlp_policy_actor | default('tsj-guardian-bot') }}" } - - { key: "AW_RUS_WORKTIME_HEAL_CMD", value: "{{ tsj_bot_aw_rus_worktime_heal_cmd | default(\"sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S /usr/local/bin/aw-worktime-autoheal.sh && sudo -S systemctl reset-failed aw-worktime-ui-bridge.service && sudo -S systemctl start aw-worktime-ui-bridge.service'\") }}" } - - { key: "AW_RUS_DLP_HEAL_CMD", value: "{{ tsj_bot_aw_rus_dlp_heal_cmd | default(\"sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S systemctl restart activitywatch-server.service && sudo -S systemctl start activitywatch-dlp-aggregator.service || true && sudo -S /usr/local/bin/aw-health-check && sudo -S /usr/local/bin/dlp-health-check'\") }}" } + - { key: "AW_RUS_WORKTIME_HEAL_CMD", value: "{{ tsj_bot_aw_rus_worktime_heal_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal.sh && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'\") }}" } + - { key: "AW_RUS_DLP_HEAL_CMD", value: "{{ tsj_bot_aw_rus_dlp_heal_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n systemctl restart activitywatch-server.service && (sudo -n systemctl start activitywatch-dlp-aggregator.service || true) && sudo -n /usr/local/bin/aw-health-check && sudo -n /usr/local/bin/dlp-health-check'\") }}" } - { key: "AW_RUS_CASE_API_BASE", value: "{{ tsj_bot_aw_rus_case_api_base | default('http://10.10.10.13:5602') }}" } - { key: "AW_RUS_HAYABUSA_ENABLED", value: "{{ tsj_bot_aw_rus_hayabusa_enabled | default('true') }}" } - - { key: "AW_RUS_HAYABUSA_SSH_CMD", value: "{{ tsj_bot_aw_rus_hayabusa_ssh_cmd | default(\"sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13\") }}" } + - { key: "AW_RUS_HAYABUSA_SSH_CMD", value: "{{ tsj_bot_aw_rus_hayabusa_ssh_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13\") }}" } - { key: "AW_RUS_HOST", value: "{{ tsj_bot_aw_rus_host | default('SHARKON2025') }}" } - { key: "AW_RUS_PRIMARY_USER", value: "{{ tsj_bot_aw_rus_primary_user | default('USER1') }}" } - { key: "AW_RUS_STALE_SEC", value: "{{ tsj_bot_aw_rus_stale_sec | default(900) }}" } + - { key: "AW_RUS_SLO_ENABLED", value: "{{ tsj_bot_aw_rus_slo_enabled | default('true') }}" } + - { key: "AW_RUS_SLO_ALERT_WINDOW", value: "{{ tsj_bot_aw_rus_slo_alert_window | default('24h') }}" } + - { key: "AW_RUS_SLO_MIN_SAMPLES", value: "{{ tsj_bot_aw_rus_slo_min_samples | default(4) }}" } + - { key: "AW_RUS_SLO_MAX_AGE_SEC", value: "{{ tsj_bot_aw_rus_slo_max_age_sec | default(90) }}" } + - { key: "AW_RUS_SLO_SUMMARY_CMD", value: "{{ tsj_bot_aw_rus_slo_summary_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'cat /var/lib/activitywatch/slo/aw-slo-summary.json'\") }}" } - { key: "AW_RUS_WINDOWS_HOST", value: "{{ tsj_bot_aw_rus_windows_host | default(hostvars[(groups['aw_windows'] | first)].ansible_host | default('192.168.100.18')) }}" } - { key: "AW_RUS_WINDOWS_SSH_USER", value: "{{ tsj_bot_aw_rus_windows_ssh_user | default(hostvars[(groups['aw_windows'] | first)].ansible_user | default('Администратор')) }}" } - { key: "AW_RUS_WINDOWS_SSH_PASSWORD", value: "{{ tsj_bot_aw_rus_windows_ssh_password | default(hostvars[(groups['aw_windows'] | first)].ansible_password | default('')) }}" } @@ -239,6 +329,9 @@ - { key: "AW_RUS_WINDOWS_EMAIL_COLLECTOR_PATH", value: "{{ tsj_bot_aw_rus_windows_email_collector_path | default('C:\\ProgramData\\AWatch-rus\\email-outbound-collector.ps1') }}" } - { key: "AI_CHAT_WORKDIR", value: "{{ tsj_bot_ai_chat_workdir | default('/home/igor') }}" } - { key: "AI_EXEC_USER", value: "{{ tsj_bot_ai_exec_user | default('igor') }}" } + - { key: "INCIDENT_FAILURE_QUORUM_CHECKS", value: "{{ tsj_bot_incident_failure_quorum_checks | default(2) }}" } + - { key: "DETMIR_AI_STATE_FILE", value: "{{ tsj_bot_detmir_ai_state_file | default('/var/lib/detmir-ai/latest-state.json') }}" } + - { key: "TSJ_GUARDIAN_STATUS_BIN", value: "{{ tsj_bot_guardian_status_bin | default('/usr/local/bin/tsj-guardian-status') }}" } - { key: "TMUX_USER", value: "{{ tsj_bot_tmux_user | default('igor') }}" } - { key: "PFSENSE_ENV_PATH", value: "{{ tsj_bot_pfsense_env_path | default('/home/igor/.config/tsj-bot/pfsense.env.readonly') }}" } - { key: "PFSENSE_INVENTORY_PATH", value: "{{ tsj_bot_pfsense_inventory_path | default('/home/igor/.config/tsj-bot/inventory.md') }}" } @@ -272,6 +365,33 @@ - Reload systemd - Restart tsj bot + - name: Установить systemd unit watchdog бота + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ tsj_bot_watchdog_service_name }}" + owner: root + group: root + mode: "0644" + content: | + [Unit] + Description=TSJ Guardian Bot Heartbeat Watchdog + After={{ tsj_bot_service_name }} gost-tg.service + Wants={{ tsj_bot_service_name }} gost-tg.service + + [Service] + Type=simple + User=root + ExecStart=/bin/bash -lc 'while true; do {{ tsj_bot_watchdog_dest }}; sleep 60; done' + StandardOutput=null + StandardError=null + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + notify: + - Reload systemd + - Restart tsj watchdog + - name: Проверить синтаксис Python скрипта бота ansible.builtin.command: "python3 -m py_compile {{ tsj_bot_script_dest }}" changed_when: false @@ -282,12 +402,24 @@ enabled: true state: started + - name: Включить и запустить watchdog бота + ansible.builtin.systemd: + name: "{{ tsj_bot_watchdog_service_name }}" + enabled: true + state: started + - name: Проверить что сервис активен ansible.builtin.command: "systemctl is-active {{ tsj_bot_service_name }}" register: tsj_bot_active changed_when: false failed_when: tsj_bot_active.stdout.strip() != "active" + - name: Проверить что watchdog активен + ansible.builtin.command: "systemctl is-active {{ tsj_bot_watchdog_service_name }}" + register: tsj_bot_watchdog_active + changed_when: false + failed_when: tsj_bot_watchdog_active.stdout.strip() != "active" + handlers: - name: Reload systemd ansible.builtin.systemd: @@ -297,3 +429,8 @@ ansible.builtin.systemd: name: "{{ tsj_bot_service_name }}" state: restarted + + - name: Restart tsj watchdog + ansible.builtin.systemd: + name: "{{ tsj_bot_watchdog_service_name }}" + state: restarted diff --git a/ansible/group_vars/all.example.yml b/ansible/group_vars/all.example.yml index 2bcd41e..6e5d732 100644 --- a/ansible/group_vars/all.example.yml +++ b/ansible/group_vars/all.example.yml @@ -32,6 +32,11 @@ aw_monitored_windows_hostname: "SHARKON2025" aw_rus_health_worktime_api_base: "http://127.0.0.1:5610" aw_rus_health_state_dir: "{{ aw_server_data_dir }}/health" aw_rus_health_validation_dir: "{{ aw_rus_health_state_dir }}/windows-validation" +aw_browser_smoke_enabled: true +aw_browser_smoke_engine: "chromium-cli" +aw_legacy_db_merge_enabled: false +aw_browser_smoke_timeout_ms: 20000 +aw_browser_smoke_render_timeout_ms: 15000 aw_hayabusa_auto_case_enabled: true aw_hayabusa_auto_case_min_severity: "medium" aw_hayabusa_telegram_enabled: true @@ -62,7 +67,7 @@ aw_worktime_from: "00:00" aw_worktime_to: "17:00" aw_worktime_start_of_day: "{{ aw_worktime_from }}" aw_server_always_active_pattern: "aw-watcher-window" -aw_server_landingpage: "/activity/SHARKON2025/view/" +aw_server_landingpage: "/#/activity/SHARKON2025/view/" aw_health_strict_fileops: 0 aw_dlp_policy_engine_enabled: true diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index af29ce2..bf3a9c6 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -74,5 +74,5 @@ aw_worktime_from: "00:00" aw_worktime_to: "17:00" aw_worktime_start_of_day: "{{ aw_worktime_from }}" aw_server_always_active_pattern: "aw-watcher-window" -aw_server_landingpage: "/activity/SHARKON2025/view/" +aw_server_landingpage: "/#/activity/SHARKON2025/view/" aw_health_strict_fileops: 0 diff --git a/ansible/group_vars/aw_windows.yml b/ansible/group_vars/aw_windows.yml index 8ca319d..21f26fe 100644 --- a/ansible/group_vars/aw_windows.yml +++ b/ansible/group_vars/aw_windows.yml @@ -59,7 +59,7 @@ aw_windows_evtx_channels: - Microsoft-Windows-TerminalServices-LocalSessionManager/Operational - Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational aw_windows_logon_marker_enabled: true -aw_windows_process_events_enabled: true +aw_windows_process_events_enabled: false aw_windows_skip_hardening: false aw_windows_rules_path: "{{ aw_windows_deploy_root }}\\windows\\web-category-rules.example.json" @@ -69,6 +69,11 @@ aw_windows_validation_remote_path: "{{ aw_windows_state_root }}\\aw_validate_ans aw_windows_validation_local_dir: "/tmp/aw-rus-validation-{{ lookup('env','USER') | default('ansible', true) }}" aw_windows_fail_on_validation_error: true +aw_windows_collector_guard_enabled: true +aw_windows_collector_guard_mode: "enforce" +aw_windows_collector_guard_service_name: "AWatchRusCollectorGuard" +aw_windows_collector_guard_loop_seconds: 60 + aw_windows_migration_enabled: false aw_windows_legacy_install_root: "C:\\Program Files\\ActivityWatch-Phase2" aw_windows_legacy_state_root: "C:\\ProgramData\\ActivityWatch-Phase2" diff --git a/ansible/group_vars/proxmox-bot.example.yml b/ansible/group_vars/proxmox-bot.example.yml index 5e6f490..cf8a1a9 100644 --- a/ansible/group_vars/proxmox-bot.example.yml +++ b/ansible/group_vars/proxmox-bot.example.yml @@ -8,6 +8,7 @@ tsj_bot_runtime_root: "/opt/infra-admin" # Optional bot tuning tsj_bot_check_interval_sec: 60 +tsj_bot_incident_failure_quorum_checks: 2 tsj_bot_operator_timeout_sec: 900 tsj_bot_retry_autorecovery_every_sec: 300 tsj_bot_telegram_proxy_url: "http://127.0.0.1:11090" @@ -56,6 +57,11 @@ tsj_bot_aw_rus_hayabusa_ssh_cmd: "sshpass -p 'CHANGE_ME' ssh -o PubkeyAuthentica tsj_bot_aw_rus_host: "SHARKON2025" tsj_bot_aw_rus_primary_user: "USER1" tsj_bot_aw_rus_stale_sec: 900 +tsj_bot_aw_rus_slo_enabled: "true" +tsj_bot_aw_rus_slo_alert_window: "24h" +tsj_bot_aw_rus_slo_min_samples: 4 +tsj_bot_aw_rus_slo_max_age_sec: 90 +tsj_bot_aw_rus_slo_summary_cmd: "sshpass -p 'CHANGE_ME' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'cat /var/lib/activitywatch/slo/aw-slo-summary.json'" tsj_bot_aw_rus_windows_host: "192.168.100.18" tsj_bot_aw_rus_windows_ssh_user: "Администратор" tsj_bot_aw_rus_windows_ssh_password: "CHANGE_ME" diff --git a/ansible/group_vars/windows.example.yml b/ansible/group_vars/windows.example.yml index ae3eee2..9f6ad99 100644 --- a/ansible/group_vars/windows.example.yml +++ b/ansible/group_vars/windows.example.yml @@ -50,7 +50,7 @@ aw_windows_evtx_channels: - Microsoft-Windows-TerminalServices-LocalSessionManager/Operational - Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational aw_windows_logon_marker_enabled: true -aw_windows_process_events_enabled: true +aw_windows_process_events_enabled: false aw_windows_skip_hardening: false aw_windows_rules_path: "{{ aw_windows_deploy_root }}\\windows\\web-category-rules.example.json" @@ -60,6 +60,13 @@ aw_windows_validation_remote_path: "{{ aw_windows_state_root }}\\aw_validate_ans aw_windows_validation_local_dir: "/tmp/aw-rus-validation-{{ lookup('env','USER') | default('ansible', true) }}" aw_windows_fail_on_validation_error: true +# Collector Guard supervises collectors, but ActivityWatch Recovery must stay enabled +# as a fallback and as the launch-task bootstrap path for managed RDP sessions. +aw_windows_collector_guard_enabled: true +aw_windows_collector_guard_mode: "enforce" +aw_windows_collector_guard_service_name: "AWatchRusCollectorGuard" +aw_windows_collector_guard_loop_seconds: 60 + # Безопасная миграция текущего прода со старых путей в единый профиль AWatch-rus. aw_windows_migration_enabled: true aw_windows_legacy_install_root: "C:\\Program Files\\ActivityWatch-Phase2" diff --git a/ansible/roles/dlp-integrations/tasks/main.yml b/ansible/roles/dlp-integrations/tasks/main.yml index ec1e3bd..fe0e340 100644 --- a/ansible/roles/dlp-integrations/tasks/main.yml +++ b/ansible/roles/dlp-integrations/tasks/main.yml @@ -68,6 +68,117 @@ - { src: "webhook-sender.service", dest: "/etc/systemd/system/aw-dlp-webhook-sender.service" } - { src: "webhook-sender.timer", dest: "/etc/systemd/system/aw-dlp-webhook-sender.timer" } +- name: Check local Rust CEF exporter + ansible.builtin.stat: + path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-cef-exporter" + delegate_to: localhost + register: dlp_cef_exporter_rust_binary + become: false + +- name: Install Rust CEF exporter + ansible.builtin.copy: + src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-cef-exporter" + dest: /usr/local/bin/dlp-cef-exporter-rust + owner: root + group: root + mode: "0755" + when: dlp_cef_exporter_rust_binary.stat.exists | default(false) + +- name: Ensure CEF exporter drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-cef-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: dlp_cef_exporter_rust_binary.stat.exists | default(false) + +- name: Switch CEF exporter to Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-cef-exporter-rust + when: dlp_cef_exporter_rust_binary.stat.exists | default(false) + +- name: Check local Rust syslog forwarder + ansible.builtin.stat: + path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-syslog-forwarder" + delegate_to: localhost + register: dlp_syslog_forwarder_rust_binary + become: false + +- name: Install Rust syslog forwarder + ansible.builtin.copy: + src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-syslog-forwarder" + dest: /usr/local/bin/dlp-syslog-forwarder-rust + owner: root + group: root + mode: "0755" + when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + +- name: Ensure syslog forwarder drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d + state: directory + owner: root + group: root + mode: "0755" + when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + +- name: Switch syslog forwarder to Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust + when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + +- name: Check local Rust webhook sender + ansible.builtin.stat: + path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-webhook-sender" + delegate_to: localhost + register: dlp_webhook_sender_rust_binary + become: false + +- name: Install Rust webhook sender + ansible.builtin.copy: + src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-webhook-sender" + dest: /usr/local/bin/dlp-webhook-sender-rust + owner: root + group: root + mode: "0755" + when: dlp_webhook_sender_rust_binary.stat.exists | default(false) + +- name: Ensure webhook sender drop-in directory + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-webhook-sender.service.d + state: directory + owner: root + group: root + mode: "0755" + when: dlp_webhook_sender_rust_binary.stat.exists | default(false) + +- name: Switch webhook sender to Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-webhook-sender-rust + when: dlp_webhook_sender_rust_binary.stat.exists | default(false) + - name: Reload systemd ansible.builtin.systemd: daemon_reload: true diff --git a/ansible/templates/proxmox-web-gateway-index.html.j2 b/ansible/templates/proxmox-web-gateway-index.html.j2 index 7e69fa4..c77b3aa 100644 --- a/ansible/templates/proxmox-web-gateway-index.html.j2 +++ b/ansible/templates/proxmox-web-gateway-index.html.j2 @@ -78,6 +78,14 @@ transform: translateY(-2px); border-color: var(--accent); } + .card-disabled { + cursor: default; + opacity: 0.72; + } + .card-disabled:hover { + transform: none; + border-color: var(--line); + } .badge { display: inline-flex; align-items: center; @@ -124,9 +132,9 @@

Proxmox Web Gateway

Единая стартовая точка для web-сервисов контура на Proxmox host - 10.10.10.2. Gateway сознательно работает как - безопасный redirector: приложения не ломаются под subpath, а оператор - получает один адрес входа. + 10.10.10.2. Внешний вход закрыт gateway-auth, + а основные операторские страницы идут через reverse proxy без выдачи + внутренних адресов наружу.

Публичное имя gateway: {{ proxmox_web_gateway_public_hostname }} @@ -139,12 +147,21 @@

{% for route in proxmox_web_gateway_routes %} - - {{ route.category }} +{% set external_enabled = route.external_enabled | default(false) %} +{% if external_enabled and route.proxy_path is defined %} + +{% else %} + +{% endif %} {% endfor %}
diff --git a/ansible/templates/proxmox-web-gateway.conf.j2 b/ansible/templates/proxmox-web-gateway.conf.j2 index 4a91758..ea6aaeb 100644 --- a/ansible/templates/proxmox-web-gateway.conf.j2 +++ b/ansible/templates/proxmox-web-gateway.conf.j2 @@ -1,3 +1,8 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 80 default_server; listen [::]:80 default_server; @@ -24,25 +29,238 @@ server { ssl_protocols TLSv1.2 TLSv1.3; ssl_prefer_server_ciphers off; + auth_basic "{{ proxmox_web_gateway_auth_realm }}"; + auth_basic_user_file {{ proxmox_web_gateway_auth_file }}; + + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header Referrer-Policy "no-referrer" always; + add_header X-Robots-Tag "noindex, nofollow, noarchive" always; + add_header Strict-Transport-Security "max-age=31536000" always; + root {{ proxmox_web_gateway_root }}; index index.html; location = {{ proxmox_web_gateway_health_path }} { + auth_basic off; default_type text/plain; return 200 "ok\n"; } location = /robots.txt { + auth_basic off; default_type text/plain; return 200 "User-agent: *\nDisallow: /\n"; } -{% for route in proxmox_web_gateway_routes %} - location = /go/{{ route.slug }} { - return 302 {{ route.target_url }}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 120s; + proxy_send_timeout 120s; + + location /api/0/ { + proxy_set_header Origin ""; + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; } + + location /r/grafana/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000/; + proxy_redirect http://10.10.10.11:3000/ /r/grafana/; + } + + location = /login { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location = /logout { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /public/ { + proxy_set_header Origin ""; + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /api/live/ { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Origin "$scheme://$host"; + proxy_set_header Authorization ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /api/ { + proxy_set_header Origin ""; + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /apis/ { + proxy_set_header Origin ""; + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /d/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /dashboards { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /dashboard/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /avatar/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /profile/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /org/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /user/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /plugins/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /explore { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /alerting/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /connections/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /datasources/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.11:3000; + proxy_redirect off; + } + + location /r/aw/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600/; + proxy_redirect http://10.10.10.13:5600/ /r/aw/; + } + + location /reports/worktime/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5610; + proxy_redirect off; + } + + location = /dark.css { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + } + + location /css/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + } + + location = /js/aw-worktime-panel.js { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + sub_filter_once off; + sub_filter_types application/javascript text/javascript; + sub_filter 'http://10.10.10.13:5610' ''; + } + + location /js/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + } + + location /img/ { + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + } + + location /fonts/ { + auth_basic off; + proxy_set_header Origin ""; + proxy_set_header Authorization ""; + proxy_pass http://10.10.10.13:5600; + proxy_redirect off; + } + +{% for route in proxmox_web_gateway_routes %} +{% if route.proxy_target_url is defined %} + location = {{ route.proxy_path }} { + proxy_set_header Authorization ""; + proxy_pass {{ route.proxy_target_url }}; + proxy_redirect off; + } + +{% endif %} {% endfor %} + location /go/ { + try_files /index.html =404; + } + location / { try_files $uri $uri/ /index.html; } diff --git a/aw-server/aw-browser-smoke.service b/aw-server/aw-browser-smoke.service new file mode 100644 index 0000000..41b74e9 --- /dev/null +++ b/aw-server/aw-browser-smoke.service @@ -0,0 +1,28 @@ +[Unit] +Description=AW-RUS browser-visible WebUI smoke check +After=network-online.target activitywatch-server.service aw-worktime-api.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +Environment=NODE_PATH=/usr/share/nodejs +Environment=AW_BROWSER_SMOKE_ENGINE=chromium-cli +Environment=HOME=/var/lib/activitywatch/browser-smoke +Environment=XDG_CONFIG_HOME=/var/lib/activitywatch/browser-smoke/.config +Environment=XDG_CACHE_HOME=/var/lib/activitywatch/browser-smoke/.cache +WorkingDirectory=/var/lib/activitywatch/browser-smoke +ExecStart=/usr/bin/node /usr/local/lib/aw-rus-browser-smoke/aw-webui-browser-smoke.mjs +TimeoutStartSec=180 +User=activitywatch +Group=activitywatch +Nice=15 +CPUQuota=25% +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/activitywatch/browser-smoke +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-browser-smoke diff --git a/aw-server/aw-browser-smoke.timer b/aw-server/aw-browser-smoke.timer new file mode 100644 index 0000000..d956c91 --- /dev/null +++ b/aw-server/aw-browser-smoke.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AW-RUS browser-visible WebUI smoke check hourly + +[Timer] +OnCalendar=hourly +AccuracySec=5min +RandomizedDelaySec=10min +Persistent=false +Unit=aw-browser-smoke.service + +[Install] +WantedBy=timers.target diff --git a/aw-server/aw-db-maintenance.service b/aw-server/aw-db-maintenance.service new file mode 100644 index 0000000..24f2188 --- /dev/null +++ b/aw-server/aw-db-maintenance.service @@ -0,0 +1,8 @@ +[Unit] +Description=ActivityWatch SQLite guarded maintenance +After=activitywatch-server.service + +[Service] +Type=oneshot +EnvironmentFile=-/etc/activitywatch/aw-server.env +ExecStart=/usr/local/bin/aw-db-maintenance --apply --json diff --git a/aw-server/aw-db-maintenance.timer b/aw-server/aw-db-maintenance.timer new file mode 100644 index 0000000..42d9bff --- /dev/null +++ b/aw-server/aw-db-maintenance.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly ActivityWatch SQLite guarded maintenance + +[Timer] +OnCalendar=Sun *-*-* 03:30:00 +Persistent=true +RandomizedDelaySec=15m + +[Install] +WantedBy=timers.target diff --git a/aw-server/aw-dlp-influx-exporter.timer b/aw-server/aw-dlp-influx-exporter.timer index 9375117..31d6e40 100644 --- a/aw-server/aw-dlp-influx-exporter.timer +++ b/aw-server/aw-dlp-influx-exporter.timer @@ -2,9 +2,9 @@ Description=Run AW DLP InfluxDB exporter every 10 minutes [Timer] -OnBootSec=4min -OnUnitActiveSec=10min +OnCalendar=*:5/10:20 AccuracySec=1min +RandomizedDelaySec=30s Unit=aw-dlp-influx-exporter.service [Install] diff --git a/aw-server/aw-hayabusa-drop.service b/aw-server/aw-hayabusa-drop.service index 77ee804..a19e340 100644 --- a/aw-server/aw-hayabusa-drop.service +++ b/aw-server/aw-hayabusa-drop.service @@ -6,7 +6,7 @@ Wants=network-online.target activitywatch-server.service aw-dlp-case-management. [Service] Type=oneshot EnvironmentFile=-/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-hayabusa-autoprocess +ExecStart=/usr/local/bin/aw-hayabusa-autoprocess User=root Group=root StandardOutput=journal diff --git a/aw-server/aw-health-check-wrapper.sh b/aw-server/aw-health-check-wrapper.sh new file mode 100644 index 0000000..98e6321 --- /dev/null +++ b/aw-server/aw-health-check-wrapper.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -x /usr/local/bin/aw-health-check-rust ]]; then + exec /usr/local/bin/aw-health-check-rust "$@" +fi + +exec /opt/activitywatch/aw-rus-ops/health-check.sh "$@" diff --git a/aw-server/aw-prune-local-state.sh b/aw-server/aw-prune-local-state.sh index c2c6afd..0455a45 100644 --- a/aw-server/aw-prune-local-state.sh +++ b/aw-server/aw-prune-local-state.sh @@ -1,42 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -AW_DATA_DIR="${AW_DATA_DIR:-/var/lib/activitywatch}" -BACKUP_DIR="${AW_BACKUP_DIR:-${AW_DATA_DIR}/backups}" -KEEP_DAYS="${AW_BACKUP_RETENTION_DAYS:-7}" -KEEP_LAST_DB="${AW_BACKUP_KEEP_LAST_DB:-2}" -KEEP_LAST_JSON="${AW_BACKUP_KEEP_LAST_JSON:-2}" +if [[ -x /usr/local/bin/aw-prune-local-state-rust ]]; then + if [[ $# -eq 0 ]]; then + exec /usr/local/bin/aw-prune-local-state-rust --apply + fi + exec /usr/local/bin/aw-prune-local-state-rust "$@" +fi -prune_group() { - local keep_last="$1" - local keep_days="$2" - shift 2 - local files=() - local idx=0 - local cutoff - cutoff="$(date -d "-${keep_days} days" +%s)" - mapfile -t files < <(find "$@" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk '{ $1=""; sub(/^ /,""); print }') - for path in "${files[@]}"; do - idx=$((idx + 1)) - if [ "$idx" -le "$keep_last" ]; then - continue - fi - [ -f "$path" ] || continue - if [ "$(stat -c %Y "$path")" -lt "$cutoff" ]; then - rm -f -- "$path" - fi - done -} - -mkdir -p "$BACKUP_DIR" - -prune_group "$KEEP_LAST_DB" "$KEEP_DAYS" "${BACKUP_DIR}/db" -prune_group "$KEEP_LAST_JSON" "$KEEP_DAYS" "$BACKUP_DIR" - -find /tmp -maxdepth 1 -type f \ - \( -name 'activitywatch-*.zip' -o -name 'hayabusa-*.zip' -o -name 'aw-hayabusa-profiles.txt' \) \ - -mtime +0 -delete 2>/dev/null || true - -find /tmp -maxdepth 1 -type f \ - \( -name 'aw-worktime-ui-bridge.py' -o -name 'views-default.json' -o -name 'apply_webui_ru_patch.out' \) \ - -mtime +1 -delete 2>/dev/null || true +exec /opt/activitywatch/aw-rus-ops/aw-prune-local-state.sh "$@" diff --git a/aw-server/aw-rus-healthd.py b/aw-server/aw-rus-healthd.py index 234b90c..07d0538 100644 --- a/aw-server/aw-rus-healthd.py +++ b/aw-server/aw-rus-healthd.py @@ -8,6 +8,7 @@ import socket import subprocess import sys import tempfile +import time from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -21,7 +22,12 @@ ENV_FILE = Path("/etc/activitywatch/aw-server.env") def load_env_file(path: Path) -> None: if not path.exists(): return - for raw_line in path.read_text(encoding="utf-8").splitlines(): + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + # systemd EnvironmentFile has already injected the variables for service runs. + return + for raw_line in lines: line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue @@ -55,9 +61,19 @@ def age_seconds(ts: datetime | None, now: datetime) -> int | None: return max(0, int((now - ts).total_seconds())) -def http_json(url: str, timeout: int = 10) -> Any: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) +def http_json(url: str, timeout: int = 20, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: + last_exc: Exception | None = None + for attempt in range(max(1, attempts)): + try: + with request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception as exc: + last_exc = exc + if attempt + 1 >= max(1, attempts): + break + time.sleep(backoff_seconds * (2**attempt)) + assert last_exc is not None + raise last_exc def run_command(cmd: list[str]) -> tuple[int, str]: @@ -136,7 +152,7 @@ class Report: def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: - events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=20") + events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1", timeout=25) if isinstance(events, list) and events: events = [item for item in events if isinstance(item, dict)] if not events: @@ -146,6 +162,16 @@ def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: return None +def bucket_metadata_ts(buckets: dict[str, Any], bucket_id: str) -> datetime | None: + bucket = buckets.get(bucket_id) if isinstance(buckets, dict) else None + if not isinstance(bucket, dict): + return None + metadata = bucket.get("metadata") + if not isinstance(metadata, dict): + return None + return parse_ts(metadata.get("end")) + + def host_activity_from_worktime(event: dict[str, Any] | None, max_age_seconds: int) -> dict[str, Any]: now = now_utc() if not event: @@ -189,6 +215,78 @@ def bucket_health( return "ok", f"fresh ({age}s)", details +def bucket_timestamp_health( + api_base: str, + buckets: dict[str, Any], + bucket_id: str, + max_age_seconds: int, + missing_status: str, + stale_status: str, +) -> tuple[str, str, dict[str, Any]]: + metadata_ts = bucket_metadata_ts(buckets, bucket_id) + if metadata_ts is not None: + age = age_seconds(metadata_ts, now_utc()) + timestamp = metadata_ts.isoformat().replace("+00:00", "Z") + details = { + "bucket": bucket_id, + "timestamp": timestamp, + "age_seconds": age, + "timestamp_source": "bucket_metadata.end", + } + if age is None: + return "warn", "timestamp parse failed", details + if age > max_age_seconds: + return stale_status, f"stale ({age}s)", details + return "ok", f"fresh ({age}s)", details + + return bucket_health(api_base, bucket_id, max_age_seconds, missing_status, stale_status) + + +def guard_bucket_health( + api_base: str, + host: str, + max_age_seconds: int, + required: bool, +) -> tuple[str, str, dict[str, Any]]: + bucket_id = f"aw-rus-collector-guard_{host}" + try: + event = latest_bucket_event(api_base, bucket_id) + except Exception as exc: + status = "fail" if required else "warn" + return status, f"guard bucket query failed: {exc}", {"bucket": bucket_id} + + if not event: + status = "fail" if required else "warn" + return status, "no guard heartbeat", {"bucket": bucket_id, "required": required} + + ts = parse_ts(event.get("timestamp")) + age = age_seconds(ts, now_utc()) + data = event.get("data") if isinstance(event.get("data"), dict) else {} + guard_status = str(data.get("status") or "unknown").lower() + details = { + "bucket": bucket_id, + "timestamp": event.get("timestamp"), + "age_seconds": age, + "required": required, + "guard_status": guard_status, + "mode": data.get("mode"), + "live_session_count": data.get("liveSessionCount"), + "problems": data.get("problems", []), + "actions": data.get("actions", []), + } + + stale_status = "fail" if required else "warn" + if age is None: + return "warn", "guard timestamp parse failed", details + if age > max_age_seconds: + return stale_status, f"guard stale ({age}s)", details + if guard_status in ("fail", "error"): + return "fail" if required else "warn", f"guard reports {guard_status}", details + if guard_status == "warn": + return "warn", "guard reports warn", details + return "ok", f"guard fresh ({age}s)", details + + def latest_validation_report(validation_dir: Path) -> Path | None: candidates = sorted( (path for path in validation_dir.glob("*-aw_validate_ansible.json") if path.is_file()), @@ -206,7 +304,20 @@ def write_atomic(path: Path, content: str) -> None: os.replace(tmp_name, path) -def check_wrapper(report: Report, name: str, cmd: list[str], json_mode: bool = False) -> None: +def chmod_if_possible(path: Path, mode: int) -> None: + try: + path.chmod(mode) + except OSError: + pass + + +def check_wrapper( + report: Report, + name: str, + cmd: list[str], + json_mode: bool = False, + failure_status: str = "fail", +) -> None: if not Path(cmd[0]).exists(): report.add(name, "warn", "binary missing", command=cmd) return @@ -221,7 +332,7 @@ def check_wrapper(report: Report, name: str, cmd: list[str], json_mode: bool = F return else: details["output"] = output - report.add(name, "ok" if rc == 0 else "fail", "passed" if rc == 0 else "failed", **details) + report.add(name, "ok" if rc == 0 else failure_status, "passed" if rc == 0 else "failed", **details) def main() -> int: @@ -240,6 +351,8 @@ def main() -> int: parser.add_argument("--session-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_MAX_AGE_SECONDS", "900"))) parser.add_argument("--interactive-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_INTERACTIVE_MAX_AGE_SECONDS", "900"))) parser.add_argument("--session-events-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS", "86400"))) + parser.add_argument("--guard-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS", "300"))) + parser.add_argument("--guard-required", action="store_true", default=env("AW_RUS_HEALTH_GUARD_REQUIRED", "0").lower() in ("1", "true", "yes")) parser.add_argument("--validation-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_VALIDATION_MAX_AGE_SECONDS", "259200"))) parser.add_argument("--tcp-timeout-seconds", type=float, default=float(env("AW_RUS_HEALTH_TCP_TIMEOUT_SECONDS", "3"))) parser.add_argument("--json", action="store_true") @@ -250,8 +363,14 @@ def main() -> int: if not aw_api_base.endswith("/api/0"): aw_api_base = aw_api_base.rstrip("/") + "/api/0" - check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"]) - check_wrapper(report, "wrapper:dlp-health-check", ["/usr/local/bin/dlp-health-check", "--json"], json_mode=True) + check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"], failure_status="warn") + check_wrapper( + report, + "wrapper:dlp-health-check", + ["/usr/local/bin/dlp-health-check", "--json"], + json_mode=True, + failure_status="warn", + ) try: info = http_json(f"{aw_api_base}/info") @@ -279,6 +398,14 @@ def main() -> int: buckets = {} host = args.rdp_hostname + guard_status, guard_summary, guard_details = guard_bucket_health( + aw_api_base, + host, + args.guard_max_age_seconds, + args.guard_required, + ) + report.add("bucket:collector-guard", guard_status, guard_summary, **guard_details) + worktime_bucket = f"aw-worktime-sessions_{host}" worktime_event = None if buckets: @@ -315,10 +442,15 @@ def main() -> int: ) details["interactive_required"] = interactive_required details["host_activity"] = activity + if not interactive_required and status != "ok": + details["inactive_summary"] = summary + status = "ok" + summary = "inactive: no active interactive users" report.add(label, status, summary, **details) - session_status, session_summary, session_details = bucket_health( + session_status, session_summary, session_details = bucket_timestamp_health( aw_api_base, + buckets, f"aw-session-events_{host}", args.session_events_max_age_seconds, missing_status="fail", @@ -362,8 +494,12 @@ def main() -> int: payload = report.as_dict() state_dir = Path(args.state_dir) - write_atomic(state_dir / "aw-rus-health.json", json.dumps(payload, ensure_ascii=False, indent=2) + "\n") - write_atomic(state_dir / "aw-rus-health.txt", report.render_text() + "\n") + health_json_path = state_dir / "aw-rus-health.json" + health_txt_path = state_dir / "aw-rus-health.txt" + write_atomic(health_json_path, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + write_atomic(health_txt_path, report.render_text() + "\n") + chmod_if_possible(health_json_path, 0o644) + chmod_if_possible(health_txt_path, 0o644) if args.json: print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/aw-server/aw-rus-healthd.timer b/aw-server/aw-rus-healthd.timer index bc6f25b..08e74a0 100644 --- a/aw-server/aw-rus-healthd.timer +++ b/aw-server/aw-rus-healthd.timer @@ -2,10 +2,11 @@ Description=Run AW-RUS unified health orchestrator every 2 minutes [Timer] -OnBootSec=2min -OnUnitActiveSec=2min +OnCalendar=*:0/2:25 +AccuracySec=15s +RandomizedDelaySec=15s Unit=aw-rus-healthd.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target diff --git a/aw-server/aw-server.env.example b/aw-server/aw-server.env.example index 58b1309..a44ffd7 100755 --- a/aw-server/aw-server.env.example +++ b/aw-server/aw-server.env.example @@ -43,7 +43,7 @@ AW_HEALTH_CHECK_ENABLED=true AW_HEALTH_CHECK_INTERVAL=60 AW_EXPECT_START_OF_DAY=00:00 AW_EXPECT_ALWAYS_ACTIVE_PATTERN=aw-watcher-window -AW_EXPECT_LANDINGPAGE=/activity/SHARKON2025/view/ +AW_EXPECT_LANDINGPAGE=/#/activity/SHARKON2025/view/ AW_HEALTH_STRICT_FILEOPS=0 AW_MONITORED_WINDOWS_HOST=192.168.100.18 AW_MONITORED_WINDOWS_HOSTNAME=SHARKON2025 @@ -51,6 +51,19 @@ AW_RUS_HEALTH_WORKTIME_API=http://127.0.0.1:5610 AW_RUS_HEALTH_STATE_DIR=/var/lib/activitywatch/health AW_RUS_HEALTH_VALIDATION_DIR=/var/lib/activitywatch/health/windows-validation AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS=86400 +AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS=300 +AW_RUS_HEALTH_GUARD_REQUIRED=1 +AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600 +AW_RUS_SLO_WORKTIME_BASE=http://127.0.0.1:5610 +AW_RUS_SLO_TARGET_PERCENT=99.97 +AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600 +AW_BROWSER_SMOKE_WORKTIME_BASE=http://127.0.0.1:5610 +AW_BROWSER_SMOKE_HOST=SHARKON2025 +AW_BROWSER_SMOKE_OUTPUT_DIR=/var/lib/activitywatch/browser-smoke +AW_BROWSER_SMOKE_KEEP_RUNS=24 +AW_BROWSER_SMOKE_ENGINE=chromium-cli +AW_BROWSER_SMOKE_TIMEOUT_MS=20000 +AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS=15000 # Hayabusa auto-case / alerting AW_HAYABUSA_AUTO_CASE_ENABLED=true diff --git a/aw-server/aw-slo-monitor-rust-switch.conf b/aw-server/aw-slo-monitor-rust-switch.conf new file mode 100644 index 0000000..8d6835e --- /dev/null +++ b/aw-server/aw-slo-monitor-rust-switch.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/local/bin/aw-slo-monitor-rust diff --git a/aw-server/aw-slo-monitor.py b/aw-server/aw-slo-monitor.py new file mode 100644 index 0000000..854d6ff --- /dev/null +++ b/aw-server/aw-slo-monitor.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib import request + + +ENV_FILE = Path("/etc/activitywatch/aw-server.env") +DEFAULT_STATE_DIR = Path("/var/lib/activitywatch/slo") +DEFAULT_HEALTHD_CMD = "/usr/local/bin/aw-rus-healthd.py --json" +DEFAULT_HEALTHD_STATE_FILE = Path("/var/lib/activitywatch/health/aw-rus-health.json") + + +def load_env_file(path: Path) -> None: + if not path.exists(): + return + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + # systemd EnvironmentFile has already loaded the variables for the service. + return + for raw_line in lines: + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ.setdefault(key.strip(), value.strip().strip("'").strip('"')) + + +def env(name: str, default: str) -> str: + value = os.environ.get(name) + return value if value not in (None, "") else default + + +def now_utc() -> datetime: + return datetime.now(UTC) + + +def parse_ts(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) + + +def iso(dt: datetime) -> str: + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def write_atomic(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: + handle.write(content) + tmp_name = handle.name + os.replace(tmp_name, path) + + +def chmod_if_possible(path: Path, mode: int) -> None: + try: + path.chmod(mode) + except OSError: + pass + + +def chown_like_parent_if_possible(path: Path) -> None: + try: + parent_stat = path.parent.stat() + os.chown(path, parent_stat.st_uid, parent_stat.st_gid) + except OSError: + pass + + +def fetch_url(url: str, timeout_seconds: float, *, accept: str | None = None, attempts: int = 1) -> dict[str, Any]: + started = now_utc() + last_error = "" + for attempt in range(1, max(1, attempts) + 1): + try: + req = request.Request(url, headers={"Accept": accept}) if accept else url + with request.urlopen(req, timeout=timeout_seconds) as resp: + body = resp.read() + status = int(resp.status) + content_type = resp.headers.get("Content-Type", "") + ok = 200 <= status < 300 + return { + "ok": ok, + "status": status, + "body_bytes": len(body), + "content_type": content_type, + "body": body, + "attempts": attempt, + "latency_ms": int((now_utc() - started).total_seconds() * 1000), + "url": url, + } + except Exception as exc: + last_error = str(exc) + if attempt < max(1, attempts): + time.sleep(1) + return { + "ok": False, + "error": last_error, + "body_bytes": 0, + "attempts": max(1, attempts), + "latency_ms": int((now_utc() - started).total_seconds() * 1000), + "url": url, + } + + +def public_probe_result(result: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in result.items() if key != "body"} + + +def http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: + return public_probe_result(fetch_url(url, timeout_seconds, attempts=2)) + + +def html_probe( + url: str, + timeout_seconds: float, + *, + min_bytes: int, + required_markers: tuple[str, ...], +) -> dict[str, Any]: + result = fetch_url(url, timeout_seconds, accept="text/html", attempts=2) + if not result.get("ok"): + return public_probe_result(result) + + body = result.get("body", b"") + text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) + missing = [marker for marker in required_markers if marker not in text] + content_type = str(result.get("content_type") or "") + if len(body) < min_bytes: + result["ok"] = False + result["error"] = f"body too small: {len(body)} < {min_bytes}" + elif "text/html" not in content_type.lower(): + result["ok"] = False + result["error"] = f"unexpected content-type: {content_type or 'unknown'}" + elif missing: + result["ok"] = False + result["error"] = "missing markers: " + ", ".join(missing) + result["missing_markers"] = missing + return public_probe_result(result) + + +def json_probe( + url: str, + timeout_seconds: float, + *, + expected_values: dict[str, Any], + required_keys: tuple[str, ...], +) -> dict[str, Any]: + result = fetch_url(url, timeout_seconds, accept="application/json", attempts=2) + if not result.get("ok"): + return public_probe_result(result) + + body = result.get("body", b"") + try: + payload = json.loads(body.decode("utf-8") if isinstance(body, bytes) else str(body)) + except Exception as exc: + result["ok"] = False + result["error"] = f"invalid json: {exc}" + return public_probe_result(result) + + if not isinstance(payload, dict): + result["ok"] = False + result["error"] = "json root is not object" + return public_probe_result(result) + + missing_keys = [key for key in required_keys if key not in payload] + mismatched = { + key: {"expected": expected, "actual": payload.get(key)} + for key, expected in expected_values.items() + if payload.get(key) != expected + } + if missing_keys: + result["ok"] = False + result["error"] = "missing json keys: " + ", ".join(missing_keys) + result["missing_keys"] = missing_keys + elif mismatched: + result["ok"] = False + result["error"] = "unexpected json values" + result["mismatched_values"] = mismatched + else: + result["json_keys"] = sorted(payload.keys()) + return public_probe_result(result) + + +def run_healthd(command: str, timeout_seconds: int) -> dict[str, Any]: + try: + proc = subprocess.run( + command, + shell=True, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + return { + "ok": False, + "returncode": None, + "error": f"timeout after {timeout_seconds}s", + "output_tail": (exc.stdout or "")[-1000:] if isinstance(exc.stdout, str) else "", + } + + payload: dict[str, Any] = {} + try: + payload = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + pass + return { + "ok": proc.returncode == 0 and bool(payload.get("ok")), + "returncode": proc.returncode, + "counts": payload.get("counts", {}), + "payload": payload if isinstance(payload, dict) else {}, + "output_tail": (proc.stdout or "")[-1000:], + } + + +def read_healthd_state(path: Path, max_age_seconds: int) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generated = parse_ts(str(payload["generated_at_utc"])) + age = max(0, int((now_utc() - generated).total_seconds())) + except Exception as exc: + return {"ok": False, "error": str(exc), "path": str(path)} + return { + "ok": bool(payload.get("ok")) and age <= max_age_seconds, + "counts": payload.get("counts", {}), + "age_seconds": age, + "path": str(path), + "payload": payload if isinstance(payload, dict) else {}, + } + + +def load_samples(path: Path, cutoff: datetime) -> list[dict[str, Any]]: + samples: list[dict[str, Any]] = [] + if not path.exists(): + return samples + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + return samples + for line in lines: + if not line.strip(): + continue + try: + item = json.loads(line) + ts = parse_ts(str(item["ts"])) + except Exception: + continue + if ts >= cutoff: + samples.append(item) + return samples + + +def append_and_trim_sample(path: Path, sample: dict[str, Any], retention_seconds: int) -> list[dict[str, Any]]: + cutoff = parse_ts(sample["ts"]) - timedelta(seconds=retention_seconds) + samples = load_samples(path, cutoff) + samples.append(sample) + content = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n" for item in samples) + write_atomic(path, content) + chmod_if_possible(path, 0o644) + chown_like_parent_if_possible(path) + return samples + + +def summarize_window( + samples: list[dict[str, Any]], + *, + now: datetime, + window_seconds: int, + sample_interval_seconds: int, + target_percent: float, +) -> dict[str, Any]: + cutoff = now - timedelta(seconds=window_seconds) + window = [] + for item in samples: + try: + if parse_ts(str(item["ts"])) >= cutoff: + window.append(item) + except Exception: + continue + + total = len(window) + good = sum(1 for item in window if item.get("ok") is True) + bad = total - good + availability = round((good / total) * 100, 5) if total else None + observed_bad_seconds = bad * sample_interval_seconds + budget_seconds = int(window_seconds * ((100.0 - target_percent) / 100.0)) + budget_remaining_seconds = budget_seconds - observed_bad_seconds + status = "unknown" + if total: + status = "ok" if budget_remaining_seconds >= 0 else "burning" + return { + "window_seconds": window_seconds, + "samples": total, + "good_samples": good, + "bad_samples": bad, + "availability_percent": availability, + "target_percent": target_percent, + "observed_bad_seconds": observed_bad_seconds, + "budget_seconds": budget_seconds, + "budget_remaining_seconds": budget_remaining_seconds, + "status": status, + } + + +def render_summary_text(summary: dict[str, Any]) -> str: + lines = [ + "=== AW-RUS SLO ===", + f"Timestamp: {summary['generated_at_utc']}", + f"Target: {summary['target_percent']}%", + "", + ] + for name, data in summary["windows"].items(): + availability = data["availability_percent"] + availability_text = "n/a" if availability is None else f"{availability:.5f}%" + remaining = data["budget_remaining_seconds"] + lines.append( + f"{name}: {data['status']} availability={availability_text} " + f"samples={data['samples']} bad={data['bad_samples']} " + f"bad_seconds={data['observed_bad_seconds']} budget_remaining_seconds={remaining}" + ) + lines.append("") + lines.append(f"Current sample: {'OK' if summary['current_sample']['ok'] else 'FAIL'}") + for name, probe in summary["current_sample"].get("probes", {}).items(): + marker = "OK" if probe.get("ok") else "FAIL" + detail = probe.get("status", probe.get("error", "")) + lines.append(f"- {name}: {marker} {detail}") + return "\n".join(lines) + + +def build_sample(args: argparse.Namespace) -> dict[str, Any]: + ts = iso(now_utc()) + if args.healthd_mode == "run": + health = run_healthd(args.healthd_cmd, args.health_timeout_seconds) + else: + health = read_healthd_state(Path(args.healthd_state_file), args.healthd_state_max_age_seconds) + probes = { + "aw_webui_index": html_probe( + args.aw_webui_url, + args.http_timeout_seconds, + min_bytes=1000, + required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), + ), + "worktime_today_html": html_probe( + args.worktime_today_html_url, + args.http_timeout_seconds, + min_bytes=5000, + required_markers=("AW-rus", ""), + ), + "worktime_management_html": html_probe( + args.worktime_management_html_url, + args.http_timeout_seconds, + min_bytes=5000, + required_markers=("AW-rus", ""), + ), + "worktime_today_csv": http_probe(args.worktime_today_csv_url, args.http_timeout_seconds), + "worktime_management_json": json_probe( + args.worktime_management_json_url, + args.http_timeout_seconds, + expected_values={"host": args.host}, + required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), + ), + } + ok = bool(health.get("ok")) and all(probe.get("ok") for probe in probes.values()) + return { + "ts": ts, + "ok": ok, + "healthd_ok": bool(health.get("ok")), + "healthd_counts": health.get("counts", {}), + "probes": probes, + } + + +def main() -> int: + load_env_file(ENV_FILE) + parser = argparse.ArgumentParser(description="AW-RUS rolling SLO sampler") + parser.add_argument("--state-dir", default=env("AW_RUS_SLO_STATE_DIR", str(DEFAULT_STATE_DIR))) + parser.add_argument("--healthd-cmd", default=env("AW_RUS_SLO_HEALTHD_CMD", DEFAULT_HEALTHD_CMD)) + parser.add_argument("--healthd-mode", choices=["state", "run"], default=env("AW_RUS_SLO_HEALTHD_MODE", "state")) + parser.add_argument("--healthd-state-file", default=env("AW_RUS_SLO_HEALTHD_STATE_FILE", str(DEFAULT_HEALTHD_STATE_FILE))) + parser.add_argument("--healthd-state-max-age-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", "180"))) + parser.add_argument("--aw-base", default=env("AW_RUS_SLO_AW_BASE", env("AW_SERVER_URL", "http://127.0.0.1:5600"))) + parser.add_argument("--worktime-base", default=env("AW_RUS_SLO_WORKTIME_BASE", env("AW_RUS_HEALTH_WORKTIME_API", "http://127.0.0.1:5610"))) + parser.add_argument("--host", default=env("AW_RUS_SLO_HOST", env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025"))) + parser.add_argument("--target-percent", type=float, default=float(env("AW_RUS_SLO_TARGET_PERCENT", "99.97"))) + parser.add_argument("--sample-interval-seconds", type=int, default=int(env("AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", "15"))) + parser.add_argument("--retention-days", type=int, default=int(env("AW_RUS_SLO_RETENTION_DAYS", "35"))) + parser.add_argument("--http-timeout-seconds", type=float, default=float(env("AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", "15"))) + parser.add_argument("--health-timeout-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", "90"))) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + aw_base = args.aw_base.rstrip("/") + worktime_base = args.worktime_base.rstrip("/") + args.aw_webui_url = f"{aw_base}/" + args.worktime_today_html_url = f"{worktime_base}/reports/worktime/today?format=html&day=today&host={args.host}&allow_stale=1" + args.worktime_management_html_url = f"{worktime_base}/reports/worktime/management?format=html&day=today&host={args.host}&allow_stale=1" + args.worktime_today_csv_url = f"{worktime_base}/reports/worktime/today?format=csv&day=today&host={args.host}&allow_stale=1" + args.worktime_management_json_url = f"{worktime_base}/reports/worktime/management?format=json&day=today&host={args.host}&allow_stale=1" + + state_dir = Path(args.state_dir) + sample_path = state_dir / "aw-slo-samples.jsonl" + sample = build_sample(args) + samples = append_and_trim_sample( + sample_path, + sample, + retention_seconds=max(1, args.retention_days) * 86400, + ) + + generated_at = now_utc() + summary = { + "generated_at_utc": iso(generated_at), + "target_percent": args.target_percent, + "sample_interval_seconds": args.sample_interval_seconds, + "current_sample": sample, + "windows": { + "24h": summarize_window( + samples, + now=generated_at, + window_seconds=86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + "7d": summarize_window( + samples, + now=generated_at, + window_seconds=7 * 86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + "30d": summarize_window( + samples, + now=generated_at, + window_seconds=30 * 86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + }, + } + summary_json_path = state_dir / "aw-slo-summary.json" + summary_txt_path = state_dir / "aw-slo-summary.txt" + write_atomic(summary_json_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") + text = render_summary_text(summary) + write_atomic(summary_txt_path, text + "\n") + chmod_if_possible(summary_json_path, 0o644) + chmod_if_possible(summary_txt_path, 0o644) + chown_like_parent_if_possible(summary_json_path) + chown_like_parent_if_possible(summary_txt_path) + if args.json: + print(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + print(text) + return 0 if sample["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/aw-server/aw-slo-monitor.service b/aw-server/aw-slo-monitor.service new file mode 100644 index 0000000..a1bd724 --- /dev/null +++ b/aw-server/aw-slo-monitor.service @@ -0,0 +1,16 @@ +[Unit] +Description=AW-RUS SLO sampler +After=network-online.target activitywatch-server.service aw-worktime-api.service aw-rus-healthd.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +ExecStart=/usr/bin/python3 /usr/local/bin/aw-slo-monitor.py +SuccessExitStatus=1 +User=activitywatch +Group=activitywatch +Nice=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-slo-monitor diff --git a/aw-server/aw-slo-monitor.timer b/aw-server/aw-slo-monitor.timer new file mode 100644 index 0000000..b5e2ce3 --- /dev/null +++ b/aw-server/aw-slo-monitor.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AW-RUS SLO sampler every minute + +[Timer] +OnCalendar=*:0/1:50 +AccuracySec=10s +RandomizedDelaySec=10s +Persistent=false +Unit=aw-slo-monitor.service + +[Install] +WantedBy=timers.target diff --git a/aw-server/aw-worktime-api.py b/aw-server/aw-worktime-api.py index f12324a..caba965 100644 --- a/aw-server/aw-worktime-api.py +++ b/aw-server/aw-worktime-api.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import base64 import csv +import hashlib import html import io import importlib.util @@ -8,6 +10,7 @@ import os import sys import tempfile import threading +import time import urllib.request from datetime import datetime, timezone, timedelta from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -49,18 +52,30 @@ MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", " MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()} EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30"))) WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000"))) +AW_HTTP_TIMEOUT_SECONDS = max(0.5, float(os.environ.get("AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS", "5"))) +REPORT_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_REPORT_CACHE_TTL_SECONDS", "60"))) +REPORT_STALE_TTL_SECONDS = max(REPORT_CACHE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_STALE_TTL_SECONDS", "900"))) +REPORT_DISK_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_REPORT_DISK_CACHE_DIR", "/var/lib/activitywatch/worktime-report-cache")) +REPORT_DISK_STALE_TTL_SECONDS = max(REPORT_STALE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS", "86400"))) +REPORT_BUSY_WAIT_SECONDS = max(0.0, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_SECONDS", "8"))) +REPORT_BUSY_WAIT_INTERVAL_SECONDS = max(0.05, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_INTERVAL_SECONDS", "0.2"))) +REPORT_MAX_CONCURRENT = max(1, int(os.environ.get("AW_WORKTIME_REPORT_MAX_CONCURRENT", "1"))) TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180"))) TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600"))) MODULE_PATH = Path(__file__).resolve() _ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}} _EVENTS_CACHE_LOCK = threading.Lock() _EVENTS_CACHE = {} +_REPORT_CACHE_LOCK = threading.Lock() +_REPORT_RESPONSE_CACHE = {} +_REPORT_REFRESHING = set() +_REPORT_BUILD_SEMAPHORE = threading.BoundedSemaphore(REPORT_MAX_CONCURRENT) _MANAGEMENT_BUILD_LOCKS_LOCK = threading.Lock() _MANAGEMENT_BUILD_LOCKS = {} def get(u): - with urllib.request.urlopen(u, timeout=30) as r: + with urllib.request.urlopen(u, timeout=AW_HTTP_TIMEOUT_SECONDS) as r: return json.loads(r.read().decode()) @@ -784,15 +799,21 @@ def aggregate_hourly_rows(events, start, end, host): def fetch_events_for_date(host, report_date): bounds = get_report_bounds(report_date) bucket_id = get_sessions_bucket_id(host) - events = fetch_bucket_events(bucket_id, host) + events = fetch_bucket_events(bucket_id, host, start=bounds["start"], end=bounds["end"]) return bounds, events -def fetch_bucket_events(bucket_id, host): +def fetch_bucket_events(bucket_id, host, start=None, end=None): now = now_utc() + cache_key = bucket_id + params = {"limit": WORKTIME_EVENTS_LIMIT} + if start is not None and end is not None: + params["start"] = to_iso_utc(start) + params["end"] = to_iso_utc(end) + cache_key = f"{bucket_id}|{params['start']}|{params['end']}" if EVENTS_CACHE_TTL_SECONDS > 0: with _EVENTS_CACHE_LOCK: - cached = _EVENTS_CACHE.get(bucket_id) + cached = _EVENTS_CACHE.get(cache_key) if cached is not None and age_seconds(cached["stored_at"], now=now) <= EVENTS_CACHE_TTL_SECONDS: return cached["events"] try: @@ -801,13 +822,13 @@ def fetch_bucket_events(bucket_id, host): log_warning(f"bucket lookup failed for host={host} bucket={bucket_id} aw_base={AW}") return [] try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit={WORKTIME_EVENTS_LIMIT}") + events = get(f"{AW}/buckets/{bucket_id}/events?{urlencode(params)}") except Exception: log_warning(f"events fetch failed for host={host} bucket={bucket_id} aw_base={AW}") return [] if EVENTS_CACHE_TTL_SECONDS > 0: with _EVENTS_CACHE_LOCK: - _EVENTS_CACHE[bucket_id] = {"stored_at": now, "events": events} + _EVENTS_CACHE[cache_key] = {"stored_at": now, "events": events} return events @@ -840,16 +861,22 @@ def build_report_summary(rows): def latest_bucket_event(bucket_id): try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit=20") + events = get(f"{AW}/buckets/{bucket_id}/events?limit=1") + except Exception: + events = None + if isinstance(events, list) and events: + valid = [item for item in events if isinstance(item, dict)] + if valid: + valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) + return valid[0] + try: + bucket = get(f"{AW}/buckets/{bucket_id}") except Exception: return None - if not isinstance(events, list) or not events: - return None - valid = [item for item in events if isinstance(item, dict)] - if not valid: - return None - valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return valid[0] + end = ((bucket or {}).get("metadata") or {}).get("end") + if end: + return {"timestamp": end, "duration": 0, "data": {"source": "bucket_metadata"}} + return None def _priority_rank(priority): @@ -1219,6 +1246,7 @@ def _source_status_label(status): "ok": "fresh", "warn": "stale", "fail": "missing", + "inactive": "inactive", }.get(status, status) @@ -1273,7 +1301,224 @@ def save_management_cache(host, report_date, payload): write_atomic_json(management_cache_path(host, report_date), payload) -def build_source_freshness(host): +def make_report_cache_key(path, fmt, host, report_date, selected_day="", owner_filter="", department_filter=""): + return "|".join( + [ + str(path), + str(fmt), + resolve_host(host), + report_date.isoformat(), + str(selected_day or ""), + normalize_management_filter(owner_filter), + normalize_management_filter(department_filter), + ] + ) + + +def resolve_report_format(params, accept_header=""): + requested = str(params.get("format", [""])[0] or "").lower() + if requested in {"csv", "html", "json"}: + return requested + accept = str(accept_header or "").lower() + if "text/html" in accept and "application/json" not in accept: + return "html" + return "json" + + +def report_disk_cache_path(key): + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + return REPORT_DISK_CACHE_DIR / f"{digest}.json" + + +def load_report_response_disk_cache(key): + if REPORT_DISK_STALE_TTL_SECONDS <= 0: + return None + path = report_disk_cache_path(key) + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + stored_epoch = float(payload["stored_epoch"]) + age = max(0, int(time.time() - stored_epoch)) + if age > REPORT_DISK_STALE_TTL_SECONDS: + return None + data = base64.b64decode(payload["data_b64"].encode("ascii")) + content_type = str(payload["content_type"]) + except Exception as exc: + log_warning(f"failed to load report disk cache key={key}: {exc}") + return None + return { + "stored_monotonic": time.monotonic() - age, + "generated_at_utc": payload.get("generated_at_utc", ""), + "data": data, + "content_type": content_type, + "age_seconds": age, + "stale": True, + "cache_source": "disk", + } + + +def save_report_response_disk_cache(key, data, content_type, generated_at_utc): + if REPORT_DISK_STALE_TTL_SECONDS <= 0: + return + payload = { + "key": key, + "stored_epoch": time.time(), + "generated_at_utc": generated_at_utc, + "content_type": content_type, + "data_b64": base64.b64encode(data).decode("ascii"), + } + try: + write_atomic_json(report_disk_cache_path(key), payload) + except Exception as exc: + log_warning(f"failed to save report disk cache key={key}: {exc}") + + +def get_report_response_cache(key, *, allow_stale=False): + if REPORT_CACHE_TTL_SECONDS <= 0 and not allow_stale: + return None + now = time.monotonic() + with _REPORT_CACHE_LOCK: + cached = _REPORT_RESPONSE_CACHE.get(key) + if cached is not None: + age = now - cached["stored_monotonic"] + max_age = REPORT_STALE_TTL_SECONDS if allow_stale else REPORT_CACHE_TTL_SECONDS + if max_age > 0 and age <= max_age: + result = dict(cached) + result["age_seconds"] = int(age) + result["stale"] = age > REPORT_CACHE_TTL_SECONDS + result["cache_source"] = "memory" + return result + if allow_stale: + return load_report_response_disk_cache(key) + return None + + +def save_report_response_cache(key, data, content_type): + if REPORT_STALE_TTL_SECONDS <= 0: + return + generated_at_utc = now_utc().isoformat().replace("+00:00", "Z") + with _REPORT_CACHE_LOCK: + _REPORT_RESPONSE_CACHE[key] = { + "stored_monotonic": time.monotonic(), + "generated_at_utc": generated_at_utc, + "data": data, + "content_type": content_type, + } + save_report_response_disk_cache(key, data, content_type, generated_at_utc) + + +def trigger_report_background_refresh(key, path, params): + refresh_params = {name: list(values) for name, values in params.items()} + with _REPORT_CACHE_LOCK: + if key in _REPORT_REFRESHING: + return False + _REPORT_REFRESHING.add(key) + + def refresh(): + try: + if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): + return + try: + data, content_type = build_worktime_report_response(path, refresh_params) + save_report_response_cache(key, data, content_type) + except Exception as exc: + log_warning(f"background report refresh failed path={path}: {exc}") + finally: + _REPORT_BUILD_SEMAPHORE.release() + finally: + with _REPORT_CACHE_LOCK: + _REPORT_REFRESHING.discard(key) + + thread = threading.Thread(target=refresh, name="worktime-report-refresh", daemon=True) + thread.start() + return True + + +def send_cached_report(handler, cached, *, reason): + handler.send_response(200) + handler.send_header("Content-Type", cached["content_type"]) + handler.send_header("Content-Length", str(len(cached["data"]))) + handler.send_header("X-AW-Worktime-Cache", "stale" if cached.get("stale") else "fresh") + handler.send_header("X-AW-Worktime-Cache-Reason", reason) + handler.send_header("X-AW-Worktime-Cache-Age", str(cached.get("age_seconds", 0))) + handler.send_header("X-AW-Worktime-Cache-Source", cached.get("cache_source", "memory")) + handler.end_headers() + try: + handler.wfile.write(cached["data"]) + except (BrokenPipeError, ConnectionResetError): + return False + return True + + +def send_report_unavailable(handler, message): + payload = { + "ok": False, + "error": "report_unavailable", + "message": message, + "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), + } + data = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") + send_bytes(handler, data, "application/json; charset=utf-8", status=503) + + +def send_worktime_report(handler, path, params, cache_key, *, host, report_date): + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl") + return + + allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} + if allow_immediate_stale: + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + refresh_stale = str(params.get("refresh_stale", [""])[0]).lower() in {"1", "true", "yes"} + if refresh_stale: + trigger_report_background_refresh(cache_key, path, params) + send_cached_report(handler, cached, reason="stale-allowed") + return + + if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): + if not allow_immediate_stale and REPORT_BUSY_WAIT_SECONDS > 0: + deadline = time.monotonic() + REPORT_BUSY_WAIT_SECONDS + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(REPORT_BUSY_WAIT_INTERVAL_SECONDS, remaining)) + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl-after-wait") + return + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + send_cached_report(handler, cached, reason="busy") + return + send_report_unavailable(handler, "report builder is busy and no cached response is available") + return + + try: + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl-after-lock") + return + try: + data, content_type = build_worktime_report_response(path, params) + except Exception as exc: + log_warning(f"report build failed path={path} host={host} date={report_date}: {exc}") + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + send_cached_report(handler, cached, reason="build-error") + return + send_report_unavailable(handler, "report build failed and no cached response is available") + return + save_report_response_cache(cache_key, data, content_type) + send_bytes(handler, data, content_type) + finally: + _REPORT_BUILD_SEMAPHORE.release() + + +def build_source_freshness(host, interactive_required=True): source_specs = [ { "source_id": "worktime_sessions", @@ -1305,6 +1550,7 @@ def build_source_freshness(host): "bucket_candidates": [f"aw-watcher-window_{host}"], "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, "required": True, + "interactive_only": True, "owner": "ops", }, { @@ -1321,6 +1567,7 @@ def build_source_freshness(host): "bucket_candidates": [f"aw-file-operations_{host}"], "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, "required": True, + "interactive_only": True, "owner": "ops", }, { @@ -1379,6 +1626,9 @@ def build_source_freshness(host): else: status = "ok" summary = f"fresh ({age}s)" + if spec.get("interactive_only") and not interactive_required and status != "ok": + status = "inactive" + summary = "inactive: no active interactive users" detail = { "source_id": spec["source_id"], "label": spec["label"], @@ -1388,12 +1638,14 @@ def build_source_freshness(host): "timestamp": (matched_event or {}).get("timestamp", ""), "age_seconds": age, "required": spec["required"], + "interactive_only": bool(spec.get("interactive_only")), + "interactive_required": bool(interactive_required), "max_age_seconds": spec["max_age_seconds"], "summary": summary, "event_summary": _source_summary(matched_event), } sources.append(detail) - if status == "ok": + if status in {"ok", "inactive"}: continue priority = "critical" if spec["required"] else "medium" actions.append( @@ -1723,7 +1975,8 @@ def build_management_payload(rows, host, report_date, owner_filter="", departmen owner_filter = normalize_management_filter(owner_filter) department_filter = normalize_management_filter(department_filter) payload = _build_management_core(rows, host, report_date, owner_filter=owner_filter, department_filter=department_filter) - source_freshness, source_actions = build_source_freshness(resolve_host(host)) + interactive_required = int(payload.get("summary", {}).get("active_users", 0) or 0) > 0 + source_freshness, source_actions = build_source_freshness(resolve_host(host), interactive_required=interactive_required) payload["sources"] = source_freshness payload["trend"] = build_management_trend( resolve_host(host), @@ -1751,11 +2004,11 @@ def report_for_date(host, report_date): return aggregate_rows(events, bounds["start"], bounds["end"], host) -def management_report_for_date(host, report_date, owner_filter="", department_filter=""): +def management_report_for_date(host, report_date, owner_filter="", department_filter="", use_cache=True): owner_filter = normalize_management_filter(owner_filter) department_filter = normalize_management_filter(department_filter) if owner_filter or department_filter: - base_payload = management_report_for_date(host, report_date) + base_payload = management_report_for_date(host, report_date, use_cache=use_cache) filtered_payload = apply_management_filters_to_payload( base_payload, owner_filter=owner_filter, @@ -1766,14 +2019,17 @@ def management_report_for_date(host, report_date, owner_filter="", department_fi filtered_payload["trend"] = [] filtered_payload["trend_scope"] = "filtered_current_only" return filtered_payload - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - lock = get_management_build_lock(host, report_date) - with lock: + cache_allowed = use_cache or report_date < datetime.now(REPORT_TZ).date() + if cache_allowed: cached = load_management_cache(host, report_date) if cached is not None: return cached + lock = get_management_build_lock(host, report_date) + with lock: + if cache_allowed: + cached = load_management_cache(host, report_date) + if cached is not None: + return cached bounds, events = fetch_events_for_date(host, report_date) rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) payload = build_management_payload(rows, host, report_date) @@ -2265,10 +2521,11 @@ def render_management_html(payload, selected_day=None): sources_html = [] for source in payload.get("sources", []): + source_priority_class = "low" if source["status"] in {"ok", "inactive"} else ("critical" if source["required"] else "medium") sources_html.append( "" f"{html.escape(source['label'])}" - f"{html.escape(source['status_label'])}" + f"{html.escape(source['status_label'])}" f"{html.escape(source['bucket_id'])}" f"{html.escape(source.get('timestamp') or '-')}" f"{html.escape(str(source.get('age_seconds')) if source.get('age_seconds') is not None else '-')}" @@ -2619,6 +2876,85 @@ def render_management_html(payload, selected_day=None): """ +def build_worktime_report_response(path, params): + fmt = resolve_report_format(params) + host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) + day = params.get("day", ["today"])[0] + date_text = params.get("date", [None])[0] + owner_filter = normalize_management_filter(params.get("owner", [""])[0]) + department_filter = normalize_management_filter(params.get("department", [""])[0]) + report_date = resolve_report_date(day=day, date_text=date_text) + is_management = path == "/reports/worktime/management" + allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} + management_payload = ( + management_report_for_date( + host, + report_date, + owner_filter=owner_filter, + department_filter=department_filter, + use_cache=allow_immediate_stale, + ) + if is_management + else None + ) + rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] + true_active_apps = [] if is_management else build_true_active_apps(host, report_date) + + if fmt == "csv": + if is_management: + out = io.StringIO() + writer = csv.DictWriter( + out, + fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], + extrasaction="ignore", + ) + writer.writeheader() + writer.writerows(management_payload["actions"]) + return out.getvalue().encode(), "text/csv; charset=utf-8" + out = io.StringIO() + writer = csv.DictWriter( + out, + fieldnames=[ + "user", + "user_id", + "active_seconds", + "active_hhmm", + "first_activity", + "last_activity", + "idle_seconds", + "sessions_count", + "samples_count", + "active_samples", + ], + ) + writer.writeheader() + writer.writerows(rows) + return out.getvalue().encode(), "text/csv; charset=utf-8" + + if fmt == "html": + if is_management: + data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") + else: + data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") + return data, "text/html; charset=utf-8" + + if is_management: + data = json.dumps(management_payload, ensure_ascii=False, indent=2).encode("utf-8") + return data, "application/json; charset=utf-8" + + obj = { + "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "report_timezone": str(REPORT_TZ), + "host": host, + "report_date": report_date.isoformat(), + "bucket_id": get_sessions_bucket_id(host), + "rows": rows, + "true_active_apps": true_active_apps, + } + data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") + return data, "application/json; charset=utf-8" + + def send_bytes(handler, data, content_type, status=200): handler.send_response(status) handler.send_header("Content-Type", content_type) @@ -2667,82 +3003,27 @@ class H(BaseHTTPRequestHandler): return params = parse_qs(parsed.query, keep_blank_values=False) - fmt = "json" - if params.get("format", ["json"])[0] == "csv": - fmt = "csv" - elif params.get("format", ["json"])[0] == "html": - fmt = "html" + fmt = resolve_report_format(params, self.headers.get("Accept", "")) + if not params.get("format") and fmt != "json": + params = {name: list(values) for name, values in params.items()} + params["format"] = [fmt] host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) day = params.get("day", ["today"])[0] date_text = params.get("date", [None])[0] owner_filter = normalize_management_filter(params.get("owner", [""])[0]) department_filter = normalize_management_filter(params.get("department", [""])[0]) report_date = resolve_report_date(day=day, date_text=date_text) - is_management = parsed.path == "/reports/worktime/management" - management_payload = management_report_for_date(host, report_date, owner_filter=owner_filter, department_filter=department_filter) if is_management else None - rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] - true_active_apps = [] if is_management else build_true_active_apps(host, report_date) + cache_key = make_report_cache_key( + parsed.path, + fmt, + host, + report_date, + selected_day=day if day in {"today", "yesterday"} else "", + owner_filter=owner_filter, + department_filter=department_filter, + ) - if fmt == "csv": - if is_management: - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], - extrasaction="ignore", - ) - writer.writeheader() - writer.writerows(management_payload["actions"]) - data = out.getvalue().encode() - send_bytes(self, data, "text/csv; charset=utf-8") - return - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=[ - "user", - "user_id", - "active_seconds", - "active_hhmm", - "first_activity", - "last_activity", - "idle_seconds", - "sessions_count", - "samples_count", - "active_samples", - ], - ) - writer.writeheader() - writer.writerows(rows) - data = out.getvalue().encode() - send_bytes(self, data, "text/csv; charset=utf-8") - return - - if fmt == "html": - if is_management: - data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") - else: - data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") - send_bytes(self, data, "text/html; charset=utf-8") - return - - if is_management: - obj = management_payload - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") - return - - obj = { - "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "host": host, - "report_date": report_date.isoformat(), - "bucket_id": get_sessions_bucket_id(host), - "rows": rows, - "true_active_apps": true_active_apps, - } - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") + send_worktime_report(self, parsed.path, params, cache_key, host=host, report_date=report_date) class WorktimeHTTPServer(ThreadingHTTPServer): diff --git a/aw-server/aw-worktime-autoheal.timer b/aw-server/aw-worktime-autoheal.timer index f70199e..58b9c76 100644 --- a/aw-server/aw-worktime-autoheal.timer +++ b/aw-server/aw-worktime-autoheal.timer @@ -2,11 +2,10 @@ Description=Run AW Worktime Autoheal every 3 minutes [Timer] -OnBootSec=2min -OnUnitActiveSec=3min +OnCalendar=*:1/3:35 +AccuracySec=30s Unit=aw-worktime-autoheal.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target - diff --git a/aw-server/aw-worktime-influx-exporter.timer b/aw-server/aw-worktime-influx-exporter.timer index a2091e6..967e581 100644 --- a/aw-server/aw-worktime-influx-exporter.timer +++ b/aw-server/aw-worktime-influx-exporter.timer @@ -2,9 +2,9 @@ Description=Run AW Worktime InfluxDB exporter every 10 minutes [Timer] -OnBootSec=3min -OnUnitActiveSec=10min +OnCalendar=*:4/10:05 AccuracySec=1min +RandomizedDelaySec=30s Unit=aw-worktime-influx-exporter.service [Install] diff --git a/aw-server/aw-worktime-prewarm.service b/aw-server/aw-worktime-prewarm.service new file mode 100644 index 0000000..a013770 --- /dev/null +++ b/aw-server/aw-worktime-prewarm.service @@ -0,0 +1,15 @@ +[Unit] +Description=AW Worktime Report Cache Prewarm +After=network-online.target activitywatch-server.service aw-worktime-api.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +ExecStart=/usr/local/bin/aw-worktime-prewarm.sh +User=activitywatch +Group=activitywatch +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-worktime-prewarm + diff --git a/aw-server/aw-worktime-prewarm.sh b/aw-server/aw-worktime-prewarm.sh new file mode 100644 index 0000000..7ae33e9 --- /dev/null +++ b/aw-server/aw-worktime-prewarm.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +WORKTIME_BASE_URL="${WORKTIME_BASE_URL:-http://127.0.0.1:5610}" +WORKTIME_PREWARM_TIMEOUT_SECONDS="${WORKTIME_PREWARM_TIMEOUT_SECONDS:-45}" +WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS="${WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS:-10}" +WORKTIME_PREWARM_READY_TIMEOUT_SECONDS="${WORKTIME_PREWARM_READY_TIMEOUT_SECONDS:-60}" +WORKTIME_PREWARM_READY_INTERVAL_SECONDS="${WORKTIME_PREWARM_READY_INTERVAL_SECONDS:-2}" +WORKTIME_PREWARM_HOST="${WORKTIME_PREWARM_HOST:-${AW_WORKTIME_HOST:-SHARKON2025}}" +WORKTIME_PREWARM_PROFILE="${WORKTIME_PREWARM_PROFILE:-full}" + +log() { + printf '%s %s\n' "$(date '+%F %T')" "$*" +} + +probe() { + local url="$1" + local timeout="$2" + local tmp + tmp="$(mktemp)" + local code + code="$(curl -sS --max-time "$timeout" -o /dev/null -D "$tmp" -w '%{http_code}' "$url" 2>/dev/null || true)" + if [[ "$code" =~ ^2 ]]; then + local cache + cache="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache:/ {print $2; exit}' || true)" + local reason + reason="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache-reason:/ {print $2; exit}' || true)" + rm -f "$tmp" + log "ok code=$code cache=${cache:-none} reason=${reason:-none} url=$url" + return 0 + fi + rm -f "$tmp" + log "warn code=${code:-000} url=$url" + return 1 +} + +wait_until_ready() { + local deadline + deadline=$(( $(date +%s) + WORKTIME_PREWARM_READY_TIMEOUT_SECONDS )) + while true; do + if probe "$WORKTIME_BASE_URL/health" "$WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS"; then + return 0 + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + return 1 + fi + sleep "$WORKTIME_PREWARM_READY_INTERVAL_SECONDS" + done +} + +if ! wait_until_ready; then + log "health readiness timed out; skip prewarm" + exit 0 +fi + +full_urls=( + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=html&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=html&host=$WORKTIME_PREWARM_HOST" +) + +startup_urls=( + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" +) + +case "$WORKTIME_PREWARM_PROFILE" in + full) + urls=("${full_urls[@]}") + ;; + startup) + urls=("${startup_urls[@]}") + ;; + *) + log "unknown profile=$WORKTIME_PREWARM_PROFILE" + exit 0 + ;; +esac + +failures=0 +for url in "${urls[@]}"; do + if ! probe "$url" "$WORKTIME_PREWARM_TIMEOUT_SECONDS"; then + failures=$((failures + 1)) + fi +done + +if [[ "$failures" -gt 0 ]]; then + log "completed profile=$WORKTIME_PREWARM_PROFILE with failures=$failures" +else + log "completed profile=$WORKTIME_PREWARM_PROFILE successfully" +fi + +exit 0 diff --git a/aw-server/aw-worktime-prewarm.timer b/aw-server/aw-worktime-prewarm.timer new file mode 100644 index 0000000..5167be5 --- /dev/null +++ b/aw-server/aw-worktime-prewarm.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AW Worktime Report Cache Prewarm + +[Timer] +OnCalendar=*:2/5:40 +AccuracySec=30s +Persistent=false +Unit=aw-worktime-prewarm.service + +[Install] +WantedBy=timers.target diff --git a/aw-server/aw-worktime-ui-bridge.timer b/aw-server/aw-worktime-ui-bridge.timer index d56cb22..768215a 100644 --- a/aw-server/aw-worktime-ui-bridge.timer +++ b/aw-server/aw-worktime-ui-bridge.timer @@ -2,10 +2,10 @@ Description=Run AW Worktime UI bridge every 30 seconds [Timer] -OnBootSec=20s -OnUnitActiveSec=30s +OnCalendar=*:0/1:15 +AccuracySec=10s Unit=aw-worktime-ui-bridge.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target diff --git a/aw-server/check-aw-data-wrapper.sh b/aw-server/check-aw-data-wrapper.sh new file mode 100644 index 0000000..8e4ab1f --- /dev/null +++ b/aw-server/check-aw-data-wrapper.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -x /usr/local/bin/check-aw-data-rust ]]; then + if [[ -z "${AW_CHECK_SERVER:-}" && -z "${AW_SERVER_URL:-}" ]]; then + for arg in "$@"; do + if [[ "$arg" == "--server" || "$arg" == --server=* ]]; then + exec /usr/local/bin/check-aw-data-rust "$@" + fi + done + exec /usr/local/bin/check-aw-data-rust --server http://127.0.0.1:5600 "$@" + fi + exec /usr/local/bin/check-aw-data-rust "$@" +fi + +exec /opt/activitywatch/aw-rus-ops/check-aw-data.sh "$@" diff --git a/aw-server/dlp-content-analysis/aw-dlp-content-analyzer.sh b/aw-server/dlp-content-analysis/aw-dlp-content-analyzer.sh index 8cd33dd..ad42440 100644 --- a/aw-server/dlp-content-analysis/aw-dlp-content-analyzer.sh +++ b/aw-server/dlp-content-analysis/aw-dlp-content-analyzer.sh @@ -2,9 +2,14 @@ set -euo pipefail BASE_DIR="/opt/activitywatch/dlp-content-analysis" +RUST_ANALYZER="/usr/local/bin/aw-dlp-content-analyzer-rust" VENV_PY="$BASE_DIR/.venv/bin/python" ANALYZER="$BASE_DIR/content_analyzer.py" +if [ -x "$RUST_ANALYZER" ]; then + exec "$RUST_ANALYZER" "$@" +fi + if [ ! -x "$VENV_PY" ]; then echo "ERROR: content-analysis virtualenv is missing: $VENV_PY" >&2 exit 1 diff --git a/aw-server/dlp-integrations/cef-exporter.timer b/aw-server/dlp-integrations/cef-exporter.timer index 4582564..3f2d312 100644 --- a/aw-server/dlp-integrations/cef-exporter.timer +++ b/aw-server/dlp-integrations/cef-exporter.timer @@ -2,10 +2,11 @@ Description=Run AWatch DLP CEF Exporter every 5 minutes [Timer] -OnBootSec=2min -OnUnitActiveSec=5min +OnCalendar=*:6/10:30 +AccuracySec=30s +RandomizedDelaySec=30s Unit=aw-dlp-cef-exporter.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target diff --git a/aw-server/dlp-integrations/syslog-forwarder.timer b/aw-server/dlp-integrations/syslog-forwarder.timer index 218a308..331ee11 100644 --- a/aw-server/dlp-integrations/syslog-forwarder.timer +++ b/aw-server/dlp-integrations/syslog-forwarder.timer @@ -2,9 +2,10 @@ Description=Run AWatch DLP syslog forwarder every 3 minutes [Timer] -OnBootSec=3min -OnUnitActiveSec=3min -Persistent=true +OnCalendar=*:7/10:40 +AccuracySec=30s +RandomizedDelaySec=30s +Persistent=false Unit=aw-dlp-syslog-forwarder.service [Install] diff --git a/aw-server/dlp-integrations/syslog_forwarder.py b/aw-server/dlp-integrations/syslog_forwarder.py index 8c32616..4ecec71 100644 --- a/aw-server/dlp-integrations/syslog_forwarder.py +++ b/aw-server/dlp-integrations/syslog_forwarder.py @@ -57,7 +57,7 @@ def iter_new_incidents(aw_base: str, state: dict[str, Any], per_bucket_limit: in for bid in bucket_ids: try: events = http_json(f"{aw_base}/buckets/{bid}/events?limit={int(per_bucket_limit)}") - except error.HTTPError as exc: + except (TimeoutError, OSError, error.URLError, error.HTTPError) as exc: LOG.warning("skip bucket %s: %s", bid, exc) continue prev = int(last_ids.get(bid, 0)) @@ -126,7 +126,11 @@ def main() -> None: app_name = str(cfg.get("app_name", "aw-dlp")) state = load_json(state_path) - incidents, max_ids = iter_new_incidents(aw_base=aw_base, state=state, per_bucket_limit=per_bucket_limit) + try: + incidents, max_ids = iter_new_incidents(aw_base=aw_base, state=state, per_bucket_limit=per_bucket_limit) + except (TimeoutError, OSError, error.URLError, error.HTTPError) as exc: + LOG.warning("skip syslog forwarder run: AW API unavailable: %s", exc) + return sent = 0 for event in incidents: diff --git a/aw-server/dlp-integrations/test_syslog_forwarder.py b/aw-server/dlp-integrations/test_syslog_forwarder.py new file mode 100644 index 0000000..446cbe3 --- /dev/null +++ b/aw-server/dlp-integrations/test_syslog_forwarder.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import sys +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("syslog_forwarder.py") +SPEC = importlib.util.spec_from_file_location("syslog_forwarder", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def test_iter_new_incidents_skips_timed_out_bucket(monkeypatch): + def fake_http_json(url, timeout=15): + if url.endswith("/buckets/"): + return {"aw-dlp-incidents_SHARKON2025": {}} + raise TimeoutError("timed out") + + monkeypatch.setattr(MODULE, "http_json", fake_http_json) + + incidents, max_ids = MODULE.iter_new_incidents( + aw_base="http://127.0.0.1:5600/api/0", + state={"last_ids": {"aw-dlp-incidents_SHARKON2025": 42}}, + per_bucket_limit=300, + ) + + assert incidents == [] + assert max_ids == {} + + +def test_main_skips_aw_api_timeout_without_overwriting_state(monkeypatch, tmp_path): + state_path = tmp_path / "syslog-forwarder-state.json" + original_state = {"last_ids": {"aw-dlp-incidents_SHARKON2025": 99}} + state_path.write_text(json.dumps(original_state), encoding="utf-8") + + monkeypatch.setattr( + MODULE, + "load_yaml", + lambda path: { + "aw_api_base": "http://127.0.0.1:5600/api/0", + "state_path": str(state_path), + }, + ) + monkeypatch.setattr( + MODULE, + "iter_new_incidents", + lambda aw_base, state, per_bucket_limit: (_ for _ in ()).throw(TimeoutError("timed out")), + ) + monkeypatch.setattr( + MODULE, + "save_json", + lambda path, payload: (_ for _ in ()).throw(AssertionError("state should not be saved on AW API timeout")), + ) + + MODULE.main() + + assert json.loads(state_path.read_text(encoding="utf-8")) == original_state diff --git a/aw-server/dlp-integrations/webhook-sender.timer b/aw-server/dlp-integrations/webhook-sender.timer index 29ed438..f64e60d 100644 --- a/aw-server/dlp-integrations/webhook-sender.timer +++ b/aw-server/dlp-integrations/webhook-sender.timer @@ -2,10 +2,11 @@ Description=Run AWatch DLP Webhook sender every 2 minutes [Timer] -OnBootSec=90s -OnUnitActiveSec=2min +OnCalendar=*:8/10:45 +AccuracySec=30s +RandomizedDelaySec=30s Unit=aw-dlp-webhook-sender.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target diff --git a/aw-server/hayabusa/aw-hayabusa.sh b/aw-server/hayabusa/aw-hayabusa.sh index 4b017a2..b82a326 100644 --- a/aw-server/hayabusa/aw-hayabusa.sh +++ b/aw-server/hayabusa/aw-hayabusa.sh @@ -417,7 +417,7 @@ process_one_package() { fail "normalized zip extraction failed for ${package_path}" fi - local manifest_path host evtx_root archive_pkg_dir archive_pkg_path archive_extract_dir status report_dir + local manifest_path host evtx_root archive_pkg_dir archive_extract_dir status report_dir manifest_path="$(find_manifest_path "${stage_dir}")" host="${forced_host}" if [ -z "${host}" ] && [ -f "${package_path}.host" ]; then diff --git a/aw-server/health-check.sh b/aw-server/health-check.sh index 5fba2e8..7df5693 100644 --- a/aw-server/health-check.sh +++ b/aw-server/health-check.sh @@ -16,6 +16,10 @@ fi WORKTIME_HEALTH_BASE="${AW_RUS_HEALTH_WORKTIME_API:-${AW_WORKTIME_REPORT_BASE:-http://127.0.0.1:5610}}" WORKTIME_HEALTH_URL="${WORKTIME_HEALTH_BASE%/}/health" WORKTIME_HEALTH_TIMEOUT_SECONDS="${AW_RUS_HEALTH_WORKTIME_TIMEOUT_SECONDS:-15}" +AW_HEALTH_API_TIMEOUT_SECONDS="${AW_RUS_HEALTH_AW_TIMEOUT_SECONDS:-15}" +AW_HEALTH_API_ATTEMPTS="${AW_RUS_HEALTH_AW_ATTEMPTS:-3}" +SETTINGS_TIMEOUT_SECONDS="${AW_RUS_HEALTH_SETTINGS_TIMEOUT_SECONDS:-15}" +SETTINGS_ATTEMPTS="${AW_RUS_HEALTH_SETTINGS_ATTEMPTS:-3}" check_service() { local service=$1 @@ -59,8 +63,17 @@ check_api_endpoint() { read_setting_value() { local key=$1 - curl -fsS --max-time 10 "http://127.0.0.1:5600/api/0/settings/${key}" 2>/dev/null | \ - python3 -c 'import json,sys; print(json.load(sys.stdin))' + local attempt + for ((attempt=1; attempt<=SETTINGS_ATTEMPTS; attempt++)); do + if curl -fsS --connect-timeout 3 --max-time "$SETTINGS_TIMEOUT_SECONDS" "http://127.0.0.1:5600/api/0/settings/${key}" 2>/dev/null | \ + python3 -c 'import json,sys; print(json.load(sys.stdin))'; then + return 0 + fi + if (( attempt < SETTINGS_ATTEMPTS )); then + sleep 1 + fi + done + return 1 } check_expected_setting() { @@ -139,7 +152,7 @@ done echo # Check API endpoints -check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" +check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" "$AW_HEALTH_API_TIMEOUT_SECONDS" "$AW_HEALTH_API_ATTEMPTS" check_api_endpoint "$WORKTIME_HEALTH_URL" "aw-worktime-api" "$WORKTIME_HEALTH_TIMEOUT_SECONDS" 2 check_dlp_transport_freshness "http://127.0.0.1:5600/api/0" "900" "${AW_HEALTH_STRICT_FILEOPS:-0}" check_expected_setting "startOfDay" "${AW_EXPECT_START_OF_DAY:-}" "startOfDay" diff --git a/aw-server/test_aw_rus_healthd.py b/aw-server/test_aw_rus_healthd.py index 211e3e5..72fad82 100644 --- a/aw-server/test_aw_rus_healthd.py +++ b/aw-server/test_aw_rus_healthd.py @@ -38,6 +38,33 @@ def test_bucket_health_reports_missing_events_without_crash(monkeypatch): assert details["bucket"] == "aw-watcher-window_SHARKON2025" +def test_inactive_interactive_bucket_can_be_downgraded_to_ok(monkeypatch): + monkeypatch.setattr( + MODULE, + "latest_bucket_event", + lambda api_base, bucket_id: {"timestamp": "2026-05-30T07:00:00Z"}, + ) + monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T12:00:00Z")) + activity = {"active": False, "fresh": True, "age_seconds": 5} + status, summary, details = MODULE.bucket_health( + "http://127.0.0.1:5600/api/0", + "aw-watcher-window_SHARKON2025", + 900, + missing_status="warn", + stale_status="warn", + ) + details["interactive_required"] = bool(activity["active"]) + details["host_activity"] = activity + if not activity["active"] and status != "ok": + details["inactive_summary"] = summary + status = "ok" + summary = "inactive: no active interactive users" + + assert status == "ok" + assert summary == "inactive: no active interactive users" + assert details["inactive_summary"] == "stale (18000s)" + + def test_bucket_health_marks_old_session_events_as_event_driven(monkeypatch): monkeypatch.setattr( MODULE, @@ -58,3 +85,127 @@ def test_bucket_health_marks_old_session_events_as_event_driven(monkeypatch): assert status == "ok" assert summary.startswith("event-driven (") assert details["age_seconds"] > 86400 + + +def test_bucket_timestamp_health_uses_bucket_metadata_without_event_query(monkeypatch): + def fail_if_called(api_base, bucket_id): + raise AssertionError("latest event query should not run when bucket metadata has end timestamp") + + monkeypatch.setattr(MODULE, "latest_bucket_event", fail_if_called) + monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T17:05:00Z")) + + status, summary, details = MODULE.bucket_timestamp_health( + "http://127.0.0.1:5600/api/0", + {"aw-session-events_SHARKON2025": {"metadata": {"end": "2026-05-30T17:00:00Z"}}}, + "aw-session-events_SHARKON2025", + 86400, + missing_status="fail", + stale_status="warn", + ) + + assert status == "ok" + assert summary == "fresh (300s)" + assert details["timestamp_source"] == "bucket_metadata.end" + assert details["age_seconds"] == 300 + + +def test_bucket_timestamp_health_falls_back_to_event_query_without_metadata(monkeypatch): + monkeypatch.setattr( + MODULE, + "latest_bucket_event", + lambda api_base, bucket_id: {"timestamp": "2026-05-30T17:00:00Z"}, + ) + monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T17:05:00Z")) + + status, summary, details = MODULE.bucket_timestamp_health( + "http://127.0.0.1:5600/api/0", + {"aw-session-events_SHARKON2025": {}}, + "aw-session-events_SHARKON2025", + 86400, + missing_status="fail", + stale_status="warn", + ) + + assert status == "ok" + assert summary == "fresh (300s)" + assert "timestamp_source" not in details + + +def test_guard_bucket_missing_is_warn_until_required(monkeypatch): + monkeypatch.setattr(MODULE, "latest_bucket_event", lambda api_base, bucket_id: None) + status, summary, details = MODULE.guard_bucket_health( + "http://127.0.0.1:5600/api/0", + "SHARKON2025", + 300, + required=False, + ) + assert status == "warn" + assert summary == "no guard heartbeat" + assert details["required"] is False + + +def test_guard_bucket_required_stale_is_fail(monkeypatch): + monkeypatch.setattr( + MODULE, + "latest_bucket_event", + lambda api_base, bucket_id: { + "timestamp": "2026-05-30T10:00:00Z", + "data": {"status": "ok", "mode": "enforce"}, + }, + ) + monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T10:10:00Z")) + status, summary, details = MODULE.guard_bucket_health( + "http://127.0.0.1:5600/api/0", + "SHARKON2025", + 300, + required=True, + ) + assert status == "fail" + assert summary == "guard stale (600s)" + assert details["mode"] == "enforce" + + +def test_aw_wrapper_failure_can_be_advisory_warn(tmp_path): + report = MODULE.Report() + missing = tmp_path / "missing-wrapper" + MODULE.check_wrapper(report, "wrapper:test", [str(missing)], failure_status="warn") + assert report.results[0].status == "warn" + + +def test_dlp_wrapper_failure_can_be_advisory_warn(monkeypatch, tmp_path): + wrapper = tmp_path / "dlp-health-check" + wrapper.write_text("#!/bin/sh\nprintf '{\"ok\":false}'\nexit 1\n", encoding="utf-8") + wrapper.chmod(0o755) + monkeypatch.setattr(MODULE, "run_command", lambda cmd: (1, '{"ok": false}')) + + report = MODULE.Report() + MODULE.check_wrapper( + report, + "wrapper:dlp-health-check", + [str(wrapper), "--json"], + json_mode=True, + failure_status="warn", + ) + + assert report.results[0].status == "warn" + assert report.results[0].details["payload"] == {"ok": False} + + +def test_load_env_file_ignores_permission_denied(): + class DeniedPath: + def exists(self): + return True + + def read_text(self, encoding="utf-8"): + raise PermissionError("denied") + + MODULE.load_env_file(DeniedPath()) + + +def test_chmod_if_possible_sets_readable_mode(tmp_path): + path = tmp_path / "aw-rus-health.json" + path.write_text("{}", encoding="utf-8") + + MODULE.chmod_if_possible(path, 0o644) + + assert oct(path.stat().st_mode & 0o777) == "0o644" diff --git a/aw-server/test_aw_slo_monitor.py b/aw-server/test_aw_slo_monitor.py new file mode 100644 index 0000000..00b55ee --- /dev/null +++ b/aw-server/test_aw_slo_monitor.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +import importlib.util +import sys +import tempfile +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("aw-slo-monitor.py") +SPEC = importlib.util.spec_from_file_location("aw_slo_monitor", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class FakeHeaders: + def __init__(self, values): + self.values = values + + def get(self, key, default=""): + return self.values.get(key, default) + + +class FakeResponse: + def __init__(self, body, status=200, content_type="text/plain"): + self.body = body + self.status = status + self.headers = FakeHeaders({"Content-Type": content_type}) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self): + return self.body + + +def test_summarize_window_calculates_9997_budget(): + now = MODULE.parse_ts("2026-05-30T12:00:00Z") + samples = [ + {"ts": "2026-05-30T11:58:00Z", "ok": True}, + {"ts": "2026-05-30T11:59:00Z", "ok": False}, + {"ts": "2026-05-30T12:00:00Z", "ok": True}, + ] + + summary = MODULE.summarize_window( + samples, + now=now, + window_seconds=86400, + sample_interval_seconds=60, + target_percent=99.97, + ) + + assert summary["samples"] == 3 + assert summary["good_samples"] == 2 + assert summary["bad_samples"] == 1 + assert summary["availability_percent"] == 66.66667 + assert summary["budget_seconds"] == 25 + assert summary["budget_remaining_seconds"] == -35 + assert summary["status"] == "burning" + + +def test_summarize_window_status_uses_remaining_error_budget_for_partial_window(): + now = MODULE.parse_ts("2026-05-30T12:00:00Z") + samples = [ + {"ts": "2026-05-30T11:59:30Z", "ok": False}, + {"ts": "2026-05-30T11:59:45Z", "ok": True}, + {"ts": "2026-05-30T12:00:00Z", "ok": True}, + ] + + summary = MODULE.summarize_window( + samples, + now=now, + window_seconds=86400, + sample_interval_seconds=15, + target_percent=99.97, + ) + + assert summary["availability_percent"] == 66.66667 + assert summary["budget_seconds"] == 25 + assert summary["budget_remaining_seconds"] == 10 + assert summary["status"] == "ok" + + +def test_append_and_trim_sample_keeps_retention_window(): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "samples.jsonl" + path.write_text( + '{"ts":"2026-05-01T00:00:00Z","ok":true}\n' + '{"ts":"2026-05-30T11:59:00Z","ok":true}\n', + encoding="utf-8", + ) + sample = {"ts": "2026-05-30T12:00:00Z", "ok": False} + + samples = MODULE.append_and_trim_sample(path, sample, retention_seconds=3600) + + assert [item["ts"] for item in samples] == [ + "2026-05-30T11:59:00Z", + "2026-05-30T12:00:00Z", + ] + assert "2026-05-01T00:00:00Z" not in path.read_text(encoding="utf-8") + + +def test_load_samples_treats_unreadable_file_as_empty(monkeypatch, tmp_path): + path = tmp_path / "samples.jsonl" + path.write_text('{"ts":"2026-05-30T12:00:00Z","ok":true}\n', encoding="utf-8") + original_read_text = MODULE.Path.read_text + + def denied(self, *args, **kwargs): + if self == path: + raise PermissionError("denied") + return original_read_text(self, *args, **kwargs) + + monkeypatch.setattr(MODULE.Path, "read_text", denied) + + assert MODULE.load_samples(path, MODULE.parse_ts("2026-05-30T11:00:00Z")) == [] + + +def test_render_summary_text_includes_budget_remaining(): + summary = { + "generated_at_utc": "2026-05-30T12:00:00Z", + "target_percent": 99.97, + "current_sample": { + "ok": True, + "probes": {"worktime_today_csv": {"ok": True, "status": 200}}, + }, + "windows": { + "24h": { + "status": "ok", + "availability_percent": 100.0, + "samples": 10, + "bad_samples": 0, + "observed_bad_seconds": 0, + "budget_remaining_seconds": 25, + } + }, + } + + text = MODULE.render_summary_text(summary) + + assert "Target: 99.97%" in text + assert "24h: ok availability=100.00000%" in text + assert "budget_remaining_seconds=25" in text + assert "- worktime_today_csv: OK 200" in text + + +def test_html_probe_requires_non_empty_html_with_markers(monkeypatch): + body = b'ActivityWatch
' + monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="text/html; charset=utf-8")) + + result = MODULE.html_probe( + "http://127.0.0.1:5600/", + 5, + min_bytes=100, + required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), + ) + + assert result["ok"] is True + assert result["status"] == 200 + assert result["body_bytes"] == len(body) + assert "body" not in result + + +def test_html_probe_fails_on_missing_marker(monkeypatch): + monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(b"", content_type="text/html")) + + result = MODULE.html_probe( + "http://127.0.0.1:5600/", + 5, + min_bytes=1, + required_markers=("ActivityWatch",), + ) + + assert result["ok"] is False + assert result["missing_markers"] == ["ActivityWatch"] + + +def test_html_probe_retries_transient_fetch_error(monkeypatch): + calls = {"count": 0} + body = b"ActivityWatch" + + def fake_urlopen(req, timeout): + calls["count"] += 1 + if calls["count"] == 1: + raise TimeoutError("timed out") + return FakeResponse(body, content_type="text/html") + + monkeypatch.setattr(MODULE.request, "urlopen", fake_urlopen) + monkeypatch.setattr(MODULE.time, "sleep", lambda seconds: None) + + result = MODULE.html_probe( + "http://127.0.0.1:5600/", + 5, + min_bytes=1, + required_markers=("ActivityWatch",), + ) + + assert result["ok"] is True + assert result["attempts"] == 2 + assert calls["count"] == 2 + + +def test_json_probe_validates_shape_and_expected_values(monkeypatch): + body = b'{"generated_at_utc":"2026-05-30T12:00:00Z","host":"SHARKON2025","summary":{},"rows":[],"workday":{}}' + monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="application/json")) + + result = MODULE.json_probe( + "http://127.0.0.1:5610/reports/worktime/management?format=json", + 5, + expected_values={"host": "SHARKON2025"}, + required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), + ) + + assert result["ok"] is True + assert result["status"] == 200 + assert "body" not in result + + +def test_json_probe_fails_on_wrong_host(monkeypatch): + body = b'{"generated_at_utc":"2026-05-30T12:00:00Z","host":"OTHER","summary":{},"rows":[],"workday":{}}' + monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="application/json")) + + result = MODULE.json_probe( + "http://127.0.0.1:5610/reports/worktime/management?format=json", + 5, + expected_values={"host": "SHARKON2025"}, + required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), + ) + + assert result["ok"] is False + assert result["mismatched_values"]["host"]["actual"] == "OTHER" + + +def test_load_env_file_ignores_permission_denied(monkeypatch): + class DeniedPath: + def exists(self): + return True + + def read_text(self, encoding="utf-8"): + raise PermissionError("denied") + + MODULE.load_env_file(DeniedPath()) + + +def test_read_healthd_state_requires_fresh_ok_snapshot(monkeypatch, tmp_path): + path = tmp_path / "aw-rus-health.json" + path.write_text( + '{"generated_at_utc":"2026-05-30T12:00:00Z","ok":true,"counts":{"ok":1,"warn":0,"fail":0}}', + encoding="utf-8", + ) + monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T12:01:00Z")) + + fresh = MODULE.read_healthd_state(path, max_age_seconds=180) + stale = MODULE.read_healthd_state(path, max_age_seconds=30) + + assert fresh["ok"] is True + assert fresh["age_seconds"] == 60 + assert stale["ok"] is False + + +def test_chmod_if_possible_sets_readable_mode(tmp_path): + path = tmp_path / "summary.json" + path.write_text("{}", encoding="utf-8") + + MODULE.chmod_if_possible(path, 0o644) + + assert oct(path.stat().st_mode & 0o777) == "0o644" diff --git a/aw-server/test_aw_worktime_api.py b/aw-server/test_aw_worktime_api.py index 7e0f031..02acff8 100644 --- a/aw-server/test_aw_worktime_api.py +++ b/aw-server/test_aw_worktime_api.py @@ -168,6 +168,48 @@ def test_build_true_active_apps_requires_foreground_not_afk_and_evidence(): assert by_app["Total Commander"]["last_action"] == "C:\\data\\report.xlsx" +def test_latest_bucket_event_falls_back_to_bucket_metadata_end(monkeypatch): + def fake_get(url): + if url.endswith("/events?limit=1"): + raise RuntimeError("events timeout") + if url.endswith("/buckets/aw-worktime-sessions_SHARKON2025"): + return {"metadata": {"end": "2026-05-30T10:38:07.530Z"}} + raise AssertionError(url) + + monkeypatch.setattr(MODULE, "get", fake_get) + + event = MODULE.latest_bucket_event("aw-worktime-sessions_SHARKON2025") + + assert event["timestamp"] == "2026-05-30T10:38:07.530Z" + assert event["data"]["source"] == "bucket_metadata" + + +def test_fetch_events_for_date_queries_aw_with_report_bounds(monkeypatch): + urls = [] + + def fake_get(url): + urls.append(url) + if url.endswith("/buckets/aw-worktime-sessions_SHARKON2025"): + return {"metadata": {"end": "2026-05-30T10:38:07.530Z"}} + if "/events?" in url: + return [] + raise AssertionError(url) + + monkeypatch.setattr(MODULE, "get", fake_get) + monkeypatch.setattr(MODULE, "REPORT_TZ", timezone.utc) + original_cache = MODULE._EVENTS_CACHE + try: + MODULE._EVENTS_CACHE = {} + MODULE.fetch_events_for_date("SHARKON2025", datetime(2026, 5, 30, tzinfo=timezone.utc).date()) + finally: + MODULE._EVENTS_CACHE = original_cache + + event_urls = [url for url in urls if "/events?" in url] + assert len(event_urls) == 1 + assert "start=2026-05-30T00%3A00%3A00Z" in event_urls[0] + assert "end=2026-05-30T23%3A59%3A59" in event_urls[0] + + def test_render_html_contains_true_active_apps_table(): html = MODULE.render_html( [], @@ -289,7 +331,7 @@ def test_build_management_payload_applies_alias_and_executive_summary(): "notes": "Дневной контур", } } - MODULE.build_source_freshness = lambda host: ([], []) + MODULE.build_source_freshness = lambda host, **kwargs: ([], []) MODULE.build_management_trend = lambda host, report_date, owner_filter="", department_filter="", **kwargs: [] rows = [ { @@ -350,7 +392,7 @@ def test_build_management_payload_filters_by_owner(): "руководитель смены": {"display_name": "Сменный руководитель", "title": "Руководитель смены 1С"} } MODULE.build_management_trend = lambda *args, **kwargs: [] - MODULE.build_source_freshness = lambda host: ([], []) + MODULE.build_source_freshness = lambda host, **kwargs: ([], []) rows = [ { "user": "user1", @@ -697,6 +739,44 @@ def test_build_source_freshness_uses_host_fileops_bucket_only(): MODULE.now_utc = original_now_utc +def test_build_source_freshness_marks_desktop_sources_inactive_without_active_users(): + original = MODULE.latest_bucket_event + original_now_utc = MODULE.now_utc + try: + def fake_latest(bucket_id): + if bucket_id in {"aw-watcher-window_SHARKON2025", "aw-file-operations_SHARKON2025"}: + return { + "timestamp": "2026-05-23T11:00:00Z", + "data": {"signalType": "collector_health", "queueDepth": 0, "sendFailures": 0, "eventsFlushed": 50}, + } + if bucket_id.startswith("aw-worktime-sessions_") or bucket_id.startswith("aw-rdp-window_") or bucket_id.startswith("aw-rdp-afk_"): + return { + "timestamp": "2026-05-23T12:00:00Z", + "data": {"status": "ok"}, + } + return None + + MODULE.latest_bucket_event = fake_latest + MODULE.now_utc = lambda: datetime(2026, 5, 23, 12, 5, 0, tzinfo=timezone.utc) + sources, actions = MODULE.build_source_freshness("SHARKON2025", interactive_required=False) + source_by_id = {source["source_id"]: source for source in sources} + assert source_by_id["watcher_window"]["status"] == "inactive" + assert source_by_id["watcher_window"]["status_label"] == "inactive" + assert source_by_id["file_operations"]["status"] == "inactive" + assert source_by_id["file_operations"]["status_label"] == "inactive" + assert not any(action["evidence"].get("source_id") in {"watcher_window", "file_operations"} for action in actions) + finally: + MODULE.latest_bucket_event = original + MODULE.now_utc = original_now_utc + + +def test_resolve_report_format_uses_html_for_browser_accept_without_breaking_api_default(): + assert MODULE.resolve_report_format({}) == "json" + assert MODULE.resolve_report_format({}, "text/html,application/xhtml+xml") == "html" + assert MODULE.resolve_report_format({"format": ["json"]}, "text/html") == "json" + assert MODULE.resolve_report_format({"format": ["csv"]}, "text/html") == "csv" + + def test_send_bytes_returns_false_on_broken_pipe(): handler = _FakeHandler(exc=BrokenPipeError()) ok = MODULE.send_bytes(handler, b"{}", "application/json; charset=utf-8") @@ -714,3 +794,285 @@ def test_send_bytes_writes_payload_when_client_is_connected(): assert handler.status == 201 assert handler.headers["Content-Length"] == "7" assert handler.wfile.data == b"payload" + + +def test_report_response_cache_returns_fresh_and_stale_entries(): + original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS + original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS + original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR + original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS + original_cache = MODULE._REPORT_RESPONSE_CACHE + original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.REPORT_CACHE_TTL_SECONDS = 1 + MODULE.REPORT_STALE_TTL_SECONDS = 60 + MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) + MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 + MODULE.REPORT_BUSY_WAIT_SECONDS = 0 + MODULE._REPORT_RESPONSE_CACHE = {} + key = MODULE.make_report_cache_key("/reports/worktime/today", "json", "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) + MODULE.save_report_response_cache(key, b"{}", "application/json; charset=utf-8") + fresh = MODULE.get_report_response_cache(key) + assert fresh is not None + assert fresh["stale"] is False + assert fresh["cache_source"] == "memory" + MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 + assert MODULE.get_report_response_cache(key) is None + stale = MODULE.get_report_response_cache(key, allow_stale=True) + assert stale is not None + assert stale["stale"] is True + finally: + MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl + MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl + MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir + MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl + MODULE._REPORT_RESPONSE_CACHE = original_cache + MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait + + +def test_report_response_cache_uses_disk_stale_after_memory_loss(): + original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS + original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS + original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR + original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS + original_cache = MODULE._REPORT_RESPONSE_CACHE + original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.REPORT_CACHE_TTL_SECONDS = 1 + MODULE.REPORT_STALE_TTL_SECONDS = 60 + MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) + MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 + MODULE.REPORT_BUSY_WAIT_SECONDS = 0 + MODULE._REPORT_RESPONSE_CACHE = {} + key = MODULE.make_report_cache_key("/reports/worktime/today", "csv", "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) + MODULE.save_report_response_cache(key, b"user,active_seconds\nuser1,60\n", "text/csv; charset=utf-8") + MODULE._REPORT_RESPONSE_CACHE = {} + + cached = MODULE.get_report_response_cache(key, allow_stale=True) + + assert cached is not None + assert cached["data"].startswith(b"user,active_seconds") + assert cached["content_type"] == "text/csv; charset=utf-8" + assert cached["stale"] is True + assert cached["cache_source"] == "disk" + finally: + MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl + MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl + MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir + MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl + MODULE._REPORT_RESPONSE_CACHE = original_cache + MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait + + +def test_send_cached_report_marks_cache_reason_and_age(): + handler = _FakeHandler() + cached = { + "data": b"cached", + "content_type": "application/json; charset=utf-8", + "age_seconds": 42, + "stale": True, + } + ok = MODULE.send_cached_report(handler, cached, reason="busy") + assert ok is True + assert handler.status == 200 + assert handler.headers["X-AW-Worktime-Cache"] == "stale" + assert handler.headers["X-AW-Worktime-Cache-Reason"] == "busy" + assert handler.headers["X-AW-Worktime-Cache-Age"] == "42" + assert handler.headers["X-AW-Worktime-Cache-Source"] == "memory" + assert handler.wfile.data == b"cached" + + +def test_send_report_unavailable_returns_503_json(): + handler = _FakeHandler() + MODULE.send_report_unavailable(handler, "busy") + payload = json.loads(handler.wfile.data.decode("utf-8")) + assert handler.status == 503 + assert payload["ok"] is False + assert payload["error"] == "report_unavailable" + + +def test_send_worktime_report_rebuilds_stale_cache_when_builder_is_available(monkeypatch): + original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS + original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS + original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR + original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS + original_cache = MODULE._REPORT_RESPONSE_CACHE + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.REPORT_CACHE_TTL_SECONDS = 1 + MODULE.REPORT_STALE_TTL_SECONDS = 60 + MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) + MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 + MODULE._REPORT_RESPONSE_CACHE = {} + report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() + key = MODULE.make_report_cache_key("/reports/worktime/today", "html", "SHARKON2025", report_date) + MODULE.save_report_response_cache(key, b"stale", "text/html; charset=utf-8") + MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 + + calls = [] + + def fake_build(path, params): + calls.append((path, params)) + return b"fresh", "text/html; charset=utf-8" + + monkeypatch.setattr(MODULE, "build_worktime_report_response", fake_build) + handler = _FakeHandler() + + MODULE.send_worktime_report( + handler, + "/reports/worktime/today", + {"format": ["html"]}, + key, + host="SHARKON2025", + report_date=report_date, + ) + + assert calls == [("/reports/worktime/today", {"format": ["html"]})] + assert handler.status == 200 + assert handler.headers["Content-Type"] == "text/html; charset=utf-8" + assert "X-AW-Worktime-Cache" not in handler.headers + assert handler.wfile.data == b"fresh" + finally: + MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl + MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl + MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir + MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl + MODULE._REPORT_RESPONSE_CACHE = original_cache + + +def test_send_worktime_report_serves_stale_cache_only_when_builder_is_busy(monkeypatch): + original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS + original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS + original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR + original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS + original_cache = MODULE._REPORT_RESPONSE_CACHE + original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.REPORT_CACHE_TTL_SECONDS = 1 + MODULE.REPORT_STALE_TTL_SECONDS = 60 + MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) + MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 + MODULE.REPORT_BUSY_WAIT_SECONDS = 0 + MODULE._REPORT_RESPONSE_CACHE = {} + report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() + key = MODULE.make_report_cache_key("/reports/worktime/today", "html", "SHARKON2025", report_date) + MODULE.save_report_response_cache(key, b"stale", "text/html; charset=utf-8") + MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 + + semaphore = MODULE.threading.BoundedSemaphore(1) + assert semaphore.acquire(blocking=False) is True + monkeypatch.setattr(MODULE, "_REPORT_BUILD_SEMAPHORE", semaphore) + monkeypatch.setattr( + MODULE, + "build_worktime_report_response", + lambda path, params: (_ for _ in ()).throw(AssertionError("builder must not run while busy")), + ) + handler = _FakeHandler() + + MODULE.send_worktime_report( + handler, + "/reports/worktime/today", + {"format": ["html"]}, + key, + host="SHARKON2025", + report_date=report_date, + ) + + assert handler.status == 200 + assert handler.headers["X-AW-Worktime-Cache"] == "stale" + assert handler.headers["X-AW-Worktime-Cache-Reason"] == "busy" + assert handler.wfile.data == b"stale" + semaphore.release() + finally: + MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl + MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl + MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir + MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl + MODULE._REPORT_RESPONSE_CACHE = original_cache + MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait + + +def test_send_worktime_report_allows_explicit_machine_stale_without_blocking(monkeypatch): + original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS + original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS + original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR + original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS + original_cache = MODULE._REPORT_RESPONSE_CACHE + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.REPORT_CACHE_TTL_SECONDS = 1 + MODULE.REPORT_STALE_TTL_SECONDS = 60 + MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) + MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 + MODULE._REPORT_RESPONSE_CACHE = {} + report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() + key = MODULE.make_report_cache_key("/reports/worktime/today", "csv", "SHARKON2025", report_date) + MODULE.save_report_response_cache(key, b"stale", "text/csv; charset=utf-8") + MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 + + refreshes = [] + monkeypatch.setattr( + MODULE, + "trigger_report_background_refresh", + lambda key, path, params: refreshes.append((key, path, params)) or True, + ) + monkeypatch.setattr( + MODULE, + "build_worktime_report_response", + lambda path, params: (_ for _ in ()).throw(AssertionError("explicit stale request must not block on build")), + ) + handler = _FakeHandler() + + MODULE.send_worktime_report( + handler, + "/reports/worktime/today", + {"format": ["csv"], "allow_stale": ["1"]}, + key, + host="SHARKON2025", + report_date=report_date, + ) + + assert refreshes == [] + assert handler.status == 200 + assert handler.headers["X-AW-Worktime-Cache"] == "stale" + assert handler.headers["X-AW-Worktime-Cache-Reason"] == "stale-allowed" + assert handler.wfile.data == b"stale" + finally: + MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl + MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl + MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir + MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl + MODULE._REPORT_RESPONSE_CACHE = original_cache + + +def test_management_report_for_date_bypasses_today_payload_cache_for_human_requests(monkeypatch): + original_ttl = MODULE.MANAGER_CACHE_TTL_SECONDS + original_dir = MODULE.MANAGER_CACHE_DIR + try: + with tempfile.TemporaryDirectory() as tmp: + MODULE.MANAGER_CACHE_TTL_SECONDS = 300 + MODULE.MANAGER_CACHE_DIR = Path(tmp) + report_date = datetime.now(timezone.utc).date() + monkeypatch.setattr(MODULE, "REPORT_TZ", timezone.utc) + monkeypatch.setattr(MODULE, "fetch_events_for_date", lambda host, date: ({"start": datetime.combine(date, datetime.min.time(), tzinfo=timezone.utc), "end": datetime.combine(date, datetime.max.time(), tzinfo=timezone.utc)}, [])) + monkeypatch.setattr(MODULE, "aggregate_rows_with_intervals", lambda events, start, end, host: []) + + generated = iter(["old", "fresh"]) + monkeypatch.setattr( + MODULE, + "build_management_payload", + lambda rows, host, date: {"generated_at_utc": next(generated), "rows": rows}, + ) + + cached = MODULE.management_report_for_date("SHARKON2025", report_date, use_cache=True) + fresh = MODULE.management_report_for_date("SHARKON2025", report_date, use_cache=False) + + assert cached["generated_at_utc"] == "old" + assert fresh["generated_at_utc"] == "fresh" + assert MODULE.load_management_cache("SHARKON2025", report_date)["generated_at_utc"] == "fresh" + finally: + MODULE.MANAGER_CACHE_TTL_SECONDS = original_ttl + MODULE.MANAGER_CACHE_DIR = original_dir diff --git a/check-aw-data.sh b/check-aw-data.sh index d3bd7b4..c230334 100644 --- a/check-aw-data.sh +++ b/check-aw-data.sh @@ -1,117 +1,17 @@ -#!/bin/bash -# check-aw-data.sh - Проверка сбора данных ActivityWatch с RDP-сервера SHARKON2025 -# Сервер: 10.10.10.13:5600 -# Хост-источник: 192.168.100.18 (SHARKON2025) +#!/usr/bin/env bash +set -euo pipefail -SERVER="http://10.10.10.13:5600" -HOSTNAME_FILTER="SHARKON2025" -NOW=$(date -u +%s) +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" -classify_bucket_age() { - local bucket="$1" - local age_sec="$2" - - case "$bucket" in - aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) - if [ "$age_sec" -lt 86400 ]; then - printf '%s' "${GREEN}FRESH${NC}" - else - printf '%s' "${CYAN}EVENT-DRIVEN${NC}" - fi - ;; - *) - if [ "$age_sec" -lt 3600 ]; then - printf '%s' "${GREEN}FRESH${NC}" - elif [ "$age_sec" -lt 86400 ]; then - printf '%s' "${YELLOW}STALE${NC}" - else - printf '%s' "${RED}DEAD${NC}" - fi - ;; - esac -} - -# Цвета -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -# Проверка доступности сервера -echo "=== ActivityWatch Data Check: $HOSTNAME_FILTER ===" -echo "" -echo -n "Server connectivity... " -RESP=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/info" 2>&1) -if [ $? -eq 0 ] && echo "$RESP" | jq -e '.version' > /dev/null 2>&1; then - VERSION=$(echo "$RESP" | jq -r '.version') - echo -e "${GREEN}OK${NC} (aw-server v$VERSION)" -else - echo -e "${RED}FAILED${NC} (cannot reach $SERVER)" - exit 1 -fi -echo "" - -# Проверка бакетов -echo "--- Buckets ---" -printf "%-45s %-8s %-22s %s\n" "BUCKET" "EVENTS" "LAST EVENT" "STATUS" -printf "%-45s %-8s %-22s %s\n" "---------------------------------------------" "--------" "----------------------" "------" - -BUCKETS=( - "aw-dlp-endpoint-signals" - "aw-dlp-incidents" - "aw-dlp-review" - "aw-dlp-rules" - "aw-watcher-afk" - "aw-watcher-window" - "aw-session-events" - "aw-worktime-sessions" -) - -for bucket in "${BUCKETS[@]}"; do - bucket_full="${bucket}_${HOSTNAME_FILTER}" - - # Получаем последний event - EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1) - LAST_ID=$(echo "$EVENT_DATA" | jq '.[0].id // 0') - LAST_TS=$(echo "$EVENT_DATA" | jq -r '.[0].timestamp // "no events"') - - # Вычисляем возраст - if [ "$LAST_TS" != "no events" ] && [ -n "$LAST_TS" ]; then - EVENT_EPOCH=$(date -d "$LAST_TS" +%s 2>/dev/null || echo 0) - if [ "$EVENT_EPOCH" -gt 0 ]; then - AGE_SEC=$((NOW - EVENT_EPOCH)) - if [ $AGE_SEC -lt 3600 ]; then - AGE="$((AGE_SEC / 60))m ago" - elif [ $AGE_SEC -lt 86400 ]; then - AGE="$((AGE_SEC / 3600))h ago" - else - AGE="$((AGE_SEC / 86400))d ago" - fi - STATUS="$(classify_bucket_age "$bucket" "$AGE_SEC")" - else - AGE="unknown" - STATUS="${RED}?${NC}" - fi - else - AGE="none" - LAST_ID="0" - STATUS="${RED}EMPTY${NC}" +for candidate in \ + "$TARGET_ROOT/release/check-aw-data" \ + "$TARGET_ROOT/debug/check-aw-data" \ + "$ROOT_DIR/adk-rust/target/release/check-aw-data" \ + "$ROOT_DIR/adk-rust/target/debug/check-aw-data"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" "$@" fi - - printf "%-45s %-8s %-22s %b\n" "$bucket_full" "$LAST_ID" "$LAST_TS ($AGE)" "$STATUS" done -echo "" - -# Проверка CORS -echo "--- CORS Check ---" -CORS_RESP=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 -o /dev/null -w '%{http_code}' -H "Origin: http://10.10.10.13:5600" "$SERVER/api/0/settings/" 2>&1) -if [ "$CORS_RESP" = "200" ]; then - echo -e "${GREEN}CORS: OK${NC} (HTTP 200)" -else - echo -e "${RED}CORS: FAIL${NC} (HTTP $CORS_RESP)" -fi - -echo "" -echo "=== Check Complete ===" -echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" +exec "$ROOT_DIR/scripts/legacy/check-aw-data.sh" "$@" diff --git a/check-aw-full.sh b/check-aw-full.sh index 56e5963..43eb40c 100644 --- a/check-aw-full.sh +++ b/check-aw-full.sh @@ -3,15 +3,45 @@ # Сервер: 10.10.10.13:5600 # RDP-хост: 192.168.100.18 (SHARKON2025) +if [[ "${CHECK_AW_FULL_FORCE_LEGACY:-0}" != "1" ]]; then + ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + for candidate in \ + "${CHECK_AW_FULL_RUST:-}" \ + "${CARGO_TARGET_DIR:-}/release/check-aw-full" \ + "$ROOT_DIR/adk-rust/target/release/check-aw-full" \ + "/usr/local/bin/check-aw-full"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + exec "$candidate" "$@" + fi + done +fi + SERVER="http://10.10.10.13:5600" HOSTNAME_FILTER="SHARKON2025" RDP_HOST="192.168.100.18" NOW=$(date -u +%s) +HOST_INACTIVE=false +GUARD_HEALTHY=false classify_bucket_age() { local bucket="$1" local age_sec="$2" + case "$bucket" in + aw-watcher-window) + if [ "$HOST_INACTIVE" = "true" ]; then + printf 'INACTIVE|%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-endpoint-signals) + if [ "$HOST_INACTIVE" = "true" ] && [ "$GUARD_HEALTHY" = "true" ]; then + printf 'INACTIVE|%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + esac + case "$bucket" in aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) if [ "$age_sec" -lt 86400 ]; then @@ -32,6 +62,31 @@ classify_bucket_age() { esac } +classify_bucket_no_events() { + local bucket="$1" + + case "$bucket" in + aw-watcher-window) + if [ "$HOST_INACTIVE" = "true" ]; then + printf 'INACTIVE|%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-endpoint-signals) + if [ "$HOST_INACTIVE" = "true" ] && [ "$GUARD_HEALTHY" = "true" ]; then + printf 'INACTIVE|%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + printf 'EVENT|%s' "${CYAN}EVENT-DRIVEN${NC}" + return + ;; + esac + + printf 'DEAD|%s' "${RED}EMPTY${NC}" +} + # Цвета RED='\033[0;31m' GREEN='\033[0;32m' @@ -63,6 +118,34 @@ else fi echo "" +# 1b. Context for inactive/event-driven classification +WORKTIME_EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-worktime-sessions_$HOSTNAME_FILTER/events?limit=1" 2>&1) +WORKTIME_TS=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null) +WORKTIME_ACTIVE=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].data.active // false' 2>/dev/null) +if [ -n "$WORKTIME_TS" ]; then + WORKTIME_EPOCH=$(date -d "$WORKTIME_TS" +%s 2>/dev/null || echo 0) + if [ "$WORKTIME_EPOCH" -gt 0 ]; then + WORKTIME_AGE=$((NOW - WORKTIME_EPOCH)) + if [ "$WORKTIME_AGE" -lt 900 ] && [ "$WORKTIME_ACTIVE" != "true" ]; then + HOST_INACTIVE=true + fi + fi +fi + +GUARD_EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-rus-collector-guard_$HOSTNAME_FILTER/events?limit=1" 2>&1) +GUARD_TS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null) +GUARD_STATUS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].data.status // ""' 2>/dev/null) +GUARD_PROBLEMS=$(echo "$GUARD_EVENT_DATA" | jq -r '([.[0].data.problems[]?] | length) // 0' 2>/dev/null) +if [ -n "$GUARD_TS" ]; then + GUARD_EPOCH=$(date -d "$GUARD_TS" +%s 2>/dev/null || echo 0) + if [ "$GUARD_EPOCH" -gt 0 ]; then + GUARD_AGE=$((NOW - GUARD_EPOCH)) + if [ "$GUARD_AGE" -lt 300 ] && [ "$GUARD_STATUS" = "ok" ] && [ "$GUARD_PROBLEMS" = "0" ]; then + GUARD_HEALTHY=true + fi + fi +fi + # 2. Проверка бакетов echo -e "${CYAN}--- 2. Data Buckets ---${NC}" printf " %-42s %-8s %-20s %s\n" "BUCKET" "EVENTS" "LAST EVENT" "STATUS" @@ -110,8 +193,9 @@ for entry in "${BUCKETS[@]}"; do else AGE="none" LAST_ID="0" - STATUS="${RED}EMPTY${NC}" - STATUS_KEY="DEAD" + CLASSIFICATION="$(classify_bucket_no_events "$bucket")" + STATUS_KEY="${CLASSIFICATION%%|*}" + STATUS="${CLASSIFICATION#*|}" fi printf " %-42s %-8s %-20s %b\n" "$label" "$LAST_ID" "$AGE" "$STATUS" @@ -157,7 +241,7 @@ for entry in "${BUCKETS[@]}"; do CLASSIFICATION="$(classify_bucket_age "$bucket" "$AGE_SEC")" STATUS_KEY="${CLASSIFICATION%%|*}" case "$STATUS_KEY" in - FRESH|EVENT) + FRESH|EVENT|INACTIVE) FRESH_COUNT=$((FRESH_COUNT + 1)) ;; STALE) @@ -169,8 +253,13 @@ for entry in "${BUCKETS[@]}"; do esac fi else - case "$bucket" in - aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + CLASSIFICATION="$(classify_bucket_no_events "$bucket")" + STATUS_KEY="${CLASSIFICATION%%|*}" + case "$STATUS_KEY" in + FRESH|EVENT|INACTIVE) + FRESH_COUNT=$((FRESH_COUNT + 1)) + ;; + STALE) STALE_COUNT=$((STALE_COUNT + 1)) ;; *) diff --git a/clickhouse-1c/ai/test_company_intelligence_api.py b/clickhouse-1c/ai/test_company_intelligence_api.py index 3e7e5a5..d3ae701 100644 --- a/clickhouse-1c/ai/test_company_intelligence_api.py +++ b/clickhouse-1c/ai/test_company_intelligence_api.py @@ -7,7 +7,9 @@ import sys import tempfile import types import unittest +from datetime import UTC, datetime from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.modules.setdefault("clickhouse_connect", types.SimpleNamespace(get_client=lambda **_: None)) @@ -343,6 +345,11 @@ class CompanyIntelligenceApiTests(unittest.TestCase): self.assertIn("ФЕЛИЦТ ГРУПП 2026", html_page) def test_build_weekly_trend_report_and_render(self) -> None: + class FixedDatetime(datetime): + @classmethod + def now(cls, tz=None): + return cls(2026, 5, 22, 13, 0, 0, tzinfo=tz or UTC) + payloads = [ { "generated_at": "2026-05-21T12:00:00+00:00", @@ -391,7 +398,8 @@ class CompanyIntelligenceApiTests(unittest.TestCase): }, }, ] - report = api.build_weekly_trend_report(payloads, days=7) + with patch.object(api, "datetime", FixedDatetime): + report = api.build_weekly_trend_report(payloads, days=7) self.assertEqual(len(report["daily"]), 2) self.assertEqual(report["top_weekly_changes"][0]["company"], "ФЕЛИЦТ ГРУПП 2026") html_page = api.render_weekly_trend_html(report) diff --git a/docs/DETMIR_UNIFIED_OPERATING_MODEL_RU.md b/docs/DETMIR_UNIFIED_OPERATING_MODEL_RU.md index 2135a69..8a7c161 100644 --- a/docs/DETMIR_UNIFIED_OPERATING_MODEL_RU.md +++ b/docs/DETMIR_UNIFIED_OPERATING_MODEL_RU.md @@ -2,7 +2,7 @@ Дата фиксации: `2026-05-24` -Последнее runtime-уточнение: `2026-05-28` +Последнее runtime-уточнение: `2026-05-30` Этот файл предназначен как единая рабочая опора по `DetMir`: что именно входит в систему, где это живет, каким инструментарием проект надо планировать и сопровождать, и какой операционный контур считать промышленным. @@ -81,6 +81,26 @@ - Grafana dashboards без авторизации корректно редиректят на login, это не считается отказом; с сохраненной admin-учеткой проверены фактические страницы и datasource health; - gateway `/go/file1c-brief`, `/go/file1c-actions`, `/go/aw-ui` ведет на рабочие внутренние surface. +### 2.3 Runtime-семантика операторских сигналов от 2026-05-30 + +После Phase 8 операторские проверки должны читаться по смыслу статуса, а не только по возрасту последнего события. + +| Статус | Как трактовать | +|---|---| +| `FRESH` | Данные свежие, источник сейчас активен или недавно обновлялся. | +| `INACTIVE` | Нормальное состояние для desktop/window/DLP endpoint buckets, если worktime-сессия свежая, но интерактивной активности нет. Это не инцидент. | +| `EVENT-DRIVEN` | Нормальное состояние для buckets, которые пишутся только при событии: `aw-session-events_*`, `aw-dlp-incidents_*`, `aw-dlp-review_*`, `aw-dlp-rules_*`. Отсутствие новых событий само по себе не отказ. | +| `STALE` | Потенциальная деградация активного источника; проверять collector/session placement. | +| `DEAD` / `EMPTY` | Отказ или неинициализированный обязательный источник; требуется диагностика. | + +Операторские правила: + +- `./check-aw-full.sh` с `FRESH=8 STALE=0 DEAD=0` считается зеленым контуром, даже если отдельные строки показывают `INACTIVE` или `EVENT-DRIVEN`. +- `./check-aw-data.sh` должен использовать ту же семантику, что и full-check; `EVENT-DRIVEN` и `INACTIVE` не являются поводом для ручного recovery. +- SLO-строка бота `aw_rus_slo: recovered ... current_sample=OK ... budget_remaining_seconds<0` означает исторически сожженный error budget при здоровом текущем контуре. Это не активная авария. +- SLO становится инцидентом только при текущем `current_sample=FAIL` и исчерпанном бюджете или при stale SLO summary. +- `aw-browser-smoke.timer` хранит ограниченное число запусков через `AW_BROWSER_SMOKE_KEEP_RUNS` и ограничен `TimeoutStartSec=180`; рост `/var/lib/activitywatch/browser-smoke` выше нескольких сотен MB надо считать regression в retention/config. + ### 2.2 Стабилизация management report, bridge и recovery от 2026-05-28 До стабилизации слабые места были такими: diff --git a/docs/dlp-cross-os-native-enforcement.md b/docs/dlp-cross-os-native-enforcement.md new file mode 100644 index 0000000..1998138 --- /dev/null +++ b/docs/dlp-cross-os-native-enforcement.md @@ -0,0 +1,198 @@ +# Cross-OS Native DLP Enforcement + +## Цель + +Расширять AWatch-rus как легкую DLP-систему без тяжелого монолитного endpoint-agent. +Базовый принцип: сначала использовать естественные механизмы ОС и управляемых браузеров, +а собственный код держать тонким слоем политики, телеметрии и корреляции. + +Это не заменяет текущий Windows/RDP contour. Он остается основным production-контуром: + +- `AWatchRusCollectorGuard` следит за collector health; +- `aw-rus-healthd` принимает решение по свежести бакетов и активности хоста; +- `tsj-guardian-bot` наблюдает и лечит только реальные деградации; +- DLP policy engine остается центральным источником правил. + +## Единая модель политики + +Новые OS-specific механизмы должны сводиться к одному контракту: + +```json +{ + "nativeControls": { + "mode": "monitor", + "channels": { + "removableStorage": {"action": "audit"}, + "print": {"action": "audit"}, + "clipboard": {"action": "audit"}, + "browserUpload": {"action": "audit"}, + "appExecution": {"action": "audit"} + } + } +} +``` + +Допустимые действия: + +- `audit`: только событие в `aw-dlp-endpoint-signals_`; +- `warn`: событие плюс локальное уведомление пользователя; +- `block`: блокировка штатным механизмом ОС или managed browser; +- `blockWithOverride`: блокировка с управляемым исключением, если платформа это поддерживает; +- `disabled`: канал выключен. + +Для rollout по умолчанию используется `monitor/audit`. `block` включается только по одному +каналу и одной группе хостов после накопления baseline. + +На Windows это правило enforced в `dlp-endpoint-signals-collector.ps1`: если endpoint-правило +просит `action: "block"`, но `nativeControls.mode` не равен `enforce` или канал не разрешает +`block`, collector подавляет enforcement и пишет инцидент с `enforcementSuppressed=true`. + +## Windows + +Windows остается самой зрелой платформой для ближайшего enforcement. + +Предпочтительный порядок: + +1. USB/removable storage: + - мониторинг: текущий `dlp-endpoint-signals-collector.ps1`; + - enforcement: GPO/registry/device installation restrictions и `Set-Disk -IsReadOnly`; + - область блокировки: запись на removable media, не чтение. +2. Print: + - мониторинг: `Microsoft-Windows-PrintService/Operational`; + - enforcement: отмена print job штатным spooler API/CIM; + - rollout: сначала только документы с совпадением `documentRegex`. +3. Clipboard: + - мониторинг: текущий endpoint collector; + - enforcement: очистка clipboard только для high-confidence правил; + - риск: UX и ложные срабатывания, поэтому `block` не включать глобально. +4. App execution: + - enforcement: AppLocker или WDAC/App Control for Business; + - назначение: не “DLP content”, а сужение каналов утечки через запрещенные приложения. +5. Browser upload: + - легкий путь: managed browser policy + URL/category rules; + - сильный путь: расширение браузера или коммерческий DLP browser control, если нужен inline file upload block. + +Ограничение: AppLocker/WDAC управляют запуском кода, но не поведением уже запущенного приложения. +Поэтому они дополняют DLP, а не заменяют USB/print/browser enforcement. + +## macOS + +Для macOS нельзя идти через kernel extension как основной путь. Современный естественный +вариант - System Extensions и профиль управления через MDM. + +Предпочтительный порядок: + +1. Monitor-only agent: + - LaunchDaemon + Swift/Go helper; + - публикация событий в AW buckets; + - локальный health heartbeat по аналогии с `aw-rus-collector-guard_`. +2. Endpoint Security system extension: + - file open/write/exec telemetry; + - deny только после отдельного PoC и подписи/entitlements; + - обязательная MDM-подготовка approval profile. +3. Network Extension: + - для DNS/proxy/web egress контроля; + - лучше применять к managed domains, не как полный MITM по умолчанию. +4. MDM restrictions: + - screen capture, AirDrop, external media, profile-level restrictions; + - это самый легкий enforcement, если парк управляется MDM. + +Ограничение: без MDM и Apple Developer entitlements macOS enforcement будет хрупким. +Для неуправляемых Mac оставляем monitor-only. + +## Linux + +Linux должен быть легким и дистрибутивно-нейтральным. Не начинать с “универсального агента, +который перехватывает все syscalls”. + +Предпочтительный порядок: + +1. fanotify file gate: + - мониторинг и permission events для чувствительных каталогов; + - хороший MVP для removable mount points, home/project shares, export directories; + - блокировка через permission response до открытия файла. +2. auditd/journald collectors: + - дешево для exec, sudo, mount, removable media, ssh/scp hints; + - подходит для server/workstation baseline. +3. eBPF telemetry: + - использовать для observability: process, connect, file metadata; + - enforcement только через BPF LSM на поддерживаемых ядрах и после отдельной совместимости. +4. Desktop clipboard/print: + - monitor-only через DE-specific tools (`wl-paste`, `xclip`, CUPS logs); + - block режим только для управляемых рабочих станций. + +Ограничение: Linux enforcement сильно зависит от ядра, LSM stack и дистрибутива. Поэтому +первая production-версия должна быть fanotify + auditd, а eBPF оставить как расширяемый слой. + +## ChromeOS и managed browser + +Для ChromeOS не надо писать свой endpoint-agent. Если устройства управляются через Google +Admin Console, использовать ChromeOS Data Controls: + +- copy/paste; +- printing; +- screen capture/screen sharing; +- file open/upload/transfer; +- removable storage. + +Для Windows/macOS/Linux браузерный контур должен быть policy-first: + +- Edge/Chrome enterprise policies; +- URL/category lists из текущего DLP policy engine; +- расширение браузера только когда нужен inline upload/file decision, а не только telemetry. + +## Priority Backlog + +### P0 - сейчас + +- Привести все health-check скрипты к единой inactive/guard-aware классификации. +- Хранить `nativeControls` в policy document как forward-compatible секцию. +- Не включать новый `block` глобально; только `audit` и `warn`. + +### P1 - Windows production + +- USB write-block profile через GPO/PowerShell. +- Print cancel по `documentRegex`. +- Browser upload detection по managed URL categories. +- Guard heartbeat для applied nativeControls version/checksum. + +### P2 - Linux MVP + +- `aw-linux-file-gate` на fanotify для monitor/audit по каталогам. +- `aw-linux-audit-collector` для mount/usb/scp/sudo/process telemetry. +- systemd unit + heartbeat bucket. + +### P3 - macOS MVP + +- `aw-macos-monitor` LaunchDaemon без enforcement. +- MDM profile checklist. +- System Extension PoC только для managed Macs. + +### P4 - managed browser / ChromeOS + +- Транслятор DLP policy domains/categories в Chrome/Edge policy bundle. +- ChromeOS Data Controls mapping для организаций с Google Workspace. + +## Rollout Rules + +- Каждая новая платформа стартует в `monitor`. +- `block` разрешен только после минимум 7 дней clean baseline. +- Любой block должен писать: + - rule id; + - action; + - platform; + - native mechanism; + - enforcement result; + - override id, если применимо. +- Если local guard/heartbeat старше порога, центральный бот не делает широкий restart, + а переводит платформу в degraded и запускает platform-specific recovery. + +## Sources + +- Microsoft AppLocker overview: https://learn.microsoft.com/en-us/windows/security/application-security/application-control/app-control-for-business/applocker/applocker-overview +- Microsoft AppLocker policy design: https://learn.microsoft.com/en-za/windows/security/application-security/application-control/app-control-for-business/applocker/understand-applocker-policy-design-decisions +- Microsoft Purview Chrome DLP extension: https://learn.microsoft.com/en-us/purview/dlp-chrome-learn-about +- Apple system extensions deployment: https://support.apple.com/guide/deployment/system-extensions-in-macos-depa5fb8376f/web +- Linux fanotify manual: https://man7.org/linux/man-pages/man7/fanotify.7.html +- eBPF docs: https://docs.ebpf.io/ +- ChromeOS Data Controls: https://support.google.com/chrome/a/answer/11587610 diff --git a/docs/dlp-enforcement.md b/docs/dlp-enforcement.md index 2751da0..adcb066 100644 --- a/docs/dlp-enforcement.md +++ b/docs/dlp-enforcement.md @@ -13,10 +13,31 @@ Phase 2.5 расширяет DLP endpoint collector функциями **акт Во всех случаях пользователь получает Windows-уведомление (balloon notification) с описанием причины блокировки. +Для cross-OS развития используется отдельный native-first профиль: +`docs/dlp-cross-os-native-enforcement.md`. Он задает правило: Windows enforcement оставляем +на штатных GPO/AppLocker/Spooler/Storage механизмах, macOS строим через System Extensions/MDM, +Linux через fanotify/auditd/eBPF, ChromeOS через Data Controls. Тяжелый универсальный агент +не является целевой архитектурой. + ## Конфигурация политики Формат `dlp-policy.json` не изменился — поле `action` в правиле теперь поддерживает значение `"block"` наряду с `"alert"` (по умолчанию). +Важно: для production safety `action: "block"` в конкретном endpoint-правиле не включает +блокировку сам по себе. Реальное enforcement-действие выполняется только если: + +- `nativeControls.mode` равно `"enforce"`; +- `nativeControls.rollout.allowGlobalBlock=true`; +- и `nativeControls.channels..action` равно `"block"` или `"blockWithOverride"`. + +Если правило просит `block`, но native controls находятся в `monitor`, collector пишет инцидент +как `alert` и добавляет поля: + +- `requestedAction: "block"`; +- `enforcementMode`; +- `nativeChannelAction`; +- `enforcementSuppressed: true`. + ### Пример: блокировка USB записи ```json @@ -27,6 +48,17 @@ Phase 2.5 расширяет DLP endpoint collector функциями **акт "severity": "medium", "cooldownSeconds": 300 }, + "nativeControls": { + "mode": "enforce", + "rollout": { + "allowGlobalBlock": false + }, + "channels": { + "clipboard": {"action": "block"}, + "usb": {"action": "block"}, + "print": {"action": "block"} + } + }, "endpoint": { "usb": [ { diff --git a/docs/metagpt-aw-scout.md b/docs/metagpt-aw-scout.md new file mode 100644 index 0000000..611c3a5 --- /dev/null +++ b/docs/metagpt-aw-scout.md @@ -0,0 +1,58 @@ +# MetaGPT AW Scout + +`scripts/metagpt-aw-scout.sh` is the safe project wrapper for using the +MetaGPT-configured LLM provider with ActivityWatch-Russian tasks. + +Default mode is direct LLM scout through `/home/igor/.metagpt/config2.yaml`. +This avoids MetaGPT Browser/Editor tools, which are too noisy for operational +checklists. + +Optional MetaGPT team mode is still available: + +```bash +METAGPT_AW_ENGINE=team scripts/metagpt-aw-scout.sh smoke +``` + +Do not use full `--implement` mode with the free Groq tier for this project. It +pulls too much context and usually exceeds TPM limits. + +## Setup + +Use a valid provider key outside git: + +```bash +export GROQ_API_KEY="gsk_..." +``` + +The global wrapper `/home/igor/bin/metagpt-lab` can write the key into +`/home/igor/.metagpt/config2.yaml`. The scout script reads that config and does +not print secrets. + +## Presets + +```bash +scripts/metagpt-aw-scout.sh qa-rollback +scripts/metagpt-aw-scout.sh smoke +scripts/metagpt-aw-scout.sh grafana +scripts/metagpt-aw-scout.sh install-kit +scripts/metagpt-aw-scout.sh windows-i18n +``` + +Free-form task: + +```bash +scripts/metagpt-aw-scout.sh "Review risk of changing aw-worktime-ui-bridge foreground cache" +``` + +Reports are saved under: + +```text +.ai/metagpt/ +``` + +## Current Rule + +The LLM is only a scout. Codex/operator must verify every material claim against +local files, live services, Ansible output, Grafana, and ActivityWatch APIs +before changing production behavior. Use `METAGPT_AW_ENGINE=team` only for +experiments; direct mode is the operational default. diff --git a/docs/runbook.md b/docs/runbook.md index c170219..7f782b3 100755 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -11,8 +11,9 @@ systemctl status nginx --no-pager nginx -t curl -I -sS -H 'Host: dm.iri1968.dpdns.org' http://127.0.0.1/ curl -k -fsS -H 'Host: dm.iri1968.dpdns.org' https://127.0.0.1/healthz -curl -k -I -sS -H 'Host: dm.iri1968.dpdns.org' https://127.0.0.1/go/proxmox-gui -curl -k -H 'Host: dm.iri1968.dpdns.org' -fsS https://127.0.0.1/ | grep -F 'dm.iri1968.dpdns.org' +curl -k -I -sS -H 'Host: dm.iri1968.dpdns.org' https://127.0.0.1/ | head +curl -k -u "$(awk -F= '/^user=/{u=$2}/^password=/{p=$2}END{print u\":\"p}' /root/proxmox-web-gateway.credentials)" \ + -H 'Host: dm.iri1968.dpdns.org' -fsS https://127.0.0.1/ | grep -F 'dm.iri1968.dpdns.org' ``` Playbook для повторного rollout: @@ -22,6 +23,49 @@ ANSIBLE_HOST_KEY_CHECKING=False \ ansible-playbook -i ansible/inventory.ini ansible/deploy_proxmox_web_gateway.yml ``` +### Public gateway через pfSense + +Публичная схема: + +```text +Internet -> dm.iri1968.dpdns.org -> pfSense WAN 178.178.98.83 -> NAT 80/443 -> nginx 10.10.10.2 +``` + +Нормальное состояние: + +- `https://dm.iri1968.dpdns.org/healthz` -> `200 ok` без auth; +- `https://dm.iri1968.dpdns.org/` без auth -> `401`; +- `http://dm.iri1968.dpdns.org/healthz` -> `301` на HTTPS; +- после Basic Auth: + - `/` -> gateway index; + - `/r/file1c/brief` -> 1C brief; + - `/r/grafana/api/health` -> Grafana health; + - `/r/aw/api/0/info` -> AW server info. + +Gateway credential хранится только на `10.10.10.2`: + +```sh +sudo cat /root/proxmox-web-gateway.credentials +``` + +pfSense NAT backup перед автоматической правкой: + +```sh +ls -1t /opt/infra-admin/backups/pfsense-gateway-nat-*.json | head +``` + +Проверка pfSense REST/API и pending firewall changes: + +```sh +set -a +. /home/igor/.config/tsj-bot/pfsense.env.readonly +set +a +curl -ksS -H "X-API-Key: $PFSENSE_API_KEY" "$PFSENSE_URL/api/v2/firewall/apply" +``` + +NAT должен содержать `WAN tcp 443 -> 10.10.10.2:443` и `WAN tcp 80 -> 10.10.10.2:80`. +WAN rules должны содержать pass на `10.10.10.2:80` и `10.10.10.2:443`. + ### На Proxmox ```sh @@ -61,16 +105,41 @@ ss -ltnp | grep 5600 ```sh /usr/local/bin/aw-health-check +/usr/local/bin/dlp-health-check --json ``` Что проверяет дополнительно: - свежесть DLP bucket-ов (`aw-dlp-endpoint-signals_*`, `aw-file-operations_*`); - наличие transport/self-test telemetry (`queueDepth`, `eventsEnqueued`, `eventsFlushed`, `sendFailures`) в endpoint self-test; +- последние значения transport-счетчиков по каждому `aw-dlp-endpoint-signals_*` bucket; +- runtime-срез `aw-file-operations_*`: последние `collector_health`, счетчики очереди/отправки, последние операции без полного пути; +- runtime-срез `aw-dlp-incidents_*`: по умолчанию только metadata/age без тяжёлого чтения events; при ручном `--incident-sample-limit > 0` — количество реальных incident events в sample, self-test count, severity/action/rule rollup и до 5 последних incidents с коротким `message_excerpt`; - API-доступность базовых сервисов. Интерпретация: - `FAIL` — есть критичная проблема (service/API/stale transport); -- `WARN` — сигнал для оператора (например, bucket еще не активирован на хосте), но без hard-fail. +- `WARN` — сигнал для оператора (например, bucket еще не активирован на хосте, `queueDepth > 100`, новый рост `sendFailuresDelta >= 1`, нет `collector_health` в sample), но без hard-fail. +- `sendFailures` — накопительный счётчик collector-а; основной health-check предупреждает только по delta относительно сохранённого baseline в `AW_DLP_HEALTH_STATE_DIR`, чтобы старые восстановленные ошибки не висели вечным warning. + +Пороги можно переопределить без изменения collector-ов: + +```sh +/usr/local/bin/dlp-health-check --json \ + --endpoint-queue-warn-depth 100 \ + --endpoint-send-failure-warn-count 1 \ + --incident-sample-limit 0 \ + --fileops-sample-limit 20 \ + --fileops-queue-warn-depth 100 \ + --fileops-send-failure-warn-count 1 +``` + +Для разового разбора последних DLP incidents можно включить sampling вручную: + +```sh +/usr/local/bin/dlp-health-check --json --incident-sample-limit 20 +``` + +Если AW-server медленно отдаёт большой `aw-dlp-incidents_*` bucket, такой запуск может вернуть `incident-runtime: WARN` по timeout. Это не должно использоваться как hard-fail основного мониторинга. ## Проверка RU patch @@ -399,8 +468,8 @@ curl -fsS 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-incidents_SHARKON2025/even 1) Проверить/обнулить очереди: ```powershell -$q1 = 'C:\ProgramData\AWatch-rus\file-operations-queue.jsonl' -$q2 = 'C:\ProgramData\AWatch-rus\dlp-endpoint-signals-queue.jsonl' +$q1 = 'C:\ProgramData\AWatch-rus\file-operations-queue*.jsonl' +$q2 = 'C:\ProgramData\AWatch-rus\dlp-endpoint-signals-queue*.jsonl' Get-Item $q1,$q2 | Select Name,Length,LastWriteTime ``` diff --git a/docs/strategic-dlp-roadmap.md b/docs/strategic-dlp-roadmap.md index ff5d184..43de9b6 100644 --- a/docs/strategic-dlp-roadmap.md +++ b/docs/strategic-dlp-roadmap.md @@ -16,6 +16,24 @@ | Deployment | Ansible + PowerShell ensemble + InnoSetup + Proxmox LXC | | Linux | Remote worker, console/SSH logger, web category logger | +## Cross-OS enforcement strategy update + +Новый целевой путь зафиксирован в `docs/dlp-cross-os-native-enforcement.md`: AWatch-rus +расширяется не тяжелым универсальным агентом, а нативными механизмами ОС и managed browser. + +Приоритет платформ: + +| Платформа | Легкий естественный путь | Enforcement-first канал | +|-----------|--------------------------|--------------------------| +| Windows | GPO/AppLocker/Spooler/Storage + текущие PowerShell collectors | USB write-block, print cancel | +| macOS | MDM + System Extensions/Endpoint Security, без kext | monitor-first, затем managed deny | +| Linux | fanotify + auditd, eBPF как telemetry/LSM extension | sensitive directory/removable gate | +| ChromeOS | Google Admin Data Controls | copy/paste, print, file transfer/upload | +| Managed browser | Edge/Chrome enterprise policy, extension только для inline upload | cloud upload policy | + +Общее правило rollout: `audit` -> `warn` -> точечный `block`; глобальный `block` запрещен +до baseline и guard heartbeat. + ## Ключевые разрывы до InfoWatch TM уровня ### 🔴 Критические (без них это не DLP, а мониторинг) diff --git a/docs/wiki/Server-Infrastructure.md b/docs/wiki/Server-Infrastructure.md index 1302239..b571f47 100644 --- a/docs/wiki/Server-Infrastructure.md +++ b/docs/wiki/Server-Infrastructure.md @@ -6,18 +6,37 @@ ## `aw-prune-local-state.sh` -Скрипт `aw-server/aw-prune-local-state.sh` устанавливается в: +Команда `aw-server/aw-prune-local-state.sh` устанавливается в: ```text /usr/local/bin/aw-prune-local-state.sh ``` +Сейчас это Rust-first wrapper. Если доступен +`/usr/local/bin/aw-prune-local-state-rust`, wrapper использует его; иначе +fallback идет на legacy shell script из +`/opt/activitywatch/aw-rus-ops/aw-prune-local-state.sh`. + Назначение: - чистит старые backup-файлы в `{{ aw_server_data_dir }}/backups`; - отдельно удерживает последние DB backups и JSON backups; - удаляет временные архивы из `/tmp`: `activitywatch-*.zip`, `hayabusa-*.zip`, `aw-hayabusa-profiles.txt`; - удаляет временные WebUI/worktime artifacts старше одного дня: `aw-worktime-ui-bridge.py`, `views-default.json`, `apply_webui_ru_patch.out`. +- чистит старые `browser-smoke` run-директории в + `{{ aw_server_data_dir }}/browser-smoke`, удерживая последние запуски. + +Rust binary по умолчанию работает в dry-run режиме. Production wrapper без +аргументов запускает его с `--apply`, чтобы сохранить поведение systemd timer. + +Safety policy: + +- не удалять основную ActivityWatch SQLite DB; +- не удалять rollback-critical backups: `switch-backups`, `before-rust`, + `rollback`; +- не удалять корневые state-каталоги; +- удалять только пути из явного allowlist: `backups`, `backups/db`, + `browser-smoke`, ограниченный набор временных файлов в `/tmp`. ## systemd unit и timer diff --git a/docs/wiki/Windows-Collector-Suite.md b/docs/wiki/Windows-Collector-Suite.md index a45b967..37d3254 100644 --- a/docs/wiki/Windows-Collector-Suite.md +++ b/docs/wiki/Windows-Collector-Suite.md @@ -18,10 +18,10 @@ ## Process events -Новый флаг: +Флаг: ```yaml -aw_windows_process_events_enabled: true +aw_windows_process_events_enabled: false ``` Он попадает в deployment config как: @@ -30,7 +30,7 @@ aw_windows_process_events_enabled: true sessionEvents.processEventsEnabled ``` -Когда флаг включен, Windows collector публикует process-level изменения в session events bucket. Это дает server-side слою больше контекста для active session detection и forensic review. +По умолчанию флаг выключен во всех путях deploy/recovery: постоянная публикация process-level изменений в `aw-session-events_` создает чрезмерный поток событий и быстро раздувает SQLite на AW server. Включать его следует только явно и временно для forensic/debug окна, после чего возвращать `false`. ## Localized Administrator diff --git a/docs/windows-deploy-startup-model.md b/docs/windows-deploy-startup-model.md index 8ec7c61..f4bd300 100644 --- a/docs/windows-deploy-startup-model.md +++ b/docs/windows-deploy-startup-model.md @@ -6,6 +6,12 @@ Use this model on `SHARKON2025`-style hosts with multiple user sessions. +- `AWatchRusCollectorGuard` service: + - runs under `LocalSystem` + - is the preferred local control plane for collector supervision + - in `shadow` mode only publishes state/heartbeat + - in `enforce` mode starts only session-appropriate collectors/tasks with cooldown and restart budget + - publishes `aw-rus-collector-guard_` - `ActivityWatch Launch [HOST_user]` tasks: - `AtLogOn` - `InteractiveToken` @@ -13,8 +19,9 @@ Use this model on `SHARKON2025`-style hosts with multiple user sessions. - `ActivityWatch Recovery` task: - `AtStartup` - `SYSTEM` + - stays enabled even when `AWatchRusCollectorGuard` is active - keeps only the global `worktime-session-collector` alive - - may re-trigger user launch tasks, but only for users whose sessions currently exist + - may re-trigger user launch tasks for managed live or disconnected sessions - interactive collectors/watcher binaries belong to the user-session path, not to Session 0 Collector ownership in this model: @@ -53,10 +60,17 @@ Do not mix the two startup models on the same RDP host: - no permanent standalone-service loop together with per-user launch/recovery tasks - no blind `Start-ScheduledTask` for all configured users - no validation rule that treats users without sessions as failed collector startup +- no bot-driven collector recovery as the primary control plane + +During migration, `AWatchRusCollectorGuard` may run in `shadow` mode beside the existing +recovery task. In `enforce` mode it becomes the primary control plane; `ActivityWatch Recovery` +still remains enabled as fallback/bootstrap and must not be disabled by deploy scripts. ## Hardening rules - start launch tasks only for users with real sessions +- treat managed disconnected RDP sessions as real recovery targets when their launch tasks exist - keep only one global `worktime-session-collector` - validate by session-aware expectations, not by “all configured users must currently run” +- use guard heartbeat and bucket freshness as health signals - keep `deploy_aw_windows.yml`, `deploy-ensemble.ps1`, `hardening-recovery.ps1`, and installer assumptions aligned diff --git a/install-kit-awindows-20260427-211240.tar.gz b/install-kit-awindows-20260427-211240.tar.gz index 42626af..72b91d6 100644 Binary files a/install-kit-awindows-20260427-211240.tar.gz and b/install-kit-awindows-20260427-211240.tar.gz differ diff --git a/install-kit-awindows-20260427-211240.zip b/install-kit-awindows-20260427-211240.zip index e7d4c50..8dce547 100644 Binary files a/install-kit-awindows-20260427-211240.zip and b/install-kit-awindows-20260427-211240.zip differ diff --git a/install-kit-awindows-20260427-211240/MANIFEST.txt b/install-kit-awindows-20260427-211240/MANIFEST.txt index 5010de7..35550c9 100644 --- a/install-kit-awindows-20260427-211240/MANIFEST.txt +++ b/install-kit-awindows-20260427-211240/MANIFEST.txt @@ -1,13 +1,13 @@ -f3dde1e6d1532804379faf7e395deaf95cf3e0b97769d425f8a69f4572de2a2f install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt -edc20460f7be2ec676a6fd7c9687f808507b364b32915eb78b7a63b441b1ff84 install-kit-awindows-20260427-211240/ansible/README.md +1b776081c24ad8413c2f3e24aff3952949e7786a108a08ef45121536bcae9482 install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt +e973ad9aa3824a6588ce4ef1b80ce3caec0085434a2755f9b657f09fb7435632 install-kit-awindows-20260427-211240/ansible/README.md 412bb766bbf0791c3593f38daa771d5d0aa58cc1f2d3c9010fcd4588d0fe87df install-kit-awindows-20260427-211240/ansible/deploy_aw_pfsense_poller.yml -0188480546b4b18937194f2b05a0f23da5dd8536eed4d60b7499a8170c36d08e install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml -953bb39572c520d0438a621c2b4c08315bf887b84ee9e795554e5592b54a1229 install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml -0054e7ba5f342cd10db08f6bab4784d4816179d69f76898a4de293a4063ea14a install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml +951b6e22c285a95a3f45ead81fe98069eb48405368617a600f86240ccad65b70 install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml +7c1afd6088e7ec26842be4b4b8a0530780d1d7075f19f80d63eb08445857823f install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml +d666eac9d94ad71842e21aaac5d23968a0c2213bb4edab899bf432d7ad417ddf install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml a896676303be9a47ed6b0c8cc07deb5a5432aebc97231e801fda985121c539eb install-kit-awindows-20260427-211240/ansible/group_vars/pfsense-poller.example.yml c5cab36645065815571c99f6d360f910dcccbb54b780c8bfd526a6cdc3684e19 install-kit-awindows-20260427-211240/ansible/group_vars/proxmox-matrix.example.yml 35a33c8a1c75ded5e85c6b79e0b3efde07959ff61ee5f66d83b7e0c2abe87fc5 install-kit-awindows-20260427-211240/ansible/group_vars/proxmox.example.yml -a1921715cf9904b7e3b61a1a5c4300382efe2f3bd70d73e4c8e995318fdd55e9 install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml +9e4939d2d951870cacab6e42c5bf0ccd03350089dc0d32b9a681948ff96754d2 install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml 7189b5205bd25313db54e5be027b0d066199e6ae34ad74be2095a1691adaf5e3 install-kit-awindows-20260427-211240/ansible/install_full_stack.yml fea0574d7eb98ce24a1e7025afb9837c6241180095d21eaa74892225305d05a3 install-kit-awindows-20260427-211240/ansible/inventory.example.ini 8a5e4923c0f581dd4fbb32549ee7ab45ba506260056da923ba86d9f1b1081714 install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml @@ -15,33 +15,53 @@ fea0574d7eb98ce24a1e7025afb9837c6241180095d21eaa74892225305d05a3 install-kit-aw a74a49371e889dc3ea404534a939f32f2dac940d8902d20770590951ab67d532 install-kit-awindows-20260427-211240/ansible/tasks/provision_ct_and_deploy_aw.yml ba16fe9e267194459a6082045a387acb828a1d39a98e66b401ece5069ea62e64 install-kit-awindows-20260427-211240/aw-server/activitywatch-server.service fea5734c99b516b01bfe8ebbef59972d0ef553c09d7691790b08472ecfb9602d install-kit-awindows-20260427-211240/aw-server/apply_webui_ru_patch.sh +fabcdbb933b1d3b605c5ac1234af4e10495c09f18bd50931bf4a92081fcb5023 install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.service +e59d58d4cc6b2b0dceef5b1da14ca0a1b24a202ce576ee43028f44ffe43c5ef4 install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.timer ba88cc284d047f521427ac038c266624a6fe8493ce3e79bc27c172a2e70ac54a install-kit-awindows-20260427-211240/aw-server/aw-host-groups.json 509759461ce0918a2dc839832812cf5f4c77b1cea1e6ffdcab0146e02598df79 install-kit-awindows-20260427-211240/aw-server/aw-ru-patch.js -22d4ee27ffc73dcec3ba73c8ee65d12353ded1403f592ff52ef578e09dad4c43 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py +6072757b685430962f61f97f6de4b0ed6f6653454559974a6ee87936fc6c7571 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py 05b04b5f49e9c7783917e0861e3edd63f5d6db8638d6b33dbfd4dc0f1c16040f install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service -bb0f1de91862da66b0b6d9bd41e8dfe181710196141cb43b00d9b41dab6caf96 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer -b493fbf4cc15ccba44ea78d3ed17a0a6db31617f22d6aeb671677cbf11a36921 install-kit-awindows-20260427-211240/aw-server/aw-server.env.example +8ae02b42f4bc33d95268624bae79c2101298b7ec6644553a1e81f7841dda74d8 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer +28950acbfeed3f3e5729e0227fd96785dd840330ed9b94a3d6ab0a6a855ca5ec install-kit-awindows-20260427-211240/aw-server/aw-server.env.example +bc8f88fa615a74fc69f80c0cc614ddea323156e5e49d3de28a9958295737afab install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py +6db182e8e5237b1daaafdcd1003fccf7e5aaaabb7be8ab4226cb513f7b1c739b install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service +257cdc18f088e8848b1bee69e7f8b99aa860a0f600eebd74365a40ed94aa5ecb install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.timer 98c0bed353bbda0fa7a69df23f3b008cb0e8e70cdff6cc63330d4caf79fd3280 install-kit-awindows-20260427-211240/aw-server/aw-sw-cleanup.js -e220a2eb830a018fa28c02872fabda54aed7f87dbb0e14141f1a8933a98064fe install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py +9597e8e12a3472b86b222f10c2e0a71c653efc41415771aeaab2e830c9424d3c install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py f764e566d70952acc1b4f2baeccd6b7888905bc6b98b36333336c0dc77c66694 install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service afbb4be301b4940ed9b7671be3441a48d53bee968d8870749442f81c6d066650 install-kit-awindows-20260427-211240/aw-server/aw-worktime-panel.js +c9169d01469b2f49eab0f0e9a483f883d4c367719fde51fb262d74c35e798c44 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service +1733c8c84b17ad89e0f3e5b27f649b968dedd52d9048f9bc2d92e7cc10daebd1 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh +0cd38bae7ad1b4718d7decc8ef3ac932939a6e5f8c25e9e27d1d0ecf8cc328b9 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.timer +2e3a0667f8a9eddcb4803c0ba274615092cdd3eec23c2dba3f494715dac91423 install-kit-awindows-20260427-211240/aw-server/health-check.sh ca7d1ed7665e225824d4ed7ef8456c81a6d97ef2c90cb3aa799d72e07b6bcbb8 install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh aedffecfa24834968742cb2477faef80bf794345275a9679ac12c5a1f609acc2 install-kit-awindows-20260427-211240/aw-server/settings/classes-worktime.json 47c50054515506b72af2d6bca0ae959ca57861803e528991c230611e6a8893d6 install-kit-awindows-20260427-211240/aw-server/settings/views-default.json +4e5b23300ba5c9878b2c8ce6eea65043b7a4c5cb2d14619243cfca6d202aec9f install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.mjs +a851fba13a14a529f95eeb054c2f64dc108cf7a379c4ea35402e84c0b4a2733d install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh +da178cb5dfff3ae8465a6139048adc6757cb1dc23edb30d747d09a05e6b07aeb install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh +fe8d2b846a64f1357dda48b4c817c7919a3c7398921ddca2adf9a99711ac2613 install-kit-awindows-20260427-211240/scripts/quality-gate.sh +f634ea60040153fafcab6363bea8c442d05c1734e6a9d2e16844cf63a741df24 install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh +b58c572edd12e4c3620c045f3382e7b9f74664ef5550302772a828a023f8f3fe install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh +0b49e6db51d5abcecaeee0b85f186efd4f10d3360cf04e8aea2e4c7d4465dbda install-kit-awindows-20260427-211240/scripts/verify_innosetup_installer.sh dd2389e9cb199ed86d219120294fdaa64415cfbc683004d5f5db5d52aa758a92 install-kit-awindows-20260427-211240/server-configs-192.168.100.18/awatch-rus-admin.deployment-config.json 5279f9d677faed76a5f0248f9217ecc29eac977ab752552cab852f5b4b6715df install-kit-awindows-20260427-211240/server-configs-192.168.100.18/awatch-rus-u2u5.deployment-config.json 333fe336e51f9c69bd2559d18763da2b83df400fa374e540ed128ffb6765ab7e install-kit-awindows-20260427-211240/server-configs-192.168.100.18/awatch-rus-user1.deployment-config.json +e856d4b411b61dfc11e0a59b59026eff10972a47e7d8b28f6dfe367a2a5e192d install-kit-awindows-20260427-211240/windows/AWatchRusCollectorGuardService.cs 33aa34b89246d6c079ef9afe2f5cd153bd9d5946b69a175ff6fd678c77f61da5 install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psd1 -9e5e6a30bfb7789ee91ae656e306586a50509d9d416fc044408e9e6c19c61b37 install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 +6d5a7dd92619bbf972e17e455a7cda65b53c790f6f0ede47e1196246bfe8b1cf install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 +f4405a959a52e21185234463887a4fc09ea28e48d2412ac336f9715b46be9cb9 install-kit-awindows-20260427-211240/windows/aw-collector-guard.ps1 81e95c7e4b7336a2e1f0caed3ea99f6cdb9696b99380dc2148667b5fba577f68 install-kit-awindows-20260427-211240/windows/browser-domains-native-collector.ps1 -e390e00a4deff5ce45a08f5252f23c56b2655c8611e42bc7b2b6992f60bde038 install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 -2074d9ebdda069c8bd579e688473656259714fe7e13b7537a7c464143195b525 install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 -9f2637ede66b61a12e9edafa0b248a5fb496dd5175b288ebb71718330cac68d5 install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 -dc74947393b1851ca233b559ea77f29b1b12a1c4480fbb865f2b58b25f3ea7d3 install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 -a4dad0745da95a69ee55b0216d4bde39c58acef8642380092938465653c61cf4 install-kit-awindows-20260427-211240/windows/dlp-policy.example.json +51eb13f689494c279ec321bbb68dfcb9b8ff81f22f11502b72c16895b65e0b65 install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 +5bacbd48fdfc8d0d4479ba1af54cbbfe6057cc654409b385f487962576f4f1db install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 +6ab4c496220bb46f064cc927a64ea853526a2c814182a32f39b432be6b32fe89 install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 +73f3328cde22798dafa645cdd78d648ccf6425d6577432ae8c95d24daa17dc94 install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 +161c0ee7e8aa6fa172bf10b6c83f208aa5c8d555b6e6008d1b0a5de6e45d2c9d install-kit-awindows-20260427-211240/windows/dlp-policy.example.json +0df53518a43bb1f0af7fdff5b44ba859e797090dd08680b239571d2e8a478d47 install-kit-awindows-20260427-211240/windows/dlp-policy.native-cross-os.example.json 863727465497b474d13d2270d443ff96ccb6076f90a5ce3eb270bdf8088e02dc install-kit-awindows-20260427-211240/windows/email-outbound-collector.ps1 -44d941322d618d92c72c3d24fe619d71e7551b9cdb35abe19f700c0b74da3eb1 install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 +2c0e7a96f32d546ce3ff8da2059259e6e5141c97cefc76bccae4c87c502f00cf install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 +ea68b869182587b7be41bf1891401bdf93dda3b9af6028afa0e30d245c77b0f4 install-kit-awindows-20260427-211240/windows/install-collector-guard-service.ps1 5ef21a25d5e2da4eeaef17126e60f96f195f90f9dc17a776f8629334b904d096 install-kit-awindows-20260427-211240/windows/migrate-awatch-rus-paths.ps1 -672d76824b1ca17d4c65e3097cf9895160a7bbce30323ef02d195fc9624538da install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 +d50228354f4c1f15b8ea9ccfef184a3a2e5cc832259ddbdc7afea4c083d03075 install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 731098681d89b9af6f3872abd586ac3b1faba2d7f9340211e503f52ad0243b3f install-kit-awindows-20260427-211240/windows/web-category-rules.example.json 1b7c337967236474484e781dc8ac37543509b051513f01e1a3145369262f5389 install-kit-awindows-20260427-211240/windows/worktime-session-collector.ps1 diff --git a/install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt b/install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt index 8435107..e2a80a1 100644 --- a/install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt +++ b/install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt @@ -4,6 +4,7 @@ Includes: - windows/* (deploy scripts, collectors, common module, configs/examples) - ansible/* (Windows and AW server playbooks, examples, inventory, tasks) - aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) +- scripts/* (install-kit rebuild/validation, quality gates, browser/web smoke checks) - server-configs-192.168.100.18/* (working Windows/RDP config snapshots) Source: diff --git a/install-kit-awindows-20260427-211240/ansible/README.md b/install-kit-awindows-20260427-211240/ansible/README.md index 7a1acbb..d251432 100644 --- a/install-kit-awindows-20260427-211240/ansible/README.md +++ b/install-kit-awindows-20260427-211240/ansible/README.md @@ -122,8 +122,9 @@ Playbook: - выгружает полный `windows/*` toolkit на целевой хост в InnoSetup-compatible каталог `C:\Program Files\AWatch-rus\windows`, включая DLP и `worktime-session-collector.ps1`; - если найден legacy config `C:\ProgramData\ActivityWatch-Phase2\deployment-config.json`, выполняет безопасную миграцию через `migrate-awatch-rus-paths.ps1`: backup, остановка задач, перенос данных, переписывание путей, пересоздание scheduled tasks и validation; - выполняет `deploy-ensemble.ps1` (deploy + hardening/recovery) с policy/rules из AWatch-rus toolkit; -- после deploy принудительно запускает `ActivityWatch Recovery` и все `ActivityWatch Launch *` задачи; -- включает (`Enable-ScheduledTask`) `ActivityWatch Recovery` и все `ActivityWatch Launch *` задачи перед запуском (иначе WebUI может показывать `Active time: 0s`); +- после deploy принудительно запускает `ActivityWatch Recovery` и managed `ActivityWatch Launch *` задачи; +- включает (`Enable-ScheduledTask`) `ActivityWatch Recovery` и managed `ActivityWatch Launch *` задачи перед запуском (иначе WebUI может показывать `Active time: 0s`); +- оставляет `ActivityWatch Recovery` включённым даже при активном `AWatchRusCollectorGuard`: guard является основным контроллером, recovery остаётся fallback/bootstrap path; - выполняет API smoke-check bucket `aw-watcher-afk_` и ожидает свежие события; - выполняет API smoke-check bucket `aw-watcher-window_` и ожидает свежие события (по умолчанию включено); - запускает `validate-deployment.ps1`; diff --git a/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml b/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml index 371dac5..7b68704 100644 --- a/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml +++ b/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml @@ -14,6 +14,7 @@ aw_sw_cleanup_cache_bust: "{{ lookup('file', aw_repo_root + '/aw-server/aw-sw-cleanup.js') | hash('sha1') | truncate(12, true, '') }}" aw_worktime_classes: "{{ lookup('file', aw_repo_root + '/aw-server/settings/classes-worktime.json') | from_json }}" aw_default_views: "{{ lookup('file', aw_repo_root + '/aw-server/settings/views-default.json') | from_json }}" + aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}" tasks: - name: Установить базовые пакеты @@ -29,6 +30,15 @@ state: present update_cache: true + - name: Установить пакеты для browser smoke проверки + ansible.builtin.apt: + name: + - chromium + - nodejs + - node-playwright + state: present + when: aw_browser_smoke_enabled | default(true) | bool + - name: Создать системную группу сервиса ansible.builtin.group: name: "{{ aw_server_group }}" @@ -63,10 +73,13 @@ - "{{ aw_server_data_dir }}/.config/activitywatch" - "{{ aw_server_data_dir }}/.config/activitywatch/aw-server-rust" - "{{ aw_server_data_dir }}/backups" + - "{{ aw_server_data_dir }}/slo" + - "{{ aw_server_data_dir }}/browser-smoke" - "{{ aw_rus_health_state_dir }}" - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" - /etc/activitywatch + - /usr/local/lib/aw-rus-browser-smoke - "{{ aw_bootstrap_dir }}" - name: Настроить каталоги ActivityWatch с владельцем сервиса @@ -90,6 +103,8 @@ - "{{ aw_server_data_dir }}/.config/activitywatch" - "{{ aw_server_data_dir }}/.config/activitywatch/aw-server-rust" - "{{ aw_server_data_dir }}/backups" + - "{{ aw_server_data_dir }}/slo" + - "{{ aw_server_data_dir }}/browser-smoke" - "{{ aw_rus_health_state_dir }}" - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" @@ -102,6 +117,30 @@ group: root mode: "0755" + - name: Положить legacy prune script в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/legacy/aw-prune-local-state.sh" + dest: /opt/activitywatch/aw-rus-ops/aw-prune-local-state.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust prune local state + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-prune-local-state" + delegate_to: localhost + register: aw_prune_local_state_rust_binary + become: false + + - name: Установить Rust prune local state + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-prune-local-state" + dest: /usr/local/bin/aw-prune-local-state-rust + owner: root + group: root + mode: "0755" + when: aw_prune_local_state_rust_binary.stat.exists | default(false) + - name: Ограничить рост journald на aw-server ansible.builtin.copy: dest: /etc/systemd/journald.conf.d/aw-rus-retention.conf @@ -376,20 +415,96 @@ dest: /opt/activitywatch/aw-server/apply_webui_ru_patch.sh mode: "0755" + - name: Создать server-side ops bundle для ранних health helpers + ansible.builtin.file: + path: /opt/activitywatch/aw-rus-ops + state: directory + owner: root + group: root + mode: "0755" + + - name: Установить wrapper aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-health-check-wrapper.sh" + dest: /usr/local/bin/aw-health-check + owner: root + group: root + mode: "0755" + + - name: Положить legacy aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/health-check.sh" + dest: /opt/activitywatch/aw-rus-ops/health-check.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust aw-health-check до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-health-check" + delegate_to: localhost + register: aw_health_check_rust_binary_early + become: false + + - name: Установить Rust aw-health-check до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-health-check" + dest: /usr/local/bin/aw-health-check-rust + owner: root + group: root + mode: "0755" + when: aw_health_check_rust_binary_early.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-health до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-health" + delegate_to: localhost + register: aw_db_health_rust_binary_early + become: false + + - name: Установить Rust aw-db-health до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-health" + dest: /usr/local/bin/aw-db-health + owner: root + group: root + mode: "0755" + when: aw_db_health_rust_binary_early.stat.exists | default(false) + + - name: Прочитать текущий aw-server.env для сохранения Influx token + ansible.builtin.slurp: + path: /etc/activitywatch/aw-server.env + register: aw_existing_server_env_raw + failed_when: false + no_log: true + + - name: Подготовить effective Influx tokens без вывода секретов + ansible.builtin.set_fact: + aw_existing_server_env_text: "{{ aw_existing_server_env_raw.content | default('') | b64decode }}" + aw_existing_worktime_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_WORKTIME_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_WORKTIME_INFLUX_TOKEN=', '')) }}" + aw_existing_dlp_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_DLP_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_DLP_INFLUX_TOKEN=', '')) }}" + no_log: true + + - name: Выбрать effective Influx tokens + ansible.builtin.set_fact: + aw_effective_worktime_influx_token: "{{ (aw_worktime_influx_token | default('') | string) if ((aw_worktime_influx_token | default('') | string | length) > 0) else aw_existing_worktime_influx_token }}" + aw_effective_dlp_influx_token: "{{ (aw_dlp_influx_token | default('') | string) if ((aw_dlp_influx_token | default('') | string | length) > 0) else aw_existing_dlp_influx_token }}" + no_log: true + - name: Проверить Influx token для AW worktime exporter ansible.builtin.assert: that: - - aw_worktime_influx_token is defined - - aw_worktime_influx_token | length > 0 - fail_msg: "aw_worktime_influx_enabled=true, но aw_worktime_influx_token пуст. Exporter будет падать и Grafana не получит worktime-ряды." + - aw_effective_worktime_influx_token is defined + - aw_effective_worktime_influx_token | length > 0 + fail_msg: "aw_worktime_influx_enabled=true, но token пуст и в локальном env, и в текущем /etc/activitywatch/aw-server.env. Exporter будет падать и Grafana не получит worktime-ряды." when: aw_worktime_influx_enabled | default(false) | bool - name: Проверить Influx token для AW DLP exporter ansible.builtin.assert: that: - - aw_dlp_influx_token is defined - - aw_dlp_influx_token | length > 0 - fail_msg: "aw_dlp_influx_enabled=true, но aw_dlp_influx_token пуст. Exporter будет падать и Grafana не получит DLP-ряды." + - aw_effective_dlp_influx_token is defined + - aw_effective_dlp_influx_token | length > 0 + fail_msg: "aw_dlp_influx_enabled=true, но token пуст и в локальном env, и в текущем /etc/activitywatch/aw-server.env. Exporter будет падать и Grafana не получит DLP-ряды." when: aw_dlp_influx_enabled | default(false) | bool - name: Записать /etc/activitywatch/aw-server.env перед хотфиксами @@ -427,7 +542,7 @@ AW_WORKTIME_INFLUX_BUCKET={{ aw_worktime_influx_bucket | default('aw_metrics') }} AW_WORKTIME_INFLUX_HOSTS={{ aw_worktime_influx_hosts | default('SHARKON2025') }} AW_WORKTIME_INFLUX_DAYS={{ aw_worktime_influx_days | default('today,yesterday') }} - AW_WORKTIME_INFLUX_TOKEN={{ aw_worktime_influx_token | default('') }} + AW_WORKTIME_INFLUX_TOKEN={{ aw_effective_worktime_influx_token | default('') }} AW_DLP_INFLUX_ENABLED={{ 'true' if (aw_dlp_influx_enabled | default(false) | bool) else 'false' }} AW_DLP_INFLUX_URL={{ aw_dlp_influx_url | default('') }} AW_DLP_INFLUX_ORG={{ aw_dlp_influx_org | default('proxmox') }} @@ -435,7 +550,7 @@ AW_DLP_INFLUX_HOSTS={{ aw_dlp_influx_hosts | default('SHARKON2025') }} AW_DLP_INFLUX_LOOKBACK_DAYS={{ aw_dlp_influx_lookback_days | default(30) }} AW_DLP_INFLUX_EVENT_LIMIT={{ aw_dlp_influx_event_limit | default(2000) }} - AW_DLP_INFLUX_TOKEN={{ aw_dlp_influx_token | default('') }} + AW_DLP_INFLUX_TOKEN={{ aw_effective_dlp_influx_token | default('') }} AW_DLP_AW_API_BASE=http://127.0.0.1:5600/api/0 AW_DLP_CASE_API_BASE=http://127.0.0.1:5602/api/0/dlp/cases AW_EXPECT_START_OF_DAY={{ aw_worktime_start_of_day | default('') }} @@ -448,6 +563,19 @@ AW_RUS_HEALTH_STATE_DIR={{ aw_rus_health_state_dir }} AW_RUS_HEALTH_VALIDATION_DIR={{ aw_rus_health_validation_dir }} AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS={{ aw_rus_health_session_events_max_age_seconds | default(86400) }} + AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS={{ aw_rus_health_guard_max_age_seconds | default(300) }} + AW_RUS_HEALTH_GUARD_REQUIRED={{ 1 if (aw_rus_health_guard_required | default(true) | bool) else 0 }} + AW_RUS_SLO_STATE_DIR={{ aw_server_data_dir }}/slo + AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600 + AW_RUS_SLO_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }} + AW_RUS_SLO_TARGET_PERCENT={{ aw_rus_slo_target_percent | default('99.97') }} + AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600 + AW_BROWSER_SMOKE_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }} + AW_BROWSER_SMOKE_HOST={{ aw_monitored_windows_hostname }} + AW_BROWSER_SMOKE_OUTPUT_DIR={{ aw_server_data_dir }}/browser-smoke + AW_BROWSER_SMOKE_ENGINE={{ aw_browser_smoke_engine | default('chromium-cli') }} + AW_BROWSER_SMOKE_TIMEOUT_MS={{ aw_browser_smoke_timeout_ms | default(20000) }} + AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS={{ aw_browser_smoke_render_timeout_ms | default(15000) }} AW_HAYABUSA_AUTO_CASE_ENABLED={{ 'true' if (aw_hayabusa_auto_case_enabled | default(true) | bool) else 'false' }} AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY={{ aw_hayabusa_auto_case_min_severity | default('medium') }} AW_HAYABUSA_TELEGRAM_ENABLED={{ 'true' if (aw_hayabusa_telegram_enabled | default(false) | bool) else 'false' }} @@ -501,6 +629,50 @@ mode: "0644" when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Проверить локальный Rust DLP policy engine + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-policy-engine" + delegate_to: localhost + register: aw_dlp_policy_engine_rust_binary + become: false + when: aw_dlp_policy_engine_enabled | default(false) | bool + + - name: Установить Rust DLP policy engine + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-policy-engine" + dest: /usr/local/bin/aw-dlp-policy-engine-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP policy engine + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-policy-engine.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + + - name: Переключить DLP policy engine на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-policy-engine.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-policy-engine-rust + when: + - aw_dlp_policy_engine_enabled | default(false) | bool + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + - name: Создать каталог DLP content analysis ansible.builtin.file: path: /opt/activitywatch/dlp-content-analysis @@ -528,6 +700,25 @@ mode: "0755" when: aw_dlp_content_analysis_enabled | default(true) | bool + - name: Проверить локальный Rust DLP content analyzer + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-content-analyzer" + delegate_to: localhost + register: dlp_content_analyzer_rust_binary + become: false + when: aw_dlp_content_analysis_enabled | default(true) | bool + + - name: Установить Rust DLP content analyzer + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-content-analyzer" + dest: /usr/local/bin/aw-dlp-content-analyzer-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_content_analysis_enabled | default(true) | bool + - dlp_content_analyzer_rust_binary.stat.exists | default(false) + - name: Создать virtualenv DLP content analysis ansible.builtin.command: cmd: python3 -m venv /opt/activitywatch/dlp-content-analysis/.venv @@ -619,6 +810,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust CEF exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-cef-exporter" + delegate_to: localhost + register: dlp_cef_exporter_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust CEF exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-cef-exporter" + dest: /usr/local/bin/dlp-cef-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог CEF exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-cef-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить CEF exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-cef-exporter-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd unit syslog forwarder ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/syslog-forwarder.service" @@ -637,6 +872,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust syslog forwarder + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" + delegate_to: localhost + register: dlp_syslog_forwarder_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust syslog forwarder + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" + dest: /usr/local/bin/dlp-syslog-forwarder-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог syslog forwarder + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + + - name: Переключить syslog forwarder на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + - name: Установить systemd unit webhook sender ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/webhook-sender.service" @@ -655,6 +934,50 @@ mode: "0644" when: aw_dlp_integrations_enabled | default(true) | bool + - name: Проверить локальный Rust webhook sender + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-webhook-sender" + delegate_to: localhost + register: dlp_webhook_sender_rust_binary + become: false + when: aw_dlp_integrations_enabled | default(true) | bool + + - name: Установить Rust webhook sender + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-webhook-sender" + dest: /usr/local/bin/dlp-webhook-sender-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог webhook sender + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-webhook-sender.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + + - name: Переключить webhook sender на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-webhook-sender-rust + when: + - aw_dlp_integrations_enabled | default(true) | bool + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + - name: Создать каталог DLP case management ansible.builtin.file: path: /opt/activitywatch/dlp-case-management @@ -710,6 +1033,50 @@ mode: "0644" when: aw_dlp_case_management_enabled | default(true) | bool + - name: Проверить локальный Rust DLP case management + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-case-management" + delegate_to: localhost + register: aw_dlp_case_management_rust_binary + become: false + when: aw_dlp_case_management_enabled | default(true) | bool + + - name: Установить Rust DLP case management + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-case-management" + dest: /usr/local/bin/aw-dlp-case-management-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP case management + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-case-management.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + + - name: Переключить DLP case management на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-case-management.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-case-management-rust + when: + - aw_dlp_case_management_enabled | default(true) | bool + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + - name: Создать каталоги DLP compliance ansible.builtin.file: path: "{{ item }}" @@ -753,6 +1120,50 @@ virtualenv: /opt/activitywatch/dlp-compliance/.venv when: aw_dlp_compliance_enabled | default(true) | bool + - name: Проверить локальный Rust DLP compliance + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-compliance" + delegate_to: localhost + register: aw_dlp_compliance_rust_binary + become: false + when: aw_dlp_compliance_enabled | default(true) | bool + + - name: Установить Rust DLP compliance + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-compliance" + dest: /usr/local/bin/aw-dlp-compliance-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP compliance report scheduler + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-report-scheduler.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + + - name: Переключить DLP compliance report scheduler на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-report-scheduler.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-compliance-rust + when: + - aw_dlp_compliance_enabled | default(true) | bool + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + - name: Установить dlp-admin-cli ansible.builtin.copy: src: "{{ aw_repo_root }}/scripts/dlp-admin-cli.py" @@ -761,6 +1172,21 @@ group: root mode: "0755" + - name: Проверить локальный Rust dlp-admin-cli + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-admin-cli" + delegate_to: localhost + register: aw_dlp_admin_cli_rust_binary + + - name: Установить Rust dlp-admin-cli + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-admin-cli" + dest: /usr/local/bin/dlp-admin-cli + owner: root + group: root + mode: "0755" + when: aw_dlp_admin_cli_rust_binary.stat.exists + - name: Установить скрипт AW worktime API ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" @@ -769,6 +1195,22 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime API + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-api" + delegate_to: localhost + register: aw_worktime_api_rust_binary + become: false + + - name: Установить Rust AW worktime API + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-api" + dest: /usr/local/bin/aw-worktime-api-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_api_rust_binary.stat.exists | default(false) + - name: Установить systemd unit AW worktime API ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.service" @@ -777,6 +1219,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime API + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-api.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_api_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime API на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-api.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-api-rust + when: aw_worktime_api_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.py" @@ -785,6 +1248,22 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime UI bridge + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-ui-bridge" + delegate_to: localhost + register: aw_worktime_ui_bridge_rust_binary + become: false + + - name: Установить Rust AW worktime UI bridge + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-ui-bridge" + dest: /usr/local/bin/aw-worktime-ui-bridge-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.sh" @@ -793,6 +1272,46 @@ group: root mode: "0755" + - name: Проверить локальный Rust AW worktime autoheal + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-autoheal" + delegate_to: localhost + register: aw_worktime_autoheal_rust_binary + become: false + + - name: Установить Rust AW worktime autoheal + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-autoheal" + dest: /usr/local/bin/aw-worktime-autoheal-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + + - name: Установить скрипт AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.sh" + dest: /usr/local/bin/aw-worktime-prewarm.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust AW worktime prewarm + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-prewarm" + delegate_to: localhost + register: aw_worktime_prewarm_rust_binary + become: false + + - name: Установить Rust AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-prewarm" + dest: /usr/local/bin/aw-worktime-prewarm-rust + owner: root + group: root + mode: "0755" + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + - name: Установить скрипт AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.py" @@ -811,21 +1330,118 @@ mode: "0755" when: aw_dlp_influx_enabled | default(false) | bool + - name: Создать server-side ops bundle для legacy aw-health-check + ansible.builtin.file: + path: /opt/activitywatch/aw-rus-ops + state: directory + owner: root + group: root + mode: "0755" + - name: Установить скрипт aw-health-check ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" + src: "{{ aw_repo_root }}/aw-server/aw-health-check-wrapper.sh" dest: /usr/local/bin/aw-health-check owner: root group: root mode: "0755" - - name: Установить скрипт dlp-health-check + - name: Положить legacy aw-health-check в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/health-check.sh" + dest: /opt/activitywatch/aw-rus-ops/health-check.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust aw-health-check + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-health-check" + delegate_to: localhost + register: aw_health_check_rust_binary + become: false + + - name: Установить Rust aw-health-check + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-health-check" + dest: /usr/local/bin/aw-health-check-rust + owner: root + group: root + mode: "0755" + when: aw_health_check_rust_binary.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-health + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-health" + delegate_to: localhost + register: aw_db_health_rust_binary + become: false + + - name: Установить Rust aw-db-health + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-health" + dest: /usr/local/bin/aw-db-health + owner: root + group: root + mode: "0755" + when: aw_db_health_rust_binary.stat.exists | default(false) + + - name: Установить скрипт check-aw-data + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/check-aw-data-wrapper.sh" + dest: /usr/local/bin/check-aw-data + owner: root + group: root + mode: "0755" + + - name: Положить legacy check-aw-data в server-side ops bundle + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/legacy/check-aw-data.sh" + dest: /opt/activitywatch/aw-rus-ops/check-aw-data.sh + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust check-aw-data + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/check-aw-data" + delegate_to: localhost + register: check_aw_data_rust_binary + become: false + + - name: Установить Rust check-aw-data + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/check-aw-data" + dest: /usr/local/bin/check-aw-data-rust + owner: root + group: root + mode: "0755" + when: check_aw_data_rust_binary.stat.exists | default(false) + + - name: Проверить локальный Rust dlp-health-check + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-health-check" + delegate_to: localhost + register: dlp_health_check_rust_binary + become: false + + - name: Установить Rust dlp-health-check + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-health-check" + dest: /usr/local/bin/dlp-health-check + owner: root + group: root + mode: "0755" + when: dlp_health_check_rust_binary.stat.exists | default(false) + + - name: Установить legacy Python dlp-health-check ansible.builtin.copy: src: "{{ aw_repo_root }}/scripts/dlp-health-check.py" dest: /usr/local/bin/dlp-health-check owner: root group: root mode: "0755" + when: not (dlp_health_check_rust_binary.stat.exists | default(false)) - name: Установить скрипт aw-rus-healthd ansible.builtin.copy: @@ -835,6 +1451,39 @@ group: root mode: "0755" + - name: Установить скрипт aw-slo-monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.py" + dest: /usr/local/bin/aw-slo-monitor.py + owner: root + group: root + mode: "0755" + + - name: Проверить локальный Rust AW-RUS SLO monitor + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-slo-monitor" + delegate_to: localhost + register: aw_slo_monitor_rust_binary + become: false + + - name: Установить Rust AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-slo-monitor" + dest: /usr/local/bin/aw-slo-monitor-rust + owner: root + group: root + mode: "0755" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Установить browser smoke скрипт + ansible.builtin.copy: + src: "{{ aw_repo_root }}/scripts/aw-webui-browser-smoke.mjs" + dest: /usr/local/lib/aw-rus-browser-smoke/aw-webui-browser-smoke.mjs + owner: root + group: root + mode: "0755" + when: aw_browser_smoke_enabled | default(true) | bool + - name: Установить systemd unit AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.service" @@ -843,6 +1492,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime UI bridge + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-ui-bridge.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime UI bridge на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-ui-bridge.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-ui-bridge-rust + when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.timer" @@ -859,6 +1529,27 @@ group: root mode: "0644" + - name: Создать drop-in каталог AW worktime autoheal + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-autoheal.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime autoheal на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-autoheal.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-autoheal-rust + when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.timer" @@ -867,6 +1558,43 @@ group: root mode: "0644" + - name: Установить systemd unit AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.service" + dest: /etc/systemd/system/aw-worktime-prewarm.service + owner: root + group: root + mode: "0644" + + - name: Создать drop-in каталог AW worktime prewarm + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-prewarm.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime prewarm на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-prewarm.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-prewarm-rust + when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) + + - name: Установить systemd timer AW worktime prewarm + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.timer" + dest: /etc/systemd/system/aw-worktime-prewarm.timer + owner: root + group: root + mode: "0644" + - name: Установить systemd unit AW-RUS healthd ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-rus-healthd.service" @@ -883,6 +1611,58 @@ group: root mode: "0644" + - name: Установить systemd unit AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.service" + dest: /etc/systemd/system/aw-slo-monitor.service + owner: root + group: root + mode: "0644" + + - name: Создать drop-in каталог AW-RUS SLO monitor + ansible.builtin.file: + path: /etc/systemd/system/aw-slo-monitor.service.d + state: directory + owner: root + group: root + mode: "0755" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Переключить AW-RUS SLO monitor на Rust + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor-rust-switch.conf" + dest: /etc/systemd/system/aw-slo-monitor.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + when: aw_slo_monitor_rust_binary.stat.exists | default(false) + + - name: Установить systemd timer AW-RUS SLO monitor + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.timer" + dest: /etc/systemd/system/aw-slo-monitor.timer + owner: root + group: root + mode: "0644" + + - name: Установить systemd unit AW-RUS browser smoke + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-browser-smoke.service" + dest: /etc/systemd/system/aw-browser-smoke.service + owner: root + group: root + mode: "0644" + when: aw_browser_smoke_enabled | default(true) | bool + + - name: Установить systemd timer AW-RUS browser smoke + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-browser-smoke.timer" + dest: /etc/systemd/system/aw-browser-smoke.timer + owner: root + group: root + mode: "0644" + when: aw_browser_smoke_enabled | default(true) | bool + - name: Установить systemd unit AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.service" @@ -892,6 +1672,50 @@ mode: "0644" when: aw_worktime_influx_enabled | default(false) | bool + - name: Проверить локальный Rust AW worktime Influx exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/worktime-influx-exporter" + delegate_to: localhost + register: aw_worktime_influx_exporter_rust_binary + become: false + when: aw_worktime_influx_enabled | default(false) | bool + + - name: Установить Rust AW worktime Influx exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/worktime-influx-exporter" + dest: /usr/local/bin/aw-worktime-influx-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог AW worktime Influx exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-worktime-influx-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить AW worktime Influx exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-worktime-influx-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-worktime-influx-exporter-rust + when: + - aw_worktime_influx_enabled | default(false) | bool + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.timer" @@ -910,6 +1734,50 @@ mode: "0644" when: aw_dlp_influx_enabled | default(false) | bool + - name: Проверить локальный Rust AW DLP Influx exporter + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-influx-exporter" + delegate_to: localhost + register: aw_dlp_influx_exporter_rust_binary + become: false + when: aw_dlp_influx_enabled | default(false) | bool + + - name: Установить Rust AW DLP Influx exporter + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-influx-exporter" + dest: /usr/local/bin/aw-dlp-influx-exporter-rust + owner: root + group: root + mode: "0755" + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог AW DLP Influx exporter + ansible.builtin.file: + path: /etc/systemd/system/aw-dlp-influx-exporter.service.d + state: directory + owner: root + group: root + mode: "0755" + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + + - name: Переключить AW DLP Influx exporter на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/aw-dlp-influx-exporter.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/aw-dlp-influx-exporter-rust + when: + - aw_dlp_influx_enabled | default(false) | bool + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + - name: Установить systemd timer AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.timer" @@ -929,6 +1797,19 @@ enabled: true state: started + - name: Включить таймер AW-RUS SLO monitor + ansible.builtin.systemd: + name: aw-slo-monitor.timer + enabled: true + state: started + + - name: Включить таймер AW-RUS browser smoke + ansible.builtin.systemd: + name: aw-browser-smoke.timer + enabled: true + state: started + when: aw_browser_smoke_enabled | default(true) | bool + - name: Включить и перезапустить DLP policy engine ansible.builtin.systemd: name: aw-dlp-policy-engine.service @@ -1015,6 +1896,18 @@ state: started failed_when: false + - name: Включить и перезапустить AW worktime prewarm timer + ansible.builtin.systemd: + name: aw-worktime-prewarm.timer + enabled: true + state: restarted + + - name: Выполнить разовый прогон AW worktime prewarm + ansible.builtin.systemd: + name: aw-worktime-prewarm.service + state: started + failed_when: false + - name: Включить и перезапустить AW worktime Influx exporter timer ansible.builtin.systemd: name: aw-worktime-influx-exporter.timer @@ -1115,7 +2008,9 @@ ansible.builtin.systemd: name: activitywatch-server.service state: stopped - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Создать backup каталоги server DB ansible.builtin.file: @@ -1124,7 +2019,9 @@ owner: "{{ aw_server_user }}" group: "{{ aw_server_group }}" mode: "0755" - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Backup target DB перед merge ansible.builtin.copy: @@ -1135,6 +2032,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1146,7 +2044,9 @@ owner: "{{ aw_server_user }}" group: "{{ aw_server_group }}" mode: "0644" - when: aw_legacy_root_db.stat.exists | default(false) + when: + - aw_legacy_db_merge_enabled | default(false) | bool + - aw_legacy_root_db.stat.exists | default(false) - name: Merge legacy root DB в target DB ansible.builtin.command: @@ -1162,6 +2062,7 @@ register: aw_merge_result failed_when: false when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1169,6 +2070,7 @@ ansible.builtin.debug: msg: "{{ aw_merge_result.stdout | default(aw_merge_result.stderr | default('merge not executed')) }}" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) @@ -1181,6 +2083,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - aw_target_db.stat.exists | default(false) - aw_merge_result is defined @@ -1195,6 +2098,7 @@ group: "{{ aw_server_group }}" mode: "0644" when: + - aw_legacy_db_merge_enabled | default(false) | bool - aw_legacy_root_db.stat.exists | default(false) - not (aw_target_db.stat.exists | default(false)) @@ -1306,6 +2210,46 @@ [Install] WantedBy=multi-user.target + - name: Проверить локальный Rust DLP aggregator + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/dlp-aggregator" + delegate_to: localhost + register: dlp_aggregator_rust_binary + become: false + + - name: Установить Rust DLP aggregator + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/dlp-aggregator" + dest: /usr/local/bin/dlp-aggregator-rust + owner: root + group: root + mode: "0755" + when: dlp_aggregator_rust_binary.stat.exists | default(false) + + - name: Создать drop-in каталог DLP aggregator + ansible.builtin.file: + path: /etc/systemd/system/activitywatch-dlp-aggregator.service.d + state: directory + owner: root + group: root + mode: "0755" + when: dlp_aggregator_rust_binary.stat.exists | default(false) + + - name: Переключить DLP aggregator на Rust + ansible.builtin.copy: + dest: /etc/systemd/system/activitywatch-dlp-aggregator.service.d/20-rust-switch.conf + owner: root + group: root + mode: "0644" + content: | + [Service] + ExecStart= + ExecStart=/usr/local/bin/dlp-aggregator-rust \ + --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ + --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ + --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json + when: dlp_aggregator_rust_binary.stat.exists | default(false) + - name: Установить systemd timer для агрегатора ansible.builtin.copy: dest: /etc/systemd/system/activitywatch-dlp-aggregator.timer @@ -1314,9 +2258,10 @@ Description=Run ActivityWatch DLP Aggregator every 5 minutes [Timer] - OnBootSec=1min - OnUnitActiveSec=5min - AccuracySec=1s + OnCalendar=*:3/10:10 + AccuracySec=30s + RandomizedDelaySec=30s + Persistent=false [Install] WantedBy=timers.target @@ -1681,9 +2626,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper link-case для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + delegate_to: localhost + register: aw_hayabusa_link_case_rust_binary + become: false + + - name: Установить Rust helper link-case для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + dest: /usr/local/bin/aw-hayabusa-link-case-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + - name: Установить helper link-case для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-link-case.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-link-case-rust ]]; then + exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py "$@" dest: /usr/local/bin/aw-hayabusa-link-case owner: root group: root @@ -1697,9 +2664,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper from-windows для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + delegate_to: localhost + register: aw_hayabusa_from_windows_rust_binary + become: false + + - name: Установить Rust helper from-windows для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + dest: /usr/local/bin/aw-hayabusa-from-windows-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + - name: Установить helper from-windows для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-from-windows.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-from-windows-rust ]]; then + exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py "$@" dest: /usr/local/bin/aw-hayabusa-from-windows owner: root group: root @@ -1721,9 +2710,31 @@ group: root mode: "0644" + - name: Проверить локальный Rust helper autoprocess для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + delegate_to: localhost + register: aw_hayabusa_autoprocess_rust_binary + become: false + + - name: Установить Rust helper autoprocess для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + dest: /usr/local/bin/aw-hayabusa-autoprocess-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + - name: Установить helper autoprocess для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-autoprocess.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-autoprocess-rust ]]; then + exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py "$@" dest: /usr/local/bin/aw-hayabusa-autoprocess owner: root group: root @@ -1737,9 +2748,31 @@ group: root mode: "0755" + - name: Проверить локальный Rust helper case-alert для Hayabusa + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + delegate_to: localhost + register: aw_hayabusa_case_alert_rust_binary + become: false + + - name: Установить Rust helper case-alert для Hayabusa + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + dest: /usr/local/bin/aw-hayabusa-case-alert-rust + owner: root + group: root + mode: "0755" + when: aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + - name: Установить helper case-alert для Hayabusa ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-case-alert.py" + content: | + #!/usr/bin/env bash + set -euo pipefail + if [[ -x /usr/local/bin/aw-hayabusa-case-alert-rust ]]; then + exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" + fi + exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py "$@" dest: /usr/local/bin/aw-hayabusa-case-alert owner: root group: root diff --git a/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml b/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml index eec5eeb..e71a3a0 100644 --- a/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml +++ b/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml @@ -50,7 +50,7 @@ aw_windows_incident_artifacts_root: "{{ aw_windows_state_root }}\\incident-artifacts" aw_windows_forensics_root: "{{ aw_windows_state_root }}\\forensics\\evtx-exports" aw_windows_logon_marker_enabled: true - aw_windows_process_events_enabled: true + aw_windows_process_events_enabled: false aw_windows_skip_hardening: false aw_windows_rules_path: "{{ aw_windows_deploy_root }}\\windows\\web-category-rules.example.json" aw_windows_policy_path: "{{ aw_windows_deploy_root }}\\windows\\dlp-policy.example.json" @@ -58,6 +58,10 @@ aw_windows_validation_local_dir: "/tmp/aw-rus-validation-{{ lookup('env','USER') | default('ansible', true) }}" aw_windows_launch_task_pattern: "ActivityWatch Launch *" aw_windows_recovery_task_name: "ActivityWatch Recovery" + aw_windows_collector_guard_enabled: true + aw_windows_collector_guard_mode: "enforce" + aw_windows_collector_guard_service_name: "AWatchRusCollectorGuard" + aw_windows_collector_guard_loop_seconds: 60 aw_windows_force_task_restart: true aw_windows_api_smoke_check_enabled: true aw_windows_api_smoke_check_bucket: "" @@ -181,6 +185,9 @@ - deploy-domain-users.ps1 - deploy-ensemble.ps1 - hardening-recovery.ps1 + - AWatchRusCollectorGuardService.cs + - aw-collector-guard.ps1 + - install-collector-guard-service.ps1 - rebuild-worktime-tasks.ps1 - audit-cryptopro.ps1 - validate-deployment.ps1 @@ -371,27 +378,50 @@ Start-Sleep -Seconds 2 + $modulePath = "{{ aw_windows_deploy_root }}\windows\ActivityWatch.Windows.Common.psm1" + Import-Module $modulePath -Force + # Ensure tasks are enabled (some environments keep them disabled, causing "0s" in WebUI). try { Enable-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" -ErrorAction SilentlyContinue | Out-Null } catch {} $config = Get-Content -Raw -LiteralPath "{{ aw_windows_state_root }}\deployment-config.json" | ConvertFrom-Json - foreach ($taskDef in @($config.userTasks)) { - try { Enable-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue | Out-Null } catch {} + $configPaths = Get-ActivityWatchRecoveryConfigPaths -PrimaryConfigPath "{{ aw_windows_state_root }}\deployment-config.json" + $taskDefs = @(Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths) + $sessionRecords = @(Get-ActivityWatchSessionRecords) + + foreach ($taskDef in @($taskDefs)) { + try { Enable-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue | Out-Null } catch {} } $recoveryTask = Get-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" -ErrorAction SilentlyContinue if ($recoveryTask -and $recoveryTask.State -notin @('Running', 'Queued')) { Start-ScheduledTask -TaskName "{{ aw_windows_recovery_task_name }}" } - foreach ($taskDef in @($config.userTasks)) { - $launchTask = Get-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue + foreach ($taskDef in @($taskDefs)) { + if (-not (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected)) { + continue + } + $launchTask = Get-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue if ($launchTask -and $launchTask.State -notin @('Running', 'Queued')) { - Start-ScheduledTask -TaskName ([string]$taskDef.launchTaskName) -ErrorAction SilentlyContinue + Start-ScheduledTask -TaskName ([string]$taskDef.taskName) -ErrorAction SilentlyContinue } } + - name: Установить session-aware AWatch-rus Collector Guard service + when: aw_windows_collector_guard_enabled | bool + ansible.windows.win_powershell: + script: | + $ErrorActionPreference = 'Stop' + $guardParams = @{ + ConfigPath = "{{ aw_windows_state_root }}\deployment-config.json" + Mode = "{{ aw_windows_collector_guard_mode }}" + ServiceName = "{{ aw_windows_collector_guard_service_name }}" + LoopSeconds = {{ aw_windows_collector_guard_loop_seconds | int }} + } + & "{{ aw_windows_deploy_root }}\windows\install-collector-guard-service.ps1" @guardParams + - name: Получить Windows hostname для AW smoke-check bucket when: - aw_windows_api_smoke_check_enabled | bool diff --git a/install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml b/install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml index 2bcd41e..6e5d732 100644 --- a/install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml +++ b/install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml @@ -32,6 +32,11 @@ aw_monitored_windows_hostname: "SHARKON2025" aw_rus_health_worktime_api_base: "http://127.0.0.1:5610" aw_rus_health_state_dir: "{{ aw_server_data_dir }}/health" aw_rus_health_validation_dir: "{{ aw_rus_health_state_dir }}/windows-validation" +aw_browser_smoke_enabled: true +aw_browser_smoke_engine: "chromium-cli" +aw_legacy_db_merge_enabled: false +aw_browser_smoke_timeout_ms: 20000 +aw_browser_smoke_render_timeout_ms: 15000 aw_hayabusa_auto_case_enabled: true aw_hayabusa_auto_case_min_severity: "medium" aw_hayabusa_telegram_enabled: true @@ -62,7 +67,7 @@ aw_worktime_from: "00:00" aw_worktime_to: "17:00" aw_worktime_start_of_day: "{{ aw_worktime_from }}" aw_server_always_active_pattern: "aw-watcher-window" -aw_server_landingpage: "/activity/SHARKON2025/view/" +aw_server_landingpage: "/#/activity/SHARKON2025/view/" aw_health_strict_fileops: 0 aw_dlp_policy_engine_enabled: true diff --git a/install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml b/install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml index ae3eee2..9f6ad99 100644 --- a/install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml +++ b/install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml @@ -50,7 +50,7 @@ aw_windows_evtx_channels: - Microsoft-Windows-TerminalServices-LocalSessionManager/Operational - Microsoft-Windows-TerminalServices-RemoteConnectionManager/Operational aw_windows_logon_marker_enabled: true -aw_windows_process_events_enabled: true +aw_windows_process_events_enabled: false aw_windows_skip_hardening: false aw_windows_rules_path: "{{ aw_windows_deploy_root }}\\windows\\web-category-rules.example.json" @@ -60,6 +60,13 @@ aw_windows_validation_remote_path: "{{ aw_windows_state_root }}\\aw_validate_ans aw_windows_validation_local_dir: "/tmp/aw-rus-validation-{{ lookup('env','USER') | default('ansible', true) }}" aw_windows_fail_on_validation_error: true +# Collector Guard supervises collectors, but ActivityWatch Recovery must stay enabled +# as a fallback and as the launch-task bootstrap path for managed RDP sessions. +aw_windows_collector_guard_enabled: true +aw_windows_collector_guard_mode: "enforce" +aw_windows_collector_guard_service_name: "AWatchRusCollectorGuard" +aw_windows_collector_guard_loop_seconds: 60 + # Безопасная миграция текущего прода со старых путей в единый профиль AWatch-rus. aw_windows_migration_enabled: true aw_windows_legacy_install_root: "C:\\Program Files\\ActivityWatch-Phase2" diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.service b/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.service new file mode 100644 index 0000000..41b74e9 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.service @@ -0,0 +1,28 @@ +[Unit] +Description=AW-RUS browser-visible WebUI smoke check +After=network-online.target activitywatch-server.service aw-worktime-api.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +Environment=NODE_PATH=/usr/share/nodejs +Environment=AW_BROWSER_SMOKE_ENGINE=chromium-cli +Environment=HOME=/var/lib/activitywatch/browser-smoke +Environment=XDG_CONFIG_HOME=/var/lib/activitywatch/browser-smoke/.config +Environment=XDG_CACHE_HOME=/var/lib/activitywatch/browser-smoke/.cache +WorkingDirectory=/var/lib/activitywatch/browser-smoke +ExecStart=/usr/bin/node /usr/local/lib/aw-rus-browser-smoke/aw-webui-browser-smoke.mjs +TimeoutStartSec=180 +User=activitywatch +Group=activitywatch +Nice=15 +CPUQuota=25% +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/activitywatch/browser-smoke +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-browser-smoke diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.timer b/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.timer new file mode 100644 index 0000000..d956c91 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AW-RUS browser-visible WebUI smoke check hourly + +[Timer] +OnCalendar=hourly +AccuracySec=5min +RandomizedDelaySec=10min +Persistent=false +Unit=aw-browser-smoke.service + +[Install] +WantedBy=timers.target diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py index 234b90c..07d0538 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py +++ b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py @@ -8,6 +8,7 @@ import socket import subprocess import sys import tempfile +import time from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -21,7 +22,12 @@ ENV_FILE = Path("/etc/activitywatch/aw-server.env") def load_env_file(path: Path) -> None: if not path.exists(): return - for raw_line in path.read_text(encoding="utf-8").splitlines(): + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + # systemd EnvironmentFile has already injected the variables for service runs. + return + for raw_line in lines: line = raw_line.strip() if not line or line.startswith("#") or "=" not in line: continue @@ -55,9 +61,19 @@ def age_seconds(ts: datetime | None, now: datetime) -> int | None: return max(0, int((now - ts).total_seconds())) -def http_json(url: str, timeout: int = 10) -> Any: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) +def http_json(url: str, timeout: int = 20, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: + last_exc: Exception | None = None + for attempt in range(max(1, attempts)): + try: + with request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception as exc: + last_exc = exc + if attempt + 1 >= max(1, attempts): + break + time.sleep(backoff_seconds * (2**attempt)) + assert last_exc is not None + raise last_exc def run_command(cmd: list[str]) -> tuple[int, str]: @@ -136,7 +152,7 @@ class Report: def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: - events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=20") + events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1", timeout=25) if isinstance(events, list) and events: events = [item for item in events if isinstance(item, dict)] if not events: @@ -146,6 +162,16 @@ def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: return None +def bucket_metadata_ts(buckets: dict[str, Any], bucket_id: str) -> datetime | None: + bucket = buckets.get(bucket_id) if isinstance(buckets, dict) else None + if not isinstance(bucket, dict): + return None + metadata = bucket.get("metadata") + if not isinstance(metadata, dict): + return None + return parse_ts(metadata.get("end")) + + def host_activity_from_worktime(event: dict[str, Any] | None, max_age_seconds: int) -> dict[str, Any]: now = now_utc() if not event: @@ -189,6 +215,78 @@ def bucket_health( return "ok", f"fresh ({age}s)", details +def bucket_timestamp_health( + api_base: str, + buckets: dict[str, Any], + bucket_id: str, + max_age_seconds: int, + missing_status: str, + stale_status: str, +) -> tuple[str, str, dict[str, Any]]: + metadata_ts = bucket_metadata_ts(buckets, bucket_id) + if metadata_ts is not None: + age = age_seconds(metadata_ts, now_utc()) + timestamp = metadata_ts.isoformat().replace("+00:00", "Z") + details = { + "bucket": bucket_id, + "timestamp": timestamp, + "age_seconds": age, + "timestamp_source": "bucket_metadata.end", + } + if age is None: + return "warn", "timestamp parse failed", details + if age > max_age_seconds: + return stale_status, f"stale ({age}s)", details + return "ok", f"fresh ({age}s)", details + + return bucket_health(api_base, bucket_id, max_age_seconds, missing_status, stale_status) + + +def guard_bucket_health( + api_base: str, + host: str, + max_age_seconds: int, + required: bool, +) -> tuple[str, str, dict[str, Any]]: + bucket_id = f"aw-rus-collector-guard_{host}" + try: + event = latest_bucket_event(api_base, bucket_id) + except Exception as exc: + status = "fail" if required else "warn" + return status, f"guard bucket query failed: {exc}", {"bucket": bucket_id} + + if not event: + status = "fail" if required else "warn" + return status, "no guard heartbeat", {"bucket": bucket_id, "required": required} + + ts = parse_ts(event.get("timestamp")) + age = age_seconds(ts, now_utc()) + data = event.get("data") if isinstance(event.get("data"), dict) else {} + guard_status = str(data.get("status") or "unknown").lower() + details = { + "bucket": bucket_id, + "timestamp": event.get("timestamp"), + "age_seconds": age, + "required": required, + "guard_status": guard_status, + "mode": data.get("mode"), + "live_session_count": data.get("liveSessionCount"), + "problems": data.get("problems", []), + "actions": data.get("actions", []), + } + + stale_status = "fail" if required else "warn" + if age is None: + return "warn", "guard timestamp parse failed", details + if age > max_age_seconds: + return stale_status, f"guard stale ({age}s)", details + if guard_status in ("fail", "error"): + return "fail" if required else "warn", f"guard reports {guard_status}", details + if guard_status == "warn": + return "warn", "guard reports warn", details + return "ok", f"guard fresh ({age}s)", details + + def latest_validation_report(validation_dir: Path) -> Path | None: candidates = sorted( (path for path in validation_dir.glob("*-aw_validate_ansible.json") if path.is_file()), @@ -206,7 +304,20 @@ def write_atomic(path: Path, content: str) -> None: os.replace(tmp_name, path) -def check_wrapper(report: Report, name: str, cmd: list[str], json_mode: bool = False) -> None: +def chmod_if_possible(path: Path, mode: int) -> None: + try: + path.chmod(mode) + except OSError: + pass + + +def check_wrapper( + report: Report, + name: str, + cmd: list[str], + json_mode: bool = False, + failure_status: str = "fail", +) -> None: if not Path(cmd[0]).exists(): report.add(name, "warn", "binary missing", command=cmd) return @@ -221,7 +332,7 @@ def check_wrapper(report: Report, name: str, cmd: list[str], json_mode: bool = F return else: details["output"] = output - report.add(name, "ok" if rc == 0 else "fail", "passed" if rc == 0 else "failed", **details) + report.add(name, "ok" if rc == 0 else failure_status, "passed" if rc == 0 else "failed", **details) def main() -> int: @@ -240,6 +351,8 @@ def main() -> int: parser.add_argument("--session-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_MAX_AGE_SECONDS", "900"))) parser.add_argument("--interactive-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_INTERACTIVE_MAX_AGE_SECONDS", "900"))) parser.add_argument("--session-events-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS", "86400"))) + parser.add_argument("--guard-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS", "300"))) + parser.add_argument("--guard-required", action="store_true", default=env("AW_RUS_HEALTH_GUARD_REQUIRED", "0").lower() in ("1", "true", "yes")) parser.add_argument("--validation-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_VALIDATION_MAX_AGE_SECONDS", "259200"))) parser.add_argument("--tcp-timeout-seconds", type=float, default=float(env("AW_RUS_HEALTH_TCP_TIMEOUT_SECONDS", "3"))) parser.add_argument("--json", action="store_true") @@ -250,8 +363,14 @@ def main() -> int: if not aw_api_base.endswith("/api/0"): aw_api_base = aw_api_base.rstrip("/") + "/api/0" - check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"]) - check_wrapper(report, "wrapper:dlp-health-check", ["/usr/local/bin/dlp-health-check", "--json"], json_mode=True) + check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"], failure_status="warn") + check_wrapper( + report, + "wrapper:dlp-health-check", + ["/usr/local/bin/dlp-health-check", "--json"], + json_mode=True, + failure_status="warn", + ) try: info = http_json(f"{aw_api_base}/info") @@ -279,6 +398,14 @@ def main() -> int: buckets = {} host = args.rdp_hostname + guard_status, guard_summary, guard_details = guard_bucket_health( + aw_api_base, + host, + args.guard_max_age_seconds, + args.guard_required, + ) + report.add("bucket:collector-guard", guard_status, guard_summary, **guard_details) + worktime_bucket = f"aw-worktime-sessions_{host}" worktime_event = None if buckets: @@ -315,10 +442,15 @@ def main() -> int: ) details["interactive_required"] = interactive_required details["host_activity"] = activity + if not interactive_required and status != "ok": + details["inactive_summary"] = summary + status = "ok" + summary = "inactive: no active interactive users" report.add(label, status, summary, **details) - session_status, session_summary, session_details = bucket_health( + session_status, session_summary, session_details = bucket_timestamp_health( aw_api_base, + buckets, f"aw-session-events_{host}", args.session_events_max_age_seconds, missing_status="fail", @@ -362,8 +494,12 @@ def main() -> int: payload = report.as_dict() state_dir = Path(args.state_dir) - write_atomic(state_dir / "aw-rus-health.json", json.dumps(payload, ensure_ascii=False, indent=2) + "\n") - write_atomic(state_dir / "aw-rus-health.txt", report.render_text() + "\n") + health_json_path = state_dir / "aw-rus-health.json" + health_txt_path = state_dir / "aw-rus-health.txt" + write_atomic(health_json_path, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + write_atomic(health_txt_path, report.render_text() + "\n") + chmod_if_possible(health_json_path, 0o644) + chmod_if_possible(health_txt_path, 0o644) if args.json: print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer index bc6f25b..08e74a0 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer +++ b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer @@ -2,10 +2,11 @@ Description=Run AW-RUS unified health orchestrator every 2 minutes [Timer] -OnBootSec=2min -OnUnitActiveSec=2min +OnCalendar=*:0/2:25 +AccuracySec=15s +RandomizedDelaySec=15s Unit=aw-rus-healthd.service -Persistent=true +Persistent=false [Install] WantedBy=timers.target diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-server.env.example b/install-kit-awindows-20260427-211240/aw-server/aw-server.env.example index 58b1309..a44ffd7 100755 --- a/install-kit-awindows-20260427-211240/aw-server/aw-server.env.example +++ b/install-kit-awindows-20260427-211240/aw-server/aw-server.env.example @@ -43,7 +43,7 @@ AW_HEALTH_CHECK_ENABLED=true AW_HEALTH_CHECK_INTERVAL=60 AW_EXPECT_START_OF_DAY=00:00 AW_EXPECT_ALWAYS_ACTIVE_PATTERN=aw-watcher-window -AW_EXPECT_LANDINGPAGE=/activity/SHARKON2025/view/ +AW_EXPECT_LANDINGPAGE=/#/activity/SHARKON2025/view/ AW_HEALTH_STRICT_FILEOPS=0 AW_MONITORED_WINDOWS_HOST=192.168.100.18 AW_MONITORED_WINDOWS_HOSTNAME=SHARKON2025 @@ -51,6 +51,19 @@ AW_RUS_HEALTH_WORKTIME_API=http://127.0.0.1:5610 AW_RUS_HEALTH_STATE_DIR=/var/lib/activitywatch/health AW_RUS_HEALTH_VALIDATION_DIR=/var/lib/activitywatch/health/windows-validation AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS=86400 +AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS=300 +AW_RUS_HEALTH_GUARD_REQUIRED=1 +AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600 +AW_RUS_SLO_WORKTIME_BASE=http://127.0.0.1:5610 +AW_RUS_SLO_TARGET_PERCENT=99.97 +AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600 +AW_BROWSER_SMOKE_WORKTIME_BASE=http://127.0.0.1:5610 +AW_BROWSER_SMOKE_HOST=SHARKON2025 +AW_BROWSER_SMOKE_OUTPUT_DIR=/var/lib/activitywatch/browser-smoke +AW_BROWSER_SMOKE_KEEP_RUNS=24 +AW_BROWSER_SMOKE_ENGINE=chromium-cli +AW_BROWSER_SMOKE_TIMEOUT_MS=20000 +AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS=15000 # Hayabusa auto-case / alerting AW_HAYABUSA_AUTO_CASE_ENABLED=true diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py new file mode 100644 index 0000000..854d6ff --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tempfile +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib import request + + +ENV_FILE = Path("/etc/activitywatch/aw-server.env") +DEFAULT_STATE_DIR = Path("/var/lib/activitywatch/slo") +DEFAULT_HEALTHD_CMD = "/usr/local/bin/aw-rus-healthd.py --json" +DEFAULT_HEALTHD_STATE_FILE = Path("/var/lib/activitywatch/health/aw-rus-health.json") + + +def load_env_file(path: Path) -> None: + if not path.exists(): + return + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + # systemd EnvironmentFile has already loaded the variables for the service. + return + for raw_line in lines: + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ.setdefault(key.strip(), value.strip().strip("'").strip('"')) + + +def env(name: str, default: str) -> str: + value = os.environ.get(name) + return value if value not in (None, "") else default + + +def now_utc() -> datetime: + return datetime.now(UTC) + + +def parse_ts(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) + + +def iso(dt: datetime) -> str: + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def write_atomic(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: + handle.write(content) + tmp_name = handle.name + os.replace(tmp_name, path) + + +def chmod_if_possible(path: Path, mode: int) -> None: + try: + path.chmod(mode) + except OSError: + pass + + +def chown_like_parent_if_possible(path: Path) -> None: + try: + parent_stat = path.parent.stat() + os.chown(path, parent_stat.st_uid, parent_stat.st_gid) + except OSError: + pass + + +def fetch_url(url: str, timeout_seconds: float, *, accept: str | None = None, attempts: int = 1) -> dict[str, Any]: + started = now_utc() + last_error = "" + for attempt in range(1, max(1, attempts) + 1): + try: + req = request.Request(url, headers={"Accept": accept}) if accept else url + with request.urlopen(req, timeout=timeout_seconds) as resp: + body = resp.read() + status = int(resp.status) + content_type = resp.headers.get("Content-Type", "") + ok = 200 <= status < 300 + return { + "ok": ok, + "status": status, + "body_bytes": len(body), + "content_type": content_type, + "body": body, + "attempts": attempt, + "latency_ms": int((now_utc() - started).total_seconds() * 1000), + "url": url, + } + except Exception as exc: + last_error = str(exc) + if attempt < max(1, attempts): + time.sleep(1) + return { + "ok": False, + "error": last_error, + "body_bytes": 0, + "attempts": max(1, attempts), + "latency_ms": int((now_utc() - started).total_seconds() * 1000), + "url": url, + } + + +def public_probe_result(result: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in result.items() if key != "body"} + + +def http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: + return public_probe_result(fetch_url(url, timeout_seconds, attempts=2)) + + +def html_probe( + url: str, + timeout_seconds: float, + *, + min_bytes: int, + required_markers: tuple[str, ...], +) -> dict[str, Any]: + result = fetch_url(url, timeout_seconds, accept="text/html", attempts=2) + if not result.get("ok"): + return public_probe_result(result) + + body = result.get("body", b"") + text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) + missing = [marker for marker in required_markers if marker not in text] + content_type = str(result.get("content_type") or "") + if len(body) < min_bytes: + result["ok"] = False + result["error"] = f"body too small: {len(body)} < {min_bytes}" + elif "text/html" not in content_type.lower(): + result["ok"] = False + result["error"] = f"unexpected content-type: {content_type or 'unknown'}" + elif missing: + result["ok"] = False + result["error"] = "missing markers: " + ", ".join(missing) + result["missing_markers"] = missing + return public_probe_result(result) + + +def json_probe( + url: str, + timeout_seconds: float, + *, + expected_values: dict[str, Any], + required_keys: tuple[str, ...], +) -> dict[str, Any]: + result = fetch_url(url, timeout_seconds, accept="application/json", attempts=2) + if not result.get("ok"): + return public_probe_result(result) + + body = result.get("body", b"") + try: + payload = json.loads(body.decode("utf-8") if isinstance(body, bytes) else str(body)) + except Exception as exc: + result["ok"] = False + result["error"] = f"invalid json: {exc}" + return public_probe_result(result) + + if not isinstance(payload, dict): + result["ok"] = False + result["error"] = "json root is not object" + return public_probe_result(result) + + missing_keys = [key for key in required_keys if key not in payload] + mismatched = { + key: {"expected": expected, "actual": payload.get(key)} + for key, expected in expected_values.items() + if payload.get(key) != expected + } + if missing_keys: + result["ok"] = False + result["error"] = "missing json keys: " + ", ".join(missing_keys) + result["missing_keys"] = missing_keys + elif mismatched: + result["ok"] = False + result["error"] = "unexpected json values" + result["mismatched_values"] = mismatched + else: + result["json_keys"] = sorted(payload.keys()) + return public_probe_result(result) + + +def run_healthd(command: str, timeout_seconds: int) -> dict[str, Any]: + try: + proc = subprocess.run( + command, + shell=True, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + return { + "ok": False, + "returncode": None, + "error": f"timeout after {timeout_seconds}s", + "output_tail": (exc.stdout or "")[-1000:] if isinstance(exc.stdout, str) else "", + } + + payload: dict[str, Any] = {} + try: + payload = json.loads(proc.stdout or "{}") + except json.JSONDecodeError: + pass + return { + "ok": proc.returncode == 0 and bool(payload.get("ok")), + "returncode": proc.returncode, + "counts": payload.get("counts", {}), + "payload": payload if isinstance(payload, dict) else {}, + "output_tail": (proc.stdout or "")[-1000:], + } + + +def read_healthd_state(path: Path, max_age_seconds: int) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + generated = parse_ts(str(payload["generated_at_utc"])) + age = max(0, int((now_utc() - generated).total_seconds())) + except Exception as exc: + return {"ok": False, "error": str(exc), "path": str(path)} + return { + "ok": bool(payload.get("ok")) and age <= max_age_seconds, + "counts": payload.get("counts", {}), + "age_seconds": age, + "path": str(path), + "payload": payload if isinstance(payload, dict) else {}, + } + + +def load_samples(path: Path, cutoff: datetime) -> list[dict[str, Any]]: + samples: list[dict[str, Any]] = [] + if not path.exists(): + return samples + try: + lines = path.read_text(encoding="utf-8").splitlines() + except PermissionError: + return samples + for line in lines: + if not line.strip(): + continue + try: + item = json.loads(line) + ts = parse_ts(str(item["ts"])) + except Exception: + continue + if ts >= cutoff: + samples.append(item) + return samples + + +def append_and_trim_sample(path: Path, sample: dict[str, Any], retention_seconds: int) -> list[dict[str, Any]]: + cutoff = parse_ts(sample["ts"]) - timedelta(seconds=retention_seconds) + samples = load_samples(path, cutoff) + samples.append(sample) + content = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n" for item in samples) + write_atomic(path, content) + chmod_if_possible(path, 0o644) + chown_like_parent_if_possible(path) + return samples + + +def summarize_window( + samples: list[dict[str, Any]], + *, + now: datetime, + window_seconds: int, + sample_interval_seconds: int, + target_percent: float, +) -> dict[str, Any]: + cutoff = now - timedelta(seconds=window_seconds) + window = [] + for item in samples: + try: + if parse_ts(str(item["ts"])) >= cutoff: + window.append(item) + except Exception: + continue + + total = len(window) + good = sum(1 for item in window if item.get("ok") is True) + bad = total - good + availability = round((good / total) * 100, 5) if total else None + observed_bad_seconds = bad * sample_interval_seconds + budget_seconds = int(window_seconds * ((100.0 - target_percent) / 100.0)) + budget_remaining_seconds = budget_seconds - observed_bad_seconds + status = "unknown" + if total: + status = "ok" if budget_remaining_seconds >= 0 else "burning" + return { + "window_seconds": window_seconds, + "samples": total, + "good_samples": good, + "bad_samples": bad, + "availability_percent": availability, + "target_percent": target_percent, + "observed_bad_seconds": observed_bad_seconds, + "budget_seconds": budget_seconds, + "budget_remaining_seconds": budget_remaining_seconds, + "status": status, + } + + +def render_summary_text(summary: dict[str, Any]) -> str: + lines = [ + "=== AW-RUS SLO ===", + f"Timestamp: {summary['generated_at_utc']}", + f"Target: {summary['target_percent']}%", + "", + ] + for name, data in summary["windows"].items(): + availability = data["availability_percent"] + availability_text = "n/a" if availability is None else f"{availability:.5f}%" + remaining = data["budget_remaining_seconds"] + lines.append( + f"{name}: {data['status']} availability={availability_text} " + f"samples={data['samples']} bad={data['bad_samples']} " + f"bad_seconds={data['observed_bad_seconds']} budget_remaining_seconds={remaining}" + ) + lines.append("") + lines.append(f"Current sample: {'OK' if summary['current_sample']['ok'] else 'FAIL'}") + for name, probe in summary["current_sample"].get("probes", {}).items(): + marker = "OK" if probe.get("ok") else "FAIL" + detail = probe.get("status", probe.get("error", "")) + lines.append(f"- {name}: {marker} {detail}") + return "\n".join(lines) + + +def build_sample(args: argparse.Namespace) -> dict[str, Any]: + ts = iso(now_utc()) + if args.healthd_mode == "run": + health = run_healthd(args.healthd_cmd, args.health_timeout_seconds) + else: + health = read_healthd_state(Path(args.healthd_state_file), args.healthd_state_max_age_seconds) + probes = { + "aw_webui_index": html_probe( + args.aw_webui_url, + args.http_timeout_seconds, + min_bytes=1000, + required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), + ), + "worktime_today_html": html_probe( + args.worktime_today_html_url, + args.http_timeout_seconds, + min_bytes=5000, + required_markers=("AW-rus", ""), + ), + "worktime_management_html": html_probe( + args.worktime_management_html_url, + args.http_timeout_seconds, + min_bytes=5000, + required_markers=("AW-rus", ""), + ), + "worktime_today_csv": http_probe(args.worktime_today_csv_url, args.http_timeout_seconds), + "worktime_management_json": json_probe( + args.worktime_management_json_url, + args.http_timeout_seconds, + expected_values={"host": args.host}, + required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), + ), + } + ok = bool(health.get("ok")) and all(probe.get("ok") for probe in probes.values()) + return { + "ts": ts, + "ok": ok, + "healthd_ok": bool(health.get("ok")), + "healthd_counts": health.get("counts", {}), + "probes": probes, + } + + +def main() -> int: + load_env_file(ENV_FILE) + parser = argparse.ArgumentParser(description="AW-RUS rolling SLO sampler") + parser.add_argument("--state-dir", default=env("AW_RUS_SLO_STATE_DIR", str(DEFAULT_STATE_DIR))) + parser.add_argument("--healthd-cmd", default=env("AW_RUS_SLO_HEALTHD_CMD", DEFAULT_HEALTHD_CMD)) + parser.add_argument("--healthd-mode", choices=["state", "run"], default=env("AW_RUS_SLO_HEALTHD_MODE", "state")) + parser.add_argument("--healthd-state-file", default=env("AW_RUS_SLO_HEALTHD_STATE_FILE", str(DEFAULT_HEALTHD_STATE_FILE))) + parser.add_argument("--healthd-state-max-age-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", "180"))) + parser.add_argument("--aw-base", default=env("AW_RUS_SLO_AW_BASE", env("AW_SERVER_URL", "http://127.0.0.1:5600"))) + parser.add_argument("--worktime-base", default=env("AW_RUS_SLO_WORKTIME_BASE", env("AW_RUS_HEALTH_WORKTIME_API", "http://127.0.0.1:5610"))) + parser.add_argument("--host", default=env("AW_RUS_SLO_HOST", env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025"))) + parser.add_argument("--target-percent", type=float, default=float(env("AW_RUS_SLO_TARGET_PERCENT", "99.97"))) + parser.add_argument("--sample-interval-seconds", type=int, default=int(env("AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", "15"))) + parser.add_argument("--retention-days", type=int, default=int(env("AW_RUS_SLO_RETENTION_DAYS", "35"))) + parser.add_argument("--http-timeout-seconds", type=float, default=float(env("AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", "15"))) + parser.add_argument("--health-timeout-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", "90"))) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + aw_base = args.aw_base.rstrip("/") + worktime_base = args.worktime_base.rstrip("/") + args.aw_webui_url = f"{aw_base}/" + args.worktime_today_html_url = f"{worktime_base}/reports/worktime/today?format=html&day=today&host={args.host}&allow_stale=1" + args.worktime_management_html_url = f"{worktime_base}/reports/worktime/management?format=html&day=today&host={args.host}&allow_stale=1" + args.worktime_today_csv_url = f"{worktime_base}/reports/worktime/today?format=csv&day=today&host={args.host}&allow_stale=1" + args.worktime_management_json_url = f"{worktime_base}/reports/worktime/management?format=json&day=today&host={args.host}&allow_stale=1" + + state_dir = Path(args.state_dir) + sample_path = state_dir / "aw-slo-samples.jsonl" + sample = build_sample(args) + samples = append_and_trim_sample( + sample_path, + sample, + retention_seconds=max(1, args.retention_days) * 86400, + ) + + generated_at = now_utc() + summary = { + "generated_at_utc": iso(generated_at), + "target_percent": args.target_percent, + "sample_interval_seconds": args.sample_interval_seconds, + "current_sample": sample, + "windows": { + "24h": summarize_window( + samples, + now=generated_at, + window_seconds=86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + "7d": summarize_window( + samples, + now=generated_at, + window_seconds=7 * 86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + "30d": summarize_window( + samples, + now=generated_at, + window_seconds=30 * 86400, + sample_interval_seconds=args.sample_interval_seconds, + target_percent=args.target_percent, + ), + }, + } + summary_json_path = state_dir / "aw-slo-summary.json" + summary_txt_path = state_dir / "aw-slo-summary.txt" + write_atomic(summary_json_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") + text = render_summary_text(summary) + write_atomic(summary_txt_path, text + "\n") + chmod_if_possible(summary_json_path, 0o644) + chmod_if_possible(summary_txt_path, 0o644) + chown_like_parent_if_possible(summary_json_path) + chown_like_parent_if_possible(summary_txt_path) + if args.json: + print(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + print(text) + return 0 if sample["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service new file mode 100644 index 0000000..a1bd724 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service @@ -0,0 +1,16 @@ +[Unit] +Description=AW-RUS SLO sampler +After=network-online.target activitywatch-server.service aw-worktime-api.service aw-rus-healthd.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +ExecStart=/usr/bin/python3 /usr/local/bin/aw-slo-monitor.py +SuccessExitStatus=1 +User=activitywatch +Group=activitywatch +Nice=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-slo-monitor diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.timer b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.timer new file mode 100644 index 0000000..b5e2ce3 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AW-RUS SLO sampler every minute + +[Timer] +OnCalendar=*:0/1:50 +AccuracySec=10s +RandomizedDelaySec=10s +Persistent=false +Unit=aw-slo-monitor.service + +[Install] +WantedBy=timers.target diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py index f12324a..caba965 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import base64 import csv +import hashlib import html import io import importlib.util @@ -8,6 +10,7 @@ import os import sys import tempfile import threading +import time import urllib.request from datetime import datetime, timezone, timedelta from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -49,18 +52,30 @@ MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", " MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()} EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30"))) WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000"))) +AW_HTTP_TIMEOUT_SECONDS = max(0.5, float(os.environ.get("AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS", "5"))) +REPORT_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_REPORT_CACHE_TTL_SECONDS", "60"))) +REPORT_STALE_TTL_SECONDS = max(REPORT_CACHE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_STALE_TTL_SECONDS", "900"))) +REPORT_DISK_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_REPORT_DISK_CACHE_DIR", "/var/lib/activitywatch/worktime-report-cache")) +REPORT_DISK_STALE_TTL_SECONDS = max(REPORT_STALE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS", "86400"))) +REPORT_BUSY_WAIT_SECONDS = max(0.0, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_SECONDS", "8"))) +REPORT_BUSY_WAIT_INTERVAL_SECONDS = max(0.05, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_INTERVAL_SECONDS", "0.2"))) +REPORT_MAX_CONCURRENT = max(1, int(os.environ.get("AW_WORKTIME_REPORT_MAX_CONCURRENT", "1"))) TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180"))) TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600"))) MODULE_PATH = Path(__file__).resolve() _ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}} _EVENTS_CACHE_LOCK = threading.Lock() _EVENTS_CACHE = {} +_REPORT_CACHE_LOCK = threading.Lock() +_REPORT_RESPONSE_CACHE = {} +_REPORT_REFRESHING = set() +_REPORT_BUILD_SEMAPHORE = threading.BoundedSemaphore(REPORT_MAX_CONCURRENT) _MANAGEMENT_BUILD_LOCKS_LOCK = threading.Lock() _MANAGEMENT_BUILD_LOCKS = {} def get(u): - with urllib.request.urlopen(u, timeout=30) as r: + with urllib.request.urlopen(u, timeout=AW_HTTP_TIMEOUT_SECONDS) as r: return json.loads(r.read().decode()) @@ -784,15 +799,21 @@ def aggregate_hourly_rows(events, start, end, host): def fetch_events_for_date(host, report_date): bounds = get_report_bounds(report_date) bucket_id = get_sessions_bucket_id(host) - events = fetch_bucket_events(bucket_id, host) + events = fetch_bucket_events(bucket_id, host, start=bounds["start"], end=bounds["end"]) return bounds, events -def fetch_bucket_events(bucket_id, host): +def fetch_bucket_events(bucket_id, host, start=None, end=None): now = now_utc() + cache_key = bucket_id + params = {"limit": WORKTIME_EVENTS_LIMIT} + if start is not None and end is not None: + params["start"] = to_iso_utc(start) + params["end"] = to_iso_utc(end) + cache_key = f"{bucket_id}|{params['start']}|{params['end']}" if EVENTS_CACHE_TTL_SECONDS > 0: with _EVENTS_CACHE_LOCK: - cached = _EVENTS_CACHE.get(bucket_id) + cached = _EVENTS_CACHE.get(cache_key) if cached is not None and age_seconds(cached["stored_at"], now=now) <= EVENTS_CACHE_TTL_SECONDS: return cached["events"] try: @@ -801,13 +822,13 @@ def fetch_bucket_events(bucket_id, host): log_warning(f"bucket lookup failed for host={host} bucket={bucket_id} aw_base={AW}") return [] try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit={WORKTIME_EVENTS_LIMIT}") + events = get(f"{AW}/buckets/{bucket_id}/events?{urlencode(params)}") except Exception: log_warning(f"events fetch failed for host={host} bucket={bucket_id} aw_base={AW}") return [] if EVENTS_CACHE_TTL_SECONDS > 0: with _EVENTS_CACHE_LOCK: - _EVENTS_CACHE[bucket_id] = {"stored_at": now, "events": events} + _EVENTS_CACHE[cache_key] = {"stored_at": now, "events": events} return events @@ -840,16 +861,22 @@ def build_report_summary(rows): def latest_bucket_event(bucket_id): try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit=20") + events = get(f"{AW}/buckets/{bucket_id}/events?limit=1") + except Exception: + events = None + if isinstance(events, list) and events: + valid = [item for item in events if isinstance(item, dict)] + if valid: + valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) + return valid[0] + try: + bucket = get(f"{AW}/buckets/{bucket_id}") except Exception: return None - if not isinstance(events, list) or not events: - return None - valid = [item for item in events if isinstance(item, dict)] - if not valid: - return None - valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return valid[0] + end = ((bucket or {}).get("metadata") or {}).get("end") + if end: + return {"timestamp": end, "duration": 0, "data": {"source": "bucket_metadata"}} + return None def _priority_rank(priority): @@ -1219,6 +1246,7 @@ def _source_status_label(status): "ok": "fresh", "warn": "stale", "fail": "missing", + "inactive": "inactive", }.get(status, status) @@ -1273,7 +1301,224 @@ def save_management_cache(host, report_date, payload): write_atomic_json(management_cache_path(host, report_date), payload) -def build_source_freshness(host): +def make_report_cache_key(path, fmt, host, report_date, selected_day="", owner_filter="", department_filter=""): + return "|".join( + [ + str(path), + str(fmt), + resolve_host(host), + report_date.isoformat(), + str(selected_day or ""), + normalize_management_filter(owner_filter), + normalize_management_filter(department_filter), + ] + ) + + +def resolve_report_format(params, accept_header=""): + requested = str(params.get("format", [""])[0] or "").lower() + if requested in {"csv", "html", "json"}: + return requested + accept = str(accept_header or "").lower() + if "text/html" in accept and "application/json" not in accept: + return "html" + return "json" + + +def report_disk_cache_path(key): + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + return REPORT_DISK_CACHE_DIR / f"{digest}.json" + + +def load_report_response_disk_cache(key): + if REPORT_DISK_STALE_TTL_SECONDS <= 0: + return None + path = report_disk_cache_path(key) + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + stored_epoch = float(payload["stored_epoch"]) + age = max(0, int(time.time() - stored_epoch)) + if age > REPORT_DISK_STALE_TTL_SECONDS: + return None + data = base64.b64decode(payload["data_b64"].encode("ascii")) + content_type = str(payload["content_type"]) + except Exception as exc: + log_warning(f"failed to load report disk cache key={key}: {exc}") + return None + return { + "stored_monotonic": time.monotonic() - age, + "generated_at_utc": payload.get("generated_at_utc", ""), + "data": data, + "content_type": content_type, + "age_seconds": age, + "stale": True, + "cache_source": "disk", + } + + +def save_report_response_disk_cache(key, data, content_type, generated_at_utc): + if REPORT_DISK_STALE_TTL_SECONDS <= 0: + return + payload = { + "key": key, + "stored_epoch": time.time(), + "generated_at_utc": generated_at_utc, + "content_type": content_type, + "data_b64": base64.b64encode(data).decode("ascii"), + } + try: + write_atomic_json(report_disk_cache_path(key), payload) + except Exception as exc: + log_warning(f"failed to save report disk cache key={key}: {exc}") + + +def get_report_response_cache(key, *, allow_stale=False): + if REPORT_CACHE_TTL_SECONDS <= 0 and not allow_stale: + return None + now = time.monotonic() + with _REPORT_CACHE_LOCK: + cached = _REPORT_RESPONSE_CACHE.get(key) + if cached is not None: + age = now - cached["stored_monotonic"] + max_age = REPORT_STALE_TTL_SECONDS if allow_stale else REPORT_CACHE_TTL_SECONDS + if max_age > 0 and age <= max_age: + result = dict(cached) + result["age_seconds"] = int(age) + result["stale"] = age > REPORT_CACHE_TTL_SECONDS + result["cache_source"] = "memory" + return result + if allow_stale: + return load_report_response_disk_cache(key) + return None + + +def save_report_response_cache(key, data, content_type): + if REPORT_STALE_TTL_SECONDS <= 0: + return + generated_at_utc = now_utc().isoformat().replace("+00:00", "Z") + with _REPORT_CACHE_LOCK: + _REPORT_RESPONSE_CACHE[key] = { + "stored_monotonic": time.monotonic(), + "generated_at_utc": generated_at_utc, + "data": data, + "content_type": content_type, + } + save_report_response_disk_cache(key, data, content_type, generated_at_utc) + + +def trigger_report_background_refresh(key, path, params): + refresh_params = {name: list(values) for name, values in params.items()} + with _REPORT_CACHE_LOCK: + if key in _REPORT_REFRESHING: + return False + _REPORT_REFRESHING.add(key) + + def refresh(): + try: + if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): + return + try: + data, content_type = build_worktime_report_response(path, refresh_params) + save_report_response_cache(key, data, content_type) + except Exception as exc: + log_warning(f"background report refresh failed path={path}: {exc}") + finally: + _REPORT_BUILD_SEMAPHORE.release() + finally: + with _REPORT_CACHE_LOCK: + _REPORT_REFRESHING.discard(key) + + thread = threading.Thread(target=refresh, name="worktime-report-refresh", daemon=True) + thread.start() + return True + + +def send_cached_report(handler, cached, *, reason): + handler.send_response(200) + handler.send_header("Content-Type", cached["content_type"]) + handler.send_header("Content-Length", str(len(cached["data"]))) + handler.send_header("X-AW-Worktime-Cache", "stale" if cached.get("stale") else "fresh") + handler.send_header("X-AW-Worktime-Cache-Reason", reason) + handler.send_header("X-AW-Worktime-Cache-Age", str(cached.get("age_seconds", 0))) + handler.send_header("X-AW-Worktime-Cache-Source", cached.get("cache_source", "memory")) + handler.end_headers() + try: + handler.wfile.write(cached["data"]) + except (BrokenPipeError, ConnectionResetError): + return False + return True + + +def send_report_unavailable(handler, message): + payload = { + "ok": False, + "error": "report_unavailable", + "message": message, + "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), + } + data = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") + send_bytes(handler, data, "application/json; charset=utf-8", status=503) + + +def send_worktime_report(handler, path, params, cache_key, *, host, report_date): + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl") + return + + allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} + if allow_immediate_stale: + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + refresh_stale = str(params.get("refresh_stale", [""])[0]).lower() in {"1", "true", "yes"} + if refresh_stale: + trigger_report_background_refresh(cache_key, path, params) + send_cached_report(handler, cached, reason="stale-allowed") + return + + if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): + if not allow_immediate_stale and REPORT_BUSY_WAIT_SECONDS > 0: + deadline = time.monotonic() + REPORT_BUSY_WAIT_SECONDS + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(REPORT_BUSY_WAIT_INTERVAL_SECONDS, remaining)) + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl-after-wait") + return + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + send_cached_report(handler, cached, reason="busy") + return + send_report_unavailable(handler, "report builder is busy and no cached response is available") + return + + try: + cached = get_report_response_cache(cache_key) + if cached is not None: + send_cached_report(handler, cached, reason="ttl-after-lock") + return + try: + data, content_type = build_worktime_report_response(path, params) + except Exception as exc: + log_warning(f"report build failed path={path} host={host} date={report_date}: {exc}") + cached = get_report_response_cache(cache_key, allow_stale=True) + if cached is not None: + send_cached_report(handler, cached, reason="build-error") + return + send_report_unavailable(handler, "report build failed and no cached response is available") + return + save_report_response_cache(cache_key, data, content_type) + send_bytes(handler, data, content_type) + finally: + _REPORT_BUILD_SEMAPHORE.release() + + +def build_source_freshness(host, interactive_required=True): source_specs = [ { "source_id": "worktime_sessions", @@ -1305,6 +1550,7 @@ def build_source_freshness(host): "bucket_candidates": [f"aw-watcher-window_{host}"], "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, "required": True, + "interactive_only": True, "owner": "ops", }, { @@ -1321,6 +1567,7 @@ def build_source_freshness(host): "bucket_candidates": [f"aw-file-operations_{host}"], "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, "required": True, + "interactive_only": True, "owner": "ops", }, { @@ -1379,6 +1626,9 @@ def build_source_freshness(host): else: status = "ok" summary = f"fresh ({age}s)" + if spec.get("interactive_only") and not interactive_required and status != "ok": + status = "inactive" + summary = "inactive: no active interactive users" detail = { "source_id": spec["source_id"], "label": spec["label"], @@ -1388,12 +1638,14 @@ def build_source_freshness(host): "timestamp": (matched_event or {}).get("timestamp", ""), "age_seconds": age, "required": spec["required"], + "interactive_only": bool(spec.get("interactive_only")), + "interactive_required": bool(interactive_required), "max_age_seconds": spec["max_age_seconds"], "summary": summary, "event_summary": _source_summary(matched_event), } sources.append(detail) - if status == "ok": + if status in {"ok", "inactive"}: continue priority = "critical" if spec["required"] else "medium" actions.append( @@ -1723,7 +1975,8 @@ def build_management_payload(rows, host, report_date, owner_filter="", departmen owner_filter = normalize_management_filter(owner_filter) department_filter = normalize_management_filter(department_filter) payload = _build_management_core(rows, host, report_date, owner_filter=owner_filter, department_filter=department_filter) - source_freshness, source_actions = build_source_freshness(resolve_host(host)) + interactive_required = int(payload.get("summary", {}).get("active_users", 0) or 0) > 0 + source_freshness, source_actions = build_source_freshness(resolve_host(host), interactive_required=interactive_required) payload["sources"] = source_freshness payload["trend"] = build_management_trend( resolve_host(host), @@ -1751,11 +2004,11 @@ def report_for_date(host, report_date): return aggregate_rows(events, bounds["start"], bounds["end"], host) -def management_report_for_date(host, report_date, owner_filter="", department_filter=""): +def management_report_for_date(host, report_date, owner_filter="", department_filter="", use_cache=True): owner_filter = normalize_management_filter(owner_filter) department_filter = normalize_management_filter(department_filter) if owner_filter or department_filter: - base_payload = management_report_for_date(host, report_date) + base_payload = management_report_for_date(host, report_date, use_cache=use_cache) filtered_payload = apply_management_filters_to_payload( base_payload, owner_filter=owner_filter, @@ -1766,14 +2019,17 @@ def management_report_for_date(host, report_date, owner_filter="", department_fi filtered_payload["trend"] = [] filtered_payload["trend_scope"] = "filtered_current_only" return filtered_payload - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - lock = get_management_build_lock(host, report_date) - with lock: + cache_allowed = use_cache or report_date < datetime.now(REPORT_TZ).date() + if cache_allowed: cached = load_management_cache(host, report_date) if cached is not None: return cached + lock = get_management_build_lock(host, report_date) + with lock: + if cache_allowed: + cached = load_management_cache(host, report_date) + if cached is not None: + return cached bounds, events = fetch_events_for_date(host, report_date) rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) payload = build_management_payload(rows, host, report_date) @@ -2265,10 +2521,11 @@ def render_management_html(payload, selected_day=None): sources_html = [] for source in payload.get("sources", []): + source_priority_class = "low" if source["status"] in {"ok", "inactive"} else ("critical" if source["required"] else "medium") sources_html.append( "" f"{html.escape(source['label'])}" - f"{html.escape(source['status_label'])}" + f"{html.escape(source['status_label'])}" f"{html.escape(source['bucket_id'])}" f"{html.escape(source.get('timestamp') or '-')}" f"{html.escape(str(source.get('age_seconds')) if source.get('age_seconds') is not None else '-')}" @@ -2619,6 +2876,85 @@ def render_management_html(payload, selected_day=None): """ +def build_worktime_report_response(path, params): + fmt = resolve_report_format(params) + host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) + day = params.get("day", ["today"])[0] + date_text = params.get("date", [None])[0] + owner_filter = normalize_management_filter(params.get("owner", [""])[0]) + department_filter = normalize_management_filter(params.get("department", [""])[0]) + report_date = resolve_report_date(day=day, date_text=date_text) + is_management = path == "/reports/worktime/management" + allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} + management_payload = ( + management_report_for_date( + host, + report_date, + owner_filter=owner_filter, + department_filter=department_filter, + use_cache=allow_immediate_stale, + ) + if is_management + else None + ) + rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] + true_active_apps = [] if is_management else build_true_active_apps(host, report_date) + + if fmt == "csv": + if is_management: + out = io.StringIO() + writer = csv.DictWriter( + out, + fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], + extrasaction="ignore", + ) + writer.writeheader() + writer.writerows(management_payload["actions"]) + return out.getvalue().encode(), "text/csv; charset=utf-8" + out = io.StringIO() + writer = csv.DictWriter( + out, + fieldnames=[ + "user", + "user_id", + "active_seconds", + "active_hhmm", + "first_activity", + "last_activity", + "idle_seconds", + "sessions_count", + "samples_count", + "active_samples", + ], + ) + writer.writeheader() + writer.writerows(rows) + return out.getvalue().encode(), "text/csv; charset=utf-8" + + if fmt == "html": + if is_management: + data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") + else: + data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") + return data, "text/html; charset=utf-8" + + if is_management: + data = json.dumps(management_payload, ensure_ascii=False, indent=2).encode("utf-8") + return data, "application/json; charset=utf-8" + + obj = { + "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "report_timezone": str(REPORT_TZ), + "host": host, + "report_date": report_date.isoformat(), + "bucket_id": get_sessions_bucket_id(host), + "rows": rows, + "true_active_apps": true_active_apps, + } + data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") + return data, "application/json; charset=utf-8" + + def send_bytes(handler, data, content_type, status=200): handler.send_response(status) handler.send_header("Content-Type", content_type) @@ -2667,82 +3003,27 @@ class H(BaseHTTPRequestHandler): return params = parse_qs(parsed.query, keep_blank_values=False) - fmt = "json" - if params.get("format", ["json"])[0] == "csv": - fmt = "csv" - elif params.get("format", ["json"])[0] == "html": - fmt = "html" + fmt = resolve_report_format(params, self.headers.get("Accept", "")) + if not params.get("format") and fmt != "json": + params = {name: list(values) for name, values in params.items()} + params["format"] = [fmt] host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) day = params.get("day", ["today"])[0] date_text = params.get("date", [None])[0] owner_filter = normalize_management_filter(params.get("owner", [""])[0]) department_filter = normalize_management_filter(params.get("department", [""])[0]) report_date = resolve_report_date(day=day, date_text=date_text) - is_management = parsed.path == "/reports/worktime/management" - management_payload = management_report_for_date(host, report_date, owner_filter=owner_filter, department_filter=department_filter) if is_management else None - rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] - true_active_apps = [] if is_management else build_true_active_apps(host, report_date) + cache_key = make_report_cache_key( + parsed.path, + fmt, + host, + report_date, + selected_day=day if day in {"today", "yesterday"} else "", + owner_filter=owner_filter, + department_filter=department_filter, + ) - if fmt == "csv": - if is_management: - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], - extrasaction="ignore", - ) - writer.writeheader() - writer.writerows(management_payload["actions"]) - data = out.getvalue().encode() - send_bytes(self, data, "text/csv; charset=utf-8") - return - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=[ - "user", - "user_id", - "active_seconds", - "active_hhmm", - "first_activity", - "last_activity", - "idle_seconds", - "sessions_count", - "samples_count", - "active_samples", - ], - ) - writer.writeheader() - writer.writerows(rows) - data = out.getvalue().encode() - send_bytes(self, data, "text/csv; charset=utf-8") - return - - if fmt == "html": - if is_management: - data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") - else: - data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") - send_bytes(self, data, "text/html; charset=utf-8") - return - - if is_management: - obj = management_payload - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") - return - - obj = { - "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "host": host, - "report_date": report_date.isoformat(), - "bucket_id": get_sessions_bucket_id(host), - "rows": rows, - "true_active_apps": true_active_apps, - } - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") + send_worktime_report(self, parsed.path, params, cache_key, host=host, report_date=report_date) class WorktimeHTTPServer(ThreadingHTTPServer): diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service new file mode 100644 index 0000000..a013770 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service @@ -0,0 +1,15 @@ +[Unit] +Description=AW Worktime Report Cache Prewarm +After=network-online.target activitywatch-server.service aw-worktime-api.service +Wants=network-online.target activitywatch-server.service aw-worktime-api.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/activitywatch/aw-server.env +ExecStart=/usr/local/bin/aw-worktime-prewarm.sh +User=activitywatch +Group=activitywatch +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-worktime-prewarm + diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh new file mode 100644 index 0000000..7ae33e9 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +WORKTIME_BASE_URL="${WORKTIME_BASE_URL:-http://127.0.0.1:5610}" +WORKTIME_PREWARM_TIMEOUT_SECONDS="${WORKTIME_PREWARM_TIMEOUT_SECONDS:-45}" +WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS="${WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS:-10}" +WORKTIME_PREWARM_READY_TIMEOUT_SECONDS="${WORKTIME_PREWARM_READY_TIMEOUT_SECONDS:-60}" +WORKTIME_PREWARM_READY_INTERVAL_SECONDS="${WORKTIME_PREWARM_READY_INTERVAL_SECONDS:-2}" +WORKTIME_PREWARM_HOST="${WORKTIME_PREWARM_HOST:-${AW_WORKTIME_HOST:-SHARKON2025}}" +WORKTIME_PREWARM_PROFILE="${WORKTIME_PREWARM_PROFILE:-full}" + +log() { + printf '%s %s\n' "$(date '+%F %T')" "$*" +} + +probe() { + local url="$1" + local timeout="$2" + local tmp + tmp="$(mktemp)" + local code + code="$(curl -sS --max-time "$timeout" -o /dev/null -D "$tmp" -w '%{http_code}' "$url" 2>/dev/null || true)" + if [[ "$code" =~ ^2 ]]; then + local cache + cache="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache:/ {print $2; exit}' || true)" + local reason + reason="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache-reason:/ {print $2; exit}' || true)" + rm -f "$tmp" + log "ok code=$code cache=${cache:-none} reason=${reason:-none} url=$url" + return 0 + fi + rm -f "$tmp" + log "warn code=${code:-000} url=$url" + return 1 +} + +wait_until_ready() { + local deadline + deadline=$(( $(date +%s) + WORKTIME_PREWARM_READY_TIMEOUT_SECONDS )) + while true; do + if probe "$WORKTIME_BASE_URL/health" "$WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS"; then + return 0 + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + return 1 + fi + sleep "$WORKTIME_PREWARM_READY_INTERVAL_SECONDS" + done +} + +if ! wait_until_ready; then + log "health readiness timed out; skip prewarm" + exit 0 +fi + +full_urls=( + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=html&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=html&host=$WORKTIME_PREWARM_HOST" +) + +startup_urls=( + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" + "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" +) + +case "$WORKTIME_PREWARM_PROFILE" in + full) + urls=("${full_urls[@]}") + ;; + startup) + urls=("${startup_urls[@]}") + ;; + *) + log "unknown profile=$WORKTIME_PREWARM_PROFILE" + exit 0 + ;; +esac + +failures=0 +for url in "${urls[@]}"; do + if ! probe "$url" "$WORKTIME_PREWARM_TIMEOUT_SECONDS"; then + failures=$((failures + 1)) + fi +done + +if [[ "$failures" -gt 0 ]]; then + log "completed profile=$WORKTIME_PREWARM_PROFILE with failures=$failures" +else + log "completed profile=$WORKTIME_PREWARM_PROFILE successfully" +fi + +exit 0 diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.timer b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.timer new file mode 100644 index 0000000..5167be5 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run AW Worktime Report Cache Prewarm + +[Timer] +OnCalendar=*:2/5:40 +AccuracySec=30s +Persistent=false +Unit=aw-worktime-prewarm.service + +[Install] +WantedBy=timers.target diff --git a/install-kit-awindows-20260427-211240/aw-server/health-check.sh b/install-kit-awindows-20260427-211240/aw-server/health-check.sh new file mode 100644 index 0000000..7df5693 --- /dev/null +++ b/install-kit-awindows-20260427-211240/aw-server/health-check.sh @@ -0,0 +1,173 @@ +#!/bin/bash +set -euo pipefail + +# Health check script for AW services +# Returns 0 if all services are healthy, 1 otherwise + +SERVICES=("activitywatch-server" "aw-worktime-api" "aw-worktime-ui-bridge") +UNHEALTHY_SERVICES=() +WARNINGS=() + +if [[ -f /etc/activitywatch/aw-server.env ]]; then + # shellcheck disable=SC1091 + source /etc/activitywatch/aw-server.env +fi + +WORKTIME_HEALTH_BASE="${AW_RUS_HEALTH_WORKTIME_API:-${AW_WORKTIME_REPORT_BASE:-http://127.0.0.1:5610}}" +WORKTIME_HEALTH_URL="${WORKTIME_HEALTH_BASE%/}/health" +WORKTIME_HEALTH_TIMEOUT_SECONDS="${AW_RUS_HEALTH_WORKTIME_TIMEOUT_SECONDS:-15}" +AW_HEALTH_API_TIMEOUT_SECONDS="${AW_RUS_HEALTH_AW_TIMEOUT_SECONDS:-15}" +AW_HEALTH_API_ATTEMPTS="${AW_RUS_HEALTH_AW_ATTEMPTS:-3}" +SETTINGS_TIMEOUT_SECONDS="${AW_RUS_HEALTH_SETTINGS_TIMEOUT_SECONDS:-15}" +SETTINGS_ATTEMPTS="${AW_RUS_HEALTH_SETTINGS_ATTEMPTS:-3}" + +check_service() { + local service=$1 + if [[ "$service" == "aw-worktime-ui-bridge" ]]; then + if systemctl is-active --quiet aw-worktime-ui-bridge.timer && systemctl is-enabled --quiet aw-worktime-ui-bridge.timer; then + echo "✓ aw-worktime-ui-bridge.timer is running and enabled" + else + echo "✗ aw-worktime-ui-bridge.timer is not active/enabled" + UNHEALTHY_SERVICES+=("aw-worktime-ui-bridge.timer") + fi + return + fi + if systemctl is-active --quiet "$service"; then + echo "✓ $service is running" + else + echo "✗ $service is not running" + UNHEALTHY_SERVICES+=("$service") + fi +} + +check_api_endpoint() { + local url=$1 + local service_name=$2 + local timeout_seconds=${3:-5} + local attempts=${4:-1} + local attempt + + for ((attempt=1; attempt<=attempts; attempt++)); do + if curl -fsS --connect-timeout 3 --max-time "$timeout_seconds" "$url" >/dev/null 2>&1; then + echo "✓ $service_name API endpoint is responding" + return + fi + if (( attempt < attempts )); then + sleep 1 + fi + done + + echo "✗ $service_name API endpoint is not responding" + UNHEALTHY_SERVICES+=("$service_name-api") +} + +read_setting_value() { + local key=$1 + local attempt + for ((attempt=1; attempt<=SETTINGS_ATTEMPTS; attempt++)); do + if curl -fsS --connect-timeout 3 --max-time "$SETTINGS_TIMEOUT_SECONDS" "http://127.0.0.1:5600/api/0/settings/${key}" 2>/dev/null | \ + python3 -c 'import json,sys; print(json.load(sys.stdin))'; then + return 0 + fi + if (( attempt < SETTINGS_ATTEMPTS )); then + sleep 1 + fi + done + return 1 +} + +check_expected_setting() { + local key=$1 + local expected=$2 + local label=${3:-$1} + + if [[ -z "$expected" ]]; then + echo "⚠ expected value for ${label} is not configured, skipping drift check" + WARNINGS+=("${key}-expected-missing") + return + fi + + local actual + if ! actual="$(read_setting_value "$key" 2>/dev/null)"; then + echo "✗ failed to read setting ${label}" + UNHEALTHY_SERVICES+=("setting-${key}") + return + fi + + if [[ "$actual" == "$expected" ]]; then + echo "✓ ${label} matches expected value (${expected})" + else + echo "✗ ${label} drift detected: actual='${actual}' expected='${expected}'" + UNHEALTHY_SERVICES+=("setting-${key}") + fi +} + +check_dlp_transport_freshness() { + local dlp_health="${DLP_HEALTH_BIN:-/usr/local/bin/dlp-health-check}" + local result + + if [[ ! -x "$dlp_health" ]]; then + echo "⚠ dlp-health-check is not available, skipping DLP transport freshness checks" + WARNINGS+=("dlp-health-check-missing") + return + fi + + result="$("$dlp_health" --json 2>/dev/null || true)" + if [[ -z "$result" ]]; then + echo "⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks" + WARNINGS+=("dlp-health-check-empty") + return + fi + + local ok + ok="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); names={r["name"]:r for r in data.get("results", [])}; checks=["buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics"]; bad=[n for n in checks if names.get(n,{}).get("status")=="fail"]; print("1" if not bad else "0")' 2>/dev/null || echo "0")" + if [[ "$ok" == "1" ]]; then + echo "✓ DLP transport freshness check passed" + else + echo "✗ DLP transport freshness check failed" + UNHEALTHY_SERVICES+=("dlp-transport") + fi + + local errors warnings + errors="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="fail" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" + warnings="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="warn" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" + if [[ -n "$errors" ]]; then + echo " errors: $errors" + fi + if [[ -n "$warnings" ]]; then + echo " warnings: $warnings" + WARNINGS+=("$warnings") + fi +} + +echo "=== AW Services Health Check ===" +echo "Timestamp: $(date)" +echo + +# Check systemd services +for service in "${SERVICES[@]}"; do + check_service "$service" +done + +echo + +# Check API endpoints +check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" "$AW_HEALTH_API_TIMEOUT_SECONDS" "$AW_HEALTH_API_ATTEMPTS" +check_api_endpoint "$WORKTIME_HEALTH_URL" "aw-worktime-api" "$WORKTIME_HEALTH_TIMEOUT_SECONDS" 2 +check_dlp_transport_freshness "http://127.0.0.1:5600/api/0" "900" "${AW_HEALTH_STRICT_FILEOPS:-0}" +check_expected_setting "startOfDay" "${AW_EXPECT_START_OF_DAY:-}" "startOfDay" +check_expected_setting "always_active_pattern" "${AW_EXPECT_ALWAYS_ACTIVE_PATTERN:-}" "always_active_pattern" +check_expected_setting "landingpage" "${AW_EXPECT_LANDINGPAGE:-}" "landingpage" + +echo + +if [ ${#UNHEALTHY_SERVICES[@]} -eq 0 ]; then + echo "✓ All services are healthy" + if [ ${#WARNINGS[@]} -gt 0 ]; then + echo "⚠ Warnings: ${WARNINGS[*]}" + fi + exit 0 +else + echo "✗ Unhealthy services: ${UNHEALTHY_SERVICES[*]}" + exit 1 +fi diff --git a/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.mjs b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.mjs new file mode 100644 index 0000000..08fe5bd --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.mjs @@ -0,0 +1,498 @@ +#!/usr/bin/env node +import { createRequire } from "node:module"; +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { promisify } from "node:util"; + +const require = createRequire(import.meta.url); +const execFileAsync = promisify(execFile); + +function loadPlaywright() { + const candidates = [ + "playwright", + "playwright-core", + process.env.PLAYWRIGHT_CORE_MODULE, + "/home/igor/.agents/skills/playwright/node_modules/playwright-core", + ].filter(Boolean); + const errors = []; + for (const candidate of candidates) { + try { + return require(candidate); + } catch (error) { + errors.push(`${candidate}: ${error.message}`); + } + } + return null; +} + +function firstExisting(candidates) { + return candidates.find((item) => item && fs.existsSync(item)) || ""; +} + +function env(name, fallback) { + const value = process.env[name]; + return value && value.trim() ? value.trim() : fallback; +} + +function normalizeBase(url) { + return url.replace(/\/+$/, ""); +} + +function safeName(value) { + return value.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-|-$/g, ""); +} + +function envInt(name, fallback) { + const value = Number(env(name, String(fallback))); + return Number.isFinite(value) ? value : fallback; +} + +function isRunDirectoryName(name) { + return /^20\d{2}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.test(name); +} + +function pruneOutputRuns(outRoot, keepRuns) { + if (keepRuns <= 0 || !fs.existsSync(outRoot)) { + return []; + } + const entries = fs + .readdirSync(outRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && isRunDirectoryName(entry.name)) + .map((entry) => entry.name) + .sort(); + const stale = entries.slice(0, Math.max(0, entries.length - keepRuns)); + for (const name of stale) { + fs.rmSync(path.join(outRoot, name), { recursive: true, force: true }); + } + return stale; +} + +function decodeHtmlEntities(value) { + return value + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +function htmlToText(html) { + return decodeHtmlEntities( + html + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(), + ); +} + +function commandInPath(name) { + const dirs = (process.env.PATH || "").split(path.delimiter); + for (const dir of dirs) { + const candidate = path.join(dir, name); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return ""; +} + +function findChromiumExecutable(explicitPath) { + return firstExisting([ + explicitPath, + "/home/igor/.cache/ms-playwright/chromium-1217/chrome-linux64/chrome", + "/home/igor/.cache/ms-playwright/chromium-1208/chrome-linux64/chrome", + "/home/igor/.cache/rod/browser/chromium-1321438/chrome", + commandInPath("chromium"), + commandInPath("chromium-browser"), + commandInPath("google-chrome"), + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/usr/bin/google-chrome", + ]); +} + +function isBenignConsoleError(text) { + return text.includes("Failed to load resource: the server responded with a status of 404 (Not Found)"); +} + +function isBenignRequestFailure(failure) { + return failure.error === "net::ERR_ABORTED"; +} + +async function waitForTextMarkers(page, markers, timeoutMs) { + if (!markers.length) { + return; + } + try { + await page.waitForFunction( + (expected) => { + const text = `${document.title}\n${document.body ? document.body.innerText : ""}`; + return expected.every((marker) => text.includes(marker)); + }, + markers, + { timeout: timeoutMs }, + ); + } catch { + // The final assertion below reports the exact missing markers. + } +} + +async function runPageCheck(browser, spec, runDir) { + const page = await browser.newPage({ + viewport: { width: 1366, height: 768 }, + locale: "ru-RU", + timezoneId: "Europe/Moscow", + }); + const started = Date.now(); + const consoleErrors = []; + const pageErrors = []; + const requestFailures = []; + const badResponses = []; + const responses = []; + + page.on("console", (message) => { + if (message.type() === "error") { + const text = message.text(); + if (!isBenignConsoleError(text)) { + consoleErrors.push(text); + } + } + }); + page.on("pageerror", (error) => { + pageErrors.push(error.message); + }); + page.on("requestfailed", (request) => { + const url = request.url(); + if (spec.relevantHosts.some((host) => url.startsWith(host))) { + const failure = { url, error: request.failure()?.errorText || "unknown" }; + if (!isBenignRequestFailure(failure)) { + requestFailures.push(failure); + } + } + }); + page.on("response", (response) => { + const url = response.url(); + if (!spec.relevantHosts.some((host) => url.startsWith(host))) { + return; + } + const item = { url, status: response.status() }; + responses.push(item); + if (response.status() >= 400) { + badResponses.push(item); + } + }); + + let status = null; + let title = ""; + let bodyText = ""; + let screenshot = ""; + let error = ""; + try { + const response = await page.goto(spec.url, { waitUntil: "commit", timeout: spec.timeoutMs }); + status = response ? response.status() : null; + await waitForTextMarkers(page, spec.requiredText, spec.renderTimeoutMs); + await page.waitForTimeout(spec.settleMs); + title = await page.title(); + bodyText = await page.locator("body").innerText({ timeout: 5000 }); + screenshot = path.join(runDir, `${safeName(spec.name)}.png`); + await page.screenshot({ path: screenshot, fullPage: true }); + } catch (caught) { + error = caught && caught.message ? caught.message : String(caught); + try { + screenshot = path.join(runDir, `${safeName(spec.name)}-failed.png`); + await page.screenshot({ path: screenshot, fullPage: true }); + } catch { + screenshot = ""; + } + } finally { + await page.close(); + } + + const visibleText = `${title}\n${bodyText}`; + const missingMarkers = spec.requiredText.filter((marker) => !visibleText.includes(marker)); + const ok = + !error && + status !== null && + status >= 200 && + status < 300 && + bodyText.length >= spec.minBodyText && + missingMarkers.length === 0 && + consoleErrors.length === 0 && + pageErrors.length === 0 && + requestFailures.length === 0 && + badResponses.length === 0; + + return { + engine: "playwright", + name: spec.name, + ok, + url: spec.url, + status, + title, + body_text_length: bodyText.length, + missing_markers: missingMarkers, + console_errors: consoleErrors, + page_errors: pageErrors, + request_failures: requestFailures, + bad_responses: badResponses, + response_count: responses.length, + latency_ms: Date.now() - started, + screenshot, + error, + }; +} + +async function runChromiumCliPageCheck(chromiumPath, spec, runDir) { + const started = Date.now(); + const screenshot = path.join(runDir, `${safeName(spec.name)}.png`); + const profileDir = path.join(runDir, `${safeName(spec.name)}-profile`); + const screenshotRequired = spec.cliScreenshotRequired !== false; + fs.mkdirSync(profileDir, { recursive: true }); + const browserArgs = [ + "--headless", + "--no-sandbox", + "--noerrdialogs", + "--disable-gpu", + "--disable-crash-reporter", + "--disable-crashpad", + "--disable-dev-shm-usage", + `--user-data-dir=${profileDir}`, + `--virtual-time-budget=${spec.renderTimeoutMs + spec.settleMs}`, + ]; + + let status = null; + let error = ""; + let fetchedText = ""; + try { + const controller = AbortSignal.timeout(spec.timeoutMs); + const response = await fetch(spec.url, { signal: controller }); + status = response.status; + fetchedText = await response.text(); + } catch (caught) { + error = caught && caught.message ? caught.message : String(caught); + } + + let html = ""; + try { + const { stdout } = await execFileAsync(chromiumPath, [...browserArgs, "--dump-dom", spec.url], { + timeout: spec.timeoutMs + spec.renderTimeoutMs + spec.settleMs + 10000, + maxBuffer: 10 * 1024 * 1024, + }); + html = stdout || ""; + } catch (caught) { + const message = caught && caught.message ? caught.message : String(caught); + error = error ? `${error}; ${message}` : message; + } + + if (screenshotRequired) { + try { + await execFileAsync( + chromiumPath, + [...browserArgs, "--window-size=1366,768", `--screenshot=${screenshot}`, spec.url], + { + timeout: spec.timeoutMs + spec.renderTimeoutMs + spec.settleMs + 10000, + maxBuffer: 1024 * 1024, + }, + ); + } catch (caught) { + const message = caught && caught.message ? caught.message : String(caught); + error = error ? `${error}; screenshot: ${message}` : `screenshot: ${message}`; + } + } + + const rawHtml = html || fetchedText; + const titleMatch = rawHtml.match(/]*>([\s\S]*?)<\/title>/i); + const title = titleMatch ? decodeHtmlEntities(titleMatch[1].trim()) : ""; + const bodyText = htmlToText(rawHtml); + const visibleText = `${title}\n${bodyText}\n${rawHtml}`; + const requiredText = spec.cliRequiredText || spec.requiredText; + const minBodyText = spec.cliMinBodyText || spec.minBodyText; + const missingMarkers = requiredText.filter((marker) => !visibleText.includes(marker)); + const ok = + !error && + status !== null && + status >= 200 && + status < 300 && + bodyText.length >= minBodyText && + missingMarkers.length === 0 && + (!screenshotRequired || fs.existsSync(screenshot)); + + return { + engine: "chromium-cli", + name: spec.name, + ok, + url: spec.url, + status, + title, + body_text_length: bodyText.length, + missing_markers: missingMarkers, + console_errors: [], + page_errors: [], + request_failures: [], + bad_responses: [], + response_count: 0, + latency_ms: Date.now() - started, + screenshot_required: screenshotRequired, + screenshot: fs.existsSync(screenshot) ? screenshot : "", + error, + }; +} + +async function runPageCheckWithRetries(runOnce, spec, maxRetries) { + let result = await runOnce(spec); + for (let attempt = 1; !result.ok && attempt <= maxRetries; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); + const retry = await runOnce({ ...spec, name: `${spec.name}_retry${attempt}` }); + retry.retry_of = spec.name; + retry.retry_attempt = attempt; + if (retry.ok) { + retry.name = spec.name; + retry.recovered_after_retry = attempt; + return retry; + } + result = retry; + result.retry_of = spec.name; + result.retry_attempt = attempt; + } + return result; +} + +async function main() { + const requestedEngine = env("AW_BROWSER_SMOKE_ENGINE", "auto"); + const playwright = requestedEngine === "chromium-cli" ? null : loadPlaywright(); + const awBase = normalizeBase(env("AW_BROWSER_SMOKE_AW_BASE", env("AW_SMOKE_AW_SERVER", "http://10.10.10.13:5600"))); + const worktimeBase = normalizeBase(env("AW_BROWSER_SMOKE_WORKTIME_BASE", env("AW_SMOKE_WORKTIME_API", "http://10.10.10.13:5610"))); + const host = env("AW_BROWSER_SMOKE_HOST", env("AW_SMOKE_SOURCE_HOSTNAME", "SHARKON2025")); + const timeoutMs = Number(env("AW_BROWSER_SMOKE_TIMEOUT_MS", "20000")); + const settleMs = Number(env("AW_BROWSER_SMOKE_SETTLE_MS", "6000")); + const renderTimeoutMs = Number(env("AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS", "15000")); + const pageRetries = Number(env("AW_BROWSER_SMOKE_PAGE_RETRIES", "1")); + const keepRuns = envInt("AW_BROWSER_SMOKE_KEEP_RUNS", 24); + const outRoot = env("AW_BROWSER_SMOKE_OUTPUT_DIR", path.resolve("output", "browser-smoke")); + const runId = new Date().toISOString().replace(/[:.]/g, "-"); + const runDir = path.join(outRoot, runId); + fs.mkdirSync(outRoot, { recursive: true }); + const prunedRuns = pruneOutputRuns(outRoot, keepRuns); + fs.mkdirSync(runDir, { recursive: true }); + + const executablePath = findChromiumExecutable(env( + "PLAYWRIGHT_CHROMIUM_EXECUTABLE", + "", + )); + + const launchOptions = { + headless: true, + args: ["--no-sandbox", "--disable-dev-shm-usage"], + }; + if (executablePath) { + launchOptions.executablePath = executablePath; + } + + const relevantHosts = [awBase, worktimeBase]; + const specs = [ + { + name: "aw_webui_home", + url: `${awBase}/`, + relevantHosts, + requiredText: ["Активность", "Windows RDP", host, "DLP"], + cliRequiredText: ["ActivityWatch", "ru-patch-v5.js", "aw-report-links"], + cliMinBodyText: 100, + cliScreenshotRequired: false, + minBodyText: 500, + timeoutMs, + settleMs, + renderTimeoutMs, + }, + { + name: "worktime_today_html", + url: `${worktimeBase}/reports/worktime/today?format=html&day=today&host=${encodeURIComponent(host)}&allow_stale=1`, + relevantHosts, + requiredText: ["AW-rus", "Отчёт", "RDP"], + minBodyText: 500, + timeoutMs, + settleMs: 1000, + renderTimeoutMs, + }, + { + name: "worktime_management_html", + url: `${worktimeBase}/reports/worktime/management?format=html&day=today&host=${encodeURIComponent(host)}&allow_stale=1`, + relevantHosts, + requiredText: ["AW-rus", "Управленческий", "RDP"], + minBodyText: 500, + timeoutMs, + settleMs: 1000, + renderTimeoutMs, + }, + ]; + + const pages = []; + if (playwright) { + const { chromium } = playwright; + const browser = await chromium.launch(launchOptions); + try { + for (const spec of specs) { + pages.push(await runPageCheckWithRetries((item) => runPageCheck(browser, item, runDir), spec, pageRetries)); + } + } finally { + await browser.close(); + } + } else { + if (requestedEngine === "playwright") { + throw new Error("AW_BROWSER_SMOKE_ENGINE=playwright requested, but Playwright could not be loaded"); + } + if (!executablePath) { + throw new Error("Unable to load Playwright and no Chromium executable found"); + } + for (const spec of specs) { + pages.push(await runPageCheckWithRetries((item) => runChromiumCliPageCheck(executablePath, item, runDir), spec, pageRetries)); + } + } + + const ok = pages.every((page) => page.ok); + const result = { + ok, + engine: pages[0]?.engine || "unknown", + generated_at_utc: new Date().toISOString(), + aw_base: awBase, + worktime_base: worktimeBase, + host, + output_dir: runDir, + retention: { + keep_runs: keepRuns, + pruned_runs: prunedRuns, + }, + pages, + }; + const jsonPath = path.join(runDir, "result.json"); + const latestPath = path.join(outRoot, "latest-result.json"); + result.result_json = jsonPath; + result.latest_result_json = latestPath; + fs.writeFileSync(jsonPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); + fs.writeFileSync(latestPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return ok ? 0 : 2; +} + +main() + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + process.stdout.write( + `${JSON.stringify( + { + ok: false, + generated_at_utc: new Date().toISOString(), + error: error && error.message ? error.message : String(error), + }, + null, + 2, + )}\n`, + ); + process.exitCode = 2; + }); diff --git a/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh new file mode 100644 index 0000000..ebc8624 --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ -z "${NODE_PATH:-}" && -d /home/igor/.agents/skills/playwright/node_modules ]]; then + export NODE_PATH=/home/igor/.agents/skills/playwright/node_modules +fi + +exec node "$ROOT_DIR/scripts/aw-webui-browser-smoke.mjs" "$@" diff --git a/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh b/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh new file mode 100644 index 0000000..ff2600b --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +KIT_DIR="install-kit-awindows-20260427-211240" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${CHECK_INSTALL_KIT_VS_REPO_RUST:-}" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/check-install-kit-vs-repo" + "$ROOT_DIR/adk-rust/target/release/check-install-kit-vs-repo" + "/usr/local/bin/check-install-kit-vs-repo" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" --kit-dir "$KIT_DIR" "$@" + fi +done + +python3 - <<'PY' +from pathlib import Path +import hashlib +import sys + +root=Path('.') +kit=Path('install-kit-awindows-20260427-211240') +if not kit.exists(): + raise SystemExit('Install kit directory not found') + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + +all_compared=[] +mismatches=[] +missing_in_repo=[] +allowed_kit_only_prefixes=('server-configs-' ,) +allowed_kit_only_files={'README-INSTALL-KIT.txt'} + +for kp in sorted(p for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'): + rel=kp.relative_to(kit) + rp=root/rel + if not rp.exists(): + rel_str=str(rel) + if rel_str in allowed_kit_only_files or rel_str.startswith(allowed_kit_only_prefixes): + continue + missing_in_repo.append(str(rel)) + continue + all_compared.append(str(rel)) + if sha(kp)!=sha(rp): + mismatches.append(str(rel)) + +ps_mismatches=[ + p for p in mismatches + if p.startswith('windows/') and p.endswith(('.ps1', '.psm1', '.psd1')) +] + +print(f'Compared files: {len(all_compared)}') +print(f'Missing in repo: {len(missing_in_repo)}') +print(f'Mismatched content: {len(mismatches)}') +if missing_in_repo: + print('--- Missing in repo ---') + for p in missing_in_repo: + print(p) +if mismatches: + print('--- Mismatches ---') + for p in mismatches: + print(p) +print(f'PowerShell mismatches: {len(ps_mismatches)}') +if ps_mismatches: + print('--- PowerShell mismatches ---') + for p in ps_mismatches: + print(p) +if missing_in_repo or mismatches: + sys.exit(1) +PY diff --git a/install-kit-awindows-20260427-211240/scripts/quality-gate.sh b/install-kit-awindows-20260427-211240/scripts/quality-gate.sh new file mode 100644 index 0000000..7a4bcf4 --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/quality-gate.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${QUALITY_GATE_RUST:-}" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/quality-gate" + "$ROOT_DIR/adk-rust/target/release/quality-gate" + "/usr/local/bin/quality-gate" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi +done + +echo "[1/5] Bash syntax check" +find aw-server proxmox scripts -type f -name "*.sh" -print0 | xargs -0 -r -n1 bash -n + +echo "[2/5] Shellcheck (if available)" +if command -v shellcheck >/dev/null 2>&1; then + { + find aw-server proxmox -type f -name "*.sh" + printf '%s\n' scripts/aw-webui-browser-smoke.sh + } | xargs -r shellcheck -e SC1007,SC1090,SC2016 +else + echo "shellcheck not found, skipping." +fi + +echo "[3/5] Node syntax check (if node available)" +if command -v node >/dev/null 2>&1; then + node --check scripts/aw-webui-browser-smoke.mjs >/dev/null +else + echo "node not found, skipping." +fi + +echo "[4/5] PowerShell parse check (if pwsh available)" +if command -v pwsh >/dev/null 2>&1; then + pwsh -NoLogo -NoProfile -Command ' + $ErrorActionPreference = "Stop" + Get-ChildItem windows -Filter *.ps1 | ForEach-Object { + [void][System.Management.Automation.Language.Parser]::ParseFile($_.FullName,[ref]$null,[ref]$null) + } + [void][System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "windows/ActivityWatch.Windows.Common.psm1"),[ref]$null,[ref]$null) + [void][System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "windows/ActivityWatch.Windows.Common.psd1"),[ref]$null,[ref]$null) + ' + pwsh -NoLogo -NoProfile -File windows/aw-collector-guard.ps1 -SelfTest >/dev/null +else + echo "pwsh not found, skipping." +fi + + + +echo "[5/5] Ansible syntax check (if ansible-playbook available)" +if command -v ansible-playbook >/dev/null 2>&1; then + for playbook in ansible/*.yml; do + ansible-playbook --syntax-check "$playbook" -i ansible/inventory.example.ini >/dev/null + done +else + echo "ansible-playbook not found, skipping." +fi + +echo "quality-gate: OK" diff --git a/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh b/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh new file mode 100644 index 0000000..13936f0 --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${REBUILD_INSTALL_KIT_RUST:-}" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/rebuild-install-kit" + "$ROOT_DIR/adk-rust/target/release/rebuild-install-kit" + "/usr/local/bin/rebuild-install-kit" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi +done + +KIT_DIR="install-kit-awindows-20260427-211240" +ZIP_ARCHIVE="${KIT_DIR}.zip" +TAR_ARCHIVE="${KIT_DIR}.tar.gz" +SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.18" +OLD_SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.21" +TMP_SERVER_CONFIG_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_SERVER_CONFIG_DIR"' EXIT + +copy_file() { + local src="$1" + local dest="$2" + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" +} + +sync_tree() { + local base="$1" + shift + for rel in "$@"; do + copy_file "$rel" "${KIT_DIR}/${rel}" + done +} + +if [[ -d "$OLD_SERVER_CONFIG_DIR" ]]; then + cp "$OLD_SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ +fi +if [[ -d "$SERVER_CONFIG_DIR" ]]; then + cp "$SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ +fi + +rm -rf "${KIT_DIR}/ansible" "${KIT_DIR}/aw-server" "${KIT_DIR}/windows" "${KIT_DIR}/scripts" "${KIT_DIR}/server-configs-"* + +ansible_files=( + ansible/README.md + ansible/deploy_aw_pfsense_poller.yml + ansible/deploy_aw_server.yml + ansible/deploy_aw_windows.yml + ansible/group_vars/all.example.yml + ansible/group_vars/pfsense-poller.example.yml + ansible/group_vars/proxmox-matrix.example.yml + ansible/group_vars/proxmox.example.yml + ansible/group_vars/windows.example.yml + ansible/install_full_stack.yml + ansible/inventory.example.ini + ansible/provision_proxmox_ct_and_deploy_aw.yml + ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml + ansible/tasks/provision_ct_and_deploy_aw.yml +) + +aw_server_files=( + aw-server/activitywatch-server.service + aw-server/apply_webui_ru_patch.sh + aw-server/aw-host-groups.json + aw-server/aw-ru-patch.js + aw-server/aw-rus-healthd.py + aw-server/aw-rus-healthd.service + aw-server/aw-rus-healthd.timer + aw-server/aw-browser-smoke.service + aw-server/aw-browser-smoke.timer + aw-server/aw-slo-monitor.py + aw-server/aw-slo-monitor.service + aw-server/aw-slo-monitor.timer + aw-server/aw-server.env.example + aw-server/aw-sw-cleanup.js + aw-server/aw-worktime-api.py + aw-server/aw-worktime-api.service + aw-server/aw-worktime-prewarm.sh + aw-server/aw-worktime-prewarm.service + aw-server/aw-worktime-prewarm.timer + aw-server/aw-worktime-panel.js + aw-server/health-check.sh + aw-server/install_aw_server.sh + aw-server/settings/classes-worktime.json + aw-server/settings/views-default.json +) + +windows_files=( + windows/ActivityWatch.Windows.Common.psd1 + windows/ActivityWatch.Windows.Common.psm1 + windows/browser-domains-native-collector.ps1 + windows/deploy-domain-users.ps1 + windows/deploy-ensemble.ps1 + windows/deploy-single-user.ps1 + windows/AWatchRusCollectorGuardService.cs + windows/aw-collector-guard.ps1 + windows/install-collector-guard-service.ps1 + windows/dlp-endpoint-signals-collector.ps1 + windows/dlp-policy.example.json + windows/dlp-policy.native-cross-os.example.json + windows/email-outbound-collector.ps1 + windows/hardening-recovery.ps1 + windows/migrate-awatch-rus-paths.ps1 + windows/validate-deployment.ps1 + windows/web-category-rules.example.json + windows/worktime-session-collector.ps1 +) + +scripts_files=( + scripts/aw-webui-browser-smoke.mjs + scripts/aw-webui-browser-smoke.sh + scripts/check_install_kit_vs_repo.sh + scripts/quality-gate.sh + scripts/rebuild_install_kit.sh + scripts/validate_install_kit.sh + scripts/verify_innosetup_installer.sh +) + +sync_tree ansible "${ansible_files[@]}" +sync_tree aw-server "${aw_server_files[@]}" +sync_tree windows "${windows_files[@]}" +sync_tree scripts "${scripts_files[@]}" + +mkdir -p "$SERVER_CONFIG_DIR" +if compgen -G "$TMP_SERVER_CONFIG_DIR/*.deployment-config.json" >/dev/null; then + cp "$TMP_SERVER_CONFIG_DIR"/*.deployment-config.json "$SERVER_CONFIG_DIR"/ +fi + +cat > "${KIT_DIR}/README-INSTALL-KIT.txt" <<'EOF' +ActivityWatch DetMir Windows Install Kit + +Includes: +- windows/* (deploy scripts, collectors, common module, configs/examples) +- ansible/* (Windows and AW server playbooks, examples, inventory, tasks) +- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) +- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks) +- server-configs-192.168.100.18/* (working Windows/RDP config snapshots) + +Source: +- Local project snapshot at build time. +EOF + +python3 - <<'PY' +from pathlib import Path +import hashlib + +root = Path('install-kit-awindows-20260427-211240') +manifest = root / 'MANIFEST.txt' +files = sorted(p for p in root.rglob('*') if p.is_file() and p.name != 'MANIFEST.txt') +with manifest.open('w', encoding='utf-8') as handle: + for path in files: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + handle.write(f"{digest} {path.as_posix()}\n") +PY + +rm -f "$ZIP_ARCHIVE" "$TAR_ARCHIVE" +zip -rq "$ZIP_ARCHIVE" "$KIT_DIR" +tar -czf "$TAR_ARCHIVE" "$KIT_DIR" diff --git a/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh b/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh new file mode 100644 index 0000000..94b924a --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +KIT_DIR="install-kit-awindows-20260427-211240" +MANIFEST="$KIT_DIR/MANIFEST.txt" +ZIP_ARCHIVE="install-kit-awindows-20260427-211240.zip" +TAR_ARCHIVE="install-kit-awindows-20260427-211240.tar.gz" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${VALIDATE_INSTALL_KIT_RUST:-}" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/validate-install-kit" + "$ROOT_DIR/adk-rust/target/release/validate-install-kit" + "/usr/local/bin/validate-install-kit" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" \ + --root "$ROOT_DIR" \ + --kit-dir "$KIT_DIR" \ + --zip-archive "$ZIP_ARCHIVE" \ + --tar-archive "$TAR_ARCHIVE" \ + "$@" + fi +done + +required_files=( + "$MANIFEST" + "$KIT_DIR/README-INSTALL-KIT.txt" + "$KIT_DIR/windows/deploy-ensemble.ps1" + "$KIT_DIR/windows/validate-deployment.ps1" + "$KIT_DIR/ansible/deploy_aw_windows.yml" + "$KIT_DIR/aw-server/install_aw_server.sh" + "$KIT_DIR/scripts/rebuild_install_kit.sh" + "$KIT_DIR/scripts/validate_install_kit.sh" + "$KIT_DIR/scripts/check_install_kit_vs_repo.sh" + "$KIT_DIR/scripts/quality-gate.sh" +) + +echo "[1/4] Required files presence" +for file in "${required_files[@]}"; do + [[ -f "$file" ]] || { echo "Missing required file: $file"; exit 1; } +done + +echo "[2/4] Manifest checksum verification" +sha256sum -c "$MANIFEST" >/dev/null + +echo "[3/4] Manifest completeness" +python3 - <<'PY' +from pathlib import Path +import sys +kit=Path('install-kit-awindows-20260427-211240') +manifest=kit/'MANIFEST.txt' +listed=[] +for line in manifest.read_text().splitlines(): + line=line.strip() + if not line: + continue + parts=line.split(' ',1) + if len(parts)!=2: + print(f'Invalid MANIFEST line: {line}') + sys.exit(1) + listed.append(parts[1]) +listed_set=set(listed) +actual_set={str(p) for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'} +missing=sorted(listed_set-actual_set) +extra=sorted(actual_set-listed_set) +if missing or extra: + print('Missing files listed in MANIFEST:', missing) + print('Files not listed in MANIFEST:', extra) + sys.exit(1) +print(f'MANIFEST complete: {len(actual_set)} files tracked') +PY + +echo "[4/4] Archive composition check" +python3 - <<'PY' +from pathlib import Path +import tarfile, zipfile, sys +kit_prefix='install-kit-awindows-20260427-211240/' +zip_path=Path('install-kit-awindows-20260427-211240.zip') +tar_path=Path('install-kit-awindows-20260427-211240.tar.gz') +if not zip_path.exists() or not tar_path.exists(): + print('Archives not found') + sys.exit(1) +with zipfile.ZipFile(zip_path) as z: + zip_files=sorted(i for i in z.namelist() if not i.endswith('/')) +with tarfile.open(tar_path, 'r:gz') as t: + tar_files=sorted(m.name for m in t.getmembers() if m.isfile()) +if zip_files != tar_files: + print('ZIP and TAR contents differ') + sys.exit(1) +if not all(f.startswith(kit_prefix) for f in zip_files): + print('Unexpected archive prefix layout') + sys.exit(1) +print(f'Archives match: {len(zip_files)} files') +PY + +echo "validate_install_kit: OK" diff --git a/install-kit-awindows-20260427-211240/scripts/verify_innosetup_installer.sh b/install-kit-awindows-20260427-211240/scripts/verify_innosetup_installer.sh new file mode 100644 index 0000000..fb89657 --- /dev/null +++ b/install-kit-awindows-20260427-211240/scripts/verify_innosetup_installer.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +INSTALLER="${1:-windows/installkit/innosetup/AWatch-rus-InstallKit.exe}" +WINEPREFIX_VERIFY="${WINEPREFIX_VERIFY:-/tmp/aw-inno-verify-wineprefix}" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${VERIFY_INNOSETUP_INSTALLER_RUST:-}" +INSTALL_DIR_WIN='C:\AWatchRusExtract' +INSTALL_DIR_UNIX="${WINEPREFIX_VERIFY}/drive_c/AWatchRusExtract" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/verify-innosetup-installer" + "$ROOT_DIR/adk-rust/target/release/verify-innosetup-installer" + "/usr/local/bin/verify-innosetup-installer" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" "$INSTALLER" --root "$ROOT_DIR" --wineprefix "$WINEPREFIX_VERIFY" "${@:2}" + fi +done + +if [[ ! -f "$INSTALLER" ]]; then + echo "Installer not found: $INSTALLER" >&2 + exit 1 +fi + +if ! command -v wine >/dev/null 2>&1; then + echo "wine not found" >&2 + exit 1 +fi + +rm -rf "$WINEPREFIX_VERIFY" +mkdir -p "$WINEPREFIX_VERIFY" +export WINEPREFIX="$WINEPREFIX_VERIFY" +export WINEDEBUG="${WINEDEBUG:--all}" + +wineboot -u >/dev/null 2>&1 +wine "$INSTALLER" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- /TASKS="" /DIR="$INSTALL_DIR_WIN" >/dev/null 2>&1 +wineserver -w >/dev/null 2>&1 + +required_files=( + windows/AWatchRusCollectorGuardService.cs + windows/aw-collector-guard.ps1 + windows/install-collector-guard-service.ps1 + windows/dlp-policy.native-cross-os.example.json +) + +for rel in "${required_files[@]}"; do + extracted="${INSTALL_DIR_UNIX}/${rel}" + if [[ ! -f "$extracted" ]]; then + echo "Missing extracted file: $rel" >&2 + exit 1 + fi + if ! cmp -s "$rel" "$extracted"; then + echo "Extracted file differs from repo: $rel" >&2 + exit 1 + fi +done + +if ! grep -q 'collector guard self-test OK' "${INSTALL_DIR_UNIX}/windows/aw-collector-guard.ps1"; then + echo "Guard self-test marker missing in extracted installer payload" >&2 + exit 1 +fi + +echo "verify_innosetup_installer: OK" diff --git a/install-kit-awindows-20260427-211240/windows/AWatchRusCollectorGuardService.cs b/install-kit-awindows-20260427-211240/windows/AWatchRusCollectorGuardService.cs new file mode 100644 index 0000000..efcbdd1 --- /dev/null +++ b/install-kit-awindows-20260427-211240/windows/AWatchRusCollectorGuardService.cs @@ -0,0 +1,123 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.ServiceProcess; + +namespace AWatchRus +{ + public sealed class CollectorGuardService : ServiceBase + { + private Process child; + private readonly ServiceOptions options; + + public CollectorGuardService(ServiceOptions options) + { + this.options = options; + ServiceName = options.ServiceName; + CanStop = true; + CanShutdown = true; + } + + protected override void OnStart(string[] args) + { + Directory.CreateDirectory(Path.GetDirectoryName(options.LogPath)); + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " service starting" + Environment.NewLine); + + var psi = new ProcessStartInfo + { + FileName = options.PowerShellPath, + Arguments = string.Format( + "-NoProfile -ExecutionPolicy Bypass -File \"{0}\" -ConfigPath \"{1}\" -Mode {2} -LoopSeconds {3}", + options.ScriptPath, + options.ConfigPath, + options.Mode, + options.LoopSeconds), + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = false, + RedirectStandardError = false, + }; + child = Process.Start(psi); + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " child pid=" + child.Id + Environment.NewLine); + } + + protected override void OnStop() + { + StopChild("service stopping"); + } + + protected override void OnShutdown() + { + StopChild("system shutdown"); + } + + private void StopChild(string reason) + { + try + { + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " " + reason + Environment.NewLine); + if (child != null && !child.HasExited) + { + child.Kill(); + child.WaitForExit(10000); + } + } + catch (Exception ex) + { + try + { + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " stop error: " + ex.Message + Environment.NewLine); + } + catch + { + } + } + } + } + + public sealed class ServiceOptions + { + public string ServiceName = "AWatchRusCollectorGuard"; + public string PowerShellPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), "System32\\WindowsPowerShell\\v1.0\\powershell.exe"); + public string ScriptPath = @"C:\Program Files\AWatch-rus\windows\aw-collector-guard.ps1"; + public string ConfigPath = @"C:\ProgramData\AWatch-rus\deployment-config.json"; + public string Mode = "shadow"; + public int LoopSeconds = 60; + public string LogPath = @"C:\ProgramData\AWatch-rus\logs\collector-guard-service.log"; + } + + internal static class Program + { + private static void Main(string[] args) + { + var options = Parse(args); + ServiceBase.Run(new CollectorGuardService(options)); + } + + private static ServiceOptions Parse(string[] args) + { + var options = new ServiceOptions(); + for (var i = 0; i < args.Length; i++) + { + var key = args[i].ToLowerInvariant(); + var value = i + 1 < args.Length ? args[i + 1] : null; + if (value == null || value.StartsWith("--", StringComparison.Ordinal)) + { + continue; + } + if (key == "--service-name") options.ServiceName = value; + else if (key == "--script") options.ScriptPath = value; + else if (key == "--config") options.ConfigPath = value; + else if (key == "--mode") options.Mode = value; + else if (key == "--loop") + { + int parsed; + if (int.TryParse(value, out parsed)) options.LoopSeconds = parsed; + } + else if (key == "--log") options.LogPath = value; + i++; + } + return options; + } + } +} diff --git a/install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 b/install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 index 3c68781..053a497 100755 --- a/install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 +++ b/install-kit-awindows-20260427-211240/windows/ActivityWatch.Windows.Common.psm1 @@ -1,4 +1,4 @@ -Set-StrictMode -Version Latest +Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $script:ActivityWatchBuiltInAdministratorName = $null @@ -431,6 +431,22 @@ function Get-ActivityWatchTaskNameToken { return $buffer.ToString().Trim('_') } +function Test-ActivityWatchScheduledTaskExistsExact { + param([string]$TaskName) + + if ([string]::IsNullOrWhiteSpace($TaskName)) { + return $false + } + + try { + & schtasks.exe /Query /TN $TaskName *> $null + return ($LASTEXITCODE -eq 0) + } + catch { + return $false + } +} + function New-ActivityWatchUserTaskDefinitions { param( [Parameter(Mandatory = $true)] @@ -633,6 +649,119 @@ function Test-ActivityWatchUserHasLiveSession { return $false } +function Test-ActivityWatchSessionMatchesUserId { + param( + [Parameter(Mandatory = $true)] + [object]$SessionRecord, + [Parameter(Mandatory = $true)] + [string]$UserId + ) + + if ([string]::IsNullOrWhiteSpace($UserId) -or $null -eq $SessionRecord) { + return $false + } + + $sessionUser = [string]$SessionRecord.UserName + if ([string]::IsNullOrWhiteSpace($sessionUser)) { + return $false + } + + foreach ($candidate in @(Resolve-ActivityWatchUserCandidates -UserId $UserId)) { + if ($sessionUser -ieq $candidate -or + ('{0}\{1}' -f $env:COMPUTERNAME, $sessionUser) -ieq $candidate -or + ((-not [string]::IsNullOrWhiteSpace($env:USERDOMAIN)) -and ('{0}\{1}' -f $env:USERDOMAIN, $sessionUser) -ieq $candidate)) { + return $true + } + } + + return $false +} + +function Test-ActivityWatchUserHasManagedSession { + param( + [Parameter(Mandatory = $true)] + [string]$UserId, + [object[]]$SessionRecords, + [switch]$IncludeLive, + [switch]$IncludeDisconnected + ) + + foreach ($session in @($SessionRecords)) { + if ([int]$session.SessionId -le 0) { + continue + } + if ([string]::IsNullOrWhiteSpace([string]$session.UserName)) { + continue + } + if ([bool]$session.IsLive -and -not $IncludeLive.IsPresent) { + continue + } + if (-not [bool]$session.IsLive -and -not $IncludeDisconnected.IsPresent) { + continue + } + if (Test-ActivityWatchSessionMatchesUserId -SessionRecord $session -UserId $UserId) { + return $true + } + } + + return $false +} + +function Get-ActivityWatchManagedInteractiveSessions { + param( + [pscustomobject[]]$TaskDefinitions, + [object[]]$SessionRecords, + [switch]$IncludeLive, + [switch]$IncludeDisconnected + ) + + $result = New-Object System.Collections.Generic.List[object] + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($taskDef in @($TaskDefinitions)) { + $userId = [string]$taskDef.userId + $taskName = [string]$taskDef.taskName + if ([string]::IsNullOrWhiteSpace($userId) -or [string]::IsNullOrWhiteSpace($taskName)) { + continue + } + + foreach ($session in @($SessionRecords)) { + if ([int]$session.SessionId -le 0) { + continue + } + if ([string]::IsNullOrWhiteSpace([string]$session.UserName)) { + continue + } + if ([bool]$session.IsLive -and -not $IncludeLive.IsPresent) { + continue + } + if (-not [bool]$session.IsLive -and -not $IncludeDisconnected.IsPresent) { + continue + } + if (-not (Test-ActivityWatchSessionMatchesUserId -SessionRecord $session -UserId $userId)) { + continue + } + + $key = '{0}|{1}|{2}' -f $taskName, [int]$session.SessionId, $userId + if (-not $seen.Add($key)) { + continue + } + + $result.Add([pscustomobject]@{ + TaskName = $taskName + UserId = $userId + SessionName = [string]$session.SessionName + SessionId = [int]$session.SessionId + State = [string]$session.State + UserName = [string]$session.UserName + IsLive = [bool]$session.IsLive + }) | Out-Null + } + } + + return @($result.ToArray()) +} + function Copy-ActivityWatchCollectorAssets { param( [Parameter(Mandatory = $true)] @@ -777,7 +906,7 @@ function New-ActivityWatchDeploymentConfig { [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [Parameter(Mandatory = $true)] [string]$LaunchScriptPath, [Parameter(Mandatory = $true)] @@ -1410,6 +1539,10 @@ function Get-ActivityWatchRecoveryTaskDefinitions { foreach ($task in @($config.userTasks)) { $taskName = [string]$task.launchTaskName $userId = Normalize-ActivityWatchUserId -UserId ([string]$task.userId) + $canonicalTaskName = "ActivityWatch Launch [$((Get-ActivityWatchTaskNameToken -UserId $userId))]" + if ($canonicalTaskName -ne $taskName -and (Test-ActivityWatchScheduledTaskExistsExact -TaskName $canonicalTaskName)) { + $taskName = $canonicalTaskName + } if (-not [string]::IsNullOrWhiteSpace($taskName) -and -not $taskMap.Contains($taskName)) { $taskMap[$taskName] = [pscustomobject]@{ taskName = $taskName @@ -1679,13 +1812,24 @@ function Stop-ActivityWatchProcessesInNonLiveSessions { [Parameter(Mandatory = $true)] [object[]]$SessionRecords, [Parameter(Mandatory = $true)] - [pscustomobject]$Config + [pscustomobject]$Config, + [pscustomobject[]]$TaskDefinitions = @(), + [switch]$PreserveManagedSessions ) $stateRoot = if ($Config.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$Config.paths.stateRoot } else { Join-Path $env:ProgramData 'AWatch-rus' } + $preservedSessionIds = @() + if ($PreserveManagedSessions.IsPresent) { + $preservedSessionIds = @( + Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $TaskDefinitions -SessionRecords $SessionRecords -IncludeDisconnected | + ForEach-Object { [int]$_.SessionId } | + Sort-Object -Unique + ) + } + $sessionIds = @( $SessionRecords | - Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 } | + Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 -and ($preservedSessionIds -notcontains [int]$_.SessionId) } | ForEach-Object { [int]$_.SessionId } | Sort-Object -Unique ) @@ -1704,7 +1848,7 @@ function Stop-ActivityWatchProcessesInNonLiveSessions { } } - foreach ($session in @($SessionRecords | Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 })) { + foreach ($session in @($SessionRecords | Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 -and ($preservedSessionIds -notcontains [int]$_.SessionId) })) { Remove-ActivityWatchLogonMarkersForSession -StateRoot $stateRoot -SessionId ([int]$session.SessionId) -UserName ([string]$session.UserName) } @@ -1885,7 +2029,7 @@ function Invoke-ActivityWatchRecoveryLoop { $config = Read-ActivityWatchDeploymentConfig -Path $ConfigPath $taskDefs = Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths $sessionRecords = Get-ActivityWatchSessionRecords - Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config + Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config -TaskDefinitions $taskDefs -PreserveManagedSessions $sessionRecords = Get-ActivityWatchSessionRecords $stateRoot = [string]$config.paths.stateRoot $sessionCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$config.paths.sessionCollectorScript } else { Join-Path $stateRoot 'worktime-session-collector.ps1' } diff --git a/install-kit-awindows-20260427-211240/windows/aw-collector-guard.ps1 b/install-kit-awindows-20260427-211240/windows/aw-collector-guard.ps1 new file mode 100644 index 0000000..559995e --- /dev/null +++ b/install-kit-awindows-20260427-211240/windows/aw-collector-guard.ps1 @@ -0,0 +1,681 @@ +[CmdletBinding()] +param( + [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json', + [ValidateSet('shadow', 'enforce')] + [string]$Mode = 'shadow', + [int]$LoopSeconds = 60, + [int]$InteractiveMaxAgeSeconds = 900, + [int]$HeadlessMaxAgeSeconds = 900, + [int]$RestartWindowSeconds = 600, + [int]$MaxRestarts = 3, + [int]$ActionCooldownSeconds = 300, + [int]$InteractiveActionCooldownSeconds = 60, + [switch]$Once, + [switch]$HeadlessEndpointEnabled, + [switch]$HeadlessFileOpsEnabled, + [switch]$SelfTest +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$modulePath = Join-Path $PSScriptRoot 'ActivityWatch.Windows.Common.psm1' +Import-Module $modulePath -Force + +function New-GuardLock { + param([string]$StateRoot) + + if (-not (Test-Path -LiteralPath $StateRoot)) { + New-Item -Path $StateRoot -ItemType Directory -Force | Out-Null + } + + $lockPath = Join-Path $StateRoot 'collector-guard.lock' + if (Test-Path -LiteralPath $lockPath) { + try { + $lockData = Get-Content -LiteralPath $lockPath -Raw | ConvertFrom-Json + $existingPid = [int]$lockData.pid + if ($existingPid -gt 0 -and (Get-Process -Id $existingPid -ErrorAction SilentlyContinue)) { + return $null + } + } + catch { + } + } + + $payload = @{ + pid = $PID + createdAt = (Get-Date).ToUniversalTime().ToString('o') + } | ConvertTo-Json -Compress + Set-Content -LiteralPath $lockPath -Value $payload -Encoding UTF8 + return $lockPath +} + +function Write-GuardLog { + param( + [string]$LogPath, + [string]$Message + ) + + try { + $directory = Split-Path -Path $LogPath -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + Add-Content -LiteralPath $LogPath -Value ('{0} {1}' -f (Get-Date -Format s), $Message) + } + catch { + } +} + +function Get-AwApiBase { + param([pscustomobject]$Config) + + $scheme = if ($Config.server.PSObject.Properties.Name -contains 'scheme') { [string]$Config.server.scheme } else { 'http' } + $hostName = [string]$Config.server.host + $port = [int]$Config.server.port + return ('{0}://{1}:{2}/api/0' -f $scheme, $hostName, $port) +} + +function Invoke-AwJson { + param( + [Parameter(Mandatory = $true)] + [string]$Method, + [Parameter(Mandatory = $true)] + [string]$Uri, + [object]$Body + ) + + $params = @{ + Method = $Method + Uri = $Uri + TimeoutSec = 15 + ErrorAction = 'Stop' + } + if ($null -ne $Body) { + $params.Body = ($Body | ConvertTo-Json -Depth 16 -Compress) + $params.ContentType = 'application/json' + } + return Invoke-RestMethod @params +} + +function Ensure-AwBucket { + param( + [string]$ApiBase, + [string]$BucketId, + [string]$ClientName, + [string]$BucketType, + [string]$Hostname + ) + + try { + Invoke-AwJson -Method 'GET' -Uri "$ApiBase/buckets/$BucketId" | Out-Null + return $true + } + catch { + } + + try { + $body = @{ + client = $ClientName + type = $BucketType + hostname = $Hostname + } + Invoke-AwJson -Method 'POST' -Uri "$ApiBase/buckets/$BucketId" -Body $body | Out-Null + return $true + } + catch { + return $false + } +} + +function Get-LatestBucketAge { + param( + [string]$ApiBase, + [string]$BucketId + ) + + try { + $events = Invoke-AwJson -Method 'GET' -Uri "$ApiBase/buckets/$BucketId/events?limit=20" + $latest = @($events | Where-Object { $null -ne $_.timestamp } | Sort-Object timestamp -Descending | Select-Object -First 1) + if (-not $latest) { + return [pscustomobject]@{ bucket = $BucketId; found = $false; timestamp = $null; ageSeconds = $null } + } + $ts = [DateTimeOffset]::Parse([string]$latest.timestamp).UtcDateTime + $age = [Math]::Max(0, [int]((Get-Date).ToUniversalTime() - $ts).TotalSeconds) + return [pscustomobject]@{ bucket = $BucketId; found = $true; timestamp = [string]$latest.timestamp; ageSeconds = $age } + } + catch { + return [pscustomobject]@{ bucket = $BucketId; found = $false; timestamp = $null; ageSeconds = $null; error = $_.Exception.Message } + } +} + +function Send-GuardHeartbeat { + param( + [string]$ApiBase, + [string]$Hostname, + [object]$State, + [int]$PulseSeconds + ) + + $bucketId = "aw-rus-collector-guard_$Hostname" + if (-not (Ensure-AwBucket -ApiBase $ApiBase -BucketId $bucketId -ClientName 'aw-rus-collector-guard' -BucketType 'aw.rus.collector.guard' -Hostname $Hostname)) { + return $false + } + + $event = @{ + timestamp = (Get-Date).ToUniversalTime().ToString('o') + duration = 0 + data = $State + } + + try { + Invoke-AwJson -Method 'POST' -Uri "$ApiBase/buckets/$bucketId/heartbeat?pulsetime=$PulseSeconds" -Body $event | Out-Null + return $true + } + catch { + return $false + } +} + +function Read-GuardRuntime { + param([string]$Path) + + if (-not (Test-Path -LiteralPath $Path)) { + return [pscustomobject]@{ restartHistory = @{}; lastAction = @{}; quarantine = @{} } + } + try { + $state = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + if ($null -eq $state.restartHistory) { $state | Add-Member -NotePropertyName restartHistory -NotePropertyValue @{} } + if ($null -eq $state.lastAction) { $state | Add-Member -NotePropertyName lastAction -NotePropertyValue @{} } + if ($null -eq $state.quarantine) { $state | Add-Member -NotePropertyName quarantine -NotePropertyValue @{} } + return $state + } + catch { + return [pscustomobject]@{ restartHistory = @{}; lastAction = @{}; quarantine = @{} } + } +} + +function Write-GuardRuntime { + param( + [string]$Path, + [object]$Runtime + ) + + $directory = Split-Path -Path $Path -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + $Runtime | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $Path -Encoding UTF8 +} + +function Get-RuntimeMapValue { + param( + [object]$Map, + [string]$Key + ) + + if ($null -eq $Map) { + return $null + } + if ($Map -is [hashtable] -and $Map.ContainsKey($Key)) { + return $Map[$Key] + } + $propertyNames = @($Map.PSObject.Properties | ForEach-Object { $_.Name }) + if ($propertyNames -contains $Key) { + return $Map.$Key + } + return $null +} + +function Set-RuntimeMapValue { + param( + [object]$Map, + [string]$Key, + [object]$Value + ) + + if ($Map -is [hashtable]) { + $Map[$Key] = $Value + return + } + $propertyNames = @($Map.PSObject.Properties | ForEach-Object { $_.Name }) + if ($propertyNames -contains $Key) { + $Map.$Key = $Value + } + else { + $Map | Add-Member -NotePropertyName $Key -NotePropertyValue $Value -Force + } +} + +function Remove-RuntimeMapValue { + param( + [object]$Map, + [string]$Key + ) + + if ($null -eq $Map) { + return + } + if ($Map -is [hashtable]) { + if ($Map.ContainsKey($Key)) { + $Map.Remove($Key) + } + return + } + $property = $Map.PSObject.Properties[$Key] + if ($null -ne $property) { + $Map.PSObject.Properties.Remove($Key) + } +} + +function Reset-GuardActionBudget { + param( + [object]$Runtime, + [string]$Key + ) + + Remove-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key + Remove-RuntimeMapValue -Map $Runtime.lastAction -Key $Key + Remove-RuntimeMapValue -Map $Runtime.quarantine -Key $Key +} + +function Invoke-GuardSelfTest { + $emptyObject = [pscustomobject]@{} + if ($null -ne (Get-RuntimeMapValue -Map $emptyObject -Key 'missing')) { + throw 'empty PSCustomObject should not return a missing runtime-map value' + } + Set-RuntimeMapValue -Map $emptyObject -Key 'headless:worktime-session' -Value 123 + if ((Get-RuntimeMapValue -Map $emptyObject -Key 'headless:worktime-session') -ne 123) { + throw 'failed to set runtime-map value on empty PSCustomObject' + } + + $hash = @{} + Set-RuntimeMapValue -Map $hash -Key 'headless:worktime-session' -Value @(1, 2) + $hashValue = @(Get-RuntimeMapValue -Map $hash -Key 'headless:worktime-session') + if ($hashValue.Count -ne 2) { + throw 'failed to round-trip runtime-map value on hashtable' + } + + $runtime = [pscustomobject]@{ restartHistory = [pscustomobject]@{}; lastAction = [pscustomobject]@{}; quarantine = [pscustomobject]@{} } + $allowed = Test-ActionAllowed -Runtime $runtime -Key 'headless:worktime-session' -CooldownSeconds 1 -WindowSeconds 60 -MaxCount 3 + if (-not $allowed.allowed) { + throw "expected action to be allowed, got $($allowed.reason)" + } + Register-GuardAction -Runtime $runtime -Key 'headless:worktime-session' + $blocked = Test-ActionAllowed -Runtime $runtime -Key 'headless:worktime-session' -CooldownSeconds 300 -WindowSeconds 60 -MaxCount 3 + if ($blocked.allowed -or $blocked.reason -ne 'cooldown') { + throw 'expected cooldown after registering guard action' + } + $budgetRuntime = [pscustomobject]@{ restartHistory = [pscustomobject]@{}; lastAction = [pscustomobject]@{}; quarantine = [pscustomobject]@{} } + foreach ($i in 1..3) { + Register-GuardAction -Runtime $budgetRuntime -Key 'task:test' + } + $budgetBlocked = Test-ActionAllowed -Runtime $budgetRuntime -Key 'task:test' -CooldownSeconds 0 -WindowSeconds 600 -MaxCount 3 + if ($budgetBlocked.allowed -or $budgetBlocked.reason -ne 'quarantine') { + throw 'expected quarantine when restart budget is exhausted' + } + Reset-GuardActionBudget -Runtime $budgetRuntime -Key 'task:test' + $budgetAllowed = Test-ActionAllowed -Runtime $budgetRuntime -Key 'task:test' -CooldownSeconds 0 -WindowSeconds 600 -MaxCount 3 + if (-not $budgetAllowed.allowed) { + throw 'expected reset action budget to clear quarantine' + } + + $oldComputerName = $env:COMPUTERNAME + try { + $env:COMPUTERNAME = 'SHARKON2025' + $sessionRecords = @( + [pscustomobject]@{ SessionName = 'USER5'; UserName = 'USER5'; SessionId = 2; State = 'Disc'; IsLive = $false }, + [pscustomobject]@{ SessionName = 'console'; UserName = ''; SessionId = 1; State = 'Conn'; IsLive = $true } + ) + $taskDefs = @( + [pscustomobject]@{ taskName = 'ActivityWatch Launch [SHARKON2025_user5]'; userId = 'SHARKON2025\user5' } + ) + if (-not (Test-ActivityWatchUserHasManagedSession -UserId 'SHARKON2025\user5' -SessionRecords $sessionRecords -IncludeDisconnected)) { + throw 'expected disconnected managed session to match task user' + } + if (Test-ActivityWatchUserHasManagedSession -UserId 'SHARKON2025\user5' -SessionRecords $sessionRecords -IncludeLive) { + throw 'disconnected managed session should not match live-only filter' + } + $managed = @(Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $taskDefs -SessionRecords $sessionRecords -IncludeDisconnected) + if ($managed.Count -ne 1 -or [int]$managed[0].SessionId -ne 2) { + throw 'failed to enumerate disconnected managed session' + } + } + finally { + if ($null -eq $oldComputerName) { + Remove-Item Env:COMPUTERNAME -ErrorAction SilentlyContinue + } + else { + $env:COMPUTERNAME = $oldComputerName + } + } + + Write-Output 'collector guard self-test OK' +} + +function Test-ActionAllowed { + param( + [object]$Runtime, + [string]$Key, + [int]$CooldownSeconds, + [int]$WindowSeconds, + [int]$MaxCount + ) + + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $last = Get-RuntimeMapValue -Map $Runtime.lastAction -Key $Key + if ($null -ne $last -and ($now - [int64]$last) -lt $CooldownSeconds) { + return [pscustomobject]@{ allowed = $false; reason = 'cooldown' } + } + + $history = @(Get-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key) + $history = @($history | Where-Object { ($now - [int64]$_) -le $WindowSeconds }) + Set-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key -Value @($history) + if ($history.Count -ge $MaxCount) { + Set-RuntimeMapValue -Map $Runtime.quarantine -Key $Key -Value @{ + since = (Get-Date).ToUniversalTime().ToString('o') + reason = 'restart-budget-exhausted' + count = $history.Count + } + return [pscustomobject]@{ allowed = $false; reason = 'quarantine' } + } + + Remove-RuntimeMapValue -Map $Runtime.quarantine -Key $Key + return [pscustomobject]@{ allowed = $true; reason = 'ok' } +} + +function Register-GuardAction { + param( + [object]$Runtime, + [string]$Key + ) + + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $history = @(Get-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key) + $history += $now + Set-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key -Value @($history) + Set-RuntimeMapValue -Map $Runtime.lastAction -Key $Key -Value $now +} + +function Get-CollectorProcessSnapshot { + param([pscustomobject]$Config) + + $scriptPaths = [ordered]@{} + foreach ($name in @('collectorScript', 'endpointCollectorScript', 'fileCollectorScript', 'emailCollectorScript', 'sessionCollectorScript')) { + if ($Config.paths.PSObject.Properties.Name -contains $name) { + $value = [string]$Config.paths.$name + if (-not [string]::IsNullOrWhiteSpace($value)) { + $scriptPaths[$name] = $value + } + } + } + + $powershellCollectors = @() + try { + $processes = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.Name -ieq 'powershell.exe' -or $_.Name -ieq 'pwsh.exe' }) + foreach ($proc in $processes) { + $commandLine = [string]$proc.CommandLine + foreach ($entry in $scriptPaths.GetEnumerator()) { + if ($commandLine -match [Regex]::Escape([string]$entry.Value)) { + $powershellCollectors += [pscustomobject]@{ + name = [string]$entry.Key + processId = [int]$proc.ProcessId + sessionId = [int]$proc.SessionId + scriptPath = [string]$entry.Value + } + } + } + } + } + catch { + } + + $watchers = @() + try { + $watchers = @(Get-Process -Name 'aw-watcher-afk','aw-watcher-window' -ErrorAction SilentlyContinue | + Select-Object @{Name='name'; Expression={$_.Name}}, @{Name='processId'; Expression={$_.Id}}, @{Name='sessionId'; Expression={$_.SessionId}}) + } + catch { + $watchers = @() + } + + return [pscustomobject]@{ + watchers = @($watchers) + collectors = @($powershellCollectors) + } +} + +function Invoke-ExactTaskRun { + param([string]$TaskName) + + & schtasks.exe /Run /TN $TaskName | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Invoke-GuardCycle { + param( + [object]$Runtime, + [string]$RuntimePath, + [string]$LogPath + ) + + $config = Read-ActivityWatchDeploymentConfig -Path $ConfigPath + $stateRoot = if ($config.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$config.paths.stateRoot } else { Split-Path -Path $ConfigPath -Parent } + $hostname = if ($config.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]$config.awHostname)) { [string]$config.awHostname } else { [string]$env:COMPUTERNAME } + $apiBase = Get-AwApiBase -Config $config + $configPaths = Get-ActivityWatchRecoveryConfigPaths -PrimaryConfigPath $ConfigPath + $taskDefs = @(Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths) + $sessionRecords = @(Get-ActivityWatchSessionRecords) + $liveSessions = @(Get-ActivityWatchLiveInteractiveSessions -SessionRecords $sessionRecords) + $managedInteractiveSessions = @(Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $taskDefs -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected) + $processSnapshot = Get-CollectorProcessSnapshot -Config $config + $liveSessionIds = @($liveSessions | ForEach-Object { [int]$_.SessionId }) + $managedSessionIds = @($managedInteractiveSessions | ForEach-Object { [int]$_.SessionId } | Sort-Object -Unique) + + $bucketChecks = [ordered]@{} + foreach ($bucket in @( + "aw-worktime-sessions_$hostname", + "aw-watcher-afk_$hostname", + "aw-watcher-window_$hostname", + "aw-dlp-endpoint-signals_$hostname" + )) { + $bucketChecks[$bucket] = Get-LatestBucketAge -ApiBase $apiBase -BucketId $bucket + } + + $actions = New-Object System.Collections.Generic.List[object] + $problems = New-Object System.Collections.Generic.List[string] + + $worktimeAge = $bucketChecks["aw-worktime-sessions_$hostname"].ageSeconds + $sessionCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$config.paths.sessionCollectorScript } else { Join-Path $stateRoot 'worktime-session-collector.ps1' } + $sessionCollectorRunning = Test-ActivityWatchCollectorRunningGlobal -ScriptPath $sessionCollectorScript + $headlessKey = 'headless:worktime-session' + $needsHeadlessAction = (-not $sessionCollectorRunning -or $null -eq $worktimeAge -or [int]$worktimeAge -gt $HeadlessMaxAgeSeconds) + if ($needsHeadlessAction) { + $key = $headlessKey + $allowed = Test-ActionAllowed -Runtime $Runtime -Key $key -CooldownSeconds $ActionCooldownSeconds -WindowSeconds $RestartWindowSeconds -MaxCount $MaxRestarts + if ($allowed.allowed) { + if ($Mode -eq 'enforce') { + Start-ActivityWatchCollectorScriptGlobalIfNeeded -ScriptPath $sessionCollectorScript -ConfigPath $ConfigPath + Register-GuardAction -Runtime $Runtime -Key $key + Write-GuardLog -LogPath $LogPath -Message "started $key" + $actions.Add([pscustomobject]@{ action = 'start'; target = $key; applied = $true }) | Out-Null + } + else { + $actions.Add([pscustomobject]@{ action = 'start'; target = $key; applied = $false; mode = 'shadow' }) | Out-Null + } + } + else { + $problems.Add("$key action blocked: $($allowed.reason)") | Out-Null + } + } + else { + Reset-GuardActionBudget -Runtime $Runtime -Key $headlessKey + } + + if ($Mode -eq 'enforce') { + Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config -TaskDefinitions $taskDefs -PreserveManagedSessions + } + + $interactiveStale = $false + foreach ($bucket in @("aw-watcher-afk_$hostname", "aw-watcher-window_$hostname", "aw-dlp-endpoint-signals_$hostname")) { + $age = $bucketChecks[$bucket].ageSeconds + if ($null -eq $age -or [int]$age -gt $InteractiveMaxAgeSeconds) { + $interactiveStale = $true + } + } + + $watchersInLiveSessions = @($processSnapshot.watchers | Where-Object { $liveSessionIds -contains [int]$_.sessionId }) + $watchersInManagedSessions = @($processSnapshot.watchers | Where-Object { $managedSessionIds -contains [int]$_.sessionId }) + $liveWatcherMissing = $false + if ($liveSessions.Count -gt 0) { + $hasAfk = @($watchersInLiveSessions | Where-Object { [string]$_.name -ieq 'aw-watcher-afk' }).Count -gt 0 + $hasWindow = @($watchersInLiveSessions | Where-Object { [string]$_.name -ieq 'aw-watcher-window' }).Count -gt 0 + $liveWatcherMissing = (-not $hasAfk) -or (-not $hasWindow) + } + $managedWatcherMissing = $false + if ($managedInteractiveSessions.Count -gt 0) { + $afkEnabled = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'afkEnabled') { [bool]$config.collectors.afkEnabled } else { $true } + $windowEnabled = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'windowEnabled') { [bool]$config.collectors.windowEnabled } else { $true } + foreach ($managedSession in @($managedInteractiveSessions)) { + $sessionId = [int]$managedSession.SessionId + $sessionWatchers = @($watchersInManagedSessions | Where-Object { [int]$_.sessionId -eq $sessionId }) + $hasManagedAfk = @($sessionWatchers | Where-Object { [string]$_.name -ieq 'aw-watcher-afk' }).Count -gt 0 + $hasManagedWindow = @($sessionWatchers | Where-Object { [string]$_.name -ieq 'aw-watcher-window' }).Count -gt 0 + if (($afkEnabled -and -not $hasManagedAfk) -or ($windowEnabled -and -not $hasManagedWindow)) { + $managedWatcherMissing = $true + break + } + } + } + + if ($managedInteractiveSessions.Count -eq 0 -and $liveSessions.Count -gt 0 -and $interactiveStale) { + $problems.Add('interactive buckets stale but no managed interactive sessions found') | Out-Null + } + + $needsInteractiveTaskAction = $managedInteractiveSessions.Count -gt 0 -and ( + $liveWatcherMissing -or + $managedWatcherMissing -or + ($interactiveStale -and $liveSessions.Count -gt 0) + ) + + if ($needsInteractiveTaskAction) { + foreach ($taskDef in $taskDefs) { + if (-not (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected)) { + continue + } + $key = "task:$($taskDef.taskName)" + $allowed = Test-ActionAllowed -Runtime $Runtime -Key $key -CooldownSeconds $InteractiveActionCooldownSeconds -WindowSeconds $RestartWindowSeconds -MaxCount $MaxRestarts + if (-not $allowed.allowed) { + $problems.Add("$key action blocked: $($allowed.reason)") | Out-Null + continue + } + + if ($Mode -eq 'enforce') { + $ok = Invoke-ExactTaskRun -TaskName ([string]$taskDef.taskName) + if ($ok) { + Register-GuardAction -Runtime $Runtime -Key $key + } + Write-GuardLog -LogPath $LogPath -Message ("run {0} ok={1}" -f $key, $ok) + $actions.Add([pscustomobject]@{ action = 'run-task'; target = [string]$taskDef.taskName; applied = $true; ok = $ok }) | Out-Null + } + else { + $actions.Add([pscustomobject]@{ action = 'run-task'; target = [string]$taskDef.taskName; applied = $false; mode = 'shadow' }) | Out-Null + } + } + } + else { + foreach ($taskDef in $taskDefs) { + if (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected) { + Reset-GuardActionBudget -Runtime $Runtime -Key "task:$($taskDef.taskName)" + } + } + } + + $status = 'ok' + if ($problems.Count -gt 0) { + $status = 'warn' + } + if ($managedInteractiveSessions.Count -gt 0 -and $interactiveStale -and $Mode -eq 'shadow') { + $status = 'warn' + } + + $sessionState = @( + foreach ($session in @($sessionRecords)) { + [pscustomobject]@{ + SessionName = [string]$session.SessionName + UserName = [string]$session.UserName + SessionId = [int]$session.SessionId + State = [string]$session.State + IsLive = [bool]$session.IsLive + } + } + ) + + $state = @{} + $state['status'] = $status + $state['mode'] = $Mode + $state['host'] = $hostname + $state['generatedAtUtc'] = (Get-Date).ToUniversalTime().ToString('o') + $state['pid'] = $PID + $state['sessions'] = @($sessionState) + $state['liveSessionCount'] = $liveSessions.Count + $state['managedSessionCount'] = $managedInteractiveSessions.Count + $state['managedSessions'] = @($managedInteractiveSessions) + $bucketState = @{} + foreach ($key in $bucketChecks.Keys) { + $bucketState[$key] = $bucketChecks[$key] + } + + $state['processes'] = $processSnapshot + $state['buckets'] = $bucketState + $state['actions'] = @($actions.ToArray()) + $state['problems'] = @($problems.ToArray()) + $state['quarantine'] = $Runtime.quarantine + + $statePath = Join-Path $stateRoot 'collector-guard-state.json' + $state | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $statePath -Encoding UTF8 + Write-GuardRuntime -Path $RuntimePath -Runtime $Runtime + [void](Send-GuardHeartbeat -ApiBase $apiBase -Hostname $hostname -State $state -PulseSeconds ([Math]::Max($LoopSeconds * 2, 60))) + return $state +} + +if ($SelfTest) { + Invoke-GuardSelfTest + exit 0 +} + +$initialConfig = Read-ActivityWatchDeploymentConfig -Path $ConfigPath +$initialStateRoot = if ($initialConfig.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$initialConfig.paths.stateRoot } else { Split-Path -Path $ConfigPath -Parent } +$initialLogsRoot = if ($initialConfig.paths.PSObject.Properties.Name -contains 'logsRoot') { [string]$initialConfig.paths.logsRoot } else { Join-Path $initialStateRoot 'logs' } +$logPath = Join-Path $initialLogsRoot 'collector-guard.log' +$runtimePath = Join-Path $initialStateRoot 'collector-guard-runtime.json' +$lockPath = New-GuardLock -StateRoot $initialStateRoot +if (-not $lockPath) { + Write-GuardLog -LogPath $logPath -Message 'another collector guard instance is already running' + exit 0 +} + +try { + $runtime = Read-GuardRuntime -Path $runtimePath + Write-GuardLog -LogPath $logPath -Message "collector guard started mode=$Mode loop=$LoopSeconds once=$($Once.IsPresent)" + while ($true) { + try { + Invoke-GuardCycle -Runtime $runtime -RuntimePath $runtimePath -LogPath $logPath | Out-Null + } + catch { + Write-GuardLog -LogPath $logPath -Message ("cycle error: {0}; at {1}" -f $_.Exception.Message, $_.ScriptStackTrace) + } + + if ($Once) { + break + } + Start-Sleep -Seconds ([Math]::Max($LoopSeconds, 15)) + } +} +finally { + if ($lockPath -and (Test-Path -LiteralPath $lockPath)) { + Remove-Item -LiteralPath $lockPath -Force -ErrorAction SilentlyContinue + } + Write-GuardLog -LogPath $logPath -Message 'collector guard stopped' +} diff --git a/install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 b/install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 index c8b66f7..4370b43 100755 --- a/install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 +++ b/install-kit-awindows-20260427-211240/windows/deploy-domain-users.ps1 @@ -27,7 +27,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath, diff --git a/install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 b/install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 index cbcf689..e21413a 100644 --- a/install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 +++ b/install-kit-awindows-20260427-211240/windows/deploy-ensemble.ps1 @@ -27,7 +27,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath, diff --git a/install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 b/install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 index 25b7221..eb6c739 100755 --- a/install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 +++ b/install-kit-awindows-20260427-211240/windows/deploy-single-user.ps1 @@ -25,7 +25,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath diff --git a/install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 b/install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 index 1350713..8fe1831 100644 --- a/install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 +++ b/install-kit-awindows-20260427-211240/windows/dlp-endpoint-signals-collector.ps1 @@ -15,7 +15,8 @@ param( [int]$PolicyRefreshSeconds, [string]$PolicyCachePath, [string]$LogPath, - [int]$PollSeconds + [int]$PollSeconds, + [switch]$SelfTestSuppressedBlock ) Set-StrictMode -Version Latest @@ -30,6 +31,7 @@ catch { $script:TransportQueuePath = $null $script:TransportQueueLockPath = $null +$script:SessionId = [System.Diagnostics.Process]::GetCurrentProcess().SessionId $script:TransportMetrics = @{ eventsEnqueued = 0 eventsFlushed = 0 @@ -71,6 +73,21 @@ function Write-EndpointLog { } } +function Get-QueueNameToken { + param( + [string]$UserName, + [int]$SessionId + ) + $token = ('{0}-s{1}' -f $UserName, $SessionId) + foreach ($ch in [System.IO.Path]::GetInvalidFileNameChars()) { + $token = $token.Replace([string]$ch, '_') + } + if ([string]::IsNullOrWhiteSpace($token)) { + return ('session-{0}' -f $SessionId) + } + return $token +} + function Invoke-AwJsonPost { param( [Parameter(Mandatory = $true)][string]$Uri, @@ -129,11 +146,20 @@ function Invoke-AwJsonPost { function Initialize-TransportQueue { param([Parameter(Mandatory = $true)][string]$StateRoot) - $script:TransportQueuePath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.jsonl' - $script:TransportQueueLockPath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.lock' + $queueToken = Get-QueueNameToken -UserName $env:USERNAME -SessionId $script:SessionId + $script:TransportQueuePath = Join-Path $StateRoot ("dlp-endpoint-signals-queue-{0}.jsonl" -f $queueToken) + $script:TransportQueueLockPath = Join-Path $StateRoot ("dlp-endpoint-signals-queue-{0}.lock" -f $queueToken) if (-not (Test-Path -LiteralPath $script:TransportQueuePath)) { New-Item -Path $script:TransportQueuePath -ItemType File -Force | Out-Null } + $legacyQueuePath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.jsonl' + if (Test-Path -LiteralPath $legacyQueuePath) { + $legacyItems = @(Get-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($legacyItems.Count -gt 0) { + Add-Content -LiteralPath $script:TransportQueuePath -Value $legacyItems -Encoding UTF8 + Clear-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue + } + } } function Get-TransportQueueLock { @@ -617,6 +643,17 @@ function Load-DlpPolicy { regexPack = $null ocrEnabled = $false } + nativeControls = [ordered]@{ + mode = 'monitor' + rollout = [ordered]@{ + allowGlobalBlock = $false + } + channels = [ordered]@{ + clipboard = [ordered]@{ action = 'audit' } + usb = [ordered]@{ action = 'audit' } + print = [ordered]@{ action = 'audit' } + } + } } $script:PolicySource = 'defaults' @@ -656,6 +693,28 @@ function Load-DlpPolicy { $script:Policy.contentAnalysis.ocrEnabled = [bool]$raw.contentAnalysis.ocrEnabled } } + if ($raw.nativeControls) { + $nativeProps = @($raw.nativeControls.PSObject.Properties.Name) + if ($nativeProps -contains 'mode' -and $raw.nativeControls.mode) { + $script:Policy.nativeControls.mode = ([string]$raw.nativeControls.mode).ToLowerInvariant() + } + if ($nativeProps -contains 'rollout' -and $raw.nativeControls.rollout) { + $rolloutProps = @($raw.nativeControls.rollout.PSObject.Properties.Name) + if ($rolloutProps -contains 'allowGlobalBlock') { + $script:Policy.nativeControls.rollout.allowGlobalBlock = [bool]$raw.nativeControls.rollout.allowGlobalBlock + } + } + if ($nativeProps -contains 'channels' -and $raw.nativeControls.channels) { + foreach ($channelName in @('clipboard', 'usb', 'print')) { + if (@($raw.nativeControls.channels.PSObject.Properties.Name) -contains $channelName) { + $channel = $raw.nativeControls.channels.$channelName + if ($channel -and (@($channel.PSObject.Properties.Name) -contains 'action') -and $channel.action) { + $script:Policy.nativeControls.channels[$channelName].action = ([string]$channel.action).ToLowerInvariant() + } + } + } + } + } $script:PolicySource = 'local' } catch { @@ -663,6 +722,68 @@ function Load-DlpPolicy { } } +function Resolve-DlpEffectiveAction { + param( + [Parameter(Mandatory = $true)][string]$RequestedAction, + [Parameter(Mandatory = $true)][ValidateSet('clipboard', 'usb', 'print')][string]$Channel + ) + + $requested = $RequestedAction.ToLowerInvariant() + $mode = ([string]$script:Policy.nativeControls.mode).ToLowerInvariant() + $allowGlobalBlock = [bool]$script:Policy.nativeControls.rollout.allowGlobalBlock + $channelAction = 'audit' + try { + $channelAction = ([string]$script:Policy.nativeControls.channels[$Channel].action).ToLowerInvariant() + } + catch { + $channelAction = 'audit' + } + + $suppressed = $false + $effective = $requested + if ($requested -eq 'block') { + $channelAllowsBlock = $channelAction -in @('block', 'blockwithoverride') + if ($mode -ne 'enforce' -or -not $allowGlobalBlock -or -not $channelAllowsBlock) { + $effective = 'alert' + $suppressed = $true + } + } + + return [pscustomobject]@{ + requestedAction = $requested + action = $effective + enforcementMode = $mode + nativeChannelAction = $channelAction + enforcementSuppressed = $suppressed + } +} + +function Invoke-SuppressedBlockSelfTest { + $decisions = @() + foreach ($channel in @('clipboard', 'usb', 'print')) { + $decisions += (Resolve-DlpEffectiveAction -RequestedAction 'block' -Channel $channel) + } + + $failed = @( + $decisions | + Where-Object { $_.action -eq 'block' -or -not [bool]$_.enforcementSuppressed } + ) + + $result = [ordered]@{ + ok = (@($failed).Count -eq 0) + test = 'suppressed-block-in-monitor' + policySource = $script:PolicySource + policyMode = $script:PolicyMode + decisions = @($decisions) + } + + $result | ConvertTo-Json -Depth 6 + if (-not $result.ok) { + exit 2 + } + exit 0 +} + function Test-ValidInn { param([string]$Value) $digits = ($Value -replace '\D', '') @@ -925,7 +1046,9 @@ function Evaluate-ClipboardRules { $fingerprint = "clipboard|$ruleId|$ClipboardHash|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'clipboard' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "Clipboard rule matched: $ruleId" } @@ -939,13 +1062,17 @@ function Evaluate-ClipboardRules { clipboardHash = $ClipboardHash clipboardLength = $ClipboardText.Length enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed dictionaryPack = $dictionaryPack regexPack = $regexPack dictionaryMatches = @($advanced.dictionaryMatches) regexMatches = @($advanced.regexMatches) ocrRequested = $ocrEnabled } - Write-EndpointLog ("incident clipboard rule={0} action={1} severity={2} enforced={3}" -f $ruleId, $action, $severity, $enforced) + Write-EndpointLog ("incident clipboard rule={0} requested={1} action={2} severity={3} enforced={4} suppressed={5}" -f $ruleId, $requestedAction, $action, $severity, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -965,7 +1092,9 @@ function Evaluate-UsbRules { $fingerprint = "usb|$ruleId|$DriveLetter|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'usb' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "USB rule matched: $ruleId" } @@ -979,8 +1108,12 @@ function Evaluate-UsbRules { driveLetter = $DriveLetter volumeName = $VolumeName enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed } - Write-EndpointLog ("incident usb rule={0} action={1} severity={2} drive={3} enforced={4}" -f $ruleId, $action, $severity, $DriveLetter, $enforced) + Write-EndpointLog ("incident usb rule={0} requested={1} action={2} severity={3} drive={4} enforced={5} suppressed={6}" -f $ruleId, $requestedAction, $action, $severity, $DriveLetter, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -1016,7 +1149,9 @@ function Evaluate-PrintRules { $fingerprint = "print|$ruleId|$PrinterName|$Owner|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'print' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "Print rule matched: $ruleId" } @@ -1031,13 +1166,17 @@ function Evaluate-PrintRules { documentName = $DocumentName owner = $Owner enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed dictionaryPack = $dictionaryPack regexPack = $regexPack dictionaryMatches = @($advanced.dictionaryMatches) regexMatches = @($advanced.regexMatches) ocrRequested = $ocrEnabled } - Write-EndpointLog ("incident print rule={0} action={1} severity={2} printer={3} enforced={4}" -f $ruleId, $action, $severity, $PrinterName, $enforced) + Write-EndpointLog ("incident print rule={0} requested={1} action={2} severity={3} printer={4} enforced={5} suppressed={6}" -f $ruleId, $requestedAction, $action, $severity, $PrinterName, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -1320,6 +1459,9 @@ $script:LastEventTime = $null Initialize-TransportQueue -StateRoot $resolvedStateRoot Initialize-DlpPolicy +if ($SelfTestSuppressedBlock) { + Invoke-SuppressedBlockSelfTest +} Write-EndpointLog ("endpoint collector started against {0}" -f $script:ApiBase) while ($true) { diff --git a/install-kit-awindows-20260427-211240/windows/dlp-policy.example.json b/install-kit-awindows-20260427-211240/windows/dlp-policy.example.json index 7c8c40f..828f234 100644 --- a/install-kit-awindows-20260427-211240/windows/dlp-policy.example.json +++ b/install-kit-awindows-20260427-211240/windows/dlp-policy.example.json @@ -6,6 +6,19 @@ "action": "log", "severity": "low" }, + "nativeControls": { + "mode": "monitor", + "rollout": { + "baselineDays": 7, + "requireGuardHeartbeat": true, + "allowGlobalBlock": false + }, + "channels": { + "clipboard": {"action": "audit"}, + "usb": {"action": "audit"}, + "print": {"action": "audit"} + } + }, "rules": [ { "id": "personal-web-during-workhours", diff --git a/install-kit-awindows-20260427-211240/windows/dlp-policy.native-cross-os.example.json b/install-kit-awindows-20260427-211240/windows/dlp-policy.native-cross-os.example.json new file mode 100644 index 0000000..b8710b0 --- /dev/null +++ b/install-kit-awindows-20260427-211240/windows/dlp-policy.native-cross-os.example.json @@ -0,0 +1,145 @@ +{ + "version": 1, + "defaults": { + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium" + }, + "nativeControls": { + "mode": "monitor", + "rollout": { + "baselineDays": 7, + "requireGuardHeartbeat": true, + "allowGlobalBlock": false + }, + "channels": { + "removableStorage": { + "action": "audit", + "windows": { + "mechanisms": ["gpo-device-restrictions", "set-disk-readonly"], + "target": "write" + }, + "linux": { + "mechanisms": ["fanotify", "auditd"], + "target": "mountpoints" + }, + "macos": { + "mechanisms": ["mdm-restrictions", "endpoint-security"], + "target": "managed-devices" + }, + "chromeos": { + "mechanisms": ["data-controls"], + "target": "removable-storage" + } + }, + "print": { + "action": "audit", + "windows": { + "mechanisms": ["printservice-operational-log", "spooler-cancel-job"] + }, + "linux": { + "mechanisms": ["cups-logs"], + "enforcement": "monitor-only" + }, + "macos": { + "mechanisms": ["mdm-printing-restrictions"], + "enforcement": "managed-only" + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "clipboard": { + "action": "audit", + "windows": { + "mechanisms": ["clipboard-monitor", "clear-clipboard"], + "blockScope": "high-confidence-only" + }, + "linux": { + "mechanisms": ["desktop-clipboard-monitor"], + "enforcement": "monitor-only" + }, + "macos": { + "mechanisms": ["endpoint-monitor", "mdm-restrictions"], + "enforcement": "monitor-first" + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "browserUpload": { + "action": "audit", + "windows": { + "mechanisms": ["managed-browser-policy", "browser-extension"] + }, + "linux": { + "mechanisms": ["managed-browser-policy", "proxy-logs"] + }, + "macos": { + "mechanisms": ["managed-browser-policy", "network-extension"] + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "appExecution": { + "action": "audit", + "windows": { + "mechanisms": ["applocker", "wdac"] + }, + "linux": { + "mechanisms": ["auditd", "fanotify", "bpf-lsm"], + "enforcement": "fanotify-or-lsm-only" + }, + "macos": { + "mechanisms": ["mdm-restrictions", "endpoint-security"] + }, + "chromeos": { + "mechanisms": ["admin-console-app-policy"] + } + } + } + }, + "endpoint": { + "clipboard": [ + { + "id": "clipboard-sensitive-keywords", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "high", + "message": "В буфере обнаружены чувствительные ключевые слова", + "minLength": 20, + "regexPatterns": [ + "(?i)парол(ь|и)", + "(?i)password", + "(?i)secret", + "(?i)cvv", + "(?i)паспорт" + ] + } + ], + "usb": [ + { + "id": "usb-media-connected", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium", + "message": "Подключен съемный носитель" + } + ], + "print": [ + { + "id": "print-sensitive-docs", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "high", + "message": "Печать документа с признаками чувствительных данных", + "documentRegex": "(?i)(salary|зарплат|passport|паспорт|договор|contract)" + } + ] + } +} diff --git a/install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 b/install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 index b28b76c..fb4b6ef 100755 --- a/install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 +++ b/install-kit-awindows-20260427-211240/windows/hardening-recovery.ps1 @@ -94,7 +94,7 @@ $effectiveEvtxExportRoot = if ($PSBoundParameters.ContainsKey('EvtxExportRoot') $effectiveEvtxRetentionDays = if ($PSBoundParameters.ContainsKey('EvtxRetentionDays')) { [int]$EvtxRetentionDays } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'retentionDays') { [int]$existingConfig.forensics.retentionDays } else { 14 } $effectiveEvtxChannels = if ($PSBoundParameters.ContainsKey('EvtxChannels')) { @($EvtxChannels) } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'evtxChannels') { @($existingConfig.forensics.evtxChannels) } else { @() } $effectiveLogonMarkerEnabled = if ($PSBoundParameters.ContainsKey('LogonMarkerEnabled')) { [bool]$LogonMarkerEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'logonEnabled') { [bool]$existingConfig.sessionEvents.logonEnabled } else { $true } -$effectiveProcessEventsEnabled = if ($PSBoundParameters.ContainsKey('ProcessEventsEnabled')) { [bool]$ProcessEventsEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$existingConfig.sessionEvents.processEventsEnabled } else { $true } +$effectiveProcessEventsEnabled = if ($PSBoundParameters.ContainsKey('ProcessEventsEnabled')) { [bool]$ProcessEventsEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$existingConfig.sessionEvents.processEventsEnabled } else { $false } $effectiveAwHostname = if ($PSBoundParameters.ContainsKey('AwHostname') -and -not [string]::IsNullOrWhiteSpace($AwHostname)) { [string]$AwHostname } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]$existingConfig.awHostname)) { [string]$existingConfig.awHostname } else { [string]$env:COMPUTERNAME } $effectiveVersion = if ($Version) { $Version } elseif ($existingConfig) { [string]$existingConfig.package.version } else { 'v0.13.2' } $effectivePolicyMode = if ($PSBoundParameters.ContainsKey('PolicyMode') -and $PolicyMode) { [string]$PolicyMode } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'policyEngine' -and $existingConfig.policyEngine.PSObject.Properties.Name -contains 'mode') { [string]$existingConfig.policyEngine.mode } else { 'local' } diff --git a/install-kit-awindows-20260427-211240/windows/install-collector-guard-service.ps1 b/install-kit-awindows-20260427-211240/windows/install-collector-guard-service.ps1 new file mode 100644 index 0000000..a6e270c --- /dev/null +++ b/install-kit-awindows-20260427-211240/windows/install-collector-guard-service.ps1 @@ -0,0 +1,88 @@ +[CmdletBinding()] +param( + [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json', + [ValidateSet('shadow', 'enforce')] + [string]$Mode = 'shadow', + [string]$ServiceName = 'AWatchRusCollectorGuard', + [int]$LoopSeconds = 60, + [switch]$DisableRecoveryTask +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Assert-Admin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = [Security.Principal.WindowsPrincipal]::new($id) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Run as Administrator.' + } +} + +Assert-Admin + +$guardScriptPath = Join-Path $PSScriptRoot 'aw-collector-guard.ps1' +$serviceSourcePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.cs' +$serviceExePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.exe' +if (-not (Test-Path -LiteralPath $guardScriptPath)) { + throw "Collector guard script not found: $guardScriptPath" +} +if (-not (Test-Path -LiteralPath $serviceSourcePath)) { + throw "Collector guard service source not found: $serviceSourcePath" +} +if (-not (Test-Path -LiteralPath $ConfigPath)) { + throw "Config not found: $ConfigPath" +} + +$cscCandidates = @( + (Join-Path $env:WINDIR 'Microsoft.NET\Framework64\v4.0.30319\csc.exe'), + (Join-Path $env:WINDIR 'Microsoft.NET\Framework\v4.0.30319\csc.exe') +) +$csc = @($cscCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1) +if (-not $csc) { + throw 'C# compiler not found. Install .NET Framework build tools or provide AWatchRusCollectorGuardService.exe.' +} + +$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue +if ($existing) { + if ($existing.Status -ne 'Stopped') { + Stop-Service -Name $ServiceName -Force -ErrorAction SilentlyContinue + try { + $existing.WaitForStatus('Stopped', [TimeSpan]::FromSeconds(20)) + } + catch { + } + } + sc.exe delete $ServiceName | Out-Null + Start-Sleep -Seconds 2 +} + +& $csc /nologo /target:exe /optimize+ /out:$serviceExePath /reference:System.ServiceProcess.dll $serviceSourcePath | Out-Null +if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $serviceExePath)) { + throw "Failed to compile $serviceExePath" +} + +$logsRoot = Join-Path (Split-Path -Path $ConfigPath -Parent) 'logs' +$serviceLogPath = Join-Path $logsRoot 'collector-guard-service.log' +$binPath = "`"$serviceExePath`" --service-name `"$ServiceName`" --script `"$guardScriptPath`" --config `"$ConfigPath`" --mode $Mode --loop $LoopSeconds --log `"$serviceLogPath`"" + +New-Service -Name $ServiceName -BinaryPathName $binPath -DisplayName 'AWatch-rus Collector Guard' -StartupType Automatic | Out-Null +sc.exe description $ServiceName "Session-aware ActivityWatch collector guard for AWatch-rus" | Out-Null +sc.exe failure $ServiceName reset= 300 actions= restart/5000/restart/15000/restart/60000 | Out-Null + +if ($DisableRecoveryTask) { + Write-Warning 'DisableRecoveryTask is deprecated and ignored: ActivityWatch Recovery must remain enabled as collector guard fallback.' +} +else { + try { + Enable-ScheduledTask -TaskName 'ActivityWatch Recovery' -ErrorAction SilentlyContinue | Out-Null + } + catch { + } +} + +sc.exe start $ServiceName | Out-Null + +Write-Output "Collector guard service installed: $ServiceName" +Write-Output "Mode: $Mode" +Write-Output "Config: $ConfigPath" diff --git a/install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 b/install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 index 237bf69..bdfcb61 100644 --- a/install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 +++ b/install-kit-awindows-20260427-211240/windows/validate-deployment.ps1 @@ -1,4 +1,4 @@ -[CmdletBinding()] +[CmdletBinding()] param( [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json' ) @@ -38,7 +38,7 @@ $windowExpected = if ($config.PSObject.Properties.Name -contains 'collectors' -a $fileOpsExpected = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'fileOpsEnabled') { [bool]$config.collectors.fileOpsEnabled } else { $true } $sessionEventsConfig = if ($config.PSObject.Properties.Name -contains 'sessionEvents') { $config.sessionEvents } else { $null } $sessionLogonEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'logonEnabled') { [bool]$sessionEventsConfig.logonEnabled } else { $false } -$sessionProcessEventsEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$sessionEventsConfig.processEventsEnabled } else { $true } +$sessionProcessEventsEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$sessionEventsConfig.processEventsEnabled } else { $false } $sessionEventsBucketId = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'bucketPrefix' -and -not [string]::IsNullOrWhiteSpace([string]$sessionEventsConfig.bucketPrefix)) { ('{0}_{1}' -f [string]$sessionEventsConfig.bucketPrefix, $awHostname) } @@ -301,6 +301,94 @@ function Get-TransportQueueHealth { } } +function Get-TransportQueueGroupHealth { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [Parameter(Mandatory = $true)] + [string]$StateRoot, + [Parameter(Mandatory = $true)] + [string]$QueuePattern, + [Parameter(Mandatory = $true)] + [int]$StaleAfterSeconds, + [Parameter(Mandatory = $true)] + [int]$MaxDepth, + [int]$ActiveProcessCount = 0, + [bool]$Required = $true + ) + + $queues = @(Get-ChildItem -LiteralPath $StateRoot -Filter $QueuePattern -ErrorAction SilentlyContinue | Sort-Object Name) + if ($queues.Count -eq 0) { + return [pscustomobject]@{ + name = $Name + required = [bool]$Required + queuePattern = $QueuePattern + queueCount = 0 + queues = @() + depth = 0 + sizeBytes = 0 + activeProcessCount = [int]$ActiveProcessCount + staleAfterSeconds = [int]$StaleAfterSeconds + maxDepth = [int]$MaxDepth + ok = [bool](-not $Required) + } + } + + $items = @() + foreach ($queue in $queues) { + $lockPath = [System.IO.Path]::ChangeExtension($queue.FullName, '.lock') + $items += Get-TransportQueueHealth -Name $queue.BaseName -QueuePath $queue.FullName -LockPath $lockPath -StaleAfterSeconds $StaleAfterSeconds -MaxDepth $MaxDepth -ActiveProcessCount $ActiveProcessCount -Required $Required + } + + return [pscustomobject]@{ + name = $Name + required = [bool]$Required + queuePattern = $QueuePattern + queueCount = [int]$items.Count + queues = @($items) + depth = [int](($items | Measure-Object -Property depth -Sum).Sum) + sizeBytes = [int64](($items | Measure-Object -Property sizeBytes -Sum).Sum) + activeProcessCount = [int]$ActiveProcessCount + staleAfterSeconds = [int]$StaleAfterSeconds + maxDepth = [int]$MaxDepth + ok = [bool](-not ($items | Where-Object { -not $_.ok })) + } +} + +function Resolve-ActivityWatchLaunchTaskName { + param( + [Parameter(Mandatory = $true)] + [string]$TaskName + ) + + if ($TaskName -notmatch '\[[^\]]+_Administrator\]') { + return $TaskName + } + + $localizedCandidate = 'ActivityWatch Launch [{0}_Администратор]' -f $awHostname + $localizedTask = Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object { $_.TaskName -eq $localizedCandidate } | Select-Object -First 1 + if ($localizedTask) { + return $localizedCandidate + } + + try { + $builtinAdmin = Get-LocalUser -ErrorAction Stop | + Where-Object { [string]$_.SID -match '-500$' } | + Select-Object -First 1 + if ($builtinAdmin -and -not [string]::IsNullOrWhiteSpace([string]$builtinAdmin.Name)) { + $candidate = 'ActivityWatch Launch [{0}_{1}]' -f $awHostname, [string]$builtinAdmin.Name + $existing = Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object { $_.TaskName -eq $candidate } | Select-Object -First 1 + if ($existing) { + return $candidate + } + } + } + catch { + } + + return $TaskName +} + function Get-TaskSnapshot { param( [Parameter(Mandatory = $true)] @@ -337,14 +425,13 @@ function Get-TaskSnapshot { } catch { } - [pscustomobject]@{ taskName = [string]$task.TaskName present = $true enabled = [bool]$enabled state = [string]$task.State lastResult = if ($taskInfo) { [int64]$taskInfo.LastTaskResult } else { $null } - ok = [bool]($enabled) + ok = [bool]$enabled } } ) @@ -413,10 +500,14 @@ else { 0 } $sessionScopedCollectorsRequired = ($sessionScopedExpectedCount -gt 0) +$liveSessionScopedCollectorsRequired = ($liveSessionBoundUsers.Count -gt 0) + +$collectorGuardService = Get-Service -Name 'AWatchRusCollectorGuard' -ErrorAction SilentlyContinue +$collectorGuardActive = [bool]($collectorGuardService -and $collectorGuardService.Status -eq 'Running') $taskNames = @() if ($config.userTasks) { - $taskNames += @($config.userTasks | ForEach-Object { [string]$_.launchTaskName }) + $taskNames += @($config.userTasks | ForEach-Object { Resolve-ActivityWatchLaunchTaskName -TaskName ([string]$_.launchTaskName) }) } $taskNames += [string]$config.recovery.taskName $tasks = @(Get-TaskSnapshot -TaskNames $taskNames) @@ -438,24 +529,24 @@ foreach ($watcher in $runningWatchers) { $bucketChecks = @( Get-BucketHealth -BucketId ('aw-worktime-sessions_' + $awHostname) -MaxAgeSeconds $sessionFreshnessSeconds -Required $true -RequireFreshEvent $true ) -if ($sessionScopedCollectorsRequired -and $afkExpected) { +if ($liveSessionScopedCollectorsRequired -and $afkExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-watcher-afk_' + $awHostname) -MaxAgeSeconds $freshnessSeconds -Required $true -RequireFreshEvent $false } -if ($sessionScopedCollectorsRequired -and $windowExpected) { +if ($liveSessionScopedCollectorsRequired -and $windowExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-watcher-window_' + $awHostname) -MaxAgeSeconds $freshnessSeconds -Required $true -RequireFreshEvent $false } -if ($sessionScopedCollectorsRequired) { +if ($liveSessionScopedCollectorsRequired) { $bucketChecks += Get-BucketHealth -BucketId ('aw-dlp-endpoint-signals_' + $awHostname) -MaxAgeSeconds $endpointFreshnessSeconds -Required $true -RequireFreshEvent $true } -if ($sessionScopedCollectorsRequired -and $fileOpsExpected) { +if ($liveSessionScopedCollectorsRequired -and $fileOpsExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-file-operations_' + $awHostname) -MaxAgeSeconds $transportStaleSeconds -Required $false -RequireFreshEvent $true } $queueChecks = @( - Get-TransportQueueHealth -Name 'endpoint' -QueuePath (Join-Path $stateRoot 'dlp-endpoint-signals-queue.jsonl') -LockPath (Join-Path $stateRoot 'dlp-endpoint-signals-queue.lock') -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($endpointCollectorProcesses).Count -Required $sessionScopedCollectorsRequired + Get-TransportQueueGroupHealth -Name 'endpoint' -StateRoot $stateRoot -QueuePattern 'dlp-endpoint-signals-queue*.jsonl' -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($endpointCollectorProcesses).Count -Required $liveSessionScopedCollectorsRequired ) if ($fileOpsExpected) { - $queueChecks += Get-TransportQueueHealth -Name 'fileops' -QueuePath (Join-Path $stateRoot 'file-operations-queue.jsonl') -LockPath (Join-Path $stateRoot 'file-operations-queue.lock') -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($fileCollectorProcesses).Count -Required $sessionScopedCollectorsRequired + $queueChecks += Get-TransportQueueGroupHealth -Name 'fileops' -StateRoot $stateRoot -QueuePattern 'file-operations-queue*.jsonl' -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($fileCollectorProcesses).Count -Required $liveSessionScopedCollectorsRequired } $printServiceOperationalEnabled = $false @@ -502,12 +593,14 @@ $result = [ordered]@{ } tasks = [ordered]@{ list = $tasks - ok = [bool]($tasks.Count -gt 0 -and -not ($tasks | Where-Object { -not $_.present -or -not $_.enabled })) + ok = [bool]($tasks.Count -gt 0 -and -not ($tasks | Where-Object { -not $_.ok })) } processes = [ordered]@{ liveSessionBoundUsers = $liveSessionBoundUsers sessionBoundUsers = $interactiveSessionBoundUsers sessionScopedExpectedCount = [int]$sessionScopedExpectedCount + liveSessionScopedCollectorsRequired = [bool]$liveSessionScopedCollectorsRequired + collectorGuardServiceActive = [bool]$collectorGuardActive watchers = @($runningWatchers) watcherDuplicates = @($watcherDuplicates) sessionCollectors = @($sessionCollectorProcesses) diff --git a/ops/detmir-ai/ai-tool b/ops/detmir-ai/ai-tool new file mode 100644 index 0000000..5d084bb --- /dev/null +++ b/ops/detmir-ai/ai-tool @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +cmd="$(basename "$0")" +polli_chat="${POLLI_CHAT:-/usr/local/bin/polli-chat}" + +need_text() { + if [ "$#" -lt 1 ]; then + echo "Нужен текст запроса. Пример: $cmd \"объясни ошибку\"" >&2 + exit 1 + fi +} + +key() { + python3 - <<'PY' +import json +from pathlib import Path +print(json.loads(Path.home().joinpath(".pollinations/credentials.json").read_text())["apiKey"]) +PY +} + +urlencode() { + python3 - "$*" <<'PY' +import sys, urllib.parse +print(urllib.parse.quote(sys.argv[1])) +PY +} + +case "$cmd" in + ai) + need_text "$@" + exec "$polli_chat" --model text.daily "$*" + ;; + ai-fast) + need_text "$@" + exec "$polli_chat" --model text.cheap "$*" + ;; + ai-smart) + need_text "$@" + exec "$polli_chat" --model text.heavy "$*" + ;; + ai-code) + need_text "$@" + exec "$polli_chat" --model text.coding_primary "$*" + ;; + ai-search) + need_text "$@" + exec "$polli_chat" --model text.search_reasoning "$*" + ;; + ai-report) + if [ "$#" -lt 1 ]; then + echo "Пример: ai-report /tmp/check-output.txt" >&2 + exit 1 + fi + file="$1" + [ -f "$file" ] || { echo "Файл не найден: $file" >&2; exit 1; } + { + echo "Сделай короткий рабочий отчет: состояние, что важно, следующие действия." + echo + sed -n '1,1400p' "$file" + } | "$polli_chat" --model text.daily + ;; + ai-models) + k="$(key)" + curl -fsSL 'https://gen.pollinations.ai/v1/models' \ + -H 'User-Agent: curl/8.5 detmir-proxmox' \ + -H "Authorization: Bearer ${k}" | + python3 -c 'import json,sys; data=json.load(sys.stdin); [print(item.get("id") or item.get("name") or item.get("model")) for item in data.get("data", []) if isinstance(item, dict)]' + ;; + ai-test) + echo "1) text:" + "$polli_chat" --model text.cheap --max-tokens 20 'Return exactly: text-ok' + echo + echo "2) embedding:" + k="$(key)" + curl -fsSL 'https://gen.pollinations.ai/v1/embeddings' \ + -H "Authorization: Bearer ${k}" \ + -H 'User-Agent: curl/8.5 detmir-proxmox' \ + -H 'Content-Type: application/json' \ + -d '{"model":"openai-3-small","input":"embedding test","dimensions":128}' | + python3 -c 'import json,sys; data=json.load(sys.stdin); emb=data["data"][0]["embedding"]; print("embedding-ok len=" + str(len(emb)))' + ;; + ai-image) + need_text "$@" + prompt="$1" + out="${2:-$HOME/ai-image-$(date +%Y%m%d-%H%M%S).jpg}" + model="${3:-zimage}" + encoded="$(urlencode "$prompt")" + curl -fsSL "https://gen.pollinations.ai/image/${encoded}?model=${model}&width=1024&height=1024" \ + -H 'User-Agent: curl/8.5 detmir-proxmox' \ + -H "Authorization: Bearer $(key)" \ + -o "$out" + echo "$out" + ;; + ai-voice) + need_text "$@" + text="$1" + out="${2:-$HOME/ai-voice-$(date +%Y%m%d-%H%M%S).mp3}" + voice="${3:-nova}" + encoded="$(urlencode "$text")" + curl -fsSL "https://gen.pollinations.ai/audio/${encoded}?voice=${voice}" \ + -H 'User-Agent: curl/8.5 detmir-proxmox' \ + -H "Authorization: Bearer $(key)" \ + -o "$out" + echo "$out" + ;; + *) + echo "Unknown AI tool name: $cmd" >&2 + exit 2 + ;; +esac diff --git a/ops/detmir-ai/detmir-ai b/ops/detmir-ai/detmir-ai new file mode 100644 index 0000000..ce374d7 --- /dev/null +++ b/ops/detmir-ai/detmir-ai @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +out_dir="${DETMIR_AI_OUT_DIR:-/var/tmp/detmir-ai-${USER:-$(id -un)}}" +mkdir -p "$out_dir" +stamp="$(date -u +%Y%m%d-%H%M%S)" +check_file="$out_dir/detmir-check-$stamp.json" +dlp_file="$out_dir/detmir-dlp-$stamp.json" +bundle_file="$out_dir/detmir-ai-bundle-$stamp.txt" + +check_rc=0 +dlp_rc=0 + +detmir-check --json >"$check_file" || check_rc=$? +detmir-dlp >"$dlp_file" || dlp_rc=$? + +{ + echo "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет." + echo "Структура ответа:" + echo "1. Состояние: OK/WARN/FAIL" + echo "2. Что важно" + echo "3. Что сделать дальше" + echo + echo "Правила:" + echo "- Не предлагай рестарты, если факты чистые." + echo "- Отличай event-driven bucket от dead/stale." + echo "- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail." + echo + echo "=== detmir-check exit=$check_rc ===" + sed -n '1,1600p' "$check_file" + echo + echo "=== detmir-dlp exit=$dlp_rc ===" + sed -n '1,1600p' "$dlp_file" +} >"$bundle_file" + +echo "Files:" +echo " $check_file" +echo " $dlp_file" +echo " $bundle_file" +echo + +polli-chat --model text.daily --max-tokens 700 <"$bundle_file" + +if [ "$check_rc" -ne 0 ] || [ "$dlp_rc" -ne 0 ]; then + exit 2 +fi diff --git a/ops/detmir-ai/detmir-ai-profile.sh b/ops/detmir-ai/detmir-ai-profile.sh new file mode 100644 index 0000000..1d9c405 --- /dev/null +++ b/ops/detmir-ai/detmir-ai-profile.sh @@ -0,0 +1,8 @@ +# DetMir autonomous AI helpers. Installed on Proxmox host. +# Run `ai-test`, `detmir-check`, `detmir-dlp`, or `detmir-ai`. + +export POLLI_CHAT=/usr/local/bin/polli-chat + +alias detmir-report='detmir-ai' +alias detmir-auto-report='detmir-status' +alias detmir-models='ai-models' diff --git a/ops/detmir-ai/detmir-auto b/ops/detmir-ai/detmir-auto new file mode 100644 index 0000000..811eb2f --- /dev/null +++ b/ops/detmir-ai/detmir-auto @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +state_dir="${DETMIR_AI_STATE_DIR:-/var/lib/detmir-ai}" +lock_dir="${DETMIR_AI_RUN_DIR:-${XDG_RUNTIME_DIR:-/tmp}}" +auto_heal="${DETMIR_AUTO_HEAL:-1}" +retain_days="${DETMIR_AI_RETAIN_DAYS:-14}" + +mkdir -p "$state_dir"/runs "$state_dir"/reports "$state_dir"/logs + +lock_file="$lock_dir/detmir-auto.lock" +exec 9>"$lock_file" +if ! flock -n 9; then + echo "detmir-auto: another run is active" + exit 0 +fi + +stamp="$(date -u +%Y%m%d-%H%M%S)" +run_dir_full="$state_dir/runs/$stamp" +mkdir -p "$run_dir_full" + +check_file="$run_dir_full/detmir-check.json" +dlp_file="$run_dir_full/detmir-dlp.json" +heal_log="$run_dir_full/heal.log" +bundle_file="$run_dir_full/bundle.txt" +report_file="$state_dir/reports/detmir-report-$stamp.md" +state_file="$state_dir/state-$stamp.json" + +run_check() { + local rc=0 + detmir-check --json >"$check_file" || rc=$? + echo "$rc" >"$run_dir_full/check.rc" +} + +run_dlp() { + local rc=0 + detmir-dlp >"$dlp_file" || rc=$? + echo "$rc" >"$run_dir_full/dlp.rc" +} + +summarize() { + python3 - "$check_file" "$dlp_file" "$run_dir_full/check.rc" "$run_dir_full/dlp.rc" <<'PY' +import json +import sys +from pathlib import Path + +check_path, dlp_path, check_rc_path, dlp_rc_path = map(Path, sys.argv[1:]) +check_rc = int(check_rc_path.read_text().strip()) +dlp_rc = int(dlp_rc_path.read_text().strip()) +summary = { + "check_rc": check_rc, + "dlp_rc": dlp_rc, + "check_ok": False, + "dlp_ok": False, + "severity": "FAIL" if check_rc or dlp_rc else "OK", + "needs_heal": bool(check_rc or dlp_rc), + "reasons": [], +} +try: + check = json.loads(check_path.read_text()) + summary["check_ok"] = bool(check.get("ok")) + cs = check.get("summary") or {} + summary["detmir_summary"] = cs + if cs.get("bucket_dead", 0) or cs.get("bucket_stale", 0) or cs.get("service_failures", 0): + summary["reasons"].append("detmir-check has stale/dead bucket or required service failure") +except Exception as exc: + summary["reasons"].append(f"detmir-check parse failed: {exc}") + +try: + dlp = json.loads(dlp_path.read_text()) + summary["dlp_ok"] = bool(dlp.get("ok")) + summary["dlp_counts"] = dlp.get("counts") + counts = dlp.get("counts") or {} + if counts.get("fail", 0) or counts.get("warn", 0): + summary["reasons"].append("dlp-health-check has warn/fail") +except Exception as exc: + summary["reasons"].append(f"detmir-dlp parse failed: {exc}") + +if summary["check_ok"] and summary["dlp_ok"]: + summary["severity"] = "OK" + summary["needs_heal"] = False +elif not summary["reasons"]: + summary["severity"] = "WARN" +else: + summary["severity"] = "FAIL" + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +} + +run_check +run_dlp +summary_before="$(summarize)" +printf '%s\n' "$summary_before" >"$run_dir_full/summary-before.json" + +if [ "$auto_heal" = "1" ] && python3 -c 'import json,sys; print("yes" if json.load(sys.stdin).get("needs_heal") else "no")' <<<"$summary_before" | grep -qx yes; then + { + echo "detmir-heal-safe started at $(date -u --iso-8601=seconds)" + detmir-heal-safe + echo "detmir-heal-safe finished at $(date -u --iso-8601=seconds)" + } >"$heal_log" 2>&1 || true + sleep 10 + run_check + run_dlp +else + echo "auto-heal skipped" >"$heal_log" +fi + +summary_after="$(summarize)" +printf '%s\n' "$summary_after" >"$state_file" + +{ + echo "Ты операторский AI-помощник DetMir. По фактам ниже дай короткий русский отчет." + echo "Структура ответа:" + echo "1. Состояние: OK/WARN/FAIL" + echo "2. Что важно" + echo "3. Что уже сделал автомат" + echo "4. Что сделать человеку, если нужно" + echo + echo "Правила:" + echo "- Не предлагай рестарты, если факты чистые." + echo "- Отличай event-driven bucket от dead/stale." + echo "- DLP sendFailures важны только при новом sendFailuresDelta или warn/fail." + echo "- Auto-heal умеет только серверные systemd-сервисы AW/DLP; Windows/RDP не трогает." + echo + echo "=== summary-before ===" + cat "$run_dir_full/summary-before.json" + echo + echo "=== summary-after ===" + cat "$state_file" + echo + echo "=== heal-log ===" + sed -n '1,300p' "$heal_log" + echo + echo "=== detmir-check ===" + sed -n '1,1600p' "$check_file" + echo + echo "=== detmir-dlp ===" + sed -n '1,1600p' "$dlp_file" +} >"$bundle_file" + +{ + echo "# DetMir Autonomous Report" + echo + echo "- generated_at_utc: $(date -u --iso-8601=seconds)" + echo "- run_dir: $run_dir_full" + echo + polli-chat --model text.daily --max-tokens 900 <"$bundle_file" || { + echo "Pollinations report failed; raw summary follows." + cat "$state_file" + } +} >"$report_file" + +ln -sfn "$run_dir_full" "$state_dir/latest-run" +ln -sfn "$report_file" "$state_dir/latest-report.md" +ln -sfn "$state_file" "$state_dir/latest-state.json" + +find "$state_dir/runs" -mindepth 1 -maxdepth 1 -type d -mtime +"$retain_days" -exec rm -rf {} + +find "$state_dir/reports" -type f -name 'detmir-report-*.md' -mtime +"$retain_days" -delete +find "$state_dir" -maxdepth 1 -type f -name 'state-*.json' -mtime +"$retain_days" -delete + +cat "$report_file" + +python3 - "$state_file" <<'PY' +import json +import sys +state = json.load(open(sys.argv[1])) +raise SystemExit(0 if state.get("severity") == "OK" else 2) +PY diff --git a/ops/detmir-ai/detmir-auto-rust-shadow.service b/ops/detmir-ai/detmir-auto-rust-shadow.service new file mode 100644 index 0000000..8f41532 --- /dev/null +++ b/ops/detmir-ai/detmir-auto-rust-shadow.service @@ -0,0 +1,20 @@ +[Unit] +Description=DetMir Rust autonomous shadow health loop +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=igor +Group=igor +ExecCondition=/bin/sh -c '! systemctl -q is-active detmir-auto.service' +Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust +Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust/locks +Environment=no_proxy=localhost,127.0.0.1,192.168.100.18,10.10.10.13,10.10.10.2,10.10.10.0/24,192.168.100.0/24 +Environment=NO_PROXY=localhost,127.0.0.1,192.168.100.18,10.10.10.13,10.10.10.2,10.10.10.0/24,192.168.100.0/24 +ExecStart=/usr/local/bin/detmir-auto-rust --no-heal --no-report --command-timeout-seconds 180 +SuccessExitStatus=2 +TimeoutStartSec=240 +Nice=5 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/ops/detmir-ai/detmir-auto-rust-shadow.timer b/ops/detmir-ai/detmir-auto-rust-shadow.timer new file mode 100644 index 0000000..f7c7e64 --- /dev/null +++ b/ops/detmir-ai/detmir-auto-rust-shadow.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Run DetMir Rust autonomous shadow loop every 15 minutes + +[Timer] +OnBootSec=7min +OnUnitActiveSec=30min +AccuracySec=1min +RandomizedDelaySec=5min +Persistent=true +Unit=detmir-auto-rust-shadow.service + +[Install] +WantedBy=timers.target diff --git a/ops/detmir-ai/detmir-auto.service b/ops/detmir-ai/detmir-auto.service new file mode 100644 index 0000000..6c49e36 --- /dev/null +++ b/ops/detmir-ai/detmir-auto.service @@ -0,0 +1,17 @@ +[Unit] +Description=DetMir autonomous health, AI report, and safe server-side recovery +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=igor +Group=igor +Environment=DETMIR_AUTO_HEAL=1 +Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai +Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/locks +ExecStart=/usr/local/bin/detmir-auto +TimeoutStartSec=300 +Nice=5 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/ops/detmir-ai/detmir-auto.timer b/ops/detmir-ai/detmir-auto.timer new file mode 100644 index 0000000..94b73b0 --- /dev/null +++ b/ops/detmir-ai/detmir-auto.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run DetMir autonomous health loop every 15 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min +AccuracySec=1min +Persistent=true +Unit=detmir-auto.service + +[Install] +WantedBy=timers.target diff --git a/ops/detmir-ai/detmir-check b/ops/detmir-ai/detmir-check new file mode 100644 index 0000000..5ae2749 --- /dev/null +++ b/ops/detmir-ai/detmir-check @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec "${DETMIR_CHECK_BIN:-/usr/local/bin/detmir-check}" "$@" diff --git a/ops/detmir-ai/detmir-dlp b/ops/detmir-ai/detmir-dlp new file mode 100644 index 0000000..50ca5b6 --- /dev/null +++ b/ops/detmir-ai/detmir-dlp @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec "${DETMIR_DLP_BIN:-/usr/local/bin/detmir-dlp}" "$@" diff --git a/ops/detmir-ai/detmir-heal-safe b/ops/detmir-ai/detmir-heal-safe new file mode 100644 index 0000000..a60eccd --- /dev/null +++ b/ops/detmir-ai/detmir-heal-safe @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +aw_host="${DETMIR_AW_SSH_HOST:-igor@10.10.10.13}" + +ssh -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=accept-new \ + "$aw_host" \ + 'bash -s' <<'REMOTE' +set -euo pipefail + +services=( + activitywatch-server.service + aw-worktime-api.service + aw-worktime-ui-bridge.service + activitywatch-dlp-aggregator.service +) + +sudo -n systemctl reset-failed "${services[@]}" >/dev/null 2>&1 || true + +for service in activitywatch-server.service aw-worktime-api.service aw-worktime-ui-bridge.service; do + if ! systemctl is-active --quiet "$service"; then + echo "restart $service" + sudo -n systemctl restart "$service" + else + echo "active $service" + fi +done + +if systemctl list-unit-files activitywatch-dlp-aggregator.service >/dev/null 2>&1; then + if ! systemctl is-active --quiet activitywatch-dlp-aggregator.service; then + echo "start activitywatch-dlp-aggregator.service" + sudo -n systemctl start activitywatch-dlp-aggregator.service || true + else + echo "active activitywatch-dlp-aggregator.service" + fi +fi + +sudo -n /usr/local/bin/dlp-health-check --json >/tmp/detmir-heal-dlp-health.json || true +REMOTE diff --git a/ops/detmir-ai/detmir-status b/ops/detmir-ai/detmir-status new file mode 100644 index 0000000..906061f --- /dev/null +++ b/ops/detmir-ai/detmir-status @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec "${DETMIR_STATUS_BIN:-/usr/local/bin/detmir-status}" "$@" diff --git a/ops/detmir-ai/polli-chat b/ops/detmir-ai/polli-chat new file mode 100644 index 0000000..dd38008 --- /dev/null +++ b/ops/detmir-ai/polli-chat @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from urllib import error, request + + +DEFAULT_BASE_URL = "https://gen.pollinations.ai" +DEFAULT_KEY_PATH = Path.home() / ".pollinations" / "credentials.json" + +ALIASES = { + "daily": "gpt-5.4-mini", + "cheap": "gemini-flash-lite-3.1", + "heavy": "gpt-5.5", + "coding_primary": "qwen-coder", + "coding_backup": "qwen-coder-large", + "search": "perplexity-fast", + "search_reasoning": "perplexity-reasoning", + "vision_primary": "qwen-vision", + "vision_heavy": "qwen-vision-pro", + "guardrail": "qwen-safety", + "text.daily": "gpt-5.4-mini", + "text.cheap": "gemini-flash-lite-3.1", + "text.heavy": "gpt-5.5", + "text.coding_primary": "qwen-coder", + "text.coding_backup": "qwen-coder-large", + "text.search": "perplexity-fast", + "text.search_reasoning": "perplexity-reasoning", + "text.vision_primary": "qwen-vision", + "text.vision_heavy": "qwen-vision-pro", + "text.guardrail": "qwen-safety", + "embeddings.primary": "openai-3-small", + "embeddings.quality": "openai-3-large", + "image.fast_preview": "zimage", + "image.background_change": "kontext", + "image.final_quality": "gptimage-large", + "video.primary": "ltx-2", + "video.backup": "wan", + "audio.tts_primary": "qwen-tts-instruct", + "audio.tts_fast": "qwen-tts", + "audio.tts_premium": "elevenlabs", +} + + +def load_api_key() -> str: + inline = os.getenv("POLLINATIONS_API_KEY", "").strip() + if inline: + return inline + key_path = Path(os.getenv("POLLINATIONS_KEY_PATH", str(DEFAULT_KEY_PATH))).expanduser() + data = json.loads(key_path.read_text(encoding="utf-8")) + key = str(data.get("apiKey", "")).strip() + if not key: + raise RuntimeError(f"apiKey not found in {key_path}") + return key + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Pollinations OpenAI-compatible chat helper.") + parser.add_argument("prompt", nargs="?", help="Prompt. If omitted, stdin is used.") + parser.add_argument("--model", default="text.daily", help="Model alias or exact model id.") + parser.add_argument("--system", default="", help="Optional system prompt.") + parser.add_argument("--temperature", type=float, default=0.2) + parser.add_argument("--max-tokens", type=int, default=1200) + parser.add_argument("--json", action="store_true", help="Print raw JSON.") + parser.add_argument("--list-aliases", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.list_aliases: + print(json.dumps(ALIASES, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 + + prompt = args.prompt if args.prompt is not None else sys.stdin.read() + prompt = prompt.strip() + if not prompt: + raise RuntimeError("prompt is empty") + + model = ALIASES.get(args.model.strip(), args.model.strip()) + messages = [] + if args.system.strip(): + messages.append({"role": "system", "content": args.system.strip()}) + messages.append({"role": "user", "content": prompt}) + + base_url = os.getenv("POLLINATIONS_BASE_URL", DEFAULT_BASE_URL).rstrip("/") + payload = { + "model": model, + "messages": messages, + "temperature": args.temperature, + "max_tokens": args.max_tokens, + } + req = request.Request( + f"{base_url}/v1/chat/completions", + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {load_api_key()}", + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "curl/8.5 detmir-proxmox", + }, + method="POST", + ) + try: + with request.urlopen(req, timeout=180) as resp: + body = resp.read().decode("utf-8", errors="replace") + except error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"pollinations http {exc.code}: {body[:2000]}") from exc + + data = json.loads(body) + if args.json: + print(json.dumps(data, ensure_ascii=False, indent=2)) + return 0 + + choices = data.get("choices") or [] + if not choices: + raise RuntimeError(f"no choices in response: {body[:2000]}") + message = choices[0].get("message") or {} + content = message.get("content") + if isinstance(content, list): + text = "\n".join(str(part.get("text", "")) for part in content if isinstance(part, dict)) + else: + text = str(content or "") + print(text.strip()) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"polli-chat error: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/proxmox/test_tsj_guardian_bot.py b/proxmox/test_tsj_guardian_bot.py index 0bb74e3..31e77f0 100644 --- a/proxmox/test_tsj_guardian_bot.py +++ b/proxmox/test_tsj_guardian_bot.py @@ -9,6 +9,9 @@ class DummyState: def __init__(self): self.pending_incident = None self.last_warning_signature = "" + self.failure_streak_signature = "" + self.failure_streak_count = 0 + self.failure_streak_first_ts = 0 def save(self): pass @@ -18,6 +21,7 @@ def make_bot(check_rc, check_out, heal_rc, heal_out): bot = object.__new__(MODULE.TSJGuardianBot) bot.state = DummyState() bot.retry_autoheal_sec = 300 + bot.incident_failure_quorum_checks = 1 bot.exit_on_autoheal_success = True bot.fs_immediate_ai_on_critical = False bot.operator_timeout = 900 @@ -37,6 +41,42 @@ def make_bot(check_rc, check_out, heal_rc, heal_out): return bot +def make_slo_bot(summary): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.aw_rus_slo_enabled = True + bot.aw_rus_slo_alert_window = "24h" + bot.aw_rus_slo_min_samples = 4 + bot.aw_rus_slo_max_age_sec = 90 + bot.aw_rus_slo_summary_cmd = "cat /tmp/slo.json" + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._run_shell = lambda cmd, timeout_sec=30: (0, MODULE.json.dumps(summary)) + bot._aw_rus_probe_should_run = lambda failures: True + bot._aw_rus_dlp_probe = lambda: (_ for _ in ()).throw(AssertionError("raw probe should not run")) + return bot + + +def make_slo_cycle_bot(summary): + bot = make_bot(check_rc=0, check_out="", heal_rc=0, heal_out="") + bot.aw_rus_slo_enabled = True + bot.aw_rus_slo_alert_window = "24h" + bot.aw_rus_slo_min_samples = 4 + bot.aw_rus_slo_max_age_sec = 90 + bot.aw_rus_slo_summary_cmd = "cat /tmp/slo.json" + bot._run_shell = lambda cmd, timeout_sec=420: (0, MODULE.json.dumps(summary)) + bot._aw_rus_probe_should_run = lambda failures: True + bot._aw_rus_dlp_probe = lambda: (_ for _ in ()).throw(AssertionError("raw probe should not run during SLO drill")) + bot._background_aw_rus_failures = MODULE.TSJGuardianBot._background_aw_rus_failures.__get__( + bot, + MODULE.TSJGuardianBot, + ) + bot._extract_aw_rus_failure_keys = MODULE.TSJGuardianBot._extract_aw_rus_failure_keys.__get__( + bot, + MODULE.TSJGuardianBot, + ) + return bot + + class TsjGuardianBotTests(unittest.TestCase): def test_new_incident_auto_resolved_before_notification_stays_silent(self): check_out = "2026-05-24 10:01:18 [FAIL] node_13: curl failed: http://10.10.10.13:5600/\n" @@ -49,6 +89,37 @@ class TsjGuardianBotTests(unittest.TestCase): self.assertEqual(bot._notifications, []) self.assertTrue(any("auto-resolved before operator notification" in msg for _, msg in bot._logs)) + def test_transient_failure_is_suppressed_until_quorum(self): + check_out = "2026-05-24 10:01:18 [FAIL] node_13: curl failed: http://10.10.10.13:5600/\n" + heal_out = "2026-05-24 10:01:40 [FAIL] node_13: curl failed: http://10.10.10.13:5600/\n" + bot = make_bot(check_rc=1, check_out=check_out, heal_rc=1, heal_out=heal_out) + bot.incident_failure_quorum_checks = 2 + + bot._handle_check_cycle() + + self.assertIsNone(bot.state.pending_incident) + self.assertEqual(bot.state.failure_streak_count, 1) + self.assertEqual(bot._notifications, []) + + bot._handle_check_cycle() + + self.assertIsNotNone(bot.state.pending_incident) + self.assertEqual(bot.state.failure_streak_count, 0) + self.assertEqual(len(bot._notifications), 2) + self.assertIn("Обнаружен инцидент", bot._notifications[0]) + + def test_filesystem_critical_bypasses_failure_quorum(self): + check_out = "2026-05-24 10:01:18 [FAIL] filesystem_usage: /var 96%\n" + heal_out = "2026-05-24 10:01:40 [FAIL] filesystem_usage: /var 96%\n" + bot = make_bot(check_rc=1, check_out=check_out, heal_rc=1, heal_out=heal_out) + bot.incident_failure_quorum_checks = 3 + + bot._handle_check_cycle() + + self.assertIsNotNone(bot.state.pending_incident) + self.assertEqual(bot.state.failure_streak_count, 0) + self.assertTrue(bot._notifications) + def test_new_incident_notifies_only_after_autoheal_failure(self): check_out = "2026-05-24 10:01:18 [FAIL] node_13: curl failed: http://10.10.10.13:5600/\n" heal_out = "2026-05-24 10:01:40 [FAIL] node_13: curl failed: http://10.10.10.13:5600/\n" @@ -93,6 +164,201 @@ class TsjGuardianBotTests(unittest.TestCase): self.assertEqual(bot._notifications, []) self.assertTrue(any("auto-resolved before operator notification" in msg for _, msg in bot._logs)) + def test_slo_budget_exhaustion_becomes_aw_failure(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": False}, + "windows": { + "24h": { + "availability_percent": 99.96, + "samples": 10, + "bad_samples": 2, + "budget_remaining_seconds": -5, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + lines, failures = MODULE.TSJGuardianBot._aw_rus_slo_lines_and_failures(bot) + rendered, keys = MODULE.TSJGuardianBot._background_aw_rus_failures(bot, []) + + self.assertEqual(failures, ["slo"]) + self.assertTrue(any("error budget exhausted" in line for line in lines)) + self.assertEqual(keys, ["slo"]) + self.assertIn("aw-rus:slo", rendered[0]) + + def test_slo_recovered_budget_exhaustion_does_not_become_aw_failure(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": True}, + "windows": { + "24h": { + "availability_percent": 99.96, + "samples": 10, + "bad_samples": 2, + "budget_remaining_seconds": -5, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + lines, failures = MODULE.TSJGuardianBot._aw_rus_slo_lines_and_failures(bot) + rendered, keys = MODULE.TSJGuardianBot._background_aw_rus_failures(bot, []) + + self.assertEqual(failures, []) + self.assertTrue(any("RECOVERED error budget exhausted" in line for line in lines)) + self.assertEqual(keys, []) + self.assertEqual(rendered, []) + + def test_slo_status_line_marks_historical_burn_as_recovered_when_current_sample_ok(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": True}, + "windows": { + "24h": { + "availability_percent": 98.43, + "samples": 100, + "bad_samples": 3, + "budget_remaining_seconds": -20, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + line = MODULE.TSJGuardianBot._aw_rus_slo_status_line(bot) + + self.assertIn("aw_rus_slo: recovered", line) + self.assertIn("current_sample=OK", line) + self.assertIn("budget_remaining_seconds=-20", line) + + def test_slo_status_line_marks_current_failure_as_fail(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": False}, + "windows": { + "24h": { + "availability_percent": 98.43, + "samples": 100, + "bad_samples": 3, + "budget_remaining_seconds": -20, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + line = MODULE.TSJGuardianBot._aw_rus_slo_status_line(bot) + + self.assertIn("aw_rus_slo: fail", line) + self.assertIn("current_sample=FAIL", line) + + def test_slo_warmup_does_not_alert_before_min_samples(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": False}, + "windows": { + "24h": { + "availability_percent": 0.0, + "samples": 1, + "bad_samples": 1, + "budget_remaining_seconds": 10, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + lines, failures = MODULE.TSJGuardianBot._aw_rus_slo_lines_and_failures(bot) + + self.assertEqual(failures, []) + self.assertTrue(any("WARMUP" in line for line in lines)) + + def test_slo_failure_has_no_direct_autoheal_target(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": False}, + "windows": { + "24h": { + "availability_percent": 99.96, + "samples": 10, + "bad_samples": 2, + "budget_remaining_seconds": -5, + "status": "burning", + } + }, + } + bot = make_slo_bot(summary) + + ok, report, _after_lines, after_failures = MODULE.TSJGuardianBot._perform_aw_rus_autoheal(bot, ["slo"]) + + self.assertFalse(ok) + self.assertEqual(after_failures, ["slo"]) + self.assertTrue(any("no direct autoheal target" in line for line in report)) + + def test_slo_stale_autoheal_message_points_to_sampler(self): + summary = { + "generated_at_utc": "2026-05-30T10:00:00Z", + "current_sample": {"ok": True}, + "windows": {"24h": {"samples": 10, "bad_samples": 0, "budget_remaining_seconds": 25, "status": "ok"}}, + } + bot = make_slo_bot(summary) + + ok, report, after_lines, after_failures = MODULE.TSJGuardianBot._perform_aw_rus_autoheal(bot, ["slo"]) + + self.assertFalse(ok) + self.assertEqual(after_failures, ["slo"]) + self.assertTrue(any("SLO summary stale" in line for line in report)) + self.assertTrue(any("slo-summary: STALE" in line for line in after_lines)) + + def test_slo_current_failure_drill_creates_incident_and_operator_notifications(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": False}, + "windows": { + "24h": { + "availability_percent": 99.96, + "samples": 10, + "bad_samples": 2, + "budget_remaining_seconds": -5, + "status": "burning", + } + }, + } + bot = make_slo_cycle_bot(summary) + + bot._handle_check_cycle() + + self.assertIsNotNone(bot.state.pending_incident) + self.assertIn("aw-rus:slo", "\n".join(bot.state.pending_incident.failures)) + self.assertEqual(len(bot._notifications), 2) + self.assertIn("Обнаружен инцидент", bot._notifications[0]) + self.assertIn("Авто-лечение неуспешно", bot._notifications[1]) + + def test_slo_recovered_historical_burn_drill_stays_silent(self): + summary = { + "generated_at_utc": MODULE.datetime.now(MODULE.timezone.utc).isoformat().replace("+00:00", "Z"), + "current_sample": {"ok": True}, + "windows": { + "24h": { + "availability_percent": 99.96, + "samples": 10, + "bad_samples": 2, + "budget_remaining_seconds": -5, + "status": "burning", + } + }, + } + bot = make_slo_cycle_bot(summary) + + bot._handle_check_cycle() + + self.assertIsNone(bot.state.pending_incident) + self.assertEqual(bot._notifications, []) + self.assertTrue(any(msg == "Check OK" for _, msg in bot._logs)) + class TelegramApiDocumentTests(unittest.TestCase): def test_send_document_rejects_empty_payload(self): @@ -288,8 +554,27 @@ class AwRusDlpProbeTests(unittest.TestCase): "aw-file-operations_10.10.10.13": {"metadata": {"end": fresh_fileops}}, } ) + if "aw-rus-collector-guard_SHARKON2025/events" in url: + return response( + [ + { + "timestamp": fresh_worktime, + "data": {"status": "ok", "mode": "shadow", "actions": [], "problems": []}, + } + ] + ) if "aw-worktime-sessions_SHARKON2025/events" in url: return response([{"timestamp": fresh_worktime, "data": {"active": True}}]) + if "aw-watcher-window_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-watcher-afk_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-dlp-endpoint-signals_SHARKON2025/events" in url: + return response([{"timestamp": stale_endpoint, "data": {"signalType": "self_test"}}]) + if "aw-file-operations_SHARKON2025/events" in url: + return response([{"timestamp": fresh_fileops, "data": {}}]) + if "aw-file-operations_10.10.10.13/events" in url: + return response([{"timestamp": fresh_fileops, "data": {}}]) if url.endswith("/buckets/aw-watcher-window_SHARKON2025"): return response({"metadata": {"end": fresh_watcher}}) if url.endswith("/buckets/aw-watcher-afk_SHARKON2025"): @@ -323,6 +608,840 @@ class AwRusDlpProbeTests(unittest.TestCase): self.assertNotIn("dlp-endpoint", failures) self.assertTrue(any("file-operations fresh; endpoint collector degraded" in line for line in lines)) + @mock.patch("proxmox.tsj_guardian_bot.requests.get") + def test_inactive_host_and_healthy_guard_downgrades_endpoint_stale_to_warn(self, get_mock): + real_datetime = MODULE.datetime + now = real_datetime(2026, 5, 30, 3, 20, tzinfo=MODULE.timezone.utc) + stale_endpoint = "2026-05-30T02:40:00.000Z" + fresh_guard = "2026-05-30T03:19:30.000Z" + fresh_worktime = "2026-05-30T03:19:40.000Z" + fresh_watcher = "2026-05-30T03:18:00.000Z" + + def response(payload): + resp = mock.Mock() + resp.raise_for_status.return_value = None + resp.json.return_value = payload + return resp + + def fake_get(url, timeout=20): + if url.endswith("/buckets"): + return response({"aw-worktime-sessions_SHARKON2025": {"metadata": {"end": fresh_worktime}}}) + if "aw-rus-collector-guard_SHARKON2025/events" in url: + return response( + [ + { + "timestamp": fresh_guard, + "data": {"status": "ok", "mode": "shadow", "actions": [], "problems": []}, + } + ] + ) + if "aw-worktime-sessions_SHARKON2025/events" in url: + return response([{"timestamp": fresh_worktime, "data": {"active": False}}]) + if "aw-watcher-window_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-watcher-afk_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-dlp-endpoint-signals_SHARKON2025/events" in url: + return response([{"timestamp": stale_endpoint, "data": {"signalType": "self_test"}}]) + if "aw-file-operations_SHARKON2025/events" in url: + return response([]) + if "aw-file-operations_10.10.10.13/events" in url: + return response([]) + if url.endswith("/buckets/aw-file-operations_SHARKON2025"): + return response({"metadata": {"end": stale_endpoint}}) + if url.endswith("/buckets/aw-file-operations_10.10.10.13"): + return response({"metadata": {"end": stale_endpoint}}) + raise AssertionError(f"unexpected url {url}") + + get_mock.side_effect = fake_get + + bot = object.__new__(MODULE.TSJGuardianBot) + bot.aw_rus_api_base = "http://10.10.10.13:5600/api/0" + bot.aw_rus_worktime_base = "http://10.10.10.13:5610" + bot.aw_rus_host = "SHARKON2025" + bot.aw_rus_stale_sec = 900 + bot.aw_rus_primary_user = "user1" + bot._fetch_worktime_today_csv = lambda timeout_sec=20, attempts=2: ( + "user,user_id,active_seconds\n" + "user1,SHARKON2025\\\\user1,0\n" + ) + + with mock.patch("proxmox.tsj_guardian_bot.datetime") as dt_mock: + dt_mock.now.return_value = now + dt_mock.fromisoformat.side_effect = lambda value: real_datetime.fromisoformat(value) + lines, failures = MODULE.TSJGuardianBot._aw_rus_dlp_probe(bot) + + self.assertNotIn("dlp-endpoint", failures) + self.assertTrue(any("collector-guard: OK" in line for line in lines)) + self.assertTrue(any("dlp-endpoint: WARN" in line and "guard healthy" in line for line in lines)) + + @mock.patch("proxmox.tsj_guardian_bot.requests.get") + def test_worktime_report_timeout_is_warn_when_session_bucket_is_fresh(self, get_mock): + real_datetime = MODULE.datetime + now = real_datetime(2026, 5, 30, 3, 20, tzinfo=MODULE.timezone.utc) + fresh_guard = "2026-05-30T03:19:30.000Z" + fresh_worktime = "2026-05-30T03:19:40.000Z" + fresh_watcher = "2026-05-30T03:18:00.000Z" + + def response(payload): + resp = mock.Mock() + resp.raise_for_status.return_value = None + resp.json.return_value = payload + return resp + + def fake_get(url, timeout=20): + if url.endswith("/buckets"): + return response({"aw-worktime-sessions_SHARKON2025": {"metadata": {"end": fresh_worktime}}}) + if "aw-rus-collector-guard_SHARKON2025/events" in url: + return response( + [ + { + "timestamp": fresh_guard, + "data": {"status": "ok", "mode": "shadow", "actions": [], "problems": []}, + } + ] + ) + if "aw-worktime-sessions_SHARKON2025/events" in url: + return response([{"timestamp": fresh_worktime, "data": {"active": False}}]) + if "aw-watcher-window_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-watcher-afk_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {}}]) + if "aw-dlp-endpoint-signals_SHARKON2025/events" in url: + return response([{"timestamp": fresh_watcher, "data": {"signalType": "self_test"}}]) + if "aw-file-operations_SHARKON2025/events" in url: + return response([]) + if "aw-file-operations_10.10.10.13/events" in url: + return response([]) + if url.endswith("/buckets/aw-file-operations_SHARKON2025"): + return response({"metadata": {"end": fresh_watcher}}) + if url.endswith("/buckets/aw-file-operations_10.10.10.13"): + return response({"metadata": {"end": fresh_watcher}}) + raise AssertionError(f"unexpected url {url}") + + get_mock.side_effect = fake_get + + bot = object.__new__(MODULE.TSJGuardianBot) + bot.aw_rus_api_base = "http://10.10.10.13:5600/api/0" + bot.aw_rus_worktime_base = "http://10.10.10.13:5610" + bot.aw_rus_host = "SHARKON2025" + bot.aw_rus_stale_sec = 900 + bot.aw_rus_primary_user = "user1" + bot._fetch_worktime_today_csv = mock.Mock(side_effect=MODULE.requests.exceptions.ReadTimeout("read timeout")) + + with mock.patch("proxmox.tsj_guardian_bot.datetime") as dt_mock: + dt_mock.now.return_value = now + dt_mock.fromisoformat.side_effect = lambda value: real_datetime.fromisoformat(value) + lines, failures = MODULE.TSJGuardianBot._aw_rus_dlp_probe(bot) + + self.assertNotIn("worktime", failures) + self.assertTrue(any("worktime: WARN report unavailable" in line for line in lines)) + + def test_endpoint_failure_uses_windows_heal_not_server_dlp_heal(self): + bot = object.__new__(MODULE.TSJGuardianBot) + calls = [] + + def windows_heal(include_watchers, include_worktime, include_dlp=False): + calls.append(("windows", include_watchers, include_worktime, include_dlp)) + return True, ["windows-ok"] + + def dlp_heal(targets): + calls.append(("dlp", tuple(targets))) + return True, ["dlp-ok"] + + bot._aw_rus_windows_collectors_heal = windows_heal + bot._aw_rus_dlp_heal = dlp_heal + bot._aw_rus_worktime_heal = lambda: (True, ["worktime-ok"]) + bot._aw_rus_dlp_probe = lambda: (["after"], []) + + with mock.patch("proxmox.tsj_guardian_bot.time.sleep"): + ok, report, after_lines, after_failures = MODULE.TSJGuardianBot._perform_aw_rus_autoheal( + bot, + ["dlp-endpoint"], + ) + + self.assertTrue(ok) + self.assertEqual(calls, [("windows", False, False, True)]) + self.assertIn("windows-ok", report) + self.assertEqual(after_lines, ["after"]) + self.assertEqual(after_failures, []) + + def test_aw_autoheal_prefers_rust_plan_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + calls = [] + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + + def windows_heal(include_watchers, include_worktime, include_dlp=False): + calls.append(("windows", include_watchers, include_worktime, include_dlp)) + return True, ["windows-ok"] + + def dlp_heal(targets): + calls.append(("dlp", tuple(targets))) + return True, ["dlp-ok"] + + bot._aw_rus_windows_collectors_heal = windows_heal + bot._aw_rus_dlp_heal = dlp_heal + bot._aw_rus_worktime_heal = lambda: (calls.append(("worktime",)) or (True, ["worktime-ok"])) + bot._aw_rus_dlp_probe = lambda: (["after"], []) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "failures": ["watcher-window", "dlp-fileops-server", "worktime"], + "slo_only": False, + "slo_stale": False, + "include_watchers": True, + "include_worktime": True, + "include_windows_dlp": False, + "server_dlp_failures": ["dlp-fileops-server"], + "run_windows_heal": True, + "run_server_dlp_heal": True, + "run_worktime_heal": True, + "sleep_after_seconds": 5, + "report_triggers": [ + "- heal trigger: Windows session collectors degraded, starting remediation", + "- heal trigger: server-side DLP degraded, starting remediation", + "- heal trigger: worktime/watchers degraded, rebuilding server-side worktime views", + ], + "direct_autoheal_target": True, + } + ), + "", + ) + + bot._run_argv = fake_run_argv + with mock.patch("proxmox.tsj_guardian_bot.time.sleep") as sleep_mock: + ok, report, after_lines, after_failures = MODULE.TSJGuardianBot._perform_aw_rus_autoheal( + bot, + ["watcher-window", "dlp-fileops-server", "worktime"], + ) + + self.assertTrue(ok) + self.assertIn("--autoheal-plan-decision", captured["argv"]) + self.assertEqual(captured["payload"]["failures"], ["watcher-window", "dlp-fileops-server", "worktime"]) + self.assertEqual( + calls, + [ + ("windows", True, True, False), + ("dlp", ("dlp-fileops-server",)), + ("worktime",), + ], + ) + sleep_mock.assert_called_once_with(5) + self.assertIn("windows-ok", report) + self.assertIn("dlp-ok", report) + self.assertIn("worktime-ok", report) + self.assertEqual(after_lines, ["after"]) + self.assertEqual(after_failures, []) + + def test_slo_autoheal_prefers_rust_plan_backend(self): + bot = make_slo_bot( + { + "generated_at_utc": "2026-05-30T10:00:00Z", + "current_sample": {"ok": True}, + "windows": {"24h": {"samples": 10, "bad_samples": 0, "budget_remaining_seconds": 25, "status": "ok"}}, + } + ) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + payload = MODULE.json.loads(input_text) + if "--autoheal-plan-decision" not in argv: + raise AssertionError(f"unexpected argv {argv}") + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "failures": payload["failures"], + "slo_only": True, + "slo_stale": True, + "include_watchers": False, + "include_worktime": False, + "include_windows_dlp": False, + "server_dlp_failures": [], + "run_windows_heal": False, + "run_server_dlp_heal": False, + "run_worktime_heal": False, + "sleep_after_seconds": 0, + "report_triggers": ["from-rust-slo-trigger"], + "direct_autoheal_target": False, + } + ), + "", + ) + + bot._run_argv = fake_run_argv + ok, report, after_lines, after_failures = MODULE.TSJGuardianBot._perform_aw_rus_autoheal(bot, ["slo"]) + + self.assertFalse(ok) + self.assertEqual(report, ["from-rust-slo-trigger"]) + self.assertEqual(after_failures, ["slo"]) + self.assertTrue(any("slo-summary: STALE" in line for line in after_lines)) + + +class CodexExecSafetyTests(unittest.TestCase): + def test_codex_exec_does_not_forward_bearer_to_sudo(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.ai_exec_user = "igor" + bot.ai_chat_workdir = "/home/igor" + bot.ai_chat_sandbox = "workspace-write" + bot.codex_model = "gpt-test" + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["env_extra"] = env_extra or {} + captured["cwd"] = cwd + return MODULE.subprocess.CompletedProcess(argv, 1, "", "401 Unauthorized token_invalidated") + + bot._run_argv = fake_run_argv + with mock.patch.dict(MODULE.os.environ, {"PFSENSE_MCP_BEARER": "secret-token"}, clear=False): + rc, out, reply = MODULE.TSJGuardianBot._run_codex_exec_prompt(bot, "hello", timeout_sec=5) + + self.assertEqual(rc, 1) + self.assertIn("401 Unauthorized", out) + self.assertEqual(reply, "") + self.assertEqual(captured["cwd"], "/home/igor") + self.assertEqual(captured["env_extra"], {}) + self.assertNotIn("--preserve-env=PFSENSE_MCP_BEARER", captured["argv"]) + self.assertNotIn("secret-token", " ".join(captured["argv"])) + + def test_codex_auth_error_is_operator_safe(self): + message = MODULE.TSJGuardianBot._summarize_exec_error( + "HTTP error: 401 Unauthorized token_invalidated refresh_token_reused", + 1, + ) + self.assertIn("повторная авторизация Codex", message) + + def test_detmir_status_line_prefers_rust_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + bot.detmir_state_file = Path(tmp) / "latest-state.json" + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + return MODULE.subprocess.CompletedProcess(argv, 0, "- detmir_auto: OK from-rust\n", "") + + bot._run_argv = fake_run_argv + line = MODULE.TSJGuardianBot._detmir_auto_status_line(bot) + + self.assertEqual(line, "- detmir_auto: OK from-rust") + self.assertEqual(captured["argv"][0], str(bin_path)) + self.assertEqual(captured["argv"][1], "--state") + + def test_aw_slo_status_line_prefers_rust_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot.aw_rus_slo_summary_cmd = "cat /tmp/slo.json" + bot.aw_rus_slo_alert_window = "24h" + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + return MODULE.subprocess.CompletedProcess( + argv, + 0, + "- aw_rus_slo: recovered 24h current_sample=OK availability=99.00000% samples=4 budget_remaining_seconds=-1\n", + "", + ) + + bot._run_argv = fake_run_argv + line = MODULE.TSJGuardianBot._aw_rus_slo_status_line(bot) + + self.assertIn("aw_rus_slo: recovered", line) + self.assertIn("--aw-slo-status-line", captured["argv"]) + self.assertIn("--aw-slo-summary-command", captured["argv"]) + + def test_status_text_prefers_rust_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot.infra_admin_root = "/opt/infra-admin" + bot.detmir_state_file = Path("/var/lib/detmir-ai/latest-state.json") + bot.state_file = "/opt/infra-admin/.state/tsj_guardian_state.json" + bot.updates_rollback_file = Path("/opt/infra-admin/.state/rollback.json") + bot.aw_rus_slo_summary_cmd = "cat /tmp/slo.json" + bot.aw_rus_slo_alert_window = "24h" + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["timeout_sec"] = timeout_sec + return MODULE.subprocess.CompletedProcess(argv, 0, "Статус: инцидентов нет.\n- from-rust\n", "") + + bot._run_argv = fake_run_argv + text = MODULE.TSJGuardianBot._cmd_status(bot) + + self.assertIn("- from-rust", text) + self.assertIn("--status-text", captured["argv"]) + self.assertIn("--bot-state", captured["argv"]) + self.assertIn("--rollback-file", captured["argv"]) + self.assertIn("--pfsense-status-command", captured["argv"]) + self.assertGreaterEqual(captured["timeout_sec"], 90) + + def test_status_text_falls_back_when_rust_backend_fails(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot.infra_admin_root = "/opt/infra-admin" + bot.detmir_state_file = Path("/tmp/missing-detmir-state.json") + bot.state_file = "/tmp/missing-bot-state.json" + bot.updates_rollback_file = Path("/tmp/missing-rollback.json") + bot.aw_rus_slo_summary_cmd = "" + bot.aw_rus_slo_alert_window = "24h" + bot.state = DummyState() + bot.state.pending_pfsense_change = None + bot.state.pending_openvpn_config = None + bot.state.pending_proxmox_selection = None + bot.state.pending_proxmox_restore = None + bot.state.pending_update_install_confirm = False + bot.state.pending_rollback_confirm = False + bot.state.last_openvpn_expiry_signature = "" + bot._pfsense_security_status_lines = lambda: "- pfsense_security: ok" + bot._aw_rus_slo_status_line = lambda: "- aw_rus_slo: ok" + bot._detmir_auto_status_line = lambda: "- detmir_auto: OK" + bot._rollback_pending_count = lambda: 0 + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + bot._run_argv = lambda *args, **kwargs: MODULE.subprocess.CompletedProcess(args[0], 1, "", "boom") + text = MODULE.TSJGuardianBot._cmd_status(bot) + + self.assertIn("Статус: инцидентов нет.", text) + self.assertIn("- detmir_auto: OK", text) + self.assertTrue(any("status-text failed" in msg for _, msg in bot._logs)) + + def test_suggestions_prefer_rust_decision_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps({"suggestions": ["from-rust"]}), + "", + ) + + bot._run_argv = fake_run_argv + suggestions = MODULE.TSJGuardianBot._suggestions_from_failures(bot, ["[FAIL] x"]) + + self.assertEqual(suggestions, ["from-rust"]) + self.assertIn("--incident-suggestions", captured["argv"]) + self.assertEqual(captured["payload"]["failures"], ["[FAIL] x"]) + + def test_defer_transient_incident_prefers_rust_decision_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.incident_failure_quorum_checks = 2 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "defer": True, + "failure_streak_signature": "sig", + "failure_streak_count": 1, + "failure_streak_first_ts": 1000, + "reset_failure_streak": False, + "log_line": "Suppressing transient incident streak=1/2 failures=1", + } + ), + "", + ) + + bot._run_argv = fake_run_argv + defer = MODULE.TSJGuardianBot._defer_transient_new_incident(bot, ["[FAIL] x"]) + + self.assertTrue(defer) + self.assertIn("--incident-defer-decision", captured["argv"]) + self.assertEqual(bot.state.failure_streak_signature, "sig") + self.assertEqual(bot.state.failure_streak_count, 1) + self.assertTrue(any("Suppressing transient incident" in msg for _, msg in bot._logs)) + + def test_timeout_escalation_prefers_rust_decision_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_incident = MODULE.PendingIncident( + incident_id="inc-1", + created_ts=1, + failures=["f1"], + suggestions=[], + last_autoheal_ts=0, + autoheal_attempts=0, + operator_acked=False, + escalated_to_ai=False, + fallback_executed=False, + ) + bot.operator_timeout = 900 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._notifications = [] + bot._notify = lambda msg: bot._notifications.append(msg) + bot._escalate_to_ai = lambda: True + bot._run_server_fallback = lambda: True + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "should_escalate": True, + "should_fallback": True, + "timed_out": True, + "operator_acked": False, + "age_seconds": 1000, + "reason": "operator_timeout_reached", + } + ), + "", + ) + + bot._run_argv = fake_run_argv + MODULE.TSJGuardianBot._evaluate_timeout_escalation(bot) + + self.assertIn("--escalation-decision", captured["argv"]) + self.assertTrue(bot.state.pending_incident.escalated_to_ai) + self.assertTrue(bot.state.pending_incident.fallback_executed) + self.assertIn("Оператор не ответил. Выполнена эскалация.", bot._notifications) + self.assertIn("Сервер выполнил автономный fallback-план.", bot._notifications) + + def test_run_operator_action_prefers_rust_routing(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._escalate_to_ai = lambda: True + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "requested_action": "techsupport", + "canonical_action": "support", + "handler": "ai_escalation", + "allowed": True, + "requires_confirmation": False, + "risk_level": "medium", + "reason": "allowed", + "message": None, + "state_update_hints": [], + } + ), + "", + ) + + bot._run_argv = fake_run_argv + result = MODULE.TSJGuardianBot._run_operator_action(bot, "techsupport") + + self.assertEqual(result, "/run support result=ok") + self.assertIn("--operator-action-decision", captured["argv"]) + self.assertEqual(captured["payload"]["action"], "techsupport") + self.assertTrue(any("Operator action routed by Rust" in msg for _, msg in bot._logs)) + + def test_run_operator_action_uses_rust_confirmation_guard(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_update_install_confirm = False + bot.state.pending_rollback_confirm = False + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._run_shell = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("blocked action must not execute shell") + ) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "requested_action": "updates-install-confirm", + "canonical_action": "updates-install-confirm", + "handler": "updates_install_apply", + "allowed": False, + "requires_confirmation": True, + "risk_level": "high", + "reason": "missing_update_install_confirmation", + "message": "Нет ожидающего запроса на установку. Сначала нажмите \"Установить критичные и важные обновления\".", + "state_update_hints": [], + } + ), + "", + ) + + bot._run_argv = fake_run_argv + result = MODULE.TSJGuardianBot._run_operator_action(bot, "updates-install-confirm") + + self.assertIn("Нет ожидающего запроса на установку", result) + self.assertIn("Установить критичные и важные обновления", result) + + def test_pfsense_first_confirm_prefers_rust_confirmation_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_pfsense_change = MODULE.PendingPfSenseChange( + request_id="pf-1", + created_ts=1000, + operator_request="add firewall rule", + stage="awaiting_first_confirm", + confirm_code="123456", + first_confirmed_ts=0, + ) + bot.pfsense_change_confirm_ttl_sec = 900 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + captured = {} + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + captured["argv"] = argv + captured["payload"] = MODULE.json.loads(input_text) + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "kind": "pfsense", + "action": "first_confirm", + "present": True, + "expired": False, + "allowed": True, + "clear_pending": False, + "next_stage": "awaiting_second_confirm", + "first_confirmed_ts": 1200, + "reason": "first_confirmed", + "message": None, + } + ), + "", + ) + + bot._run_argv = fake_run_argv + text = MODULE.TSJGuardianBot._confirm_pfsense_change_stage_one(bot) + + self.assertIn("--confirmation-decision", captured["argv"]) + self.assertEqual(captured["payload"]["kind"], "pfsense") + self.assertEqual(captured["payload"]["action"], "first_confirm") + self.assertEqual(bot.state.pending_pfsense_change.stage, "awaiting_second_confirm") + self.assertEqual(bot.state.pending_pfsense_change.first_confirmed_ts, 1200) + self.assertIn("/pfsense_apply 123456", text) + + def test_pfsense_apply_uses_rust_confirmation_guard_before_side_effect(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_pfsense_change = MODULE.PendingPfSenseChange( + request_id="pf-1", + created_ts=1000, + operator_request="add firewall rule", + stage="awaiting_second_confirm", + confirm_code="123456", + first_confirmed_ts=1100, + ) + bot.pfsense_change_confirm_ttl_sec = 900 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._run_pfsense_change_codex_exec = lambda pending: (_ for _ in ()).throw( + AssertionError("blocked confirmation must not execute pfSense side effect") + ) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "kind": "pfsense", + "action": "apply", + "present": True, + "expired": False, + "allowed": False, + "clear_pending": False, + "reason": "wrong_code", + "message": "Неверный код второго подтверждения pfSense.", + } + ), + "", + ) + + bot._run_argv = fake_run_argv + result = MODULE.TSJGuardianBot._apply_pfsense_change(bot, "000000") + + self.assertEqual(result, "Неверный код второго подтверждения pfSense.") + + def test_openvpn_apply_uses_rust_confirmation_guard_before_prepare(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_openvpn_config = MODULE.PendingOpenVpnConfig( + request_id="ovpn-1", + created_ts=1000, + common_name="user1", + stage="awaiting_second_confirm", + confirm_code="123456", + first_confirmed_ts=1100, + ) + bot.openvpn_config_confirm_ttl_sec = 900 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + sent = [] + bot._send_text = lambda chat_id, text: sent.append((chat_id, text)) + bot._prepare_openvpn_config = lambda pending: (_ for _ in ()).throw( + AssertionError("blocked confirmation must not prepare OpenVPN config") + ) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "kind": "openvpn", + "action": "apply", + "present": True, + "expired": False, + "allowed": False, + "clear_pending": False, + "reason": "wrong_stage", + "message": "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения.", + } + ), + "", + ) + + bot._run_argv = fake_run_argv + MODULE.TSJGuardianBot._apply_openvpn_config(bot, 42, "123456") + + self.assertEqual(sent, [(42, "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения.")]) + + def test_proxmox_restore_apply_uses_rust_confirmation_guard_before_snapshot_check(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.state = DummyState() + bot.state.pending_proxmox_restore = MODULE.PendingProxmoxRestore( + request_id="pm-1", + created_ts=1000, + kind="lxc", + guest_id="200", + guest_name="ct", + node="pve", + snapshot="tsj-guardian-manual", + confirm_code="123456", + ) + bot.proxmox_restore_confirm_ttl_sec = 900 + bot._logs = [] + bot._log = lambda level, msg: bot._logs.append((level, msg)) + bot._snapshot_exists = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError("blocked confirmation must not check snapshot") + ) + with MODULE.tempfile.TemporaryDirectory() as tmp: + bin_path = Path(tmp) / "tsj-guardian-status" + bin_path.write_text("#!/bin/sh\n", encoding="utf-8") + bot.tsj_guardian_status_bin = str(bin_path) + + def fake_run_argv(argv, timeout_sec=180, input_text="", env_extra=None, cwd=None): + return MODULE.subprocess.CompletedProcess( + argv, + 0, + MODULE.json.dumps( + { + "kind": "proxmox_restore", + "action": "apply", + "present": True, + "expired": False, + "allowed": False, + "clear_pending": False, + "reason": "wrong_code", + "message": "Неверный код подтверждения восстановления Proxmox.", + } + ), + "", + ) + + bot._run_argv = fake_run_argv + result = MODULE.TSJGuardianBot._apply_proxmox_restore(bot, "000000") + + self.assertEqual(result, "Неверный код подтверждения восстановления Proxmox.") + class BotUiLabelTests(unittest.TestCase): def test_dlp_toggle_button_text_reflects_monitor_mode(self): @@ -439,6 +1558,70 @@ class BotUiLabelTests(unittest.TestCase): class DlpPolicyToggleTests(unittest.TestCase): + def test_aw_dlp_policy_mode_text_prefers_rust_decision_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot._aw_dlp_policy_request = lambda method, path, timeout_sec=15: { + "name": "active", + "policyId": 7, + "version": 3, + "policy": {"endpoint": {"clipboard": [{"id": "c1", "action": "alert"}]}}, + } + captured = {} + + def fake_decision(policy, target_mode=""): + captured["policy"] = policy + captured["target_mode"] = target_mode + return { + "current_mode": "monitor", + "groups": [{"name": "endpoint.clipboard", "blocked": 0, "total": 1}], + } + + bot._aw_dlp_policy_decision = fake_decision + + text = MODULE.TSJGuardianBot._aw_dlp_policy_mode_text(bot) + + self.assertEqual(captured["target_mode"], "") + self.assertIn("- mode: monitor", text) + self.assertIn("- endpoint.clipboard: block=0/1", text) + + def test_aw_dlp_policy_toggle_text_prefers_rust_decision_backend(self): + bot = object.__new__(MODULE.TSJGuardianBot) + bot.aw_dlp_policy_actor = "tester" + active_policy = { + "endpoint": {"clipboard": [{"id": "c1", "enabled": True, "action": "alert"}]}, + } + updated_policy = { + "endpoint": {"clipboard": [{"id": "c1", "enabled": True, "action": "block"}]}, + "_tsj_meta": {"dlp_mode": "enforce"}, + } + requests_seen = [] + + def fake_request(method, path, payload=None, timeout_sec=20): + requests_seen.append((method, path, payload)) + if method == "GET": + return {"name": "active", "policyId": 7, "version": 3, "policy": active_policy} + return {"item": {"current_version": 4}} + + bot._aw_dlp_policy_request = fake_request + bot._aw_dlp_policy_decision = lambda policy, target_mode="": { + "current_mode": "monitor", + "target_mode": "enforce", + "changed_count": 1, + "changed_rules": ["endpoint.clipboard:c1 alert->block"], + "updated_policy": updated_policy, + } + bot._aw_rus_windows_sync_dlp_policy = lambda policy: (True, ["- windows-policy-sync: OK"]) + bot._aw_dlp_toggle_button_text = lambda current_mode=None: "DLP сейчас: блокировка | включить наблюдение" + + text = MODULE.TSJGuardianBot._aw_dlp_policy_toggle_text(bot) + + self.assertEqual(requests_seen[1][0], "PUT") + self.assertEqual(requests_seen[1][1], "/dlp/policies/7") + self.assertEqual(requests_seen[1][2]["policy"], updated_policy) + self.assertIn("- mode: monitor -> enforce", text) + self.assertIn("- changed_rules: 1", text) + self.assertIn("endpoint.clipboard:c1 alert->block", text) + def test_aw_dlp_mode_from_policy_detects_monitor_and_enforce(self): monitor_policy = { "endpoint": { diff --git a/proxmox/tsj_guardian_bot.py b/proxmox/tsj_guardian_bot.py index 0fd2ead..d8fd442 100644 --- a/proxmox/tsj_guardian_bot.py +++ b/proxmox/tsj_guardian_bot.py @@ -115,6 +115,9 @@ class GuardianState: self.pending_update_install_confirm = False self.pending_rollback_confirm = False self.last_warning_signature = "" + self.failure_streak_signature = "" + self.failure_streak_count = 0 + self.failure_streak_first_ts = 0 self.last_openvpn_expiry_signature = "" self.ai_chat_intro_variant = -1 @@ -128,6 +131,9 @@ class GuardianState: self.pending_update_install_confirm = bool(raw.get("pending_update_install_confirm", False)) self.pending_rollback_confirm = bool(raw.get("pending_rollback_confirm", False)) self.last_warning_signature = str(raw.get("last_warning_signature", "")) + self.failure_streak_signature = str(raw.get("failure_streak_signature", "")) + self.failure_streak_count = int(raw.get("failure_streak_count", 0)) + self.failure_streak_first_ts = int(raw.get("failure_streak_first_ts", 0)) self.last_openvpn_expiry_signature = str(raw.get("last_openvpn_expiry_signature", "")) self.ai_chat_intro_variant = int(raw.get("ai_chat_intro_variant", -1)) pi = raw.get("pending_incident") @@ -159,6 +165,9 @@ class GuardianState: "pending_update_install_confirm": self.pending_update_install_confirm, "pending_rollback_confirm": self.pending_rollback_confirm, "last_warning_signature": self.last_warning_signature, + "failure_streak_signature": self.failure_streak_signature, + "failure_streak_count": self.failure_streak_count, + "failure_streak_first_ts": self.failure_streak_first_ts, "last_openvpn_expiry_signature": self.last_openvpn_expiry_signature, "ai_chat_intro_variant": self.ai_chat_intro_variant, } @@ -369,8 +378,7 @@ class TSJGuardianBot: self.aw_dlp_policy_actor = os.getenv("AW_DLP_POLICY_ACTOR", "tsj-guardian-bot").strip() or "tsj-guardian-bot" self.aw_rus_worktime_heal_cmd = os.getenv( "AW_RUS_WORKTIME_HEAL_CMD", - "sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 " - "'sudo -S /usr/local/bin/aw-worktime-autoheal.sh && sudo -S systemctl reset-failed aw-worktime-ui-bridge.service && sudo -S systemctl start aw-worktime-ui-bridge.service'", + "", ).strip() self.aw_rus_dlp_heal_cmd = os.getenv( "AW_RUS_DLP_HEAL_CMD", @@ -380,11 +388,19 @@ class TSJGuardianBot: self.aw_rus_hayabusa_enabled = env_bool("AW_RUS_HAYABUSA_ENABLED", True) self.aw_rus_hayabusa_ssh_cmd = os.getenv( "AW_RUS_HAYABUSA_SSH_CMD", - "sshpass -p '04091968' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13", + "", ).strip() self.aw_rus_host = os.getenv("AW_RUS_HOST", "SHARKON2025").strip() self.aw_rus_primary_user = os.getenv("AW_RUS_PRIMARY_USER", "USER1").strip() self.aw_rus_stale_sec = max(60, env_int("AW_RUS_STALE_SEC", 900)) + self.aw_rus_slo_enabled = env_bool("AW_RUS_SLO_ENABLED", True) + self.aw_rus_slo_alert_window = os.getenv("AW_RUS_SLO_ALERT_WINDOW", "24h").strip() or "24h" + self.aw_rus_slo_min_samples = max(1, env_int("AW_RUS_SLO_MIN_SAMPLES", 4)) + self.aw_rus_slo_max_age_sec = max(15, env_int("AW_RUS_SLO_MAX_AGE_SEC", 90)) + self.aw_rus_slo_summary_cmd = os.getenv( + "AW_RUS_SLO_SUMMARY_CMD", + "", + ).strip() self.aw_rus_windows_host = os.getenv("AW_RUS_WINDOWS_HOST", "192.168.100.18").strip() or "192.168.100.18" self.aw_rus_windows_ssh_user = os.getenv("AW_RUS_WINDOWS_SSH_USER", "Администратор").strip() or "Администратор" self.aw_rus_windows_ssh_password = os.getenv("AW_RUS_WINDOWS_SSH_PASSWORD", "").strip() @@ -400,6 +416,11 @@ class TSJGuardianBot: "AW_RUS_WINDOWS_SESSION_COLLECTOR_PATH", r"C:\ProgramData\AWatch-rus\worktime-session-collector.ps1", ).strip() or r"C:\ProgramData\AWatch-rus\worktime-session-collector.ps1" + launch_tasks_raw = os.getenv( + "AW_RUS_WINDOWS_LAUNCH_TASKS", + "ActivityWatch Launch [SHARKON2025_Администратор];ActivityWatch Launch [SHARKON2025_user5]", + ) + self.aw_rus_windows_launch_tasks = [item.strip() for item in launch_tasks_raw.split(";") if item.strip()] self.aw_rus_windows_policy_path = os.getenv( "AW_RUS_WINDOWS_POLICY_PATH", r"C:\ProgramData\AWatch-rus\dlp-policy.json", @@ -425,6 +446,7 @@ class TSJGuardianBot: "HEARTBEAT_FILE", f"{self.infra_admin_root}/.state/tsj_guardian_heartbeat" ) self.check_interval = env_int("CHECK_INTERVAL_SEC", 60) + self.incident_failure_quorum_checks = max(1, env_int("INCIDENT_FAILURE_QUORUM_CHECKS", 2)) self.operator_timeout = env_int("OPERATOR_TIMEOUT_SEC", 900) # 15 min self.retry_autoheal_sec = env_int("RETRY_AUTORECOVERY_EVERY_SEC", 300) self.exit_on_autoheal_success = env_bool("EXIT_ON_AUTORECOVERY_SUCCESS", True) @@ -496,6 +518,8 @@ class TSJGuardianBot: f"{self.infra_admin_root}/.state/proxmox_lxc_pending_rollback.json", ) ) + self.detmir_state_file = Path(os.getenv("DETMIR_AI_STATE_FILE", "/var/lib/detmir-ai/latest-state.json")) + self.tsj_guardian_status_bin = os.getenv("TSJ_GUARDIAN_STATUS_BIN", "/usr/local/bin/tsj-guardian-status").strip() self.proxmox_selection_ttl_sec = env_int("PROXMOX_SELECTION_TTL_SEC", 900) self.proxmox_restore_confirm_ttl_sec = env_int("PROXMOX_RESTORE_CONFIRM_TTL_SEC", 900) self.proxmox_manual_snapshot_name = ( @@ -883,6 +907,7 @@ class TSJGuardianBot: timeout_sec: int = 180, input_text: str = "", env_extra: Optional[Dict[str, str]] = None, + cwd: Optional[str] = None, ) -> subprocess.CompletedProcess: env = os.environ.copy() if env_extra: @@ -895,6 +920,7 @@ class TSJGuardianBot: timeout=timeout_sec, check=False, env=env, + cwd=cwd, ) return subprocess.CompletedProcess( proc.args, @@ -961,24 +987,62 @@ class TSJGuardianBot: cmd = f"sudo -u {shlex.quote(self.ai_exec_user)} bash -lc {shlex.quote(cmd)}" return self._run_shell(cmd, timeout_sec=timeout_sec) + def _run_ai_user_argv( + self, + argv: List[str], + timeout_sec: int = 180, + env_extra: Optional[Dict[str, str]] = None, + ) -> subprocess.CompletedProcess: + cmd = list(argv) + if self.ai_exec_user: + preserve = [] + if env_extra and env_extra.get("PFSENSE_MCP_BEARER"): + preserve.append("PFSENSE_MCP_BEARER") + sudo_argv = ["sudo", "-u", self.ai_exec_user] + if preserve: + sudo_argv.append(f"--preserve-env={','.join(preserve)}") + cmd = [*sudo_argv, "--", *cmd] + return self._run_argv( + cmd, + timeout_sec=timeout_sec, + env_extra=env_extra, + cwd=self.ai_chat_workdir, + ) + def _run_codex_exec_prompt(self, prompt: str, timeout_sec: int, model: Optional[str] = None) -> Tuple[int, str, str]: with tempfile.NamedTemporaryFile("w+", encoding="utf-8", delete=False) as tmp: tmp_path = tmp.name os.chmod(tmp_path, 0o666) selected_model = (model or self.codex_model).strip() or self.codex_model - cmd = ( - f"cd {shlex.quote(self.ai_chat_workdir)} && " - f"PFSENSE_MCP_BEARER={shlex.quote(os.getenv('PFSENSE_MCP_BEARER', ''))} " - f"codex exec --ephemeral --skip-git-repo-check " - f"--model {shlex.quote(selected_model)} " - f"-C {shlex.quote(self.ai_chat_workdir)} " - f"-s {shlex.quote(self.ai_chat_sandbox)} " - f"--color never -o {shlex.quote(tmp_path)} " - f"{shlex.quote(prompt)}" - ) + # Do not forward PFSENSE_MCP_BEARER into sudo/codex. sudo logs + # preserved env values, so passing the bearer here leaks it to journald. + env_extra = None + argv = [ + "codex", + "exec", + "--ephemeral", + "--skip-git-repo-check", + "--model", + selected_model, + "-C", + self.ai_chat_workdir, + "-s", + self.ai_chat_sandbox, + "--color", + "never", + "-o", + tmp_path, + prompt, + ] try: - rc, out = self._run_ai_user_shell(cmd, timeout_sec=timeout_sec) + proc = self._run_ai_user_argv( + argv, + timeout_sec=timeout_sec, + env_extra=env_extra, + ) + rc = proc.returncode + out = (proc.stdout or "") + (("\n" + proc.stderr) if proc.stderr else "") try: with open(tmp_path, "r", encoding="utf-8") as f: reply = f.read().strip() @@ -1027,6 +1091,8 @@ class TSJGuardianBot: @staticmethod def _summarize_exec_error(output: str, rc: int) -> str: lowered = (output or "").lower() + if TSJGuardianBot._is_codex_auth_error(output): + return "Канал расширенной диагностики недоступен: требуется повторная авторизация Codex на сервере." if "403 forbidden" in lowered or "unable to load site" in lowered: return "Сервис ответов временно недоступен. Повторите запрос чуть позже." if "selected model is at capacity" in lowered: @@ -1035,6 +1101,20 @@ class TSJGuardianBot: return "Сервис ответов временно недоступен из-за сетевой ошибки. Повторите запрос чуть позже." return f"Обработка запроса завершилась с ошибкой.\nrc={rc}" + @staticmethod + def _is_codex_auth_error(output: str) -> bool: + lowered = (output or "").lower() + return any( + marker in lowered + for marker in ( + "401 unauthorized", + "token_invalidated", + "refresh_token_reused", + "access token could not be refreshed", + "authentication token has been invalidated", + ) + ) + def _run_ai_chat_codex_exec(self, operator_text: str) -> str: clean_text = operator_text.strip() if not clean_text: @@ -1093,6 +1173,14 @@ class TSJGuardianBot: ppc = self.state.pending_pfsense_change if not ppc: return False + decision = self._confirmation_decision( + "pfsense", + "expire", + ppc, + self.pfsense_change_confirm_ttl_sec, + ) + if isinstance(decision, dict): + return bool(decision.get("expired")) return int(time.time()) - ppc.created_ts > self.pfsense_change_confirm_ttl_sec def _expire_pending_pfsense_change_if_needed(self) -> None: @@ -1140,6 +1228,30 @@ class TSJGuardianBot: ppc = self.state.pending_pfsense_change if not ppc: return "Нет ожидающего изменения pfSense." + decision = self._confirmation_decision( + "pfsense", + "first_confirm", + ppc, + self.pfsense_change_confirm_ttl_sec, + ) + if isinstance(decision, dict): + if decision.get("clear_pending"): + self.state.pending_pfsense_change = None + self.state.save() + return str(decision.get("message") or "Нет ожидающего изменения pfSense.") + if not decision.get("allowed"): + return str(decision.get("message") or "Первое подтверждение уже принято.") + ppc.stage = str(decision.get("next_stage") or "awaiting_second_confirm") + ppc.first_confirmed_ts = int(decision.get("first_confirmed_ts") or time.time()) + self.state.save() + return ( + "Первое подтверждение принято.\n" + f"- request_id: {ppc.request_id}\n" + f"- запрос: {ppc.operator_request}\n" + "- Второе подтверждение должно быть отдельным действием.\n" + f"- Для выполнения отправьте: `/pfsense_apply {ppc.confirm_code}`\n" + f"- Код подтверждения действует {self.pfsense_change_confirm_ttl_sec} секунд с момента создания запроса." + ) if ppc.stage != "awaiting_first_confirm": return ( "Первое подтверждение уже принято.\n" @@ -1159,14 +1271,30 @@ class TSJGuardianBot: ) def _cancel_pfsense_change(self) -> str: + decision = self._confirmation_decision( + "pfsense", + "cancel", + self.state.pending_pfsense_change, + self.pfsense_change_confirm_ttl_sec, + ) self.state.pending_pfsense_change = None self.state.save() + if isinstance(decision, dict): + return str(decision.get("message") or "Ожидающее изменение pfSense отменено.") return "Ожидающее изменение pfSense отменено." def _pending_openvpn_config_expired(self) -> bool: povpn = self.state.pending_openvpn_config if not povpn: return False + decision = self._confirmation_decision( + "openvpn", + "expire", + povpn, + self.openvpn_config_confirm_ttl_sec, + ) + if isinstance(decision, dict): + return bool(decision.get("expired")) return int(time.time()) - povpn.created_ts > self.openvpn_config_confirm_ttl_sec def _expire_pending_openvpn_config_if_needed(self) -> None: @@ -1409,6 +1537,29 @@ class TSJGuardianBot: povpn = self.state.pending_openvpn_config if not povpn: return "Нет ожидающего запроса на OpenVPN конфиг." + decision = self._confirmation_decision( + "openvpn", + "first_confirm", + povpn, + self.openvpn_config_confirm_ttl_sec, + ) + if isinstance(decision, dict): + if decision.get("clear_pending"): + self.state.pending_openvpn_config = None + self.state.save() + return str(decision.get("message") or "Нет ожидающего запроса на OpenVPN конфиг.") + if not decision.get("allowed"): + return str(decision.get("message") or "Первое подтверждение уже принято.") + povpn.stage = str(decision.get("next_stage") or "awaiting_second_confirm") + povpn.first_confirmed_ts = int(decision.get("first_confirmed_ts") or time.time()) + self.state.save() + return ( + "Первое подтверждение OpenVPN-конфига принято.\n" + f"- request_id: {povpn.request_id}\n" + f"- common_name: {povpn.common_name}\n" + f"- Для второго подтверждения отправьте: `/openvpn_config_apply {povpn.confirm_code}`\n" + f"- Код действует {self.openvpn_config_confirm_ttl_sec} секунд с момента создания запроса." + ) if povpn.stage != "awaiting_first_confirm": return ( "Первое подтверждение уже принято.\n" @@ -1426,14 +1577,30 @@ class TSJGuardianBot: ) def _cancel_openvpn_config(self) -> str: + decision = self._confirmation_decision( + "openvpn", + "cancel", + self.state.pending_openvpn_config, + self.openvpn_config_confirm_ttl_sec, + ) self.state.pending_openvpn_config = None self.state.save() + if isinstance(decision, dict): + return str(decision.get("message") or "Ожидающий запрос на OpenVPN конфиг отменён.") return "Ожидающий запрос на OpenVPN конфиг отменён." def _pending_proxmox_selection_expired(self) -> bool: pending = self.state.pending_proxmox_selection if not pending: return False + decision = self._confirmation_decision( + "proxmox_selection", + "expire", + pending, + self.proxmox_selection_ttl_sec, + ) + if isinstance(decision, dict): + return bool(decision.get("expired")) return int(time.time()) - pending.created_ts > self.proxmox_selection_ttl_sec def _expire_pending_proxmox_selection_if_needed(self) -> None: @@ -1445,6 +1612,14 @@ class TSJGuardianBot: pending = self.state.pending_proxmox_restore if not pending: return False + decision = self._confirmation_decision( + "proxmox_restore", + "expire", + pending, + self.proxmox_restore_confirm_ttl_sec, + ) + if isinstance(decision, dict): + return bool(decision.get("expired")) return int(time.time()) - pending.created_ts > self.proxmox_restore_confirm_ttl_sec def _expire_pending_proxmox_restore_if_needed(self) -> None: @@ -1668,13 +1843,29 @@ class TSJGuardianBot: ) def _cancel_proxmox_selection(self) -> str: + decision = self._confirmation_decision( + "proxmox_selection", + "cancel", + self.state.pending_proxmox_selection, + self.proxmox_selection_ttl_sec, + ) self.state.pending_proxmox_selection = None self.state.save() + if isinstance(decision, dict): + return str(decision.get("message") or "Выбор узла Proxmox отменён.") return "Выбор узла Proxmox отменён." def _cancel_proxmox_restore(self) -> str: + decision = self._confirmation_decision( + "proxmox_restore", + "cancel", + self.state.pending_proxmox_restore, + self.proxmox_restore_confirm_ttl_sec, + ) self.state.pending_proxmox_restore = None self.state.save() + if isinstance(decision, dict): + return str(decision.get("message") or "Ожидающее восстановление Proxmox отменено.") return "Ожидающее восстановление Proxmox отменено." def _apply_proxmox_restore(self, code: str) -> str: @@ -1682,7 +1873,21 @@ class TSJGuardianBot: pending = self.state.pending_proxmox_restore if not pending: return "Нет ожидающего восстановления Proxmox." - if code.strip() != pending.confirm_code: + decision = self._confirmation_decision( + "proxmox_restore", + "apply", + pending, + self.proxmox_restore_confirm_ttl_sec, + code=code, + ) + if isinstance(decision, dict): + if decision.get("clear_pending"): + self.state.pending_proxmox_restore = None + self.state.save() + return str(decision.get("message") or "Нет ожидающего восстановления Proxmox.") + if not decision.get("allowed"): + return str(decision.get("message") or "Неверный код подтверждения восстановления Proxmox.") + elif code.strip() != pending.confirm_code: return "Неверный код подтверждения восстановления Proxmox." if not self._snapshot_exists(pending.kind, pending.guest_id, pending.snapshot): self.state.pending_proxmox_restore = None @@ -1949,12 +2154,32 @@ class TSJGuardianBot: if not povpn: self._send_text(chat_id, "Нет ожидающего запроса на OpenVPN конфиг.") return - if povpn.stage != "awaiting_second_confirm": - self._send_text(chat_id, "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения.") - return - if code.strip() != povpn.confirm_code: - self._send_text(chat_id, "Неверный код второго подтверждения OpenVPN-конфига.") - return + decision = self._confirmation_decision( + "openvpn", + "apply", + povpn, + self.openvpn_config_confirm_ttl_sec, + code=code, + ) + if isinstance(decision, dict): + if decision.get("clear_pending"): + self.state.pending_openvpn_config = None + self.state.save() + self._send_text(chat_id, str(decision.get("message") or "Нет ожидающего запроса на OpenVPN конфиг.")) + return + if not decision.get("allowed"): + self._send_text( + chat_id, + str(decision.get("message") or "Неверный код второго подтверждения OpenVPN-конфига."), + ) + return + else: + if povpn.stage != "awaiting_second_confirm": + self._send_text(chat_id, "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения.") + return + if code.strip() != povpn.confirm_code: + self._send_text(chat_id, "Неверный код второго подтверждения OpenVPN-конфига.") + return try: filename, summary, config_bytes = self._prepare_openvpn_config(povpn) @@ -1999,10 +2224,25 @@ class TSJGuardianBot: ppc = self.state.pending_pfsense_change if not ppc: return "Нет ожидающего изменения pfSense." - if ppc.stage != "awaiting_second_confirm": - return "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения." - if code.strip() != ppc.confirm_code: - return "Неверный код второго подтверждения pfSense." + decision = self._confirmation_decision( + "pfsense", + "apply", + ppc, + self.pfsense_change_confirm_ttl_sec, + code=code, + ) + if isinstance(decision, dict): + if decision.get("clear_pending"): + self.state.pending_pfsense_change = None + self.state.save() + return str(decision.get("message") or "Нет ожидающего изменения pfSense.") + if not decision.get("allowed"): + return str(decision.get("message") or "Неверный код второго подтверждения pfSense.") + else: + if ppc.stage != "awaiting_second_confirm": + return "Второе подтверждение пока недоступно. Сначала выполните первый шаг подтверждения." + if code.strip() != ppc.confirm_code: + return "Неверный код второго подтверждения pfSense." result = self._run_pfsense_change_codex_exec(ppc) self.state.pending_pfsense_change = None @@ -2055,6 +2295,167 @@ class TSJGuardianBot: def _warning_signature(self, warnings: List[str]) -> str: return "\n".join(sorted(set(warnings))) + def _failure_signature(self, failures: List[str]) -> str: + return "\n".join(sorted(set(failures))) + + def _state_decision_payload(self) -> Dict: + return { + "pending_incident": asdict(self.state.pending_incident) if self.state.pending_incident else None, + "failure_streak_signature": self.state.failure_streak_signature, + "failure_streak_count": self.state.failure_streak_count, + "failure_streak_first_ts": self.state.failure_streak_first_ts, + } + + def _operator_action_state_payload(self) -> Dict: + return { + "pending_update_install_confirm": bool( + getattr(self.state, "pending_update_install_confirm", False) + ), + "pending_rollback_confirm": bool( + getattr(self.state, "pending_rollback_confirm", False) + ), + "pending_incident": asdict(self.state.pending_incident) + if getattr(self.state, "pending_incident", None) + else None, + } + + def _run_status_helper_json( + self, + args: List[str], + payload: Dict, + timeout_sec: int = 10, + ) -> Optional[Dict]: + status_bin_raw = (getattr(self, "tsj_guardian_status_bin", "") or "").strip() + if not status_bin_raw: + return None + status_bin = Path(status_bin_raw) + if not status_bin.exists(): + return None + try: + proc = self._run_argv( + [str(status_bin), *args], + timeout_sec=timeout_sec, + input_text=json.dumps(payload, ensure_ascii=False), + ) + if proc.returncode != 0: + self._log( + "WARN", + f"tsj-guardian-status decision failed rc={proc.returncode}: {((proc.stderr or '') + (proc.stdout or ''))[-500:]}", + ) + return None + return json.loads(proc.stdout or "{}") + except Exception as exc: + self._log("WARN", f"tsj-guardian-status decision unavailable: {exc}") + return None + + def _operator_action_decision(self, action: str) -> Optional[Dict]: + return self._run_status_helper_json( + ["--operator-action-decision"], + { + "action": action, + "state": self._operator_action_state_payload(), + }, + ) + + def _confirmation_decision( + self, + kind: str, + action: str, + pending: object, + ttl_seconds: int, + code: str = "", + ) -> Optional[Dict]: + state = asdict(pending) if pending else {} + return self._run_status_helper_json( + [ + "--confirmation-decision", + "--confirmation-ttl-seconds", + str(ttl_seconds), + "--now-epoch", + str(int(time.time())), + ], + { + "kind": kind, + "action": action, + "code": code, + "state": state, + }, + ) + + def _autoheal_plan_decision(self, failures: List[str], slo_stale: bool = False) -> Optional[Dict]: + return self._run_status_helper_json( + ["--autoheal-plan-decision"], + { + "failures": failures, + "slo_stale": bool(slo_stale), + }, + ) + + def _reset_failure_streak(self) -> None: + changed = bool( + self.state.failure_streak_signature + or self.state.failure_streak_count + or self.state.failure_streak_first_ts + ) + self.state.failure_streak_signature = "" + self.state.failure_streak_count = 0 + self.state.failure_streak_first_ts = 0 + if changed: + self.state.save() + + def _defer_transient_new_incident(self, failures: List[str]) -> bool: + threshold = getattr(self, "incident_failure_quorum_checks", 1) + decision = self._run_status_helper_json( + [ + "--incident-defer-decision", + "--incident-failure-quorum-checks", + str(threshold), + "--now-epoch", + str(int(time.time())), + ], + { + "failures": failures, + "state": self._state_decision_payload(), + }, + ) + if isinstance(decision, dict): + self.state.failure_streak_signature = str(decision.get("failure_streak_signature") or "") + self.state.failure_streak_count = int(decision.get("failure_streak_count") or 0) + self.state.failure_streak_first_ts = int(decision.get("failure_streak_first_ts") or 0) + if decision.get("reset_failure_streak"): + self.state.save() + self._reset_failure_streak() + else: + self.state.save() + if decision.get("defer"): + log_line = str(decision.get("log_line") or "") + self._log("WARN", log_line or f"Suppressing transient incident failures={len(failures)}") + return True + return False + + if threshold <= 1 or self._has_filesystem_critical(failures): + return False + + now = int(time.time()) + signature = self._failure_signature(failures) + if signature == self.state.failure_streak_signature: + self.state.failure_streak_count += 1 + else: + self.state.failure_streak_signature = signature + self.state.failure_streak_count = 1 + self.state.failure_streak_first_ts = now + self.state.save() + + if self.state.failure_streak_count < threshold: + self._log( + "WARN", + "Suppressing transient incident " + f"streak={self.state.failure_streak_count}/{threshold} " + f"failures={len(failures)}", + ) + return True + return False + def _filesystem_failures(self, failures: List[str]) -> List[str]: return [line for line in failures if "filesystem_usage" in line.lower()] @@ -2084,6 +2485,8 @@ class TSJGuardianBot: return False if key == "worktime": return text.startswith("- worktime(") or text.startswith("- worktime:") + if key == "slo": + return text.startswith("- slo-alert:") return text.startswith(f"- {key}:") def _aw_rus_failure_log_lines(self, lines: List[str], failures: List[str]) -> List[str]: @@ -2094,13 +2497,141 @@ class TSJGuardianBot: rendered.append(f"[FAIL] aw-rus:{key}: {summary}") return rendered + def _load_aw_rus_slo_summary(self) -> Optional[Dict]: + cmd = (getattr(self, "aw_rus_slo_summary_cmd", "") or "").strip() + if not cmd: + return None + rc, out = self._run_shell(cmd, timeout_sec=30) + if rc != 0: + self._log("WARN", f"AW-Rus SLO summary command failed rc={rc}: {out[-500:]}") + return None + try: + payload = json.loads(out) + except json.JSONDecodeError as exc: + self._log("WARN", f"AW-Rus SLO summary JSON parse failed: {exc}") + return None + return payload if isinstance(payload, dict) else None + + def _aw_rus_slo_lines_and_failures(self) -> Tuple[List[str], List[str]]: + if not getattr(self, "aw_rus_slo_enabled", True): + return [], [] + summary = self._load_aw_rus_slo_summary() + if not summary: + return [], [] + + lines = ["Проверка AW-Rus SLO:"] + failures: List[str] = [] + generated_raw = str(summary.get("generated_at_utc") or "") + age = None + try: + generated = datetime.fromisoformat(generated_raw.replace("Z", "+00:00")).astimezone(timezone.utc) + age = int((datetime.now(timezone.utc) - generated).total_seconds()) + except Exception: + generated = None + + if generated is None or age is None or age > self.aw_rus_slo_max_age_sec: + lines.append(f"- slo-summary: STALE age={age}s generated={generated_raw or 'unknown'}") + failures.append("slo") + return lines, failures + + windows = summary.get("windows") if isinstance(summary.get("windows"), dict) else {} + window_name = self.aw_rus_slo_alert_window if self.aw_rus_slo_alert_window in windows else "24h" + window = windows.get(window_name) if isinstance(windows.get(window_name), dict) else {} + samples = int(window.get("samples") or 0) + availability = window.get("availability_percent") + remaining = int(window.get("budget_remaining_seconds") or 0) + bad = int(window.get("bad_samples") or 0) + current = summary.get("current_sample") if isinstance(summary.get("current_sample"), dict) else {} + current_ok = bool(current.get("ok")) + lines.append( + f"- {window_name}: availability={availability if availability is not None else 'n/a'}% " + f"samples={samples} bad={bad} budget_remaining_seconds={remaining}" + ) + lines.append(f"- current_sample: {'OK' if current_ok else 'FAIL'} age={age}s") + + if samples < self.aw_rus_slo_min_samples: + lines.append(f"- slo-alert: WARMUP samples={samples}/{self.aw_rus_slo_min_samples}") + return lines, [] + if remaining < 0 and not current_ok: + lines.append(f"- slo-alert: FAIL error budget exhausted ({remaining}s)") + failures.append("slo") + elif remaining < 0: + lines.append(f"- slo-alert: RECOVERED error budget exhausted ({remaining}s), current sample OK") + return lines, failures + def _background_aw_rus_failures(self, failures: List[str]) -> Tuple[List[str], List[str]]: if not self._aw_rus_probe_should_run(failures): return [], [] + slo_lines, slo_failures = self._aw_rus_slo_lines_and_failures() + if slo_lines: + return self._aw_rus_failure_log_lines(slo_lines, slo_failures), slo_failures lines, aw_failures = self._aw_rus_dlp_probe() return self._aw_rus_failure_log_lines(lines, aw_failures), aw_failures + def _aw_rus_slo_status_line(self) -> str: + if getattr(self, "tsj_guardian_status_bin", ""): + status_bin = Path(self.tsj_guardian_status_bin) + if status_bin.exists(): + try: + proc = self._run_argv( + [ + str(status_bin), + "--aw-slo-status-line", + "--aw-slo-summary-command", + self.aw_rus_slo_summary_cmd, + "--aw-slo-alert-window", + self.aw_rus_slo_alert_window, + ], + timeout_sec=35, + ) + line = (proc.stdout or "").strip().splitlines() + if proc.returncode == 0 and line: + return line[-1] + if proc.returncode != 0: + self._log( + "WARN", + f"tsj-guardian-status aw-slo failed rc={proc.returncode}: {((proc.stderr or '') + (proc.stdout or ''))[-500:]}", + ) + except Exception as exc: + self._log("WARN", f"tsj-guardian-status aw-slo unavailable: {exc}") + + summary = self._load_aw_rus_slo_summary() + if not summary: + return "- aw_rus_slo: unavailable" + windows = summary.get("windows") if isinstance(summary.get("windows"), dict) else {} + window_name = self.aw_rus_slo_alert_window if self.aw_rus_slo_alert_window in windows else "24h" + window = windows.get(window_name) if isinstance(windows.get(window_name), dict) else {} + availability = window.get("availability_percent") + remaining = window.get("budget_remaining_seconds") + samples = window.get("samples") + status = window.get("status", "unknown") + current = summary.get("current_sample") if isinstance(summary.get("current_sample"), dict) else {} + current_ok = bool(current.get("ok")) + display_status = status + try: + remaining_value = int(remaining) + except (TypeError, ValueError): + remaining_value = 0 + if remaining_value < 0 and current_ok: + display_status = "recovered" + elif remaining_value < 0 and not current_ok: + display_status = "fail" + availability_text = "n/a" if availability is None else f"{float(availability):.5f}%" + return ( + f"- aw_rus_slo: {display_status} {window_name} current_sample={'OK' if current_ok else 'FAIL'} " + f"availability={availability_text} samples={samples} budget_remaining_seconds={remaining}" + ) + def _suggestions_from_failures(self, failures: List[str]) -> List[str]: + decision = self._run_status_helper_json( + ["--incident-suggestions"], + {"failures": failures}, + ) + if isinstance(decision, dict) and isinstance(decision.get("suggestions"), list): + suggestions = [str(item) for item in decision["suggestions"] if str(item)] + if suggestions: + return suggestions + suggestions = [] text = "\n".join(failures).lower() if "proxmox_api" in text: @@ -2201,6 +2732,7 @@ class TSJGuardianBot: if self.state.pending_incident: self._notify("Инцидент закрыт: система снова в норме.") self.state.pending_incident = None + self._reset_failure_streak() self._sync_warning_state(warnings) self.state.save() self._log("INFO", "Check OK") @@ -2221,6 +2753,8 @@ class TSJGuardianBot: now = int(time.time()) new_incident_created = False if not self.state.pending_incident: + if self._defer_transient_new_incident(failures): + return incident_id = time.strftime("%Y%m%d-%H%M%S") suggestions = self._suggestions_from_failures(failures) self.state.pending_incident = PendingIncident( @@ -2235,6 +2769,7 @@ class TSJGuardianBot: fallback_executed=False, ) new_incident_created = True + self._reset_failure_streak() else: self.state.pending_incident.failures = failures or self.state.pending_incident.failures self.state.pending_incident.suggestions = self._suggestions_from_failures( @@ -2380,6 +2915,10 @@ class TSJGuardianBot: if rc != 0: self._log("WARN", f"Incident codex exec returned rc={rc} but produced final message") return True + if self._is_codex_auth_error(out): + self._log("WARN", "Incident codex exec skipped after authentication failure; server fallback remains active") + self._notify(self._summarize_exec_error(out, rc)) + return True self._log("ERROR", f"Incident codex exec failed rc={rc}: {out[-2000:]}") return False @@ -2410,13 +2949,31 @@ class TSJGuardianBot: pi = self.state.pending_incident if not pi: return - if pi.operator_acked: - return now = int(time.time()) - if now - pi.created_ts < self.operator_timeout: - return + decision = self._run_status_helper_json( + [ + "--escalation-decision", + "--operator-timeout-seconds", + str(self.operator_timeout), + "--now-epoch", + str(now), + ], + {"state": self._state_decision_payload()}, + ) + if isinstance(decision, dict): + if not decision.get("timed_out"): + return + should_escalate = bool(decision.get("should_escalate")) + should_fallback = bool(decision.get("should_fallback")) + else: + if pi.operator_acked: + return + if now - pi.created_ts < self.operator_timeout: + return + should_escalate = not pi.escalated_to_ai + should_fallback = not pi.fallback_executed - if not pi.escalated_to_ai: + if should_escalate: ai_ok = self._escalate_to_ai() pi.escalated_to_ai = ai_ok self.state.save() @@ -2425,7 +2982,7 @@ class TSJGuardianBot: else: self._notify("Оператор не ответил. Эскалация не удалась.") - if not pi.fallback_executed: + if should_fallback: fallback_ok = self._run_server_fallback() pi.fallback_executed = True self.state.save() @@ -2564,8 +3121,105 @@ class TSJGuardianBot: "Резервные slash-команды: /status /check /aw_dlp_check /dlp_mode /dlp_mode_toggle /aw_dfir PACKAGE HOST [CASE_ID] [MODE] /heal /ack /resolve /run ... /openvpn_certs [filter] /openvpn_expiring /openvpn_config USER /openvpn_config_confirm /openvpn_config_cancel /openvpn_config_apply CODE /pfsense_confirm /pfsense_cancel /pfsense_apply CODE /proxmox_snapshot TARGET /proxmox_restore TARGET /proxmox_restore_apply CODE /proxmox_restore_cancel /proxmox_selection_cancel" ) + def _detmir_auto_status_line(self) -> str: + if self.tsj_guardian_status_bin: + status_bin = Path(self.tsj_guardian_status_bin) + if status_bin.exists(): + try: + proc = self._run_argv( + [ + str(status_bin), + "--state", + str(self.detmir_state_file), + ], + timeout_sec=5, + ) + line = (proc.stdout or "").strip().splitlines() + if proc.returncode == 0 and line: + return line[-1] + if proc.returncode != 0: + self._log( + "WARN", + f"tsj-guardian-status failed rc={proc.returncode}: {((proc.stderr or '') + (proc.stdout or ''))[-500:]}", + ) + except Exception as exc: + self._log("WARN", f"tsj-guardian-status unavailable: {exc}") + + try: + state = json.loads(self.detmir_state_file.read_text(encoding="utf-8")) + except FileNotFoundError: + return f"- detmir_auto: unavailable state_file={self.detmir_state_file}" + except Exception as exc: + return f"- detmir_auto: unavailable ({exc})" + + summary = state.get("detmir_summary") + if not isinstance(summary, dict): + check = state.get("check") if isinstance(state.get("check"), dict) else {} + summary = check.get("summary") if isinstance(check.get("summary"), dict) else {} + dlp_counts = state.get("dlp_counts") if isinstance(state.get("dlp_counts"), dict) else {} + severity = str(state.get("severity", "UNKNOWN")) + check_ok = state.get("check_ok") + if check_ok is None and isinstance(state.get("check"), dict): + check_ok = state["check"].get("ok") + dlp_ok = state.get("dlp_ok") + + return ( + f"- detmir_auto: {severity} check_ok={check_ok} dlp_ok={dlp_ok} " + f"bucket_stale={summary.get('bucket_stale', 'n/a')} " + f"bucket_dead={summary.get('bucket_dead', 'n/a')} " + f"service_fail={summary.get('service_failures', 'n/a')} " + f"service_warn={summary.get('service_warnings', 'n/a')} " + f"dlp_warn={dlp_counts.get('warn', 'n/a')} " + f"dlp_fail={dlp_counts.get('fail', 'n/a')}" + ) + + def _cmd_status_from_rust(self) -> Optional[str]: + if not getattr(self, "tsj_guardian_status_bin", ""): + return None + status_bin = Path(self.tsj_guardian_status_bin) + if not status_bin.exists(): + return None + pfsense_cmd = f"/usr/bin/python3 {shlex.quote(self.infra_admin_root)}/scripts/pfsense_security_status.py" + try: + proc = self._run_argv( + [ + str(status_bin), + "--status-text", + "--state", + str(self.detmir_state_file), + "--bot-state", + str(self.state_file), + "--rollback-file", + str(self.updates_rollback_file), + "--pfsense-status-command", + pfsense_cmd, + "--aw-slo-summary-command", + self.aw_rus_slo_summary_cmd, + "--aw-slo-alert-window", + self.aw_rus_slo_alert_window, + ], + timeout_sec=90, + ) + text = (proc.stdout or "").strip() + if proc.returncode == 0 and text: + return text + if proc.returncode != 0: + self._log( + "WARN", + f"tsj-guardian-status status-text failed rc={proc.returncode}: {((proc.stderr or '') + (proc.stdout or ''))[-500:]}", + ) + except Exception as exc: + self._log("WARN", f"tsj-guardian-status status-text unavailable: {exc}") + return None + def _cmd_status(self) -> str: + rust_status = self._cmd_status_from_rust() + if rust_status: + return rust_status + pfsense_status = self._pfsense_security_status_lines() + aw_slo_status = self._aw_rus_slo_status_line() + detmir_status = self._detmir_auto_status_line() pi = self.state.pending_incident if not pi: ppc = self.state.pending_pfsense_change @@ -2600,6 +3254,8 @@ class TSJGuardianBot: return ( "Статус: инцидентов нет.\n" f"{pfsense_status}\n" + f"{aw_slo_status}\n" + f"{detmir_status}\n" f"{ppc_line}\n" f"{ovpn_warn_line}\n" f"{pps_line}\n" @@ -2642,6 +3298,8 @@ class TSJGuardianBot: return ( f"Статус: активный инцидент {pi.incident_id}\n" f"{pfsense_status}\n" + f"{aw_slo_status}\n" + f"{detmir_status}\n" f"- возраст: {age}s\n" f"- autoheal attempts: {pi.autoheal_attempts}\n" f"- operator_acked: {pi.operator_acked}\n" @@ -2664,7 +3322,44 @@ class TSJGuardianBot: host = self.aw_rus_host now = datetime.now(timezone.utc) + def latest_bucket_event(bucket_id: str) -> Tuple[Optional[Dict], str]: + try: + r = requests.get(f"{base}/buckets/{bucket_id}/events?limit=20", timeout=20) + r.raise_for_status() + events = r.json() + if not isinstance(events, list) or not events: + return None, "no-events" + + latest_event = None + latest_ts = None + for event in events: + raw_ts = event.get("timestamp") if isinstance(event, dict) else None + if not raw_ts: + continue + try: + event_ts = datetime.fromisoformat(raw_ts.replace("Z", "+00:00")).astimezone(timezone.utc) + except Exception: + continue + if latest_ts is None or event_ts > latest_ts: + latest_ts = event_ts + latest_event = event + if latest_event is None: + return None, "no-valid-events" + return latest_event, str(latest_event.get("timestamp") or "unknown") + except Exception as exc: + return None, f"events-error:{exc}" + def bucket_age(bucket_id: str) -> Tuple[Optional[int], str]: + event, event_tail = latest_bucket_event(bucket_id) + if event: + end = str(event.get("timestamp") or "").strip() + try: + end_dt = datetime.fromisoformat(end.replace("Z", "+00:00")).astimezone(timezone.utc) + age = int((now - end_dt).total_seconds()) + return age, end + except Exception as exc: + return None, f"event-ts-error:{exc}" + try: r = requests.get(f"{base}/buckets/{bucket_id}", timeout=20) r.raise_for_status() @@ -2676,7 +3371,7 @@ class TSJGuardianBot: age = int((now - end_dt).total_seconds()) return age, end except Exception as exc: - return None, f"error:{exc}" + return None, f"{event_tail}; metadata-error:{exc}" def load_worktime_activity() -> Tuple[Optional[Dict[str, Dict[str, Optional[int]]]], Optional[str]]: try: @@ -2723,6 +3418,7 @@ class TSJGuardianBot: activity[bucket_host] = { "active": bool(latest_ts and latest_active and (age_seconds or 0) <= self.aw_rus_stale_sec), "age_seconds": age_seconds, + "fresh": bool(latest_ts and age_seconds is not None and age_seconds <= self.aw_rus_stale_sec), } return activity, None @@ -2733,6 +3429,33 @@ class TSJGuardianBot: current_host_activity = worktime_activity.get(host) if worktime_activity else None interactive_required = bool(current_host_activity and current_host_activity.get("active")) + guard_event, guard_tail = latest_bucket_event(f"aw-rus-collector-guard_{host}") + guard_data = (guard_event or {}).get("data") if guard_event else {} + guard_problems = guard_data.get("problems") if isinstance(guard_data, dict) else [] + guard_actions = guard_data.get("actions") if isinstance(guard_data, dict) else [] + guard_age = None + if guard_event: + try: + guard_ts = datetime.fromisoformat(str(guard_event["timestamp"]).replace("Z", "+00:00")).astimezone(timezone.utc) + guard_age = int((now - guard_ts).total_seconds()) + except Exception: + guard_age = None + guard_healthy = bool( + guard_age is not None + and guard_age <= max(300, self.aw_rus_stale_sec) + and isinstance(guard_data, dict) + and guard_data.get("status") == "ok" + and not guard_problems + ) + if guard_event: + mode = guard_data.get("mode", "unknown") if isinstance(guard_data, dict) else "unknown" + if guard_healthy: + lines.append(f"- collector-guard: OK age={guard_age}s mode={mode} actions={len(guard_actions or [])}") + else: + lines.append(f"- collector-guard: WARN age={guard_age}s mode={mode} tail={guard_tail}") + else: + lines.append(f"- collector-guard: WARN ({guard_tail})") + checks = [ (f"aw-watcher-window_{host}", "watcher-window"), (f"aw-watcher-afk_{host}", "watcher-afk"), @@ -2748,6 +3471,8 @@ class TSJGuardianBot: if age is None: if label.startswith("watcher-") and not interactive_required: lines.append(f"- {label}: WARN ({tail}; host inactive)") + elif label == "dlp-endpoint" and not interactive_required and guard_healthy: + lines.append(f"- {label}: WARN ({tail}; host inactive; guard healthy)") else: lines.append(f"- {label}: FAIL ({tail})") failures.append(label) @@ -2755,6 +3480,8 @@ class TSJGuardianBot: if age > self.aw_rus_stale_sec: if label.startswith("watcher-") and not interactive_required: lines.append(f"- {label}: WARN age={age}s end={tail} (host inactive)") + elif label == "dlp-endpoint" and not interactive_required and guard_healthy: + lines.append(f"- {label}: WARN age={age}s end={tail} (host inactive; guard healthy)") else: lines.append(f"- {label}: STALE age={age}s end={tail}") failures.append(label) @@ -2852,8 +3579,13 @@ class TSJGuardianBot: else: lines.append(f"- worktime({target}): OK active_seconds=0 (no active sessions)") except Exception as exc: - lines.append(f"- worktime: FAIL ({exc})") - failures.append("worktime") + if current_host_activity and current_host_activity.get("fresh"): + age_seconds = current_host_activity.get("age_seconds") + age_tail = f"; session bucket age={age_seconds}s" if age_seconds is not None else "" + lines.append(f"- worktime: WARN report unavailable ({exc}{age_tail})") + else: + lines.append(f"- worktime: FAIL ({exc})") + failures.append("worktime") return lines, failures @@ -2977,38 +3709,56 @@ class TSJGuardianBot: out = (proc.stdout or "") + (("\n" + proc.stderr) if proc.stderr else "") return proc.returncode, out.strip() - def _aw_rus_windows_collectors_heal(self, include_watchers: bool, include_worktime: bool) -> Tuple[bool, List[str]]: + def _aw_rus_windows_collectors_heal( + self, + include_watchers: bool, + include_worktime: bool, + include_dlp: bool = False, + ) -> Tuple[bool, List[str]]: report: List[str] = [] - if not (include_watchers or include_worktime): - report.append("- windows-heal: skipped (no watcher/worktime targets)") + if not (include_watchers or include_worktime or include_dlp): + report.append("- windows-heal: skipped (no Windows collector targets)") return True, report if not self.aw_rus_windows_ssh_password: report.append("- windows-heal: FAIL (AW_RUS_WINDOWS_SSH_PASSWORD not configured)") return False, report + task_items = ", ".join("'" + task.replace("'", "''") + "'" for task in self.aw_rus_windows_launch_tasks) + launch_tasks_expr = f"@({task_items})" if task_items else "@()" script = ( "$ErrorActionPreference = 'Stop'\n" "$ProgressPreference = 'SilentlyContinue'\n" f"$sessionCollectorScript = '{self.aw_rus_windows_session_collector_path}'\n" + f"$launchTasks = {launch_tasks_expr}\n" + "$stopSessionCollector = " + ("$true" if include_worktime else "$false") + "\n" + "$runRecovery = " + ("$true" if (include_watchers or include_worktime) else "$false") + "\n" + "if ($stopSessionCollector) {\n" "Get-CimInstance Win32_Process -ErrorAction SilentlyContinue |\n" " Where-Object { ($_.Name -ieq 'powershell.exe' -or $_.Name -ieq 'pwsh.exe') -and $_.CommandLine -and $_.CommandLine -match [Regex]::Escape($sessionCollectorScript) } |\n" " ForEach-Object { Stop-Process -Id ([int]$_.ProcessId) -Force -ErrorAction SilentlyContinue }\n" - f"if (Test-Path '{self.aw_rus_windows_hardening_recovery_path}') {{ try {{ & '{self.aw_rus_windows_hardening_recovery_path}' -ConfigPath '{self.aw_rus_windows_config_path}' }} catch {{ Write-Output ('HARDENING_RECOVERY_WARN: ' + $_.Exception.Message) }} }}\n" + "}\n" + "if ($runRecovery) {\n" + f" if (Test-Path '{self.aw_rus_windows_hardening_recovery_path}') {{ try {{ & '{self.aw_rus_windows_hardening_recovery_path}' -ConfigPath '{self.aw_rus_windows_config_path}' }} catch {{ Write-Output ('HARDENING_RECOVERY_WARN: ' + $_.Exception.Message) }} }}\n" + " Start-Sleep -Seconds 2\n" + " schtasks /Run /TN 'ActivityWatch Recovery' | Out-Null\n" + "}\n" "Start-Sleep -Seconds 2\n" - "schtasks /Run /TN 'ActivityWatch Recovery' | Out-Null\n" - "Start-Sleep -Seconds 2\n" - "Get-ScheduledTask -ErrorAction SilentlyContinue |\n" - " Where-Object { $_.TaskName -like 'ActivityWatch Launch *' } |\n" - " ForEach-Object { try { Start-ScheduledTask -TaskName $_.TaskName -ErrorAction Stop } catch {} }\n" - "Start-Sleep -Seconds 8\n" + "$allTasks = Get-ScheduledTask -ErrorAction SilentlyContinue\n" + "foreach ($taskName in $launchTasks) {\n" + " $task = $allTasks | Where-Object { $_.TaskName -eq $taskName } | Select-Object -First 1\n" + " if (-not $task) { Write-Output ('MISSING ' + $taskName); continue }\n" + " if ($task.State -eq 'Running') { Write-Output ('SKIP running ' + $taskName); continue }\n" + " try { schtasks /Run /TN $taskName | Out-String | Write-Output } catch { Write-Output ('START_WARN ' + $taskName + ': ' + $_.Exception.Message) }\n" + "}\n" + "Start-Sleep -Seconds 20\n" "Write-Output 'WATCHERS'\n" "Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue |\n" " Select-Object Name, Id, SessionId, StartTime |\n" " Sort-Object SessionId, Name |\n" " Format-Table -AutoSize\n" - "Write-Output 'WORKTIME'\n" + "Write-Output 'AW_POWERSHELL'\n" "Get-CimInstance Win32_Process -ErrorAction SilentlyContinue |\n" - " Where-Object { ($_.Name -ieq 'powershell.exe' -or $_.Name -ieq 'pwsh.exe') -and $_.CommandLine -and $_.CommandLine -match [Regex]::Escape($sessionCollectorScript) } |\n" + " Where-Object { ($_.Name -ieq 'powershell.exe' -or $_.Name -ieq 'pwsh.exe') -and $_.CommandLine -and $_.CommandLine -match 'AWatch-rus' } |\n" " Select-Object Name, ProcessId, SessionId, CommandLine |\n" " Format-List\n" ) @@ -3026,34 +3776,81 @@ class TSJGuardianBot: return True, report def _perform_aw_rus_autoheal(self, failures: List[str]) -> Tuple[bool, List[str], List[str], List[str]]: - watcher_failures = [item for item in failures if item.startswith("watcher-")] - dlp_failures = [item for item in failures if item.startswith("dlp-")] - worktime_failed = "worktime" in failures + if "slo" in failures: + slo_lines, slo_failures = self._aw_rus_slo_lines_and_failures() + stale = any(line.strip().startswith("- slo-summary: STALE") for line in slo_lines) + plan = self._autoheal_plan_decision(failures, slo_stale=stale) + if isinstance(plan, dict) and plan.get("slo_only"): + triggers = [str(item) for item in (plan.get("report_triggers") or []) if str(item)] + return False, triggers, slo_lines, slo_failures + trigger = ( + "- heal trigger: SLO summary stale, check aw-slo-monitor.timer/service" + if stale + else "- heal trigger: SLO error budget exhausted, no direct autoheal target" + ) + return False, [trigger], slo_lines, slo_failures + + plan = self._autoheal_plan_decision(failures) + if isinstance(plan, dict): + watcher_failed = bool(plan.get("include_watchers")) + worktime_failed = bool(plan.get("include_worktime")) + windows_dlp_failed = bool(plan.get("include_windows_dlp")) + server_dlp_failures = [str(item) for item in (plan.get("server_dlp_failures") or []) if str(item)] + run_windows_heal = bool(plan.get("run_windows_heal")) + run_server_dlp_heal = bool(plan.get("run_server_dlp_heal")) + run_worktime_heal = bool(plan.get("run_worktime_heal")) + report_triggers = [str(item) for item in (plan.get("report_triggers") or []) if str(item)] + sleep_after_seconds = int(plan.get("sleep_after_seconds") or 5) + else: + watcher_failures = [item for item in failures if item.startswith("watcher-")] + dlp_failures = [item for item in failures if item.startswith("dlp-")] + windows_dlp_failures = [item for item in dlp_failures if item in {"dlp-endpoint", "dlp-fileops-host"}] + server_dlp_failures = [item for item in dlp_failures if item not in set(windows_dlp_failures)] + worktime_failed = "worktime" in failures + watcher_failed = bool(watcher_failures) + windows_dlp_failed = bool(windows_dlp_failures) + run_windows_heal = watcher_failed or worktime_failed or windows_dlp_failed + run_server_dlp_heal = bool(server_dlp_failures) + run_worktime_heal = watcher_failed or worktime_failed + report_triggers = [] + if run_windows_heal: + report_triggers.append("- heal trigger: Windows session collectors degraded, starting remediation") + if run_server_dlp_heal: + report_triggers.append("- heal trigger: server-side DLP degraded, starting remediation") + if run_worktime_heal: + report_triggers.append("- heal trigger: worktime/watchers degraded, rebuilding server-side worktime views") + sleep_after_seconds = 30 if (watcher_failed or windows_dlp_failed) else 5 report: List[str] = [] heal_ok = True - if watcher_failures or worktime_failed: + if run_windows_heal: windows_ok, windows_lines = self._aw_rus_windows_collectors_heal( - include_watchers=bool(watcher_failures), + include_watchers=watcher_failed, include_worktime=worktime_failed, + include_dlp=windows_dlp_failed, ) heal_ok = heal_ok and windows_ok - report.append("- heal trigger: Windows session collectors degraded, starting remediation") + if report_triggers: + report.append(report_triggers.pop(0)) + else: + report.append("- heal trigger: Windows session collectors degraded, starting remediation") report.extend(windows_lines) - if dlp_failures: - dlp_ok, dlp_lines = self._aw_rus_dlp_heal(dlp_failures) + if run_server_dlp_heal: + dlp_ok, dlp_lines = self._aw_rus_dlp_heal(server_dlp_failures) heal_ok = heal_ok and dlp_ok - report.append("- heal trigger: DLP degraded, starting remediation") + trigger = next((item for item in report_triggers if "server-side DLP" in item), None) + report.append(trigger or "- heal trigger: server-side DLP degraded, starting remediation") report.extend(dlp_lines) - if watcher_failures or worktime_failed: + if run_worktime_heal: wt_ok, wt_lines = self._aw_rus_worktime_heal() heal_ok = heal_ok and wt_ok - report.append("- heal trigger: worktime/watchers degraded, rebuilding server-side worktime views") + trigger = next((item for item in report_triggers if "worktime/watchers" in item), None) + report.append(trigger or "- heal trigger: worktime/watchers degraded, rebuilding server-side worktime views") report.extend(wt_lines) - time.sleep(5) + time.sleep(sleep_after_seconds) after_lines, after_failures = self._aw_rus_dlp_probe() return heal_ok and not after_failures, report, after_lines, after_failures @@ -3187,7 +3984,7 @@ class TSJGuardianBot: return "\n".join(lines) def _pfsense_security_status_lines(self) -> str: - cmd = "/usr/bin/python3 /home/codex/infra-admin/scripts/pfsense_security_status.py" + cmd = f"/usr/bin/python3 {shlex.quote(self.infra_admin_root)}/scripts/pfsense_security_status.py" try: rc, out = self._run_shell(cmd, timeout_sec=40) except Exception as exc: @@ -3290,6 +4087,22 @@ class TSJGuardianBot: updated["_tsj_meta"] = meta return updated, changed, changed_rules + def _aw_dlp_policy_decision(self, policy: Dict, target_mode: str = "") -> Optional[Dict]: + decision = self._run_status_helper_json( + [ + "--dlp-policy-decision", + "--now-epoch", + str(int(time.time())), + ], + { + "policy": policy if isinstance(policy, dict) else {}, + "target_mode": target_mode, + }, + ) + if isinstance(decision, dict): + return decision + return None + def _aw_rus_windows_sync_dlp_policy(self, policy: Dict) -> Tuple[bool, List[str]]: report: List[str] = [] if not self.aw_rus_windows_ssh_password: @@ -3345,18 +4158,27 @@ class TSJGuardianBot: def _aw_dlp_policy_mode_text(self) -> str: active = self._aw_dlp_policy_request("GET", "/dlp/policies/active", timeout_sec=15) policy = active.get("policy") or {} - mode = self._aw_dlp_mode_from_policy(policy if isinstance(policy, dict) else {}) - details = [] - for group_name, rules in self._aw_dlp_toggle_rule_groups(policy if isinstance(policy, dict) else {}): - block_count = 0 - total_count = 0 - for rule in rules: - if not isinstance(rule, dict) or rule.get("enabled") is False: - continue - total_count += 1 - if str(rule.get("action") or "").strip().lower() == "block": - block_count += 1 - details.append(f"- {group_name}: block={block_count}/{total_count}") + decision = self._aw_dlp_policy_decision(policy if isinstance(policy, dict) else {}) + if isinstance(decision, dict): + mode = str(decision.get("current_mode") or "unknown") + details = [ + f"- {item.get('name')}: block={item.get('blocked', 0)}/{item.get('total', 0)}" + for item in (decision.get("groups") or []) + if isinstance(item, dict) + ] + else: + mode = self._aw_dlp_mode_from_policy(policy if isinstance(policy, dict) else {}) + details = [] + for group_name, rules in self._aw_dlp_toggle_rule_groups(policy if isinstance(policy, dict) else {}): + block_count = 0 + total_count = 0 + for rule in rules: + if not isinstance(rule, dict) or rule.get("enabled") is False: + continue + total_count += 1 + if str(rule.get("action") or "").strip().lower() == "block": + block_count += 1 + details.append(f"- {group_name}: block={block_count}/{total_count}") lines = [ "DLP режим:", f"- policy: {active.get('name') or '-'} (id={active.get('policyId') or '-'})", @@ -3376,9 +4198,23 @@ class TSJGuardianBot: if not isinstance(policy, dict): raise RuntimeError("Active DLP policy payload is invalid") - current_mode = self._aw_dlp_mode_from_policy(policy) - target_mode = "monitor" if current_mode in {"enforce", "mixed"} else "enforce" - updated_policy, changed_count, changed_rules = self._aw_dlp_policy_for_mode(policy, target_mode) + decision = self._aw_dlp_policy_decision(policy, target_mode="toggle") + if isinstance(decision, dict): + current_mode = str(decision.get("current_mode") or "unknown") + target_mode = str(decision.get("target_mode") or "") + updated_policy = decision.get("updated_policy") + changed_count = int(decision.get("changed_count") or 0) + changed_rules = [ + str(item) + for item in (decision.get("changed_rules") or []) + if str(item) + ] + if target_mode not in {"monitor", "enforce"} or not isinstance(updated_policy, dict): + raise RuntimeError("Rust DLP policy decision returned invalid toggle plan") + else: + current_mode = self._aw_dlp_mode_from_policy(policy) + target_mode = "monitor" if current_mode in {"enforce", "mixed"} else "enforce" + updated_policy, changed_count, changed_rules = self._aw_dlp_policy_for_mode(policy, target_mode) if changed_count == 0: return ( "DLP режим не изменён.\n" @@ -3416,7 +4252,19 @@ class TSJGuardianBot: return "\n".join(lines) def _run_operator_action(self, action: str) -> str: - action = action.strip().lower() + requested_action = action + decision = self._operator_action_decision(requested_action) + if isinstance(decision, dict): + if not bool(decision.get("allowed")): + return str(decision.get("message") or "Неизвестное действие.") + action = str(decision.get("canonical_action") or requested_action).strip().lower() + if action and action != requested_action.strip().lower(): + self._log( + "INFO", + f"Operator action routed by Rust: {requested_action!r} -> {action!r}", + ) + else: + action = requested_action.strip().lower() if action == "check": rc, out, started = self._run_check_script_once(timeout_sec=240) if not started: diff --git a/proxmox/tsj_guardian_watchdog.sh b/proxmox/tsj_guardian_watchdog.sh new file mode 100644 index 0000000..c0ea80b --- /dev/null +++ b/proxmox/tsj_guardian_watchdog.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +HEARTBEAT_FILE="${HEARTBEAT_FILE:-/opt/infra-admin/.state/tsj_guardian_heartbeat}" +MAX_AGE_SEC="${MAX_AGE_SEC:-180}" +SERVICE_NAME="${SERVICE_NAME:-tsj-guardian-bot.service}" +GOST_SERVICE_NAME="${GOST_SERVICE_NAME:-gost-tg.service}" +GOST_DUP_PATTERN="${GOST_DUP_PATTERN:-/usr/local/bin/gost -L http+socks5://127.0.0.1:11090 -F socks5+wss://gw.iri1968.dpdns.org:4443}" + +dedupe_gost_instances() { + local main_pid + main_pid="$(systemctl show -p MainPID --value "${GOST_SERVICE_NAME}" 2>/dev/null || true)" + mapfile -t pids < <(pgrep -f -- "${GOST_DUP_PATTERN}" 2>/dev/null || true) + if [[ "${#pids[@]}" -le 1 ]]; then + return 0 + fi + + local keep_pid="" + if [[ "${main_pid}" =~ ^[0-9]+$ ]] && [[ "${main_pid}" -gt 1 ]]; then + keep_pid="${main_pid}" + else + keep_pid="${pids[0]}" + fi + + for pid in "${pids[@]}"; do + [[ "${pid}" == "${keep_pid}" ]] && continue + kill -TERM "${pid}" 2>/dev/null || true + done + + sleep 2 + + for pid in "${pids[@]}"; do + [[ "${pid}" == "${keep_pid}" ]] && continue + kill -0 "${pid}" 2>/dev/null || continue + kill -KILL "${pid}" 2>/dev/null || true + done +} + +dedupe_gost_instances + +if [[ ! -f "${HEARTBEAT_FILE}" ]]; then + systemctl restart "${SERVICE_NAME}" + exit 0 +fi + +now="$(date +%s)" +hb="$(cat "${HEARTBEAT_FILE}" 2>/dev/null || printf '0')" +if [[ ! "${hb}" =~ ^[0-9]+$ ]]; then + systemctl restart "${SERVICE_NAME}" + exit 0 +fi + +age=$((now - hb)) +if [[ "${age}" -gt "${MAX_AGE_SEC}" ]]; then + systemctl restart "${SERVICE_NAME}" +fi diff --git a/scripts/aw-contour-smoke-10.10.10.2.sh b/scripts/aw-contour-smoke-10.10.10.2.sh index 798c507..64cd38c 100644 --- a/scripts/aw-contour-smoke-10.10.10.2.sh +++ b/scripts/aw-contour-smoke-10.10.10.2.sh @@ -3,6 +3,28 @@ set -uo pipefail +TARGET_ROOT="${CARGO_TARGET_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd)/adk-rust/target}" +RUST_BIN="${AW_CONTOUR_SMOKE_RUST:-}" +rust_candidates=() +if [ -n "$RUST_BIN" ]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/aw-contour-smoke" + "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." 2>/dev/null && pwd)/adk-rust/target/release/aw-contour-smoke" + "/usr/local/sbin/aw-contour-smoke" + "/usr/local/bin/aw-contour-smoke" +) + +for candidate in "${rust_candidates[@]}"; do + if [ -x "$candidate" ]; then + if [ "$#" -gt 0 ]; then + exec "$candidate" "$@" + fi + exec "$candidate" --mode proxmox-remote + fi +done + OK_COUNT=0 WARN_COUNT=0 FAIL_COUNT=0 diff --git a/scripts/aw-contour-smoke-local.sh b/scripts/aw-contour-smoke-local.sh index 1226443..be9e08e 100644 --- a/scripts/aw-contour-smoke-local.sh +++ b/scripts/aw-contour-smoke-local.sh @@ -19,6 +19,8 @@ ANSIBLE_DIR="$REPO_ROOT/ansible" INVENTORY="${AW_SMOKE_INVENTORY:-$ANSIBLE_DIR/inventory.ini}" REMOTE_SCRIPT_SRC="$REPO_ROOT/scripts/aw-contour-smoke-10.10.10.2.sh" REMOTE_SCRIPT_DST="${AW_SMOKE_REMOTE_SCRIPT:-/usr/local/sbin/aw-contour-smoke.sh}" +REMOTE_RUST_SRC="${AW_SMOKE_REMOTE_RUST_SRC:-${CARGO_TARGET_DIR:-$REPO_ROOT/adk-rust/target}/release/aw-contour-smoke}" +REMOTE_RUST_DST="${AW_SMOKE_REMOTE_RUST_BIN:-/usr/local/sbin/aw-contour-smoke}" AW_SERVER="${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}" WORKTIME_API="${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}" GRAFANA_URL="${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}" @@ -447,6 +449,18 @@ run_remote_proxmox_script() { fi section "Deploy Remote Script To 10.10.10.2" + if [ -x "$REMOTE_RUST_SRC" ]; then + if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_RUST_SRC dest=$REMOTE_RUST_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy-rust.$$ 2>&1; then + pass "remote Rust smoke deployed to $REMOTE_RUST_DST" + else + warn "remote Rust smoke deploy failed; shell fallback remains available" + sed 's/^/ /' /tmp/aw-smoke-copy-rust.$$ | head -120 + fi + rm -f /tmp/aw-smoke-copy-rust.$$ + else + skip "remote Rust smoke binary not found: $REMOTE_RUST_SRC" + fi + if ANSIBLE_NOCOLOR=1 ansible proxmox -i "$INVENTORY" -m copy -a "src=$REMOTE_SCRIPT_SRC dest=$REMOTE_SCRIPT_DST owner=root group=root mode=0755" >/tmp/aw-smoke-copy.$$ 2>&1; then pass "remote script deployed to $REMOTE_SCRIPT_DST" else diff --git a/scripts/aw-webui-browser-smoke.mjs b/scripts/aw-webui-browser-smoke.mjs new file mode 100644 index 0000000..08fe5bd --- /dev/null +++ b/scripts/aw-webui-browser-smoke.mjs @@ -0,0 +1,498 @@ +#!/usr/bin/env node +import { createRequire } from "node:module"; +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { promisify } from "node:util"; + +const require = createRequire(import.meta.url); +const execFileAsync = promisify(execFile); + +function loadPlaywright() { + const candidates = [ + "playwright", + "playwright-core", + process.env.PLAYWRIGHT_CORE_MODULE, + "/home/igor/.agents/skills/playwright/node_modules/playwright-core", + ].filter(Boolean); + const errors = []; + for (const candidate of candidates) { + try { + return require(candidate); + } catch (error) { + errors.push(`${candidate}: ${error.message}`); + } + } + return null; +} + +function firstExisting(candidates) { + return candidates.find((item) => item && fs.existsSync(item)) || ""; +} + +function env(name, fallback) { + const value = process.env[name]; + return value && value.trim() ? value.trim() : fallback; +} + +function normalizeBase(url) { + return url.replace(/\/+$/, ""); +} + +function safeName(value) { + return value.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-|-$/g, ""); +} + +function envInt(name, fallback) { + const value = Number(env(name, String(fallback))); + return Number.isFinite(value) ? value : fallback; +} + +function isRunDirectoryName(name) { + return /^20\d{2}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.test(name); +} + +function pruneOutputRuns(outRoot, keepRuns) { + if (keepRuns <= 0 || !fs.existsSync(outRoot)) { + return []; + } + const entries = fs + .readdirSync(outRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && isRunDirectoryName(entry.name)) + .map((entry) => entry.name) + .sort(); + const stale = entries.slice(0, Math.max(0, entries.length - keepRuns)); + for (const name of stale) { + fs.rmSync(path.join(outRoot, name), { recursive: true, force: true }); + } + return stale; +} + +function decodeHtmlEntities(value) { + return value + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + +function htmlToText(html) { + return decodeHtmlEntities( + html + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(), + ); +} + +function commandInPath(name) { + const dirs = (process.env.PATH || "").split(path.delimiter); + for (const dir of dirs) { + const candidate = path.join(dir, name); + if (fs.existsSync(candidate)) { + return candidate; + } + } + return ""; +} + +function findChromiumExecutable(explicitPath) { + return firstExisting([ + explicitPath, + "/home/igor/.cache/ms-playwright/chromium-1217/chrome-linux64/chrome", + "/home/igor/.cache/ms-playwright/chromium-1208/chrome-linux64/chrome", + "/home/igor/.cache/rod/browser/chromium-1321438/chrome", + commandInPath("chromium"), + commandInPath("chromium-browser"), + commandInPath("google-chrome"), + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/usr/bin/google-chrome", + ]); +} + +function isBenignConsoleError(text) { + return text.includes("Failed to load resource: the server responded with a status of 404 (Not Found)"); +} + +function isBenignRequestFailure(failure) { + return failure.error === "net::ERR_ABORTED"; +} + +async function waitForTextMarkers(page, markers, timeoutMs) { + if (!markers.length) { + return; + } + try { + await page.waitForFunction( + (expected) => { + const text = `${document.title}\n${document.body ? document.body.innerText : ""}`; + return expected.every((marker) => text.includes(marker)); + }, + markers, + { timeout: timeoutMs }, + ); + } catch { + // The final assertion below reports the exact missing markers. + } +} + +async function runPageCheck(browser, spec, runDir) { + const page = await browser.newPage({ + viewport: { width: 1366, height: 768 }, + locale: "ru-RU", + timezoneId: "Europe/Moscow", + }); + const started = Date.now(); + const consoleErrors = []; + const pageErrors = []; + const requestFailures = []; + const badResponses = []; + const responses = []; + + page.on("console", (message) => { + if (message.type() === "error") { + const text = message.text(); + if (!isBenignConsoleError(text)) { + consoleErrors.push(text); + } + } + }); + page.on("pageerror", (error) => { + pageErrors.push(error.message); + }); + page.on("requestfailed", (request) => { + const url = request.url(); + if (spec.relevantHosts.some((host) => url.startsWith(host))) { + const failure = { url, error: request.failure()?.errorText || "unknown" }; + if (!isBenignRequestFailure(failure)) { + requestFailures.push(failure); + } + } + }); + page.on("response", (response) => { + const url = response.url(); + if (!spec.relevantHosts.some((host) => url.startsWith(host))) { + return; + } + const item = { url, status: response.status() }; + responses.push(item); + if (response.status() >= 400) { + badResponses.push(item); + } + }); + + let status = null; + let title = ""; + let bodyText = ""; + let screenshot = ""; + let error = ""; + try { + const response = await page.goto(spec.url, { waitUntil: "commit", timeout: spec.timeoutMs }); + status = response ? response.status() : null; + await waitForTextMarkers(page, spec.requiredText, spec.renderTimeoutMs); + await page.waitForTimeout(spec.settleMs); + title = await page.title(); + bodyText = await page.locator("body").innerText({ timeout: 5000 }); + screenshot = path.join(runDir, `${safeName(spec.name)}.png`); + await page.screenshot({ path: screenshot, fullPage: true }); + } catch (caught) { + error = caught && caught.message ? caught.message : String(caught); + try { + screenshot = path.join(runDir, `${safeName(spec.name)}-failed.png`); + await page.screenshot({ path: screenshot, fullPage: true }); + } catch { + screenshot = ""; + } + } finally { + await page.close(); + } + + const visibleText = `${title}\n${bodyText}`; + const missingMarkers = spec.requiredText.filter((marker) => !visibleText.includes(marker)); + const ok = + !error && + status !== null && + status >= 200 && + status < 300 && + bodyText.length >= spec.minBodyText && + missingMarkers.length === 0 && + consoleErrors.length === 0 && + pageErrors.length === 0 && + requestFailures.length === 0 && + badResponses.length === 0; + + return { + engine: "playwright", + name: spec.name, + ok, + url: spec.url, + status, + title, + body_text_length: bodyText.length, + missing_markers: missingMarkers, + console_errors: consoleErrors, + page_errors: pageErrors, + request_failures: requestFailures, + bad_responses: badResponses, + response_count: responses.length, + latency_ms: Date.now() - started, + screenshot, + error, + }; +} + +async function runChromiumCliPageCheck(chromiumPath, spec, runDir) { + const started = Date.now(); + const screenshot = path.join(runDir, `${safeName(spec.name)}.png`); + const profileDir = path.join(runDir, `${safeName(spec.name)}-profile`); + const screenshotRequired = spec.cliScreenshotRequired !== false; + fs.mkdirSync(profileDir, { recursive: true }); + const browserArgs = [ + "--headless", + "--no-sandbox", + "--noerrdialogs", + "--disable-gpu", + "--disable-crash-reporter", + "--disable-crashpad", + "--disable-dev-shm-usage", + `--user-data-dir=${profileDir}`, + `--virtual-time-budget=${spec.renderTimeoutMs + spec.settleMs}`, + ]; + + let status = null; + let error = ""; + let fetchedText = ""; + try { + const controller = AbortSignal.timeout(spec.timeoutMs); + const response = await fetch(spec.url, { signal: controller }); + status = response.status; + fetchedText = await response.text(); + } catch (caught) { + error = caught && caught.message ? caught.message : String(caught); + } + + let html = ""; + try { + const { stdout } = await execFileAsync(chromiumPath, [...browserArgs, "--dump-dom", spec.url], { + timeout: spec.timeoutMs + spec.renderTimeoutMs + spec.settleMs + 10000, + maxBuffer: 10 * 1024 * 1024, + }); + html = stdout || ""; + } catch (caught) { + const message = caught && caught.message ? caught.message : String(caught); + error = error ? `${error}; ${message}` : message; + } + + if (screenshotRequired) { + try { + await execFileAsync( + chromiumPath, + [...browserArgs, "--window-size=1366,768", `--screenshot=${screenshot}`, spec.url], + { + timeout: spec.timeoutMs + spec.renderTimeoutMs + spec.settleMs + 10000, + maxBuffer: 1024 * 1024, + }, + ); + } catch (caught) { + const message = caught && caught.message ? caught.message : String(caught); + error = error ? `${error}; screenshot: ${message}` : `screenshot: ${message}`; + } + } + + const rawHtml = html || fetchedText; + const titleMatch = rawHtml.match(/]*>([\s\S]*?)<\/title>/i); + const title = titleMatch ? decodeHtmlEntities(titleMatch[1].trim()) : ""; + const bodyText = htmlToText(rawHtml); + const visibleText = `${title}\n${bodyText}\n${rawHtml}`; + const requiredText = spec.cliRequiredText || spec.requiredText; + const minBodyText = spec.cliMinBodyText || spec.minBodyText; + const missingMarkers = requiredText.filter((marker) => !visibleText.includes(marker)); + const ok = + !error && + status !== null && + status >= 200 && + status < 300 && + bodyText.length >= minBodyText && + missingMarkers.length === 0 && + (!screenshotRequired || fs.existsSync(screenshot)); + + return { + engine: "chromium-cli", + name: spec.name, + ok, + url: spec.url, + status, + title, + body_text_length: bodyText.length, + missing_markers: missingMarkers, + console_errors: [], + page_errors: [], + request_failures: [], + bad_responses: [], + response_count: 0, + latency_ms: Date.now() - started, + screenshot_required: screenshotRequired, + screenshot: fs.existsSync(screenshot) ? screenshot : "", + error, + }; +} + +async function runPageCheckWithRetries(runOnce, spec, maxRetries) { + let result = await runOnce(spec); + for (let attempt = 1; !result.ok && attempt <= maxRetries; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); + const retry = await runOnce({ ...spec, name: `${spec.name}_retry${attempt}` }); + retry.retry_of = spec.name; + retry.retry_attempt = attempt; + if (retry.ok) { + retry.name = spec.name; + retry.recovered_after_retry = attempt; + return retry; + } + result = retry; + result.retry_of = spec.name; + result.retry_attempt = attempt; + } + return result; +} + +async function main() { + const requestedEngine = env("AW_BROWSER_SMOKE_ENGINE", "auto"); + const playwright = requestedEngine === "chromium-cli" ? null : loadPlaywright(); + const awBase = normalizeBase(env("AW_BROWSER_SMOKE_AW_BASE", env("AW_SMOKE_AW_SERVER", "http://10.10.10.13:5600"))); + const worktimeBase = normalizeBase(env("AW_BROWSER_SMOKE_WORKTIME_BASE", env("AW_SMOKE_WORKTIME_API", "http://10.10.10.13:5610"))); + const host = env("AW_BROWSER_SMOKE_HOST", env("AW_SMOKE_SOURCE_HOSTNAME", "SHARKON2025")); + const timeoutMs = Number(env("AW_BROWSER_SMOKE_TIMEOUT_MS", "20000")); + const settleMs = Number(env("AW_BROWSER_SMOKE_SETTLE_MS", "6000")); + const renderTimeoutMs = Number(env("AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS", "15000")); + const pageRetries = Number(env("AW_BROWSER_SMOKE_PAGE_RETRIES", "1")); + const keepRuns = envInt("AW_BROWSER_SMOKE_KEEP_RUNS", 24); + const outRoot = env("AW_BROWSER_SMOKE_OUTPUT_DIR", path.resolve("output", "browser-smoke")); + const runId = new Date().toISOString().replace(/[:.]/g, "-"); + const runDir = path.join(outRoot, runId); + fs.mkdirSync(outRoot, { recursive: true }); + const prunedRuns = pruneOutputRuns(outRoot, keepRuns); + fs.mkdirSync(runDir, { recursive: true }); + + const executablePath = findChromiumExecutable(env( + "PLAYWRIGHT_CHROMIUM_EXECUTABLE", + "", + )); + + const launchOptions = { + headless: true, + args: ["--no-sandbox", "--disable-dev-shm-usage"], + }; + if (executablePath) { + launchOptions.executablePath = executablePath; + } + + const relevantHosts = [awBase, worktimeBase]; + const specs = [ + { + name: "aw_webui_home", + url: `${awBase}/`, + relevantHosts, + requiredText: ["Активность", "Windows RDP", host, "DLP"], + cliRequiredText: ["ActivityWatch", "ru-patch-v5.js", "aw-report-links"], + cliMinBodyText: 100, + cliScreenshotRequired: false, + minBodyText: 500, + timeoutMs, + settleMs, + renderTimeoutMs, + }, + { + name: "worktime_today_html", + url: `${worktimeBase}/reports/worktime/today?format=html&day=today&host=${encodeURIComponent(host)}&allow_stale=1`, + relevantHosts, + requiredText: ["AW-rus", "Отчёт", "RDP"], + minBodyText: 500, + timeoutMs, + settleMs: 1000, + renderTimeoutMs, + }, + { + name: "worktime_management_html", + url: `${worktimeBase}/reports/worktime/management?format=html&day=today&host=${encodeURIComponent(host)}&allow_stale=1`, + relevantHosts, + requiredText: ["AW-rus", "Управленческий", "RDP"], + minBodyText: 500, + timeoutMs, + settleMs: 1000, + renderTimeoutMs, + }, + ]; + + const pages = []; + if (playwright) { + const { chromium } = playwright; + const browser = await chromium.launch(launchOptions); + try { + for (const spec of specs) { + pages.push(await runPageCheckWithRetries((item) => runPageCheck(browser, item, runDir), spec, pageRetries)); + } + } finally { + await browser.close(); + } + } else { + if (requestedEngine === "playwright") { + throw new Error("AW_BROWSER_SMOKE_ENGINE=playwright requested, but Playwright could not be loaded"); + } + if (!executablePath) { + throw new Error("Unable to load Playwright and no Chromium executable found"); + } + for (const spec of specs) { + pages.push(await runPageCheckWithRetries((item) => runChromiumCliPageCheck(executablePath, item, runDir), spec, pageRetries)); + } + } + + const ok = pages.every((page) => page.ok); + const result = { + ok, + engine: pages[0]?.engine || "unknown", + generated_at_utc: new Date().toISOString(), + aw_base: awBase, + worktime_base: worktimeBase, + host, + output_dir: runDir, + retention: { + keep_runs: keepRuns, + pruned_runs: prunedRuns, + }, + pages, + }; + const jsonPath = path.join(runDir, "result.json"); + const latestPath = path.join(outRoot, "latest-result.json"); + result.result_json = jsonPath; + result.latest_result_json = latestPath; + fs.writeFileSync(jsonPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); + fs.writeFileSync(latestPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + return ok ? 0 : 2; +} + +main() + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + process.stdout.write( + `${JSON.stringify( + { + ok: false, + generated_at_utc: new Date().toISOString(), + error: error && error.message ? error.message : String(error), + }, + null, + 2, + )}\n`, + ); + process.exitCode = 2; + }); diff --git a/scripts/aw-webui-browser-smoke.sh b/scripts/aw-webui-browser-smoke.sh new file mode 100644 index 0000000..66d2670 --- /dev/null +++ b/scripts/aw-webui-browser-smoke.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +rust_candidates=( + "${AW_BROWSER_SMOKE_RUST:-}" + "${CARGO_TARGET_DIR:-}/release/aw-browser-smoke" + "$ROOT_DIR/adk-rust/target/release/aw-browser-smoke" + "/usr/local/bin/aw-browser-smoke" +) + +for rust_bin in "${rust_candidates[@]}"; do + if [[ -n "$rust_bin" && -x "$rust_bin" ]]; then + exec "$rust_bin" --root "$ROOT_DIR" -- "$@" + fi +done + +if [[ -z "${NODE_PATH:-}" && -d /home/igor/.agents/skills/playwright/node_modules ]]; then + export NODE_PATH=/home/igor/.agents/skills/playwright/node_modules +fi + +exec node "$ROOT_DIR/scripts/aw-webui-browser-smoke.mjs" "$@" diff --git a/scripts/build_dlp_ioc_from_hayabusa.sh b/scripts/build_dlp_ioc_from_hayabusa.sh index 7dcf42b..21e120e 100644 --- a/scripts/build_dlp_ioc_from_hayabusa.sh +++ b/scripts/build_dlp_ioc_from_hayabusa.sh @@ -13,6 +13,8 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" RULES_ROOT="${1:-/mnt/usb_hdd1/Projects/hayabusa/rules}" OUT_DIR="${2:-$REPO_ROOT/data/dlp-ioc}" +TARGET_ROOT="${CARGO_TARGET_DIR:-$REPO_ROOT/adk-rust/target}" +RUST_BIN="${EXTRACT_IOC_FROM_SIGMA_RUST:-}" if [[ ! -d "$RULES_ROOT" ]]; then echo "ERROR: rules root not found: $RULES_ROOT" >&2 @@ -21,9 +23,28 @@ fi mkdir -p "$OUT_DIR" +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/extract-ioc-from-sigma" + "$REPO_ROOT/adk-rust/target/release/extract-ioc-from-sigma" + "/usr/local/bin/extract-ioc-from-sigma" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + "$candidate" \ + --rules-root "$RULES_ROOT" \ + --out-dir "$OUT_DIR" + echo "IOC artifacts generated in: $OUT_DIR" + exit 0 + fi +done + python3 "$REPO_ROOT/scripts/extract_ioc_from_sigma.py" \ --rules-root "$RULES_ROOT" \ --out-dir "$OUT_DIR" echo "IOC artifacts generated in: $OUT_DIR" - diff --git a/scripts/check_detmir_rust_release_artifacts.sh b/scripts/check_detmir_rust_release_artifacts.sh new file mode 100644 index 0000000..1d7880c --- /dev/null +++ b/scripts/check_detmir_rust_release_artifacts.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RELEASE_DIR="$TARGET_ROOT/release" + +required_bins=( + detmir-status + detmir-adk-status + detmir-check + detmir-dlp + detmir-auto + detmir-heal-safe + tsj-guardian-watchdog + tsj-guardian-status + aw-rus-healthd + aw-db-health + aw-db-maintenance + aw-health-check + aw-prune-local-state + check-aw-data + check-aw-full + check-install-kit-vs-repo + validate-install-kit + verify-innosetup-installer + rebuild-install-kit + quality-gate + extract-ioc-from-sigma + merge-aw-server-dbs + prod-backup-restore + prod-rollout + rdp-worktime-report + aw-contour-smoke + aw-browser-smoke + diag-and-manual-restart + dlp-health-check + dlp-content-analyzer + dlp-admin-cli + dlp-policy-engine + dlp-case-management + dlp-compliance + dlp-aggregator + dlp-syslog-forwarder + dlp-webhook-sender + dlp-cef-exporter + dlp-influx-exporter + worktime-influx-exporter + worktime-prewarm + worktime-ui-bridge + worktime-autoheal + worktime-api + aw-slo-monitor + aw-hayabusa-case-alert-rust + aw-hayabusa-link-case-rust + aw-hayabusa-from-windows-rust + aw-hayabusa-autoprocess-rust +) + +missing=0 +for bin in "${required_bins[@]}"; do + if [[ -x "$RELEASE_DIR/$bin" ]]; then + printf 'OK %s\n' "$bin" + else + printf 'MISSING %s (%s)\n' "$bin" "$RELEASE_DIR/$bin" >&2 + missing=1 + fi +done + +if (( missing != 0 )); then + cat >&2 < str: return value if value not in (None, "") else default -def _http_json(url: str, timeout: int = 10) -> Any: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) +def _http_json(url: str, timeout: int = 15, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: + last_exc: Exception | None = None + for attempt in range(max(1, attempts)): + try: + with request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception as exc: + last_exc = exc + if attempt + 1 >= max(1, attempts): + break + time.sleep(backoff_seconds * (2**attempt)) + assert last_exc is not None + raise last_exc def _parse_ts(value: str | None) -> datetime | None: @@ -42,6 +53,76 @@ def _age_seconds(ts: datetime | None, now: datetime) -> int | None: return max(0, int((now - ts).total_seconds())) +def _int_or_zero(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _path_tail(value: Any, parts: int = 2) -> str: + text = str(value or "").replace("\\", "/").strip() + if not text: + return "" + tokens = [item for item in text.split("/") if item] + return "/".join(tokens[-parts:]) if tokens else text + + +def _text_excerpt(value: Any, limit: int = 120) -> str: + text = " ".join(str(value or "").split()) + if len(text) <= limit: + return text + return text[: max(0, limit - 1)].rstrip() + "…" + + +def _load_counter_state(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {"counters": {}} + except Exception: + return {"counters": {}} + if not isinstance(payload, dict): + return {"counters": {}} + counters = payload.get("counters") + if not isinstance(counters, dict): + payload["counters"] = {} + return payload + + +def _save_counter_state(path: Path, state: dict[str, Any]) -> str | None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.parent / f".{path.name}.{os.getpid()}.tmp" + tmp_path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(tmp_path, path) + return None + except Exception as exc: + try: + path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return None + except Exception: + return str(exc) + + +def _counter_delta(counter_state: dict[str, Any] | None, key: str, current_value: int) -> tuple[int | None, int]: + if counter_state is None: + return None, current_value + counters = counter_state.setdefault("counters", {}) + if not isinstance(counters, dict): + counters = {} + counter_state["counters"] = counters + previous: int | None = None + try: + previous = int(counters[key]) + except (KeyError, TypeError, ValueError): + previous = None + counters[key] = current_value + if previous is None or current_value < previous: + return previous, 0 + return previous, current_value - previous + + def _run_systemctl(*args: str) -> tuple[int, str]: proc = subprocess.run( ["systemctl", *args], @@ -268,6 +349,134 @@ def check_incident_buckets( ) +def check_incident_runtime( + report: HealthReport, + api_base: str, + buckets: dict[str, Any], + sample_limit: int = 20, +) -> None: + now = _now_utc() + prefix = "aw-dlp-incidents_" + matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) + if not matched: + report.add("incident-runtime", "ok", "no incident buckets to sample", bucket_count=0) + return + + if sample_limit <= 0: + metadata = [] + for bucket_id in matched: + ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) + metadata.append( + { + "bucket": bucket_id, + "end": ts.isoformat().replace("+00:00", "Z") if ts else None, + "age_seconds": _age_seconds(ts, now), + } + ) + report.add( + "incident-runtime", + "ok", + "incident event sampling disabled", + bucket_count=len(matched), + sample_limit=sample_limit, + metadata=metadata, + ) + return + + sampled: list[dict[str, Any]] = [] + latest_incidents: list[dict[str, Any]] = [] + read_failed: list[dict[str, str]] = [] + totals = { + "sampled_events": 0, + "real_incidents": 0, + "self_tests": 0, + } + severity_counts: dict[str, int] = {} + action_counts: dict[str, int] = {} + rule_counts: dict[str, int] = {} + + for bucket_id in matched: + try: + events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}", timeout=5, attempts=1) + except Exception as exc: + read_failed.append({"bucket": bucket_id, "error": str(exc)}) + continue + if not isinstance(events, list): + read_failed.append({"bucket": bucket_id, "error": "events response is not a list"}) + continue + + bucket_summary = { + "bucket": bucket_id, + "sampled_events": len(events), + "real_incidents": 0, + "self_tests": 0, + } + sampled.append(bucket_summary) + totals["sampled_events"] += len(events) + + for event in events: + data = event.get("data") or {} + if not isinstance(data, dict): + continue + + signal_type = str(data.get("signalType") or "").strip() + source = str(data.get("source") or "").strip() + rule_id = str(data.get("ruleId") or data.get("rule_id") or "").strip() + is_self_test = signal_type == "self_test" or source == "self-test" or rule_id.startswith("selftest-") + if is_self_test: + bucket_summary["self_tests"] += 1 + totals["self_tests"] += 1 + continue + + ts = _parse_ts(event.get("timestamp")) + severity = str(data.get("severity") or "unknown").strip().lower() or "unknown" + action = str(data.get("action") or "unknown").strip().lower() or "unknown" + rule_key = rule_id or "unknown" + severity_counts[severity] = severity_counts.get(severity, 0) + 1 + action_counts[action] = action_counts.get(action, 0) + 1 + rule_counts[rule_key] = rule_counts.get(rule_key, 0) + 1 + bucket_summary["real_incidents"] += 1 + totals["real_incidents"] += 1 + latest_incidents.append( + { + "bucket": bucket_id, + "timestamp": event.get("timestamp"), + "age_seconds": _age_seconds(ts, now), + "ruleId": rule_id, + "severity": severity, + "action": action, + "username": str(data.get("username") or ""), + "hostname": str(data.get("hostname") or ""), + "source": source, + "message_excerpt": _text_excerpt(data.get("message")), + } + ) + + latest_incidents.sort(key=lambda item: item.get("timestamp") or "", reverse=True) + status = "ok" + summary = f"{totals['real_incidents']} real incidents in sampled events" + if read_failed: + status = "warn" + summary = f"{len(read_failed)} incident buckets failed to sample" + elif totals["real_incidents"] == 0: + summary = "no real incidents in sampled events" + + report.add( + "incident-runtime", + status, + summary, + bucket_count=len(matched), + sample_limit=sample_limit, + totals=totals, + sampled=sampled, + severity_counts=severity_counts, + action_counts=action_counts, + rule_counts=dict(sorted(rule_counts.items(), key=lambda item: (-item[1], item[0]))[:10]), + latest_incidents=latest_incidents[:5], + read_failed=read_failed, + ) + + def _worktime_activity_map(api_base: str, buckets: dict[str, Any], max_age_seconds: int) -> dict[str, dict[str, Any]]: now = _now_utc() activity: dict[str, dict[str, Any]] = {} @@ -377,8 +586,239 @@ def check_file_operations_buckets( ) -def check_endpoint_self_test_metrics(report: HealthReport, api_base: str, buckets: dict[str, Any]) -> None: +def check_file_operations_runtime( + report: HealthReport, + api_base: str, + buckets: dict[str, Any], + sample_limit: int = 20, + queue_warn_depth: int = 100, + send_failure_warn_count: int = 1, + counter_state: dict[str, Any] | None = None, +) -> None: + now = _now_utc() + prefix = "aw-file-operations_" + matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) + if not matched: + report.add("file-operations-runtime", "warn", "no file-operations buckets to sample", bucket_count=0) + return + + sampled: list[dict[str, Any]] = [] + latest_operations: list[dict[str, Any]] = [] + latest_health: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] + read_failed: list[dict[str, str]] = [] + + for bucket_id in matched: + try: + events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}") + except Exception as exc: + read_failed.append({"bucket": bucket_id, "error": str(exc)}) + continue + if not isinstance(events, list): + read_failed.append({"bucket": bucket_id, "error": "events response is not a list"}) + continue + + operation_counts: dict[str, int] = {} + latest_health_event: dict[str, Any] | None = None + latest_health_ts: datetime | None = None + + for event in events: + data = event.get("data") or {} + if not isinstance(data, dict): + continue + ts = _parse_ts(event.get("timestamp")) + signal_type = str(data.get("signalType") or "") + operation = str(data.get("operation") or "") + + if signal_type == "collector_health": + if latest_health_event is None or (ts is not None and (latest_health_ts is None or ts > latest_health_ts)): + latest_health_event = event + latest_health_ts = ts + continue + + if operation: + operation_counts[operation] = operation_counts.get(operation, 0) + 1 + latest_operations.append( + { + "bucket": bucket_id, + "timestamp": event.get("timestamp"), + "age_seconds": _age_seconds(ts, now), + "operation": operation, + "username": str(data.get("username") or ""), + "hostname": str(data.get("hostname") or ""), + "extension": str(data.get("extension") or ""), + "archiveHint": bool(data.get("archiveHint")), + "path_tail": _path_tail(data.get("path")), + "size": _int_or_zero(data.get("size")), + } + ) + + bucket_sample = { + "bucket": bucket_id, + "sampled_events": len(events), + "operation_counts": operation_counts, + } + sampled.append(bucket_sample) + + if latest_health_event is None: + warnings.append({"bucket": bucket_id, "metric": "collector_health", "value": "missing_in_sample"}) + continue + + health_data = latest_health_event.get("data") or {} + send_failures = _int_or_zero(health_data.get("sendFailures")) + previous_send_failures, send_failures_delta = _counter_delta( + counter_state, + f"file-operations:{bucket_id}:sendFailures", + send_failures, + ) + health_item = { + "bucket": bucket_id, + "timestamp": latest_health_event.get("timestamp"), + "age_seconds": _age_seconds(latest_health_ts, now), + "queueDepth": _int_or_zero(health_data.get("queueDepth")), + "eventsEnqueued": _int_or_zero(health_data.get("eventsEnqueued")), + "eventsFlushed": _int_or_zero(health_data.get("eventsFlushed")), + "sendFailures": send_failures, + "sendFailuresPrevious": previous_send_failures, + "sendFailuresDelta": send_failures_delta, + "username": str(health_data.get("username") or ""), + "hostname": str(health_data.get("hostname") or ""), + "sessionId": _int_or_zero(health_data.get("sessionId")), + } + latest_health.append(health_item) + if health_item["queueDepth"] > queue_warn_depth: + warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": health_item["queueDepth"], "threshold": queue_warn_depth}) + if send_failure_warn_count > 0 and health_item["sendFailuresDelta"] >= send_failure_warn_count: + warnings.append( + { + "bucket": bucket_id, + "metric": "sendFailuresDelta", + "value": health_item["sendFailuresDelta"], + "current": health_item["sendFailures"], + "previous": health_item["sendFailuresPrevious"], + "threshold": send_failure_warn_count, + } + ) + + latest_operations.sort(key=lambda item: item.get("timestamp") or "", reverse=True) + status = "ok" + summary = f"{len(matched)} file-operations buckets sampled" + if read_failed: + status = "warn" + summary = f"{len(read_failed)} file-operations buckets failed to sample" + elif warnings: + status = "warn" + summary = "file-operations runtime counters outside expectations" + + report.add( + "file-operations-runtime", + status, + summary, + bucket_count=len(matched), + sample_limit=sample_limit, + sampled=sampled, + latest_health=latest_health, + latest_operations=latest_operations[:5], + warnings=warnings, + read_failed=read_failed, + thresholds={ + "queueDepth": queue_warn_depth, + "sendFailures": send_failure_warn_count, + }, + ) + + +def check_endpoint_signal_buckets( + report: HealthReport, + api_base: str, + buckets: dict[str, Any], + max_age_seconds: int, +) -> None: + now = _now_utc() + prefix = "aw-dlp-endpoint-signals_" + matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) + worktime = _worktime_activity_map(api_base, buckets, max_age_seconds) + active_hosts = sorted(host for host, meta in worktime.items() if meta.get("active")) + matched_by_host = {_bucket_suffix(bucket_id, prefix): bucket_id for bucket_id in matched} + + ignored_unmanaged: list[str] = [] + ignored_inactive: list[str] = [] + missing_active: list[str] = [] + stale: list[dict[str, Any]] = [] + unknown: list[str] = [] + fresh: list[str] = [] + + for host, bucket_id in matched_by_host.items(): + if host not in worktime: + ignored_unmanaged.append(bucket_id) + continue + if host not in active_hosts: + ignored_inactive.append(bucket_id) + continue + ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) + age = _age_seconds(ts, now) + if age is None: + unknown.append(bucket_id) + continue + if age > max_age_seconds: + stale.append({"bucket": bucket_id, "age_seconds": age}) + else: + fresh.append(bucket_id) + + for host in active_hosts: + if host not in matched_by_host: + missing_active.append(host) + + if not active_hosts: + report.add( + "buckets:endpoint-signals", + "ok", + "no active managed hosts require endpoint-signals freshness", + active_hosts=[], + ignored_unmanaged=ignored_unmanaged, + ignored_inactive=ignored_inactive, + worktime_hosts=sorted(worktime), + ) + return + + status = "ok" + summary = f"{len(fresh)} active host buckets fresh" + if missing_active: + status = "fail" + summary = f"{len(missing_active)} active hosts missing endpoint-signals buckets" + elif stale: + status = "fail" + summary = f"{len(stale)} active host buckets stale" + elif unknown: + status = "warn" + summary = f"{len(unknown)} active host buckets without timestamp" + + report.add( + "buckets:endpoint-signals", + status, + summary, + active_hosts=active_hosts, + fresh=fresh, + stale=stale, + missing_active=missing_active, + unknown=unknown, + ignored_unmanaged=ignored_unmanaged, + ignored_inactive=ignored_inactive, + ) + + +def check_endpoint_self_test_metrics( + report: HealthReport, + api_base: str, + buckets: dict[str, Any], + queue_warn_depth: int = 100, + send_failure_warn_count: int = 1, + counter_state: dict[str, Any] | None = None, +) -> None: + now = _now_utc() missing: list[str] = [] + latest_self_tests: list[dict[str, Any]] = [] + warnings: list[dict[str, Any]] = [] expected = ("queueDepth", "eventsEnqueued", "eventsFlushed", "sendFailures") for bucket_id in sorted(k for k in buckets if k.startswith("aw-dlp-endpoint-signals_")): try: @@ -386,20 +826,89 @@ def check_endpoint_self_test_metrics(report: HealthReport, api_base: str, bucket except Exception as exc: report.add(f"endpoint-self-test:{bucket_id}", "warn", f"failed to read events: {exc}", bucket=bucket_id) continue - found = False + latest_event: dict[str, Any] | None = None + latest_ts: datetime | None = None if isinstance(events, list): for event in events: data = event.get("data") or {} - if data.get("signalType") == "self_test" and all(key in data for key in expected): - found = True - break - if not found: + if data.get("signalType") != "self_test" or not all(key in data for key in expected): + continue + ts = _parse_ts(event.get("timestamp")) + if latest_event is None or (ts is not None and (latest_ts is None or ts > latest_ts)): + latest_event = event + latest_ts = ts + if latest_event is None: missing.append(bucket_id) + continue + + data = latest_event.get("data") or {} + send_failures = _int_or_zero(data.get("sendFailures")) + previous_send_failures, send_failures_delta = _counter_delta( + counter_state, + f"endpoint-self-test:{bucket_id}:sendFailures", + send_failures, + ) + item = { + "bucket": bucket_id, + "timestamp": latest_event.get("timestamp"), + "age_seconds": _age_seconds(latest_ts, now), + "queueDepth": _int_or_zero(data.get("queueDepth")), + "eventsEnqueued": _int_or_zero(data.get("eventsEnqueued")), + "eventsFlushed": _int_or_zero(data.get("eventsFlushed")), + "sendFailures": send_failures, + "sendFailuresPrevious": previous_send_failures, + "sendFailuresDelta": send_failures_delta, + } + latest_self_tests.append(item) + if item["queueDepth"] > queue_warn_depth: + warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": item["queueDepth"], "threshold": queue_warn_depth}) + if send_failure_warn_count > 0 and item["sendFailuresDelta"] >= send_failure_warn_count: + warnings.append( + { + "bucket": bucket_id, + "metric": "sendFailuresDelta", + "value": item["sendFailuresDelta"], + "current": item["sendFailures"], + "previous": item["sendFailuresPrevious"], + "threshold": send_failure_warn_count, + } + ) if missing: - report.add("endpoint-self-test-metrics", "warn", "missing transport metrics in recent self_test events", buckets=missing) + report.add( + "endpoint-self-test-metrics", + "warn", + "missing transport metrics in sampled self_test events", + buckets=missing, + latest_self_tests=latest_self_tests, + thresholds={ + "queueDepth": queue_warn_depth, + "sendFailures": send_failure_warn_count, + }, + ) + elif warnings: + report.add( + "endpoint-self-test-metrics", + "warn", + "endpoint transport counters outside thresholds", + latest_self_tests=latest_self_tests, + warnings=warnings, + thresholds={ + "queueDepth": queue_warn_depth, + "sendFailures": send_failure_warn_count, + }, + ) else: - report.add("endpoint-self-test-metrics", "ok", "recent self_test metrics present") + report.add( + "endpoint-self-test-metrics", + "ok", + "self_test transport metrics present", + latest_self_tests=latest_self_tests, + thresholds={ + "queueDepth": queue_warn_depth, + "sendFailures": send_failure_warn_count, + }, + ) def check_compliance_reports(report: HealthReport, report_dir: Path, profiles: list[str], month: str) -> None: @@ -425,6 +934,13 @@ def main() -> int: parser.add_argument("--case-server", default=_env("AW_HEALTH_CASE_SERVER", "http://127.0.0.1:5602")) parser.add_argument("--max-age-seconds", type=int, default=int(_env("AW_HEALTH_MAX_AGE_SECONDS", "900"))) parser.add_argument("--strict-fileops", action="store_true", default=_env("AW_HEALTH_STRICT_FILEOPS", "0").lower() in {"1", "true", "yes", "on"}) + parser.add_argument("--endpoint-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_QUEUE_WARN_DEPTH", "100"))) + parser.add_argument("--endpoint-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_SEND_FAILURE_WARN_COUNT", "1"))) + parser.add_argument("--fileops-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SAMPLE_LIMIT", "20"))) + parser.add_argument("--fileops-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_QUEUE_WARN_DEPTH", "100"))) + parser.add_argument("--fileops-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SEND_FAILURE_WARN_COUNT", "1"))) + parser.add_argument("--incident-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_INCIDENT_SAMPLE_LIMIT", "0"))) + parser.add_argument("--state-dir", default=_env("AW_DLP_HEALTH_STATE_DIR", "/var/lib/activitywatch/health")) parser.add_argument("--report-dir", default=_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports")) parser.add_argument("--profiles", default=_env("AW_DLP_COMPLIANCE_PROFILES", "152-fz,pci-dss")) parser.add_argument("--json", action="store_true") @@ -432,6 +948,8 @@ def main() -> int: report = HealthReport() aw_api_base = args.aw_server.rstrip("/") + "/api/0" + counter_state_path = Path(args.state_dir) / "dlp-health-check-counters.json" + counter_state = _load_counter_state(counter_state_path) check_http_endpoint(report, "http:aw", f"{aw_api_base}/info") check_http_endpoint(report, "http:policy", args.policy_server.rstrip("/") + "/healthz") @@ -461,13 +979,34 @@ def main() -> int: if not isinstance(buckets, dict): raise RuntimeError("bucket list is not a dict") report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets)) - check_bucket_group(report, aw_api_base, buckets, "endpoint-signals", "aw-dlp-endpoint-signals_", args.max_age_seconds) + check_endpoint_signal_buckets(report, aw_api_base, buckets, args.max_age_seconds) check_file_operations_buckets(report, aw_api_base, buckets, args.max_age_seconds, args.strict_fileops) + check_file_operations_runtime( + report, + aw_api_base, + buckets, + args.fileops_sample_limit, + args.fileops_queue_warn_depth, + args.fileops_send_failure_warn_count, + counter_state, + ) check_incident_buckets(report, aw_api_base, buckets, args.max_age_seconds * 24) - check_endpoint_self_test_metrics(report, aw_api_base, buckets) + check_incident_runtime(report, aw_api_base, buckets, args.incident_sample_limit) + check_endpoint_self_test_metrics( + report, + aw_api_base, + buckets, + args.endpoint_queue_warn_depth, + args.endpoint_send_failure_warn_count, + counter_state, + ) except Exception as exc: report.add("aw:buckets-index", "fail", f"failed to inspect bucket index: {exc}") + state_error = _save_counter_state(counter_state_path, counter_state) + if state_error: + report.add("state:counters", "warn", f"failed to save counter baseline: {state_error}", path=str(counter_state_path)) + month = _now_utc().strftime("%Y-%m") profiles = [x.strip() for x in args.profiles.split(",") if x.strip()] check_compliance_reports(report, Path(args.report_dir), profiles, month) diff --git a/scripts/legacy/aw-prune-local-state.sh b/scripts/legacy/aw-prune-local-state.sh new file mode 100644 index 0000000..c2c6afd --- /dev/null +++ b/scripts/legacy/aw-prune-local-state.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +AW_DATA_DIR="${AW_DATA_DIR:-/var/lib/activitywatch}" +BACKUP_DIR="${AW_BACKUP_DIR:-${AW_DATA_DIR}/backups}" +KEEP_DAYS="${AW_BACKUP_RETENTION_DAYS:-7}" +KEEP_LAST_DB="${AW_BACKUP_KEEP_LAST_DB:-2}" +KEEP_LAST_JSON="${AW_BACKUP_KEEP_LAST_JSON:-2}" + +prune_group() { + local keep_last="$1" + local keep_days="$2" + shift 2 + local files=() + local idx=0 + local cutoff + cutoff="$(date -d "-${keep_days} days" +%s)" + mapfile -t files < <(find "$@" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk '{ $1=""; sub(/^ /,""); print }') + for path in "${files[@]}"; do + idx=$((idx + 1)) + if [ "$idx" -le "$keep_last" ]; then + continue + fi + [ -f "$path" ] || continue + if [ "$(stat -c %Y "$path")" -lt "$cutoff" ]; then + rm -f -- "$path" + fi + done +} + +mkdir -p "$BACKUP_DIR" + +prune_group "$KEEP_LAST_DB" "$KEEP_DAYS" "${BACKUP_DIR}/db" +prune_group "$KEEP_LAST_JSON" "$KEEP_DAYS" "$BACKUP_DIR" + +find /tmp -maxdepth 1 -type f \ + \( -name 'activitywatch-*.zip' -o -name 'hayabusa-*.zip' -o -name 'aw-hayabusa-profiles.txt' \) \ + -mtime +0 -delete 2>/dev/null || true + +find /tmp -maxdepth 1 -type f \ + \( -name 'aw-worktime-ui-bridge.py' -o -name 'views-default.json' -o -name 'apply_webui_ru_patch.out' \) \ + -mtime +1 -delete 2>/dev/null || true diff --git a/scripts/legacy/check-aw-data.sh b/scripts/legacy/check-aw-data.sh new file mode 100644 index 0000000..b028e4f --- /dev/null +++ b/scripts/legacy/check-aw-data.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# check-aw-data.sh - Проверка сбора данных ActivityWatch с RDP-сервера SHARKON2025 +# Сервер: 10.10.10.13:5600 +# Хост-источник: 192.168.100.18 (SHARKON2025) + +SERVER="http://10.10.10.13:5600" +HOSTNAME_FILTER="SHARKON2025" +NOW=$(date -u +%s) +HOST_INACTIVE=false +GUARD_HEALTHY=false + +classify_bucket_age() { + local bucket="$1" + local age_sec="$2" + + case "$bucket" in + aw-watcher-window) + if [ "$HOST_INACTIVE" = "true" ]; then + printf '%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-endpoint-signals) + if [ "$HOST_INACTIVE" = "true" ] && [ "$GUARD_HEALTHY" = "true" ]; then + printf '%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + esac + + case "$bucket" in + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + if [ "$age_sec" -lt 86400 ]; then + printf '%s' "${GREEN}FRESH${NC}" + else + printf '%s' "${CYAN}EVENT-DRIVEN${NC}" + fi + ;; + *) + if [ "$age_sec" -lt 3600 ]; then + printf '%s' "${GREEN}FRESH${NC}" + elif [ "$age_sec" -lt 86400 ]; then + printf '%s' "${YELLOW}STALE${NC}" + else + printf '%s' "${RED}DEAD${NC}" + fi + ;; + esac +} + +classify_bucket_no_events() { + local bucket="$1" + + case "$bucket" in + aw-watcher-window) + if [ "$HOST_INACTIVE" = "true" ]; then + printf '%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-endpoint-signals) + if [ "$HOST_INACTIVE" = "true" ] && [ "$GUARD_HEALTHY" = "true" ]; then + printf '%s' "${CYAN}INACTIVE${NC}" + return + fi + ;; + aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events) + printf '%s' "${CYAN}EVENT-DRIVEN${NC}" + return + ;; + esac + + printf '%s' "${RED}EMPTY${NC}" +} + +# Цвета +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +# Проверка доступности сервера +echo "=== ActivityWatch Data Check: $HOSTNAME_FILTER ===" +echo "" +echo -n "Server connectivity... " +RESP=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/info" 2>&1) +if [ $? -eq 0 ] && echo "$RESP" | jq -e '.version' > /dev/null 2>&1; then + VERSION=$(echo "$RESP" | jq -r '.version') + echo -e "${GREEN}OK${NC} (aw-server $VERSION)" +else + echo -e "${RED}FAILED${NC} (cannot reach $SERVER)" + exit 1 +fi +echo "" + +# Context for inactive/event-driven classification. +WORKTIME_EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-worktime-sessions_$HOSTNAME_FILTER/events?limit=1" 2>&1) +WORKTIME_TS=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null) +WORKTIME_ACTIVE=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].data.active // false' 2>/dev/null) +if [ -n "$WORKTIME_TS" ]; then + WORKTIME_EPOCH=$(date -d "$WORKTIME_TS" +%s 2>/dev/null || echo 0) + if [ "$WORKTIME_EPOCH" -gt 0 ]; then + WORKTIME_AGE=$((NOW - WORKTIME_EPOCH)) + if [ "$WORKTIME_AGE" -lt 900 ] && [ "$WORKTIME_ACTIVE" != "true" ]; then + HOST_INACTIVE=true + fi + fi +fi + +GUARD_EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-rus-collector-guard_$HOSTNAME_FILTER/events?limit=1" 2>&1) +GUARD_TS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null) +GUARD_STATUS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].data.status // ""' 2>/dev/null) +GUARD_PROBLEMS=$(echo "$GUARD_EVENT_DATA" | jq -r '([.[0].data.problems[]?] | length) // 0' 2>/dev/null) +if [ -n "$GUARD_TS" ]; then + GUARD_EPOCH=$(date -d "$GUARD_TS" +%s 2>/dev/null || echo 0) + if [ "$GUARD_EPOCH" -gt 0 ]; then + GUARD_AGE=$((NOW - GUARD_EPOCH)) + if [ "$GUARD_AGE" -lt 300 ] && [ "$GUARD_STATUS" = "ok" ] && [ "$GUARD_PROBLEMS" = "0" ]; then + GUARD_HEALTHY=true + fi + fi +fi + +# Проверка бакетов +echo "--- Buckets ---" +printf "%-45s %-8s %-22s %s\n" "BUCKET" "EVENTS" "LAST EVENT" "STATUS" +printf "%-45s %-8s %-22s %s\n" "---------------------------------------------" "--------" "----------------------" "------" + +BUCKETS=( + "aw-dlp-endpoint-signals" + "aw-dlp-incidents" + "aw-dlp-review" + "aw-dlp-rules" + "aw-watcher-afk" + "aw-watcher-window" + "aw-session-events" + "aw-worktime-sessions" +) + +for bucket in "${BUCKETS[@]}"; do + bucket_full="${bucket}_${HOSTNAME_FILTER}" + + # Получаем последний event + EVENT_DATA=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1) + LAST_ID=$(echo "$EVENT_DATA" | jq '.[0].id // 0') + LAST_TS=$(echo "$EVENT_DATA" | jq -r '.[0].timestamp // "no events"') + + # Вычисляем возраст + if [ "$LAST_TS" != "no events" ] && [ -n "$LAST_TS" ]; then + EVENT_EPOCH=$(date -d "$LAST_TS" +%s 2>/dev/null || echo 0) + if [ "$EVENT_EPOCH" -gt 0 ]; then + AGE_SEC=$((NOW - EVENT_EPOCH)) + if [ $AGE_SEC -lt 3600 ]; then + AGE="$((AGE_SEC / 60))m ago" + elif [ $AGE_SEC -lt 86400 ]; then + AGE="$((AGE_SEC / 3600))h ago" + else + AGE="$((AGE_SEC / 86400))d ago" + fi + STATUS="$(classify_bucket_age "$bucket" "$AGE_SEC")" + else + AGE="unknown" + STATUS="${RED}?${NC}" + fi + else + AGE="none" + LAST_ID="0" + STATUS="$(classify_bucket_no_events "$bucket")" + fi + + printf "%-45s %-8s %-22s %b\n" "$bucket_full" "$LAST_ID" "$LAST_TS ($AGE)" "$STATUS" +done + +echo "" + +# Проверка CORS +echo "--- CORS Check ---" +CORS_RESP=$(no_proxy=10.10.10.13 curl -s --connect-timeout 10 --max-time 15 -o /dev/null -w '%{http_code}' -H "Origin: http://10.10.10.13:5600" "$SERVER/api/0/settings/" 2>&1) +if [ "$CORS_RESP" = "200" ]; then + echo -e "${GREEN}CORS: OK${NC} (HTTP 200)" +else + echo -e "${RED}CORS: FAIL${NC} (HTTP $CORS_RESP)" +fi + +echo "" +echo "=== Check Complete ===" +echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" diff --git a/scripts/merge_aw_server_dbs.py b/scripts/merge_aw_server_dbs.py index 0426f23..9f7408f 100644 --- a/scripts/merge_aw_server_dbs.py +++ b/scripts/merge_aw_server_dbs.py @@ -2,11 +2,34 @@ import argparse import json import os -import shutil import sqlite3 +import sys from pathlib import Path +def maybe_exec_rust() -> None: + if os.environ.get("MERGE_AW_SERVER_DBS_FORCE_LEGACY") == "1": + return + script_path = Path(__file__).resolve() + repo_root = script_path.parent.parent if script_path.parent.name == "scripts" else None + candidates = [ + os.environ.get("MERGE_AW_SERVER_DBS_RUST"), + str(Path(os.environ.get("CARGO_TARGET_DIR", "")) / "release" / "merge-aw-server-dbs") + if os.environ.get("CARGO_TARGET_DIR") + else None, + str(repo_root / "adk-rust" / "target" / "release" / "merge-aw-server-dbs") + if repo_root + else None, + "/usr/local/bin/merge-aw-server-dbs", + ] + for candidate in candidates: + if candidate and os.path.isfile(candidate) and os.access(candidate, os.X_OK): + os.execv(candidate, [candidate, *sys.argv[1:]]) + + +maybe_exec_rust() + + def connect(path: Path) -> sqlite3.Connection: connection = sqlite3.connect(str(path)) connection.execute("PRAGMA journal_mode=WAL") @@ -130,43 +153,48 @@ def main() -> int: if dest_rowid is None: dest_rowid = dest_bucket_map.get(key) if dest_rowid is None: - # Use UPSERT to handle UNIQUE(name) constraint gracefully - cursor = dest.execute( - """ - INSERT INTO buckets (name, type, client, hostname, created, data_deprecated, data) - VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(name) DO UPDATE SET - type=excluded.type, - client=excluded.client, - hostname=excluded.hostname, - created=excluded.created, - data_deprecated=excluded.data_deprecated, - data=excluded.data - WHERE rowid = (SELECT rowid FROM buckets WHERE name = ? LIMIT 1) - """, - ( - src_bucket["name"], - src_bucket["type"], - src_bucket["client"], - src_bucket["hostname"], - src_bucket["created"], - src_bucket["data_deprecated"], - src_bucket["data"], - str(src_bucket["name"]), - ), - ) - # Get the rowid of the affected bucket (either inserted or updated) - cursor.execute("SELECT last_insert_rowid(), (SELECT rowid FROM buckets WHERE name = ? LIMIT 1)", - (str(src_bucket["name"]),)) - result = cursor.fetchone() - dest_rowid = result[0] if result[0] != 0 else result[1] - - if dest_rowid: - dest_bucket_map[key] = dest_rowid - # Only count as inserted if it was a true insert (not update) - cursor.execute("SELECT changes() FROM buckets WHERE rowid = ?", (dest_rowid,)) - if cursor.fetchone()[0] > 0: - inserted_buckets += 1 + dest_rowid = find_bucket_by_name(dest, str(src_bucket["name"])) + if dest_rowid is not None: + dest.execute( + """ + UPDATE buckets + SET type = ?, client = ?, hostname = ?, created = ?, + data_deprecated = ?, data = ? + WHERE rowid = ? + """, + ( + src_bucket["type"], + src_bucket["client"], + src_bucket["hostname"], + src_bucket["created"], + src_bucket["data_deprecated"], + src_bucket["data"], + dest_rowid, + ), + ) + else: + cursor = dest.execute( + """ + INSERT INTO buckets (id, name, type, client, hostname, created, data_deprecated, data) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + src_bucket["id"], + src_bucket["name"], + src_bucket["type"], + src_bucket["client"], + src_bucket["hostname"], + src_bucket["created"], + src_bucket["data_deprecated"], + src_bucket["data"], + ), + ) + dest_rowid = int(cursor.lastrowid) + inserted_buckets += 1 + + dest_bucket_map[key] = dest_rowid + if src_id: + dest_id_map[str(src_id)] = dest_rowid existing_events = load_existing_events(dest, dest_rowid) for starttime, endtime, data in source.execute( diff --git a/scripts/metagpt-aw-scout.sh b/scripts/metagpt-aw-scout.sh new file mode 100644 index 0000000..0a764dd --- /dev/null +++ b/scripts/metagpt-aw-scout.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +OUT_DIR="${METAGPT_AW_OUT_DIR:-$ROOT/.ai/metagpt}" +METAGPT_BIN="${METAGPT_BIN:-/home/igor/bin/metagpt-lab}" +METAGPT_AW_INVESTMENT="${METAGPT_AW_INVESTMENT:-0.1}" +METAGPT_AW_N_ROUND="${METAGPT_AW_N_ROUND:-2}" +METAGPT_AW_TIMEOUT="${METAGPT_AW_TIMEOUT:-90}" +METAGPT_AW_ENGINE="${METAGPT_AW_ENGINE:-direct}" +METAGPT_CONFIG="${METAGPT_CONFIG:-/home/igor/.metagpt/config2.yaml}" + +usage() { + cat <<'USAGE' +Usage: + scripts/metagpt-aw-scout.sh + +Presets: + qa-rollback QA checklist and rollback plan for Windows collectors + smoke End-to-end smoke-test plan for AW server, RDP collectors, Grafana, Influx exporters + grafana Management-facing Grafana dashboard review plan + install-kit Install-kit rebuild and validation checklist + windows-i18n Windows localized Administrator / Cyrillic collector checks + +Examples: + scripts/metagpt-aw-scout.sh qa-rollback + scripts/metagpt-aw-scout.sh "Review risk of changing aw-worktime-ui-bridge foreground cache" +USAGE +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" || $# -eq 0 ]]; then + usage + exit 0 +fi + +case "$1" in + qa-rollback) + TASK="Prepare a practical QA checklist and rollback plan for ActivityWatch-Russian Windows collectors after changes in PowerShell collectors, process events, localized Administrator task names, and recovery hardening." + ;; + smoke) + TASK="Prepare a minimal but complete end-to-end smoke-test plan for ActivityWatch-Russian: AW server buckets, worktime API, RDP/WinRM collectors, Windows SSH access, Grafana dashboards, Influx worktime exporter, DLP exporter, and install-kit sanity." + ;; + grafana) + TASK="Review the management-facing Grafana dashboards for ActivityWatch-Russian. Identify confusing technical labels, panels that should be hidden or renamed, and checks needed to prove 'true user work' by application." + ;; + install-kit) + TASK="Prepare a rebuild and validation checklist for install-kit-awindows, including rebuild_install_kit.sh, check_install_kit_vs_repo.sh, validate_install_kit.sh, and optional InnoSetup exe rebuild." + ;; + windows-i18n) + TASK="Prepare a validation checklist for Windows localized account names and Cyrillic handling: CP866 query.exe decoding, SHARKON2025_Администратор scheduled task, WinRM output, SSH checks, and ActivityWatch Launch fallback rules." + ;; + *) + TASK="$*" + ;; +esac + +if [[ ! -x "$METAGPT_BIN" ]]; then + echo "MetaGPT wrapper not found or not executable: $METAGPT_BIN" >&2 + exit 2 +fi + +mkdir -p "$OUT_DIR" +STAMP="$(date +%Y%m%d-%H%M%S)" +SLUG="$(printf '%s' "$TASK" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//; s/--*/-/g; s/^$/task/' | cut -c1-80)" +OUT="$OUT_DIR/$STAMP-$SLUG.md" + +PROMPT="$(cat < "$OUT" + +set +e +if [[ "$METAGPT_AW_ENGINE" == "team" ]]; then + timeout "$METAGPT_AW_TIMEOUT" "$METAGPT_BIN" --investment "$METAGPT_AW_INVESTMENT" --n-round "$METAGPT_AW_N_ROUND" --no-implement --project-name "aw-scout-$STAMP" "$PROMPT" 2>&1 | tee -a "$OUT" +else + /home/igor/labs/metagpt-lab/.venv/bin/python - "$METAGPT_CONFIG" "$PROMPT" <<'PY' 2>&1 | tee -a "$OUT" +import sys +from pathlib import Path + +import yaml +from openai import OpenAI + +config_path = Path(sys.argv[1]) +prompt = sys.argv[2] +cfg = yaml.safe_load(config_path.read_text(encoding="utf-8"))["llm"] +client = OpenAI( + api_key=cfg["api_key"], + base_url=(cfg.get("base_url") or "https://api.openai.com/v1").rstrip("/"), +) +response = client.chat.completions.create( + model=cfg.get("model") or "gpt-4.1-mini", + messages=[ + { + "role": "system", + "content": ( + "Ты практичный технический ревьюер ActivityWatch-Russian. " + "Отвечай по-русски, кратко, с командами и критериями проверки. " + "Не выдумывай факты, помечай непроверенное как гипотезу." + ), + }, + {"role": "user", "content": prompt}, + ], + temperature=0.2, + max_tokens=1800, +) +print(response.choices[0].message.content.strip()) +PY +fi +status=${PIPESTATUS[0]} +set -e + +{ + echo + echo "## Exit Status" + echo + echo "$status" +} >> "$OUT" + +echo +echo "Saved: $OUT" +exit "$status" diff --git a/scripts/prod_backup_restore.sh b/scripts/prod_backup_restore.sh index a731ae6..9780df2 100644 --- a/scripts/prod_backup_restore.sh +++ b/scripts/prod_backup_restore.sh @@ -11,6 +11,29 @@ if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then set +a fi +if [[ "${1:-}" == "--apply-legacy" ]]; then + shift +else + for candidate in \ + "${PROD_BACKUP_RESTORE_RUST:-}" \ + "${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \ + "$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \ + "/usr/local/bin/prod-backup-restore"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi + done + cat >&2 <<'EOF' +prod_backup_restore.sh is destructive and now requires the Rust planner. +Build it first: + cd adk-rust && cargo build --release -p prod-backup-restore + +To run the old destructive restore explicitly: + scripts/prod_backup_restore.sh --apply-legacy +EOF + exit 2 +fi + : "${AW_SSH_PASSWORD:?AW_SSH_PASSWORD is required}" : "${AW_WINRM_PASSWORD:?AW_WINRM_PASSWORD is required}" diff --git a/scripts/prod_rollout.sh b/scripts/prod_rollout.sh index 61583f7..9c0f636 100644 --- a/scripts/prod_rollout.sh +++ b/scripts/prod_rollout.sh @@ -4,6 +4,43 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" +if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${ROOT_DIR}/secrets/runtime.env" + set +a +fi + +if [[ "${1:-}" == "--apply-legacy" ]]; then + shift +else + TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" + for candidate in \ + "${PROD_ROLLOUT_RUST:-}" \ + "$TARGET_ROOT/release/prod-rollout" \ + "$ROOT_DIR/adk-rust/target/release/prod-rollout" \ + "/usr/local/bin/prod-rollout"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi + done + cat >&2 <<'EOF' +prod_rollout.sh now requires the Rust planner/orchestrator for safe default runs. +Build it first: + cd adk-rust && cargo build --release -p prod-rollout + +Safe checks: + scripts/prod_rollout.sh --check-inputs --json + +Explicit Rust rollout: + scripts/prod_rollout.sh --apply + +Old Bash rollout: + scripts/prod_rollout.sh --apply-legacy +EOF + exit 2 +fi + timestamp() { date +"%Y%m%d-%H%M%S"; } LOG_DIR="${ROOT_DIR}/.rollout-logs/$(timestamp)" diff --git a/scripts/quality-gate.sh b/scripts/quality-gate.sh index 3b88862..7a4bcf4 100755 --- a/scripts/quality-gate.sh +++ b/scripts/quality-gate.sh @@ -4,17 +4,46 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -echo "[1/4] Bash syntax check" -find aw-server proxmox -type f -name "*.sh" -print0 | xargs -0 -r -n1 bash -n +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${QUALITY_GATE_RUST:-}" -echo "[2/4] Shellcheck (if available)" +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/quality-gate" + "$ROOT_DIR/adk-rust/target/release/quality-gate" + "/usr/local/bin/quality-gate" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi +done + +echo "[1/5] Bash syntax check" +find aw-server proxmox scripts -type f -name "*.sh" -print0 | xargs -0 -r -n1 bash -n + +echo "[2/5] Shellcheck (if available)" if command -v shellcheck >/dev/null 2>&1; then - find aw-server proxmox -type f -name "*.sh" -print0 | xargs -0 -r shellcheck -e SC1007,SC1090,SC2016 + { + find aw-server proxmox -type f -name "*.sh" + printf '%s\n' scripts/aw-webui-browser-smoke.sh + } | xargs -r shellcheck -e SC1007,SC1090,SC2016 else echo "shellcheck not found, skipping." fi -echo "[3/4] PowerShell parse check (if pwsh available)" +echo "[3/5] Node syntax check (if node available)" +if command -v node >/dev/null 2>&1; then + node --check scripts/aw-webui-browser-smoke.mjs >/dev/null +else + echo "node not found, skipping." +fi + +echo "[4/5] PowerShell parse check (if pwsh available)" if command -v pwsh >/dev/null 2>&1; then pwsh -NoLogo -NoProfile -Command ' $ErrorActionPreference = "Stop" @@ -24,13 +53,14 @@ if command -v pwsh >/dev/null 2>&1; then [void][System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "windows/ActivityWatch.Windows.Common.psm1"),[ref]$null,[ref]$null) [void][System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "windows/ActivityWatch.Windows.Common.psd1"),[ref]$null,[ref]$null) ' + pwsh -NoLogo -NoProfile -File windows/aw-collector-guard.ps1 -SelfTest >/dev/null else echo "pwsh not found, skipping." fi -echo "[4/4] Ansible syntax check (if ansible-playbook available)" +echo "[5/5] Ansible syntax check (if ansible-playbook available)" if command -v ansible-playbook >/dev/null 2>&1; then for playbook in ansible/*.yml; do ansible-playbook --syntax-check "$playbook" -i ansible/inventory.example.ini >/dev/null diff --git a/scripts/rdp-worktime-report.sh b/scripts/rdp-worktime-report.sh index b56b55c..371ba0e 100644 --- a/scripts/rdp-worktime-report.sh +++ b/scripts/rdp-worktime-report.sh @@ -9,6 +9,8 @@ AW_WORKTIME_HOST="${AW_WORKTIME_HOST:-SHARKON2025}" AW_WORKTIME_DEFAULT_SAMPLE_SECONDS="${AW_WORKTIME_DEFAULT_SAMPLE_SECONDS:-30}" AW_WORKTIME_MAX_SAMPLE_SECONDS="${AW_WORKTIME_MAX_SAMPLE_SECONDS:-300}" OUT_DIR="${OUT_DIR:-reports}" +TARGET_ROOT="${CARGO_TARGET_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/adk-rust/target}" +RUST_BIN="${RDP_WORKTIME_REPORT_RUST:-}" usage() { cat </dev/null; then @@ -104,6 +147,7 @@ Includes: - windows/* (deploy scripts, collectors, common module, configs/examples) - ansible/* (Windows and AW server playbooks, examples, inventory, tasks) - aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) +- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks) - server-configs-192.168.100.18/* (working Windows/RDP config snapshots) Source: diff --git a/scripts/test_dlp_health_check.py b/scripts/test_dlp_health_check.py new file mode 100644 index 0000000..13ad931 --- /dev/null +++ b/scripts/test_dlp_health_check.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +import importlib.util +import sys +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("dlp-health-check.py") +SPEC = importlib.util.spec_from_file_location("dlp_health_check", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def test_endpoint_signals_ok_when_no_active_managed_hosts(monkeypatch): + buckets = { + "aw-worktime-sessions_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}, + "aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T07:00:00Z"}}, + } + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:00:10Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url: [{"timestamp": "2026-05-30T12:00:00Z", "data": {"active": False}}] + if "aw-worktime-sessions_SHARKON2025/events" in url + else [], + ) + report = MODULE.HealthReport() + + MODULE.check_endpoint_signal_buckets(report, "http://127.0.0.1:5600/api/0", buckets, 900) + + result = report.results[0] + assert result.name == "buckets:endpoint-signals" + assert result.status == "ok" + assert result.summary == "no active managed hosts require endpoint-signals freshness" + assert result.details["ignored_inactive"] == ["aw-dlp-endpoint-signals_SHARKON2025"] + + +def test_endpoint_self_test_metrics_reports_transport_counters(monkeypatch): + buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url, **kwargs: [ + { + "timestamp": "2026-05-30T11:59:00Z", + "data": { + "signalType": "self_test", + "queueDepth": 8, + "eventsEnqueued": 100, + "eventsFlushed": 92, + "sendFailures": 0, + }, + }, + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "self_test", + "queueDepth": "3", + "eventsEnqueued": "120", + "eventsFlushed": "117", + "sendFailures": "0", + }, + }, + ], + ) + report = MODULE.HealthReport() + + MODULE.check_endpoint_self_test_metrics(report, "http://127.0.0.1:5600/api/0", buckets) + + result = report.results[0] + assert result.name == "endpoint-self-test-metrics" + assert result.status == "ok" + latest = result.details["latest_self_tests"][0] + assert latest["bucket"] == "aw-dlp-endpoint-signals_SHARKON2025" + assert latest["timestamp"] == "2026-05-30T12:00:30Z" + assert latest["age_seconds"] == 30 + assert latest["queueDepth"] == 3 + assert latest["eventsEnqueued"] == 120 + assert latest["eventsFlushed"] == 117 + assert latest["sendFailures"] == 0 + assert latest["sendFailuresDelta"] == 0 + + +def test_endpoint_self_test_metrics_warns_on_transport_counters(monkeypatch): + buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url, **kwargs: [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "self_test", + "queueDepth": 101, + "eventsEnqueued": 120, + "eventsFlushed": 19, + "sendFailures": 1, + }, + } + ], + ) + report = MODULE.HealthReport() + + MODULE.check_endpoint_self_test_metrics( + report, + "http://127.0.0.1:5600/api/0", + buckets, + queue_warn_depth=100, + send_failure_warn_count=1, + ) + + result = report.results[0] + assert result.status == "warn" + assert result.summary == "endpoint transport counters outside thresholds" + assert result.details["warnings"] == [ + {"bucket": "aw-dlp-endpoint-signals_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100}, + { + "bucket": "aw-dlp-endpoint-signals_SHARKON2025", + "metric": "sendFailuresDelta", + "value": 1, + "current": 1, + "previous": None, + "threshold": 1, + }, + ] + + +def test_file_operations_runtime_reports_health_and_latest_operations(monkeypatch): + buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url: [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "collector_health", + "queueDepth": "0", + "eventsEnqueued": "5", + "eventsFlushed": "5", + "sendFailures": "0", + "username": "USER1", + "hostname": "SHARKON2025", + "sessionId": "3", + }, + }, + { + "timestamp": "2026-05-30T12:00:20Z", + "data": { + "operation": "Created", + "path": "C:\\Users\\USER1\\Downloads\\report.zip", + "extension": ".zip", + "archiveHint": True, + "username": "USER1", + "hostname": "SHARKON2025", + "size": "42", + }, + }, + ], + ) + report = MODULE.HealthReport() + + MODULE.check_file_operations_runtime(report, "http://127.0.0.1:5600/api/0", buckets) + + result = report.results[0] + assert result.name == "file-operations-runtime" + assert result.status == "ok" + assert result.details["latest_health"][0]["queueDepth"] == 0 + assert result.details["latest_health"][0]["eventsEnqueued"] == 5 + assert result.details["latest_health"][0]["sendFailuresDelta"] == 0 + latest_op = result.details["latest_operations"][0] + assert latest_op["operation"] == "Created" + assert latest_op["extension"] == ".zip" + assert latest_op["archiveHint"] is True + assert latest_op["path_tail"] == "Downloads/report.zip" + assert latest_op["size"] == 42 + assert result.details["sampled"][0]["operation_counts"] == {"Created": 1} + + +def test_file_operations_runtime_warns_on_transport_counters(monkeypatch): + buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url: [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "collector_health", + "queueDepth": 101, + "eventsEnqueued": 5, + "eventsFlushed": 2, + "sendFailures": 1, + "username": "USER1", + "hostname": "SHARKON2025", + "sessionId": 3, + }, + } + ], + ) + report = MODULE.HealthReport() + + MODULE.check_file_operations_runtime( + report, + "http://127.0.0.1:5600/api/0", + buckets, + queue_warn_depth=100, + send_failure_warn_count=1, + ) + + result = report.results[0] + assert result.status == "warn" + assert result.summary == "file-operations runtime counters outside expectations" + assert result.details["warnings"] == [ + {"bucket": "aw-file-operations_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100}, + { + "bucket": "aw-file-operations_SHARKON2025", + "metric": "sendFailuresDelta", + "value": 1, + "current": 1, + "previous": None, + "threshold": 1, + }, + ] + + +def test_endpoint_send_failures_uses_delta_baseline(monkeypatch): + buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + state = {"counters": {}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + values = [12, 12, 13] + + def fake_http(url, **kwargs): + value = values.pop(0) + return [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "self_test", + "queueDepth": 0, + "eventsEnqueued": 120, + "eventsFlushed": 117, + "sendFailures": value, + }, + } + ] + + monkeypatch.setattr(MODULE, "_http_json", fake_http) + + first = MODULE.HealthReport() + MODULE.check_endpoint_self_test_metrics(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert first.results[0].status == "ok" + assert first.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] is None + assert first.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0 + + second = MODULE.HealthReport() + MODULE.check_endpoint_self_test_metrics(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert second.results[0].status == "ok" + assert second.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] == 12 + assert second.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0 + + third = MODULE.HealthReport() + MODULE.check_endpoint_self_test_metrics(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert third.results[0].status == "warn" + assert third.results[0].details["warnings"] == [ + { + "bucket": "aw-dlp-endpoint-signals_SHARKON2025", + "metric": "sendFailuresDelta", + "value": 1, + "current": 13, + "previous": 12, + "threshold": 1, + } + ] + + +def test_file_operations_send_failures_uses_delta_baseline(monkeypatch): + buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + state = {"counters": {}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + values = [28, 28, 29] + + def fake_http(url, **kwargs): + value = values.pop(0) + return [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "signalType": "collector_health", + "queueDepth": 0, + "eventsEnqueued": 120, + "eventsFlushed": 117, + "sendFailures": value, + }, + } + ] + + monkeypatch.setattr(MODULE, "_http_json", fake_http) + + first = MODULE.HealthReport() + MODULE.check_file_operations_runtime(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert first.results[0].status == "ok" + assert first.results[0].details["latest_health"][0]["sendFailuresPrevious"] is None + assert first.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0 + + second = MODULE.HealthReport() + MODULE.check_file_operations_runtime(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert second.results[0].status == "ok" + assert second.results[0].details["latest_health"][0]["sendFailuresPrevious"] == 28 + assert second.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0 + + third = MODULE.HealthReport() + MODULE.check_file_operations_runtime(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) + assert third.results[0].status == "warn" + assert third.results[0].details["warnings"] == [ + { + "bucket": "aw-file-operations_SHARKON2025", + "metric": "sendFailuresDelta", + "value": 1, + "current": 29, + "previous": 28, + "threshold": 1, + } + ] + + +def test_incident_runtime_reports_counts_and_latest_real_incidents(monkeypatch): + buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + monkeypatch.setattr( + MODULE, + "_http_json", + lambda url, **kwargs: [ + { + "timestamp": "2026-05-30T12:00:30Z", + "data": { + "ruleId": "usb-archive-copy", + "severity": "high", + "action": "alert", + "message": "Archive copied to removable device with a long message that should not leak full raw content.", + "username": "USER1", + "hostname": "SHARKON2025", + "source": "endpoint", + }, + }, + { + "timestamp": "2026-05-30T12:00:00Z", + "data": { + "ruleId": "selftest-dlp-incident", + "severity": "low", + "action": "alert", + "message": "Self-test DLP incident from validation", + "signalType": "self_test", + "source": "self-test", + }, + }, + ], + ) + report = MODULE.HealthReport() + + MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20) + + result = report.results[0] + assert result.name == "incident-runtime" + assert result.status == "ok" + assert result.summary == "1 real incidents in sampled events" + assert result.details["totals"] == {"sampled_events": 2, "real_incidents": 1, "self_tests": 1} + assert result.details["severity_counts"] == {"high": 1} + assert result.details["action_counts"] == {"alert": 1} + assert result.details["rule_counts"] == {"usb-archive-copy": 1} + latest = result.details["latest_incidents"][0] + assert latest["age_seconds"] == 30 + assert latest["ruleId"] == "usb-archive-copy" + assert latest["message_excerpt"].endswith(".") + + +def test_incident_runtime_defaults_to_metadata_without_event_sampling(monkeypatch): + buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) + + def fail_if_called(url, **kwargs): + raise AssertionError("event sampling should stay disabled") + + monkeypatch.setattr(MODULE, "_http_json", fail_if_called) + report = MODULE.HealthReport() + + MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=0) + + result = report.results[0] + assert result.name == "incident-runtime" + assert result.status == "ok" + assert result.summary == "incident event sampling disabled" + assert result.details["metadata"] == [ + { + "bucket": "aw-dlp-incidents_SHARKON2025", + "end": "2026-05-30T12:00:00Z", + "age_seconds": 60, + } + ] + + +def test_incident_runtime_warns_on_read_failure(monkeypatch): + buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} + + def fail_http(url, **kwargs): + raise RuntimeError("timeout") + + monkeypatch.setattr(MODULE, "_http_json", fail_http) + report = MODULE.HealthReport() + + MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20) + + result = report.results[0] + assert result.name == "incident-runtime" + assert result.status == "warn" + assert result.summary == "1 incident buckets failed to sample" + assert result.details["read_failed"] == [{"bucket": "aw-dlp-incidents_SHARKON2025", "error": "timeout"}] diff --git a/scripts/validate_install_kit.sh b/scripts/validate_install_kit.sh index 0c0e037..94b924a 100755 --- a/scripts/validate_install_kit.sh +++ b/scripts/validate_install_kit.sh @@ -8,6 +8,29 @@ KIT_DIR="install-kit-awindows-20260427-211240" MANIFEST="$KIT_DIR/MANIFEST.txt" ZIP_ARCHIVE="install-kit-awindows-20260427-211240.zip" TAR_ARCHIVE="install-kit-awindows-20260427-211240.tar.gz" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${VALIDATE_INSTALL_KIT_RUST:-}" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/validate-install-kit" + "$ROOT_DIR/adk-rust/target/release/validate-install-kit" + "/usr/local/bin/validate-install-kit" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" \ + --root "$ROOT_DIR" \ + --kit-dir "$KIT_DIR" \ + --zip-archive "$ZIP_ARCHIVE" \ + --tar-archive "$TAR_ARCHIVE" \ + "$@" + fi +done required_files=( "$MANIFEST" @@ -16,6 +39,10 @@ required_files=( "$KIT_DIR/windows/validate-deployment.ps1" "$KIT_DIR/ansible/deploy_aw_windows.yml" "$KIT_DIR/aw-server/install_aw_server.sh" + "$KIT_DIR/scripts/rebuild_install_kit.sh" + "$KIT_DIR/scripts/validate_install_kit.sh" + "$KIT_DIR/scripts/check_install_kit_vs_repo.sh" + "$KIT_DIR/scripts/quality-gate.sh" ) echo "[1/4] Required files presence" diff --git a/scripts/verify_innosetup_installer.sh b/scripts/verify_innosetup_installer.sh new file mode 100644 index 0000000..fb89657 --- /dev/null +++ b/scripts/verify_innosetup_installer.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +INSTALLER="${1:-windows/installkit/innosetup/AWatch-rus-InstallKit.exe}" +WINEPREFIX_VERIFY="${WINEPREFIX_VERIFY:-/tmp/aw-inno-verify-wineprefix}" +TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" +RUST_BIN="${VERIFY_INNOSETUP_INSTALLER_RUST:-}" +INSTALL_DIR_WIN='C:\AWatchRusExtract' +INSTALL_DIR_UNIX="${WINEPREFIX_VERIFY}/drive_c/AWatchRusExtract" + +rust_candidates=() +if [[ -n "$RUST_BIN" ]]; then + rust_candidates+=("$RUST_BIN") +fi +rust_candidates+=( + "$TARGET_ROOT/release/verify-innosetup-installer" + "$ROOT_DIR/adk-rust/target/release/verify-innosetup-installer" + "/usr/local/bin/verify-innosetup-installer" +) + +for candidate in "${rust_candidates[@]}"; do + if [[ -x "$candidate" ]]; then + exec "$candidate" "$INSTALLER" --root "$ROOT_DIR" --wineprefix "$WINEPREFIX_VERIFY" "${@:2}" + fi +done + +if [[ ! -f "$INSTALLER" ]]; then + echo "Installer not found: $INSTALLER" >&2 + exit 1 +fi + +if ! command -v wine >/dev/null 2>&1; then + echo "wine not found" >&2 + exit 1 +fi + +rm -rf "$WINEPREFIX_VERIFY" +mkdir -p "$WINEPREFIX_VERIFY" +export WINEPREFIX="$WINEPREFIX_VERIFY" +export WINEDEBUG="${WINEDEBUG:--all}" + +wineboot -u >/dev/null 2>&1 +wine "$INSTALLER" /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- /TASKS="" /DIR="$INSTALL_DIR_WIN" >/dev/null 2>&1 +wineserver -w >/dev/null 2>&1 + +required_files=( + windows/AWatchRusCollectorGuardService.cs + windows/aw-collector-guard.ps1 + windows/install-collector-guard-service.ps1 + windows/dlp-policy.native-cross-os.example.json +) + +for rel in "${required_files[@]}"; do + extracted="${INSTALL_DIR_UNIX}/${rel}" + if [[ ! -f "$extracted" ]]; then + echo "Missing extracted file: $rel" >&2 + exit 1 + fi + if ! cmp -s "$rel" "$extracted"; then + echo "Extracted file differs from repo: $rel" >&2 + exit 1 + fi +done + +if ! grep -q 'collector guard self-test OK' "${INSTALL_DIR_UNIX}/windows/aw-collector-guard.ps1"; then + echo "Guard self-test marker missing in extracted installer payload" >&2 + exit 1 +fi + +echo "verify_innosetup_installer: OK" diff --git a/windows/AWatchRusCollectorGuardService.cs b/windows/AWatchRusCollectorGuardService.cs new file mode 100644 index 0000000..efcbdd1 --- /dev/null +++ b/windows/AWatchRusCollectorGuardService.cs @@ -0,0 +1,123 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.ServiceProcess; + +namespace AWatchRus +{ + public sealed class CollectorGuardService : ServiceBase + { + private Process child; + private readonly ServiceOptions options; + + public CollectorGuardService(ServiceOptions options) + { + this.options = options; + ServiceName = options.ServiceName; + CanStop = true; + CanShutdown = true; + } + + protected override void OnStart(string[] args) + { + Directory.CreateDirectory(Path.GetDirectoryName(options.LogPath)); + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " service starting" + Environment.NewLine); + + var psi = new ProcessStartInfo + { + FileName = options.PowerShellPath, + Arguments = string.Format( + "-NoProfile -ExecutionPolicy Bypass -File \"{0}\" -ConfigPath \"{1}\" -Mode {2} -LoopSeconds {3}", + options.ScriptPath, + options.ConfigPath, + options.Mode, + options.LoopSeconds), + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = false, + RedirectStandardError = false, + }; + child = Process.Start(psi); + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " child pid=" + child.Id + Environment.NewLine); + } + + protected override void OnStop() + { + StopChild("service stopping"); + } + + protected override void OnShutdown() + { + StopChild("system shutdown"); + } + + private void StopChild(string reason) + { + try + { + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " " + reason + Environment.NewLine); + if (child != null && !child.HasExited) + { + child.Kill(); + child.WaitForExit(10000); + } + } + catch (Exception ex) + { + try + { + File.AppendAllText(options.LogPath, DateTime.Now.ToString("s") + " stop error: " + ex.Message + Environment.NewLine); + } + catch + { + } + } + } + } + + public sealed class ServiceOptions + { + public string ServiceName = "AWatchRusCollectorGuard"; + public string PowerShellPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), "System32\\WindowsPowerShell\\v1.0\\powershell.exe"); + public string ScriptPath = @"C:\Program Files\AWatch-rus\windows\aw-collector-guard.ps1"; + public string ConfigPath = @"C:\ProgramData\AWatch-rus\deployment-config.json"; + public string Mode = "shadow"; + public int LoopSeconds = 60; + public string LogPath = @"C:\ProgramData\AWatch-rus\logs\collector-guard-service.log"; + } + + internal static class Program + { + private static void Main(string[] args) + { + var options = Parse(args); + ServiceBase.Run(new CollectorGuardService(options)); + } + + private static ServiceOptions Parse(string[] args) + { + var options = new ServiceOptions(); + for (var i = 0; i < args.Length; i++) + { + var key = args[i].ToLowerInvariant(); + var value = i + 1 < args.Length ? args[i + 1] : null; + if (value == null || value.StartsWith("--", StringComparison.Ordinal)) + { + continue; + } + if (key == "--service-name") options.ServiceName = value; + else if (key == "--script") options.ScriptPath = value; + else if (key == "--config") options.ConfigPath = value; + else if (key == "--mode") options.Mode = value; + else if (key == "--loop") + { + int parsed; + if (int.TryParse(value, out parsed)) options.LoopSeconds = parsed; + } + else if (key == "--log") options.LogPath = value; + i++; + } + return options; + } + } +} diff --git a/windows/ActivityWatch.Windows.Common.psm1 b/windows/ActivityWatch.Windows.Common.psm1 index 3c68781..053a497 100755 --- a/windows/ActivityWatch.Windows.Common.psm1 +++ b/windows/ActivityWatch.Windows.Common.psm1 @@ -1,4 +1,4 @@ -Set-StrictMode -Version Latest +Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $script:ActivityWatchBuiltInAdministratorName = $null @@ -431,6 +431,22 @@ function Get-ActivityWatchTaskNameToken { return $buffer.ToString().Trim('_') } +function Test-ActivityWatchScheduledTaskExistsExact { + param([string]$TaskName) + + if ([string]::IsNullOrWhiteSpace($TaskName)) { + return $false + } + + try { + & schtasks.exe /Query /TN $TaskName *> $null + return ($LASTEXITCODE -eq 0) + } + catch { + return $false + } +} + function New-ActivityWatchUserTaskDefinitions { param( [Parameter(Mandatory = $true)] @@ -633,6 +649,119 @@ function Test-ActivityWatchUserHasLiveSession { return $false } +function Test-ActivityWatchSessionMatchesUserId { + param( + [Parameter(Mandatory = $true)] + [object]$SessionRecord, + [Parameter(Mandatory = $true)] + [string]$UserId + ) + + if ([string]::IsNullOrWhiteSpace($UserId) -or $null -eq $SessionRecord) { + return $false + } + + $sessionUser = [string]$SessionRecord.UserName + if ([string]::IsNullOrWhiteSpace($sessionUser)) { + return $false + } + + foreach ($candidate in @(Resolve-ActivityWatchUserCandidates -UserId $UserId)) { + if ($sessionUser -ieq $candidate -or + ('{0}\{1}' -f $env:COMPUTERNAME, $sessionUser) -ieq $candidate -or + ((-not [string]::IsNullOrWhiteSpace($env:USERDOMAIN)) -and ('{0}\{1}' -f $env:USERDOMAIN, $sessionUser) -ieq $candidate)) { + return $true + } + } + + return $false +} + +function Test-ActivityWatchUserHasManagedSession { + param( + [Parameter(Mandatory = $true)] + [string]$UserId, + [object[]]$SessionRecords, + [switch]$IncludeLive, + [switch]$IncludeDisconnected + ) + + foreach ($session in @($SessionRecords)) { + if ([int]$session.SessionId -le 0) { + continue + } + if ([string]::IsNullOrWhiteSpace([string]$session.UserName)) { + continue + } + if ([bool]$session.IsLive -and -not $IncludeLive.IsPresent) { + continue + } + if (-not [bool]$session.IsLive -and -not $IncludeDisconnected.IsPresent) { + continue + } + if (Test-ActivityWatchSessionMatchesUserId -SessionRecord $session -UserId $UserId) { + return $true + } + } + + return $false +} + +function Get-ActivityWatchManagedInteractiveSessions { + param( + [pscustomobject[]]$TaskDefinitions, + [object[]]$SessionRecords, + [switch]$IncludeLive, + [switch]$IncludeDisconnected + ) + + $result = New-Object System.Collections.Generic.List[object] + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + + foreach ($taskDef in @($TaskDefinitions)) { + $userId = [string]$taskDef.userId + $taskName = [string]$taskDef.taskName + if ([string]::IsNullOrWhiteSpace($userId) -or [string]::IsNullOrWhiteSpace($taskName)) { + continue + } + + foreach ($session in @($SessionRecords)) { + if ([int]$session.SessionId -le 0) { + continue + } + if ([string]::IsNullOrWhiteSpace([string]$session.UserName)) { + continue + } + if ([bool]$session.IsLive -and -not $IncludeLive.IsPresent) { + continue + } + if (-not [bool]$session.IsLive -and -not $IncludeDisconnected.IsPresent) { + continue + } + if (-not (Test-ActivityWatchSessionMatchesUserId -SessionRecord $session -UserId $userId)) { + continue + } + + $key = '{0}|{1}|{2}' -f $taskName, [int]$session.SessionId, $userId + if (-not $seen.Add($key)) { + continue + } + + $result.Add([pscustomobject]@{ + TaskName = $taskName + UserId = $userId + SessionName = [string]$session.SessionName + SessionId = [int]$session.SessionId + State = [string]$session.State + UserName = [string]$session.UserName + IsLive = [bool]$session.IsLive + }) | Out-Null + } + } + + return @($result.ToArray()) +} + function Copy-ActivityWatchCollectorAssets { param( [Parameter(Mandatory = $true)] @@ -777,7 +906,7 @@ function New-ActivityWatchDeploymentConfig { [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [Parameter(Mandatory = $true)] [string]$LaunchScriptPath, [Parameter(Mandatory = $true)] @@ -1410,6 +1539,10 @@ function Get-ActivityWatchRecoveryTaskDefinitions { foreach ($task in @($config.userTasks)) { $taskName = [string]$task.launchTaskName $userId = Normalize-ActivityWatchUserId -UserId ([string]$task.userId) + $canonicalTaskName = "ActivityWatch Launch [$((Get-ActivityWatchTaskNameToken -UserId $userId))]" + if ($canonicalTaskName -ne $taskName -and (Test-ActivityWatchScheduledTaskExistsExact -TaskName $canonicalTaskName)) { + $taskName = $canonicalTaskName + } if (-not [string]::IsNullOrWhiteSpace($taskName) -and -not $taskMap.Contains($taskName)) { $taskMap[$taskName] = [pscustomobject]@{ taskName = $taskName @@ -1679,13 +1812,24 @@ function Stop-ActivityWatchProcessesInNonLiveSessions { [Parameter(Mandatory = $true)] [object[]]$SessionRecords, [Parameter(Mandatory = $true)] - [pscustomobject]$Config + [pscustomobject]$Config, + [pscustomobject[]]$TaskDefinitions = @(), + [switch]$PreserveManagedSessions ) $stateRoot = if ($Config.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$Config.paths.stateRoot } else { Join-Path $env:ProgramData 'AWatch-rus' } + $preservedSessionIds = @() + if ($PreserveManagedSessions.IsPresent) { + $preservedSessionIds = @( + Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $TaskDefinitions -SessionRecords $SessionRecords -IncludeDisconnected | + ForEach-Object { [int]$_.SessionId } | + Sort-Object -Unique + ) + } + $sessionIds = @( $SessionRecords | - Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 } | + Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 -and ($preservedSessionIds -notcontains [int]$_.SessionId) } | ForEach-Object { [int]$_.SessionId } | Sort-Object -Unique ) @@ -1704,7 +1848,7 @@ function Stop-ActivityWatchProcessesInNonLiveSessions { } } - foreach ($session in @($SessionRecords | Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 })) { + foreach ($session in @($SessionRecords | Where-Object { -not $_.IsLive -and $_.SessionId -gt 0 -and ($preservedSessionIds -notcontains [int]$_.SessionId) })) { Remove-ActivityWatchLogonMarkersForSession -StateRoot $stateRoot -SessionId ([int]$session.SessionId) -UserName ([string]$session.UserName) } @@ -1885,7 +2029,7 @@ function Invoke-ActivityWatchRecoveryLoop { $config = Read-ActivityWatchDeploymentConfig -Path $ConfigPath $taskDefs = Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths $sessionRecords = Get-ActivityWatchSessionRecords - Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config + Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config -TaskDefinitions $taskDefs -PreserveManagedSessions $sessionRecords = Get-ActivityWatchSessionRecords $stateRoot = [string]$config.paths.stateRoot $sessionCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$config.paths.sessionCollectorScript } else { Join-Path $stateRoot 'worktime-session-collector.ps1' } diff --git a/windows/aw-collector-guard.ps1 b/windows/aw-collector-guard.ps1 new file mode 100644 index 0000000..559995e --- /dev/null +++ b/windows/aw-collector-guard.ps1 @@ -0,0 +1,681 @@ +[CmdletBinding()] +param( + [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json', + [ValidateSet('shadow', 'enforce')] + [string]$Mode = 'shadow', + [int]$LoopSeconds = 60, + [int]$InteractiveMaxAgeSeconds = 900, + [int]$HeadlessMaxAgeSeconds = 900, + [int]$RestartWindowSeconds = 600, + [int]$MaxRestarts = 3, + [int]$ActionCooldownSeconds = 300, + [int]$InteractiveActionCooldownSeconds = 60, + [switch]$Once, + [switch]$HeadlessEndpointEnabled, + [switch]$HeadlessFileOpsEnabled, + [switch]$SelfTest +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$modulePath = Join-Path $PSScriptRoot 'ActivityWatch.Windows.Common.psm1' +Import-Module $modulePath -Force + +function New-GuardLock { + param([string]$StateRoot) + + if (-not (Test-Path -LiteralPath $StateRoot)) { + New-Item -Path $StateRoot -ItemType Directory -Force | Out-Null + } + + $lockPath = Join-Path $StateRoot 'collector-guard.lock' + if (Test-Path -LiteralPath $lockPath) { + try { + $lockData = Get-Content -LiteralPath $lockPath -Raw | ConvertFrom-Json + $existingPid = [int]$lockData.pid + if ($existingPid -gt 0 -and (Get-Process -Id $existingPid -ErrorAction SilentlyContinue)) { + return $null + } + } + catch { + } + } + + $payload = @{ + pid = $PID + createdAt = (Get-Date).ToUniversalTime().ToString('o') + } | ConvertTo-Json -Compress + Set-Content -LiteralPath $lockPath -Value $payload -Encoding UTF8 + return $lockPath +} + +function Write-GuardLog { + param( + [string]$LogPath, + [string]$Message + ) + + try { + $directory = Split-Path -Path $LogPath -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + Add-Content -LiteralPath $LogPath -Value ('{0} {1}' -f (Get-Date -Format s), $Message) + } + catch { + } +} + +function Get-AwApiBase { + param([pscustomobject]$Config) + + $scheme = if ($Config.server.PSObject.Properties.Name -contains 'scheme') { [string]$Config.server.scheme } else { 'http' } + $hostName = [string]$Config.server.host + $port = [int]$Config.server.port + return ('{0}://{1}:{2}/api/0' -f $scheme, $hostName, $port) +} + +function Invoke-AwJson { + param( + [Parameter(Mandatory = $true)] + [string]$Method, + [Parameter(Mandatory = $true)] + [string]$Uri, + [object]$Body + ) + + $params = @{ + Method = $Method + Uri = $Uri + TimeoutSec = 15 + ErrorAction = 'Stop' + } + if ($null -ne $Body) { + $params.Body = ($Body | ConvertTo-Json -Depth 16 -Compress) + $params.ContentType = 'application/json' + } + return Invoke-RestMethod @params +} + +function Ensure-AwBucket { + param( + [string]$ApiBase, + [string]$BucketId, + [string]$ClientName, + [string]$BucketType, + [string]$Hostname + ) + + try { + Invoke-AwJson -Method 'GET' -Uri "$ApiBase/buckets/$BucketId" | Out-Null + return $true + } + catch { + } + + try { + $body = @{ + client = $ClientName + type = $BucketType + hostname = $Hostname + } + Invoke-AwJson -Method 'POST' -Uri "$ApiBase/buckets/$BucketId" -Body $body | Out-Null + return $true + } + catch { + return $false + } +} + +function Get-LatestBucketAge { + param( + [string]$ApiBase, + [string]$BucketId + ) + + try { + $events = Invoke-AwJson -Method 'GET' -Uri "$ApiBase/buckets/$BucketId/events?limit=20" + $latest = @($events | Where-Object { $null -ne $_.timestamp } | Sort-Object timestamp -Descending | Select-Object -First 1) + if (-not $latest) { + return [pscustomobject]@{ bucket = $BucketId; found = $false; timestamp = $null; ageSeconds = $null } + } + $ts = [DateTimeOffset]::Parse([string]$latest.timestamp).UtcDateTime + $age = [Math]::Max(0, [int]((Get-Date).ToUniversalTime() - $ts).TotalSeconds) + return [pscustomobject]@{ bucket = $BucketId; found = $true; timestamp = [string]$latest.timestamp; ageSeconds = $age } + } + catch { + return [pscustomobject]@{ bucket = $BucketId; found = $false; timestamp = $null; ageSeconds = $null; error = $_.Exception.Message } + } +} + +function Send-GuardHeartbeat { + param( + [string]$ApiBase, + [string]$Hostname, + [object]$State, + [int]$PulseSeconds + ) + + $bucketId = "aw-rus-collector-guard_$Hostname" + if (-not (Ensure-AwBucket -ApiBase $ApiBase -BucketId $bucketId -ClientName 'aw-rus-collector-guard' -BucketType 'aw.rus.collector.guard' -Hostname $Hostname)) { + return $false + } + + $event = @{ + timestamp = (Get-Date).ToUniversalTime().ToString('o') + duration = 0 + data = $State + } + + try { + Invoke-AwJson -Method 'POST' -Uri "$ApiBase/buckets/$bucketId/heartbeat?pulsetime=$PulseSeconds" -Body $event | Out-Null + return $true + } + catch { + return $false + } +} + +function Read-GuardRuntime { + param([string]$Path) + + if (-not (Test-Path -LiteralPath $Path)) { + return [pscustomobject]@{ restartHistory = @{}; lastAction = @{}; quarantine = @{} } + } + try { + $state = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + if ($null -eq $state.restartHistory) { $state | Add-Member -NotePropertyName restartHistory -NotePropertyValue @{} } + if ($null -eq $state.lastAction) { $state | Add-Member -NotePropertyName lastAction -NotePropertyValue @{} } + if ($null -eq $state.quarantine) { $state | Add-Member -NotePropertyName quarantine -NotePropertyValue @{} } + return $state + } + catch { + return [pscustomobject]@{ restartHistory = @{}; lastAction = @{}; quarantine = @{} } + } +} + +function Write-GuardRuntime { + param( + [string]$Path, + [object]$Runtime + ) + + $directory = Split-Path -Path $Path -Parent + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -Path $directory -ItemType Directory -Force | Out-Null + } + $Runtime | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $Path -Encoding UTF8 +} + +function Get-RuntimeMapValue { + param( + [object]$Map, + [string]$Key + ) + + if ($null -eq $Map) { + return $null + } + if ($Map -is [hashtable] -and $Map.ContainsKey($Key)) { + return $Map[$Key] + } + $propertyNames = @($Map.PSObject.Properties | ForEach-Object { $_.Name }) + if ($propertyNames -contains $Key) { + return $Map.$Key + } + return $null +} + +function Set-RuntimeMapValue { + param( + [object]$Map, + [string]$Key, + [object]$Value + ) + + if ($Map -is [hashtable]) { + $Map[$Key] = $Value + return + } + $propertyNames = @($Map.PSObject.Properties | ForEach-Object { $_.Name }) + if ($propertyNames -contains $Key) { + $Map.$Key = $Value + } + else { + $Map | Add-Member -NotePropertyName $Key -NotePropertyValue $Value -Force + } +} + +function Remove-RuntimeMapValue { + param( + [object]$Map, + [string]$Key + ) + + if ($null -eq $Map) { + return + } + if ($Map -is [hashtable]) { + if ($Map.ContainsKey($Key)) { + $Map.Remove($Key) + } + return + } + $property = $Map.PSObject.Properties[$Key] + if ($null -ne $property) { + $Map.PSObject.Properties.Remove($Key) + } +} + +function Reset-GuardActionBudget { + param( + [object]$Runtime, + [string]$Key + ) + + Remove-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key + Remove-RuntimeMapValue -Map $Runtime.lastAction -Key $Key + Remove-RuntimeMapValue -Map $Runtime.quarantine -Key $Key +} + +function Invoke-GuardSelfTest { + $emptyObject = [pscustomobject]@{} + if ($null -ne (Get-RuntimeMapValue -Map $emptyObject -Key 'missing')) { + throw 'empty PSCustomObject should not return a missing runtime-map value' + } + Set-RuntimeMapValue -Map $emptyObject -Key 'headless:worktime-session' -Value 123 + if ((Get-RuntimeMapValue -Map $emptyObject -Key 'headless:worktime-session') -ne 123) { + throw 'failed to set runtime-map value on empty PSCustomObject' + } + + $hash = @{} + Set-RuntimeMapValue -Map $hash -Key 'headless:worktime-session' -Value @(1, 2) + $hashValue = @(Get-RuntimeMapValue -Map $hash -Key 'headless:worktime-session') + if ($hashValue.Count -ne 2) { + throw 'failed to round-trip runtime-map value on hashtable' + } + + $runtime = [pscustomobject]@{ restartHistory = [pscustomobject]@{}; lastAction = [pscustomobject]@{}; quarantine = [pscustomobject]@{} } + $allowed = Test-ActionAllowed -Runtime $runtime -Key 'headless:worktime-session' -CooldownSeconds 1 -WindowSeconds 60 -MaxCount 3 + if (-not $allowed.allowed) { + throw "expected action to be allowed, got $($allowed.reason)" + } + Register-GuardAction -Runtime $runtime -Key 'headless:worktime-session' + $blocked = Test-ActionAllowed -Runtime $runtime -Key 'headless:worktime-session' -CooldownSeconds 300 -WindowSeconds 60 -MaxCount 3 + if ($blocked.allowed -or $blocked.reason -ne 'cooldown') { + throw 'expected cooldown after registering guard action' + } + $budgetRuntime = [pscustomobject]@{ restartHistory = [pscustomobject]@{}; lastAction = [pscustomobject]@{}; quarantine = [pscustomobject]@{} } + foreach ($i in 1..3) { + Register-GuardAction -Runtime $budgetRuntime -Key 'task:test' + } + $budgetBlocked = Test-ActionAllowed -Runtime $budgetRuntime -Key 'task:test' -CooldownSeconds 0 -WindowSeconds 600 -MaxCount 3 + if ($budgetBlocked.allowed -or $budgetBlocked.reason -ne 'quarantine') { + throw 'expected quarantine when restart budget is exhausted' + } + Reset-GuardActionBudget -Runtime $budgetRuntime -Key 'task:test' + $budgetAllowed = Test-ActionAllowed -Runtime $budgetRuntime -Key 'task:test' -CooldownSeconds 0 -WindowSeconds 600 -MaxCount 3 + if (-not $budgetAllowed.allowed) { + throw 'expected reset action budget to clear quarantine' + } + + $oldComputerName = $env:COMPUTERNAME + try { + $env:COMPUTERNAME = 'SHARKON2025' + $sessionRecords = @( + [pscustomobject]@{ SessionName = 'USER5'; UserName = 'USER5'; SessionId = 2; State = 'Disc'; IsLive = $false }, + [pscustomobject]@{ SessionName = 'console'; UserName = ''; SessionId = 1; State = 'Conn'; IsLive = $true } + ) + $taskDefs = @( + [pscustomobject]@{ taskName = 'ActivityWatch Launch [SHARKON2025_user5]'; userId = 'SHARKON2025\user5' } + ) + if (-not (Test-ActivityWatchUserHasManagedSession -UserId 'SHARKON2025\user5' -SessionRecords $sessionRecords -IncludeDisconnected)) { + throw 'expected disconnected managed session to match task user' + } + if (Test-ActivityWatchUserHasManagedSession -UserId 'SHARKON2025\user5' -SessionRecords $sessionRecords -IncludeLive) { + throw 'disconnected managed session should not match live-only filter' + } + $managed = @(Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $taskDefs -SessionRecords $sessionRecords -IncludeDisconnected) + if ($managed.Count -ne 1 -or [int]$managed[0].SessionId -ne 2) { + throw 'failed to enumerate disconnected managed session' + } + } + finally { + if ($null -eq $oldComputerName) { + Remove-Item Env:COMPUTERNAME -ErrorAction SilentlyContinue + } + else { + $env:COMPUTERNAME = $oldComputerName + } + } + + Write-Output 'collector guard self-test OK' +} + +function Test-ActionAllowed { + param( + [object]$Runtime, + [string]$Key, + [int]$CooldownSeconds, + [int]$WindowSeconds, + [int]$MaxCount + ) + + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $last = Get-RuntimeMapValue -Map $Runtime.lastAction -Key $Key + if ($null -ne $last -and ($now - [int64]$last) -lt $CooldownSeconds) { + return [pscustomobject]@{ allowed = $false; reason = 'cooldown' } + } + + $history = @(Get-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key) + $history = @($history | Where-Object { ($now - [int64]$_) -le $WindowSeconds }) + Set-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key -Value @($history) + if ($history.Count -ge $MaxCount) { + Set-RuntimeMapValue -Map $Runtime.quarantine -Key $Key -Value @{ + since = (Get-Date).ToUniversalTime().ToString('o') + reason = 'restart-budget-exhausted' + count = $history.Count + } + return [pscustomobject]@{ allowed = $false; reason = 'quarantine' } + } + + Remove-RuntimeMapValue -Map $Runtime.quarantine -Key $Key + return [pscustomobject]@{ allowed = $true; reason = 'ok' } +} + +function Register-GuardAction { + param( + [object]$Runtime, + [string]$Key + ) + + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + $history = @(Get-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key) + $history += $now + Set-RuntimeMapValue -Map $Runtime.restartHistory -Key $Key -Value @($history) + Set-RuntimeMapValue -Map $Runtime.lastAction -Key $Key -Value $now +} + +function Get-CollectorProcessSnapshot { + param([pscustomobject]$Config) + + $scriptPaths = [ordered]@{} + foreach ($name in @('collectorScript', 'endpointCollectorScript', 'fileCollectorScript', 'emailCollectorScript', 'sessionCollectorScript')) { + if ($Config.paths.PSObject.Properties.Name -contains $name) { + $value = [string]$Config.paths.$name + if (-not [string]::IsNullOrWhiteSpace($value)) { + $scriptPaths[$name] = $value + } + } + } + + $powershellCollectors = @() + try { + $processes = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { $_.Name -ieq 'powershell.exe' -or $_.Name -ieq 'pwsh.exe' }) + foreach ($proc in $processes) { + $commandLine = [string]$proc.CommandLine + foreach ($entry in $scriptPaths.GetEnumerator()) { + if ($commandLine -match [Regex]::Escape([string]$entry.Value)) { + $powershellCollectors += [pscustomobject]@{ + name = [string]$entry.Key + processId = [int]$proc.ProcessId + sessionId = [int]$proc.SessionId + scriptPath = [string]$entry.Value + } + } + } + } + } + catch { + } + + $watchers = @() + try { + $watchers = @(Get-Process -Name 'aw-watcher-afk','aw-watcher-window' -ErrorAction SilentlyContinue | + Select-Object @{Name='name'; Expression={$_.Name}}, @{Name='processId'; Expression={$_.Id}}, @{Name='sessionId'; Expression={$_.SessionId}}) + } + catch { + $watchers = @() + } + + return [pscustomobject]@{ + watchers = @($watchers) + collectors = @($powershellCollectors) + } +} + +function Invoke-ExactTaskRun { + param([string]$TaskName) + + & schtasks.exe /Run /TN $TaskName | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Invoke-GuardCycle { + param( + [object]$Runtime, + [string]$RuntimePath, + [string]$LogPath + ) + + $config = Read-ActivityWatchDeploymentConfig -Path $ConfigPath + $stateRoot = if ($config.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$config.paths.stateRoot } else { Split-Path -Path $ConfigPath -Parent } + $hostname = if ($config.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]$config.awHostname)) { [string]$config.awHostname } else { [string]$env:COMPUTERNAME } + $apiBase = Get-AwApiBase -Config $config + $configPaths = Get-ActivityWatchRecoveryConfigPaths -PrimaryConfigPath $ConfigPath + $taskDefs = @(Get-ActivityWatchRecoveryTaskDefinitions -ConfigPaths $configPaths) + $sessionRecords = @(Get-ActivityWatchSessionRecords) + $liveSessions = @(Get-ActivityWatchLiveInteractiveSessions -SessionRecords $sessionRecords) + $managedInteractiveSessions = @(Get-ActivityWatchManagedInteractiveSessions -TaskDefinitions $taskDefs -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected) + $processSnapshot = Get-CollectorProcessSnapshot -Config $config + $liveSessionIds = @($liveSessions | ForEach-Object { [int]$_.SessionId }) + $managedSessionIds = @($managedInteractiveSessions | ForEach-Object { [int]$_.SessionId } | Sort-Object -Unique) + + $bucketChecks = [ordered]@{} + foreach ($bucket in @( + "aw-worktime-sessions_$hostname", + "aw-watcher-afk_$hostname", + "aw-watcher-window_$hostname", + "aw-dlp-endpoint-signals_$hostname" + )) { + $bucketChecks[$bucket] = Get-LatestBucketAge -ApiBase $apiBase -BucketId $bucket + } + + $actions = New-Object System.Collections.Generic.List[object] + $problems = New-Object System.Collections.Generic.List[string] + + $worktimeAge = $bucketChecks["aw-worktime-sessions_$hostname"].ageSeconds + $sessionCollectorScript = if ($config.paths.PSObject.Properties.Name -contains 'sessionCollectorScript') { [string]$config.paths.sessionCollectorScript } else { Join-Path $stateRoot 'worktime-session-collector.ps1' } + $sessionCollectorRunning = Test-ActivityWatchCollectorRunningGlobal -ScriptPath $sessionCollectorScript + $headlessKey = 'headless:worktime-session' + $needsHeadlessAction = (-not $sessionCollectorRunning -or $null -eq $worktimeAge -or [int]$worktimeAge -gt $HeadlessMaxAgeSeconds) + if ($needsHeadlessAction) { + $key = $headlessKey + $allowed = Test-ActionAllowed -Runtime $Runtime -Key $key -CooldownSeconds $ActionCooldownSeconds -WindowSeconds $RestartWindowSeconds -MaxCount $MaxRestarts + if ($allowed.allowed) { + if ($Mode -eq 'enforce') { + Start-ActivityWatchCollectorScriptGlobalIfNeeded -ScriptPath $sessionCollectorScript -ConfigPath $ConfigPath + Register-GuardAction -Runtime $Runtime -Key $key + Write-GuardLog -LogPath $LogPath -Message "started $key" + $actions.Add([pscustomobject]@{ action = 'start'; target = $key; applied = $true }) | Out-Null + } + else { + $actions.Add([pscustomobject]@{ action = 'start'; target = $key; applied = $false; mode = 'shadow' }) | Out-Null + } + } + else { + $problems.Add("$key action blocked: $($allowed.reason)") | Out-Null + } + } + else { + Reset-GuardActionBudget -Runtime $Runtime -Key $headlessKey + } + + if ($Mode -eq 'enforce') { + Stop-ActivityWatchProcessesInNonLiveSessions -SessionRecords $sessionRecords -Config $config -TaskDefinitions $taskDefs -PreserveManagedSessions + } + + $interactiveStale = $false + foreach ($bucket in @("aw-watcher-afk_$hostname", "aw-watcher-window_$hostname", "aw-dlp-endpoint-signals_$hostname")) { + $age = $bucketChecks[$bucket].ageSeconds + if ($null -eq $age -or [int]$age -gt $InteractiveMaxAgeSeconds) { + $interactiveStale = $true + } + } + + $watchersInLiveSessions = @($processSnapshot.watchers | Where-Object { $liveSessionIds -contains [int]$_.sessionId }) + $watchersInManagedSessions = @($processSnapshot.watchers | Where-Object { $managedSessionIds -contains [int]$_.sessionId }) + $liveWatcherMissing = $false + if ($liveSessions.Count -gt 0) { + $hasAfk = @($watchersInLiveSessions | Where-Object { [string]$_.name -ieq 'aw-watcher-afk' }).Count -gt 0 + $hasWindow = @($watchersInLiveSessions | Where-Object { [string]$_.name -ieq 'aw-watcher-window' }).Count -gt 0 + $liveWatcherMissing = (-not $hasAfk) -or (-not $hasWindow) + } + $managedWatcherMissing = $false + if ($managedInteractiveSessions.Count -gt 0) { + $afkEnabled = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'afkEnabled') { [bool]$config.collectors.afkEnabled } else { $true } + $windowEnabled = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'windowEnabled') { [bool]$config.collectors.windowEnabled } else { $true } + foreach ($managedSession in @($managedInteractiveSessions)) { + $sessionId = [int]$managedSession.SessionId + $sessionWatchers = @($watchersInManagedSessions | Where-Object { [int]$_.sessionId -eq $sessionId }) + $hasManagedAfk = @($sessionWatchers | Where-Object { [string]$_.name -ieq 'aw-watcher-afk' }).Count -gt 0 + $hasManagedWindow = @($sessionWatchers | Where-Object { [string]$_.name -ieq 'aw-watcher-window' }).Count -gt 0 + if (($afkEnabled -and -not $hasManagedAfk) -or ($windowEnabled -and -not $hasManagedWindow)) { + $managedWatcherMissing = $true + break + } + } + } + + if ($managedInteractiveSessions.Count -eq 0 -and $liveSessions.Count -gt 0 -and $interactiveStale) { + $problems.Add('interactive buckets stale but no managed interactive sessions found') | Out-Null + } + + $needsInteractiveTaskAction = $managedInteractiveSessions.Count -gt 0 -and ( + $liveWatcherMissing -or + $managedWatcherMissing -or + ($interactiveStale -and $liveSessions.Count -gt 0) + ) + + if ($needsInteractiveTaskAction) { + foreach ($taskDef in $taskDefs) { + if (-not (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected)) { + continue + } + $key = "task:$($taskDef.taskName)" + $allowed = Test-ActionAllowed -Runtime $Runtime -Key $key -CooldownSeconds $InteractiveActionCooldownSeconds -WindowSeconds $RestartWindowSeconds -MaxCount $MaxRestarts + if (-not $allowed.allowed) { + $problems.Add("$key action blocked: $($allowed.reason)") | Out-Null + continue + } + + if ($Mode -eq 'enforce') { + $ok = Invoke-ExactTaskRun -TaskName ([string]$taskDef.taskName) + if ($ok) { + Register-GuardAction -Runtime $Runtime -Key $key + } + Write-GuardLog -LogPath $LogPath -Message ("run {0} ok={1}" -f $key, $ok) + $actions.Add([pscustomobject]@{ action = 'run-task'; target = [string]$taskDef.taskName; applied = $true; ok = $ok }) | Out-Null + } + else { + $actions.Add([pscustomobject]@{ action = 'run-task'; target = [string]$taskDef.taskName; applied = $false; mode = 'shadow' }) | Out-Null + } + } + } + else { + foreach ($taskDef in $taskDefs) { + if (Test-ActivityWatchUserHasManagedSession -UserId ([string]$taskDef.userId) -SessionRecords $sessionRecords -IncludeLive -IncludeDisconnected) { + Reset-GuardActionBudget -Runtime $Runtime -Key "task:$($taskDef.taskName)" + } + } + } + + $status = 'ok' + if ($problems.Count -gt 0) { + $status = 'warn' + } + if ($managedInteractiveSessions.Count -gt 0 -and $interactiveStale -and $Mode -eq 'shadow') { + $status = 'warn' + } + + $sessionState = @( + foreach ($session in @($sessionRecords)) { + [pscustomobject]@{ + SessionName = [string]$session.SessionName + UserName = [string]$session.UserName + SessionId = [int]$session.SessionId + State = [string]$session.State + IsLive = [bool]$session.IsLive + } + } + ) + + $state = @{} + $state['status'] = $status + $state['mode'] = $Mode + $state['host'] = $hostname + $state['generatedAtUtc'] = (Get-Date).ToUniversalTime().ToString('o') + $state['pid'] = $PID + $state['sessions'] = @($sessionState) + $state['liveSessionCount'] = $liveSessions.Count + $state['managedSessionCount'] = $managedInteractiveSessions.Count + $state['managedSessions'] = @($managedInteractiveSessions) + $bucketState = @{} + foreach ($key in $bucketChecks.Keys) { + $bucketState[$key] = $bucketChecks[$key] + } + + $state['processes'] = $processSnapshot + $state['buckets'] = $bucketState + $state['actions'] = @($actions.ToArray()) + $state['problems'] = @($problems.ToArray()) + $state['quarantine'] = $Runtime.quarantine + + $statePath = Join-Path $stateRoot 'collector-guard-state.json' + $state | ConvertTo-Json -Depth 16 | Set-Content -LiteralPath $statePath -Encoding UTF8 + Write-GuardRuntime -Path $RuntimePath -Runtime $Runtime + [void](Send-GuardHeartbeat -ApiBase $apiBase -Hostname $hostname -State $state -PulseSeconds ([Math]::Max($LoopSeconds * 2, 60))) + return $state +} + +if ($SelfTest) { + Invoke-GuardSelfTest + exit 0 +} + +$initialConfig = Read-ActivityWatchDeploymentConfig -Path $ConfigPath +$initialStateRoot = if ($initialConfig.paths.PSObject.Properties.Name -contains 'stateRoot') { [string]$initialConfig.paths.stateRoot } else { Split-Path -Path $ConfigPath -Parent } +$initialLogsRoot = if ($initialConfig.paths.PSObject.Properties.Name -contains 'logsRoot') { [string]$initialConfig.paths.logsRoot } else { Join-Path $initialStateRoot 'logs' } +$logPath = Join-Path $initialLogsRoot 'collector-guard.log' +$runtimePath = Join-Path $initialStateRoot 'collector-guard-runtime.json' +$lockPath = New-GuardLock -StateRoot $initialStateRoot +if (-not $lockPath) { + Write-GuardLog -LogPath $logPath -Message 'another collector guard instance is already running' + exit 0 +} + +try { + $runtime = Read-GuardRuntime -Path $runtimePath + Write-GuardLog -LogPath $logPath -Message "collector guard started mode=$Mode loop=$LoopSeconds once=$($Once.IsPresent)" + while ($true) { + try { + Invoke-GuardCycle -Runtime $runtime -RuntimePath $runtimePath -LogPath $logPath | Out-Null + } + catch { + Write-GuardLog -LogPath $logPath -Message ("cycle error: {0}; at {1}" -f $_.Exception.Message, $_.ScriptStackTrace) + } + + if ($Once) { + break + } + Start-Sleep -Seconds ([Math]::Max($LoopSeconds, 15)) + } +} +finally { + if ($lockPath -and (Test-Path -LiteralPath $lockPath)) { + Remove-Item -LiteralPath $lockPath -Force -ErrorAction SilentlyContinue + } + Write-GuardLog -LogPath $logPath -Message 'collector guard stopped' +} diff --git a/windows/deploy-domain-users.ps1 b/windows/deploy-domain-users.ps1 index c8b66f7..4370b43 100755 --- a/windows/deploy-domain-users.ps1 +++ b/windows/deploy-domain-users.ps1 @@ -27,7 +27,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath, diff --git a/windows/deploy-ensemble.ps1 b/windows/deploy-ensemble.ps1 index cbcf689..e21413a 100644 --- a/windows/deploy-ensemble.ps1 +++ b/windows/deploy-ensemble.ps1 @@ -27,7 +27,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath, diff --git a/windows/deploy-single-user.ps1 b/windows/deploy-single-user.ps1 index 25b7221..eb6c739 100755 --- a/windows/deploy-single-user.ps1 +++ b/windows/deploy-single-user.ps1 @@ -25,7 +25,7 @@ param( [int]$EvtxRetentionDays = 14, [string[]]$EvtxChannels = @(), [bool]$LogonMarkerEnabled = $true, - [bool]$ProcessEventsEnabled = $true, + [bool]$ProcessEventsEnabled = $false, [string]$AwHostname, [string]$CustomRulesPath, [string]$CustomPolicyPath diff --git a/windows/dlp-endpoint-signals-collector.ps1 b/windows/dlp-endpoint-signals-collector.ps1 index 1350713..8fe1831 100644 --- a/windows/dlp-endpoint-signals-collector.ps1 +++ b/windows/dlp-endpoint-signals-collector.ps1 @@ -15,7 +15,8 @@ param( [int]$PolicyRefreshSeconds, [string]$PolicyCachePath, [string]$LogPath, - [int]$PollSeconds + [int]$PollSeconds, + [switch]$SelfTestSuppressedBlock ) Set-StrictMode -Version Latest @@ -30,6 +31,7 @@ catch { $script:TransportQueuePath = $null $script:TransportQueueLockPath = $null +$script:SessionId = [System.Diagnostics.Process]::GetCurrentProcess().SessionId $script:TransportMetrics = @{ eventsEnqueued = 0 eventsFlushed = 0 @@ -71,6 +73,21 @@ function Write-EndpointLog { } } +function Get-QueueNameToken { + param( + [string]$UserName, + [int]$SessionId + ) + $token = ('{0}-s{1}' -f $UserName, $SessionId) + foreach ($ch in [System.IO.Path]::GetInvalidFileNameChars()) { + $token = $token.Replace([string]$ch, '_') + } + if ([string]::IsNullOrWhiteSpace($token)) { + return ('session-{0}' -f $SessionId) + } + return $token +} + function Invoke-AwJsonPost { param( [Parameter(Mandatory = $true)][string]$Uri, @@ -129,11 +146,20 @@ function Invoke-AwJsonPost { function Initialize-TransportQueue { param([Parameter(Mandatory = $true)][string]$StateRoot) - $script:TransportQueuePath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.jsonl' - $script:TransportQueueLockPath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.lock' + $queueToken = Get-QueueNameToken -UserName $env:USERNAME -SessionId $script:SessionId + $script:TransportQueuePath = Join-Path $StateRoot ("dlp-endpoint-signals-queue-{0}.jsonl" -f $queueToken) + $script:TransportQueueLockPath = Join-Path $StateRoot ("dlp-endpoint-signals-queue-{0}.lock" -f $queueToken) if (-not (Test-Path -LiteralPath $script:TransportQueuePath)) { New-Item -Path $script:TransportQueuePath -ItemType File -Force | Out-Null } + $legacyQueuePath = Join-Path $StateRoot 'dlp-endpoint-signals-queue.jsonl' + if (Test-Path -LiteralPath $legacyQueuePath) { + $legacyItems = @(Get-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($legacyItems.Count -gt 0) { + Add-Content -LiteralPath $script:TransportQueuePath -Value $legacyItems -Encoding UTF8 + Clear-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue + } + } } function Get-TransportQueueLock { @@ -617,6 +643,17 @@ function Load-DlpPolicy { regexPack = $null ocrEnabled = $false } + nativeControls = [ordered]@{ + mode = 'monitor' + rollout = [ordered]@{ + allowGlobalBlock = $false + } + channels = [ordered]@{ + clipboard = [ordered]@{ action = 'audit' } + usb = [ordered]@{ action = 'audit' } + print = [ordered]@{ action = 'audit' } + } + } } $script:PolicySource = 'defaults' @@ -656,6 +693,28 @@ function Load-DlpPolicy { $script:Policy.contentAnalysis.ocrEnabled = [bool]$raw.contentAnalysis.ocrEnabled } } + if ($raw.nativeControls) { + $nativeProps = @($raw.nativeControls.PSObject.Properties.Name) + if ($nativeProps -contains 'mode' -and $raw.nativeControls.mode) { + $script:Policy.nativeControls.mode = ([string]$raw.nativeControls.mode).ToLowerInvariant() + } + if ($nativeProps -contains 'rollout' -and $raw.nativeControls.rollout) { + $rolloutProps = @($raw.nativeControls.rollout.PSObject.Properties.Name) + if ($rolloutProps -contains 'allowGlobalBlock') { + $script:Policy.nativeControls.rollout.allowGlobalBlock = [bool]$raw.nativeControls.rollout.allowGlobalBlock + } + } + if ($nativeProps -contains 'channels' -and $raw.nativeControls.channels) { + foreach ($channelName in @('clipboard', 'usb', 'print')) { + if (@($raw.nativeControls.channels.PSObject.Properties.Name) -contains $channelName) { + $channel = $raw.nativeControls.channels.$channelName + if ($channel -and (@($channel.PSObject.Properties.Name) -contains 'action') -and $channel.action) { + $script:Policy.nativeControls.channels[$channelName].action = ([string]$channel.action).ToLowerInvariant() + } + } + } + } + } $script:PolicySource = 'local' } catch { @@ -663,6 +722,68 @@ function Load-DlpPolicy { } } +function Resolve-DlpEffectiveAction { + param( + [Parameter(Mandatory = $true)][string]$RequestedAction, + [Parameter(Mandatory = $true)][ValidateSet('clipboard', 'usb', 'print')][string]$Channel + ) + + $requested = $RequestedAction.ToLowerInvariant() + $mode = ([string]$script:Policy.nativeControls.mode).ToLowerInvariant() + $allowGlobalBlock = [bool]$script:Policy.nativeControls.rollout.allowGlobalBlock + $channelAction = 'audit' + try { + $channelAction = ([string]$script:Policy.nativeControls.channels[$Channel].action).ToLowerInvariant() + } + catch { + $channelAction = 'audit' + } + + $suppressed = $false + $effective = $requested + if ($requested -eq 'block') { + $channelAllowsBlock = $channelAction -in @('block', 'blockwithoverride') + if ($mode -ne 'enforce' -or -not $allowGlobalBlock -or -not $channelAllowsBlock) { + $effective = 'alert' + $suppressed = $true + } + } + + return [pscustomobject]@{ + requestedAction = $requested + action = $effective + enforcementMode = $mode + nativeChannelAction = $channelAction + enforcementSuppressed = $suppressed + } +} + +function Invoke-SuppressedBlockSelfTest { + $decisions = @() + foreach ($channel in @('clipboard', 'usb', 'print')) { + $decisions += (Resolve-DlpEffectiveAction -RequestedAction 'block' -Channel $channel) + } + + $failed = @( + $decisions | + Where-Object { $_.action -eq 'block' -or -not [bool]$_.enforcementSuppressed } + ) + + $result = [ordered]@{ + ok = (@($failed).Count -eq 0) + test = 'suppressed-block-in-monitor' + policySource = $script:PolicySource + policyMode = $script:PolicyMode + decisions = @($decisions) + } + + $result | ConvertTo-Json -Depth 6 + if (-not $result.ok) { + exit 2 + } + exit 0 +} + function Test-ValidInn { param([string]$Value) $digits = ($Value -replace '\D', '') @@ -925,7 +1046,9 @@ function Evaluate-ClipboardRules { $fingerprint = "clipboard|$ruleId|$ClipboardHash|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'clipboard' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "Clipboard rule matched: $ruleId" } @@ -939,13 +1062,17 @@ function Evaluate-ClipboardRules { clipboardHash = $ClipboardHash clipboardLength = $ClipboardText.Length enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed dictionaryPack = $dictionaryPack regexPack = $regexPack dictionaryMatches = @($advanced.dictionaryMatches) regexMatches = @($advanced.regexMatches) ocrRequested = $ocrEnabled } - Write-EndpointLog ("incident clipboard rule={0} action={1} severity={2} enforced={3}" -f $ruleId, $action, $severity, $enforced) + Write-EndpointLog ("incident clipboard rule={0} requested={1} action={2} severity={3} enforced={4} suppressed={5}" -f $ruleId, $requestedAction, $action, $severity, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -965,7 +1092,9 @@ function Evaluate-UsbRules { $fingerprint = "usb|$ruleId|$DriveLetter|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'usb' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "USB rule matched: $ruleId" } @@ -979,8 +1108,12 @@ function Evaluate-UsbRules { driveLetter = $DriveLetter volumeName = $VolumeName enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed } - Write-EndpointLog ("incident usb rule={0} action={1} severity={2} drive={3} enforced={4}" -f $ruleId, $action, $severity, $DriveLetter, $enforced) + Write-EndpointLog ("incident usb rule={0} requested={1} action={2} severity={3} drive={4} enforced={5} suppressed={6}" -f $ruleId, $requestedAction, $action, $severity, $DriveLetter, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -1016,7 +1149,9 @@ function Evaluate-PrintRules { $fingerprint = "print|$ruleId|$PrinterName|$Owner|$env:USERNAME" if (-not (Should-EmitByCooldown -Fingerprint $fingerprint -CooldownSeconds ([Math]::Max($cooldown, 30)))) { continue } - $action = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $requestedAction = if ($rule.action) { [string]$rule.action } else { [string]$script:Policy.defaults.action } + $actionDecision = Resolve-DlpEffectiveAction -RequestedAction $requestedAction -Channel 'print' + $action = [string]$actionDecision.action $severity = if ($rule.severity) { [string]$rule.severity } else { [string]$script:Policy.defaults.severity } $message = if ($rule.message) { [string]$rule.message } else { "Print rule matched: $ruleId" } @@ -1031,13 +1166,17 @@ function Evaluate-PrintRules { documentName = $DocumentName owner = $Owner enforced = $enforced + requestedAction = [string]$actionDecision.requestedAction + enforcementMode = [string]$actionDecision.enforcementMode + nativeChannelAction = [string]$actionDecision.nativeChannelAction + enforcementSuppressed = [bool]$actionDecision.enforcementSuppressed dictionaryPack = $dictionaryPack regexPack = $regexPack dictionaryMatches = @($advanced.dictionaryMatches) regexMatches = @($advanced.regexMatches) ocrRequested = $ocrEnabled } - Write-EndpointLog ("incident print rule={0} action={1} severity={2} printer={3} enforced={4}" -f $ruleId, $action, $severity, $PrinterName, $enforced) + Write-EndpointLog ("incident print rule={0} requested={1} action={2} severity={3} printer={4} enforced={5} suppressed={6}" -f $ruleId, $requestedAction, $action, $severity, $PrinterName, $enforced, [bool]$actionDecision.enforcementSuppressed) } } @@ -1320,6 +1459,9 @@ $script:LastEventTime = $null Initialize-TransportQueue -StateRoot $resolvedStateRoot Initialize-DlpPolicy +if ($SelfTestSuppressedBlock) { + Invoke-SuppressedBlockSelfTest +} Write-EndpointLog ("endpoint collector started against {0}" -f $script:ApiBase) while ($true) { diff --git a/windows/dlp-policy-client.ps1 b/windows/dlp-policy-client.ps1 index 541df91..fd23786 100644 --- a/windows/dlp-policy-client.ps1 +++ b/windows/dlp-policy-client.ps1 @@ -154,4 +154,6 @@ function Save-CachedDlpPolicyBundle { Set-Content -LiteralPath $CachePath -Value $json -Encoding UTF8 } -Export-ModuleMember -Function Invoke-DlpPolicyGetJson, Get-RemoteDlpPolicyBundle, Get-RemoteDlpPolicyDesired, Send-DlpPolicyAgentHeartbeat, Read-CachedDlpPolicyBundle, Save-CachedDlpPolicyBundle +if ($ExecutionContext.SessionState.Module) { + Export-ModuleMember -Function Invoke-DlpPolicyGetJson, Get-RemoteDlpPolicyBundle, Get-RemoteDlpPolicyDesired, Send-DlpPolicyAgentHeartbeat, Read-CachedDlpPolicyBundle, Save-CachedDlpPolicyBundle +} diff --git a/windows/dlp-policy.example.json b/windows/dlp-policy.example.json index 7c8c40f..828f234 100644 --- a/windows/dlp-policy.example.json +++ b/windows/dlp-policy.example.json @@ -6,6 +6,19 @@ "action": "log", "severity": "low" }, + "nativeControls": { + "mode": "monitor", + "rollout": { + "baselineDays": 7, + "requireGuardHeartbeat": true, + "allowGlobalBlock": false + }, + "channels": { + "clipboard": {"action": "audit"}, + "usb": {"action": "audit"}, + "print": {"action": "audit"} + } + }, "rules": [ { "id": "personal-web-during-workhours", diff --git a/windows/dlp-policy.native-cross-os.example.json b/windows/dlp-policy.native-cross-os.example.json new file mode 100644 index 0000000..b8710b0 --- /dev/null +++ b/windows/dlp-policy.native-cross-os.example.json @@ -0,0 +1,145 @@ +{ + "version": 1, + "defaults": { + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium" + }, + "nativeControls": { + "mode": "monitor", + "rollout": { + "baselineDays": 7, + "requireGuardHeartbeat": true, + "allowGlobalBlock": false + }, + "channels": { + "removableStorage": { + "action": "audit", + "windows": { + "mechanisms": ["gpo-device-restrictions", "set-disk-readonly"], + "target": "write" + }, + "linux": { + "mechanisms": ["fanotify", "auditd"], + "target": "mountpoints" + }, + "macos": { + "mechanisms": ["mdm-restrictions", "endpoint-security"], + "target": "managed-devices" + }, + "chromeos": { + "mechanisms": ["data-controls"], + "target": "removable-storage" + } + }, + "print": { + "action": "audit", + "windows": { + "mechanisms": ["printservice-operational-log", "spooler-cancel-job"] + }, + "linux": { + "mechanisms": ["cups-logs"], + "enforcement": "monitor-only" + }, + "macos": { + "mechanisms": ["mdm-printing-restrictions"], + "enforcement": "managed-only" + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "clipboard": { + "action": "audit", + "windows": { + "mechanisms": ["clipboard-monitor", "clear-clipboard"], + "blockScope": "high-confidence-only" + }, + "linux": { + "mechanisms": ["desktop-clipboard-monitor"], + "enforcement": "monitor-only" + }, + "macos": { + "mechanisms": ["endpoint-monitor", "mdm-restrictions"], + "enforcement": "monitor-first" + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "browserUpload": { + "action": "audit", + "windows": { + "mechanisms": ["managed-browser-policy", "browser-extension"] + }, + "linux": { + "mechanisms": ["managed-browser-policy", "proxy-logs"] + }, + "macos": { + "mechanisms": ["managed-browser-policy", "network-extension"] + }, + "chromeos": { + "mechanisms": ["data-controls"] + } + }, + "appExecution": { + "action": "audit", + "windows": { + "mechanisms": ["applocker", "wdac"] + }, + "linux": { + "mechanisms": ["auditd", "fanotify", "bpf-lsm"], + "enforcement": "fanotify-or-lsm-only" + }, + "macos": { + "mechanisms": ["mdm-restrictions", "endpoint-security"] + }, + "chromeos": { + "mechanisms": ["admin-console-app-policy"] + } + } + } + }, + "endpoint": { + "clipboard": [ + { + "id": "clipboard-sensitive-keywords", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "high", + "message": "В буфере обнаружены чувствительные ключевые слова", + "minLength": 20, + "regexPatterns": [ + "(?i)парол(ь|и)", + "(?i)password", + "(?i)secret", + "(?i)cvv", + "(?i)паспорт" + ] + } + ], + "usb": [ + { + "id": "usb-media-connected", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "medium", + "message": "Подключен съемный носитель" + } + ], + "print": [ + { + "id": "print-sensitive-docs", + "enabled": true, + "cooldownSeconds": 300, + "action": "alert", + "severity": "high", + "message": "Печать документа с признаками чувствительных данных", + "documentRegex": "(?i)(salary|зарплат|passport|паспорт|договор|contract)" + } + ] + } +} diff --git a/windows/file-operations-collector.ps1 b/windows/file-operations-collector.ps1 index 09ddb59..dd6f1e9 100644 --- a/windows/file-operations-collector.ps1 +++ b/windows/file-operations-collector.ps1 @@ -66,6 +66,21 @@ function Write-StartupTrace { } } +function Get-QueueNameToken { + param( + [string]$UserName, + [int]$SessionId + ) + $token = ('{0}-s{1}' -f $UserName, $SessionId) + foreach ($ch in [System.IO.Path]::GetInvalidFileNameChars()) { + $token = $token.Replace([string]$ch, '_') + } + if ([string]::IsNullOrWhiteSpace($token)) { + return ('session-{0}' -f $SessionId) + } + return $token +} + function Invoke-AwJsonPost { param( [Parameter(Mandatory = $true)][string]$Uri, @@ -97,11 +112,20 @@ function Initialize-TransportQueue { param( [Parameter(Mandatory = $true)][string]$StateRoot ) - $script:TransportQueuePath = Join-Path $StateRoot 'file-operations-queue.jsonl' - $script:TransportQueueLockPath = Join-Path $StateRoot 'file-operations-queue.lock' + $queueToken = Get-QueueNameToken -UserName $env:USERNAME -SessionId $script:SessionId + $script:TransportQueuePath = Join-Path $StateRoot ("file-operations-queue-{0}.jsonl" -f $queueToken) + $script:TransportQueueLockPath = Join-Path $StateRoot ("file-operations-queue-{0}.lock" -f $queueToken) if (-not (Test-Path -LiteralPath $script:TransportQueuePath)) { New-Item -Path $script:TransportQueuePath -ItemType File -Force | Out-Null } + $legacyQueuePath = Join-Path $StateRoot 'file-operations-queue.jsonl' + if (Test-Path -LiteralPath $legacyQueuePath) { + $legacyItems = @(Get-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($legacyItems.Count -gt 0) { + Add-Content -LiteralPath $script:TransportQueuePath -Value $legacyItems -Encoding UTF8 + Clear-Content -LiteralPath $legacyQueuePath -ErrorAction SilentlyContinue + } + } } function Get-TransportQueueLock { diff --git a/windows/hardening-recovery.ps1 b/windows/hardening-recovery.ps1 index b28b76c..fb4b6ef 100755 --- a/windows/hardening-recovery.ps1 +++ b/windows/hardening-recovery.ps1 @@ -94,7 +94,7 @@ $effectiveEvtxExportRoot = if ($PSBoundParameters.ContainsKey('EvtxExportRoot') $effectiveEvtxRetentionDays = if ($PSBoundParameters.ContainsKey('EvtxRetentionDays')) { [int]$EvtxRetentionDays } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'retentionDays') { [int]$existingConfig.forensics.retentionDays } else { 14 } $effectiveEvtxChannels = if ($PSBoundParameters.ContainsKey('EvtxChannels')) { @($EvtxChannels) } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'forensics' -and $existingConfig.forensics.PSObject.Properties.Name -contains 'evtxChannels') { @($existingConfig.forensics.evtxChannels) } else { @() } $effectiveLogonMarkerEnabled = if ($PSBoundParameters.ContainsKey('LogonMarkerEnabled')) { [bool]$LogonMarkerEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'logonEnabled') { [bool]$existingConfig.sessionEvents.logonEnabled } else { $true } -$effectiveProcessEventsEnabled = if ($PSBoundParameters.ContainsKey('ProcessEventsEnabled')) { [bool]$ProcessEventsEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$existingConfig.sessionEvents.processEventsEnabled } else { $true } +$effectiveProcessEventsEnabled = if ($PSBoundParameters.ContainsKey('ProcessEventsEnabled')) { [bool]$ProcessEventsEnabled } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'sessionEvents' -and $existingConfig.sessionEvents.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$existingConfig.sessionEvents.processEventsEnabled } else { $false } $effectiveAwHostname = if ($PSBoundParameters.ContainsKey('AwHostname') -and -not [string]::IsNullOrWhiteSpace($AwHostname)) { [string]$AwHostname } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'awHostname' -and -not [string]::IsNullOrWhiteSpace([string]$existingConfig.awHostname)) { [string]$existingConfig.awHostname } else { [string]$env:COMPUTERNAME } $effectiveVersion = if ($Version) { $Version } elseif ($existingConfig) { [string]$existingConfig.package.version } else { 'v0.13.2' } $effectivePolicyMode = if ($PSBoundParameters.ContainsKey('PolicyMode') -and $PolicyMode) { [string]$PolicyMode } elseif ($existingConfig -and $existingConfig.PSObject.Properties.Name -contains 'policyEngine' -and $existingConfig.policyEngine.PSObject.Properties.Name -contains 'mode') { [string]$existingConfig.policyEngine.mode } else { 'local' } diff --git a/windows/install-collector-guard-service.ps1 b/windows/install-collector-guard-service.ps1 new file mode 100644 index 0000000..a6e270c --- /dev/null +++ b/windows/install-collector-guard-service.ps1 @@ -0,0 +1,88 @@ +[CmdletBinding()] +param( + [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json', + [ValidateSet('shadow', 'enforce')] + [string]$Mode = 'shadow', + [string]$ServiceName = 'AWatchRusCollectorGuard', + [int]$LoopSeconds = 60, + [switch]$DisableRecoveryTask +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Assert-Admin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = [Security.Principal.WindowsPrincipal]::new($id) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Run as Administrator.' + } +} + +Assert-Admin + +$guardScriptPath = Join-Path $PSScriptRoot 'aw-collector-guard.ps1' +$serviceSourcePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.cs' +$serviceExePath = Join-Path $PSScriptRoot 'AWatchRusCollectorGuardService.exe' +if (-not (Test-Path -LiteralPath $guardScriptPath)) { + throw "Collector guard script not found: $guardScriptPath" +} +if (-not (Test-Path -LiteralPath $serviceSourcePath)) { + throw "Collector guard service source not found: $serviceSourcePath" +} +if (-not (Test-Path -LiteralPath $ConfigPath)) { + throw "Config not found: $ConfigPath" +} + +$cscCandidates = @( + (Join-Path $env:WINDIR 'Microsoft.NET\Framework64\v4.0.30319\csc.exe'), + (Join-Path $env:WINDIR 'Microsoft.NET\Framework\v4.0.30319\csc.exe') +) +$csc = @($cscCandidates | Where-Object { Test-Path -LiteralPath $_ } | Select-Object -First 1) +if (-not $csc) { + throw 'C# compiler not found. Install .NET Framework build tools or provide AWatchRusCollectorGuardService.exe.' +} + +$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue +if ($existing) { + if ($existing.Status -ne 'Stopped') { + Stop-Service -Name $ServiceName -Force -ErrorAction SilentlyContinue + try { + $existing.WaitForStatus('Stopped', [TimeSpan]::FromSeconds(20)) + } + catch { + } + } + sc.exe delete $ServiceName | Out-Null + Start-Sleep -Seconds 2 +} + +& $csc /nologo /target:exe /optimize+ /out:$serviceExePath /reference:System.ServiceProcess.dll $serviceSourcePath | Out-Null +if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $serviceExePath)) { + throw "Failed to compile $serviceExePath" +} + +$logsRoot = Join-Path (Split-Path -Path $ConfigPath -Parent) 'logs' +$serviceLogPath = Join-Path $logsRoot 'collector-guard-service.log' +$binPath = "`"$serviceExePath`" --service-name `"$ServiceName`" --script `"$guardScriptPath`" --config `"$ConfigPath`" --mode $Mode --loop $LoopSeconds --log `"$serviceLogPath`"" + +New-Service -Name $ServiceName -BinaryPathName $binPath -DisplayName 'AWatch-rus Collector Guard' -StartupType Automatic | Out-Null +sc.exe description $ServiceName "Session-aware ActivityWatch collector guard for AWatch-rus" | Out-Null +sc.exe failure $ServiceName reset= 300 actions= restart/5000/restart/15000/restart/60000 | Out-Null + +if ($DisableRecoveryTask) { + Write-Warning 'DisableRecoveryTask is deprecated and ignored: ActivityWatch Recovery must remain enabled as collector guard fallback.' +} +else { + try { + Enable-ScheduledTask -TaskName 'ActivityWatch Recovery' -ErrorAction SilentlyContinue | Out-Null + } + catch { + } +} + +sc.exe start $ServiceName | Out-Null + +Write-Output "Collector guard service installed: $ServiceName" +Write-Output "Mode: $Mode" +Write-Output "Config: $ConfigPath" diff --git a/windows/installkit/innosetup/AWatch-rus-InnoSetup.iss b/windows/installkit/innosetup/AWatch-rus-InnoSetup.iss index 7103626..b510572 100644 --- a/windows/installkit/innosetup/AWatch-rus-InnoSetup.iss +++ b/windows/installkit/innosetup/AWatch-rus-InnoSetup.iss @@ -44,6 +44,9 @@ Source: "..\..\aw-standalone-service.ps1"; DestDir: "{app}\windows"; Flags: igno Source: "..\..\deploy-single-user.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\deploy-domain-users.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\deploy-ensemble.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion +Source: "..\..\AWatchRusCollectorGuardService.cs"; DestDir: "{app}\windows"; Flags: ignoreversion +Source: "..\..\aw-collector-guard.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion +Source: "..\..\install-collector-guard-service.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\hardening-recovery.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\validate-deployment.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\migrate-awatch-rus-paths.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion @@ -54,6 +57,7 @@ Source: "..\..\file-operations-collector.ps1"; DestDir: "{app}\windows"; Flags: Source: "..\..\email-outbound-collector.ps1"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\web-category-rules.example.json"; DestDir: "{app}\windows"; Flags: ignoreversion Source: "..\..\dlp-policy.example.json"; DestDir: "{app}\windows"; Flags: ignoreversion +Source: "..\..\dlp-policy.native-cross-os.example.json"; DestDir: "{app}\windows"; Flags: ignoreversion ; Offline payload (optional): place ZIP into windows/installkit/innosetup/payload/ before compiling. Source: "payload\{#AwDefaultZipName}"; DestDir: "{app}\payload"; Flags: ignoreversion skipifsourcedoesntexist Source: "innosetup-rdp-package-filelist.md"; DestDir: "{app}\windows\installkit\innosetup"; Flags: ignoreversion diff --git a/windows/installkit/innosetup/AWatch-rus-InstallKit.exe b/windows/installkit/innosetup/AWatch-rus-InstallKit.exe index 122cdaf..fda871d 100644 --- a/windows/installkit/innosetup/AWatch-rus-InstallKit.exe +++ b/windows/installkit/innosetup/AWatch-rus-InstallKit.exe @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:736323b56689b86ecbd3c3758a7ced394a9e9613f9de411057c80a0578e8f790 -size 115989031 +oid sha256:2736b7cf6cad3259c6b4c30e311e701d3617d2f013ceaed9b8b6702b593c442e +size 115998720 diff --git a/windows/installkit/innosetup/innosetup-rdp-package-filelist.md b/windows/installkit/innosetup/innosetup-rdp-package-filelist.md index fba1cc1..450b516 100644 --- a/windows/installkit/innosetup/innosetup-rdp-package-filelist.md +++ b/windows/installkit/innosetup/innosetup-rdp-package-filelist.md @@ -29,6 +29,9 @@ - `windows/deploy-single-user.ps1` - `windows/deploy-domain-users.ps1` - `windows/deploy-ensemble.ps1` +- `windows/AWatchRusCollectorGuardService.cs` +- `windows/aw-collector-guard.ps1` +- `windows/install-collector-guard-service.ps1` - `windows/hardening-recovery.ps1` - `windows/validate-deployment.ps1` - `windows/migrate-awatch-rus-paths.ps1` @@ -41,6 +44,7 @@ ### 1.4 Шаблоны конфигурации - `windows/web-category-rules.example.json` - `windows/dlp-policy.example.json` +- `windows/dlp-policy.native-cross-os.example.json` ## 2) Бинарный payload ActivityWatch @@ -83,6 +87,7 @@ - `windows\dlp-endpoint-signals-collector.ps1` - `windows\web-category-rules.example.json` - `windows\dlp-policy.example.json` +- `windows\dlp-policy.native-cross-os.example.json` - `payload\activitywatch-v0.13.2-windows-x86_64.zip` (только для offline-режима) ## 6) Контроль перед сборкой .iss diff --git a/windows/validate-deployment.ps1 b/windows/validate-deployment.ps1 index 237bf69..bdfcb61 100644 --- a/windows/validate-deployment.ps1 +++ b/windows/validate-deployment.ps1 @@ -1,4 +1,4 @@ -[CmdletBinding()] +[CmdletBinding()] param( [string]$ConfigPath = 'C:\ProgramData\AWatch-rus\deployment-config.json' ) @@ -38,7 +38,7 @@ $windowExpected = if ($config.PSObject.Properties.Name -contains 'collectors' -a $fileOpsExpected = if ($config.PSObject.Properties.Name -contains 'collectors' -and $config.collectors.PSObject.Properties.Name -contains 'fileOpsEnabled') { [bool]$config.collectors.fileOpsEnabled } else { $true } $sessionEventsConfig = if ($config.PSObject.Properties.Name -contains 'sessionEvents') { $config.sessionEvents } else { $null } $sessionLogonEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'logonEnabled') { [bool]$sessionEventsConfig.logonEnabled } else { $false } -$sessionProcessEventsEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$sessionEventsConfig.processEventsEnabled } else { $true } +$sessionProcessEventsEnabled = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'processEventsEnabled') { [bool]$sessionEventsConfig.processEventsEnabled } else { $false } $sessionEventsBucketId = if ($sessionEventsConfig -and $sessionEventsConfig.PSObject.Properties.Name -contains 'bucketPrefix' -and -not [string]::IsNullOrWhiteSpace([string]$sessionEventsConfig.bucketPrefix)) { ('{0}_{1}' -f [string]$sessionEventsConfig.bucketPrefix, $awHostname) } @@ -301,6 +301,94 @@ function Get-TransportQueueHealth { } } +function Get-TransportQueueGroupHealth { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [Parameter(Mandatory = $true)] + [string]$StateRoot, + [Parameter(Mandatory = $true)] + [string]$QueuePattern, + [Parameter(Mandatory = $true)] + [int]$StaleAfterSeconds, + [Parameter(Mandatory = $true)] + [int]$MaxDepth, + [int]$ActiveProcessCount = 0, + [bool]$Required = $true + ) + + $queues = @(Get-ChildItem -LiteralPath $StateRoot -Filter $QueuePattern -ErrorAction SilentlyContinue | Sort-Object Name) + if ($queues.Count -eq 0) { + return [pscustomobject]@{ + name = $Name + required = [bool]$Required + queuePattern = $QueuePattern + queueCount = 0 + queues = @() + depth = 0 + sizeBytes = 0 + activeProcessCount = [int]$ActiveProcessCount + staleAfterSeconds = [int]$StaleAfterSeconds + maxDepth = [int]$MaxDepth + ok = [bool](-not $Required) + } + } + + $items = @() + foreach ($queue in $queues) { + $lockPath = [System.IO.Path]::ChangeExtension($queue.FullName, '.lock') + $items += Get-TransportQueueHealth -Name $queue.BaseName -QueuePath $queue.FullName -LockPath $lockPath -StaleAfterSeconds $StaleAfterSeconds -MaxDepth $MaxDepth -ActiveProcessCount $ActiveProcessCount -Required $Required + } + + return [pscustomobject]@{ + name = $Name + required = [bool]$Required + queuePattern = $QueuePattern + queueCount = [int]$items.Count + queues = @($items) + depth = [int](($items | Measure-Object -Property depth -Sum).Sum) + sizeBytes = [int64](($items | Measure-Object -Property sizeBytes -Sum).Sum) + activeProcessCount = [int]$ActiveProcessCount + staleAfterSeconds = [int]$StaleAfterSeconds + maxDepth = [int]$MaxDepth + ok = [bool](-not ($items | Where-Object { -not $_.ok })) + } +} + +function Resolve-ActivityWatchLaunchTaskName { + param( + [Parameter(Mandatory = $true)] + [string]$TaskName + ) + + if ($TaskName -notmatch '\[[^\]]+_Administrator\]') { + return $TaskName + } + + $localizedCandidate = 'ActivityWatch Launch [{0}_Администратор]' -f $awHostname + $localizedTask = Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object { $_.TaskName -eq $localizedCandidate } | Select-Object -First 1 + if ($localizedTask) { + return $localizedCandidate + } + + try { + $builtinAdmin = Get-LocalUser -ErrorAction Stop | + Where-Object { [string]$_.SID -match '-500$' } | + Select-Object -First 1 + if ($builtinAdmin -and -not [string]::IsNullOrWhiteSpace([string]$builtinAdmin.Name)) { + $candidate = 'ActivityWatch Launch [{0}_{1}]' -f $awHostname, [string]$builtinAdmin.Name + $existing = Get-ScheduledTask -ErrorAction SilentlyContinue | Where-Object { $_.TaskName -eq $candidate } | Select-Object -First 1 + if ($existing) { + return $candidate + } + } + } + catch { + } + + return $TaskName +} + function Get-TaskSnapshot { param( [Parameter(Mandatory = $true)] @@ -337,14 +425,13 @@ function Get-TaskSnapshot { } catch { } - [pscustomobject]@{ taskName = [string]$task.TaskName present = $true enabled = [bool]$enabled state = [string]$task.State lastResult = if ($taskInfo) { [int64]$taskInfo.LastTaskResult } else { $null } - ok = [bool]($enabled) + ok = [bool]$enabled } } ) @@ -413,10 +500,14 @@ else { 0 } $sessionScopedCollectorsRequired = ($sessionScopedExpectedCount -gt 0) +$liveSessionScopedCollectorsRequired = ($liveSessionBoundUsers.Count -gt 0) + +$collectorGuardService = Get-Service -Name 'AWatchRusCollectorGuard' -ErrorAction SilentlyContinue +$collectorGuardActive = [bool]($collectorGuardService -and $collectorGuardService.Status -eq 'Running') $taskNames = @() if ($config.userTasks) { - $taskNames += @($config.userTasks | ForEach-Object { [string]$_.launchTaskName }) + $taskNames += @($config.userTasks | ForEach-Object { Resolve-ActivityWatchLaunchTaskName -TaskName ([string]$_.launchTaskName) }) } $taskNames += [string]$config.recovery.taskName $tasks = @(Get-TaskSnapshot -TaskNames $taskNames) @@ -438,24 +529,24 @@ foreach ($watcher in $runningWatchers) { $bucketChecks = @( Get-BucketHealth -BucketId ('aw-worktime-sessions_' + $awHostname) -MaxAgeSeconds $sessionFreshnessSeconds -Required $true -RequireFreshEvent $true ) -if ($sessionScopedCollectorsRequired -and $afkExpected) { +if ($liveSessionScopedCollectorsRequired -and $afkExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-watcher-afk_' + $awHostname) -MaxAgeSeconds $freshnessSeconds -Required $true -RequireFreshEvent $false } -if ($sessionScopedCollectorsRequired -and $windowExpected) { +if ($liveSessionScopedCollectorsRequired -and $windowExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-watcher-window_' + $awHostname) -MaxAgeSeconds $freshnessSeconds -Required $true -RequireFreshEvent $false } -if ($sessionScopedCollectorsRequired) { +if ($liveSessionScopedCollectorsRequired) { $bucketChecks += Get-BucketHealth -BucketId ('aw-dlp-endpoint-signals_' + $awHostname) -MaxAgeSeconds $endpointFreshnessSeconds -Required $true -RequireFreshEvent $true } -if ($sessionScopedCollectorsRequired -and $fileOpsExpected) { +if ($liveSessionScopedCollectorsRequired -and $fileOpsExpected) { $bucketChecks += Get-BucketHealth -BucketId ('aw-file-operations_' + $awHostname) -MaxAgeSeconds $transportStaleSeconds -Required $false -RequireFreshEvent $true } $queueChecks = @( - Get-TransportQueueHealth -Name 'endpoint' -QueuePath (Join-Path $stateRoot 'dlp-endpoint-signals-queue.jsonl') -LockPath (Join-Path $stateRoot 'dlp-endpoint-signals-queue.lock') -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($endpointCollectorProcesses).Count -Required $sessionScopedCollectorsRequired + Get-TransportQueueGroupHealth -Name 'endpoint' -StateRoot $stateRoot -QueuePattern 'dlp-endpoint-signals-queue*.jsonl' -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($endpointCollectorProcesses).Count -Required $liveSessionScopedCollectorsRequired ) if ($fileOpsExpected) { - $queueChecks += Get-TransportQueueHealth -Name 'fileops' -QueuePath (Join-Path $stateRoot 'file-operations-queue.jsonl') -LockPath (Join-Path $stateRoot 'file-operations-queue.lock') -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($fileCollectorProcesses).Count -Required $sessionScopedCollectorsRequired + $queueChecks += Get-TransportQueueGroupHealth -Name 'fileops' -StateRoot $stateRoot -QueuePattern 'file-operations-queue*.jsonl' -StaleAfterSeconds $transportStaleSeconds -MaxDepth $queueMaxDepth -ActiveProcessCount @($fileCollectorProcesses).Count -Required $liveSessionScopedCollectorsRequired } $printServiceOperationalEnabled = $false @@ -502,12 +593,14 @@ $result = [ordered]@{ } tasks = [ordered]@{ list = $tasks - ok = [bool]($tasks.Count -gt 0 -and -not ($tasks | Where-Object { -not $_.present -or -not $_.enabled })) + ok = [bool]($tasks.Count -gt 0 -and -not ($tasks | Where-Object { -not $_.ok })) } processes = [ordered]@{ liveSessionBoundUsers = $liveSessionBoundUsers sessionBoundUsers = $interactiveSessionBoundUsers sessionScopedExpectedCount = [int]$sessionScopedExpectedCount + liveSessionScopedCollectorsRequired = [bool]$liveSessionScopedCollectorsRequired + collectorGuardServiceActive = [bool]$collectorGuardActive watchers = @($runningWatchers) watcherDuplicates = @($watcherDuplicates) sessionCollectors = @($sessionCollectorProcesses)