diff --git a/AGENTS.md b/AGENTS.md index 84b4279..8b35631 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ AGENTS for OpenCode Keep this file minimal and high-signal: only include facts an agent would otherwise miss. 1) Repo purpose (one line) -- This repository bundles an ActivityWatch Server deployment, RU WebUI patch, Windows collectors (PowerShell), and small Python utilities for aggregation and monitoring. +- This repository bundles an ActivityWatch Server deployment, RU WebUI patch, Windows collectors (PowerShell), and Rust operational utilities for aggregation, monitoring, health checks, SLO, DLP helpers, and DetMir automation. Python is retained only for explicit exceptions: Telegram bot runtime, pfSense tooling, OCR/content-analysis, 1C AI/ETL, and detmir-mcp. 2) Highest-value entrypoints & commands - Read README.md and docs/preparation.md first (they are the authoritative onboarding flow). @@ -12,7 +12,7 @@ Keep this file minimal and high-signal: only include facts an agent would otherw - Install AW server on the CT (runs inside CT): `aw-server/install_aw_server.sh` (requires `/etc/activitywatch/aw-server.env`). - Apply RU WebUI patch (must run on the CT and after webui is present): `aw-server/apply_webui_ru_patch.sh`. - Run the Windows ensemble deploy from a Windows admin host: `windows/deploy-ensemble.ps1` (see its parameters; it calls `deploy-domain-users.ps1`). -- Quick DLP aggregation (local): `python3 scripts/aggregate_dlp_events.py`. +- Quick DLP aggregation (local): `adk-rust/target/release/dlp-aggregator`. 3) Exact env/secrets behavior agents often miss - Secrets live in `secrets/deploy.secrets.env` (actual file is intentionally local-only). Many scripts default to that path if no arg provided. Never add real secrets to commits. Use `.example` files as templates. diff --git a/adk-rust/RUNBOOK.md b/adk-rust/RUNBOOK.md index 658eaab..0ad7fa4 100644 --- a/adk-rust/RUNBOOK.md +++ b/adk-rust/RUNBOOK.md @@ -52,8 +52,8 @@ Ansible, PowerShell или Playwright, он остается там до поя | `detmir-heal-safe` | Rust binary deployed + legacy script retained | switched for Rust auto | | `tsj_guardian_watchdog.sh` | Rust service via systemd drop-in + legacy shell retained | switched | | Telegram `/status`/decision backend | Rust helper + permanent Python bot runtime | backend only | -| `aw-rus-healthd.py` | Rust production via systemd drop-in + legacy Python retained | switched | -| `scripts/dlp-health-check.py` | Rust production binary + legacy Python backup retained | switched | +| `aw-rus-healthd` | Rust production unit, Python entrypoint removed from repo | done | +| `dlp-health-check` | Rust production binary, Python entrypoint removed from repo | done | | DLP aggregator | Rust production via systemd drop-in + legacy Python retained | switched | | AW DLP Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched | | AW worktime Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched | @@ -763,7 +763,8 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg `cases list/create`, `health check`; - HTTP client использует no-proxy для локальных AW/DLP сервисов; - `/usr/local/bin/dlp-admin-cli` установлен как Rust production CLI, - legacy `/usr/local/bin/dlp-admin-cli.py` оставлен для rollback/reference; + legacy `/usr/local/bin/dlp-admin-cli.py` больше не является repo/runtime + path; рабочий CLI: `/usr/local/bin/dlp-admin-cli`; - parity на AW server совпал с Python для `health`, `policies active`, `incidents list --since-hours 24 --limit 5`, `cases list --limit 5`; - production verification зеленый: AW failed units 0, @@ -885,9 +886,9 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg freshness через `dlp-health-check --json`, и drift checks для `startOfDay`, `always_active_pattern`, `landingpage`; - env contract читается из процесса и `/etc/activitywatch/aw-server.env`; - - production `/usr/local/bin/aw-health-check` теперь wrapper, который - предпочитает `/usr/local/bin/aw-health-check-rust` и fallback на - `/opt/activitywatch/aw-rus-ops/health-check.sh`; + - production `/usr/local/bin/aw-health-check` теперь Rust-required wrapper: + вызывает `/usr/local/bin/aw-health-check-rust` и падает с ошибкой, если + Rust binary отсутствует; - shadow parity на AW server совпал с shell по exit code и ключевым health-строкам; - production verification зеленый: `aw-health-check` OK, @@ -1380,32 +1381,24 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg warnings`, release build OK, `bash -n` OK, artifact check OK, `quality-gate.sh` OK, DetMir read-only OK with `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. -50. `[done]` Перенести `scripts/merge_aw_server_dbs.py` merge engine на Rust: +50. `[done]` Перенести merge engine на Rust: - добавлен crate `merge-aw-server-dbs`; - - `scripts/merge_aw_server_dbs.py` остается совместимым с текущим - `python3 merge_aw_server_dbs.py` вызовом, но в начале exec'ает Rust - binary при наличии `MERGE_AW_SERVER_DBS_RUST`, - `$CARGO_TARGET_DIR/release/merge-aw-server-dbs`, - `adk-rust/target/release/merge-aw-server-dbs` или + - рабочий runtime path: `adk-rust/target/release/merge-aw-server-dbs` и `/usr/local/bin/merge-aw-server-dbs`; - - `MERGE_AW_SERVER_DBS_FORCE_LEGACY=1` форсирует Python fallback; - - `ansible/deploy_aw_server.yml` теперь optional устанавливает - `/usr/local/bin/merge-aw-server-dbs`, если release artifact существует, - сохраняя прежний `python3 /usr/local/bin/merge_aw_server_dbs.py` command; + - legacy `scripts/merge_aw_server_dbs.py` удален из repo/runtime path; + - `ansible/deploy_aw_server.yml` устанавливает + `/usr/local/bin/merge-aw-server-dbs` и использует его без Python; - Rust сохраняет контракт `--base`, `--overlay`, `--output`, SQLite backup-copy base DB, bucket matching by id/key/name, duplicate event suppression by `(starttime,endtime,data)`, JSON summary; - - fallback bugfix: Python legacy больше не падает на `UNIQUE(name)` - name-conflict/update path и считает только реально вставленные buckets; - - offline parity fixture: Rust и Python fallback дали одинаковый JSON + - offline parity fixture: Rust replacement дал ожидаемый JSON `inserted_buckets=1`, `inserted_events=3` и одинаковые SQLite table rows (`buckets=3`, `events=5`) на кейсе duplicate event + name-conflict bucket; - gates: `cargo fmt --all -- --check`, `cargo test -p merge-aw-server-dbs` (`2 passed`), `cargo clippy -p merge-aw-server-dbs --all-targets -- -D warnings`, release build OK, - `python3 -m py_compile scripts/merge_aw_server_dbs.py`, artifact check - OK, `quality-gate.sh` OK, DetMir read-only OK after DLP baseline retry + artifact check OK, `quality-gate.sh` OK, DetMir read-only OK after DLP baseline retry with `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`. 51. `[done]` Перенести `scripts/prod_backup_restore.sh` в safe-by-default Rust planner/checker: @@ -1419,13 +1412,13 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg он печатает plan-only flow и помечает destructive steps; - если Rust artifact отсутствует, wrapper отказывается запускать destructive legacy случайно и просит собрать Rust planner; - - старый destructive flow доступен только явно: - `scripts/prod_backup_restore.sh --apply-legacy`; + - старый Python destructive flow удален; `--apply-legacy` больше не + поддерживается; - Rust `--apply` намеренно запрещен на этом этапе; - `--check-inputs --json` валидирует env/files/commands без вывода секретов, проверяет `AW_SSH_PASSWORD`, `AW_WINRM_PASSWORD`, `sshpass`, - `ansible-playbook`, inventory и `merge_aw_server_dbs.py`; - - план сохраняет порядок legacy flow: scp merge script, remote backup dir, + `ansible-playbook`, inventory и `merge-aw-server-dbs`; + - план сохраняет порядок restore flow: scp Rust merge binary, remote backup dir, DB existence checks, DB backups, stop `activitywatch-server`, merge DB, install merged DB, 3 ansible playbooks, final AW validation; - safe smoke: plan text OK, JSON valid (`steps=12`, `missing_count=0`), diff --git a/adk-rust/crates/aw-ensure-reliability/src/main.rs b/adk-rust/crates/aw-ensure-reliability/src/main.rs index 6af41c4..55ad4eb 100644 --- a/adk-rust/crates/aw-ensure-reliability/src/main.rs +++ b/adk-rust/crates/aw-ensure-reliability/src/main.rs @@ -56,9 +56,6 @@ struct Cli { #[arg(long, default_value = "/etc/systemd/system/aw-health-check.service")] health_service_target: PathBuf, - #[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/health-check.sh")] - health_script_source: PathBuf, - #[arg(long, default_value_t = false)] apply: bool, @@ -72,7 +69,6 @@ enum StepKind { Check, Chown, Chmod, - Copy, Write, Systemd, Sleep, @@ -138,11 +134,10 @@ fn build_report(cli: &Cli) -> Report { if !cli.env_file.is_file() { missing_required.push(format!("env file missing: {}", cli.env_file.display())); } - if !cli.health_script_target.is_file() && !cli.health_script_source.is_file() { + if !cli.health_script_target.is_file() { missing_required.push(format!( - "health script target and source missing: {}, {}", - cli.health_script_target.display(), - cli.health_script_source.display() + "health script target missing: {}", + cli.health_script_target.display() )); } @@ -193,17 +188,13 @@ fn build_report(cli: &Cli) -> Report { ); push_step( &mut steps, - "install-health-script", - StepKind::Copy, - format!( - "copy {} {}", - shell_quote(&cli.health_script_source), - shell_quote(&cli.health_script_target) - ), - true, + "check-health-script", + StepKind::Check, + format!("test -x {}", shell_quote(&cli.health_script_target)), + false, !cli.health_script_target.is_file(), if cli.health_script_target.is_file() { - "health script already installed".to_string() + "health script already installed by Ansible".to_string() } else { "health script target missing".to_string() }, @@ -331,8 +322,16 @@ fn apply_steps(report: &mut Report) -> Result<()> { 1 }), }, + "check-health-script" => { + let target = Path::new("/usr/local/bin/aw-health-check"); + ExecResult { + order: step.order, + name: step.name.clone(), + ok: target.is_file(), + exit_code: Some(if target.is_file() { 0 } else { 1 }), + } + } "install-logrotate" => write_file_result(step, report, LOGROTATE_CONTENT, 0o644)?, - "install-health-script" => copy_health_script(step, report)?, "install-health-timer" => write_file_result(step, report, HEALTH_TIMER_CONTENT, 0o644)?, "install-health-service" => { write_file_result(step, report, HEALTH_SERVICE_CONTENT, 0o644)? @@ -371,24 +370,6 @@ fn write_file_result(step: &Step, report: &Report, content: &str, mode: u32) -> }) } -fn copy_health_script(step: &Step, report: &Report) -> Result { - let source = source_from_copy_command(&step.command)?; - let target = copy_target_from_command(&step.command)?; - let _ = report; - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; - } - fs::copy(&source, &target) - .with_context(|| format!("copy {} -> {}", source.display(), target.display()))?; - set_mode(&target, 0o755).with_context(|| format!("chmod 0755 {}", target.display()))?; - Ok(ExecResult { - order: step.order, - name: step.name.clone(), - ok: true, - exit_code: Some(0), - }) -} - fn run_shell_step(step: &Step) -> Result { let status = Command::new("sh") .arg("-c") @@ -412,22 +393,6 @@ fn target_from_command(command: &str) -> Result { Ok(PathBuf::from(raw.trim_matches('\''))) } -fn source_from_copy_command(command: &str) -> Result { - let raw = command - .split_whitespace() - .nth(1) - .context("parse source from copy command")?; - Ok(PathBuf::from(raw.trim_matches('\''))) -} - -fn copy_target_from_command(command: &str) -> Result { - let raw = command - .split_whitespace() - .nth(2) - .context("parse target from copy command")?; - Ok(PathBuf::from(raw.trim_matches('\''))) -} - #[cfg(unix)] fn set_mode(path: &Path, mode: u32) -> Result<()> { use std::os::unix::fs::PermissionsExt; @@ -494,19 +459,19 @@ mod tests { fn dry_run_marks_legacy_mutations() { let dir = tempfile::tempdir().unwrap(); let env_file = dir.path().join("aw-server.env"); - let health = dir.path().join("health-check.sh"); + let health_target = dir.path().join("bin/aw-health-check"); fs::write(&env_file, "AW_BASE_URL=http://127.0.0.1:5600\n").unwrap(); - fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap(); + fs::create_dir_all(health_target.parent().unwrap()).unwrap(); + fs::write(&health_target, "#!/bin/sh\nexit 0\n").unwrap(); let cli = Cli { env_file, data_dir: dir.path().join("data"), log_dir: dir.path().join("log"), opt_dir: dir.path().join("opt"), logrotate_target: dir.path().join("logrotate/activitywatch"), - health_script_target: dir.path().join("bin/aw-health-check"), + health_script_target: health_target, health_timer_target: dir.path().join("systemd/aw-health-check.timer"), health_service_target: dir.path().join("systemd/aw-health-check.service"), - health_script_source: health, apply: false, json: true, }; @@ -525,18 +490,18 @@ mod tests { #[test] fn missing_env_blocks_apply() { let dir = tempfile::tempdir().unwrap(); - let health = dir.path().join("health-check.sh"); - fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap(); + let health_target = dir.path().join("bin/aw-health-check"); + fs::create_dir_all(health_target.parent().unwrap()).unwrap(); + fs::write(&health_target, "#!/bin/sh\nexit 0\n").unwrap(); let cli = Cli { env_file: dir.path().join("missing.env"), data_dir: dir.path().join("data"), log_dir: dir.path().join("log"), opt_dir: dir.path().join("opt"), logrotate_target: dir.path().join("logrotate/activitywatch"), - health_script_target: dir.path().join("bin/aw-health-check"), + health_script_target: health_target, health_timer_target: dir.path().join("systemd/aw-health-check.timer"), health_service_target: dir.path().join("systemd/aw-health-check.service"), - health_script_source: health, apply: true, json: true, }; diff --git a/adk-rust/crates/aw-slo-monitor/src/main.rs b/adk-rust/crates/aw-slo-monitor/src/main.rs index bb1fe14..3a7f77e 100644 --- a/adk-rust/crates/aw-slo-monitor/src/main.rs +++ b/adk-rust/crates/aw-slo-monitor/src/main.rs @@ -20,7 +20,7 @@ use serde_json::{Value, json}; const ENV_FILE: &str = "/etc/activitywatch/aw-server.env"; const DEFAULT_STATE_DIR: &str = "/var/lib/activitywatch/slo"; -const DEFAULT_HEALTHD_CMD: &str = "/usr/local/bin/aw-rus-healthd.py --json"; +const DEFAULT_HEALTHD_CMD: &str = "/usr/local/bin/aw-rus-healthd-rust --json"; const DEFAULT_HEALTHD_STATE_FILE: &str = "/var/lib/activitywatch/health/aw-rus-health.json"; #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] diff --git a/adk-rust/crates/prod-backup-restore/src/main.rs b/adk-rust/crates/prod-backup-restore/src/main.rs index 1d4a00f..5767bf5 100644 --- a/adk-rust/crates/prod-backup-restore/src/main.rs +++ b/adk-rust/crates/prod-backup-restore/src/main.rs @@ -13,7 +13,7 @@ const DEFAULT_SERVER_USER: &str = "igor"; const DEFAULT_LEGACY_DB: &str = "/root/.local/share/activitywatch/aw-server-rust/sqlite.db"; const DEFAULT_TARGET_DB: &str = "/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db"; -const DEFAULT_REMOTE_MERGE_SCRIPT: &str = "/tmp/merge_aw_server_dbs.py"; +const DEFAULT_REMOTE_MERGE_BIN: &str = "/tmp/merge-aw-server-dbs"; #[derive(Debug, Parser)] #[command(about = "Safe planner/checker for the destructive prod backup restore flow")] @@ -45,8 +45,8 @@ struct Cli { #[arg(long, default_value = DEFAULT_TARGET_DB)] target_db: String, - #[arg(long, default_value = DEFAULT_REMOTE_MERGE_SCRIPT)] - remote_merge_script: String, + #[arg(long, default_value = DEFAULT_REMOTE_MERGE_BIN)] + remote_merge_bin: String, #[arg(long)] apply: bool, @@ -63,7 +63,7 @@ struct Plan { remote_backup_dir: String, legacy_db: String, target_db: String, - remote_merge_script: String, + remote_merge_bin: String, required_env: Vec, required_commands: Vec, required_files: Vec, @@ -101,7 +101,7 @@ fn run() -> Result { let cli = Cli::parse(); if cli.apply { bail!( - "Rust apply is intentionally disabled for this stage; use scripts/prod_backup_restore.sh --apply-legacy only for explicit legacy restore" + "Rust apply is intentionally disabled for this stage; run without --apply to review the safe restore plan" ); } let root = cli @@ -128,7 +128,7 @@ fn run() -> Result { } else { root.join(&cli.inventory) }; - let merge_script = root.join("scripts/merge_aw_server_dbs.py"); + let merge_bin = root.join("adk-rust/target/release/merge-aw-server-dbs"); let plan = build_plan( &root, &env_file, @@ -139,9 +139,9 @@ fn run() -> Result { &remote_backup_dir, &cli.legacy_db, &cli.target_db, - &cli.remote_merge_script, + &cli.remote_merge_bin, &inventory, - &merge_script, + &merge_bin, ); if cli.json { @@ -168,9 +168,9 @@ fn build_plan( remote_backup_dir: &str, legacy_db: &str, target_db: &str, - remote_merge_script: &str, + remote_merge_bin: &str, inventory: &Path, - merge_script: &Path, + merge_bin: &Path, ) -> Plan { let required_env = ["AW_SSH_PASSWORD", "AW_WINRM_PASSWORD"] .into_iter() @@ -199,24 +199,21 @@ fn build_plan( }, }) .collect::>(); - let required_files = [ - ("inventory", inventory), - ("merge_aw_server_dbs.py", merge_script), - ] - .into_iter() - .map(|(name, path)| Requirement { - name: name.to_string(), - ok: path.is_file(), - detail: path.display().to_string(), - }) - .collect::>(); + let required_files = [("inventory", inventory), ("merge-aw-server-dbs", merge_bin)] + .into_iter() + .map(|(name, path)| Requirement { + name: name.to_string(), + ok: path.is_file(), + detail: path.display().to_string(), + }) + .collect::>(); let mut steps = Vec::new(); push_step( &mut steps, "scp", format!( - "sshpass scp scripts/merge_aw_server_dbs.py {server_user}@{server_host}:{remote_merge_script}" + "sshpass scp adk-rust/target/release/merge-aw-server-dbs {server_user}@{server_host}:{remote_merge_bin}" ), false, ); @@ -258,7 +255,7 @@ fn build_plan( &mut steps, "ssh", format!( - "sudo python3 '{remote_merge_script}' --base '{legacy_db}' --overlay '{target_db}' --output '{remote_backup_dir}/sqlite.merged.db'" + "sudo chmod 0755 '{remote_merge_bin}' && sudo '{remote_merge_bin}' --base '{legacy_db}' --overlay '{target_db}' --output '{remote_backup_dir}/sqlite.merged.db'" ), true, ); @@ -307,7 +304,7 @@ fn build_plan( remote_backup_dir: remote_backup_dir.to_string(), legacy_db: legacy_db.to_string(), target_db: target_db.to_string(), - remote_merge_script: remote_merge_script.to_string(), + remote_merge_bin: remote_merge_bin.to_string(), required_env, required_commands, required_files, @@ -443,7 +440,9 @@ mod tests { fn plan_marks_destructive_steps() { let dir = tempfile::tempdir().unwrap(); let inventory = dir.path().join("ansible/inventory.ini"); - let merge = dir.path().join("scripts/merge_aw_server_dbs.py"); + let merge = dir + .path() + .join("adk-rust/target/release/merge-aw-server-dbs"); fs::create_dir_all(inventory.parent().unwrap()).unwrap(); fs::create_dir_all(merge.parent().unwrap()).unwrap(); fs::write(&inventory, "").unwrap(); @@ -461,7 +460,7 @@ mod tests { "/var/lib/activitywatch/backups/prod-restore-20260602-000000", DEFAULT_LEGACY_DB, DEFAULT_TARGET_DB, - DEFAULT_REMOTE_MERGE_SCRIPT, + DEFAULT_REMOTE_MERGE_BIN, &inventory, &merge, ); diff --git a/adk-rust/crates/rebuild-install-kit/src/main.rs b/adk-rust/crates/rebuild-install-kit/src/main.rs index 31c34db..74c39b4 100644 --- a/adk-rust/crates/rebuild-install-kit/src/main.rs +++ b/adk-rust/crates/rebuild-install-kit/src/main.rs @@ -47,23 +47,18 @@ const AW_SERVER_FILES: &[&str] = &[ "aw-server/apply_webui_ru_patch.sh", "aw-server/aw-host-groups.json", "aw-server/aw-ru-patch.js", - "aw-server/aw-rus-healthd.py", "aw-server/aw-rus-healthd.service", "aw-server/aw-rus-healthd.timer", "aw-server/aw-browser-smoke.service", "aw-server/aw-browser-smoke.timer", - "aw-server/aw-slo-monitor.py", "aw-server/aw-slo-monitor.service", "aw-server/aw-slo-monitor.timer", "aw-server/aw-server.env.example", "aw-server/aw-sw-cleanup.js", - "aw-server/aw-worktime-api.py", "aw-server/aw-worktime-api.service", - "aw-server/aw-worktime-prewarm.sh", "aw-server/aw-worktime-prewarm.service", "aw-server/aw-worktime-prewarm.timer", "aw-server/aw-worktime-panel.js", - "aw-server/health-check.sh", "aw-server/install_aw_server.sh", "aw-server/settings/classes-worktime.json", "aw-server/settings/views-default.json", diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index 66defd1..6a81578 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -27,6 +27,7 @@ - tesseract-ocr - tesseract-ocr-rus - unzip + - jq state: present update_cache: true @@ -109,6 +110,64 @@ - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" + - name: Проверить существующие health state files + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ aw_rus_health_state_dir }}/aw-rus-health.json" + - "{{ aw_rus_health_state_dir }}/aw-rus-health.txt" + - "{{ aw_rus_health_state_dir }}/dlp-health-check-counters.json" + register: aw_rus_health_state_files + + - name: Нормализовать владельца существующих health state files + ansible.builtin.file: + path: "{{ item.stat.path }}" + owner: "{{ aw_server_user }}" + group: "{{ aw_server_group }}" + mode: "0644" + loop: "{{ aw_rus_health_state_files.results }}" + when: item.stat.exists | default(false) + + - name: Удалить устаревшие Python entrypoints после Rust migration + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /opt/activitywatch/aw-rus-ops/health-check.sh + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py + - /opt/activitywatch/dlp-case-management/.venv + - /opt/activitywatch/dlp-case-management/case_rules.py + - /opt/activitywatch/dlp-case-management/case_schema.py + - /opt/activitywatch/dlp-case-management/case_service.py + - /opt/activitywatch/dlp-case-management/case_storage.py + - /opt/activitywatch/dlp-case-management/evidence_chain.py + - /opt/activitywatch/dlp-compliance/compliance_scheduler.py + - /opt/activitywatch/dlp-compliance/report_generator.py + - /opt/activitywatch/dlp-compliance/.venv + - /opt/activitywatch/dlp-integrations/cef_exporter.py + - /opt/activitywatch/dlp-integrations/syslog_forwarder.py + - /opt/activitywatch/dlp-integrations/webhook_sender.py + - /opt/activitywatch/dlp-integrations/.venv + - /opt/activitywatch/dlp-policy-engine/.venv + - /opt/activitywatch/dlp-policy-engine/policy_distributor.py + - /opt/activitywatch/dlp-policy-engine/policy_schema.py + - /opt/activitywatch/dlp-policy-engine/policy_service.py + - /opt/activitywatch/dlp-policy-engine/policy_storage.py + - /opt/activitywatch/scripts/aggregate_dlp_events.py + - /usr/local/bin/aw-dlp-influx-exporter.py + - /usr/local/bin/aw-extract-ioc-from-sigma.py + - /usr/local/bin/aw-rus-healthd.py + - /usr/local/bin/aw-slo-monitor.py + - /usr/local/bin/aw-worktime-afk-bridge.py + - /usr/local/bin/aw-worktime-api.py + - /usr/local/bin/aw-worktime-influx-exporter.py + - /usr/local/bin/aw-worktime-ui-bridge.py + - /usr/local/bin/dlp-admin-cli.py + - /usr/local/bin/merge_aw_server_dbs.py + - name: Установить prune script для локального state ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-prune-local-state.sh" @@ -431,14 +490,6 @@ group: root mode: "0755" - - name: Положить legacy aw-health-check до Influx проверок - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" - dest: /opt/activitywatch/aw-rus-ops/health-check.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust aw-health-check до Influx проверок ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-health-check" @@ -455,6 +506,12 @@ mode: "0755" when: aw_health_check_rust_binary_early.stat.exists | default(false) + - name: Требовать Rust aw-health-check до Influx проверок + ansible.builtin.assert: + that: + - aw_health_check_rust_binary_early.stat.exists | default(false) + fail_msg: "Rust aw-health-check artifact is required at {{ aw_rust_release_dir }}/aw-health-check" + - name: Проверить локальный Rust aw-db-health до Influx проверок ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-db-health" @@ -650,34 +707,6 @@ mode: "0755" when: aw_dlp_policy_engine_enabled | default(false) | bool - - name: Скопировать файлы DLP policy engine - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/{{ item }}" - dest: "/opt/activitywatch/dlp-policy-engine/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0644" - loop: - - policy_service.py - - policy_schema.py - - policy_storage.py - - policy_distributor.py - - requirements.txt - when: aw_dlp_policy_engine_enabled | default(false) | bool - - - name: Создать virtualenv DLP policy engine - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-policy-engine/.venv - args: - creates: /opt/activitywatch/dlp-policy-engine/.venv/bin/python - when: aw_dlp_policy_engine_enabled | default(false) | bool - - - name: Установить зависимости DLP policy engine - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-policy-engine/requirements.txt - virtualenv: /opt/activitywatch/dlp-policy-engine/.venv - when: aw_dlp_policy_engine_enabled | default(false) | bool - - name: Установить systemd unit DLP policy engine ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/dlp-policy-engine.service" @@ -695,6 +724,13 @@ become: false when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Требовать Rust DLP policy engine artifact + ansible.builtin.assert: + that: + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-policy-engine" + when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Установить Rust DLP policy engine ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-policy-engine" @@ -706,31 +742,6 @@ - aw_dlp_policy_engine_enabled | default(false) | bool - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP policy engine - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-policy-engine.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_policy_engine_enabled | default(false) | bool - - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - - name: Переключить DLP policy engine на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-policy-engine.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-policy-engine-rust - when: - - aw_dlp_policy_engine_enabled | default(false) | bool - - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - name: Создать каталог DLP content analysis ansible.builtin.file: path: /opt/activitywatch/dlp-content-analysis @@ -807,38 +818,9 @@ group: "{{ aw_server_group }}" mode: "0644" loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - cef-config.yaml - syslog-forwarder-config.yaml - webhook-config.yaml - - requirements.txt - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Выставить executable для python scripts DLP integrations - ansible.builtin.file: - path: "/opt/activitywatch/dlp-integrations/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0755" - loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Создать virtualenv DLP integrations - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-integrations/.venv - args: - creates: /opt/activitywatch/dlp-integrations/.venv/bin/python - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Установить зависимости DLP integrations - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-integrations/requirements.txt - virtualenv: /opt/activitywatch/dlp-integrations/.venv when: aw_dlp_integrations_enabled | default(true) | bool - name: Создать state каталог DLP integrations @@ -876,6 +858,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust CEF exporter artifact + ansible.builtin.assert: + that: + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-cef-exporter" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust CEF exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-cef-exporter" @@ -887,31 +876,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог CEF exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-cef-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить CEF exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-cef-exporter-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd unit syslog forwarder ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/syslog-forwarder.service" @@ -938,6 +902,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust syslog forwarder artifact + ansible.builtin.assert: + that: + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-syslog-forwarder" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust syslog forwarder ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" @@ -949,31 +920,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог syslog forwarder - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - - name: Переключить syslog forwarder на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - name: Установить systemd unit webhook sender ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/webhook-sender.service" @@ -1000,6 +946,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust webhook sender artifact + ansible.builtin.assert: + that: + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-webhook-sender" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust webhook sender ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-webhook-sender" @@ -1011,31 +964,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог webhook sender - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-webhook-sender.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - - name: Переключить webhook sender на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-webhook-sender-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - name: Создать каталог DLP case management ansible.builtin.file: path: /opt/activitywatch/dlp-case-management @@ -1045,43 +973,6 @@ mode: "0755" when: aw_dlp_case_management_enabled | default(true) | bool - - name: Скопировать файлы DLP case management - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/dlp-case-management/{{ item }}" - dest: "/opt/activitywatch/dlp-case-management/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0644" - loop: - - case_service.py - - case_rules.py - - case_schema.py - - case_storage.py - - evidence_chain.py - - requirements.txt - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Выставить executable для case_service - ansible.builtin.file: - path: /opt/activitywatch/dlp-case-management/case_service.py - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0755" - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Создать virtualenv DLP case management - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-case-management/.venv - args: - creates: /opt/activitywatch/dlp-case-management/.venv/bin/python - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Установить зависимости DLP case management - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-case-management/requirements.txt - virtualenv: /opt/activitywatch/dlp-case-management/.venv - when: aw_dlp_case_management_enabled | default(true) | bool - - name: Установить systemd unit DLP case management ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-case-management/case-service.service" @@ -1099,6 +990,13 @@ become: false when: aw_dlp_case_management_enabled | default(true) | bool + - name: Требовать Rust DLP case management artifact + ansible.builtin.assert: + that: + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-case-management" + when: aw_dlp_case_management_enabled | default(true) | bool + - name: Установить Rust DLP case management ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-case-management" @@ -1110,31 +1008,6 @@ - aw_dlp_case_management_enabled | default(true) | bool - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP case management - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-case-management.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_case_management_enabled | default(true) | bool - - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - - name: Переключить DLP case management на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-case-management.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-case-management-rust - when: - - aw_dlp_case_management_enabled | default(true) | bool - - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - name: Создать каталоги DLP compliance ansible.builtin.file: path: "{{ item }}" @@ -1156,28 +1029,12 @@ group: "{{ aw_server_group }}" mode: "{{ item.mode }}" loop: - - { src: "report_generator.py", dest: "/opt/activitywatch/dlp-compliance/report_generator.py", mode: "0755" } - - { src: "compliance_scheduler.py", dest: "/opt/activitywatch/dlp-compliance/compliance_scheduler.py", mode: "0755" } - - { src: "requirements.txt", dest: "/opt/activitywatch/dlp-compliance/requirements.txt", mode: "0644" } - { src: "templates/152-fz-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html", mode: "0644" } - { src: "templates/pci-dss-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/pci-dss-report.html", mode: "0644" } - { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" } - { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" } when: aw_dlp_compliance_enabled | default(true) | bool - - name: Создать virtualenv DLP compliance - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-compliance/.venv - args: - creates: /opt/activitywatch/dlp-compliance/.venv/bin/python - when: aw_dlp_compliance_enabled | default(true) | bool - - - name: Установить зависимости DLP compliance - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-compliance/requirements.txt - virtualenv: /opt/activitywatch/dlp-compliance/.venv - when: aw_dlp_compliance_enabled | default(true) | bool - - name: Проверить локальный Rust DLP compliance ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-compliance" @@ -1186,6 +1043,13 @@ become: false when: aw_dlp_compliance_enabled | default(true) | bool + - name: Требовать Rust DLP compliance artifact + ansible.builtin.assert: + that: + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-compliance" + when: aw_dlp_compliance_enabled | default(true) | bool + - name: Установить Rust DLP compliance ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-compliance" @@ -1197,45 +1061,18 @@ - aw_dlp_compliance_enabled | default(true) | bool - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP compliance report scheduler - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-report-scheduler.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_compliance_enabled | default(true) | bool - - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - - name: Переключить DLP compliance report scheduler на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-report-scheduler.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-compliance-rust - when: - - aw_dlp_compliance_enabled | default(true) | bool - - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - - name: Установить dlp-admin-cli - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/dlp-admin-cli.py" - dest: /usr/local/bin/dlp-admin-cli.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust dlp-admin-cli ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-admin-cli" delegate_to: localhost register: aw_dlp_admin_cli_rust_binary + - name: Требовать Rust dlp-admin-cli artifact + ansible.builtin.assert: + that: + - aw_dlp_admin_cli_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-admin-cli" + - name: Установить Rust dlp-admin-cli ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-admin-cli" @@ -1245,14 +1082,6 @@ mode: "0755" when: aw_dlp_admin_cli_rust_binary.stat.exists - - name: Установить скрипт AW worktime API - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" - dest: /usr/local/bin/aw-worktime-api.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime API ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-api" @@ -1260,6 +1089,12 @@ register: aw_worktime_api_rust_binary become: false + - name: Требовать Rust AW worktime API artifact + ansible.builtin.assert: + that: + - aw_worktime_api_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-api" + - name: Установить Rust AW worktime API ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-api" @@ -1277,35 +1112,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime API - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-api.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_api_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime API на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-api.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-api-rust - when: aw_worktime_api_rust_binary.stat.exists | default(false) - - - name: Установить скрипт AW worktime UI bridge - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.py" - dest: /usr/local/bin/aw-worktime-ui-bridge.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime UI bridge ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-ui-bridge" @@ -1313,6 +1119,12 @@ register: aw_worktime_ui_bridge_rust_binary become: false + - name: Требовать Rust AW worktime UI bridge artifact + ansible.builtin.assert: + that: + - aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-ui-bridge" + - name: Установить Rust AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-ui-bridge" @@ -1322,14 +1134,6 @@ mode: "0755" when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime autoheal - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.sh" - dest: /usr/local/bin/aw-worktime-autoheal.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime autoheal ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-autoheal" @@ -1337,6 +1141,12 @@ register: aw_worktime_autoheal_rust_binary become: false + - name: Требовать Rust AW worktime autoheal artifact + ansible.builtin.assert: + that: + - aw_worktime_autoheal_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-autoheal" + - name: Установить Rust AW worktime autoheal ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-autoheal" @@ -1346,14 +1156,6 @@ mode: "0755" when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime prewarm - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.sh" - dest: /usr/local/bin/aw-worktime-prewarm.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime prewarm ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-prewarm" @@ -1361,6 +1163,12 @@ register: aw_worktime_prewarm_rust_binary become: false + - name: Требовать Rust AW worktime prewarm artifact + ansible.builtin.assert: + that: + - aw_worktime_prewarm_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-prewarm" + - name: Установить Rust AW worktime prewarm ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-prewarm" @@ -1370,25 +1178,7 @@ mode: "0755" when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime Influx exporter - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.py" - dest: /usr/local/bin/aw-worktime-influx-exporter.py - owner: root - group: root - mode: "0755" - when: aw_worktime_influx_enabled | default(false) | bool - - - name: Установить скрипт AW DLP Influx exporter - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.py" - dest: /usr/local/bin/aw-dlp-influx-exporter.py - owner: root - group: root - mode: "0755" - when: aw_dlp_influx_enabled | default(false) | bool - - - name: Создать server-side ops bundle для legacy aw-health-check + - name: Создать server-side ops bundle для AW ops helpers ansible.builtin.file: path: /opt/activitywatch/aw-rus-ops state: directory @@ -1404,14 +1194,6 @@ group: root mode: "0755" - - name: Положить legacy aw-health-check в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" - dest: /opt/activitywatch/aw-rus-ops/health-check.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust aw-health-check ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-health-check" @@ -1428,6 +1210,12 @@ mode: "0755" when: aw_health_check_rust_binary.stat.exists | default(false) + - name: Требовать Rust aw-health-check artifact + ansible.builtin.assert: + that: + - aw_health_check_rust_binary.stat.exists | default(false) + fail_msg: "Rust aw-health-check artifact is required at {{ aw_rust_release_dir }}/aw-health-check" + - name: Проверить локальный Rust aw-db-health ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-db-health" @@ -1492,30 +1280,12 @@ mode: "0755" when: dlp_health_check_rust_binary.stat.exists | default(false) - - name: Установить legacy Python dlp-health-check - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/dlp-health-check.py" - dest: /usr/local/bin/dlp-health-check - owner: root - group: root - mode: "0755" - when: not (dlp_health_check_rust_binary.stat.exists | default(false)) - - - name: Установить скрипт aw-rus-healthd - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-rus-healthd.py" - dest: /usr/local/bin/aw-rus-healthd.py - owner: root - group: root - mode: "0755" - - - name: Установить скрипт aw-slo-monitor - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.py" - dest: /usr/local/bin/aw-slo-monitor.py - owner: root - group: root - mode: "0755" + - name: Проверить локальный Rust AW-RUS healthd + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-rus-healthd" + delegate_to: localhost + register: aw_rus_healthd_rust_binary + become: false - name: Проверить локальный Rust AW-RUS SLO monitor ansible.builtin.stat: @@ -1524,6 +1294,32 @@ register: aw_slo_monitor_rust_binary become: false + - name: Требовать Rust dlp-health-check artifact + ansible.builtin.assert: + that: + - dlp_health_check_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-health-check" + + - name: Требовать Rust AW-RUS healthd artifact + ansible.builtin.assert: + that: + - aw_rus_healthd_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-rus-healthd" + + - name: Требовать Rust AW-RUS SLO monitor artifact + ansible.builtin.assert: + that: + - aw_slo_monitor_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-slo-monitor" + + - name: Установить Rust AW-RUS healthd + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-rus-healthd" + dest: /usr/local/bin/aw-rus-healthd-rust + owner: root + group: root + mode: "0755" + - name: Установить Rust AW-RUS SLO monitor ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/aw-slo-monitor" @@ -1550,27 +1346,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime UI bridge - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-ui-bridge.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime UI bridge на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-ui-bridge.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-ui-bridge-rust - when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.timer" @@ -1587,27 +1362,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime autoheal - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-autoheal.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime autoheal на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-autoheal.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-autoheal-rust - when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.timer" @@ -1624,27 +1378,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime prewarm - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-prewarm.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime prewarm на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-prewarm.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-prewarm-rust - when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime prewarm ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.timer" @@ -1677,24 +1410,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW-RUS SLO monitor - ansible.builtin.file: - path: /etc/systemd/system/aw-slo-monitor.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_slo_monitor_rust_binary.stat.exists | default(false) - - - name: Переключить AW-RUS SLO monitor на Rust - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor-rust-switch.conf" - dest: /etc/systemd/system/aw-slo-monitor.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - when: aw_slo_monitor_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW-RUS SLO monitor ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.timer" @@ -1738,6 +1453,13 @@ become: false when: aw_worktime_influx_enabled | default(false) | bool + - name: Требовать Rust AW worktime Influx exporter artifact + ansible.builtin.assert: + that: + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-influx-exporter" + when: aw_worktime_influx_enabled | default(false) | bool + - name: Установить Rust AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-influx-exporter" @@ -1749,31 +1471,6 @@ - aw_worktime_influx_enabled | default(false) | bool - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог AW worktime Influx exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-influx-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_worktime_influx_enabled | default(false) | bool - - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime Influx exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-influx-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-influx-exporter-rust - when: - - aw_worktime_influx_enabled | default(false) | bool - - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.timer" @@ -1800,6 +1497,13 @@ become: false when: aw_dlp_influx_enabled | default(false) | bool + - name: Требовать Rust AW DLP Influx exporter artifact + ansible.builtin.assert: + that: + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-influx-exporter" + when: aw_dlp_influx_enabled | default(false) | bool + - name: Установить Rust AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-influx-exporter" @@ -1811,31 +1515,6 @@ - aw_dlp_influx_enabled | default(false) | bool - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог AW DLP Influx exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-influx-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_influx_enabled | default(false) | bool - - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить AW DLP Influx exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-influx-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-influx-exporter-rust - when: - - aw_dlp_influx_enabled | default(false) | bool - - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.timer" @@ -2044,14 +1723,6 @@ regexp: '' replace: '' - - name: Скопировать merge script AW DB на сервер - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/merge_aw_server_dbs.py" - dest: /usr/local/bin/merge_aw_server_dbs.py - owner: root - group: root - mode: "0755" - - name: Check Rust merge AW DB binary artifact ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" @@ -2059,6 +1730,13 @@ delegate_to: localhost become: false + - name: Требовать Rust merge AW DB artifact + ansible.builtin.assert: + that: + - aw_merge_db_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/merge-aw-server-dbs" + when: aw_legacy_db_merge_enabled | default(false) | bool + - name: Install Rust merge AW DB binary ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" @@ -2125,8 +1803,7 @@ - name: Merge legacy root DB в target DB ansible.builtin.command: argv: - - python3 - - /usr/local/bin/merge_aw_server_dbs.py + - /usr/local/bin/merge-aw-server-dbs - --base - /root/.local/share/activitywatch/aw-server-rust/sqlite.db - --overlay @@ -2256,34 +1933,6 @@ group: root mode: "0755" - - name: Скопировать агрегатор событий DLP - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/aggregate_dlp_events.py" - dest: "/opt/activitywatch/scripts/aggregate_dlp_events.py" - owner: root - group: root - mode: "0755" - - - name: Установить systemd unit для агрегатора - ansible.builtin.copy: - dest: /etc/systemd/system/activitywatch-dlp-aggregator.service - content: | - [Unit] - Description=ActivityWatch DLP Event Aggregator - After=activitywatch-server.service - - [Service] - Type=oneshot - User={{ aw_server_user }} - WorkingDirectory={{ aw_server_data_dir }} - ExecStart=/usr/bin/python3 /opt/activitywatch/scripts/aggregate_dlp_events.py \ - --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ - --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ - --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json - - [Install] - WantedBy=multi-user.target - - name: Проверить локальный Rust DLP aggregator ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-aggregator" @@ -2291,6 +1940,12 @@ register: dlp_aggregator_rust_binary become: false + - name: Требовать Rust DLP aggregator artifact + ansible.builtin.assert: + that: + - dlp_aggregator_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-aggregator" + - name: Установить Rust DLP aggregator ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-aggregator" @@ -2300,29 +1955,28 @@ mode: "0755" when: dlp_aggregator_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP aggregator - ansible.builtin.file: - path: /etc/systemd/system/activitywatch-dlp-aggregator.service.d - state: directory - owner: root - group: root - mode: "0755" - when: dlp_aggregator_rust_binary.stat.exists | default(false) - - - name: Переключить DLP aggregator на Rust + - name: Установить systemd unit для агрегатора ansible.builtin.copy: - dest: /etc/systemd/system/activitywatch-dlp-aggregator.service.d/20-rust-switch.conf + dest: /etc/systemd/system/activitywatch-dlp-aggregator.service owner: root group: root mode: "0644" content: | + [Unit] + Description=ActivityWatch DLP Event Aggregator + After=activitywatch-server.service + [Service] - ExecStart= + Type=oneshot + User={{ aw_server_user }} + WorkingDirectory={{ aw_server_data_dir }} ExecStart=/usr/local/bin/dlp-aggregator-rust \ --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json - when: dlp_aggregator_rust_binary.stat.exists | default(false) + + [Install] + WantedBy=multi-user.target - name: Установить systemd timer для агрегатора ansible.builtin.copy: @@ -2350,13 +2004,6 @@ - name: Настроить IOC enrichment из Hayabusa Sigma when: aw_dlp_ioc_enabled | default(false) | bool block: - - name: Установить зависимости IOC refresh - ansible.builtin.apt: - name: - - python3-yaml - state: present - update_cache: true - - name: Создать каталог IOC enrichment ansible.builtin.file: path: "{{ aw_dlp_ioc_workdir }}/{{ item }}" @@ -2369,10 +2016,23 @@ - output - tmp - - name: Установить экстрактор IOC из Sigma + - name: Проверить локальный Rust extractor IOC из Sigma + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/extract-ioc-from-sigma" + delegate_to: localhost + register: aw_extract_ioc_from_sigma_rust_binary + become: false + + - name: Требовать Rust extractor IOC из Sigma + ansible.builtin.assert: + that: + - aw_extract_ioc_from_sigma_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/extract-ioc-from-sigma" + + - name: Установить Rust extractor IOC из Sigma ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/extract_ioc_from_sigma.py" - dest: /usr/local/bin/aw-extract-ioc-from-sigma.py + src: "{{ aw_rust_release_dir }}/extract-ioc-from-sigma" + dest: /usr/local/bin/aw-extract-ioc-from-sigma owner: root group: root mode: "0755" @@ -2404,7 +2064,7 @@ fi mv "${EXTRACTED_DIR}" "${RULES_DIR}" - /usr/bin/python3 /usr/local/bin/aw-extract-ioc-from-sigma.py \ + /usr/local/bin/aw-extract-ioc-from-sigma \ --rules-root "${RULES_DIR}" \ --out-dir "${OUTDIR}" @@ -2716,28 +2376,23 @@ mode: "0755" when: aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper link-case для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + - name: Установить helper link-case для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-link-case-rust ]]; then - exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py "$@" + exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" dest: /usr/local/bin/aw-hayabusa-link-case owner: root group: root mode: "0755" - - name: Положить helper link-case в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-link-case.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust helper from-windows для Hayabusa ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" @@ -2754,28 +2409,23 @@ mode: "0755" when: aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper from-windows для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + - name: Установить helper from-windows для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-from-windows-rust ]]; then - exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py "$@" + exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" dest: /usr/local/bin/aw-hayabusa-from-windows owner: root group: root mode: "0755" - - name: Положить helper from-windows в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-from-windows.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py - owner: root - group: root - mode: "0755" - - name: Положить README Hayabusa в server-side ops bundle ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/hayabusa/README.md" @@ -2800,28 +2450,23 @@ mode: "0755" when: aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper autoprocess для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + - name: Установить helper autoprocess для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-autoprocess-rust ]]; then - exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py "$@" + exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" dest: /usr/local/bin/aw-hayabusa-autoprocess owner: root group: root mode: "0755" - - name: Положить helper autoprocess в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-autoprocess.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust helper case-alert для Hayabusa ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" @@ -2838,28 +2483,23 @@ mode: "0755" when: aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper case-alert для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + - name: Установить helper case-alert для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-case-alert-rust ]]; then - exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py "$@" + exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" dest: /usr/local/bin/aw-hayabusa-case-alert owner: root group: root mode: "0755" - - name: Положить helper case-alert в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-case-alert.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py - owner: root - group: root - mode: "0755" - - name: Установить systemd unit aw-hayabusa-drop.service ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-hayabusa-drop.service" @@ -2907,9 +2547,9 @@ - not ansible_check_mode - aw_server_post_deploy_health_check_enabled | default(true) | bool block: - - name: Запустить /usr/local/bin/aw-rus-healthd.py --json + - name: Запустить /usr/local/bin/aw-rus-healthd-rust --json ansible.builtin.command: - cmd: /usr/local/bin/aw-rus-healthd.py --json + cmd: /usr/local/bin/aw-rus-healthd-rust --json register: aw_post_deploy_health changed_when: false failed_when: false diff --git a/ansible/deploy_tsj_guardian_bot_proxmox.yml b/ansible/deploy_tsj_guardian_bot_proxmox.yml index ed3e24e..bb93181 100644 --- a/ansible/deploy_tsj_guardian_bot_proxmox.yml +++ b/ansible/deploy_tsj_guardian_bot_proxmox.yml @@ -258,7 +258,7 @@ AW_RUS_WORKTIME_BASE={{ tsj_bot_aw_rus_worktime_base | default('http://10.10.10.13:5610') }} AW_DLP_POLICY_API_BASE={{ tsj_bot_aw_dlp_policy_api_base | default('http://10.10.10.13:5601/api/0') }} AW_DLP_POLICY_ACTOR={{ tsj_bot_aw_dlp_policy_actor | default('tsj-guardian-bot') }} - AW_RUS_WORKTIME_HEAL_CMD={{ tsj_bot_aw_rus_worktime_heal_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal.sh && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'") }} + AW_RUS_WORKTIME_HEAL_CMD={{ tsj_bot_aw_rus_worktime_heal_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal-rust && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'") }} AW_RUS_DLP_HEAL_CMD={{ tsj_bot_aw_rus_dlp_heal_cmd | default("ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n systemctl restart activitywatch-server.service && (sudo -n systemctl start activitywatch-dlp-aggregator.service || true) && sudo -n /usr/local/bin/aw-health-check && sudo -n /usr/local/bin/dlp-health-check'") }} AW_RUS_CASE_API_BASE={{ tsj_bot_aw_rus_case_api_base | default('http://10.10.10.13:5602') }} AW_RUS_HAYABUSA_ENABLED={{ tsj_bot_aw_rus_hayabusa_enabled | default('true') }} @@ -305,7 +305,7 @@ - { key: "AW_RUS_WORKTIME_BASE", value: "{{ tsj_bot_aw_rus_worktime_base | default('http://10.10.10.13:5610') }}" } - { key: "AW_DLP_POLICY_API_BASE", value: "{{ tsj_bot_aw_dlp_policy_api_base | default('http://10.10.10.13:5601/api/0') }}" } - { key: "AW_DLP_POLICY_ACTOR", value: "{{ tsj_bot_aw_dlp_policy_actor | default('tsj-guardian-bot') }}" } - - { key: "AW_RUS_WORKTIME_HEAL_CMD", value: "{{ tsj_bot_aw_rus_worktime_heal_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal.sh && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'\") }}" } + - { key: "AW_RUS_WORKTIME_HEAL_CMD", value: "{{ tsj_bot_aw_rus_worktime_heal_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n /usr/local/bin/aw-worktime-autoheal-rust && sudo -n systemctl reset-failed aw-worktime-ui-bridge.service && sudo -n systemctl start aw-worktime-ui-bridge.service'\") }}" } - { key: "AW_RUS_DLP_HEAL_CMD", value: "{{ tsj_bot_aw_rus_dlp_heal_cmd | default(\"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new igor@10.10.10.13 'sudo -n systemctl restart activitywatch-server.service && (sudo -n systemctl start activitywatch-dlp-aggregator.service || true) && sudo -n /usr/local/bin/aw-health-check && sudo -n /usr/local/bin/dlp-health-check'\") }}" } - { key: "AW_RUS_CASE_API_BASE", value: "{{ tsj_bot_aw_rus_case_api_base | default('http://10.10.10.13:5602') }}" } - { key: "AW_RUS_HAYABUSA_ENABLED", value: "{{ tsj_bot_aw_rus_hayabusa_enabled | default('true') }}" } diff --git a/ansible/group_vars/proxmox-bot.example.yml b/ansible/group_vars/proxmox-bot.example.yml index cf8a1a9..6c9c124 100644 --- a/ansible/group_vars/proxmox-bot.example.yml +++ b/ansible/group_vars/proxmox-bot.example.yml @@ -49,7 +49,7 @@ tsj_bot_aw_rus_api_base: "http://10.10.10.13:5600/api/0" tsj_bot_aw_rus_worktime_base: "http://10.10.10.13:5610" tsj_bot_aw_dlp_policy_api_base: "http://10.10.10.13:5601/api/0" tsj_bot_aw_dlp_policy_actor: "tsj-guardian-bot" -tsj_bot_aw_rus_worktime_heal_cmd: "sshpass -p 'CHANGE_ME' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S /usr/local/bin/aw-worktime-autoheal.sh && sudo -S systemctl reset-failed aw-worktime-ui-bridge.service && sudo -S systemctl start aw-worktime-ui-bridge.service'" +tsj_bot_aw_rus_worktime_heal_cmd: "sshpass -p 'CHANGE_ME' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S /usr/local/bin/aw-worktime-autoheal-rust && sudo -S systemctl reset-failed aw-worktime-ui-bridge.service && sudo -S systemctl start aw-worktime-ui-bridge.service'" tsj_bot_aw_rus_dlp_heal_cmd: "sshpass -p 'CHANGE_ME' ssh -o PubkeyAuthentication=no -o StrictHostKeyChecking=no igor@10.10.10.13 'sudo -S systemctl restart activitywatch-server.service && sudo -S systemctl start activitywatch-dlp-aggregator.service || true && sudo -S /usr/local/bin/aw-health-check && sudo -S /usr/local/bin/dlp-health-check'" tsj_bot_aw_rus_case_api_base: "http://10.10.10.13:5602" tsj_bot_aw_rus_hayabusa_enabled: "true" diff --git a/ansible/provision_proxmox_ct_and_deploy_aw.yml b/ansible/provision_proxmox_ct_and_deploy_aw.yml index 1ac786e..c6318db 100644 --- a/ansible/provision_proxmox_ct_and_deploy_aw.yml +++ b/ansible/provision_proxmox_ct_and_deploy_aw.yml @@ -1,46 +1,14 @@ --- -- name: Создать один Proxmox CT и развернуть AWatch-rus +- name: Deprecated Proxmox CT bootstrap path hosts: proxmox gather_facts: false - vars: - proxmox_bootstrap_dir: "/tmp/aw-rus-bootstrap" - aw_bootstrap_files: - - install_aw_server.sh - - apply_webui_ru_patch.sh - - activitywatch-server.service - - aw-worktime-api.py - - aw-worktime-api.service - - aw-worktime-ui-bridge.py - - aw-worktime-ui-bridge.service - - aw-worktime-ui-bridge.timer - - aw-worktime-panel.js - - aw-server.env.example - - aw-ru-patch.js - - aw-sw-cleanup.js - - aw-host-groups.json - - settings/classes-worktime.json - - settings/views-default.json - tasks: - - name: Выполнить workflow создания одного CT - ansible.builtin.include_tasks: tasks/provision_ct_and_deploy_aw.yml - vars: - ct_id: "{{ proxmox_ct_id }}" - ct_hostname: "{{ proxmox_ct_hostname }}" - ct_storage: "{{ proxmox_ct_storage }}" - ct_template: "{{ proxmox_ct_template }}" - ct_rootfs_size: "{{ proxmox_ct_rootfs_size }}" - ct_cores: "{{ proxmox_ct_cores }}" - ct_memory: "{{ proxmox_ct_memory }}" - ct_swap: "{{ proxmox_ct_swap }}" - ct_bridge: "{{ proxmox_ct_bridge }}" - ct_ip: "{{ proxmox_ct_ip }}" - ct_gw: "{{ proxmox_ct_gw }}" - ct_vlan: "{{ proxmox_ct_vlan | default('') }}" - ct_nameserver: "{{ proxmox_ct_nameserver | default('') }}" - ct_searchdomain: "{{ proxmox_ct_searchdomain | default('') }}" - ct_password: "{{ proxmox_ct_password }}" - ct_unprivileged: "{{ proxmox_ct_unprivileged }}" - ct_onboot: "{{ proxmox_ct_onboot }}" - ct_features: "{{ proxmox_ct_features }}" + - name: Stop deprecated Python-era CT bootstrap + ansible.builtin.fail: + msg: >- + ansible/provision_proxmox_ct_and_deploy_aw.yml is deprecated because + its bootstrap path copied removed Python service entrypoints. Use the + existing DetMir AW server and ansible/deploy_aw_server.yml for the + Rust application deploy. Rebuild this CT bootstrap path separately + before using it for new infrastructure. diff --git a/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml b/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml index e0ff9bc..3819d24 100644 --- a/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml +++ b/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml @@ -1,56 +1,14 @@ --- -- name: Создать матрицу Proxmox CT и развернуть AWatch-rus с RU patch +- name: Deprecated Proxmox CT matrix bootstrap path hosts: proxmox gather_facts: false - vars: - proxmox_bootstrap_dir: "/tmp/aw-rus-bootstrap" - aw_bootstrap_files: - - install_aw_server.sh - - apply_webui_ru_patch.sh - - activitywatch-server.service - - aw-worktime-api.py - - aw-worktime-api.service - - aw-worktime-ui-bridge.py - - aw-worktime-ui-bridge.service - - aw-worktime-ui-bridge.timer - - aw-worktime-panel.js - - aw-server.env.example - - aw-ru-patch.js - - aw-sw-cleanup.js - - aw-host-groups.json - - settings/classes-worktime.json - - settings/views-default.json - tasks: - - name: Проверить, что матрица CT задана - ansible.builtin.assert: - that: - - proxmox_ct_matrix is defined - - proxmox_ct_matrix | length > 0 - fail_msg: "Задайте proxmox_ct_matrix в group_vars/proxmox-matrix.yml" - - - name: Выполнить workflow создания для каждого CT - ansible.builtin.include_tasks: tasks/provision_ct_and_deploy_aw.yml - vars: - ct_id: "{{ item.id }}" - ct_hostname: "{{ item.hostname }}" - ct_storage: "{{ item.storage }}" - ct_template: "{{ item.template }}" - ct_rootfs_size: "{{ item.rootfs_size }}" - ct_cores: "{{ item.cores }}" - ct_memory: "{{ item.memory }}" - ct_swap: "{{ item.swap }}" - ct_bridge: "{{ item.bridge }}" - ct_ip: "{{ item.ip }}" - ct_gw: "{{ item.gw }}" - ct_vlan: "{{ item.vlan | default('') }}" - ct_nameserver: "{{ item.nameserver | default('') }}" - ct_searchdomain: "{{ item.searchdomain | default('') }}" - ct_password: "{{ item.password }}" - ct_unprivileged: "{{ item.unprivileged }}" - ct_onboot: "{{ item.onboot }}" - ct_features: "{{ item.features }}" - loop: "{{ proxmox_ct_matrix }}" - loop_control: - label: "ct={{ item.id }} host={{ item.hostname }} ip={{ item.ip }}" + - name: Stop deprecated Python-era CT matrix bootstrap + ansible.builtin.fail: + msg: >- + ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml is deprecated + because its bootstrap path copied removed Python service entrypoints. + Use the existing DetMir AW server and ansible/deploy_aw_server.yml for + the Rust application deploy. Rebuild this CT bootstrap path separately + before using it for new infrastructure. diff --git a/ansible/roles/dlp-case-management/tasks/main.yml b/ansible/roles/dlp-case-management/tasks/main.yml index 0b121a5..b7aba1a 100644 --- a/ansible/roles/dlp-case-management/tasks/main.yml +++ b/ansible/roles/dlp-case-management/tasks/main.yml @@ -1,55 +1,7 @@ --- -- name: Ensure DLP case management directory - ansible.builtin.file: - path: /opt/activitywatch/dlp-case-management - state: directory - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - -- name: Copy DLP case management files - ansible.builtin.copy: - src: "{{ playbook_dir }}/../aw-server/dlp-case-management/{{ item }}" - dest: "/opt/activitywatch/dlp-case-management/{{ item }}" - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0644" - loop: - - case_service.py - - case_schema.py - - case_storage.py - - evidence_chain.py - - requirements.txt - -- name: Make case service executable - ansible.builtin.file: - path: /opt/activitywatch/dlp-case-management/case_service.py - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - -- name: Create venv for DLP case management - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-case-management/.venv - args: - creates: /opt/activitywatch/dlp-case-management/.venv/bin/python - -- name: Install Python dependencies for DLP case management - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-case-management/requirements.txt - virtualenv: /opt/activitywatch/dlp-case-management/.venv - -- name: Install systemd unit DLP case management - ansible.builtin.copy: - src: "{{ playbook_dir }}/../aw-server/dlp-case-management/case-service.service" - dest: /etc/systemd/system/aw-dlp-case-management.service - owner: root - group: root - mode: "0644" - -- name: Reload systemd and restart DLP case management - ansible.builtin.systemd: - name: aw-dlp-case-management.service - daemon_reload: true - enabled: true - state: restarted +- name: Deprecated role guard - DLP case management is Rust-only + ansible.builtin.fail: + msg: >- + ansible/roles/dlp-case-management is deprecated because it deployed the old + Python runtime. Use ansible/deploy_aw_server.yml, which installs + /usr/local/bin/aw-dlp-case-management-rust from adk-rust. diff --git a/ansible/roles/dlp-compliance/tasks/main.yml b/ansible/roles/dlp-compliance/tasks/main.yml index 3f681de..997d37f 100644 --- a/ansible/roles/dlp-compliance/tasks/main.yml +++ b/ansible/roles/dlp-compliance/tasks/main.yml @@ -1,54 +1,7 @@ --- -- name: Ensure DLP compliance directory tree - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - loop: - - /opt/activitywatch/dlp-compliance - - /opt/activitywatch/dlp-compliance/templates - - /opt/activitywatch/dlp-compliance/reports - -- name: Copy DLP compliance files - ansible.builtin.copy: - src: "{{ playbook_dir }}/../aw-server/dlp-compliance/{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "{{ item.mode }}" - loop: - - { src: "report_generator.py", dest: "/opt/activitywatch/dlp-compliance/report_generator.py", mode: "0755" } - - { src: "compliance_scheduler.py", dest: "/opt/activitywatch/dlp-compliance/compliance_scheduler.py", mode: "0755" } - - { src: "requirements.txt", dest: "/opt/activitywatch/dlp-compliance/requirements.txt", mode: "0644" } - - { src: "templates/152-fz-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html", mode: "0644" } - - { src: "templates/pci-dss-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/pci-dss-report.html", mode: "0644" } - - { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" } - - { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" } - -- name: Create venv for DLP compliance - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-compliance/.venv - args: - creates: /opt/activitywatch/dlp-compliance/.venv/bin/python - -- name: Install Python dependencies for DLP compliance - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-compliance/requirements.txt - virtualenv: /opt/activitywatch/dlp-compliance/.venv - -- name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true - -- name: Enable and start DLP compliance timer - ansible.builtin.systemd: - name: aw-dlp-report-scheduler.timer - enabled: true - state: restarted - -- name: Run DLP compliance report once - ansible.builtin.systemd: - name: aw-dlp-report-scheduler.service - state: started +- name: Deprecated role guard - DLP compliance is Rust-only + ansible.builtin.fail: + msg: >- + ansible/roles/dlp-compliance is deprecated because it deployed the old + Python runtime. Use ansible/deploy_aw_server.yml, which installs + /usr/local/bin/aw-dlp-compliance-rust from adk-rust. diff --git a/ansible/roles/dlp-integrations/tasks/main.yml b/ansible/roles/dlp-integrations/tasks/main.yml index fe0e340..915e150 100644 --- a/ansible/roles/dlp-integrations/tasks/main.yml +++ b/ansible/roles/dlp-integrations/tasks/main.yml @@ -1,194 +1,7 @@ --- -- name: Ensure DLP integrations directory - ansible.builtin.file: - path: /opt/activitywatch/dlp-integrations - state: directory - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - -- name: Copy DLP integrations files - ansible.builtin.copy: - src: "{{ playbook_dir }}/../aw-server/dlp-integrations/{{ item }}" - dest: "/opt/activitywatch/dlp-integrations/{{ item }}" - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0644" - loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - - cef-config.yaml - - syslog-forwarder-config.yaml - - webhook-config.yaml - - requirements.txt - -- name: Make DLP integrations scripts executable - ansible.builtin.file: - path: "/opt/activitywatch/dlp-integrations/{{ item }}" - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - -- name: Create venv for DLP integrations - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-integrations/.venv - args: - creates: /opt/activitywatch/dlp-integrations/.venv/bin/python - -- name: Install Python dependencies for DLP integrations - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-integrations/requirements.txt - virtualenv: /opt/activitywatch/dlp-integrations/.venv - -- name: Ensure integrations state directory - ansible.builtin.file: - path: /var/lib/activitywatch/dlp-integrations - state: directory - owner: "{{ aw_server_user | default('activitywatch') }}" - group: "{{ aw_server_group | default('activitywatch') }}" - mode: "0755" - -- name: Install systemd units/timers for DLP integrations - ansible.builtin.copy: - src: "{{ playbook_dir }}/../aw-server/dlp-integrations/{{ item.src }}" - dest: "{{ item.dest }}" - owner: root - group: root - mode: "0644" - loop: - - { src: "cef-exporter.service", dest: "/etc/systemd/system/aw-dlp-cef-exporter.service" } - - { src: "cef-exporter.timer", dest: "/etc/systemd/system/aw-dlp-cef-exporter.timer" } - - { src: "syslog-forwarder.service", dest: "/etc/systemd/system/aw-dlp-syslog-forwarder.service" } - - { src: "syslog-forwarder.timer", dest: "/etc/systemd/system/aw-dlp-syslog-forwarder.timer" } - - { src: "webhook-sender.service", dest: "/etc/systemd/system/aw-dlp-webhook-sender.service" } - - { src: "webhook-sender.timer", dest: "/etc/systemd/system/aw-dlp-webhook-sender.timer" } - -- name: Check local Rust CEF exporter - ansible.builtin.stat: - path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-cef-exporter" - delegate_to: localhost - register: dlp_cef_exporter_rust_binary - become: false - -- name: Install Rust CEF exporter - ansible.builtin.copy: - src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-cef-exporter" - dest: /usr/local/bin/dlp-cef-exporter-rust - owner: root - group: root - mode: "0755" - when: dlp_cef_exporter_rust_binary.stat.exists | default(false) - -- name: Ensure CEF exporter drop-in directory - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-cef-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: dlp_cef_exporter_rust_binary.stat.exists | default(false) - -- name: Switch CEF exporter to Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-cef-exporter-rust - when: dlp_cef_exporter_rust_binary.stat.exists | default(false) - -- name: Check local Rust syslog forwarder - ansible.builtin.stat: - path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-syslog-forwarder" - delegate_to: localhost - register: dlp_syslog_forwarder_rust_binary - become: false - -- name: Install Rust syslog forwarder - ansible.builtin.copy: - src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-syslog-forwarder" - dest: /usr/local/bin/dlp-syslog-forwarder-rust - owner: root - group: root - mode: "0755" - when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - -- name: Ensure syslog forwarder drop-in directory - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d - state: directory - owner: root - group: root - mode: "0755" - when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - -- name: Switch syslog forwarder to Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust - when: dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - -- name: Check local Rust webhook sender - ansible.builtin.stat: - path: "{{ playbook_dir }}/../adk-rust/target/release/dlp-webhook-sender" - delegate_to: localhost - register: dlp_webhook_sender_rust_binary - become: false - -- name: Install Rust webhook sender - ansible.builtin.copy: - src: "{{ playbook_dir }}/../adk-rust/target/release/dlp-webhook-sender" - dest: /usr/local/bin/dlp-webhook-sender-rust - owner: root - group: root - mode: "0755" - when: dlp_webhook_sender_rust_binary.stat.exists | default(false) - -- name: Ensure webhook sender drop-in directory - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-webhook-sender.service.d - state: directory - owner: root - group: root - mode: "0755" - when: dlp_webhook_sender_rust_binary.stat.exists | default(false) - -- name: Switch webhook sender to Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-webhook-sender-rust - when: dlp_webhook_sender_rust_binary.stat.exists | default(false) - -- name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true - -- name: Enable and restart DLP integration timers - ansible.builtin.systemd: - name: "{{ item }}" - enabled: true - state: restarted - loop: - - aw-dlp-cef-exporter.timer - - aw-dlp-syslog-forwarder.timer - - aw-dlp-webhook-sender.timer +- name: Deprecated role guard - DLP integrations are Rust-only + ansible.builtin.fail: + msg: >- + ansible/roles/dlp-integrations is deprecated because it deployed the old + Python runtime. Use ansible/deploy_aw_server.yml, which installs the + Rust CEF, syslog, and webhook exporters from adk-rust. diff --git a/ansible/roles/dlp-policy-engine/tasks/main.yml b/ansible/roles/dlp-policy-engine/tasks/main.yml index ec4d99e..7dfa7d0 100644 --- a/ansible/roles/dlp-policy-engine/tasks/main.yml +++ b/ansible/roles/dlp-policy-engine/tasks/main.yml @@ -1,46 +1,7 @@ --- -- name: Ensure policy engine directory exists - become: true - ansible.builtin.file: - path: /opt/activitywatch/dlp-policy-engine - state: directory - owner: activitywatch - group: activitywatch - mode: "0755" - -- name: Deploy policy engine files - become: true - ansible.builtin.copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: activitywatch - group: activitywatch - mode: "{{ item.mode | default('0644') }}" - loop: - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_service.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_service.py" } - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_schema.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_schema.py" } - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_storage.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_storage.py" } - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/policy_distributor.py", dest: "/opt/activitywatch/dlp-policy-engine/policy_distributor.py" } - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/requirements.txt", dest: "/opt/activitywatch/dlp-policy-engine/requirements.txt" } - - { src: "{{ playbook_dir }}/../aw-server/dlp-policy-engine/dlp-policy-engine.service", dest: "/etc/systemd/system/aw-dlp-policy-engine.service" } - -- name: Ensure policy engine venv exists - become: true - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-policy-engine/.venv - args: - creates: /opt/activitywatch/dlp-policy-engine/.venv/bin/python - -- name: Install python deps for policy engine venv - become: true - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-policy-engine/requirements.txt - virtualenv: /opt/activitywatch/dlp-policy-engine/.venv - -- name: Enable and restart policy engine - become: true - ansible.builtin.systemd: - daemon_reload: true - name: aw-dlp-policy-engine.service - enabled: true - state: restarted +- name: Deprecated role guard - DLP policy engine is Rust-only + ansible.builtin.fail: + msg: >- + ansible/roles/dlp-policy-engine is deprecated because it deployed the old + Python runtime. Use ansible/deploy_aw_server.yml, which installs + /usr/local/bin/aw-dlp-policy-engine-rust from adk-rust. diff --git a/aw-server/aw-dlp-influx-exporter.py b/aw-server/aw-dlp-influx-exporter.py deleted file mode 100644 index 85005a3..0000000 --- a/aw-server/aw-dlp-influx-exporter.py +++ /dev/null @@ -1,417 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import sys -import urllib.parse -import urllib.request -from datetime import UTC, datetime, timedelta - - -def _env_bool(name: str, default: bool = False) -> bool: - value = os.environ.get(name, "").strip().lower() - if not value: - return default - return value in {"1", "true", "yes", "on"} - - -AW_API_BASE = os.environ.get("AW_DLP_AW_API_BASE", "http://127.0.0.1:5600/api/0").strip().rstrip("/") -CASE_API_BASE = os.environ.get("AW_DLP_CASE_API_BASE", "http://127.0.0.1:5602/api/0/dlp/cases").strip().rstrip("/") -INFLUX_URL = os.environ.get("AW_DLP_INFLUX_URL", "").strip().rstrip("/") -INFLUX_ORG = os.environ.get("AW_DLP_INFLUX_ORG", "proxmox").strip() or "proxmox" -INFLUX_BUCKET = os.environ.get("AW_DLP_INFLUX_BUCKET", "aw_metrics").strip() or "aw_metrics" -INFLUX_TOKEN = os.environ.get("AW_DLP_INFLUX_TOKEN", "").strip() -INFLUX_ENABLED = _env_bool("AW_DLP_INFLUX_ENABLED", False) -HOSTS = [item.strip() for item in os.environ.get("AW_DLP_INFLUX_HOSTS", "SHARKON2025").split(",") if item.strip()] -LOOKBACK_DAYS = int(os.environ.get("AW_DLP_INFLUX_LOOKBACK_DAYS", "30") or "30") -EVENT_LIMIT = int(os.environ.get("AW_DLP_INFLUX_EVENT_LIMIT", "2000") or "2000") -CASE_LIMIT = int(os.environ.get("AW_DLP_CASE_LIMIT", "500") or "500") - - -def utc_now() -> datetime: - return datetime.now(tz=UTC) - - -def pts(value: str | None) -> datetime: - if not value: - return utc_now() - normalized = value.replace("Z", "+00:00") - parsed = datetime.fromisoformat(normalized) - if parsed.tzinfo is None: - return parsed.replace(tzinfo=UTC) - return parsed.astimezone(UTC) - - -def _escape_tag(value: object) -> str: - return ( - str(value or "") - .replace("\\", "\\\\") - .replace(" ", "\\ ") - .replace(",", "\\,") - .replace("=", "\\=") - ) - - -def _line(measurement: str, tags: dict[str, object], fields: dict[str, object], timestamp_ns: int) -> str: - tag_part = ",".join(f"{key}={_escape_tag(value)}" for key, value in sorted(tags.items()) if value not in (None, "")) - field_parts: list[str] = [] - for key, value in fields.items(): - if value is None: - continue - if isinstance(value, bool): - field_parts.append(f"{key}={'true' if value else 'false'}") - elif isinstance(value, int): - field_parts.append(f"{key}={value}i") - elif isinstance(value, float): - field_parts.append(f"{key}={value}") - else: - text = str(value).replace("\\", "\\\\").replace('"', '\\"') - field_parts.append(f'{key}="{text}"') - if not field_parts: - return "" - if tag_part: - return f"{measurement},{tag_part} {','.join(field_parts)} {timestamp_ns}" - return f"{measurement} {','.join(field_parts)} {timestamp_ns}" - - -def _timestamp_ns(dt: datetime) -> int: - return int(dt.astimezone(UTC).timestamp() * 1_000_000_000) - - -def _get_json(url: str) -> object: - req = urllib.request.Request(url, headers={"Accept": "application/json"}) - with urllib.request.urlopen(req, timeout=30) as response: - return json.loads(response.read().decode("utf-8")) - - -def fetch_bucket_events(bucket_id: str, start: datetime, end: datetime, limit: int) -> list[dict]: - query = urllib.parse.urlencode( - { - "start": start.astimezone(UTC).isoformat().replace("+00:00", "Z"), - "end": end.astimezone(UTC).isoformat().replace("+00:00", "Z"), - "limit": str(limit), - } - ) - url = f"{AW_API_BASE}/buckets/{urllib.parse.quote(bucket_id, safe='')}/events?{query}" - payload = _get_json(url) - return payload if isinstance(payload, list) else [] - - -def fetch_cases(host: str) -> list[dict]: - query = urllib.parse.urlencode({"host": host, "limit": str(CASE_LIMIT)}) - payload = _get_json(f"{CASE_API_BASE}?{query}") - return payload if isinstance(payload, list) else [] - - -def _s(value: object) -> str: - return str(value or "").strip() - - -def _first_nonempty(*values: object, default: str = "") -> str: - for value in values: - text = _s(value) - if text: - return text - return default - - -def normalize_incident(event: dict, default_host: str) -> dict[str, object]: - data = event.get("data") or {} - source_event = data.get("sourceEvent") or {} - source_data = source_event.get("data") or {} - nested = data.get("incident") or {} - - signal_type = _first_nonempty(data.get("signalType"), source_data.get("signalType"), default="unknown") - username = _first_nonempty( - data.get("username"), - source_data.get("username"), - source_data.get("owner"), - source_data.get("host"), - default="unknown", - ) - host = _first_nonempty(data.get("hostname"), data.get("host"), source_data.get("hostname"), default=default_host) - severity = _first_nonempty(data.get("severity"), nested.get("severity"), default="unknown") - action = _first_nonempty(data.get("action"), nested.get("verdict"), default="incident") - message = _first_nonempty( - data.get("message"), - source_data.get("documentName"), - source_data.get("documentNameOriginal"), - nested.get("comment"), - ) - return { - "host": host, - "signal_type": signal_type, - "username": username, - "severity": severity, - "action": action, - "message": message, - "rule_id": _first_nonempty(data.get("ruleId")), - "source": _first_nonempty(data.get("source"), source_data.get("source"), data.get("sourceBucket")), - "document_name": _first_nonempty(source_data.get("documentName"), source_data.get("documentNameOriginal")), - "printer_name": _first_nonempty(source_data.get("printerName")), - "incident_status": _first_nonempty(nested.get("status")), - "incident_verdict": _first_nonempty(nested.get("verdict")), - "regex_matches": len(data.get("regexMatches") or []), - "dictionary_matches": len(data.get("dictionaryMatches") or []), - "ocr_requested": bool(data.get("ocrRequested")), - } - - -def build_endpoint_lines(host: str, events: list[dict]) -> list[str]: - lines: list[str] = [] - for item in events: - data = item.get("data") or {} - signal_type = _first_nonempty(data.get("signalType"), default="unknown") - timestamp_ns = _timestamp_ns(pts(item.get("timestamp"))) - event_id = item.get("id") or f"{signal_type}-{timestamp_ns}" - username = _first_nonempty(data.get("username"), data.get("owner"), default="unknown") - if signal_type == "self_test": - lines.append( - _line( - "aw_dlp_endpoint_self_test", - { - "host": _first_nonempty(data.get("hostname"), default=host), - "event_id": event_id, - "username": username, - "policy_mode": _first_nonempty(data.get("policyMode"), default="unknown"), - "policy_source": _first_nonempty(data.get("policySource"), default="unknown"), - }, - { - "count": 1, - "queue_depth": int(data.get("queueDepth") or 0), - "events_enqueued": int(data.get("eventsEnqueued") or 0), - "events_flushed": int(data.get("eventsFlushed") or 0), - "send_failures": int(data.get("sendFailures") or 0), - "policy_enabled": bool(data.get("policyEnabled")), - }, - timestamp_ns, - ) - ) - continue - lines.append( - _line( - "aw_dlp_signal", - { - "host": _first_nonempty(data.get("hostname"), default=host), - "event_id": event_id, - "signal_type": signal_type, - "username": username, - "source": _first_nonempty(data.get("source"), default="unknown"), - }, - { - "count": 1, - "document_name": _first_nonempty(data.get("documentName"), data.get("documentNameOriginal")), - "printer_name": _first_nonempty(data.get("printerName")), - "owner": _first_nonempty(data.get("owner")), - "session_id": int(data.get("sessionId") or 0), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_incident_lines(host: str, events: list[dict]) -> list[str]: - lines: list[str] = [] - for item in events: - normalized = normalize_incident(item, host) - timestamp_ns = _timestamp_ns(pts(item.get("timestamp"))) - event_id = item.get("id") or f"incident-{timestamp_ns}" - lines.append( - _line( - "aw_dlp_incident", - { - "host": normalized["host"], - "event_id": event_id, - "signal_type": normalized["signal_type"], - "severity": normalized["severity"], - "action": normalized["action"], - "username": normalized["username"], - "source": normalized["source"], - }, - { - "count": 1, - "message": normalized["message"], - "rule_id": normalized["rule_id"], - "document_name": normalized["document_name"], - "printer_name": normalized["printer_name"], - "incident_status": normalized["incident_status"], - "incident_verdict": normalized["incident_verdict"], - "regex_matches": int(normalized["regex_matches"]), - "dictionary_matches": int(normalized["dictionary_matches"]), - "ocr_requested": bool(normalized["ocr_requested"]), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_review_lines(host: str, events: list[dict]) -> list[str]: - lines: list[str] = [] - for item in events: - data = item.get("data") or {} - review = data.get("review") or {} - source_data = (data.get("sourceEvent") or {}).get("data") or {} - timestamp_ns = _timestamp_ns(pts(item.get("timestamp"))) - review_id = _first_nonempty(review.get("reviewId"), default=f"review-{timestamp_ns}") - lines.append( - _line( - "aw_dlp_review", - { - "host": _first_nonempty(data.get("host"), source_data.get("hostname"), default=host), - "review_id": review_id, - "verdict": _first_nonempty(review.get("verdict"), default="unknown"), - "signal_type": _first_nonempty(source_data.get("signalType"), default="unknown"), - "username": _first_nonempty(source_data.get("username"), source_data.get("owner"), default="unknown"), - }, - { - "count": 1, - "archived": bool(review.get("archived")), - "comment": _first_nonempty(review.get("comment")), - "category": _first_nonempty(review.get("category")), - "document_name": _first_nonempty(source_data.get("documentName"), source_data.get("documentNameOriginal")), - "printer_name": _first_nonempty(source_data.get("printerName")), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_rule_lines(host: str, events: list[dict]) -> list[str]: - lines: list[str] = [] - for item in events: - data = item.get("data") or {} - match = data.get("match") or {} - timestamp_ns = _timestamp_ns(pts(item.get("timestamp"))) - rule_id = _first_nonempty(data.get("ruleId"), default=f"rule-{timestamp_ns}") - lines.append( - _line( - "aw_dlp_rule", - { - "host": _first_nonempty(data.get("host"), match.get("hostname"), default=host), - "rule_id": rule_id, - "action": _first_nonempty(data.get("action"), default="unknown"), - "signal_type": _first_nonempty(match.get("signalType"), default="unknown"), - "username": _first_nonempty(match.get("username"), match.get("owner"), default="unknown"), - "enabled": "true" if bool(data.get("enabled", True)) else "false", - }, - { - "count": 1, - "category": _first_nonempty(data.get("category")), - "comment": _first_nonempty(data.get("comment")), - "document_name": _first_nonempty(match.get("documentName")), - "printer_name": _first_nonempty(match.get("printerName")), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_fileops_lines(host: str, events: list[dict]) -> list[str]: - lines: list[str] = [] - for item in events: - data = item.get("data") or {} - signal_type = _first_nonempty(data.get("signalType"), default="unknown") - if signal_type != "collector_health": - continue - timestamp_ns = _timestamp_ns(pts(item.get("timestamp"))) - event_id = item.get("id") or f"fileops-{timestamp_ns}" - lines.append( - _line( - "aw_dlp_fileops_health", - { - "host": _first_nonempty(data.get("hostname"), default=host), - "event_id": event_id, - "username": _first_nonempty(data.get("username"), default="unknown"), - }, - { - "count": 1, - "queue_depth": int(data.get("queueDepth") or 0), - "events_enqueued": int(data.get("eventsEnqueued") or 0), - "events_flushed": int(data.get("eventsFlushed") or 0), - "send_failures": int(data.get("sendFailures") or 0), - "session_id": int(data.get("sessionId") or 0), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_case_lines(host: str, cases: list[dict]) -> list[str]: - lines: list[str] = [] - for item in cases: - timestamp_ns = _timestamp_ns(pts(item.get("updated_at") or item.get("created_at"))) - evidence = item.get("evidence") or {} - evidence_items = evidence.get("items") or [] - lines.append( - _line( - "aw_dlp_case", - { - "host": _first_nonempty(item.get("host"), default=host), - "case_id": item.get("id"), - "status": _first_nonempty(item.get("status"), default="unknown"), - "severity": _first_nonempty(item.get("severity"), default="unknown"), - "assignee": _first_nonempty(item.get("assignee"), default="unassigned"), - }, - { - "count": 1, - "title": _first_nonempty(item.get("title")), - "incident_id": _first_nonempty(item.get("incident_id")), - "has_forensics": item.get("forensics") is not None, - "evidence_items": len(evidence_items), - "chain_length": int(evidence.get("chain_length") or 0), - }, - timestamp_ns, - ) - ) - return [line for line in lines if line] - - -def build_lines_for_host(host: str, start: datetime, end: datetime) -> list[str]: - lines: list[str] = [] - lines.extend(build_endpoint_lines(host, fetch_bucket_events(f"aw-dlp-endpoint-signals_{host}", start, end, EVENT_LIMIT))) - lines.extend(build_incident_lines(host, fetch_bucket_events(f"aw-dlp-incidents_{host}", start, end, EVENT_LIMIT))) - lines.extend(build_review_lines(host, fetch_bucket_events(f"aw-dlp-review_{host}", start, end, EVENT_LIMIT))) - lines.extend(build_rule_lines(host, fetch_bucket_events(f"aw-dlp-rules_{host}", start, end, EVENT_LIMIT))) - lines.extend(build_fileops_lines(host, fetch_bucket_events(f"aw-file-operations_{host}", start, end, EVENT_LIMIT))) - lines.extend(build_case_lines(host, fetch_cases(host))) - return [line for line in lines if line] - - -def write_lines(lines: list[str]) -> int: - if not lines: - return 0 - if not INFLUX_URL or not INFLUX_TOKEN: - raise RuntimeError("InfluxDB destination is not configured") - payload = ("\n".join(lines) + "\n").encode("utf-8") - req = urllib.request.Request( - f"{INFLUX_URL}/api/v2/write?org={INFLUX_ORG}&bucket={INFLUX_BUCKET}&precision=ns", - data=payload, - method="POST", - headers={"Authorization": f"Token {INFLUX_TOKEN}", "Content-Type": "text/plain; charset=utf-8"}, - ) - with urllib.request.urlopen(req, timeout=30) as response: - if response.status not in {200, 204}: - raise RuntimeError(f"InfluxDB write failed with status={response.status}") - return len(lines) - - -def main() -> int: - if not INFLUX_ENABLED: - print("[aw-dlp-influx-exporter] disabled by AW_DLP_INFLUX_ENABLED", file=sys.stderr) - return 0 - end = utc_now() - start = end - timedelta(days=LOOKBACK_DAYS) - lines: list[str] = [] - for host in HOSTS: - lines.extend(build_lines_for_host(host, start, end)) - written = write_lines(lines) - print(f"[aw-dlp-influx-exporter] wrote {written} points to {INFLUX_BUCKET}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/aw-server/aw-dlp-influx-exporter.service b/aw-server/aw-dlp-influx-exporter.service index 3de628b..7d354c3 100644 --- a/aw-server/aw-dlp-influx-exporter.service +++ b/aw-server/aw-dlp-influx-exporter.service @@ -6,7 +6,7 @@ Wants=network-online.target aw-dlp-case-management.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-dlp-influx-exporter.py +ExecStart=/usr/local/bin/aw-dlp-influx-exporter-rust User=activitywatch Group=activitywatch StandardOutput=journal diff --git a/aw-server/aw-health-check-wrapper.sh b/aw-server/aw-health-check-wrapper.sh index 98e6321..8d368e2 100644 --- a/aw-server/aw-health-check-wrapper.sh +++ b/aw-server/aw-health-check-wrapper.sh @@ -5,4 +5,5 @@ if [[ -x /usr/local/bin/aw-health-check-rust ]]; then exec /usr/local/bin/aw-health-check-rust "$@" fi -exec /opt/activitywatch/aw-rus-ops/health-check.sh "$@" +echo "ERROR: /usr/local/bin/aw-health-check-rust is missing or not executable" >&2 +exit 127 diff --git a/aw-server/aw-rus-healthd.py b/aw-server/aw-rus-healthd.py deleted file mode 100644 index 07d0538..0000000 --- a/aw-server/aw-rus-healthd.py +++ /dev/null @@ -1,512 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import socket -import subprocess -import sys -import tempfile -import time -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -from urllib import request - - -ENV_FILE = Path("/etc/activitywatch/aw-server.env") - - -def load_env_file(path: Path) -> None: - if not path.exists(): - return - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - # systemd EnvironmentFile has already injected the variables for service runs. - return - for raw_line in lines: - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - key = key.strip() - value = value.strip().strip("'").strip('"') - os.environ.setdefault(key, value) - - -def env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def now_utc() -> datetime: - return datetime.now(UTC) - - -def parse_ts(value: str | None) -> datetime | None: - if not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) - except ValueError: - return None - - -def age_seconds(ts: datetime | None, now: datetime) -> int | None: - if ts is None: - return None - return max(0, int((now - ts).total_seconds())) - - -def http_json(url: str, timeout: int = 20, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: - last_exc: Exception | None = None - for attempt in range(max(1, attempts)): - try: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) - except Exception as exc: - last_exc = exc - if attempt + 1 >= max(1, attempts): - break - time.sleep(backoff_seconds * (2**attempt)) - assert last_exc is not None - raise last_exc - - -def run_command(cmd: list[str]) -> tuple[int, str]: - proc = subprocess.run( - cmd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - return proc.returncode, proc.stdout.strip() - - -def tcp_connect(host: str, port: int, timeout: float) -> tuple[bool, str]: - try: - with socket.create_connection((host, port), timeout=timeout): - return True, "connected" - except OSError as exc: - return False, str(exc) - - -@dataclass -class CheckResult: - name: str - status: str - summary: str - details: dict[str, Any] - - -class Report: - def __init__(self) -> None: - self.results: list[CheckResult] = [] - - def add(self, name: str, status: str, summary: str, **details: Any) -> None: - self.results.append(CheckResult(name=name, status=status, summary=summary, details=details)) - - @property - def ok(self) -> bool: - return not any(item.status == "fail" for item in self.results) - - def as_dict(self) -> dict[str, Any]: - counts = {"ok": 0, "warn": 0, "fail": 0} - for item in self.results: - counts[item.status] = counts.get(item.status, 0) + 1 - return { - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "ok": self.ok, - "counts": counts, - "results": [ - { - "name": item.name, - "status": item.status, - "summary": item.summary, - "details": item.details, - } - for item in self.results - ], - } - - def render_text(self) -> str: - icon = {"ok": "OK", "warn": "WARN", "fail": "FAIL"} - lines = ["=== AW-RUS Health ===", f"Timestamp: {now_utc().isoformat().replace('+00:00', 'Z')}", ""] - for item in self.results: - lines.append(f"[{icon.get(item.status, item.status.upper())}] {item.name}: {item.summary}") - lines.append("") - payload = self.as_dict() - lines.append( - "Counts: ok={ok} warn={warn} fail={fail}".format( - ok=payload["counts"]["ok"], - warn=payload["counts"]["warn"], - fail=payload["counts"]["fail"], - ) - ) - lines.append(f"Overall: {'OK' if payload['ok'] else 'FAIL'}") - return "\n".join(lines) - - -def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: - events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1", timeout=25) - if isinstance(events, list) and events: - events = [item for item in events if isinstance(item, dict)] - if not events: - return None - events.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return events[0] - return None - - -def bucket_metadata_ts(buckets: dict[str, Any], bucket_id: str) -> datetime | None: - bucket = buckets.get(bucket_id) if isinstance(buckets, dict) else None - if not isinstance(bucket, dict): - return None - metadata = bucket.get("metadata") - if not isinstance(metadata, dict): - return None - return parse_ts(metadata.get("end")) - - -def host_activity_from_worktime(event: dict[str, Any] | None, max_age_seconds: int) -> dict[str, Any]: - now = now_utc() - if not event: - return {"fresh": False, "active": False, "age_seconds": None, "timestamp": None} - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now) - data = event.get("data") or {} - is_fresh = age is not None and age <= max_age_seconds - is_active = bool(is_fresh and data.get("active")) - return { - "fresh": bool(is_fresh), - "active": bool(is_active), - "age_seconds": age, - "timestamp": event.get("timestamp"), - "data": data, - } - - -def bucket_health( - api_base: str, - bucket_id: str, - max_age_seconds: int, - missing_status: str, - stale_status: str, -) -> tuple[str, str, dict[str, Any]]: - try: - event = latest_bucket_event(api_base, bucket_id) - except Exception as exc: - return "fail", f"bucket query failed: {exc}", {"bucket": bucket_id} - - if not event: - return missing_status, "no events", {"bucket": bucket_id} - - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now_utc()) - details = {"bucket": bucket_id, "timestamp": event.get("timestamp"), "age_seconds": age} - if age is None: - return "warn", "timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"stale ({age}s)", details - return "ok", f"fresh ({age}s)", details - - -def bucket_timestamp_health( - api_base: str, - buckets: dict[str, Any], - bucket_id: str, - max_age_seconds: int, - missing_status: str, - stale_status: str, -) -> tuple[str, str, dict[str, Any]]: - metadata_ts = bucket_metadata_ts(buckets, bucket_id) - if metadata_ts is not None: - age = age_seconds(metadata_ts, now_utc()) - timestamp = metadata_ts.isoformat().replace("+00:00", "Z") - details = { - "bucket": bucket_id, - "timestamp": timestamp, - "age_seconds": age, - "timestamp_source": "bucket_metadata.end", - } - if age is None: - return "warn", "timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"stale ({age}s)", details - return "ok", f"fresh ({age}s)", details - - return bucket_health(api_base, bucket_id, max_age_seconds, missing_status, stale_status) - - -def guard_bucket_health( - api_base: str, - host: str, - max_age_seconds: int, - required: bool, -) -> tuple[str, str, dict[str, Any]]: - bucket_id = f"aw-rus-collector-guard_{host}" - try: - event = latest_bucket_event(api_base, bucket_id) - except Exception as exc: - status = "fail" if required else "warn" - return status, f"guard bucket query failed: {exc}", {"bucket": bucket_id} - - if not event: - status = "fail" if required else "warn" - return status, "no guard heartbeat", {"bucket": bucket_id, "required": required} - - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now_utc()) - data = event.get("data") if isinstance(event.get("data"), dict) else {} - guard_status = str(data.get("status") or "unknown").lower() - details = { - "bucket": bucket_id, - "timestamp": event.get("timestamp"), - "age_seconds": age, - "required": required, - "guard_status": guard_status, - "mode": data.get("mode"), - "live_session_count": data.get("liveSessionCount"), - "problems": data.get("problems", []), - "actions": data.get("actions", []), - } - - stale_status = "fail" if required else "warn" - if age is None: - return "warn", "guard timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"guard stale ({age}s)", details - if guard_status in ("fail", "error"): - return "fail" if required else "warn", f"guard reports {guard_status}", details - if guard_status == "warn": - return "warn", "guard reports warn", details - return "ok", f"guard fresh ({age}s)", details - - -def latest_validation_report(validation_dir: Path) -> Path | None: - candidates = sorted( - (path for path in validation_dir.glob("*-aw_validate_ansible.json") if path.is_file()), - key=lambda item: item.stat().st_mtime, - reverse=True, - ) - return candidates[0] if candidates else None - - -def write_atomic(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - handle.write(content) - tmp_name = handle.name - os.replace(tmp_name, path) - - -def chmod_if_possible(path: Path, mode: int) -> None: - try: - path.chmod(mode) - except OSError: - pass - - -def check_wrapper( - report: Report, - name: str, - cmd: list[str], - json_mode: bool = False, - failure_status: str = "fail", -) -> None: - if not Path(cmd[0]).exists(): - report.add(name, "warn", "binary missing", command=cmd) - return - rc, output = run_command(cmd) - details: dict[str, Any] = {"command": cmd, "returncode": rc} - if json_mode: - try: - details["payload"] = json.loads(output) if output else {} - except json.JSONDecodeError: - details["raw_output"] = output - report.add(name, "fail", "invalid JSON output", **details) - return - else: - details["output"] = output - report.add(name, "ok" if rc == 0 else failure_status, "passed" if rc == 0 else "failed", **details) - - -def main() -> int: - load_env_file(ENV_FILE) - - parser = argparse.ArgumentParser(description="Unified AW-RUS health orchestrator") - parser.add_argument("--aw-server", default=env("AW_SERVER_URL", "http://127.0.0.1:5600")) - parser.add_argument( - "--worktime-api", - default=env("AW_RUS_HEALTH_WORKTIME_API", env("AW_WORKTIME_REPORT_BASE", "http://127.0.0.1:5610")), - ) - parser.add_argument("--rdp-host", default=env("AW_MONITORED_WINDOWS_HOST", "192.168.100.18")) - parser.add_argument("--rdp-hostname", default=env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025")) - parser.add_argument("--state-dir", default=env("AW_RUS_HEALTH_STATE_DIR", "/var/lib/activitywatch/health")) - parser.add_argument("--validation-dir", default=env("AW_RUS_HEALTH_VALIDATION_DIR", "/var/lib/activitywatch/health/windows-validation")) - parser.add_argument("--session-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_MAX_AGE_SECONDS", "900"))) - parser.add_argument("--interactive-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_INTERACTIVE_MAX_AGE_SECONDS", "900"))) - parser.add_argument("--session-events-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS", "86400"))) - parser.add_argument("--guard-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS", "300"))) - parser.add_argument("--guard-required", action="store_true", default=env("AW_RUS_HEALTH_GUARD_REQUIRED", "0").lower() in ("1", "true", "yes")) - parser.add_argument("--validation-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_VALIDATION_MAX_AGE_SECONDS", "259200"))) - parser.add_argument("--tcp-timeout-seconds", type=float, default=float(env("AW_RUS_HEALTH_TCP_TIMEOUT_SECONDS", "3"))) - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - - report = Report() - aw_api_base = args.aw_server.rstrip("/") - if not aw_api_base.endswith("/api/0"): - aw_api_base = aw_api_base.rstrip("/") + "/api/0" - - check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"], failure_status="warn") - check_wrapper( - report, - "wrapper:dlp-health-check", - ["/usr/local/bin/dlp-health-check", "--json"], - json_mode=True, - failure_status="warn", - ) - - try: - info = http_json(f"{aw_api_base}/info") - report.add("http:aw-server", "ok", "activitywatch API responded", version=info.get("version")) - except Exception as exc: - report.add("http:aw-server", "fail", f"activitywatch API failed: {exc}", url=f"{aw_api_base}/info") - - try: - payload = http_json(args.worktime_api.rstrip("/") + "/health") - report.add("http:worktime-api", "ok", "worktime API responded", payload=payload if isinstance(payload, dict) else {}) - except Exception as exc: - report.add("http:worktime-api", "fail", f"worktime API failed: {exc}", url=args.worktime_api) - - for port, label in ((5985, "winrm"), (3389, "rdp")): - ok, message = tcp_connect(args.rdp_host, port, args.tcp_timeout_seconds) - report.add(f"tcp:{label}", "ok" if ok else "fail", message if ok else f"unreachable: {message}", host=args.rdp_host, port=port) - - try: - buckets = http_json(f"{aw_api_base}/buckets") - if not isinstance(buckets, dict): - raise RuntimeError("bucket index is not a dict") - report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets)) - except Exception as exc: - report.add("aw:buckets-index", "fail", f"failed to load bucket index: {exc}") - buckets = {} - - host = args.rdp_hostname - guard_status, guard_summary, guard_details = guard_bucket_health( - aw_api_base, - host, - args.guard_max_age_seconds, - args.guard_required, - ) - report.add("bucket:collector-guard", guard_status, guard_summary, **guard_details) - - worktime_bucket = f"aw-worktime-sessions_{host}" - worktime_event = None - if buckets: - try: - worktime_event = latest_bucket_event(aw_api_base, worktime_bucket) - except Exception: - worktime_event = None - activity = host_activity_from_worktime(worktime_event, args.session_max_age_seconds) - if worktime_event: - status, summary, details = bucket_health( - aw_api_base, - worktime_bucket, - args.session_max_age_seconds, - missing_status="fail", - stale_status="fail", - ) - details["host_activity"] = activity - report.add("bucket:worktime-sessions", status, summary, **details) - else: - report.add("bucket:worktime-sessions", "fail", "no events", bucket=worktime_bucket, host_activity=activity) - - interactive_required = bool(activity["active"]) - for bucket_name, label in ( - ("aw-watcher-afk", "bucket:afk"), - ("aw-watcher-window", "bucket:window"), - ("aw-dlp-endpoint-signals", "bucket:endpoint-signals"), - ): - status, summary, details = bucket_health( - aw_api_base, - f"{bucket_name}_{host}", - args.interactive_max_age_seconds, - missing_status="fail" if interactive_required else "warn", - stale_status="fail" if interactive_required else "warn", - ) - details["interactive_required"] = interactive_required - details["host_activity"] = activity - if not interactive_required and status != "ok": - details["inactive_summary"] = summary - status = "ok" - summary = "inactive: no active interactive users" - report.add(label, status, summary, **details) - - session_status, session_summary, session_details = bucket_timestamp_health( - aw_api_base, - buckets, - f"aw-session-events_{host}", - args.session_events_max_age_seconds, - missing_status="fail", - stale_status="warn", - ) - if session_status == "warn" and session_details.get("age_seconds") is not None: - session_status = "ok" - session_summary = f"event-driven ({session_details['age_seconds']}s since last logon marker)" - report.add("bucket:session-events", session_status, session_summary, **session_details) - - validation_dir = Path(args.validation_dir) - validation_report = latest_validation_report(validation_dir) - if validation_report is None: - report.add("validation:windows", "warn", "no validation report snapshot", directory=str(validation_dir)) - else: - try: - payload = json.loads(validation_report.read_text(encoding="utf-8-sig")) - age = age_seconds(datetime.fromtimestamp(validation_report.stat().st_mtime, tz=UTC), now_utc()) - if age is not None and age > args.validation_max_age_seconds: - report.add( - "validation:windows", - "warn", - f"validation snapshot is stale ({age}s)", - path=str(validation_report), - overall_ok=payload.get("overallOk"), - failed_sections=payload.get("summary", {}).get("failedSections", []), - ) - elif payload.get("overallOk") is True: - report.add("validation:windows", "ok", "validation snapshot OK", path=str(validation_report), age_seconds=age) - else: - report.add( - "validation:windows", - "fail", - "validation snapshot reports failure", - path=str(validation_report), - age_seconds=age, - failed_sections=payload.get("summary", {}).get("failedSections", []), - ) - except Exception as exc: - report.add("validation:windows", "fail", f"invalid validation snapshot: {exc}", path=str(validation_report)) - - payload = report.as_dict() - state_dir = Path(args.state_dir) - health_json_path = state_dir / "aw-rus-health.json" - health_txt_path = state_dir / "aw-rus-health.txt" - write_atomic(health_json_path, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") - write_atomic(health_txt_path, report.render_text() + "\n") - chmod_if_possible(health_json_path, 0o644) - chmod_if_possible(health_txt_path, 0o644) - - if args.json: - print(json.dumps(payload, ensure_ascii=False, indent=2)) - else: - print(report.render_text()) - return 0 if payload["ok"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/aw-server/aw-rus-healthd.service b/aw-server/aw-rus-healthd.service index 015a906..b5ba629 100644 --- a/aw-server/aw-rus-healthd.service +++ b/aw-server/aw-rus-healthd.service @@ -6,7 +6,7 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-rus-healthd.py +ExecStart=/usr/local/bin/aw-rus-healthd-rust User=root Group=root StandardOutput=journal diff --git a/aw-server/aw-slo-monitor.py b/aw-server/aw-slo-monitor.py deleted file mode 100644 index 854d6ff..0000000 --- a/aw-server/aw-slo-monitor.py +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import tempfile -import time -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from urllib import request - - -ENV_FILE = Path("/etc/activitywatch/aw-server.env") -DEFAULT_STATE_DIR = Path("/var/lib/activitywatch/slo") -DEFAULT_HEALTHD_CMD = "/usr/local/bin/aw-rus-healthd.py --json" -DEFAULT_HEALTHD_STATE_FILE = Path("/var/lib/activitywatch/health/aw-rus-health.json") - - -def load_env_file(path: Path) -> None: - if not path.exists(): - return - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - # systemd EnvironmentFile has already loaded the variables for the service. - return - for raw_line in lines: - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - os.environ.setdefault(key.strip(), value.strip().strip("'").strip('"')) - - -def env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def now_utc() -> datetime: - return datetime.now(UTC) - - -def parse_ts(value: str) -> datetime: - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) - - -def iso(dt: datetime) -> str: - return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -def write_atomic(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - handle.write(content) - tmp_name = handle.name - os.replace(tmp_name, path) - - -def chmod_if_possible(path: Path, mode: int) -> None: - try: - path.chmod(mode) - except OSError: - pass - - -def chown_like_parent_if_possible(path: Path) -> None: - try: - parent_stat = path.parent.stat() - os.chown(path, parent_stat.st_uid, parent_stat.st_gid) - except OSError: - pass - - -def fetch_url(url: str, timeout_seconds: float, *, accept: str | None = None, attempts: int = 1) -> dict[str, Any]: - started = now_utc() - last_error = "" - for attempt in range(1, max(1, attempts) + 1): - try: - req = request.Request(url, headers={"Accept": accept}) if accept else url - with request.urlopen(req, timeout=timeout_seconds) as resp: - body = resp.read() - status = int(resp.status) - content_type = resp.headers.get("Content-Type", "") - ok = 200 <= status < 300 - return { - "ok": ok, - "status": status, - "body_bytes": len(body), - "content_type": content_type, - "body": body, - "attempts": attempt, - "latency_ms": int((now_utc() - started).total_seconds() * 1000), - "url": url, - } - except Exception as exc: - last_error = str(exc) - if attempt < max(1, attempts): - time.sleep(1) - return { - "ok": False, - "error": last_error, - "body_bytes": 0, - "attempts": max(1, attempts), - "latency_ms": int((now_utc() - started).total_seconds() * 1000), - "url": url, - } - - -def public_probe_result(result: dict[str, Any]) -> dict[str, Any]: - return {key: value for key, value in result.items() if key != "body"} - - -def http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: - return public_probe_result(fetch_url(url, timeout_seconds, attempts=2)) - - -def html_probe( - url: str, - timeout_seconds: float, - *, - min_bytes: int, - required_markers: tuple[str, ...], -) -> dict[str, Any]: - result = fetch_url(url, timeout_seconds, accept="text/html", attempts=2) - if not result.get("ok"): - return public_probe_result(result) - - body = result.get("body", b"") - text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) - missing = [marker for marker in required_markers if marker not in text] - content_type = str(result.get("content_type") or "") - if len(body) < min_bytes: - result["ok"] = False - result["error"] = f"body too small: {len(body)} < {min_bytes}" - elif "text/html" not in content_type.lower(): - result["ok"] = False - result["error"] = f"unexpected content-type: {content_type or 'unknown'}" - elif missing: - result["ok"] = False - result["error"] = "missing markers: " + ", ".join(missing) - result["missing_markers"] = missing - return public_probe_result(result) - - -def json_probe( - url: str, - timeout_seconds: float, - *, - expected_values: dict[str, Any], - required_keys: tuple[str, ...], -) -> dict[str, Any]: - result = fetch_url(url, timeout_seconds, accept="application/json", attempts=2) - if not result.get("ok"): - return public_probe_result(result) - - body = result.get("body", b"") - try: - payload = json.loads(body.decode("utf-8") if isinstance(body, bytes) else str(body)) - except Exception as exc: - result["ok"] = False - result["error"] = f"invalid json: {exc}" - return public_probe_result(result) - - if not isinstance(payload, dict): - result["ok"] = False - result["error"] = "json root is not object" - return public_probe_result(result) - - missing_keys = [key for key in required_keys if key not in payload] - mismatched = { - key: {"expected": expected, "actual": payload.get(key)} - for key, expected in expected_values.items() - if payload.get(key) != expected - } - if missing_keys: - result["ok"] = False - result["error"] = "missing json keys: " + ", ".join(missing_keys) - result["missing_keys"] = missing_keys - elif mismatched: - result["ok"] = False - result["error"] = "unexpected json values" - result["mismatched_values"] = mismatched - else: - result["json_keys"] = sorted(payload.keys()) - return public_probe_result(result) - - -def run_healthd(command: str, timeout_seconds: int) -> dict[str, Any]: - try: - proc = subprocess.run( - command, - shell=True, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as exc: - return { - "ok": False, - "returncode": None, - "error": f"timeout after {timeout_seconds}s", - "output_tail": (exc.stdout or "")[-1000:] if isinstance(exc.stdout, str) else "", - } - - payload: dict[str, Any] = {} - try: - payload = json.loads(proc.stdout or "{}") - except json.JSONDecodeError: - pass - return { - "ok": proc.returncode == 0 and bool(payload.get("ok")), - "returncode": proc.returncode, - "counts": payload.get("counts", {}), - "payload": payload if isinstance(payload, dict) else {}, - "output_tail": (proc.stdout or "")[-1000:], - } - - -def read_healthd_state(path: Path, max_age_seconds: int) -> dict[str, Any]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - generated = parse_ts(str(payload["generated_at_utc"])) - age = max(0, int((now_utc() - generated).total_seconds())) - except Exception as exc: - return {"ok": False, "error": str(exc), "path": str(path)} - return { - "ok": bool(payload.get("ok")) and age <= max_age_seconds, - "counts": payload.get("counts", {}), - "age_seconds": age, - "path": str(path), - "payload": payload if isinstance(payload, dict) else {}, - } - - -def load_samples(path: Path, cutoff: datetime) -> list[dict[str, Any]]: - samples: list[dict[str, Any]] = [] - if not path.exists(): - return samples - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - return samples - for line in lines: - if not line.strip(): - continue - try: - item = json.loads(line) - ts = parse_ts(str(item["ts"])) - except Exception: - continue - if ts >= cutoff: - samples.append(item) - return samples - - -def append_and_trim_sample(path: Path, sample: dict[str, Any], retention_seconds: int) -> list[dict[str, Any]]: - cutoff = parse_ts(sample["ts"]) - timedelta(seconds=retention_seconds) - samples = load_samples(path, cutoff) - samples.append(sample) - content = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n" for item in samples) - write_atomic(path, content) - chmod_if_possible(path, 0o644) - chown_like_parent_if_possible(path) - return samples - - -def summarize_window( - samples: list[dict[str, Any]], - *, - now: datetime, - window_seconds: int, - sample_interval_seconds: int, - target_percent: float, -) -> dict[str, Any]: - cutoff = now - timedelta(seconds=window_seconds) - window = [] - for item in samples: - try: - if parse_ts(str(item["ts"])) >= cutoff: - window.append(item) - except Exception: - continue - - total = len(window) - good = sum(1 for item in window if item.get("ok") is True) - bad = total - good - availability = round((good / total) * 100, 5) if total else None - observed_bad_seconds = bad * sample_interval_seconds - budget_seconds = int(window_seconds * ((100.0 - target_percent) / 100.0)) - budget_remaining_seconds = budget_seconds - observed_bad_seconds - status = "unknown" - if total: - status = "ok" if budget_remaining_seconds >= 0 else "burning" - return { - "window_seconds": window_seconds, - "samples": total, - "good_samples": good, - "bad_samples": bad, - "availability_percent": availability, - "target_percent": target_percent, - "observed_bad_seconds": observed_bad_seconds, - "budget_seconds": budget_seconds, - "budget_remaining_seconds": budget_remaining_seconds, - "status": status, - } - - -def render_summary_text(summary: dict[str, Any]) -> str: - lines = [ - "=== AW-RUS SLO ===", - f"Timestamp: {summary['generated_at_utc']}", - f"Target: {summary['target_percent']}%", - "", - ] - for name, data in summary["windows"].items(): - availability = data["availability_percent"] - availability_text = "n/a" if availability is None else f"{availability:.5f}%" - remaining = data["budget_remaining_seconds"] - lines.append( - f"{name}: {data['status']} availability={availability_text} " - f"samples={data['samples']} bad={data['bad_samples']} " - f"bad_seconds={data['observed_bad_seconds']} budget_remaining_seconds={remaining}" - ) - lines.append("") - lines.append(f"Current sample: {'OK' if summary['current_sample']['ok'] else 'FAIL'}") - for name, probe in summary["current_sample"].get("probes", {}).items(): - marker = "OK" if probe.get("ok") else "FAIL" - detail = probe.get("status", probe.get("error", "")) - lines.append(f"- {name}: {marker} {detail}") - return "\n".join(lines) - - -def build_sample(args: argparse.Namespace) -> dict[str, Any]: - ts = iso(now_utc()) - if args.healthd_mode == "run": - health = run_healthd(args.healthd_cmd, args.health_timeout_seconds) - else: - health = read_healthd_state(Path(args.healthd_state_file), args.healthd_state_max_age_seconds) - probes = { - "aw_webui_index": html_probe( - args.aw_webui_url, - args.http_timeout_seconds, - min_bytes=1000, - required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), - ), - "worktime_today_html": html_probe( - args.worktime_today_html_url, - args.http_timeout_seconds, - min_bytes=5000, - required_markers=("AW-rus", ""), - ), - "worktime_management_html": html_probe( - args.worktime_management_html_url, - args.http_timeout_seconds, - min_bytes=5000, - required_markers=("AW-rus", ""), - ), - "worktime_today_csv": http_probe(args.worktime_today_csv_url, args.http_timeout_seconds), - "worktime_management_json": json_probe( - args.worktime_management_json_url, - args.http_timeout_seconds, - expected_values={"host": args.host}, - required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), - ), - } - ok = bool(health.get("ok")) and all(probe.get("ok") for probe in probes.values()) - return { - "ts": ts, - "ok": ok, - "healthd_ok": bool(health.get("ok")), - "healthd_counts": health.get("counts", {}), - "probes": probes, - } - - -def main() -> int: - load_env_file(ENV_FILE) - parser = argparse.ArgumentParser(description="AW-RUS rolling SLO sampler") - parser.add_argument("--state-dir", default=env("AW_RUS_SLO_STATE_DIR", str(DEFAULT_STATE_DIR))) - parser.add_argument("--healthd-cmd", default=env("AW_RUS_SLO_HEALTHD_CMD", DEFAULT_HEALTHD_CMD)) - parser.add_argument("--healthd-mode", choices=["state", "run"], default=env("AW_RUS_SLO_HEALTHD_MODE", "state")) - parser.add_argument("--healthd-state-file", default=env("AW_RUS_SLO_HEALTHD_STATE_FILE", str(DEFAULT_HEALTHD_STATE_FILE))) - parser.add_argument("--healthd-state-max-age-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", "180"))) - parser.add_argument("--aw-base", default=env("AW_RUS_SLO_AW_BASE", env("AW_SERVER_URL", "http://127.0.0.1:5600"))) - parser.add_argument("--worktime-base", default=env("AW_RUS_SLO_WORKTIME_BASE", env("AW_RUS_HEALTH_WORKTIME_API", "http://127.0.0.1:5610"))) - parser.add_argument("--host", default=env("AW_RUS_SLO_HOST", env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025"))) - parser.add_argument("--target-percent", type=float, default=float(env("AW_RUS_SLO_TARGET_PERCENT", "99.97"))) - parser.add_argument("--sample-interval-seconds", type=int, default=int(env("AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", "15"))) - parser.add_argument("--retention-days", type=int, default=int(env("AW_RUS_SLO_RETENTION_DAYS", "35"))) - parser.add_argument("--http-timeout-seconds", type=float, default=float(env("AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", "15"))) - parser.add_argument("--health-timeout-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", "90"))) - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - - aw_base = args.aw_base.rstrip("/") - worktime_base = args.worktime_base.rstrip("/") - args.aw_webui_url = f"{aw_base}/" - args.worktime_today_html_url = f"{worktime_base}/reports/worktime/today?format=html&day=today&host={args.host}&allow_stale=1" - args.worktime_management_html_url = f"{worktime_base}/reports/worktime/management?format=html&day=today&host={args.host}&allow_stale=1" - args.worktime_today_csv_url = f"{worktime_base}/reports/worktime/today?format=csv&day=today&host={args.host}&allow_stale=1" - args.worktime_management_json_url = f"{worktime_base}/reports/worktime/management?format=json&day=today&host={args.host}&allow_stale=1" - - state_dir = Path(args.state_dir) - sample_path = state_dir / "aw-slo-samples.jsonl" - sample = build_sample(args) - samples = append_and_trim_sample( - sample_path, - sample, - retention_seconds=max(1, args.retention_days) * 86400, - ) - - generated_at = now_utc() - summary = { - "generated_at_utc": iso(generated_at), - "target_percent": args.target_percent, - "sample_interval_seconds": args.sample_interval_seconds, - "current_sample": sample, - "windows": { - "24h": summarize_window( - samples, - now=generated_at, - window_seconds=86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - "7d": summarize_window( - samples, - now=generated_at, - window_seconds=7 * 86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - "30d": summarize_window( - samples, - now=generated_at, - window_seconds=30 * 86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - }, - } - summary_json_path = state_dir / "aw-slo-summary.json" - summary_txt_path = state_dir / "aw-slo-summary.txt" - write_atomic(summary_json_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") - text = render_summary_text(summary) - write_atomic(summary_txt_path, text + "\n") - chmod_if_possible(summary_json_path, 0o644) - chmod_if_possible(summary_txt_path, 0o644) - chown_like_parent_if_possible(summary_json_path) - chown_like_parent_if_possible(summary_txt_path) - if args.json: - print(json.dumps(summary, ensure_ascii=False, indent=2)) - else: - print(text) - return 0 if sample["ok"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/aw-server/aw-slo-monitor.service b/aw-server/aw-slo-monitor.service index a1bd724..0fbba5e 100644 --- a/aw-server/aw-slo-monitor.service +++ b/aw-server/aw-slo-monitor.service @@ -6,7 +6,7 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-slo-monitor.py +ExecStart=/usr/local/bin/aw-slo-monitor-rust SuccessExitStatus=1 User=activitywatch Group=activitywatch diff --git a/aw-server/aw-worktime-api.py b/aw-server/aw-worktime-api.py deleted file mode 100644 index caba965..0000000 --- a/aw-server/aw-worktime-api.py +++ /dev/null @@ -1,3039 +0,0 @@ -#!/usr/bin/env python3 -import base64 -import csv -import hashlib -import html -import io -import importlib.util -import json -import os -import sys -import tempfile -import threading -import time -import urllib.request -from datetime import datetime, timezone, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from urllib.parse import parse_qs, urlencode, urlparse -from zoneinfo import ZoneInfo - - -def build_aw_api_base(raw_url): - url = (raw_url or "http://127.0.0.1:5600").strip().rstrip("/") - if url.endswith("/api/0"): - return url - return url + "/api/0" - - -AW_SERVER_URL = os.environ.get("AW_SERVER_URL", "http://127.0.0.1:5600") -AW = build_aw_api_base(AW_SERVER_URL) -REPORT_TZ = ZoneInfo(os.environ.get("AW_WORKTIME_TZ", "Europe/Moscow")) -IOC_DIR = os.environ.get("AW_DLP_IOC_DIR", "/opt/activitywatch/dlp-ioc/output") -DEFAULT_HOST = os.environ.get("AW_WORKTIME_HOST", "SHARKON2025").strip() or "SHARKON2025" -DEFAULT_SAMPLE_SECONDS = max(1.0, float(os.environ.get("AW_WORKTIME_DEFAULT_SAMPLE_SECONDS", "30"))) -MAX_SAMPLE_SECONDS = max(DEFAULT_SAMPLE_SECONDS, float(os.environ.get("AW_WORKTIME_MAX_SAMPLE_SECONDS", "300"))) -LISTEN_HOST = os.environ.get("AW_WORKTIME_LISTEN_HOST", "0.0.0.0") -LISTEN_PORT = int(os.environ.get("AW_WORKTIME_PORT", "5610")) -WORKDAY_START_HOUR = int(os.environ.get("AW_WORKTIME_MANAGER_START_HOUR", "9")) -WORKDAY_END_HOUR = int(os.environ.get("AW_WORKTIME_MANAGER_END_HOUR", "18")) -MANAGER_TARGET_COVERAGE_PCT = max(1, min(100, int(os.environ.get("AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT", "75")))) -MANAGER_LOW_COVERAGE_PCT = max(1, min(100, int(os.environ.get("AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT", "35")))) -MANAGER_LATE_START_GRACE_MINUTES = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_LATE_START_GRACE_MINUTES", "60"))) -MANAGER_EARLY_FINISH_GRACE_MINUTES = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_EARLY_FINISH_GRACE_MINUTES", "90"))) -MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS = max(60, int(os.environ.get("AW_WORKTIME_MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS", "900"))) -MANAGER_WEB_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_WEB_SOURCE_MAX_AGE_SECONDS", "259200"))) -MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS", "604800"))) -MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS", "172800"))) -MANAGER_TREND_DAYS = max(3, min(31, int(os.environ.get("AW_WORKTIME_MANAGER_TREND_DAYS", "7")))) -MANAGER_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_CACHE_TTL_SECONDS", "300"))) -MANAGER_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_MANAGER_CACHE_DIR", "/var/lib/activitywatch/worktime-cache")) -MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", "/etc/activitywatch/worktime-manager-aliases.json")) -MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()} -EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30"))) -WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000"))) -AW_HTTP_TIMEOUT_SECONDS = max(0.5, float(os.environ.get("AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS", "5"))) -REPORT_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_REPORT_CACHE_TTL_SECONDS", "60"))) -REPORT_STALE_TTL_SECONDS = max(REPORT_CACHE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_STALE_TTL_SECONDS", "900"))) -REPORT_DISK_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_REPORT_DISK_CACHE_DIR", "/var/lib/activitywatch/worktime-report-cache")) -REPORT_DISK_STALE_TTL_SECONDS = max(REPORT_STALE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS", "86400"))) -REPORT_BUSY_WAIT_SECONDS = max(0.0, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_SECONDS", "8"))) -REPORT_BUSY_WAIT_INTERVAL_SECONDS = max(0.05, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_INTERVAL_SECONDS", "0.2"))) -REPORT_MAX_CONCURRENT = max(1, int(os.environ.get("AW_WORKTIME_REPORT_MAX_CONCURRENT", "1"))) -TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180"))) -TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600"))) -MODULE_PATH = Path(__file__).resolve() -_ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}} -_EVENTS_CACHE_LOCK = threading.Lock() -_EVENTS_CACHE = {} -_REPORT_CACHE_LOCK = threading.Lock() -_REPORT_RESPONSE_CACHE = {} -_REPORT_REFRESHING = set() -_REPORT_BUILD_SEMAPHORE = threading.BoundedSemaphore(REPORT_MAX_CONCURRENT) -_MANAGEMENT_BUILD_LOCKS_LOCK = threading.Lock() -_MANAGEMENT_BUILD_LOCKS = {} - - -def get(u): - with urllib.request.urlopen(u, timeout=AW_HTTP_TIMEOUT_SECONDS) as r: - return json.loads(r.read().decode()) - - -def log_warning(message): - print(f"[aw-worktime-api] {message}", file=sys.stderr, flush=True) - - -def pts(s): - return datetime.fromisoformat(s.replace("Z", "+00:00")).astimezone(timezone.utc) - - -def to_iso_utc(dt): - return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") - - -def parse_iso_utc(value): - if not value: - return None - return pts(value) - - -def hhmm(total_seconds): - total_seconds = max(0, int(total_seconds)) - return "%02d:%02d" % (total_seconds // 3600, (total_seconds % 3600) // 60) - - -def human_duration_ru(total_seconds): - total_seconds = max(0, int(total_seconds)) - hours = total_seconds // 3600 - minutes = (total_seconds % 3600) // 60 - if hours and minutes: - return f"{hours} ч {minutes} мин" - if hours: - return f"{hours} ч" - if minutes: - return f"{minutes} мин" - return f"{total_seconds} сек" - - -def now_utc(): - return datetime.now(timezone.utc) - - -def worktime_health_payload(): - return { - "ok": True, - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "default_host": DEFAULT_HOST, - "aw_api_base": AW, - } - - -def age_seconds(ts, now=None): - if ts is None: - return None - ref = now or now_utc() - return max(0, int((ref - ts).total_seconds())) - - -def write_atomic_json(path, payload): - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - json.dump(payload, handle, ensure_ascii=False, indent=2) - handle.write("\n") - tmp_name = handle.name - os.replace(tmp_name, path) - - -def safe_slug(value): - text = str(value or "").strip().lower() - slug = [] - for char in text: - if char.isalnum(): - slug.append(char) - else: - slug.append("-") - normalized = "".join(slug).strip("-") - while "--" in normalized: - normalized = normalized.replace("--", "-") - return normalized or "user" - - -def _normalize_identity_key(value): - return str(value or "").strip().lower() - - -def _default_display_name(user, user_id): - base = str(user_id or user or "").strip() - if "\\" in base: - base = base.split("\\", 1)[1] - if not base: - base = str(user or "").strip() - if base and base.isascii() and base.lower() == base and any(ch.isalpha() for ch in base): - return base.upper() - return base or str(user or "").strip() - - -def _load_manager_directory(): - try: - stat = MANAGER_ALIASES_JSON.stat() - except FileNotFoundError: - _ALIASES_CACHE["mtime"] = None - _ALIASES_CACHE["users"] = {} - _ALIASES_CACHE["owners"] = {} - _ALIASES_CACHE["raw"] = {} - return {"users": {}, "owners": {}, "raw": {}} - mtime = stat.st_mtime - if _ALIASES_CACHE["mtime"] == mtime: - return { - "users": _ALIASES_CACHE["users"], - "owners": _ALIASES_CACHE["owners"], - "raw": _ALIASES_CACHE["raw"], - } - try: - raw = json.loads(MANAGER_ALIASES_JSON.read_text(encoding="utf-8")) - except Exception as exc: - log_warning(f"failed to load manager aliases from {MANAGER_ALIASES_JSON}: {exc}") - raw = {} - users_payload = {} - owners_payload = {} - if isinstance(raw, dict): - users = raw.get("users", raw) - if isinstance(users, dict): - for key, value in users.items(): - norm_key = _normalize_identity_key(key) - if not norm_key: - continue - if isinstance(value, str): - users_payload[norm_key] = {"display_name": value} - elif isinstance(value, dict): - users_payload[norm_key] = dict(value) - owners = raw.get("owners", {}) - if isinstance(owners, dict): - for key, value in owners.items(): - norm_key = _normalize_identity_key(key) - if not norm_key: - continue - if isinstance(value, str): - owners_payload[norm_key] = {"display_name": value} - elif isinstance(value, dict): - owners_payload[norm_key] = dict(value) - _ALIASES_CACHE["mtime"] = mtime - _ALIASES_CACHE["users"] = users_payload - _ALIASES_CACHE["owners"] = owners_payload - _ALIASES_CACHE["raw"] = raw if isinstance(raw, dict) else {} - return { - "users": users_payload, - "owners": owners_payload, - "raw": _ALIASES_CACHE["raw"], - } - - -def load_manager_aliases(): - return _load_manager_directory()["users"] - - -def load_manager_owners(): - return _load_manager_directory()["owners"] - - -def resolve_user_alias(user, user_id, host): - aliases = load_manager_aliases() - candidates = [ - _normalize_identity_key(user_id), - _normalize_identity_key(f"{resolve_host(host)}\\{user}"), - _normalize_identity_key(user), - ] - alias = {} - for candidate in candidates: - if candidate and candidate in aliases: - alias = dict(aliases[candidate]) - break - display_name = str(alias.get("display_name") or alias.get("name") or _default_display_name(user, user_id)).strip() - manager_owner = str(alias.get("manager") or alias.get("owner") or display_name).strip() or display_name - department = str(alias.get("department") or "").strip() - role = str(alias.get("role") or "").strip() - notes = str(alias.get("notes") or "").strip() - canonical_user_id = str(alias.get("canonical_user_id") or user_id or "").strip() - exclude = bool(alias.get("exclude")) or _normalize_identity_key(user) in MANAGER_EXCLUDE_USERS or _normalize_identity_key(display_name) in MANAGER_EXCLUDE_USERS - return { - "display_name": display_name, - "manager_owner": manager_owner, - "department": department, - "role": role, - "notes": notes, - "canonical_user_id": canonical_user_id, - "exclude": exclude, - } - - -def resolve_owner_profile(owner_name): - owner_name = normalize_management_filter(owner_name) - if not owner_name: - owner_name = "unassigned" - owners = load_manager_owners() - profile = dict(owners.get(_normalize_identity_key(owner_name), {})) - display_name = str(profile.get("display_name") or profile.get("name") or owner_name).strip() or owner_name - title = str(profile.get("title") or profile.get("role") or "").strip() - department = str(profile.get("department") or "").strip() - escalation_to = str(profile.get("escalation_to") or profile.get("escalate_to") or "").strip() - contact = str(profile.get("contact") or profile.get("telegram") or profile.get("email") or "").strip() - notes = str(profile.get("notes") or "").strip() - return { - "owner_name": owner_name, - "display_name": display_name, - "title": title, - "department": department, - "escalation_to": escalation_to, - "contact": contact, - "notes": notes, - } - - -def clamp_seconds(value, fallback=DEFAULT_SAMPLE_SECONDS): - try: - seconds = float(value) - except Exception: - seconds = float(fallback) - if seconds <= 0: - seconds = float(fallback) - return min(seconds, MAX_SAMPLE_SECONDS) - - -def resolve_host(request_host=None): - host = (request_host or DEFAULT_HOST).strip() - if not host: - host = DEFAULT_HOST - return host - - -def get_sessions_bucket_id(host): - return f"aw-worktime-sessions_{resolve_host(host)}" - - -def get_management_build_lock(host, report_date): - key = (resolve_host(host), report_date.isoformat()) - with _MANAGEMENT_BUILD_LOCKS_LOCK: - lock = _MANAGEMENT_BUILD_LOCKS.get(key) - if lock is None: - lock = threading.Lock() - _MANAGEMENT_BUILD_LOCKS[key] = lock - return lock - - -def resolve_report_date(day=None, date_text=None): - now_local = datetime.now(REPORT_TZ) - if date_text: - return datetime.strptime(date_text, "%Y-%m-%d").date() - if day == "yesterday": - return (now_local - timedelta(days=1)).date() - return now_local.date() - - -def get_report_bounds(report_date): - start_local = datetime(report_date.year, report_date.month, report_date.day, tzinfo=REPORT_TZ) - end_local = start_local + timedelta(days=1) - timedelta(seconds=1) - start = start_local.astimezone(timezone.utc) - end = end_local.astimezone(timezone.utc) - end_exclusive = end + timedelta(seconds=1) - return { - "start_local": start_local, - "end_local": end_local, - "start": start, - "end": end, - "end_exclusive": end_exclusive, - } - - -def get_workday_bounds(report_date): - start_local = datetime(report_date.year, report_date.month, report_date.day, WORKDAY_START_HOUR, 0, 0, tzinfo=REPORT_TZ) - end_local = datetime(report_date.year, report_date.month, report_date.day, WORKDAY_END_HOUR, 0, 0, tzinfo=REPORT_TZ) - if end_local <= start_local: - end_local = start_local + timedelta(hours=8) - return { - "start_local": start_local, - "end_local": end_local, - "duration_seconds": int((end_local - start_local).total_seconds()), - } - - -def _is_machine_user(user: str): - u = (user or "").strip().lower() - return u.endswith("$") or u in {"system", "localservice", "networkservice"} - - -def _is_active_sample(data: dict): - state = str(data.get("state") or "").strip().lower() - if isinstance(data.get("active"), bool) and data.get("active"): - return True - if ("актив" in state) or (state == "active"): - return True - if state == "unknown": - try: - sid = int(data.get("sessionId")) - except Exception: - sid = -1 - user = str(data.get("username") or "").strip() - session_name = str(data.get("sessionName") or "").strip().lower() - if sid > 0 and user and (not _is_machine_user(user)) and (session_name.startswith("rdp-") or session_name == "console"): - return True - return False - - -def _normalize_user_id(data, host, username): - user_id = str(data.get("userId") or "").strip() - if user_id: - left, sep, right = user_id.partition("\\") - if sep and right: - return f"{resolve_host(host)}\\{right}" - return user_id - return f"{resolve_host(host)}\\{username}" - - -def _event_sample_seconds(event, next_same_session_ts=None): - data = event.get("data") or {} - for key in ("sampleSeconds", "pollSeconds"): - value = data.get(key) - try: - if float(value) > 0: - return clamp_seconds(value) - except Exception: - pass - try: - duration = float(event.get("duration") or 0.0) - except Exception: - duration = 0.0 - if duration > 0: - return clamp_seconds(duration) - if next_same_session_ts is not None: - delta = (next_same_session_ts - event["_ts"]).total_seconds() - if delta > 0: - return clamp_seconds(delta) - return clamp_seconds(DEFAULT_SAMPLE_SECONDS) - - -def _merge_intervals(intervals): - if not intervals: - return [] - ordered = sorted(intervals, key=lambda item: item[0]) - merged = [ordered[0]] - for start, end in ordered[1:]: - last_start, last_end = merged[-1] - if start <= last_end: - if end > last_end: - merged[-1] = (last_start, end) - continue - merged.append((start, end)) - return merged - - -def _overlap_interval(left, right): - start = max(left[0], right[0]) - end = min(left[1], right[1]) - if end <= start: - return None - return start, end - - -def _interval_contains(interval, ts): - return interval[0] <= ts < interval[1] - - -def _event_timestamp(event): - try: - return pts(event.get("timestamp")) - except Exception: - return None - - -def _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS): - try: - duration = float(event.get("duration") or 0.0) - except Exception: - duration = 0.0 - if duration <= 0: - duration = default_seconds - return max(1.0, min(float(TRUE_ACTIVE_MAX_EVENT_SECONDS), duration)) - - -def _normalize_app_name(app, title=""): - app_raw = str(app or "").strip() - app_l = app_raw.lower() - title_raw = str(title or "").strip() - if app_l.startswith(("1cv8", "1cestart")): - return "1С" - if app_l in {"chrome.exe", "google chrome"} or "google chrome" in title_raw.lower(): - return "Chrome" - if app_l in {"msedge.exe", "microsoft edge"} or "microsoft edge" in title_raw.lower(): - return "Edge" - if app_l in {"browser.exe", "browser"} or "яндекс" in title_raw.lower(): - return "Яндекс Браузер" - if app_l in {"excel.exe"}: - return "Excel" - if app_l in {"winword.exe"}: - return "Word" - if app_l in {"powerpnt.exe"}: - return "PowerPoint" - if app_l in {"outlook.exe"}: - return "Outlook" - if app_l in {"explorer.exe"}: - return "Проводник" - if app_l in {"totalcmd.exe", "totalcmd64.exe"}: - return "Total Commander" - if app_l in {"cmd.exe"}: - return "Command Prompt" - if app_l in {"powershell.exe", "pwsh.exe"}: - return "PowerShell" - if app_l in {"acrord32.exe", "acrobat.exe"}: - return "Adobe Acrobat Reader" - if app_l in {"windowsterminal.exe", "windowsterminal"}: - return "Windows Terminal" - if app_raw: - return app_raw[:-4] if app_l.endswith(".exe") else app_raw - if title_raw: - return title_raw - return "Неизвестное приложение" - - -def _event_context(event): - data = event.get("data") or {} - for key in ("title", "url", "path", "filePath", "targetPath", "windowTitle", "foregroundTitle", "signalType"): - value = str(data.get(key) or "").strip() - if value: - return value - return "активность" - - -def _is_not_afk_event(event): - data = event.get("data") or {} - status = str(data.get("status") or data.get("state") or "").strip().lower() - return status in {"not-afk", "not_afk", "active", "активно"} - - -def _is_real_evidence_event(event): - data = event.get("data") or {} - signal_type = str(data.get("signalType") or data.get("type") or "").strip().lower() - if signal_type in {"collector_health", "self_test", "heartbeat", "health"}: - return False - if data.get("url") or data.get("title") or data.get("path") or data.get("filePath") or data.get("targetPath"): - return True - if signal_type: - return True - return False - - -def _events_for_bounds(events, start, end): - result = [] - for event in events: - ts = _event_timestamp(event) - if ts is None or ts < start or ts > end: - continue - result.append((ts, event)) - result.sort(key=lambda item: item[0]) - return result - - -def _build_window_intervals(window_events, start, end): - intervals = [] - previous_key = None - for ts, event in _events_for_bounds(window_events, start, end): - data = event.get("data") or {} - app = str(data.get("app") or data.get("process") or data.get("processName") or "").strip() - title = str(data.get("title") or data.get("windowTitle") or "").strip() - if not app and not title: - continue - duration = _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS) - interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) - if interval[1] <= interval[0]: - continue - app_name = _normalize_app_name(app, title) - current_key = (app_name, title) - title_changed = previous_key is not None and current_key != previous_key - previous_key = current_key - intervals.append( - { - "app": app_name, - "raw_app": app, - "title": title, - "start": interval[0], - "end": interval[1], - "title_changed": title_changed, - "timestamp": ts, - } - ) - return intervals - - -def _build_not_afk_intervals(afk_events, start, end): - intervals = [] - for ts, event in _events_for_bounds(afk_events, start, end): - if not _is_not_afk_event(event): - continue - duration = _event_duration_seconds(event, default_seconds=5) - interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) - if interval[1] > interval[0]: - intervals.append(interval) - return _merge_intervals(intervals) - - -def _find_window_at(window_intervals, ts): - for item in window_intervals: - if item["start"] <= ts < item["end"]: - return item - return None - - -def _add_app_evidence(evidence_by_app, app, ts, context): - evidence_by_app.setdefault(app, []).append((ts, str(context or "").strip() or "активность")) - - -def build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end): - window_intervals = _build_window_intervals(window_events, start, end) - not_afk_intervals = _build_not_afk_intervals(afk_events, start, end) - evidence_by_app = {} - - for item in window_intervals: - if item["title_changed"]: - _add_app_evidence(evidence_by_app, item["app"], item["timestamp"], item["title"] or item["raw_app"]) - - for events in evidence_events_by_bucket.values(): - for ts, event in _events_for_bounds(events, start, end): - if not _is_real_evidence_event(event): - continue - window = _find_window_at(window_intervals, ts) - if window is None: - continue - _add_app_evidence(evidence_by_app, window["app"], ts, _event_context(event)) - - rows = [] - evidence_delta = timedelta(seconds=TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS) - for app in sorted({item["app"] for item in window_intervals} | set(evidence_by_app)): - app_evidence = sorted(evidence_by_app.get(app, []), key=lambda item: item[0]) - if not app_evidence: - continue - evidence_windows = _merge_intervals([(ts - evidence_delta, ts + evidence_delta) for ts, _context in app_evidence]) - proved_intervals = [] - for window in [item for item in window_intervals if item["app"] == app]: - base = (window["start"], window["end"]) - for afk_interval in not_afk_intervals: - active_overlap = _overlap_interval(base, afk_interval) - if active_overlap is None: - continue - for evidence_interval in evidence_windows: - proved = _overlap_interval(active_overlap, evidence_interval) - if proved is not None: - proved_intervals.append(proved) - proved_intervals = _merge_intervals(proved_intervals) - proved_seconds = int(sum((right - left).total_seconds() for left, right in proved_intervals)) - if proved_seconds <= 0: - continue - last_ts, last_context = app_evidence[-1] - rows.append( - { - "application": app, - "proved_work_seconds": proved_seconds, - "proved_work_hhmm": hhmm(proved_seconds), - "proved_work_human": human_duration_ru(proved_seconds), - "last_action_utc": to_iso_utc(last_ts), - "last_action_local": last_ts.astimezone(REPORT_TZ).strftime("%H:%M"), - "last_action": last_context, - "evidence_events": len(app_evidence), - } - ) - rows.sort(key=lambda item: (-item["proved_work_seconds"], item["application"].lower())) - return rows - - -def build_true_active_apps(host, report_date): - bounds = get_report_bounds(report_date) - host = resolve_host(host) - window_events = fetch_bucket_events(f"aw-watcher-window_{host}", host) or fetch_bucket_events(f"aw-rdp-window_{host}", host) - afk_events = fetch_bucket_events(f"aw-watcher-afk_{host}", host) or fetch_bucket_events(f"aw-rdp-afk_{host}", host) - evidence_events_by_bucket = {} - for bucket_id in ( - f"aw-file-operations_{host}", - f"aw-dlp-endpoint-signals_{host}", - f"aw-watcher-web-chrome_{host}", - f"aw-watcher-web-edge_{host}", - f"aw-detmir-web-category_{host}", - ): - evidence_events_by_bucket[bucket_id] = fetch_bucket_events(bucket_id, host) - return build_true_active_apps_from_events( - window_events, - afk_events, - evidence_events_by_bucket, - bounds["start"], - bounds["end"], - ) - - -def _collect_user_rows(events, start, end, host): - end_exclusive = end + timedelta(seconds=1) - by_user = {} - by_identity = {} - - for event in events: - ts = pts(event.get("timestamp")) - if ts < start or ts > end: - continue - data = event.get("data") or {} - username = str(data.get("username") or "").strip() - if not username: - continue - session_id = str(data.get("sessionId") or "").strip() or "unknown" - event_copy = { - "_ts": ts, - "data": data, - "duration": event.get("duration"), - } - by_identity.setdefault((username, session_id), []).append(event_copy) - - for (username, session_id), samples in by_identity.items(): - ordered = sorted(samples, key=lambda item: item["_ts"]) - for idx, sample in enumerate(ordered): - data = sample["data"] - active = _is_active_sample(data) - next_ts = ordered[idx + 1]["_ts"] if idx + 1 < len(ordered) else None - sample_seconds = _event_sample_seconds(sample, next_ts) - row = by_user.setdefault( - username, - { - "user": username, - "user_id": _normalize_user_id(data, host, username), - "samples_count": 0, - "active_samples": 0, - "session_ids": set(), - "intervals": [], - }, - ) - row["samples_count"] += 1 - row["session_ids"].add(session_id) - if active: - row["active_samples"] += 1 - interval_start = sample["_ts"] - interval_end = min(sample["_ts"] + timedelta(seconds=sample_seconds), end_exclusive) - if interval_end > interval_start: - row["intervals"].append((interval_start, interval_end)) - return by_user - - -def aggregate_rows(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - return _build_rows_from_user_map(by_user, start, end, include_intervals=False) - - -def aggregate_rows_with_intervals(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - return _build_rows_from_user_map(by_user, start, end, include_intervals=True) - - -def _build_rows_from_user_map(by_user, start, end, include_intervals): - rows = [] - full_range = int((end - start).total_seconds()) + 1 - for username in sorted(by_user): - row = by_user[username] - merged = _merge_intervals(row["intervals"]) - active_seconds = int(sum((end_dt - start_dt).total_seconds() for start_dt, end_dt in merged)) - active_seconds = min(active_seconds, full_range) - first_activity = to_iso_utc(merged[0][0]) if merged else "" - last_activity = to_iso_utc(merged[-1][1]) if merged else "" - rows.append( - { - "user": row["user"], - "user_id": row["user_id"], - "active_seconds": active_seconds, - "active_hhmm": hhmm(active_seconds), - "first_activity": first_activity, - "last_activity": last_activity, - "idle_seconds": max(0, full_range - active_seconds), - "sessions_count": len(row["session_ids"]), - "samples_count": row["samples_count"], - "active_samples": row["active_samples"], - } - ) - if include_intervals: - rows[-1]["_intervals"] = merged - return rows - - -def aggregate_hourly_rows(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - rows = [] - for username in sorted(by_user): - row = by_user[username] - merged = _merge_intervals(row["intervals"]) - per_bucket = {} - for interval_start, interval_end in merged: - cursor = interval_start - while cursor < interval_end: - bucket_local = cursor.astimezone(REPORT_TZ).replace(minute=0, second=0, microsecond=0) - bucket_start = bucket_local.astimezone(timezone.utc) - bucket_end = (bucket_local + timedelta(hours=1)).astimezone(timezone.utc) - overlap_start = max(interval_start, bucket_start) - overlap_end = min(interval_end, bucket_end) - if overlap_end > overlap_start: - key = bucket_start - per_bucket[key] = per_bucket.get(key, 0) + int((overlap_end - overlap_start).total_seconds()) - cursor = bucket_end - - for bucket_start in sorted(per_bucket): - active_seconds = per_bucket[bucket_start] - if active_seconds <= 0: - continue - bucket_local = bucket_start.astimezone(REPORT_TZ) - rows.append( - { - "user": row["user"], - "user_id": row["user_id"], - "bucket_start_utc": to_iso_utc(bucket_start), - "bucket_start_local": bucket_local.isoformat(), - "report_date": bucket_local.date().isoformat(), - "hour_local": bucket_local.strftime("%H:00"), - "active_seconds": active_seconds, - "active_hhmm": hhmm(active_seconds), - } - ) - return rows - - -def fetch_events_for_date(host, report_date): - bounds = get_report_bounds(report_date) - bucket_id = get_sessions_bucket_id(host) - events = fetch_bucket_events(bucket_id, host, start=bounds["start"], end=bounds["end"]) - return bounds, events - - -def fetch_bucket_events(bucket_id, host, start=None, end=None): - now = now_utc() - cache_key = bucket_id - params = {"limit": WORKTIME_EVENTS_LIMIT} - if start is not None and end is not None: - params["start"] = to_iso_utc(start) - params["end"] = to_iso_utc(end) - cache_key = f"{bucket_id}|{params['start']}|{params['end']}" - if EVENTS_CACHE_TTL_SECONDS > 0: - with _EVENTS_CACHE_LOCK: - cached = _EVENTS_CACHE.get(cache_key) - if cached is not None and age_seconds(cached["stored_at"], now=now) <= EVENTS_CACHE_TTL_SECONDS: - return cached["events"] - try: - get(f"{AW}/buckets/{bucket_id}") - except Exception: - log_warning(f"bucket lookup failed for host={host} bucket={bucket_id} aw_base={AW}") - return [] - try: - events = get(f"{AW}/buckets/{bucket_id}/events?{urlencode(params)}") - except Exception: - log_warning(f"events fetch failed for host={host} bucket={bucket_id} aw_base={AW}") - return [] - if EVENTS_CACHE_TTL_SECONDS > 0: - with _EVENTS_CACHE_LOCK: - _EVENTS_CACHE[cache_key] = {"stored_at": now, "events": events} - return events - - -def build_report_summary(rows): - if not rows: - return { - "users_count": 0, - "total_active_seconds": 0, - "total_active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "top_user": "", - "top_user_active_hhmm": "00:00", - } - - total_active_seconds = sum(int(row.get("active_seconds", 0) or 0) for row in rows) - first_values = [row.get("first_activity") for row in rows if row.get("first_activity")] - last_values = [row.get("last_activity") for row in rows if row.get("last_activity")] - top_row = max(rows, key=lambda row: int(row.get("active_seconds", 0) or 0)) - return { - "users_count": len(rows), - "total_active_seconds": total_active_seconds, - "total_active_hhmm": hhmm(total_active_seconds), - "first_activity": min(first_values) if first_values else "", - "last_activity": max(last_values) if last_values else "", - "top_user": top_row.get("user", ""), - "top_user_active_hhmm": top_row.get("active_hhmm", "00:00"), - } - - -def latest_bucket_event(bucket_id): - try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit=1") - except Exception: - events = None - if isinstance(events, list) and events: - valid = [item for item in events if isinstance(item, dict)] - if valid: - valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return valid[0] - try: - bucket = get(f"{AW}/buckets/{bucket_id}") - except Exception: - return None - end = ((bucket or {}).get("metadata") or {}).get("end") - if end: - return {"timestamp": end, "duration": 0, "data": {"source": "bucket_metadata"}} - return None - - -def _priority_rank(priority): - return {"critical": 0, "high": 1, "medium": 2, "low": 3}.get(priority, 9) - - -def _clamp_pct(value): - return round(min(100.0, max(0.0, float(value))), 2) - - -def _action(action_id, priority, owner, deadline_hint, reason, recommended_action, *, user_id="", evidence=None): - return { - "action_id": action_id, - "priority": priority, - "owner": owner, - "user_id": user_id, - "deadline_hint": deadline_hint, - "reason": reason, - "recommended_action": recommended_action, - "evidence": evidence or {}, - } - - -def build_executive_summary(summary, actions, sources): - critical = [action for action in actions if action.get("priority") == "critical"] - high = [action for action in actions if action.get("priority") == "high"] - stale_sources = [source for source in sources if source.get("status") != "ok"] - if critical: - portfolio_state = "critical" - headline = f"Есть {len(critical)} критичных вопроса, требующих решения сегодня." - elif high: - portfolio_state = "attention" - headline = f"Критичных провалов нет, но есть {len(high)} вопроса повышенного внимания." - elif summary.get("portfolio_coverage_pct", 0.0) < MANAGER_TARGET_COVERAGE_PCT: - portfolio_state = "attention" - headline = "Покрытие ниже целевого порога, но явных критичных кейсов не найдено." - else: - portfolio_state = "stable" - headline = "Критичных отклонений не найдено, рабочий день идёт в пределах нормы." - - message_parts = [ - f"Активны {summary.get('active_users', 0)} из {summary.get('users_count', 0)} сотрудников.", - f"Покрытие рабочего окна {summary.get('portfolio_coverage_pct', 0.0)}%.", - ] - if stale_sources: - message_parts.append(f"Есть {len(stale_sources)} проблем(ы) со свежестью источников.") - message = " ".join(message_parts) - - focus_items = [] - for action in actions[:5]: - focus_items.append( - { - "priority": action["priority"], - "owner": action["owner"], - "title": action["action_id"], - "reason": action["reason"], - "recommended_action": action["recommended_action"], - } - ) - stale_items = [] - for source in stale_sources[:3]: - stale_items.append( - { - "source_id": source["source_id"], - "label": source["label"], - "status": source["status_label"], - "summary": source["summary"], - } - ) - return { - "portfolio_state": portfolio_state, - "headline": headline, - "message": message, - "focus_items": focus_items, - "stale_sources": stale_items, - } - - -def _empty_rollup(name): - return { - "name": name or "unassigned", - "users_count": 0, - "active_users": 0, - "inactive_users": 0, - "below_target_users": 0, - "workday_total_active_seconds": 0, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 0, - "critical_actions_count": 0, - "high_actions_count": 0, - "medium_actions_count": 0, - "low_actions_count": 0, - "users": [], - } - - -def normalize_management_filter(value): - if value is None: - return "" - return " ".join(str(value).split()).strip() - - -def _filter_key(value): - return normalize_management_filter(value).casefold() - - -def _matches_management_filters(alias, owner_filter="", department_filter=""): - if owner_filter and _filter_key(alias.get("manager_owner")) != _filter_key(owner_filter): - return False - if department_filter and _filter_key(alias.get("department")) != _filter_key(department_filter): - return False - return True - - -def _action_department(action): - evidence = action.get("evidence") or {} - department = normalize_management_filter(evidence.get("department")) - if department: - return department - if normalize_management_filter(action.get("owner")) == "ops": - return "Инфраструктура" - return "" - - -def filter_management_actions(actions, rows, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - if not owner_filter and not department_filter: - return list(actions) - row_user_ids = { - normalize_management_filter(row.get("canonical_user_id") or row.get("user_id")) - for row in rows - if normalize_management_filter(row.get("canonical_user_id") or row.get("user_id")) - } - filtered = [] - for action in actions: - if owner_filter and _filter_key(action.get("owner")) != _filter_key(owner_filter): - continue - if department_filter: - action_department = _action_department(action) - if action_department: - if _filter_key(action_department) != _filter_key(department_filter): - continue - else: - user_id = normalize_management_filter(action.get("user_id")) - if not user_id or user_id not in row_user_ids: - continue - filtered.append(action) - return filtered - - -def _row_matches_payload_filters(row, owner_filter="", department_filter=""): - if owner_filter and _filter_key(row.get("manager_owner")) != _filter_key(owner_filter): - return False - if department_filter and _filter_key(row.get("department")) != _filter_key(department_filter): - return False - return True - - -def summarize_management_rows(rows, actions, workday): - expected_seconds_per_user = int(workday.get("expected_seconds_per_user", 0) or 0) - workday_total_active_seconds = sum(int(row.get("workday_active_seconds", 0) or 0) for row in rows) - calendar_total_active_seconds = sum(int(row.get("calendar_active_seconds", 0) or 0) for row in rows) - active_users = sum(1 for row in rows if row.get("status") != "inactive") - inactive_users = sum(1 for row in rows if row.get("status") == "inactive") - below_target_users = sum(1 for row in rows if row.get("status") == "below_target") - on_target_users = sum(1 for row in rows if row.get("status") == "ok") - workday_first_values = sorted(str(row.get("workday_first_activity_local") or "") for row in rows if row.get("workday_first_activity_local")) - workday_last_values = sorted(str(row.get("workday_last_activity_local") or "") for row in rows if row.get("workday_last_activity_local")) - calendar_first_values = sorted(str(row.get("first_activity_local") or "") for row in rows if row.get("first_activity_local")) - calendar_last_values = sorted(str(row.get("last_activity_local") or "") for row in rows if row.get("last_activity_local")) - top_row = max(rows, key=lambda row: int(row.get("workday_active_seconds", 0) or 0), default=None) - portfolio_coverage_pct = _clamp_pct((workday_total_active_seconds / (expected_seconds_per_user * len(rows))) * 100.0) if rows and expected_seconds_per_user > 0 else 0.0 - return { - "users_count": len(rows), - "active_users": active_users, - "inactive_users": inactive_users, - "on_target_users": on_target_users, - "below_target_users": below_target_users, - "portfolio_coverage_pct": portfolio_coverage_pct, - "actions_count": len(actions), - "critical_actions_count": sum(1 for action in actions if action.get("priority") == "critical"), - "high_actions_count": sum(1 for action in actions if action.get("priority") == "high"), - "calendar_total_active_seconds": calendar_total_active_seconds, - "calendar_total_active_hhmm": hhmm(calendar_total_active_seconds), - "calendar_first_activity": calendar_first_values[0] if calendar_first_values else "", - "calendar_last_activity": calendar_last_values[-1] if calendar_last_values else "", - "workday_total_active_seconds": workday_total_active_seconds, - "workday_total_active_hhmm": hhmm(workday_total_active_seconds), - "workday_first_activity": workday_first_values[0] if workday_first_values else "", - "workday_last_activity": workday_last_values[-1] if workday_last_values else "", - "total_active_seconds": workday_total_active_seconds, - "total_active_hhmm": hhmm(workday_total_active_seconds), - "first_activity": workday_first_values[0] if workday_first_values else "", - "last_activity": workday_last_values[-1] if workday_last_values else "", - "top_user": str((top_row or {}).get("user") or ""), - "top_user_active_hhmm": hhmm(int((top_row or {}).get("workday_active_seconds", 0) or 0)), - } - - -def apply_management_filters_to_payload(payload, owner_filter="", department_filter="", include_sources=True, include_source_actions=True): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - filtered_rows = [ - dict(row) - for row in payload.get("rows", []) - if _row_matches_payload_filters(row, owner_filter=owner_filter, department_filter=department_filter) - ] - base_actions = list(payload.get("actions", [])) - if not include_source_actions: - base_actions = [action for action in base_actions if action.get("action_id") != "source_freshness_review"] - filtered_actions = filter_management_actions(base_actions, filtered_rows, owner_filter=owner_filter, department_filter=department_filter) - filtered_payload = dict(payload) - filtered_payload["filters"] = { - "owner": owner_filter, - "department": department_filter, - } - filtered_payload["rows"] = filtered_rows - filtered_payload["actions"] = filtered_actions - filtered_payload["summary"] = summarize_management_rows(filtered_rows, filtered_actions, payload.get("workday") or {}) - filtered_payload["owner_rollups"] = build_owner_rollups(filtered_rows, filtered_actions) - filtered_payload["department_rollups"] = build_department_rollups(filtered_rows, filtered_actions) - filtered_payload["owner_roster"] = build_owner_roster(filtered_rows, filtered_actions) - if include_sources: - filtered_payload["sources"] = list(payload.get("sources", [])) - else: - filtered_payload["sources"] = [] - filtered_payload["executive"] = build_executive_summary(filtered_payload["summary"], filtered_actions, filtered_payload["sources"]) - return filtered_payload - - -def build_owner_rollups(rows, actions): - groups = {} - for row in rows: - owner = str(row.get("manager_owner") or row.get("user") or "unassigned").strip() or "unassigned" - group = groups.setdefault(owner, _empty_rollup(owner)) - group["users_count"] += 1 - if row.get("status") == "inactive": - group["inactive_users"] += 1 - else: - group["active_users"] += 1 - if row.get("status") == "below_target": - group["below_target_users"] += 1 - group["workday_total_active_seconds"] += int(row.get("workday_active_seconds", 0) or 0) - group["users"].append(row.get("user", "unknown")) - - for action in actions: - owner = str(action.get("owner") or "unassigned").strip() or "unassigned" - group = groups.setdefault(owner, _empty_rollup(owner)) - group["actions_count"] += 1 - prio = str(action.get("priority") or "").lower() - if prio == "critical": - group["critical_actions_count"] += 1 - elif prio == "high": - group["high_actions_count"] += 1 - elif prio == "medium": - group["medium_actions_count"] += 1 - elif prio == "low": - group["low_actions_count"] += 1 - - for group in groups.values(): - expected = group["users_count"] * max(1, WORKDAY_END_HOUR - WORKDAY_START_HOUR) * 3600 - group["workday_total_active_hhmm"] = hhmm(group["workday_total_active_seconds"]) - group["portfolio_coverage_pct"] = _clamp_pct((group["workday_total_active_seconds"] / expected) * 100.0) if expected > 0 else 0.0 - group["users"] = sorted(set(str(user or "").strip() for user in group["users"] if str(user or "").strip())) - - return sorted( - groups.values(), - key=lambda item: ( - -item["critical_actions_count"], - -item["high_actions_count"], - -item["inactive_users"], - item["name"].lower(), - ), - ) - - -def build_department_rollups(rows, actions): - groups = {} - for row in rows: - department = str(row.get("department") or "Без подразделения").strip() or "Без подразделения" - group = groups.setdefault(department, _empty_rollup(department)) - group["users_count"] += 1 - if row.get("status") == "inactive": - group["inactive_users"] += 1 - else: - group["active_users"] += 1 - if row.get("status") == "below_target": - group["below_target_users"] += 1 - group["workday_total_active_seconds"] += int(row.get("workday_active_seconds", 0) or 0) - group["users"].append(row.get("user", "unknown")) - - for action in actions: - evidence = action.get("evidence") or {} - department = str(evidence.get("department") or ("Инфраструктура" if str(action.get("owner") or "").strip().lower() == "ops" else "Без подразделения")).strip() - group = groups.setdefault(department, _empty_rollup(department)) - group["actions_count"] += 1 - prio = str(action.get("priority") or "").lower() - if prio == "critical": - group["critical_actions_count"] += 1 - elif prio == "high": - group["high_actions_count"] += 1 - elif prio == "medium": - group["medium_actions_count"] += 1 - elif prio == "low": - group["low_actions_count"] += 1 - - for group in groups.values(): - expected = group["users_count"] * max(1, WORKDAY_END_HOUR - WORKDAY_START_HOUR) * 3600 - group["workday_total_active_hhmm"] = hhmm(group["workday_total_active_seconds"]) - group["portfolio_coverage_pct"] = _clamp_pct((group["workday_total_active_seconds"] / expected) * 100.0) if expected > 0 else 0.0 - group["users"] = sorted(set(str(user or "").strip() for user in group["users"] if str(user or "").strip())) - - return sorted( - groups.values(), - key=lambda item: ( - -item["critical_actions_count"], - -item["high_actions_count"], - -item["inactive_users"], - item["name"].lower(), - ), - ) - - -def build_owner_roster(rows, actions): - owner_rollups = build_owner_rollups(rows, actions) - roster = [] - for item in owner_rollups: - profile = resolve_owner_profile(item["name"]) - roster.append( - { - **item, - "display_name": profile["display_name"], - "title": profile["title"], - "department": profile["department"], - "contact": profile["contact"], - "escalation_to": profile["escalation_to"], - "notes": profile["notes"], - } - ) - return roster - - -def _interval_overlap_seconds(intervals, start, end): - total = 0 - first = None - last = None - for interval_start, interval_end in intervals or []: - overlap_start = max(interval_start, start) - overlap_end = min(interval_end, end) - if overlap_end <= overlap_start: - continue - seconds = int((overlap_end - overlap_start).total_seconds()) - if seconds <= 0: - continue - total += seconds - if first is None or overlap_start < first: - first = overlap_start - if last is None or overlap_end > last: - last = overlap_end - return total, first, last - - -def _source_status_label(status): - return { - "ok": "fresh", - "warn": "stale", - "fail": "missing", - "inactive": "inactive", - }.get(status, status) - - -def _source_summary(event): - data = (event or {}).get("data") or {} - signal_type = str(data.get("signalType") or "").strip() - if signal_type == "collector_health": - return ( - f"queue={data.get('queueDepth', 0)} " - f"failures={data.get('sendFailures', 0)} " - f"flushed={data.get('eventsFlushed', 0)}" - ) - if data.get("domain"): - return f"{data.get('domain')} ({data.get('category', 'uncategorized')})" - if data.get("eventType"): - return f"{data.get('eventType')} {data.get('username', '')}".strip() - if data.get("action"): - return f"{data.get('action')} {data.get('result', '')}".strip() - if data.get("title"): - return str(data.get("title"))[:120] - if data.get("status"): - return str(data.get("status")) - if data.get("app"): - return str(data.get("app")) - return "" - - -def management_cache_path(host, report_date): - host_slug = safe_slug(host) - return MANAGER_CACHE_DIR / f"{host_slug}-{report_date.isoformat()}.json" - - -def load_management_cache(host, report_date): - if MANAGER_CACHE_TTL_SECONDS <= 0: - return None - path = management_cache_path(host, report_date) - if not path.exists(): - return None - if report_date >= datetime.now(REPORT_TZ).date(): - age = age_seconds(datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)) - if age is None or age > MANAGER_CACHE_TTL_SECONDS: - return None - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return None - - -def save_management_cache(host, report_date, payload): - if MANAGER_CACHE_TTL_SECONDS <= 0: - return - write_atomic_json(management_cache_path(host, report_date), payload) - - -def make_report_cache_key(path, fmt, host, report_date, selected_day="", owner_filter="", department_filter=""): - return "|".join( - [ - str(path), - str(fmt), - resolve_host(host), - report_date.isoformat(), - str(selected_day or ""), - normalize_management_filter(owner_filter), - normalize_management_filter(department_filter), - ] - ) - - -def resolve_report_format(params, accept_header=""): - requested = str(params.get("format", [""])[0] or "").lower() - if requested in {"csv", "html", "json"}: - return requested - accept = str(accept_header or "").lower() - if "text/html" in accept and "application/json" not in accept: - return "html" - return "json" - - -def report_disk_cache_path(key): - digest = hashlib.sha256(key.encode("utf-8")).hexdigest() - return REPORT_DISK_CACHE_DIR / f"{digest}.json" - - -def load_report_response_disk_cache(key): - if REPORT_DISK_STALE_TTL_SECONDS <= 0: - return None - path = report_disk_cache_path(key) - if not path.exists(): - return None - try: - payload = json.loads(path.read_text(encoding="utf-8")) - stored_epoch = float(payload["stored_epoch"]) - age = max(0, int(time.time() - stored_epoch)) - if age > REPORT_DISK_STALE_TTL_SECONDS: - return None - data = base64.b64decode(payload["data_b64"].encode("ascii")) - content_type = str(payload["content_type"]) - except Exception as exc: - log_warning(f"failed to load report disk cache key={key}: {exc}") - return None - return { - "stored_monotonic": time.monotonic() - age, - "generated_at_utc": payload.get("generated_at_utc", ""), - "data": data, - "content_type": content_type, - "age_seconds": age, - "stale": True, - "cache_source": "disk", - } - - -def save_report_response_disk_cache(key, data, content_type, generated_at_utc): - if REPORT_DISK_STALE_TTL_SECONDS <= 0: - return - payload = { - "key": key, - "stored_epoch": time.time(), - "generated_at_utc": generated_at_utc, - "content_type": content_type, - "data_b64": base64.b64encode(data).decode("ascii"), - } - try: - write_atomic_json(report_disk_cache_path(key), payload) - except Exception as exc: - log_warning(f"failed to save report disk cache key={key}: {exc}") - - -def get_report_response_cache(key, *, allow_stale=False): - if REPORT_CACHE_TTL_SECONDS <= 0 and not allow_stale: - return None - now = time.monotonic() - with _REPORT_CACHE_LOCK: - cached = _REPORT_RESPONSE_CACHE.get(key) - if cached is not None: - age = now - cached["stored_monotonic"] - max_age = REPORT_STALE_TTL_SECONDS if allow_stale else REPORT_CACHE_TTL_SECONDS - if max_age > 0 and age <= max_age: - result = dict(cached) - result["age_seconds"] = int(age) - result["stale"] = age > REPORT_CACHE_TTL_SECONDS - result["cache_source"] = "memory" - return result - if allow_stale: - return load_report_response_disk_cache(key) - return None - - -def save_report_response_cache(key, data, content_type): - if REPORT_STALE_TTL_SECONDS <= 0: - return - generated_at_utc = now_utc().isoformat().replace("+00:00", "Z") - with _REPORT_CACHE_LOCK: - _REPORT_RESPONSE_CACHE[key] = { - "stored_monotonic": time.monotonic(), - "generated_at_utc": generated_at_utc, - "data": data, - "content_type": content_type, - } - save_report_response_disk_cache(key, data, content_type, generated_at_utc) - - -def trigger_report_background_refresh(key, path, params): - refresh_params = {name: list(values) for name, values in params.items()} - with _REPORT_CACHE_LOCK: - if key in _REPORT_REFRESHING: - return False - _REPORT_REFRESHING.add(key) - - def refresh(): - try: - if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): - return - try: - data, content_type = build_worktime_report_response(path, refresh_params) - save_report_response_cache(key, data, content_type) - except Exception as exc: - log_warning(f"background report refresh failed path={path}: {exc}") - finally: - _REPORT_BUILD_SEMAPHORE.release() - finally: - with _REPORT_CACHE_LOCK: - _REPORT_REFRESHING.discard(key) - - thread = threading.Thread(target=refresh, name="worktime-report-refresh", daemon=True) - thread.start() - return True - - -def send_cached_report(handler, cached, *, reason): - handler.send_response(200) - handler.send_header("Content-Type", cached["content_type"]) - handler.send_header("Content-Length", str(len(cached["data"]))) - handler.send_header("X-AW-Worktime-Cache", "stale" if cached.get("stale") else "fresh") - handler.send_header("X-AW-Worktime-Cache-Reason", reason) - handler.send_header("X-AW-Worktime-Cache-Age", str(cached.get("age_seconds", 0))) - handler.send_header("X-AW-Worktime-Cache-Source", cached.get("cache_source", "memory")) - handler.end_headers() - try: - handler.wfile.write(cached["data"]) - except (BrokenPipeError, ConnectionResetError): - return False - return True - - -def send_report_unavailable(handler, message): - payload = { - "ok": False, - "error": "report_unavailable", - "message": message, - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - } - data = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(handler, data, "application/json; charset=utf-8", status=503) - - -def send_worktime_report(handler, path, params, cache_key, *, host, report_date): - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl") - return - - allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} - if allow_immediate_stale: - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - refresh_stale = str(params.get("refresh_stale", [""])[0]).lower() in {"1", "true", "yes"} - if refresh_stale: - trigger_report_background_refresh(cache_key, path, params) - send_cached_report(handler, cached, reason="stale-allowed") - return - - if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): - if not allow_immediate_stale and REPORT_BUSY_WAIT_SECONDS > 0: - deadline = time.monotonic() + REPORT_BUSY_WAIT_SECONDS - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - time.sleep(min(REPORT_BUSY_WAIT_INTERVAL_SECONDS, remaining)) - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl-after-wait") - return - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - send_cached_report(handler, cached, reason="busy") - return - send_report_unavailable(handler, "report builder is busy and no cached response is available") - return - - try: - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl-after-lock") - return - try: - data, content_type = build_worktime_report_response(path, params) - except Exception as exc: - log_warning(f"report build failed path={path} host={host} date={report_date}: {exc}") - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - send_cached_report(handler, cached, reason="build-error") - return - send_report_unavailable(handler, "report build failed and no cached response is available") - return - save_report_response_cache(cache_key, data, content_type) - send_bytes(handler, data, content_type) - finally: - _REPORT_BUILD_SEMAPHORE.release() - - -def build_source_freshness(host, interactive_required=True): - source_specs = [ - { - "source_id": "worktime_sessions", - "label": "RDP worktime sessions", - "bucket_candidates": [f"aw-worktime-sessions_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "rdp_window", - "label": "RDP current window", - "bucket_candidates": [f"aw-rdp-window_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "rdp_afk", - "label": "RDP AFK", - "bucket_candidates": [f"aw-rdp-afk_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "watcher_window", - "label": "Local watcher window", - "bucket_candidates": [f"aw-watcher-window_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "interactive_only": True, - "owner": "ops", - }, - { - "source_id": "watcher_afk", - "label": "Local watcher AFK", - "bucket_candidates": [f"aw-watcher-afk_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "file_operations", - "label": "File operations collector", - "bucket_candidates": [f"aw-file-operations_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "interactive_only": True, - "owner": "ops", - }, - { - "source_id": "web_categories", - "label": "Browser/web categories", - "bucket_candidates": [f"aw-detmir-web-category_{host}"], - "max_age_seconds": MANAGER_WEB_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - { - "source_id": "session_events", - "label": "Windows session events", - "bucket_candidates": [f"aw-session-events_{host}"], - "max_age_seconds": MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - { - "source_id": "pve_tasks", - "label": "PVE task feed", - "bucket_candidates": ["aw-pve-task-events_pve-detmir"], - "max_age_seconds": MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - ] - now = now_utc() - sources = [] - actions = [] - for spec in source_specs: - matched_bucket = "" - matched_event = None - matched_age = None - for candidate in spec["bucket_candidates"]: - event = latest_bucket_event(candidate) - if not event: - continue - ts = parse_iso_utc(event.get("timestamp")) - candidate_age = age_seconds(ts, now=now) - if matched_event is None or (candidate_age is not None and (matched_age is None or candidate_age < matched_age)): - matched_bucket = candidate - matched_event = event - matched_age = candidate_age - ts = parse_iso_utc((matched_event or {}).get("timestamp")) - age = matched_age if matched_event is not None else age_seconds(ts, now=now) - if matched_event is None: - status = "fail" if spec["required"] else "warn" - summary = "bucket missing or empty" - elif age is None: - status = "warn" - summary = "timestamp parse failed" - elif age > spec["max_age_seconds"]: - status = "fail" if spec["required"] else "warn" - summary = f"stale ({age}s)" - else: - status = "ok" - summary = f"fresh ({age}s)" - if spec.get("interactive_only") and not interactive_required and status != "ok": - status = "inactive" - summary = "inactive: no active interactive users" - detail = { - "source_id": spec["source_id"], - "label": spec["label"], - "status": status, - "status_label": _source_status_label(status), - "bucket_id": matched_bucket or spec["bucket_candidates"][0], - "timestamp": (matched_event or {}).get("timestamp", ""), - "age_seconds": age, - "required": spec["required"], - "interactive_only": bool(spec.get("interactive_only")), - "interactive_required": bool(interactive_required), - "max_age_seconds": spec["max_age_seconds"], - "summary": summary, - "event_summary": _source_summary(matched_event), - } - sources.append(detail) - if status in {"ok", "inactive"}: - continue - priority = "critical" if spec["required"] else "medium" - actions.append( - _action( - "source_freshness_review", - priority, - spec["owner"], - "today" if spec["required"] else "3d", - f"Источник '{spec['label']}' в состоянии {detail['status_label']}: {summary}.", - "Проверить collector/service, причину отставания и подтвердить, что управленческие выводы по данным ещё надёжны.", - evidence={ - "source_id": spec["source_id"], - "bucket_id": detail["bucket_id"], - "age_seconds": age, - "required": spec["required"], - }, - ) - ) - return sources, actions - - -def _build_management_core(rows, host, report_date, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - report_bounds = get_report_bounds(report_date) - workday = get_workday_bounds(report_date) - now_local = datetime.now(REPORT_TZ) - is_today = report_date == now_local.date() - effective_end_local = min(now_local, workday["end_local"]) if is_today else workday["end_local"] - elapsed_seconds = max(0, int((effective_end_local - workday["start_local"]).total_seconds())) - if not is_today: - elapsed_seconds = workday["duration_seconds"] - expected_seconds_per_user = min(workday["duration_seconds"], max(0, elapsed_seconds)) - target_seconds = int(expected_seconds_per_user * (MANAGER_TARGET_COVERAGE_PCT / 100.0)) - low_seconds = int(expected_seconds_per_user * (MANAGER_LOW_COVERAGE_PCT / 100.0)) - late_start_local = workday["start_local"] + timedelta(minutes=MANAGER_LATE_START_GRACE_MINUTES) - early_finish_local = workday["end_local"] - timedelta(minutes=MANAGER_EARLY_FINISH_GRACE_MINUTES) - - roster = [] - actions = [] - active_users = 0 - on_target_users = 0 - below_target_users = 0 - workday_total_active_seconds = 0 - workday_first_values = [] - workday_last_values = [] - top_workday_user = "" - top_workday_seconds = -1 - filtered_rows = [] - - for row in rows: - alias = resolve_user_alias(row.get("user", ""), row.get("user_id", ""), host) - if alias["exclude"]: - continue - if not _matches_management_filters(alias, owner_filter=owner_filter, department_filter=department_filter): - continue - filtered_rows.append(row) - public_row = {key: value for key, value in row.items() if key != "_intervals"} - calendar_active_seconds = int(row.get("active_seconds", 0) or 0) - intervals = row.get("_intervals") or [] - workday_active_seconds, workday_first, workday_last = _interval_overlap_seconds( - intervals, - workday["start_local"].astimezone(timezone.utc), - effective_end_local.astimezone(timezone.utc), - ) - first_activity = parse_iso_utc(row.get("first_activity")) - last_activity = parse_iso_utc(row.get("last_activity")) - first_local = first_activity.astimezone(REPORT_TZ) if first_activity else None - last_local = last_activity.astimezone(REPORT_TZ) if last_activity else None - workday_first_local = workday_first.astimezone(REPORT_TZ) if workday_first else None - workday_last_local = workday_last.astimezone(REPORT_TZ) if workday_last else None - coverage_pct = _clamp_pct((workday_active_seconds / expected_seconds_per_user) * 100.0) if expected_seconds_per_user > 0 else 0.0 - status = "ok" - if workday_active_seconds <= 0: - status = "inactive" - elif workday_active_seconds < target_seconds: - status = "below_target" - if workday_active_seconds > 0: - active_users += 1 - workday_total_active_seconds += workday_active_seconds - if workday_first_local: - workday_first_values.append(workday_first_local.isoformat()) - if workday_last_local: - workday_last_values.append(workday_last_local.isoformat()) - if workday_active_seconds >= target_seconds and workday_active_seconds > 0: - on_target_users += 1 - elif workday_active_seconds > 0: - below_target_users += 1 - if workday_active_seconds > top_workday_seconds: - top_workday_seconds = workday_active_seconds - top_workday_user = alias["display_name"] - - roster.append( - { - **public_row, - "user": alias["display_name"], - "user_original": row.get("user", ""), - "manager_owner": alias["manager_owner"], - "department": alias["department"], - "role": alias["role"], - "notes": alias["notes"], - "canonical_user_id": alias["canonical_user_id"] or row.get("user_id", ""), - "calendar_active_seconds": calendar_active_seconds, - "calendar_active_hhmm": row.get("active_hhmm", "00:00"), - "workday_active_seconds": workday_active_seconds, - "workday_active_hhmm": hhmm(workday_active_seconds), - "coverage_pct": coverage_pct, - "status": status, - "first_activity_local": first_local.isoformat() if first_local else "", - "last_activity_local": last_local.isoformat() if last_local else "", - "workday_first_activity_local": workday_first_local.isoformat() if workday_first_local else "", - "workday_last_activity_local": workday_last_local.isoformat() if workday_last_local else "", - } - ) - - owner = alias["display_name"] - user_id = alias["canonical_user_id"] or row.get("user_id", "") - evidence = { - "calendar_active_hhmm": row.get("active_hhmm", "00:00"), - "workday_active_hhmm": hhmm(workday_active_seconds), - "coverage_pct": coverage_pct, - "first_activity": row.get("first_activity", ""), - "last_activity": row.get("last_activity", ""), - "sessions_count": row.get("sessions_count", 0), - "manager_owner": alias["manager_owner"], - "department": alias["department"], - "role": alias["role"], - } - if workday_active_seconds <= 0: - actions.append( - _action( - "missing_activity", - "critical", - alias["manager_owner"], - "today", - f"За {report_date.isoformat()} у сотрудника {owner} нет подтверждённой активности в рабочем окне RDP.", - f"Проверить сотрудника {owner}: работал ли он в рабочее время, была ли потеря сбора данных или отсутствие входа в систему.", - user_id=user_id, - evidence=evidence, - ) - ) - continue - if expected_seconds_per_user > 0 and workday_active_seconds < low_seconds: - actions.append( - _action( - "low_activity_review", - "high", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} активное время в рабочем окне {hhmm(workday_active_seconds)} ниже {MANAGER_LOW_COVERAGE_PCT}% от ожидаемого окна.", - f"Проверить загрузку сотрудника {owner}, задачи и фактическое присутствие в рабочем процессе.", - user_id=user_id, - evidence=evidence, - ) - ) - elif expected_seconds_per_user > 0 and workday_active_seconds < target_seconds: - actions.append( - _action( - "target_gap_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} активное время в рабочем окне {hhmm(workday_active_seconds)} ниже управленческого целевого порога {MANAGER_TARGET_COVERAGE_PCT}%.", - f"Уточнить причину отклонения по сотруднику {owner} и подтвердить план работ.", - user_id=user_id, - evidence=evidence, - ) - ) - if workday_first_local and workday_first_local > late_start_local: - actions.append( - _action( - "late_start_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} первая активность в рабочем окне зафиксирована поздно: {workday_first_local.strftime('%H:%M')}.", - f"Проверить причину позднего старта сотрудника {owner} и подтвердить, что это не проблема доступа или дисциплины.", - user_id=user_id, - evidence=evidence, - ) - ) - if (not is_today) and workday_last_local and workday_last_local < early_finish_local: - actions.append( - _action( - "early_finish_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} последняя активность в рабочем окне завершилась рано: {workday_last_local.strftime('%H:%M')}.", - f"Проверить, было ли досрочное завершение рабочего дня сотрудника {owner} согласовано и чем оно объясняется.", - user_id=user_id, - evidence=evidence, - ) - ) - - actions.sort(key=lambda item: (_priority_rank(item["priority"]), item["owner"].lower(), item["action_id"])) - calendar_summary = build_report_summary(filtered_rows) - inactive_users = sum(1 for row in roster if row["status"] == "inactive") - portfolio_coverage_pct = _clamp_pct((workday_total_active_seconds / (expected_seconds_per_user * len(roster))) * 100.0) if roster and expected_seconds_per_user > 0 else 0.0 - calendar_first = calendar_summary.get("first_activity", "") - calendar_last = calendar_summary.get("last_activity", "") - return { - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "host": resolve_host(host), - "report_date": report_date.isoformat(), - "report_timezone": str(REPORT_TZ), - "filters": { - "owner": owner_filter, - "department": department_filter, - }, - "workday": { - "start_local": workday["start_local"].isoformat(), - "end_local": workday["end_local"].isoformat(), - "expected_seconds_per_user": expected_seconds_per_user, - "expected_hhmm_per_user": hhmm(expected_seconds_per_user), - "target_coverage_pct": MANAGER_TARGET_COVERAGE_PCT, - "low_coverage_pct": MANAGER_LOW_COVERAGE_PCT, - }, - "summary": { - **calendar_summary, - "calendar_total_active_seconds": calendar_summary["total_active_seconds"], - "calendar_total_active_hhmm": calendar_summary["total_active_hhmm"], - "calendar_first_activity": calendar_first, - "calendar_last_activity": calendar_last, - "workday_total_active_seconds": workday_total_active_seconds, - "workday_total_active_hhmm": hhmm(workday_total_active_seconds), - "workday_first_activity": min(workday_first_values) if workday_first_values else "", - "workday_last_activity": max(workday_last_values) if workday_last_values else "", - "total_active_seconds": workday_total_active_seconds, - "total_active_hhmm": hhmm(workday_total_active_seconds), - "first_activity": min(workday_first_values) if workday_first_values else "", - "last_activity": max(workday_last_values) if workday_last_values else "", - "top_user": top_workday_user, - "top_user_active_hhmm": hhmm(top_workday_seconds if top_workday_seconds > 0 else 0), - "active_users": active_users, - "inactive_users": inactive_users, - "on_target_users": on_target_users, - "below_target_users": below_target_users, - "portfolio_coverage_pct": portfolio_coverage_pct, - "actions_count": len(actions), - "critical_actions_count": sum(1 for action in actions if action["priority"] == "critical"), - "high_actions_count": sum(1 for action in actions if action["priority"] == "high"), - }, - "actions": actions, - "rows": roster, - "bucket_id": get_sessions_bucket_id(host), - "report_bounds": { - "start_utc": to_iso_utc(report_bounds["start"]), - "end_utc": to_iso_utc(report_bounds["end"]), - }, - } - - -def _management_trend_item(payload, report_date): - summary = payload["summary"] - return { - "report_date": report_date.isoformat(), - "users_count": summary["users_count"], - "active_users": summary["active_users"], - "inactive_users": summary["inactive_users"], - "workday_total_active_seconds": summary["workday_total_active_seconds"], - "workday_total_active_hhmm": summary["workday_total_active_hhmm"], - "portfolio_coverage_pct": summary["portfolio_coverage_pct"], - "actions_count": summary["actions_count"], - "critical_actions_count": summary["critical_actions_count"], - } - - -def build_management_trend(host, anchor_date, owner_filter="", department_filter="", precomputed_payloads=None): - trend = [] - precomputed_payloads = precomputed_payloads or {} - for offset in range(MANAGER_TREND_DAYS - 1, -1, -1): - current_date = anchor_date - timedelta(days=offset) - payload = precomputed_payloads.get(current_date) - if payload is None: - payload = load_management_cache(host, current_date) - if payload is None: - bounds, events = fetch_events_for_date(host, current_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - payload = _build_management_core(rows, host, current_date, owner_filter=owner_filter, department_filter=department_filter) - elif owner_filter or department_filter: - payload = apply_management_filters_to_payload( - payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=False, - include_source_actions=False, - ) - trend.append(_management_trend_item(payload, current_date)) - return trend - - -def build_filtered_management_trend(host, anchor_date, owner_filter="", department_filter=""): - trend = [] - for offset in range(MANAGER_TREND_DAYS - 1, -1, -1): - current_date = anchor_date - timedelta(days=offset) - base_payload = load_management_cache(host, current_date) - if base_payload is None: - bounds, events = fetch_events_for_date(host, current_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - base_payload = _build_management_core(rows, host, current_date) - filtered_payload = apply_management_filters_to_payload( - base_payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=False, - include_source_actions=False, - ) - summary = filtered_payload["summary"] - trend.append( - { - "report_date": current_date.isoformat(), - "users_count": summary["users_count"], - "active_users": summary["active_users"], - "inactive_users": summary["inactive_users"], - "workday_total_active_seconds": summary["workday_total_active_seconds"], - "workday_total_active_hhmm": summary["workday_total_active_hhmm"], - "portfolio_coverage_pct": summary["portfolio_coverage_pct"], - "actions_count": summary["actions_count"], - "critical_actions_count": summary["critical_actions_count"], - } - ) - return trend - - -def build_management_payload(rows, host, report_date, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - payload = _build_management_core(rows, host, report_date, owner_filter=owner_filter, department_filter=department_filter) - interactive_required = int(payload.get("summary", {}).get("active_users", 0) or 0) > 0 - source_freshness, source_actions = build_source_freshness(resolve_host(host), interactive_required=interactive_required) - payload["sources"] = source_freshness - payload["trend"] = build_management_trend( - resolve_host(host), - report_date, - owner_filter=owner_filter, - department_filter=department_filter, - precomputed_payloads={report_date: payload}, - ) - payload["trend_scope"] = "portfolio" - if source_actions: - payload["actions"].extend(filter_management_actions(source_actions, payload["rows"], owner_filter=owner_filter, department_filter=department_filter)) - payload["actions"].sort(key=lambda item: (_priority_rank(item["priority"]), item["owner"].lower(), item["action_id"])) - payload["summary"]["actions_count"] = len(payload["actions"]) - payload["summary"]["critical_actions_count"] = sum(1 for action in payload["actions"] if action["priority"] == "critical") - payload["summary"]["high_actions_count"] = sum(1 for action in payload["actions"] if action["priority"] == "high") - payload["executive"] = build_executive_summary(payload["summary"], payload["actions"], payload["sources"]) - payload["owner_rollups"] = build_owner_rollups(payload["rows"], payload["actions"]) - payload["department_rollups"] = build_department_rollups(payload["rows"], payload["actions"]) - payload["owner_roster"] = build_owner_roster(payload["rows"], payload["actions"]) - return payload - - -def report_for_date(host, report_date): - bounds, events = fetch_events_for_date(host, report_date) - return aggregate_rows(events, bounds["start"], bounds["end"], host) - - -def management_report_for_date(host, report_date, owner_filter="", department_filter="", use_cache=True): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - if owner_filter or department_filter: - base_payload = management_report_for_date(host, report_date, use_cache=use_cache) - filtered_payload = apply_management_filters_to_payload( - base_payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=True, - include_source_actions=True, - ) - filtered_payload["trend"] = [] - filtered_payload["trend_scope"] = "filtered_current_only" - return filtered_payload - cache_allowed = use_cache or report_date < datetime.now(REPORT_TZ).date() - if cache_allowed: - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - lock = get_management_build_lock(host, report_date) - with lock: - if cache_allowed: - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - bounds, events = fetch_events_for_date(host, report_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - payload = build_management_payload(rows, host, report_date) - save_management_cache(host, report_date, payload) - return payload - - -def report_today(host): - return report_for_date(host, resolve_report_date()) - - -def report_for_date_fresh(host, report_date): - spec = importlib.util.spec_from_file_location("aw_worktime_runtime", MODULE_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module.report_for_date(host, report_date) - - -def render_html(rows, host, report_date, selected_day=None, true_active_apps=None): - generated = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - date_local = report_date.strftime("%Y-%m-%d") - day_query = f"&day={selected_day}" if selected_day in {"today", "yesterday"} else "" - date_query = f"&date={date_local}" if not day_query else "" - summary = build_report_summary(rows) - today_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": resolve_host(host), "day": "today"}) - yesterday_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": resolve_host(host), "day": "yesterday"}) - csv_url = "/reports/worktime/today?" + urlencode({"format": "csv", "host": resolve_host(host), **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": date_local})}) - json_url = "/reports/worktime/today?" + urlencode({"host": resolve_host(host), **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": date_local})}) - form_action = "/reports/worktime/today" - cards = [ - ("Пользователи", str(summary["users_count"])), - ("Активное время", summary["total_active_hhmm"]), - ("Лидер дня", f"{summary['top_user']} · {summary['top_user_active_hhmm']}" if summary["top_user"] else "н/д"), - ("Диапазон", f"{summary['first_activity']} -> {summary['last_activity']}" if summary["first_activity"] else "нет активности"), - ] - trs = [] - detail_cards = [] - true_active_apps = true_active_apps or [] - true_active_rows = [] - for app_row in true_active_apps: - last_action = app_row.get("last_action") or "-" - last_time = app_row.get("last_action_local") or "-" - true_active_rows.append( - "" - f"{html.escape(app_row.get('application') or '-')}" - f"{html.escape(app_row.get('proved_work_human') or app_row.get('proved_work_hhmm') or '0 сек')}" - f"{html.escape(last_time)} · {html.escape(last_action)}" - "" - ) - if not true_active_rows: - true_active_rows.append("Пока нет доказанной активной работы по приложениям за выбранную дату.") - for row in rows: - user_slug = safe_slug(row["user"]) - active_seconds = int(row.get("active_seconds", 0) or 0) - utilization = 0.0 - day_total = 24 * 3600 - if day_total > 0: - utilization = round((active_seconds / day_total) * 100.0, 2) - trs.append( - "" - f"{html.escape(row['user'])}" - f"{html.escape(row['user_id'])}" - f"{row['active_hhmm']}" - f"{row['active_seconds']}" - f"{html.escape(row['first_activity'])}" - f"{html.escape(row['last_activity'])}" - f"{row['idle_seconds']}" - f"{row['sessions_count']}" - f"{row['samples_count']}" - "" - ) - detail_cards.append( - "
" - "
" - "

{user}

" - "{active}" - "
" - "
" - "
Пользователь{user_id}
" - "
Загрузка{utilization}%
" - "
Начало активности{first_activity}
" - "
Конец активности{last_activity}
" - "
Сессии{sessions}
" - "
Активные сэмплы{active_samples} / {samples}
" - "
" - "
" - .format( - slug=user_slug, - user=html.escape(row["user"]), - active=html.escape(row["active_hhmm"]), - user_id=html.escape(row["user_id"]), - utilization=utilization, - first_activity=html.escape(row["first_activity"] or "н/д"), - last_activity=html.escape(row["last_activity"] or "н/д"), - sessions=row["sessions_count"], - active_samples=row["active_samples"], - samples=row["samples_count"], - )) - if not trs: - trs.append('За выбранную дату данных пока нет.') - detail_cards.append("

За выбранную дату нет активности пользователей.

") - return f""" - - - - - AW-rus Отчёт по работе в RDP - - - -
-
-

Отчёт по работе в RDP

-
Хост: {resolve_host(host)} · Дата: {date_local} · Часовой пояс: {REPORT_TZ} · Сформировано UTC: {generated}
-
- Сегодня - Вчера - Скачать CSV - Открыть JSON -
-
-
- - - - -
-
-
- {''.join(f"
{html.escape(label)}{html.escape(value)}
" for label, value in cards)} -
-
-
-

Доказанная работа по приложениям

- - - - - - - - - - {''.join(true_active_rows)} - -
ПриложениеДоказанная работаПоследнее действие
-
-
-

Таблица по пользователям

- - - - - - - - - - - - - - - - {''.join(trs)} - -
ПользовательУчётная записьАктивноАктивно, секНачало активностиКонец активностиПростой, секСессииСэмплы
-
-
-

Детали по пользователям

-
- {''.join(detail_cards)} -
-
-
- -""" - - -def build_management_report_url(host, report_date, selected_day=None, fmt=None, owner_filter="", department_filter=""): - params = {"host": resolve_host(host)} - if fmt: - params["format"] = fmt - if selected_day in {"today", "yesterday"}: - params["day"] = selected_day - else: - params["date"] = report_date - if normalize_management_filter(owner_filter): - params["owner"] = normalize_management_filter(owner_filter) - if normalize_management_filter(department_filter): - params["department"] = normalize_management_filter(department_filter) - return "/reports/worktime/management?" + urlencode(params) - - -def render_management_html(payload, selected_day=None): - summary = payload["summary"] - workday = payload["workday"] - report_date = payload["report_date"] - host = payload["host"] - executive = payload.get("executive") or {} - active_filters = payload.get("filters") or {} - owner_filter = normalize_management_filter(active_filters.get("owner")) - department_filter = normalize_management_filter(active_filters.get("department")) - today_url = build_management_report_url(host, report_date, selected_day="today", fmt="html", owner_filter=owner_filter, department_filter=department_filter) - yesterday_url = build_management_report_url(host, report_date, selected_day="yesterday", fmt="html", owner_filter=owner_filter, department_filter=department_filter) - json_url = build_management_report_url(host, report_date, selected_day=selected_day, owner_filter=owner_filter, department_filter=department_filter) - classic_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": host, **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": report_date})}) - reset_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html") - actions_html = [] - for action in payload["actions"]: - actions_html.append( - "" - f"{html.escape(action['priority'])}" - f"{html.escape(action['owner'])}" - f"{html.escape(action['action_id'])}" - f"{html.escape(action['deadline_hint'])}" - f"{html.escape(action['reason'])}" - f"{html.escape(action['recommended_action'])}" - "" - ) - if not actions_html: - actions_html.append("Отклонений по текущим правилам не найдено.") - - roster_html = [] - for row in payload["rows"]: - roster_html.append( - "" - f"{html.escape(row['user'])}" - f"{html.escape(row.get('canonical_user_id') or row['user_id'])}" - f"{html.escape(row.get('manager_owner') or row['user'])}" - f"{html.escape(row.get('department') or '-')}" - f"{html.escape(row['workday_active_hhmm'])}" - f"{html.escape(row['calendar_active_hhmm'])}" - f"{row['coverage_pct']}" - f"{html.escape(row['status'])}" - f"{html.escape(row.get('workday_first_activity_local') or '-')}" - f"{html.escape(row.get('workday_last_activity_local') or '-')}" - f"{row['sessions_count']}" - "" - ) - if not roster_html: - roster_html.append("За выбранную дату данных нет.") - - trend_html = [] - for row in payload.get("trend", []): - trend_html.append( - "" - f"{html.escape(row['report_date'])}" - f"{row['users_count']}" - f"{row['active_users']}" - f"{row['inactive_users']}" - f"{html.escape(row['workday_total_active_hhmm'])}" - f"{row['portfolio_coverage_pct']}" - f"{row['actions_count']}" - f"{row['critical_actions_count']}" - "" - ) - if not trend_html: - trend_message = "Тренд по выбранному фильтру пока отключён, чтобы current-scope отчёт отвечал быстро." if (owner_filter or department_filter) else "Тренд пока недоступен." - trend_html.append(f"{html.escape(trend_message)}") - - owner_rollup_html = [] - for item in payload.get("owner_rollups", []): - owner_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html", owner_filter=item["name"], department_filter=department_filter) - owner_rollup_html.append( - "" - f"{html.escape(item['name'])}" - f"{item['users_count']}" - f"{item['inactive_users']}" - f"{item['below_target_users']}" - f"{item['critical_actions_count']}" - f"{item['high_actions_count']}" - f"{html.escape(item['workday_total_active_hhmm'])}" - f"{item['portfolio_coverage_pct']}" - f"{html.escape(', '.join(item['users']) if item['users'] else '-')}" - "" - ) - if not owner_rollup_html: - owner_rollup_html.append("Нет данных по ответственным.") - - department_rollup_html = [] - for item in payload.get("department_rollups", []): - department_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html", owner_filter=owner_filter, department_filter=item["name"]) - department_rollup_html.append( - "" - f"{html.escape(item['name'])}" - f"{item['users_count']}" - f"{item['inactive_users']}" - f"{item['below_target_users']}" - f"{item['critical_actions_count']}" - f"{item['high_actions_count']}" - f"{html.escape(item['workday_total_active_hhmm'])}" - f"{item['portfolio_coverage_pct']}" - f"{html.escape(', '.join(item['users']) if item['users'] else '-')}" - "" - ) - if not department_rollup_html: - department_rollup_html.append("Нет данных по подразделениям.") - - owner_profile_html = [] - for item in payload.get("owner_roster", []): - owner_profile_html.append( - "
" - f"
0 else ('high' if item['high_actions_count'] > 0 else 'low')}'>{html.escape(item['display_name'])}
" - f"

{html.escape(item['title'] or item['name'])}

" - f"
Подразделение: {html.escape(item['department'] or '-')}
" - f"

Пользователи: {item['users_count']} · inactive: {item['inactive_users']} · actions: {item['actions_count']}

" - f"

Контакт: {html.escape(item['contact'] or '-')}

" - f"

Эскалация: {html.escape(item['escalation_to'] or '-')}

" - f"

{html.escape(item['notes'] or 'Без дополнительных заметок.')}

" - "
" - ) - if not owner_profile_html: - owner_profile_html.append("

Каталог ответственных пуст

Добавьте блок owners в worktime-manager-aliases.json, чтобы отчёт показывал роли, контакты и эскалацию.

") - - sources_html = [] - for source in payload.get("sources", []): - source_priority_class = "low" if source["status"] in {"ok", "inactive"} else ("critical" if source["required"] else "medium") - sources_html.append( - "" - f"{html.escape(source['label'])}" - f"{html.escape(source['status_label'])}" - f"{html.escape(source['bucket_id'])}" - f"{html.escape(source.get('timestamp') or '-')}" - f"{html.escape(str(source.get('age_seconds')) if source.get('age_seconds') is not None else '-')}" - f"{html.escape(source.get('event_summary') or source.get('summary') or '-')}" - "" - ) - if not sources_html: - sources_html.append("Статусы источников недоступны.") - - focus_html = [] - for item in executive.get("focus_items", []): - focus_html.append( - "
" - f"
{html.escape(item['priority'])}
" - f"

{html.escape(item['title'])}

" - f"
Ответственный: {html.escape(item['owner'])}
" - f"

{html.escape(item['reason'])}

" - f"{html.escape(item['recommended_action'])}" - "
" - ) - if not focus_html: - focus_html.append("

Критичных действий нет

На текущий момент менеджерских отклонений по активным правилам не найдено.

") - - stale_html = [] - for item in executive.get("stale_sources", []): - stale_html.append( - "
  • " - f"{html.escape(item['label'])}: {html.escape(item['status'])} · {html.escape(item['summary'])}" - "
    • " - ) - stale_block = ( - "
    Проблемы со свежестью источников:
      " - + "".join(stale_html) - + "
    " - ) if stale_html else "" - filter_parts = [] - if owner_filter: - filter_parts.append(f"ответственный: {html.escape(owner_filter)}") - if department_filter: - filter_parts.append(f"подразделение: {html.escape(department_filter)}") - filter_block = ( - "
    Фильтр: " - + " · ".join(filter_parts) - + f" Сбросить
    " - ) if filter_parts else "" - - cards = [ - ("Пользователи", str(summary["users_count"])), - ("Активны", str(summary["active_users"])), - ("Без активности", str(summary["inactive_users"])), - ("Ниже цели", str(summary["below_target_users"])), - ("Покрытие", f"{summary['portfolio_coverage_pct']}%"), - ("Действия", str(summary["actions_count"])), - ("Рабочее окно", summary["workday_total_active_hhmm"]), - ("Календарный день", summary["calendar_total_active_hhmm"]), - ] - - return f""" - - - - - AW-rus Управленческий отчёт по работе в RDP - - - -
    -
    -

    Управленческий отчёт по работе в RDP

    -
    Хост: {html.escape(host)} · Дата: {html.escape(report_date)} · Рабочее окно: {html.escape(workday['start_local'])} -> {html.escape(workday['end_local'])} · Сформировано UTC: {html.escape(payload['generated_at_utc'])}
    -
    - Сегодня - Вчера - Открыть JSON - Классический RDP отчёт -
    -
    - {''.join(f"
    {html.escape(label)}{html.escape(value)}
    " for label, value in cards)} -
    -
    - Целевое покрытие: {MANAGER_TARGET_COVERAGE_PCT}% от ожидаемого рабочего окна на пользователя. - Критический провал: ниже {MANAGER_LOW_COVERAGE_PCT}% или полное отсутствие активности. - Рабочее окно считается отдельно от календарной активности, чтобы ночная работа не маскировала дневной провал. -
    -
    -
    -

    Что делать сегодня

    -
    - {html.escape(executive.get('headline') or 'Сводка недоступна')} -

    {html.escape(executive.get('message') or '')}

    - {filter_block} - {stale_block} -
    -
    - {''.join(focus_html)} -
    -
    -
    -

    Тренд за {MANAGER_TREND_DAYS} дней

    - - - - - - - - - - - - - - - {''.join(trend_html)} - -
    ДатаПользователиАктивныБез активностиРабочее окноПокрытие, %ДействияCritical
    -
    -
    -

    По ответственным

    - - - - - - - - - - - - - - - - {''.join(owner_rollup_html)} - -
    ОтветственныйСотрудникиБез активностиНиже целиCriticalHighРабочее окноПокрытие, %Кого затрагивает
    -
    -
    -

    Ответственные и эскалация

    -
    - {''.join(owner_profile_html)} -
    -
    -
    -

    По подразделениям

    - - - - - - - - - - - - - - - - {''.join(department_rollup_html)} - -
    ПодразделениеСотрудникиБез активностиНиже целиCriticalHighРабочее окноПокрытие, %Кого затрагивает
    -
    -
    -

    Очередь действий руководителя

    - - - - - - - - - - - - - {''.join(actions_html)} - -
    ПриоритетСотрудникТипСрокПочему это важноЧто сделать
    -
    -
    -

    Покрытие по сотрудникам

    - - - - - - - - - - - - - - - - - - {''.join(roster_html)} - -
    СотрудникУчётная записьОтветственныйПодразделениеАктивно в окнеАктивно за деньПокрытие, %СтатусПервая активность в окнеПоследняя активность в окнеСессии
    -
    -
    -

    Свежесть источников данных

    - - - - - - - - - - - - - {''.join(sources_html)} - -
    ИсточникСтатусБакетПоследнее событие UTCВозраст, секКонтекст
    -
    -
    - -""" - - -def build_worktime_report_response(path, params): - fmt = resolve_report_format(params) - host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) - day = params.get("day", ["today"])[0] - date_text = params.get("date", [None])[0] - owner_filter = normalize_management_filter(params.get("owner", [""])[0]) - department_filter = normalize_management_filter(params.get("department", [""])[0]) - report_date = resolve_report_date(day=day, date_text=date_text) - is_management = path == "/reports/worktime/management" - allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} - management_payload = ( - management_report_for_date( - host, - report_date, - owner_filter=owner_filter, - department_filter=department_filter, - use_cache=allow_immediate_stale, - ) - if is_management - else None - ) - rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] - true_active_apps = [] if is_management else build_true_active_apps(host, report_date) - - if fmt == "csv": - if is_management: - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], - extrasaction="ignore", - ) - writer.writeheader() - writer.writerows(management_payload["actions"]) - return out.getvalue().encode(), "text/csv; charset=utf-8" - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=[ - "user", - "user_id", - "active_seconds", - "active_hhmm", - "first_activity", - "last_activity", - "idle_seconds", - "sessions_count", - "samples_count", - "active_samples", - ], - ) - writer.writeheader() - writer.writerows(rows) - return out.getvalue().encode(), "text/csv; charset=utf-8" - - if fmt == "html": - if is_management: - data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") - else: - data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") - return data, "text/html; charset=utf-8" - - if is_management: - data = json.dumps(management_payload, ensure_ascii=False, indent=2).encode("utf-8") - return data, "application/json; charset=utf-8" - - obj = { - "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "host": host, - "report_date": report_date.isoformat(), - "bucket_id": get_sessions_bucket_id(host), - "rows": rows, - "true_active_apps": true_active_apps, - } - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - return data, "application/json; charset=utf-8" - - -def send_bytes(handler, data, content_type, status=200): - handler.send_response(status) - handler.send_header("Content-Type", content_type) - handler.send_header("Content-Length", str(len(data))) - handler.end_headers() - try: - handler.wfile.write(data) - except (BrokenPipeError, ConnectionResetError): - return False - return True - - -class H(BaseHTTPRequestHandler): - def do_GET(self): - parsed = urlparse(self.path) - if parsed.path in {"/health", "/api/health"}: - data = json.dumps(worktime_health_payload(), ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") - return - - if parsed.path.startswith("/dlp-ioc/"): - name = parsed.path.rsplit("/", 1)[-1] - if name not in {"ioc_blacklist.json", "ioc_blacklist.csv", "ioc_blacklist.sql"}: - self.send_response(404) - self.end_headers() - return - path = os.path.join(IOC_DIR, name) - if not os.path.isfile(path): - self.send_response(404) - self.end_headers() - return - with open(path, "rb") as f: - data = f.read() - if name.endswith(".json"): - ctype = "application/json; charset=utf-8" - elif name.endswith(".csv"): - ctype = "text/csv; charset=utf-8" - else: - ctype = "text/plain; charset=utf-8" - send_bytes(self, data, ctype) - return - - if parsed.path not in {"/reports/worktime/today", "/reports/worktime/management"}: - self.send_response(404) - self.end_headers() - return - - params = parse_qs(parsed.query, keep_blank_values=False) - fmt = resolve_report_format(params, self.headers.get("Accept", "")) - if not params.get("format") and fmt != "json": - params = {name: list(values) for name, values in params.items()} - params["format"] = [fmt] - host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) - day = params.get("day", ["today"])[0] - date_text = params.get("date", [None])[0] - owner_filter = normalize_management_filter(params.get("owner", [""])[0]) - department_filter = normalize_management_filter(params.get("department", [""])[0]) - report_date = resolve_report_date(day=day, date_text=date_text) - cache_key = make_report_cache_key( - parsed.path, - fmt, - host, - report_date, - selected_day=day if day in {"today", "yesterday"} else "", - owner_filter=owner_filter, - department_filter=department_filter, - ) - - send_worktime_report(self, parsed.path, params, cache_key, host=host, report_date=report_date) - - -class WorktimeHTTPServer(ThreadingHTTPServer): - daemon_threads = True - request_queue_size = 64 - - -def main(): - WorktimeHTTPServer((LISTEN_HOST, LISTEN_PORT), H).serve_forever(poll_interval=0.5) - - -if __name__ == "__main__": - main() diff --git a/aw-server/aw-worktime-api.service b/aw-server/aw-worktime-api.service index e85d3bf..07e32d8 100644 --- a/aw-server/aw-worktime-api.service +++ b/aw-server/aw-worktime-api.service @@ -8,7 +8,7 @@ StartLimitIntervalSec=60 [Service] Type=simple EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-api.py +ExecStart=/usr/local/bin/aw-worktime-api-rust Restart=on-failure RestartSec=5 User=activitywatch diff --git a/aw-server/aw-worktime-autoheal.service b/aw-server/aw-worktime-autoheal.service index e5bd3ec..37244ff 100644 --- a/aw-server/aw-worktime-autoheal.service +++ b/aw-server/aw-worktime-autoheal.service @@ -7,10 +7,9 @@ Wants=network-online.target Type=oneshot Environment=AW_URL=http://127.0.0.1:5600 Environment=AW_WORKTIME_HOST=SHARKON2025 -ExecStart=/usr/local/bin/aw-worktime-autoheal.sh +ExecStart=/usr/local/bin/aw-worktime-autoheal-rust User=root Group=root StandardOutput=journal StandardError=journal SyslogIdentifier=aw-worktime-autoheal - diff --git a/aw-server/aw-worktime-autoheal.sh b/aw-server/aw-worktime-autoheal.sh deleted file mode 100644 index d027072..0000000 --- a/aw-server/aw-worktime-autoheal.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -AW_URL="${AW_URL:-http://127.0.0.1:5600}" -HOST="${AW_WORKTIME_HOST:-SHARKON2025}" -PYTHON_BIN="${PYTHON_BIN:-/usr/bin/python3}" -WORKTIME_HEALTH_URL="${WORKTIME_HEALTH_URL:-http://127.0.0.1:5610/health}" -WORKTIME_REPORT_TIMEOUT_SECONDS="${WORKTIME_REPORT_TIMEOUT_SECONDS:-20}" -WORKTIME_MANAGEMENT_WARM_ENABLED="${WORKTIME_MANAGEMENT_WARM_ENABLED:-1}" -WORKTIME_MANAGEMENT_WARM_URL="${WORKTIME_MANAGEMENT_WARM_URL:-http://127.0.0.1:5610/reports/worktime/management?day=today&format=json}" -WORKTIME_MANAGEMENT_WARM_TIMEOUT_SECONDS="${WORKTIME_MANAGEMENT_WARM_TIMEOUT_SECONDS:-60}" -WORKTIME_TODAY_PROBE_ENABLED="${WORKTIME_TODAY_PROBE_ENABLED:-1}" -WORKTIME_TODAY_PROBE_URL="${WORKTIME_TODAY_PROBE_URL:-http://127.0.0.1:5610/reports/worktime/today?day=today&format=json}" -WORKTIME_TODAY_PROBE_TIMEOUT_SECONDS="${WORKTIME_TODAY_PROBE_TIMEOUT_SECONDS:-20}" -WORKTIME_SESSION_FRESHNESS_SECONDS="${WORKTIME_SESSION_FRESHNESS_SECONDS:-600}" -LOG_TAG="aw-worktime-autoheal" - -log() { - logger -t "$LOG_TAG" "$*" - printf '%s %s\n' "$(date '+%F %T')" "$*" -} - -probe_url() { - local url="$1" - local timeout="$2" - curl -fsS --max-time "$timeout" "$url" >/dev/null 2>&1 -} - -probe_reports() { - probe_url "$WORKTIME_HEALTH_URL" "$WORKTIME_REPORT_TIMEOUT_SECONDS" || return 1 - if [[ "$WORKTIME_TODAY_PROBE_ENABLED" == "1" ]]; then - probe_url "$WORKTIME_TODAY_PROBE_URL" "$WORKTIME_TODAY_PROBE_TIMEOUT_SECONDS" || return 1 - fi -} - -worktime_ok=1 -if ! probe_reports; then - log "worktime API probe failed, restarting aw-worktime-api.service" - systemctl restart aw-worktime-api.service || true - sleep 2 - if ! probe_reports; then - log "worktime API still degraded after restart" - worktime_ok=0 - else - log "worktime API recovered after restart" - fi -fi - -if [[ "$worktime_ok" != "1" ]]; then - log "skip warm/heal because worktime API is still unavailable" - exit 0 -fi - -if [[ "$WORKTIME_MANAGEMENT_WARM_ENABLED" == "1" ]]; then - if probe_url "$WORKTIME_MANAGEMENT_WARM_URL" "$WORKTIME_MANAGEMENT_WARM_TIMEOUT_SECONDS"; then - log "management cache warm ok" - else - log "management cache warm failed" - fi -fi - -need_heal="$("$PYTHON_BIN" - <<'PY' -import json, urllib.request, datetime, os, sys -AW=os.environ.get("AW_URL","http://127.0.0.1:5600") -host=os.environ.get("HOST","SHARKON2025") -window_bucket=f"aw-rdp-window_{host}" -session_bucket=f"aw-worktime-sessions_{host}" -freshness=int(os.environ.get("WORKTIME_SESSION_FRESHNESS_SECONDS","600")) -msk=datetime.timezone(datetime.timedelta(hours=3)) -start=datetime.datetime.now(msk).replace(hour=0,minute=0,second=0,microsecond=0).astimezone(datetime.timezone.utc) -now=datetime.datetime.now(datetime.timezone.utc) - -def p(ts): - if ts.endswith("Z"): ts=ts[:-1] + "+00:00" - return datetime.datetime.fromisoformat(ts).astimezone(datetime.timezone.utc) - -def is_active(d): - if isinstance(d.get("active"), bool) and d.get("active"): - return True - st=str(d.get("state","")).strip().lower() - if st in ("active","активно"): - return True - if st == "unknown": - try: - sid=int(d.get("sessionId")) - except Exception: - sid=-1 - u=str(d.get("username","")).strip().lower() - sn=str(d.get("sessionName","")).strip().lower() - if sid > 0 and u and (not u.endswith("$")) and (sn.startswith("rdp-") or sn == "console"): - return True - return False - -try: - with urllib.request.urlopen(f"{AW}/api/0/buckets/{window_bucket}/events?limit=12000", timeout=25) as r: - window_events=json.loads(r.read().decode("utf-8")) - with urllib.request.urlopen(f"{AW}/api/0/buckets/{session_bucket}/events?limit=12000", timeout=25) as r: - session_events=json.loads(r.read().decode("utf-8")) -except Exception: - print("1") - sys.exit(0) - -latest_ts=None -active_users=set() -for e in session_events: - ts=e.get("timestamp") - if not ts: - continue - try: - cur=p(ts) - except Exception: - continue - if latest_ts is None or cur > latest_ts: - latest_ts=cur - active_users.clear() - if cur == latest_ts: - d=e.get("data") or {} - u=str(d.get("username","")).strip() - if u and is_active(d): - active_users.add(u) - -if latest_ts is None or (now - latest_ts).total_seconds() > freshness: - print("0") - sys.exit(0) - -if not active_users: - print("0") - sys.exit(0) - -active=0.0 -for e in window_events: - ts=e.get("timestamp") - if not ts: - continue - try: - if p(ts) < start: - continue - except Exception: - continue - d=float(e.get("duration",0) or 0) - title=((e.get("data") or {}).get("title") or "").strip().lower() - if d > 0 and "rdp active" in title: - active += d - -print("1" if active <= 0 else "0") -PY -)" - -if [[ "$need_heal" != "1" ]]; then - log "health ok: activity present for ${HOST}, no action" - exit 0 -fi - -log "detected zero activity for ${HOST}, running heal" -systemctl restart aw-worktime-ui-bridge.timer -systemctl start aw-worktime-ui-bridge.service || true - -"$PYTHON_BIN" - <<'PY' -import json, urllib.request, datetime, os -AW=os.environ.get("AW_URL","http://127.0.0.1:5600") -host=os.environ.get("HOST","SHARKON2025") -sb=f"aw-worktime-sessions_{host}" -afk=f"aw-rdp-afk_{host}" -win=f"aw-rdp-window_{host}" -msk=datetime.timezone(datetime.timedelta(hours=3)) -start=datetime.datetime.now(msk).replace(hour=0,minute=0,second=0,microsecond=0).astimezone(datetime.timezone.utc) - -def req(method,path,payload=None): - data=None; headers={} - if payload is not None: - data=json.dumps(payload,ensure_ascii=False).encode("utf-8"); headers["Content-Type"]="application/json" - r=urllib.request.Request(AW+path,data=data,headers=headers,method=method) - try: - with urllib.request.urlopen(r,timeout=30) as resp: - body=resp.read() - return json.loads(body.decode("utf-8")) if body else None - except urllib.error.HTTPError as e: - if e.code in (304, 409): - return None - raise - -def reset_bucket(bucket_id, event_type, client, hostname): - try: - req("DELETE", f"/api/0/buckets/{bucket_id}") - except Exception: - pass - req("POST", f"/api/0/buckets/{bucket_id}", { - "client": client, - "type": event_type, - "hostname": hostname, - }) - -def parse(ts): - if ts.endswith("Z"): ts=ts[:-1]+"+00:00" - return datetime.datetime.fromisoformat(ts).astimezone(datetime.timezone.utc) - -def is_active(d): - if isinstance(d.get("active"), bool) and d.get("active"): return True - st=str(d.get("state","")).strip().lower() - if st in ("active","активно"): return True - if st=="unknown": - try: sid=int(d.get("sessionId")) - except: sid=-1 - u=str(d.get("username","")).strip().lower() - sn=str(d.get("sessionName","")).strip().lower() - if sid>0 and u and (not u.endswith("$")) and (sn.startswith("rdp-") or sn=="console"): return True - return False - -rows=req("GET",f"/api/0/buckets/{sb}/events?limit=12000") or [] -rows=[e for e in rows if e.get("timestamp") and parse(e["timestamp"])>=start] -if not rows: - raise SystemExit(0) - -# Hard normalization: drop corrupted/mixed watcher buckets and rebuild from source sessions. -reset_bucket(afk, "afkstatus", "aw-worktime-ui-bridge", host) -reset_bucket(win, "currentwindow", "aw-worktime-ui-bridge", host) - -by={} -for e in rows: - by.setdefault(e["timestamp"],[]).append(e) -keys=sorted(by.keys()) -out_afk=[]; out_win=[] -for i,ts in enumerate(keys): - cur=parse(ts) - nxt=parse(keys[i+1]) if i+10 else 10.0) - act=[] - for r in by[ts]: - d=r.get("data") or {} - u=str(d.get("username","")).strip() - if u and is_active(d): act.append(u) - act=sorted(set(act)) - active=bool(act) - out_afk.append({"timestamp":ts,"duration":dur,"data":{"status":"not-afk" if active else "afk","source":"aw-worktime-autoheal"}}) - out_win.append({"timestamp":ts,"duration":dur,"data":{"app":"RDP","title":("RDP active (%d): %s"%(len(act),", ".join(act))) if active else "RDP idle","source":"aw-worktime-autoheal"}}) - -req("POST",f"/api/0/buckets/{afk}/events",out_afk) -req("POST",f"/api/0/buckets/{win}/events",out_win) -print(f"autoheal backfill posted afk={len(out_afk)} win={len(out_win)}") -PY - -log "heal completed for ${HOST}" diff --git a/aw-server/aw-worktime-influx-exporter.py b/aw-server/aw-worktime-influx-exporter.py deleted file mode 100644 index eb2993f..0000000 --- a/aw-server/aw-worktime-influx-exporter.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import os -import sys -import urllib.request -from datetime import datetime, timezone -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-worktime-api.py") -SPEC = importlib.util.spec_from_file_location("aw_worktime_api", MODULE_PATH) -WORKTIME = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(WORKTIME) - -INFLUX_URL = os.environ.get("AW_WORKTIME_INFLUX_URL", "").strip().rstrip("/") -INFLUX_ORG = os.environ.get("AW_WORKTIME_INFLUX_ORG", "proxmox").strip() or "proxmox" -INFLUX_BUCKET = os.environ.get("AW_WORKTIME_INFLUX_BUCKET", "aw_metrics").strip() or "aw_metrics" -INFLUX_TOKEN = os.environ.get("AW_WORKTIME_INFLUX_TOKEN", "").strip() -INFLUX_ENABLED = os.environ.get("AW_WORKTIME_INFLUX_ENABLED", "").strip().lower() in {"1", "true", "yes", "on"} -HOSTS = [item.strip() for item in os.environ.get("AW_WORKTIME_INFLUX_HOSTS", WORKTIME.DEFAULT_HOST).split(",") if item.strip()] -DAYS = [item.strip() for item in os.environ.get("AW_WORKTIME_INFLUX_DAYS", "today,yesterday").split(",") if item.strip()] - - -def _escape_tag(value): - return ( - str(value or "") - .replace("\\", "\\\\") - .replace(" ", "\\ ") - .replace(",", "\\,") - .replace("=", "\\=") - ) - - -def _line(measurement, tags, fields, timestamp_ns): - tag_part = ",".join(f"{key}={_escape_tag(value)}" for key, value in sorted(tags.items()) if value is not None and value != "") - field_parts = [] - for key, value in fields.items(): - if isinstance(value, bool): - field_parts.append(f"{key}={'true' if value else 'false'}") - elif isinstance(value, int): - field_parts.append(f"{key}={value}i") - elif isinstance(value, float): - field_parts.append(f"{key}={value}") - else: - text = str(value or "").replace("\\", "\\\\").replace('"', '\\"') - field_parts.append(f'{key}="{text}"') - if not field_parts: - return "" - if tag_part: - return f"{measurement},{tag_part} {','.join(field_parts)} {timestamp_ns}" - return f"{measurement} {','.join(field_parts)} {timestamp_ns}" - - -def _timestamp_ns(dt): - return int(dt.astimezone(timezone.utc).timestamp() * 1_000_000_000) - - -def build_lines_for_day(host, report_date): - bounds, events = WORKTIME.fetch_events_for_date(host, report_date) - rows = WORKTIME.aggregate_rows(events, bounds["start"], bounds["end"], host) - hourly_rows = WORKTIME.aggregate_hourly_rows(events, bounds["start"], bounds["end"], host) - summary = WORKTIME.build_report_summary(rows) - true_active_apps = WORKTIME.build_true_active_apps(host, report_date) - - lines = [] - daily_ts = _timestamp_ns(bounds["start"]) - - for row in rows: - lines.append( - _line( - "aw_rdp_worktime_daily", - { - "host": host, - "user": row["user"], - "user_id": row["user_id"], - "report_date": report_date.isoformat(), - }, - { - "active_seconds": int(row["active_seconds"]), - "idle_seconds": int(row["idle_seconds"]), - "sessions_count": int(row["sessions_count"]), - "samples_count": int(row["samples_count"]), - "active_samples": int(row["active_samples"]), - }, - daily_ts, - ) - ) - - for row in hourly_rows: - lines.append( - _line( - "aw_rdp_worktime_hourly", - { - "host": host, - "user": row["user"], - "user_id": row["user_id"], - "report_date": row["report_date"], - "hour_local": row["hour_local"], - }, - { - "active_seconds": int(row["active_seconds"]), - }, - _timestamp_ns(WORKTIME.pts(row["bucket_start_utc"])), - ) - ) - - lines.append( - _line( - "aw_rdp_worktime_summary_daily", - { - "host": host, - "report_date": report_date.isoformat(), - }, - { - "users_count": int(summary["users_count"]), - "total_active_seconds": int(summary["total_active_seconds"]), - "top_user": summary["top_user"], - }, - daily_ts, - ) - ) - - for app in true_active_apps: - lines.append( - _line( - "aw_true_active_app_daily", - { - "host": host, - "application": app["application"], - "report_date": report_date.isoformat(), - }, - { - "proved_work_seconds": int(app.get("proved_work_seconds", 0) or 0), - "evidence_events": int(app.get("evidence_events", 0) or 0), - "proved_work_human": app.get("proved_work_human", ""), - "proved_work_hhmm": app.get("proved_work_hhmm", ""), - "last_action": app.get("last_action", ""), - "last_action_local": app.get("last_action_local", ""), - "last_action_utc": app.get("last_action_utc", ""), - }, - daily_ts, - ) - ) - return [line for line in lines if line] - - -def write_lines(lines): - if not lines: - return 0 - if not INFLUX_URL or not INFLUX_TOKEN: - raise RuntimeError("InfluxDB destination is not configured") - payload = ("\n".join(lines) + "\n").encode("utf-8") - req = urllib.request.Request( - f"{INFLUX_URL}/api/v2/write?org={INFLUX_ORG}&bucket={INFLUX_BUCKET}&precision=ns", - data=payload, - method="POST", - headers={"Authorization": f"Token {INFLUX_TOKEN}", "Content-Type": "text/plain; charset=utf-8"}, - ) - with urllib.request.urlopen(req, timeout=30) as response: - if response.status not in {204, 200}: - raise RuntimeError(f"InfluxDB write failed with status={response.status}") - return len(lines) - - -def main(): - if not INFLUX_ENABLED: - print("[aw-worktime-influx-exporter] disabled by AW_WORKTIME_INFLUX_ENABLED", file=sys.stderr) - return 0 - - lines = [] - for host in HOSTS: - for day in DAYS: - report_date = WORKTIME.resolve_report_date(day=day) - lines.extend(build_lines_for_day(host, report_date)) - written = write_lines(lines) - print(f"[aw-worktime-influx-exporter] wrote {written} points to {INFLUX_BUCKET}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/aw-server/aw-worktime-influx-exporter.service b/aw-server/aw-worktime-influx-exporter.service index 4cdf451..25ffc9f 100644 --- a/aw-server/aw-worktime-influx-exporter.service +++ b/aw-server/aw-worktime-influx-exporter.service @@ -6,10 +6,9 @@ Wants=network-online.target aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-influx-exporter.py +ExecStart=/usr/local/bin/aw-worktime-influx-exporter-rust User=activitywatch Group=activitywatch StandardOutput=journal StandardError=journal SyslogIdentifier=aw-worktime-influx-exporter - diff --git a/aw-server/aw-worktime-prewarm.service b/aw-server/aw-worktime-prewarm.service index a013770..6f3cb29 100644 --- a/aw-server/aw-worktime-prewarm.service +++ b/aw-server/aw-worktime-prewarm.service @@ -6,10 +6,9 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/local/bin/aw-worktime-prewarm.sh +ExecStart=/usr/local/bin/aw-worktime-prewarm-rust User=activitywatch Group=activitywatch StandardOutput=journal StandardError=journal SyslogIdentifier=aw-worktime-prewarm - diff --git a/aw-server/aw-worktime-ui-bridge.py b/aw-server/aw-worktime-ui-bridge.py deleted file mode 100644 index 5621fb5..0000000 --- a/aw-server/aw-worktime-ui-bridge.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import urllib.error -import urllib.request -from datetime import datetime, timedelta, timezone - - -AW_URL = os.environ.get("AW_SERVER_URL", "http://127.0.0.1:5600") -HOST = os.environ.get("AW_WORKTIME_HOST", "SHARKON2025") -STATE_PATH = os.environ.get( - "AW_WORKTIME_UI_BRIDGE_STATE", - "/var/lib/activitywatch/aw-worktime-ui-bridge-state.json", -) -TIMEOUT = float(os.environ.get("AW_WORKTIME_UI_BRIDGE_TIMEOUT", "60")) -WATCHER_FALLBACK_ENABLED = os.environ.get("AW_WORKTIME_UI_BRIDGE_WATCHER_FALLBACK", "1").strip().lower() not in { - "0", - "false", - "no", - "off", -} -WATCHER_FALLBACK_STALE_SECONDS = float(os.environ.get("AW_WORKTIME_UI_BRIDGE_WATCHER_STALE_SECONDS", "600")) - - -SESSIONS_BUCKET = f"aw-worktime-sessions_{HOST}" -AFK_BUCKET = f"aw-rdp-afk_{HOST}" -WINDOW_BUCKET = f"aw-rdp-window_{HOST}" -WATCHER_AFK_BUCKET = f"aw-watcher-afk_{HOST}" -WATCHER_WINDOW_BUCKET = f"aw-watcher-window_{HOST}" -WEB_CATEGORY_BUCKET = f"aw-detmir-web-category_{HOST}" -COLLECTOR_HEALTH_MAX_AGE_SECONDS = float(os.environ.get("AW_WORKTIME_UI_BRIDGE_COLLECTOR_HEALTH_MAX_AGE_SECONDS", "300")) -COLLECTOR_HEALTH_QUERY_LIMIT = int(os.environ.get("AW_WORKTIME_UI_BRIDGE_COLLECTOR_HEALTH_QUERY_LIMIT", "200")) -FOREGROUND_CONTEXT_CACHE_SECONDS = float(os.environ.get("AW_WORKTIME_UI_BRIDGE_FOREGROUND_CACHE_SECONDS", "900")) - - -def _req(method: str, path: str, payload=None): - data = None - headers = {} - if payload is not None: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers["Content-Type"] = "application/json" - req = urllib.request.Request(AW_URL + path, data=data, headers=headers, method=method) - with urllib.request.urlopen(req, timeout=TIMEOUT) as r: - raw = r.read() - if not raw: - return None - return json.loads(raw.decode("utf-8")) - - -def ensure_bucket(bucket_id: str, event_type: str, client: str): - payload = {"client": client, "type": event_type, "hostname": HOST} - try: - _req("POST", f"/api/0/buckets/{bucket_id}", payload) - except urllib.error.HTTPError as e: - if e.code != 304: - raise - - -def get_latest_bucket_event(bucket_id: str): - try: - events = _req("GET", f"/api/0/buckets/{bucket_id}/events?limit=1") or [] - except urllib.error.HTTPError as e: - if e.code == 404: - return None - raise - if not events: - return None - return events[0] - - -def get_latest_bucket_event_ts(bucket_id: str): - event = get_latest_bucket_event(bucket_id) - if not event: - return None - ts = event.get("timestamp") - if not ts: - return None - try: - return parse_iso_utc(ts) - except Exception: - return None - - -def bucket_needs_fallback(bucket_id: str, now_utc: datetime, stale_after_seconds: float): - latest_dt = get_latest_bucket_event_ts(bucket_id) - if latest_dt is None: - return True - return (now_utc - latest_dt).total_seconds() >= stale_after_seconds - - -def watcher_window_needs_bridge_sync(now_utc: datetime): - latest_event = get_latest_bucket_event(WATCHER_WINDOW_BUCKET) - if latest_event is None: - return True - latest_dt = get_latest_bucket_event_ts(WATCHER_WINDOW_BUCKET) - if latest_dt is None: - return True - if (now_utc - latest_dt).total_seconds() >= WATCHER_FALLBACK_STALE_SECONDS: - return True - - data = latest_event.get("data") or {} - source = str(data.get("source", "")).strip().lower() - app = str(data.get("app", "")).strip() - title = str(data.get("title", "")).strip() - if source == "aw-worktime-ui-bridge": - return True - if source != "aw-worktime-ui-bridge": - return False - if app.upper() == "RDP": - return True - if title == "RDP idle" or title.startswith("RDP active"): - return True - return False - - -def load_state(): - try: - with open(STATE_PATH, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, dict) and "last_ts" in data: - return data - except FileNotFoundError: - pass - except json.JSONDecodeError: - pass - return {"last_ts": "1970-01-01T00:00:00Z"} - - -def save_state(state): - os.makedirs(os.path.dirname(STATE_PATH), exist_ok=True) - tmp = STATE_PATH + ".tmp" - with open(tmp, "w", encoding="utf-8") as f: - json.dump(state, f, ensure_ascii=False) - os.replace(tmp, STATE_PATH) - - -def to_iso_utc(ts): - if ts.endswith("Z"): - return ts - return ts.replace("+00:00", "Z") - - -def parse_iso_utc(ts: str): - if ts.endswith("Z"): - ts = ts[:-1] + "+00:00" - return datetime.fromisoformat(ts).astimezone(timezone.utc) - - -def build_window_title(users, active_count): - if not users: - return "RDP idle" - return f"RDP active ({active_count}): " + ", ".join(users) - - -def get_latest_active_session_ids(events): - grouped = {} - for event in events: - ts = event.get("timestamp") - if not ts: - continue - grouped.setdefault(ts, []).append(event) - if not grouped: - return set() - latest_ts = max(grouped.keys(), key=lambda item: parse_iso_utc(item)) - active_session_ids = set() - for event in grouped.get(latest_ts, []): - data = event.get("data") or {} - if not _is_session_active(data): - continue - try: - active_session_ids.add(int(data.get("sessionId"))) - except Exception: - continue - return active_session_ids - - -def _normalize_foreground_context(data): - foreground_process = str(data.get("foregroundProcess", "")).strip() - foreground_title = str(data.get("foregroundTitle", "")).strip() - if not foreground_process and not foreground_title: - return None - return { - "app": foreground_process if foreground_process.endswith(".exe") else f"{foreground_process}.exe", - "title": foreground_title or foreground_process, - } - - -def get_latest_foreground_context(now_utc: datetime, active_session_ids=None, state=None): - try: - events = _req("GET", f"/api/0/buckets/{WEB_CATEGORY_BUCKET}/events?limit={COLLECTOR_HEALTH_QUERY_LIMIT}") or [] - except urllib.error.HTTPError as e: - if e.code == 404: - events = [] - else: - raise - - active_session_ids = set(active_session_ids or []) - recent_candidates = [] - for event in reversed(events): - data = event.get("data") or {} - if str(data.get("signalType", "")).strip().lower() != "collector_health": - continue - ts = event.get("timestamp") - if not ts: - continue - try: - event_dt = parse_iso_utc(ts) - except Exception: - continue - if (now_utc - event_dt).total_seconds() > COLLECTOR_HEALTH_MAX_AGE_SECONDS: - continue - normalized = _normalize_foreground_context(data) - if not normalized: - continue - session_id = data.get("sessionId") - try: - session_id = int(session_id) - except Exception: - session_id = None - recent_candidates.append((session_id, event_dt, normalized)) - - for session_id, event_dt, normalized in recent_candidates: - if active_session_ids and session_id in active_session_ids: - normalized["timestamp"] = to_iso_utc(event_dt.isoformat()) - return normalized - - if recent_candidates: - session_id, event_dt, normalized = recent_candidates[0] - normalized["timestamp"] = to_iso_utc(event_dt.isoformat()) - return normalized - - cached = (state or {}).get("last_foreground_context") - if isinstance(cached, dict): - cached_ts = str(cached.get("timestamp", "")).strip() - if cached_ts: - try: - cached_dt = parse_iso_utc(cached_ts) - except Exception: - cached_dt = None - if cached_dt and (now_utc - cached_dt).total_seconds() <= FOREGROUND_CONTEXT_CACHE_SECONDS: - app = str(cached.get("app", "")).strip() - title = str(cached.get("title", "")).strip() - if app or title: - return { - "app": app or "RDP", - "title": title or app or "RDP", - "timestamp": cached_ts, - } - return None - - -def _is_session_active(row_data): - if isinstance(row_data.get("active"), bool): - if row_data.get("active"): - return True - state = str(row_data.get("state", "")).strip().lower() - if state in {"active", "активно"}: - return True - # query user can intermittently return Unknown on RDP hosts. - if state == "unknown": - try: - sid = int(row_data.get("sessionId")) - except Exception: - sid = -1 - user = str(row_data.get("username", "")).strip().lower() - session_name = str(row_data.get("sessionName", "")).strip().lower() - if sid > 0 and user and (not user.endswith("$")) and (session_name.startswith("rdp-") or session_name == "console"): - return True - return False - - -def transform(events, foreground_context=None): - out_afk = [] - out_win = [] - last_ts = None - - grouped = {} - for e in events: - ts = e.get("timestamp") - if not ts: - continue - try: - normalized_ts = to_iso_utc(parse_iso_utc(ts).replace(microsecond=0).isoformat()) - except Exception: - normalized_ts = ts - grouped.setdefault(normalized_ts, []).append(e) - - ordered_ts = sorted(grouped.keys()) - parsed_ts = {} - for ts in ordered_ts: - try: - parsed_ts[ts] = parse_iso_utc(ts) - except Exception: - parsed_ts[ts] = None - - for idx, ts in enumerate(ordered_ts): - rows = grouped[ts] - src_duration = max(float(r.get("duration", 0.0)) for r in rows) - duration = src_duration - cur_dt = parsed_ts.get(ts) - next_dt = parsed_ts.get(ordered_ts[idx + 1]) if idx + 1 < len(ordered_ts) else None - next_gap = None - if cur_dt and next_dt: - next_gap = max(0.0, (next_dt - cur_dt).total_seconds()) - if duration <= 0: - if cur_dt and next_dt: - duration = next_gap or 0.0 - if duration <= 0: - duration = 10.0 - elif next_gap is not None and next_gap > 0: - # Do not let a sampled session interval extend past the next sample. - duration = min(duration, next_gap) - duration = min(duration, 30.0) - active_users = [] - for r in rows: - data = r.get("data") or {} - user = str(data.get("username", "")).strip() - if user and _is_session_active(data): - active_users.append(user) - active_users = sorted(set(active_users)) - active_count = len(active_users) - is_active = active_count > 0 - - afk_data = {"status": "not-afk" if is_active else "afk", "source": "aw-worktime-ui-bridge"} - out_afk.append({"timestamp": ts, "duration": duration, "data": afk_data}) - - if is_active and foreground_context: - title = str(foreground_context.get("title") or "").strip() - if active_count > 1: - title = f"{title} | {build_window_title(active_users, active_count)}" if title else build_window_title(active_users, active_count) - win_data = { - "app": str(foreground_context.get("app") or "RDP"), - "title": title or build_window_title(active_users, active_count), - "source": "aw-worktime-ui-bridge", - } - else: - win_data = { - "app": "RDP", - "title": build_window_title(active_users, active_count), - "source": "aw-worktime-ui-bridge", - } - out_win.append({"timestamp": ts, "duration": duration, "data": win_data}) - last_ts = ts - - return out_afk, out_win, last_ts - - -def normalize_watcher_window_events(win_events): - normalized = [] - for event in win_events: - cloned = dict(event) - data = dict(event.get("data") or {}) - app = str(data.get("app") or "").strip() - title = str(data.get("title") or "").strip() - if not app or app.upper() == "RDP": - continue - if app and app.upper() != "RDP" and " | RDP active (" in title: - data["title"] = title.split(" | RDP active (", 1)[0].strip() - cloned["data"] = data - normalized.append(cloned) - return normalized - - -def main(): - state = load_state() - last_ts = state.get("last_ts", "1970-01-01T00:00:00Z") - - ensure_bucket(AFK_BUCKET, "afkstatus", "aw-worktime-ui-bridge") - ensure_bucket(WINDOW_BUCKET, "currentwindow", "aw-worktime-ui-bridge") - - now_utc = datetime.now(timezone.utc) - recent = _req("GET", f"/api/0/buckets/{SESSIONS_BUCKET}/events?limit=5000") or [] - if not recent: - return - - try: - last_dt = parse_iso_utc(last_ts) - except Exception: - last_dt = datetime(1970, 1, 1, tzinfo=timezone.utc) - - events = [] - for e in recent: - ts = e.get("timestamp") - if not ts: - continue - try: - if parse_iso_utc(ts) > last_dt: - events.append(e) - except Exception: - continue - if not events: - return - - active_session_ids = get_latest_active_session_ids(events) - foreground_context = get_latest_foreground_context(now_utc, active_session_ids=active_session_ids, state=state) - afk_events, win_events, new_last_ts = transform(events, foreground_context=foreground_context) - if not afk_events or not win_events or not new_last_ts: - return - watcher_win_events = normalize_watcher_window_events(win_events) - - _req("POST", f"/api/0/buckets/{AFK_BUCKET}/events", afk_events) - _req("POST", f"/api/0/buckets/{WINDOW_BUCKET}/events", win_events) - if WATCHER_FALLBACK_ENABLED: - if bucket_needs_fallback(WATCHER_AFK_BUCKET, now_utc, WATCHER_FALLBACK_STALE_SECONDS): - ensure_bucket(WATCHER_AFK_BUCKET, "afkstatus", "aw-watcher-afk") - _req("POST", f"/api/0/buckets/{WATCHER_AFK_BUCKET}/events", afk_events) - if watcher_win_events and watcher_window_needs_bridge_sync(now_utc): - ensure_bucket(WATCHER_WINDOW_BUCKET, "currentwindow", "aw-watcher-window") - _req("POST", f"/api/0/buckets/{WATCHER_WINDOW_BUCKET}/events", watcher_win_events) - next_state = {"last_ts": new_last_ts} - if foreground_context: - next_state["last_foreground_context"] = { - "app": str(foreground_context.get("app") or ""), - "title": str(foreground_context.get("title") or ""), - "timestamp": str(foreground_context.get("timestamp") or to_iso_utc(now_utc.isoformat())), - } - elif isinstance(state.get("last_foreground_context"), dict): - next_state["last_foreground_context"] = state["last_foreground_context"] - save_state(next_state) - print(f"posted_afk={len(afk_events)} posted_win={len(win_events)} last_ts={new_last_ts}") - - -if __name__ == "__main__": - main() diff --git a/aw-server/aw-worktime-ui-bridge.service b/aw-server/aw-worktime-ui-bridge.service index 3ca931d..b836e4f 100644 --- a/aw-server/aw-worktime-ui-bridge.service +++ b/aw-server/aw-worktime-ui-bridge.service @@ -9,7 +9,7 @@ StartLimitIntervalSec=120 Type=simple Environment=AW_SERVER_URL=http://127.0.0.1:5600 Environment=AW_WORKTIME_HOST=SHARKON2025 -ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-ui-bridge.py +ExecStart=/usr/local/bin/aw-worktime-ui-bridge-rust Restart=on-failure RestartSec=10 User=activitywatch diff --git a/aw-server/dlp-case-management/case-service.service b/aw-server/dlp-case-management/case-service.service index 054e7be..65fa604 100644 --- a/aw-server/dlp-case-management/case-service.service +++ b/aw-server/dlp-case-management/case-service.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple EnvironmentFile=-/etc/activitywatch/aw-server.env -WorkingDirectory=/opt/activitywatch/dlp-case-management -ExecStart=/opt/activitywatch/dlp-case-management/.venv/bin/uvicorn case_service:APP --host ${AW_DLP_CASE_BIND_HOST} --port ${AW_DLP_CASE_PORT} +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/aw-dlp-case-management-rust Restart=on-failure RestartSec=3 User=activitywatch diff --git a/aw-server/dlp-case-management/case_rules.py b/aw-server/dlp-case-management/case_rules.py deleted file mode 100644 index 899d0c2..0000000 --- a/aw-server/dlp-case-management/case_rules.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - - -def is_self_test_case(incident_id: str | None, title: str | None) -> bool: - incident = str(incident_id or "").lower() - label = str(title or "").lower() - return "|self_test|" in incident or label.startswith("dlp self_test") diff --git a/aw-server/dlp-case-management/case_schema.py b/aw-server/dlp-case-management/case_schema.py deleted file mode 100644 index ec6493d..0000000 --- a/aw-server/dlp-case-management/case_schema.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -from datetime import datetime -from typing import Literal - -from pydantic import BaseModel, Field - -CaseStatus = Literal["open", "investigating", "resolved", "closed"] - - -class CaseHayabusaLink(BaseModel): - tool: Literal["hayabusa"] = "hayabusa" - host: str = Field(min_length=1, max_length=128) - mode: str = Field(min_length=1, max_length=32) - status: str = Field(min_length=1, max_length=64) - intake_id: str | None = Field(default=None, max_length=256) - package_path: str | None = Field(default=None, max_length=1024) - sha256: str | None = Field(default=None, max_length=128) - report_dir: str | None = Field(default=None, max_length=1024) - summary_html: str | None = Field(default=None, max_length=1024) - timeline_path: str | None = Field(default=None, max_length=1024) - manifest_path: str | None = Field(default=None, max_length=1024) - linked_at: str | None = Field(default=None, max_length=64) - link_source: str | None = Field(default=None, max_length=64) - - -class CaseCreate(BaseModel): - incident_id: str = Field(min_length=1, max_length=256) - host: str | None = Field(default=None, max_length=128) - title: str = Field(min_length=1, max_length=512) - severity: str = Field(default="medium", max_length=32) - assignee: str | None = Field(default=None, max_length=128) - source_bucket: str | None = Field(default=None, max_length=256) - source_event_ts: str | None = Field(default=None, max_length=64) - evidence: dict | None = None - - -class CaseUpdate(BaseModel): - status: CaseStatus | None = None - assignee: str | None = Field(default=None, max_length=128) - title: str | None = Field(default=None, max_length=512) - severity: str | None = Field(default=None, max_length=32) - - -class CaseCommentCreate(BaseModel): - comment: str = Field(min_length=1, max_length=2000) - author: str | None = Field(default=None, max_length=128) - - -class CaseComment(BaseModel): - id: int - case_id: int - comment: str - author: str | None - created_at: datetime - - -class CaseRecord(BaseModel): - id: int - incident_id: str - host: str | None - title: str - severity: str - assignee: str | None - status: CaseStatus - source_bucket: str | None - source_event_ts: str | None - evidence: dict | None - forensics: dict | None - created_at: datetime - updated_at: datetime diff --git a/aw-server/dlp-case-management/case_service.py b/aw-server/dlp-case-management/case_service.py deleted file mode 100644 index 8663a9c..0000000 --- a/aw-server/dlp-case-management/case_service.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -from fastapi import FastAPI, HTTPException, Query -from fastapi.middleware.cors import CORSMiddleware - -from case_rules import is_self_test_case -from case_schema import CaseCommentCreate, CaseCreate, CaseHayabusaLink, CaseUpdate -from case_storage import CaseStorage, ForensicsHostMismatchError - -DB = Path(os.environ.get("AW_DLP_CASE_DB_PATH", "/opt/activitywatch/dlp-case-management/cases.db")) -APP = FastAPI(title="AWatch DLP Case Management") -APP.add_middleware( - CORSMiddleware, - allow_origins=["http://127.0.0.1:5600", "http://localhost:5600", "http://10.10.10.13:5600", "*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) -STORE = CaseStorage(DB) - - -@APP.get("/health") -def health() -> dict[str, Any]: - return {"ok": True, "db": str(DB)} - - -@APP.post("/api/0/dlp/cases") -def create_case(payload: CaseCreate) -> dict[str, Any]: - if is_self_test_case(payload.incident_id, payload.title): - raise HTTPException(status_code=422, detail="self_test cases are not allowed") - return STORE.create_case(payload.model_dump(exclude_none=True), actor="api") - - -@APP.get("/api/0/dlp/cases") -def list_cases( - status: str | None = Query(default=None), - host: str | None = Query(default=None), - limit: int = Query(default=200, ge=1, le=2000), -) -> list[dict[str, Any]]: - return STORE.list_cases(status=status, host=host, limit=limit) - - -@APP.get("/api/0/dlp/cases/{case_id}") -def get_case(case_id: int) -> dict[str, Any]: - try: - case = STORE.get_case(case_id) - except KeyError: - raise HTTPException(status_code=404, detail="case not found") - case["comments"] = STORE.list_comments(case_id, limit=200) - case["audit"] = STORE.list_audit(case_id, limit=200) - return case - - -@APP.patch("/api/0/dlp/cases/{case_id}") -def update_case(case_id: int, payload: CaseUpdate) -> dict[str, Any]: - patch = payload.model_dump(exclude_none=True) - if not patch: - return STORE.get_case(case_id) - try: - return STORE.update_case(case_id, patch=patch, actor="api") - except KeyError: - raise HTTPException(status_code=404, detail="case not found") - - -@APP.post("/api/0/dlp/cases/{case_id}/comments") -def add_comment(case_id: int, payload: CaseCommentCreate) -> dict[str, Any]: - try: - STORE.get_case(case_id) - except KeyError: - raise HTTPException(status_code=404, detail="case not found") - return STORE.add_comment(case_id=case_id, comment=payload.comment, author=payload.author) - - -@APP.get("/api/0/dlp/cases/{case_id}/comments") -def list_comments(case_id: int, limit: int = Query(default=200, ge=1, le=2000)) -> list[dict[str, Any]]: - return STORE.list_comments(case_id=case_id, limit=limit) - - -@APP.post("/api/0/dlp/cases/{case_id}/forensics/hayabusa") -def link_hayabusa(case_id: int, payload: CaseHayabusaLink) -> dict[str, Any]: - try: - return STORE.link_hayabusa(case_id=case_id, payload=payload.model_dump(exclude_none=True), actor="api") - except ForensicsHostMismatchError as exc: - raise HTTPException(status_code=409, detail=str(exc)) - except KeyError: - raise HTTPException(status_code=404, detail="case not found") diff --git a/aw-server/dlp-case-management/case_storage.py b/aw-server/dlp-case-management/case_storage.py deleted file mode 100644 index 3b3f8a4..0000000 --- a/aw-server/dlp-case-management/case_storage.py +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import json -import sqlite3 -from contextlib import contextmanager -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Iterator - -from evidence_chain import evidence_sha256, normalize_evidence_chain - - -class ForensicsHostMismatchError(ValueError): - pass - - -class CaseStorage: - def __init__(self, db_path: Path) -> None: - self.db_path = db_path - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init_db() - - @contextmanager - def conn(self) -> Iterator[sqlite3.Connection]: - c = sqlite3.connect(self.db_path) - c.row_factory = sqlite3.Row - c.execute("PRAGMA journal_mode=WAL") - c.execute("PRAGMA foreign_keys=ON") - try: - yield c - finally: - c.close() - - def _init_db(self) -> None: - with self.conn() as c: - c.executescript( - """ - CREATE TABLE IF NOT EXISTS cases ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - incident_id TEXT NOT NULL, - host TEXT, - title TEXT NOT NULL, - severity TEXT NOT NULL DEFAULT 'medium', - assignee TEXT, - status TEXT NOT NULL DEFAULT 'open', - source_bucket TEXT, - source_event_ts TEXT, - evidence_json TEXT, - forensics_json TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_cases_incident_id ON cases(incident_id); - CREATE INDEX IF NOT EXISTS idx_cases_status ON cases(status); - - CREATE TABLE IF NOT EXISTS case_comments ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - case_id INTEGER NOT NULL, - comment TEXT NOT NULL, - author TEXT, - created_at TEXT NOT NULL, - FOREIGN KEY(case_id) REFERENCES cases(id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS case_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - case_id INTEGER NOT NULL, - action TEXT NOT NULL, - actor TEXT, - details_json TEXT, - created_at TEXT NOT NULL, - FOREIGN KEY(case_id) REFERENCES cases(id) ON DELETE CASCADE - ); - """ - ) - self._ensure_column(c, "cases", "forensics_json", "TEXT") - c.commit() - - @staticmethod - def _ensure_column(c: sqlite3.Connection, table: str, column: str, definition: str) -> None: - columns = { - str(row["name"]) - for row in c.execute(f"PRAGMA table_info({table})").fetchall() - } - if column not in columns: - c.execute(f"ALTER TABLE {table} ADD COLUMN {column} {definition}") - - @staticmethod - def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - @staticmethod - def _normalize_host(value: Any) -> str: - return str(value or "").strip().lower() - - @staticmethod - def _load_json_field(raw: Any) -> dict[str, Any] | None: - if not raw: - return None - try: - return json.loads(raw) - except Exception: - return None - - @classmethod - def _to_case_dict(cls, row: sqlite3.Row) -> dict[str, Any]: - evidence = cls._load_json_field(row["evidence_json"]) - forensics = cls._load_json_field(row["forensics_json"]) - return { - "id": int(row["id"]), - "incident_id": row["incident_id"], - "host": row["host"], - "title": row["title"], - "severity": row["severity"], - "assignee": row["assignee"], - "status": row["status"], - "source_bucket": row["source_bucket"], - "source_event_ts": row["source_event_ts"], - "evidence": evidence, - "forensics": forensics, - "created_at": row["created_at"], - "updated_at": row["updated_at"], - } - - def create_case(self, payload: dict[str, Any], actor: str | None = None) -> dict[str, Any]: - now = self._now() - normalized_evidence = None - evidence_digest = None - if payload.get("evidence") is not None: - normalized_evidence = normalize_evidence_chain( - payload=payload.get("evidence"), - source_bucket=payload.get("source_bucket"), - source_event_ts=payload.get("source_event_ts"), - ) - evidence_digest = normalized_evidence.get("latest_sha256") or evidence_sha256(payload.get("evidence")) - with self.conn() as c: - existing = c.execute( - """ - SELECT * FROM cases - WHERE incident_id = ? AND COALESCE(host, '') = COALESCE(?, '') - ORDER BY id DESC - LIMIT 1 - """, - (payload["incident_id"], payload.get("host")), - ).fetchone() - if existing: - return self._to_case_dict(existing) - cur = c.execute( - """ - INSERT INTO cases ( - incident_id, host, title, severity, assignee, status, - source_bucket, source_event_ts, evidence_json, forensics_json, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, 'open', ?, ?, ?, ?, ?, ?) - """, - ( - payload["incident_id"], - payload.get("host"), - payload["title"], - payload.get("severity", "medium"), - payload.get("assignee"), - payload.get("source_bucket"), - payload.get("source_event_ts"), - json.dumps(normalized_evidence, ensure_ascii=False) if normalized_evidence is not None else None, - None, - now, - now, - ), - ) - case_id = int(cur.lastrowid) - self._insert_audit( - c, - case_id=case_id, - action="create", - actor=actor, - details={ - "fields": {k: v for k, v in payload.items() if k != "evidence"}, - "evidence_sha256": evidence_digest, - }, - ) - c.commit() - return self.get_case(case_id, c) - - def list_cases(self, status: str | None = None, host: str | None = None, limit: int = 200) -> list[dict[str, Any]]: - q = "SELECT * FROM cases" - clauses = [] - args: list[Any] = [] - if status: - clauses.append("status = ?") - args.append(status) - if host: - clauses.append("host = ?") - args.append(host) - if clauses: - q += " WHERE " + " AND ".join(clauses) - q += " ORDER BY id DESC LIMIT ?" - args.append(int(limit)) - with self.conn() as c: - rows = c.execute(q, args).fetchall() - return [self._to_case_dict(r) for r in rows] - - def get_case(self, case_id: int, c: sqlite3.Connection | None = None) -> dict[str, Any]: - own = False - if c is None: - own = True - c = sqlite3.connect(self.db_path) - c.row_factory = sqlite3.Row - try: - row = c.execute("SELECT * FROM cases WHERE id = ?", (int(case_id),)).fetchone() - if not row: - raise KeyError(case_id) - return self._to_case_dict(row) - finally: - if own: - c.close() - - def update_case(self, case_id: int, patch: dict[str, Any], actor: str | None = None) -> dict[str, Any]: - fields = [] - args: list[Any] = [] - for key in ("status", "assignee", "title", "severity"): - if key in patch and patch[key] is not None: - fields.append(f"{key} = ?") - args.append(patch[key]) - if not fields: - return self.get_case(case_id) - fields.append("updated_at = ?") - args.append(self._now()) - args.append(int(case_id)) - with self.conn() as c: - c.execute(f"UPDATE cases SET {', '.join(fields)} WHERE id = ?", args) - self._insert_audit(c, case_id=case_id, action="update", actor=actor, details=patch) - c.commit() - return self.get_case(case_id, c) - - def link_hayabusa(self, case_id: int, payload: dict[str, Any], actor: str | None = None) -> dict[str, Any]: - now = self._now() - with self.conn() as c: - existing = self.get_case(case_id, c) - case_host = self._normalize_host(existing.get("host")) - forensic_host = self._normalize_host(payload.get("host")) - if case_host and forensic_host and case_host != forensic_host: - raise ForensicsHostMismatchError( - f"hayabusa host mismatch: case host={existing.get('host')} payload host={payload.get('host')}" - ) - forensics = existing.get("forensics") or {} - forensics["hayabusa"] = { - "tool": "hayabusa", - "host": payload["host"], - "mode": payload["mode"], - "status": payload["status"], - "intake_id": payload.get("intake_id"), - "package_path": payload.get("package_path"), - "sha256": payload.get("sha256"), - "report_dir": payload.get("report_dir"), - "summary_html": payload.get("summary_html"), - "timeline_path": payload.get("timeline_path"), - "manifest_path": payload.get("manifest_path"), - "linked_at": payload.get("linked_at") or now, - "link_source": payload.get("link_source") or "api", - } - c.execute( - "UPDATE cases SET forensics_json = ?, updated_at = ? WHERE id = ?", - (json.dumps(forensics, ensure_ascii=False), now, int(case_id)), - ) - self._insert_audit( - c, - case_id=case_id, - action="link_hayabusa", - actor=actor, - details={ - "host": payload["host"], - "mode": payload["mode"], - "status": payload["status"], - "intake_id": payload.get("intake_id"), - "report_dir": payload.get("report_dir"), - }, - ) - c.commit() - return self.get_case(case_id, c) - - def add_comment(self, case_id: int, comment: str, author: str | None = None) -> dict[str, Any]: - now = self._now() - with self.conn() as c: - cur = c.execute( - "INSERT INTO case_comments (case_id, comment, author, created_at) VALUES (?, ?, ?, ?)", - (int(case_id), comment, author, now), - ) - cid = int(cur.lastrowid) - self._insert_audit( - c, - case_id=case_id, - action="comment", - actor=author, - details={"comment_id": cid}, - ) - c.commit() - row = c.execute("SELECT id, case_id, comment, author, created_at FROM case_comments WHERE id = ?", (cid,)).fetchone() - return dict(row) - - def list_comments(self, case_id: int, limit: int = 200) -> list[dict[str, Any]]: - with self.conn() as c: - rows = c.execute( - "SELECT id, case_id, comment, author, created_at FROM case_comments WHERE case_id = ? ORDER BY id DESC LIMIT ?", - (int(case_id), int(limit)), - ).fetchall() - return [dict(r) for r in rows] - - def list_audit(self, case_id: int, limit: int = 200) -> list[dict[str, Any]]: - with self.conn() as c: - rows = c.execute( - "SELECT id, case_id, action, actor, details_json, created_at FROM case_audit WHERE case_id = ? ORDER BY id DESC LIMIT ?", - (int(case_id), int(limit)), - ).fetchall() - out: list[dict[str, Any]] = [] - for r in rows: - details = None - if r["details_json"]: - try: - details = json.loads(r["details_json"]) - except Exception: - details = None - out.append( - { - "id": int(r["id"]), - "case_id": int(r["case_id"]), - "action": r["action"], - "actor": r["actor"], - "details": details, - "created_at": r["created_at"], - } - ) - return out - - def _insert_audit( - self, - c: sqlite3.Connection, - case_id: int, - action: str, - actor: str | None, - details: dict[str, Any] | None = None, - ) -> None: - c.execute( - "INSERT INTO case_audit (case_id, action, actor, details_json, created_at) VALUES (?, ?, ?, ?, ?)", - ( - int(case_id), - action, - actor, - json.dumps(details, ensure_ascii=False) if details is not None else None, - self._now(), - ), - ) diff --git a/aw-server/dlp-case-management/evidence_chain.py b/aw-server/dlp-case-management/evidence_chain.py deleted file mode 100644 index c8c4294..0000000 --- a/aw-server/dlp-case-management/evidence_chain.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from typing import Any - - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _canonical_json(payload: Any) -> str: - return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - - -def evidence_sha256(payload: Any) -> str: - return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest() - - -def build_evidence_record( - payload: Any, - source_bucket: str | None = None, - source_event_ts: str | None = None, -) -> dict[str, Any]: - return { - "recorded_at": _utc_now(), - "source_bucket": source_bucket, - "source_event_ts": source_event_ts, - "sha256": evidence_sha256(payload), - "payload": payload, - } - - -def normalize_evidence_chain( - payload: Any, - source_bucket: str | None = None, - source_event_ts: str | None = None, -) -> dict[str, Any]: - if isinstance(payload, dict) and isinstance(payload.get("items"), list): - return payload - record = build_evidence_record( - payload=payload, - source_bucket=source_bucket, - source_event_ts=source_event_ts, - ) - return { - "items": [record], - "latest_sha256": record["sha256"], - "chain_length": 1, - } diff --git a/aw-server/dlp-case-management/test_case_rules.py b/aw-server/dlp-case-management/test_case_rules.py deleted file mode 100644 index 7ca1eb3..0000000 --- a/aw-server/dlp-case-management/test_case_rules.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import unittest - -from case_rules import is_self_test_case - - -class CaseRulesTest(unittest.TestCase): - def test_is_self_test_case_by_incident_id(self) -> None: - self.assertTrue(is_self_test_case("2026-05-14T17:00:52.186Z|self_test|Администратор|||", "Normal")) - - def test_is_self_test_case_by_title(self) -> None: - self.assertTrue(is_self_test_case("inc-1", "DLP self_test · Администратор")) - - def test_is_self_test_case_false_for_normal_case(self) -> None: - self.assertFalse(is_self_test_case("inc-1", "DLP print incident")) - - -if __name__ == "__main__": - unittest.main() diff --git a/aw-server/dlp-case-management/test_case_storage.py b/aw-server/dlp-case-management/test_case_storage.py deleted file mode 100644 index faf6f72..0000000 --- a/aw-server/dlp-case-management/test_case_storage.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import tempfile -import unittest -from pathlib import Path - -from case_storage import CaseStorage, ForensicsHostMismatchError - - -class CaseStorageHayabusaLinkTest(unittest.TestCase): - def test_link_hayabusa_metadata(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "cases.db" - storage = CaseStorage(db_path) - created = storage.create_case( - { - "incident_id": "inc-1", - "host": "SHARKON2025", - "title": "DLP print incident", - "severity": "high", - }, - actor="test", - ) - linked = storage.link_hayabusa( - case_id=int(created["id"]), - payload={ - "host": "SHARKON2025", - "mode": "incident", - "status": "ok", - "intake_id": "pkg-1", - "report_dir": "/opt/hayabusa/reports/SHARKON2025/run-1", - "package_path": "/opt/hayabusa/archive/packages/SHARKON2025/pkg-1.zip", - "sha256": "abc123", - "link_source": "unit-test", - }, - actor="test", - ) - hayabusa = (linked.get("forensics") or {}).get("hayabusa") or {} - self.assertEqual(hayabusa.get("tool"), "hayabusa") - self.assertEqual(hayabusa.get("host"), "SHARKON2025") - self.assertEqual(hayabusa.get("mode"), "incident") - self.assertEqual(hayabusa.get("status"), "ok") - self.assertEqual(hayabusa.get("intake_id"), "pkg-1") - self.assertEqual(hayabusa.get("link_source"), "unit-test") - audit = storage.list_audit(int(created["id"])) - self.assertTrue(any(row.get("action") == "link_hayabusa" for row in audit)) - - def test_create_case_deduplicates_by_incident_and_host(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "cases.db" - storage = CaseStorage(db_path) - payload = { - "incident_id": "2026-05-14T17:00:52.186Z|self_test|Администратор|||", - "host": "SHARKON2025", - "title": "DLP self_test · Администратор", - "severity": "medium", - } - created = storage.create_case(payload, actor="test") - duplicate = storage.create_case(payload, actor="test") - self.assertEqual(created["id"], duplicate["id"]) - cases = storage.list_cases(host="SHARKON2025", limit=10) - self.assertEqual(len(cases), 1) - - def test_link_hayabusa_rejects_host_mismatch(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "cases.db" - storage = CaseStorage(db_path) - created = storage.create_case( - { - "incident_id": "inc-2", - "host": "SHARKON2025", - "title": "DLP print incident", - "severity": "high", - }, - actor="test", - ) - with self.assertRaises(ForensicsHostMismatchError): - storage.link_hayabusa( - case_id=int(created["id"]), - payload={ - "host": "stability", - "mode": "incident", - "status": "ok", - "intake_id": "pkg-2", - "report_dir": "/opt/hayabusa/reports/stability/run-1", - }, - actor="test", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/aw-server/dlp-compliance/compliance_scheduler.py b/aw-server/dlp-compliance/compliance_scheduler.py deleted file mode 100644 index fb55a9a..0000000 --- a/aw-server/dlp-compliance/compliance_scheduler.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json - -from report_generator import generate_report - - -def main() -> None: - parser = argparse.ArgumentParser(description="Run one or more DLP compliance report profiles") - parser.add_argument("--month", help="Month in YYYY-MM format") - parser.add_argument("--profiles", default="152-fz,pci-dss", help="Comma-separated profiles to generate") - parser.add_argument("--stdout-json", action="store_true") - args = parser.parse_args() - - profiles = [item.strip() for item in str(args.profiles).split(",") if item.strip()] - results = [generate_report(month=args.month, profile=profile) for profile in profiles] - if args.stdout_json: - print(json.dumps({"items": results}, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/aw-server/dlp-compliance/report-scheduler.service b/aw-server/dlp-compliance/report-scheduler.service index d0ad7da..3f380f3 100644 --- a/aw-server/dlp-compliance/report-scheduler.service +++ b/aw-server/dlp-compliance/report-scheduler.service @@ -5,7 +5,7 @@ After=network-online.target [Service] Type=oneshot EnvironmentFile=-/etc/activitywatch/aw-server.env -WorkingDirectory=/opt/activitywatch/dlp-compliance -ExecStart=/opt/activitywatch/dlp-compliance/.venv/bin/python /opt/activitywatch/dlp-compliance/compliance_scheduler.py +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/aw-dlp-compliance-rust User=activitywatch Group=activitywatch diff --git a/aw-server/dlp-compliance/report_generator.py b/aw-server/dlp-compliance/report_generator.py deleted file mode 100644 index c7b6905..0000000 --- a/aw-server/dlp-compliance/report_generator.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from urllib.parse import quote -from urllib.request import Request, urlopen - - -def _env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def build_aw_api_base(raw_url: str | None) -> str: - url = (raw_url or "http://127.0.0.1:5600").strip().rstrip("/") - if url.endswith("/api/0"): - return url - return url + "/api/0" - - -AW_API_BASE = build_aw_api_base(_env("AW_DLP_AW_API_BASE", _env("AW_SERVER_URL", "http://127.0.0.1:5600"))) -OUTPUT_DIR = Path(_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports")) -BASE_DIR = Path(__file__).resolve().parent -PROFILE_TEMPLATE_MAP = { - "152-fz": BASE_DIR / "templates" / "152-fz-report.html", - "pci-dss": BASE_DIR / "templates" / "pci-dss-report.html", -} - - -@dataclass -class ReportStats: - total_incidents: int - high: int - medium: int - low: int - by_host: dict[str, int] - channels: dict[str, int] - - -def _http_json(url: str) -> object: - req = Request(url, headers={"Accept": "application/json"}) - with urlopen(req, timeout=30) as response: - return json.loads(response.read().decode("utf-8")) - - -def _parse_ts(value: str | None) -> datetime | None: - if not value: - return None - text = value.replace("Z", "+00:00") - try: - return datetime.fromisoformat(text).astimezone(UTC) - except ValueError: - return None - - -def _load_incidents(start: datetime, end: datetime) -> list[dict]: - buckets = _http_json(f"{AW_API_BASE}/buckets") - if not isinstance(buckets, dict): - return [] - bucket_ids = sorted([bid for bid in buckets.keys() if str(bid).startswith("aw-dlp-incidents_")]) - - incidents: list[dict] = [] - for bucket_id in bucket_ids: - encoded = quote(str(bucket_id), safe="") - events = _http_json(f"{AW_API_BASE}/buckets/{encoded}/events?limit=2000") - if not isinstance(events, list): - continue - for event in events: - if not isinstance(event, dict): - continue - ts = _parse_ts(event.get("timestamp")) - if ts is None or ts < start or ts > end: - continue - incidents.append(event) - return incidents - - -def _build_stats(incidents: list[dict]) -> ReportStats: - by_host: dict[str, int] = {} - channels: dict[str, int] = {} - high = medium = low = 0 - for event in incidents: - data = event.get("data") or {} - if not isinstance(data, dict): - data = {} - host = str(data.get("hostname") or "unknown") - by_host[host] = by_host.get(host, 0) + 1 - - severity = str(data.get("severity") or "low").lower() - if severity == "high": - high += 1 - elif severity == "medium": - medium += 1 - else: - low += 1 - - channel = str(data.get("signalType") or data.get("source") or "unknown") - channels[channel] = channels.get(channel, 0) + 1 - - return ReportStats( - total_incidents=len(incidents), - high=high, - medium=medium, - low=low, - by_host=dict(sorted(by_host.items(), key=lambda item: item[1], reverse=True)), - channels=dict(sorted(channels.items(), key=lambda item: item[1], reverse=True)), - ) - - -def _render_table(title: str, rows: list[tuple[str, int]]) -> str: - if not rows: - return f"

    {title}

    Нет данных

    " - body = "".join([f"{name}{count}" for name, count in rows]) - return f"

    {title}

    {body}
    ПараметрЗначение
    " - - -def _resolve_template_path(profile: str) -> Path: - if profile == "152-fz": - explicit = _env("AW_DLP_COMPLIANCE_TEMPLATE", str(PROFILE_TEMPLATE_MAP["152-fz"])) - return Path(explicit) - return PROFILE_TEMPLATE_MAP.get(profile, PROFILE_TEMPLATE_MAP["152-fz"]) - - -def _render_html(profile: str, period_label: str, stats: ReportStats, generated_at: str) -> str: - template = _resolve_template_path(profile).read_text(encoding="utf-8") - return ( - template.replace("{{PERIOD}}", period_label) - .replace("{{PROFILE}}", profile) - .replace("{{GENERATED_AT}}", generated_at) - .replace("{{TOTAL}}", str(stats.total_incidents)) - .replace("{{HIGH}}", str(stats.high)) - .replace("{{MEDIUM}}", str(stats.medium)) - .replace("{{LOW}}", str(stats.low)) - .replace("{{HOST_TABLE}}", _render_table("Инциденты по хостам", list(stats.by_host.items()))) - .replace("{{CHANNEL_TABLE}}", _render_table("Инциденты по каналам", list(stats.channels.items()))) - ) - - -def _period_bounds(month: str | None) -> tuple[datetime, datetime, str]: - if month: - start = datetime.fromisoformat(f"{month}-01T00:00:00+00:00").astimezone(UTC) - else: - now = datetime.now(UTC) - start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - if start.month == 12: - end = start.replace(year=start.year + 1, month=1) - else: - end = start.replace(month=start.month + 1) - end = end.replace(second=0, microsecond=0) - return start, end, start.strftime("%Y-%m") - - -def generate_report(month: str | None = None, profile: str = "152-fz") -> dict[str, object]: - start, end, period_label = _period_bounds(month) - incidents = _load_incidents(start, end) - stats = _build_stats(incidents) - - OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - generated_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") - html_out = OUTPUT_DIR / f"{profile}-{period_label}.html" - html_out.write_text(_render_html(profile, period_label, stats, generated_at), encoding="utf-8") - - metadata = { - "profile": profile, - "period": period_label, - "generated_at": generated_at, - "aw_api_base": AW_API_BASE, - "report_path": str(html_out), - "stats": { - "total_incidents": stats.total_incidents, - "high": stats.high, - "medium": stats.medium, - "low": stats.low, - }, - } - (OUTPUT_DIR / f"{profile}-{period_label}.json").write_text( - json.dumps(metadata, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - return metadata - - -def main() -> None: - parser = argparse.ArgumentParser(description="Generate DLP compliance report from AW DLP incidents") - parser.add_argument("--month", help="Month in YYYY-MM format (default: current month)") - parser.add_argument("--profile", default="152-fz", help="Profile name: 152-fz or pci-dss") - parser.add_argument("--stdout-json", action="store_true", help="Print report metadata as JSON") - args = parser.parse_args() - metadata = generate_report(month=args.month, profile=args.profile) - - if args.stdout_json: - print(json.dumps(metadata, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/aw-server/dlp-integrations/cef-exporter.service b/aw-server/dlp-integrations/cef-exporter.service index 41186a0..9c147c7 100644 --- a/aw-server/dlp-integrations/cef-exporter.service +++ b/aw-server/dlp-integrations/cef-exporter.service @@ -6,8 +6,8 @@ After=network-online.target Type=oneshot User=activitywatch Group=activitywatch -WorkingDirectory=/opt/activitywatch/dlp-integrations -ExecStart=/opt/activitywatch/dlp-integrations/.venv/bin/python /opt/activitywatch/dlp-integrations/cef_exporter.py +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/dlp-cef-exporter-rust [Install] WantedBy=multi-user.target diff --git a/aw-server/dlp-integrations/cef_exporter.py b/aw-server/dlp-integrations/cef_exporter.py deleted file mode 100644 index a21e966..0000000 --- a/aw-server/dlp-integrations/cef_exporter.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import json -import logging -import socket -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from urllib import error, request - -import yaml - -LOG = logging.getLogger("aw.dlp.cef_exporter") - - -def setup_logging() -> None: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - - -def load_yaml(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - if not isinstance(data, dict): - return {} - return data - - -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - if isinstance(data, dict): - return data - except Exception: - return {} - return {} - - -def save_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") - - -def http_json(url: str, timeout: int = 15) -> Any: - req = request.Request(url, method="GET") - with request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8", errors="ignore")) - - -def escape_cef(v: Any) -> str: - s = "" if v is None else str(v) - return s.replace("\\", "\\\\").replace("|", "\\|").replace("=", "\\=").replace("\n", "\\n").replace("\r", "") - - -def map_severity(name: str, mapping: dict[str, int]) -> int: - return int(mapping.get((name or "").lower(), 3)) - - -def build_cef(event: dict[str, Any], mapping: dict[str, int]) -> str: - data = event.get("data") or {} - sev_name = str(data.get("severity") or "low").lower() - sev_num = map_severity(sev_name, mapping) - rt = event.get("timestamp") or datetime.now(timezone.utc).isoformat() - rule = data.get("ruleId") or "dlp-incident" - msg = data.get("message") or "AWatch DLP incident" - sig = data.get("signalType") or "unknown" - host = data.get("hostname") or "unknown" - user = data.get("username") or "unknown" - action = data.get("action") or "alert" - ext = ( - f"rt={escape_cef(rt)} " - f"shost={escape_cef(host)} " - f"suser={escape_cef(user)} " - f"cs1Label=signalType cs1={escape_cef(sig)} " - f"cs2Label=action cs2={escape_cef(action)} " - f"cs3Label=ruleId cs3={escape_cef(rule)}" - ) - return ( - f"CEF:0|AWatch-rus|DLP|1.0|{escape_cef(rule)}|{escape_cef(msg)}|{sev_num}|{ext}" - ) - - -def send_syslog_udp(line: str, host: str, port: int) -> None: - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - sock.sendto(line.encode("utf-8", errors="ignore"), (host, port)) - finally: - sock.close() - - -def send_syslog_tcp(line: str, host: str, port: int, timeout: int = 10) -> None: - sock = socket.create_connection((host, port), timeout=timeout) - try: - sock.sendall((line + "\n").encode("utf-8", errors="ignore")) - finally: - sock.close() - - -def iter_new_incidents( - aw_base: str, - state: dict[str, Any], - per_bucket_limit: int, -) -> tuple[list[dict[str, Any]], dict[str, int]]: - buckets = http_json(f"{aw_base}/buckets/") - bucket_ids = sorted([bid for bid in buckets.keys() if bid.startswith("aw-dlp-incidents_")]) - last_ids = state.get("last_ids", {}) - if not isinstance(last_ids, dict): - last_ids = {} - max_ids: dict[str, int] = {} - out: list[dict[str, Any]] = [] - for bid in bucket_ids: - try: - events = http_json(f"{aw_base}/buckets/{bid}/events?limit={int(per_bucket_limit)}") - except error.HTTPError as exc: - LOG.warning("skip bucket %s: %s", bid, exc) - continue - prev = int(last_ids.get(bid, 0)) - bucket_max = prev - for ev in events: - eid = int(ev.get("id") or 0) - if eid <= prev: - continue - out.append(ev) - if eid > bucket_max: - bucket_max = eid - max_ids[bid] = bucket_max - out.sort(key=lambda x: int(x.get("id") or 0)) - return out, max_ids - - -def main() -> None: - setup_logging() - cfg_path = Path("/opt/activitywatch/dlp-integrations/cef-config.yaml") - cfg = load_yaml(cfg_path) - aw_base = str(cfg.get("aw_api_base", "http://127.0.0.1:5600/api/0")).rstrip("/") - syslog_host = str(cfg.get("syslog_host", "127.0.0.1")) - syslog_port = int(cfg.get("syslog_port", 514)) - syslog_proto = str(cfg.get("syslog_proto", "udp")).lower() - per_bucket_limit = int(cfg.get("per_bucket_limit", 300)) - state_path = Path(str(cfg.get("state_path", "/var/lib/activitywatch/dlp-integrations/cef-state.json"))) - sev_mapping = cfg.get("severity_mapping", {"low": 3, "medium": 6, "high": 10}) - if not isinstance(sev_mapping, dict): - sev_mapping = {"low": 3, "medium": 6, "high": 10} - - state = load_json(state_path) - incidents, max_ids = iter_new_incidents(aw_base=aw_base, state=state, per_bucket_limit=per_bucket_limit) - sent = 0 - for ev in incidents: - line = build_cef(ev, sev_mapping) - if syslog_proto == "tcp": - send_syslog_tcp(line, syslog_host, syslog_port) - else: - send_syslog_udp(line, syslog_host, syslog_port) - sent += 1 - state["last_ids"] = max_ids - state["updated_at"] = datetime.now(timezone.utc).isoformat() - save_json(state_path, state) - LOG.info("CEF exporter done: sent=%d buckets=%d target=%s:%d/%s", sent, len(max_ids), syslog_host, syslog_port, syslog_proto) - - -if __name__ == "__main__": - main() diff --git a/aw-server/dlp-integrations/syslog-forwarder.service b/aw-server/dlp-integrations/syslog-forwarder.service index d354551..bbd573d 100644 --- a/aw-server/dlp-integrations/syslog-forwarder.service +++ b/aw-server/dlp-integrations/syslog-forwarder.service @@ -4,7 +4,7 @@ After=network-online.target [Service] Type=oneshot -WorkingDirectory=/opt/activitywatch/dlp-integrations -ExecStart=/opt/activitywatch/dlp-integrations/.venv/bin/python /opt/activitywatch/dlp-integrations/syslog_forwarder.py +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust User=activitywatch Group=activitywatch diff --git a/aw-server/dlp-integrations/syslog_forwarder.py b/aw-server/dlp-integrations/syslog_forwarder.py deleted file mode 100644 index 4ecec71..0000000 --- a/aw-server/dlp-integrations/syslog_forwarder.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import json -import logging -import socket -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from urllib import error, request - -import yaml - -LOG = logging.getLogger("aw.dlp.syslog_forwarder") - - -def setup_logging() -> None: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - - -def load_yaml(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - return data if isinstance(data, dict) else {} - - -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return {} - return data if isinstance(data, dict) else {} - - -def save_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") - - -def http_json(url: str, timeout: int = 15) -> Any: - req = request.Request(url, method="GET") - with request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8", errors="ignore")) - - -def iter_new_incidents(aw_base: str, state: dict[str, Any], per_bucket_limit: int) -> tuple[list[dict[str, Any]], dict[str, int]]: - buckets = http_json(f"{aw_base}/buckets/") - bucket_ids = sorted([bid for bid in buckets.keys() if bid.startswith("aw-dlp-incidents_")]) - last_ids = state.get("last_ids", {}) - if not isinstance(last_ids, dict): - last_ids = {} - max_ids: dict[str, int] = {} - out: list[dict[str, Any]] = [] - for bid in bucket_ids: - try: - events = http_json(f"{aw_base}/buckets/{bid}/events?limit={int(per_bucket_limit)}") - except (TimeoutError, OSError, error.URLError, error.HTTPError) as exc: - LOG.warning("skip bucket %s: %s", bid, exc) - continue - prev = int(last_ids.get(bid, 0)) - bucket_max = prev - for ev in events: - eid = int(ev.get("id") or 0) - if eid <= prev: - continue - out.append(ev) - if eid > bucket_max: - bucket_max = eid - max_ids[bid] = bucket_max - out.sort(key=lambda x: int(x.get("id") or 0)) - return out, max_ids - - -def build_message(event: dict[str, Any], app_name: str, facility: int) -> str: - pri = facility * 8 + 6 - ts = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - data = event.get("data") or {} - host = str(data.get("hostname") or "unknown") - payload = json.dumps( - { - "event_id": event.get("id"), - "timestamp": event.get("timestamp"), - "host": host, - "severity": data.get("severity"), - "signalType": data.get("signalType"), - "username": data.get("username"), - "action": data.get("action"), - "message": data.get("message"), - "data": data, - }, - ensure_ascii=False, - separators=(",", ":"), - ) - return f"<{pri}>1 {ts} {host} {app_name} - - - {payload}" - - -def send_syslog(line: str, host: str, port: int, proto: str, timeout: int = 10) -> None: - if proto.lower() == "tcp": - sock = socket.create_connection((host, port), timeout=timeout) - try: - sock.sendall((line + "\n").encode("utf-8", errors="ignore")) - finally: - sock.close() - return - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - sock.sendto(line.encode("utf-8", errors="ignore"), (host, port)) - finally: - sock.close() - - -def main() -> None: - setup_logging() - cfg_path = Path("/opt/activitywatch/dlp-integrations/syslog-forwarder-config.yaml") - cfg = load_yaml(cfg_path) - aw_base = str(cfg.get("aw_api_base", "http://127.0.0.1:5600/api/0")).rstrip("/") - state_path = Path(str(cfg.get("state_path", "/var/lib/activitywatch/dlp-integrations/syslog-forwarder-state.json"))) - per_bucket_limit = int(cfg.get("per_bucket_limit", 300)) - syslog_host = str(cfg.get("syslog_host", "127.0.0.1")) - syslog_port = int(cfg.get("syslog_port", 514)) - syslog_proto = str(cfg.get("syslog_proto", "udp")) - facility = int(cfg.get("facility", 16)) - app_name = str(cfg.get("app_name", "aw-dlp")) - - state = load_json(state_path) - try: - incidents, max_ids = iter_new_incidents(aw_base=aw_base, state=state, per_bucket_limit=per_bucket_limit) - except (TimeoutError, OSError, error.URLError, error.HTTPError) as exc: - LOG.warning("skip syslog forwarder run: AW API unavailable: %s", exc) - return - - sent = 0 - for event in incidents: - line = build_message(event, app_name=app_name, facility=facility) - send_syslog(line=line, host=syslog_host, port=syslog_port, proto=syslog_proto) - sent += 1 - - save_json(state_path, {"last_ids": max_ids}) - LOG.info("syslog forwarder sent=%d buckets=%d", sent, len(max_ids)) - - -if __name__ == "__main__": - main() diff --git a/aw-server/dlp-integrations/test_syslog_forwarder.py b/aw-server/dlp-integrations/test_syslog_forwarder.py deleted file mode 100644 index 446cbe3..0000000 --- a/aw-server/dlp-integrations/test_syslog_forwarder.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import json -import sys -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("syslog_forwarder.py") -SPEC = importlib.util.spec_from_file_location("syslog_forwarder", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -def test_iter_new_incidents_skips_timed_out_bucket(monkeypatch): - def fake_http_json(url, timeout=15): - if url.endswith("/buckets/"): - return {"aw-dlp-incidents_SHARKON2025": {}} - raise TimeoutError("timed out") - - monkeypatch.setattr(MODULE, "http_json", fake_http_json) - - incidents, max_ids = MODULE.iter_new_incidents( - aw_base="http://127.0.0.1:5600/api/0", - state={"last_ids": {"aw-dlp-incidents_SHARKON2025": 42}}, - per_bucket_limit=300, - ) - - assert incidents == [] - assert max_ids == {} - - -def test_main_skips_aw_api_timeout_without_overwriting_state(monkeypatch, tmp_path): - state_path = tmp_path / "syslog-forwarder-state.json" - original_state = {"last_ids": {"aw-dlp-incidents_SHARKON2025": 99}} - state_path.write_text(json.dumps(original_state), encoding="utf-8") - - monkeypatch.setattr( - MODULE, - "load_yaml", - lambda path: { - "aw_api_base": "http://127.0.0.1:5600/api/0", - "state_path": str(state_path), - }, - ) - monkeypatch.setattr( - MODULE, - "iter_new_incidents", - lambda aw_base, state, per_bucket_limit: (_ for _ in ()).throw(TimeoutError("timed out")), - ) - monkeypatch.setattr( - MODULE, - "save_json", - lambda path, payload: (_ for _ in ()).throw(AssertionError("state should not be saved on AW API timeout")), - ) - - MODULE.main() - - assert json.loads(state_path.read_text(encoding="utf-8")) == original_state diff --git a/aw-server/dlp-integrations/webhook-sender.service b/aw-server/dlp-integrations/webhook-sender.service index 425dd98..5e6fa2d 100644 --- a/aw-server/dlp-integrations/webhook-sender.service +++ b/aw-server/dlp-integrations/webhook-sender.service @@ -6,8 +6,8 @@ After=network-online.target Type=oneshot User=activitywatch Group=activitywatch -WorkingDirectory=/opt/activitywatch/dlp-integrations -ExecStart=/opt/activitywatch/dlp-integrations/.venv/bin/python /opt/activitywatch/dlp-integrations/webhook_sender.py +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/dlp-webhook-sender-rust [Install] WantedBy=multi-user.target diff --git a/aw-server/dlp-integrations/webhook_sender.py b/aw-server/dlp-integrations/webhook_sender.py deleted file mode 100644 index 49ddd6c..0000000 --- a/aw-server/dlp-integrations/webhook_sender.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import json -import logging -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from urllib import error, request - -import yaml - -LOG = logging.getLogger("aw.dlp.webhook_sender") - - -def setup_logging() -> None: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - - -def load_yaml(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - if not isinstance(data, dict): - return {} - return data - - -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - if isinstance(data, dict): - return data - except Exception: - return {} - return {} - - -def save_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") - - -def http_json(url: str, timeout: int = 15) -> Any: - req = request.Request(url, method="GET") - with request.urlopen(req, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8", errors="ignore")) - - -def post_with_retry(url: str, payload: dict[str, Any], retries: int, timeout: int, backoff_base: float) -> bool: - body = json.dumps(payload, ensure_ascii=False).encode("utf-8") - headers = {"Content-Type": "application/json; charset=utf-8"} - for attempt in range(1, retries + 1): - try: - req = request.Request(url, data=body, headers=headers, method="POST") - with request.urlopen(req, timeout=timeout) as resp: - code = getattr(resp, "status", 200) - if 200 <= code < 300: - return True - except error.HTTPError as exc: - LOG.warning("webhook http error url=%s code=%s attempt=%d/%d", url, exc.code, attempt, retries) - except Exception as exc: - LOG.warning("webhook transport error url=%s err=%s attempt=%d/%d", url, exc, attempt, retries) - if attempt < retries: - time.sleep(backoff_base ** (attempt - 1)) - return False - - -def iter_new_incidents(aw_base: str, state: dict[str, Any], per_bucket_limit: int) -> tuple[list[dict[str, Any]], dict[str, int]]: - buckets = http_json(f"{aw_base}/buckets/") - bucket_ids = sorted([bid for bid in buckets.keys() if bid.startswith("aw-dlp-incidents_")]) - last_ids = state.get("last_ids", {}) - if not isinstance(last_ids, dict): - last_ids = {} - max_ids: dict[str, int] = {} - out: list[dict[str, Any]] = [] - for bid in bucket_ids: - events = http_json(f"{aw_base}/buckets/{bid}/events?limit={int(per_bucket_limit)}") - prev = int(last_ids.get(bid, 0)) - bucket_max = prev - for ev in events: - eid = int(ev.get("id") or 0) - if eid <= prev: - continue - out.append(ev) - if eid > bucket_max: - bucket_max = eid - max_ids[bid] = bucket_max - out.sort(key=lambda x: int(x.get("id") or 0)) - return out, max_ids - - -def should_send(severity: str, allowed: list[str]) -> bool: - return severity.lower() in {s.lower() for s in allowed} - - -def main() -> None: - setup_logging() - cfg_path = Path("/opt/activitywatch/dlp-integrations/webhook-config.yaml") - cfg = load_yaml(cfg_path) - aw_base = str(cfg.get("aw_api_base", "http://127.0.0.1:5600/api/0")).rstrip("/") - state_path = Path(str(cfg.get("state_path", "/var/lib/activitywatch/dlp-integrations/webhook-state.json"))) - retries = int(cfg.get("retries", 4)) - timeout = int(cfg.get("timeout_sec", 15)) - backoff_base = float(cfg.get("backoff_base", 2.0)) - per_bucket_limit = int(cfg.get("per_bucket_limit", 300)) - hooks = cfg.get("critical_webhooks", []) - if not isinstance(hooks, list): - hooks = [] - - state = load_json(state_path) - incidents, max_ids = iter_new_incidents(aw_base=aw_base, state=state, per_bucket_limit=per_bucket_limit) - - sent = 0 - for ev in incidents: - data = ev.get("data") or {} - severity = str(data.get("severity") or "low") - for hook in hooks: - if not isinstance(hook, dict): - continue - url = str(hook.get("url") or "").strip() - if not url: - continue - allowed = hook.get("severity", ["high"]) - if isinstance(allowed, str): - allowed = [allowed] - if not should_send(severity, [str(x) for x in allowed]): - continue - payload = { - "source": "AWatch-rus DLP", - "timestamp": ev.get("timestamp"), - "event_id": ev.get("id"), - "severity": severity, - "message": data.get("message"), - "ruleId": data.get("ruleId"), - "signalType": data.get("signalType"), - "hostname": data.get("hostname"), - "username": data.get("username"), - "action": data.get("action"), - "raw": data, - } - if post_with_retry(url=url, payload=payload, retries=retries, timeout=timeout, backoff_base=backoff_base): - sent += 1 - - state["last_ids"] = max_ids - state["updated_at"] = datetime.now(timezone.utc).isoformat() - save_json(state_path, state) - LOG.info("Webhook sender done: delivered=%d incidents_seen=%d", sent, len(incidents)) - - -if __name__ == "__main__": - main() diff --git a/aw-server/dlp-monitoring/metrics_exporter.py b/aw-server/dlp-monitoring/metrics_exporter.py deleted file mode 100644 index 8a52a8a..0000000 --- a/aw-server/dlp-monitoring/metrics_exporter.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -from fastapi import FastAPI -from fastapi.responses import PlainTextResponse - -app = FastAPI(title="AWatch DLP Metrics") - - -@app.get("/metrics", response_class=PlainTextResponse) -def metrics() -> str: - # Minimal exporter baseline for Prometheus scraping. - return "\n".join( - [ - "# HELP aw_dlp_exporter_up Exporter availability.", - "# TYPE aw_dlp_exporter_up gauge", - "aw_dlp_exporter_up 1", - ] - ) diff --git a/aw-server/dlp-policy-engine/dlp-policy-engine.service b/aw-server/dlp-policy-engine/dlp-policy-engine.service index 989be92..65c2c4b 100644 --- a/aw-server/dlp-policy-engine/dlp-policy-engine.service +++ b/aw-server/dlp-policy-engine/dlp-policy-engine.service @@ -8,8 +8,8 @@ StartLimitIntervalSec=60 [Service] Type=simple EnvironmentFile=/etc/activitywatch/aw-server.env -WorkingDirectory=/opt/activitywatch/dlp-policy-engine -ExecStart=/opt/activitywatch/dlp-policy-engine/.venv/bin/uvicorn policy_service:app --host ${AW_DLP_POLICY_ENGINE_BIND_HOST} --port ${AW_DLP_POLICY_ENGINE_PORT} +WorkingDirectory=/var/lib/activitywatch +ExecStart=/usr/local/bin/aw-dlp-policy-engine-rust Restart=on-failure RestartSec=5 User=activitywatch diff --git a/aw-server/dlp-policy-engine/policy_distributor.py b/aw-server/dlp-policy-engine/policy_distributor.py deleted file mode 100644 index 915bb13..0000000 --- a/aw-server/dlp-policy-engine/policy_distributor.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -from typing import Any - - -def build_policy_bundle(record: dict[str, Any] | None) -> dict[str, Any]: - if not record: - return { - "active": False, - "policyId": None, - "name": None, - "version": None, - "checksum": None, - "updatedAtUtc": None, - "policy": None, - } - - return { - "active": True, - "policyId": record["id"], - "name": record["name"], - "version": record["current_version"], - "checksum": record["checksum"], - "updatedAtUtc": record["updated_at"], - "policy": record["policy"], - } diff --git a/aw-server/dlp-policy-engine/policy_schema.py b/aw-server/dlp-policy-engine/policy_schema.py deleted file mode 100644 index 3109f4e..0000000 --- a/aw-server/dlp-policy-engine/policy_schema.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from pydantic import BaseModel, Field, ConfigDict - - -class PolicyDocument(BaseModel): - model_config = ConfigDict(extra="allow") - - version: int = 1 - defaults: dict[str, Any] = Field( - default_factory=lambda: { - "enabled": True, - "cooldownSeconds": 300, - "action": "alert", - "severity": "medium", - } - ) - endpoint: dict[str, list[dict[str, Any]]] = Field( - default_factory=lambda: { - "clipboard": [], - "usb": [], - "print": [], - } - ) - - -class PolicyCreateRequest(BaseModel): - name: str = Field(min_length=1, max_length=128) - description: str | None = Field(default=None, max_length=2048) - policy: PolicyDocument - activate: bool = False - actor: str | None = Field(default="api") - - -class PolicyUpdateRequest(BaseModel): - name: str | None = Field(default=None, min_length=1, max_length=128) - description: str | None = Field(default=None, max_length=2048) - policy: PolicyDocument | None = None - activate: bool = False - actor: str | None = Field(default="api") - - -class PolicyActivateRequest(BaseModel): - actor: str | None = Field(default="api") - - -class PolicyStatusRequest(BaseModel): - actor: str | None = Field(default="api") - comment: str | None = Field(default=None, max_length=2048) - - -class PolicyRecord(BaseModel): - id: int - name: str - description: str | None - is_active: bool - current_version: int - checksum: str - created_at: datetime - updated_at: datetime - - -class PolicyVersionRecord(BaseModel): - policy_id: int - version: int - checksum: str - created_at: datetime - created_by: str | None diff --git a/aw-server/dlp-policy-engine/policy_service.py b/aw-server/dlp-policy-engine/policy_service.py deleted file mode 100644 index e9dac58..0000000 --- a/aw-server/dlp-policy-engine/policy_service.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -from fastapi import FastAPI, HTTPException - -from policy_distributor import build_policy_bundle -from policy_schema import PolicyActivateRequest, PolicyCreateRequest, PolicyStatusRequest, PolicyUpdateRequest -from policy_storage import PolicyStorage - - -def _env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -APP_NAME = "aw-dlp-policy-engine" -APP_VERSION = "0.1.0" -DB_PATH = _env("AW_DLP_POLICY_ENGINE_DB_PATH", "/var/lib/activitywatch/dlp-policy-engine.sqlite") -storage = PolicyStorage(DB_PATH) - -app = FastAPI(title=APP_NAME, version=APP_VERSION) -AGENT_STATE: dict[str, dict[str, Any]] = {} - - -@app.get("/healthz") -def healthz() -> dict[str, str]: - return { - "status": "ok", - "service": APP_NAME, - "db_path": DB_PATH, - "db_exists": str(Path(DB_PATH).exists()).lower(), - } - - -@app.get("/api/0/dlp/policies") -def list_policies() -> dict[str, object]: - return {"items": storage.list_policies()} - - -@app.post("/api/0/dlp/policies", status_code=201) -def create_policy(payload: PolicyCreateRequest) -> dict[str, object]: - try: - item = storage.create_policy( - name=payload.name, - description=payload.description, - policy=payload.policy.model_dump(mode="json"), - activate=payload.activate, - actor=payload.actor, - ) - except Exception as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - return {"item": item} - - -@app.get("/api/0/dlp/policies/active") -def get_active_policy() -> dict[str, object]: - item = storage.get_active_policy() - if not item: - raise HTTPException(status_code=404, detail="no active policy configured") - return build_policy_bundle(item) - - -@app.get("/api/0/dlp/policies/active/version") -def get_active_policy_version() -> dict[str, object]: - item = storage.get_active_policy() - if not item: - raise HTTPException(status_code=404, detail="no active policy configured") - return { - "active": True, - "policyId": item["id"], - "version": item["current_version"], - "checksum": item["checksum"], - "updatedAtUtc": item["updated_at"], - } - - -@app.post("/api/0/dlp/policies/agents/{agent_id}/heartbeat") -def update_agent_policy_heartbeat(agent_id: str, payload: dict[str, Any]) -> dict[str, object]: - AGENT_STATE[agent_id] = { - "agentId": agent_id, - "hostname": payload.get("hostname") or agent_id, - "version": payload.get("version"), - "checksum": payload.get("checksum"), - "updatedAtUtc": payload.get("updatedAtUtc"), - } - return {"ok": True, "agent": AGENT_STATE[agent_id]} - - -@app.get("/api/0/dlp/policies/agents/{agent_id}/desired") -def get_agent_desired_policy(agent_id: str) -> dict[str, object]: - item = storage.get_active_policy() - if not item: - raise HTTPException(status_code=404, detail="no active policy configured") - - current = AGENT_STATE.get(agent_id, {}) - current_version = current.get("version") - current_checksum = current.get("checksum") - desired_version = item["current_version"] - desired_checksum = item["checksum"] - refresh_now = (str(current_version) != str(desired_version)) or (str(current_checksum) != str(desired_checksum)) - - return { - "agentId": agent_id, - "refreshNow": refresh_now, - "reason": "mismatch" if refresh_now else "up-to-date", - "current": { - "version": current_version, - "checksum": current_checksum, - }, - "desired": { - "policyId": item["id"], - "version": desired_version, - "checksum": desired_checksum, - "updatedAtUtc": item["updated_at"], - }, - } - - -@app.post("/api/0/dlp/policies/rollback") -def rollback_active_policy(payload: PolicyActivateRequest) -> dict[str, object]: - item = storage.rollback_active_policy(actor=payload.actor) - if not item: - raise HTTPException(status_code=404, detail="no active policy configured") - return {"item": item} - - -@app.get("/api/0/dlp/policies/{policy_id}") -def get_policy(policy_id: int) -> dict[str, object]: - item = storage.get_policy(policy_id) - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.put("/api/0/dlp/policies/{policy_id}") -def update_policy(policy_id: int, payload: PolicyUpdateRequest) -> dict[str, object]: - try: - item = storage.update_policy( - policy_id=policy_id, - name=payload.name, - description=payload.description, - policy=payload.policy.model_dump(mode="json") if payload.policy is not None else None, - activate=payload.activate, - actor=payload.actor, - ) - except Exception as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.post("/api/0/dlp/policies/{policy_id}/activate") -def activate_policy(policy_id: int, payload: PolicyActivateRequest) -> dict[str, object]: - try: - item = storage.activate_policy(policy_id=policy_id, actor=payload.actor) - except Exception as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.post("/api/0/dlp/policies/{policy_id}/submit") -def submit_policy_for_approval(policy_id: int, payload: PolicyStatusRequest) -> dict[str, object]: - item = storage.set_policy_status(policy_id, "pending_approval", payload.actor, payload.comment) - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.post("/api/0/dlp/policies/{policy_id}/approve") -def approve_policy(policy_id: int, payload: PolicyStatusRequest) -> dict[str, object]: - item = storage.set_policy_status(policy_id, "approved", payload.actor, payload.comment) - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.post("/api/0/dlp/policies/{policy_id}/draft") -def return_policy_to_draft(policy_id: int, payload: PolicyStatusRequest) -> dict[str, object]: - item = storage.set_policy_status(policy_id, "draft", payload.actor, payload.comment) - if not item: - raise HTTPException(status_code=404, detail="policy not found") - return {"item": item} - - -@app.delete("/api/0/dlp/policies/{policy_id}") -def delete_policy(policy_id: int) -> dict[str, bool]: - try: - deleted = storage.delete_policy(policy_id) - except ValueError as exc: - raise HTTPException(status_code=409, detail=str(exc)) from exc - - if not deleted: - raise HTTPException(status_code=404, detail="policy not found") - return {"deleted": True} - - -@app.get("/api/0/dlp/policies/audit") -def list_policy_audit(limit: int = 200) -> dict[str, object]: - return {"items": storage.list_audit(policy_id=None, limit=max(1, min(limit, 1000)))} - - -@app.get("/api/0/dlp/policies/{policy_id}/audit") -def list_single_policy_audit(policy_id: int, limit: int = 200) -> dict[str, object]: - if not storage.get_policy(policy_id): - raise HTTPException(status_code=404, detail="policy not found") - return {"items": storage.list_audit(policy_id=policy_id, limit=max(1, min(limit, 1000)))} diff --git a/aw-server/dlp-policy-engine/policy_storage.py b/aw-server/dlp-policy-engine/policy_storage.py deleted file mode 100644 index 3365579..0000000 --- a/aw-server/dlp-policy-engine/policy_storage.py +++ /dev/null @@ -1,367 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import sqlite3 -from contextlib import contextmanager -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Iterator - - -def utc_now() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def canonical_policy_json(policy: dict[str, Any]) -> str: - return json.dumps(policy, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - - -def checksum_policy(policy: dict[str, Any]) -> str: - return hashlib.sha256(canonical_policy_json(policy).encode("utf-8")).hexdigest() - - -class PolicyStorage: - def __init__(self, db_path: str) -> None: - self.db_path = Path(db_path) - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init_schema() - - @contextmanager - def connect(self) -> Iterator[sqlite3.Connection]: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - try: - yield conn - conn.commit() - finally: - conn.close() - - def _init_schema(self) -> None: - with self.connect() as conn: - conn.executescript( - """ - PRAGMA journal_mode=WAL; - - CREATE TABLE IF NOT EXISTS policies ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, - description TEXT, - status TEXT NOT NULL DEFAULT 'draft', - is_active INTEGER NOT NULL DEFAULT 0, - current_version INTEGER NOT NULL DEFAULT 1, - checksum TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS policy_versions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - policy_id INTEGER NOT NULL, - version INTEGER NOT NULL, - policy_json TEXT NOT NULL, - checksum TEXT NOT NULL, - created_at TEXT NOT NULL, - created_by TEXT, - rollback_of_version INTEGER, - FOREIGN KEY(policy_id) REFERENCES policies(id), - UNIQUE(policy_id, version) - ); - - CREATE TABLE IF NOT EXISTS policy_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - policy_id INTEGER, - action TEXT NOT NULL, - actor TEXT, - comment TEXT, - details_json TEXT, - created_at TEXT NOT NULL, - FOREIGN KEY(policy_id) REFERENCES policies(id) - ); - - CREATE INDEX IF NOT EXISTS idx_policies_active ON policies(is_active); - CREATE INDEX IF NOT EXISTS idx_policy_versions_policy ON policy_versions(policy_id, version DESC); - CREATE INDEX IF NOT EXISTS idx_policy_audit_policy ON policy_audit(policy_id, id DESC); - """ - ) - cols = [r["name"] for r in conn.execute("PRAGMA table_info(policies)").fetchall()] - if "status" not in cols: - conn.execute("ALTER TABLE policies ADD COLUMN status TEXT NOT NULL DEFAULT 'draft'") - - def _audit( - self, - conn: sqlite3.Connection, - policy_id: int | None, - action: str, - actor: str | None, - comment: str | None = None, - details: dict[str, Any] | None = None, - ) -> None: - conn.execute( - """ - INSERT INTO policy_audit(policy_id, action, actor, comment, details_json, created_at) - VALUES(?, ?, ?, ?, ?, ?) - """, - ( - policy_id, - action, - actor, - comment, - canonical_policy_json(details) if details is not None else None, - utc_now(), - ), - ) - - def list_policies(self) -> list[dict[str, Any]]: - with self.connect() as conn: - rows = conn.execute( - """ - SELECT id, name, description, status, is_active, current_version, checksum, created_at, updated_at - FROM policies - ORDER BY is_active DESC, updated_at DESC, id DESC - """ - ).fetchall() - return [dict(row) for row in rows] - - def get_policy(self, policy_id: int) -> dict[str, Any] | None: - with self.connect() as conn: - policy_row = conn.execute( - """ - SELECT id, name, description, status, is_active, current_version, checksum, created_at, updated_at - FROM policies - WHERE id = ? - """, - (policy_id,), - ).fetchone() - if not policy_row: - return None - - version_row = conn.execute( - """ - SELECT version, policy_json, checksum, created_at, created_by - FROM policy_versions - WHERE policy_id = ? AND version = ? - """, - (policy_id, policy_row["current_version"]), - ).fetchone() - if not version_row: - return None - - result = dict(policy_row) - result["policy"] = json.loads(version_row["policy_json"]) - result["version_created_at"] = version_row["created_at"] - result["version_created_by"] = version_row["created_by"] - return result - - def get_active_policy(self) -> dict[str, Any] | None: - with self.connect() as conn: - row = conn.execute("SELECT id FROM policies WHERE is_active = 1 ORDER BY updated_at DESC LIMIT 1").fetchone() - if not row: - return None - return self.get_policy(int(row["id"])) - - def create_policy(self, name: str, description: str | None, policy: dict[str, Any], activate: bool, actor: str | None) -> dict[str, Any]: - checksum = checksum_policy(policy) - now = utc_now() - status = "deployed" if activate else "draft" - policy_json = canonical_policy_json(policy) - with self.connect() as conn: - if activate: - conn.execute("UPDATE policies SET is_active = 0") - cursor = conn.execute( - """ - INSERT INTO policies(name, description, status, is_active, current_version, checksum, created_at, updated_at) - VALUES(?, ?, ?, ?, 1, ?, ?, ?) - """, - (name, description, status, 1 if activate else 0, checksum, now, now), - ) - policy_id = int(cursor.lastrowid) - conn.execute( - """ - INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) - VALUES(?, 1, ?, ?, ?, ?, NULL) - """, - (policy_id, policy_json, checksum, now, actor), - ) - self._audit(conn, policy_id, "create", actor, details={"activate": activate, "status": status}) - return self.get_policy(policy_id) # type: ignore[return-value] - - def update_policy( - self, - policy_id: int, - name: str | None, - description: str | None, - policy: dict[str, Any] | None, - activate: bool, - actor: str | None, - ) -> dict[str, Any] | None: - current = self.get_policy(policy_id) - if not current: - return None - - with self.connect() as conn: - new_name = name if name is not None else current["name"] - new_description = description if description is not None else current["description"] - new_version = int(current["current_version"]) - new_checksum = current["checksum"] - new_status = current.get("status", "draft") - - if policy is not None: - new_version += 1 - new_checksum = checksum_policy(policy) - new_status = "draft" - policy_json = canonical_policy_json(policy) - conn.execute( - """ - INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) - VALUES(?, ?, ?, ?, ?, ?, NULL) - """, - (policy_id, new_version, policy_json, new_checksum, utc_now(), actor), - ) - - if activate: - conn.execute("UPDATE policies SET is_active = 0") - new_status = "deployed" - - conn.execute( - """ - UPDATE policies - SET name = ?, description = ?, status = ?, is_active = ?, current_version = ?, checksum = ?, updated_at = ? - WHERE id = ? - """, - ( - new_name, - new_description, - new_status, - 1 if activate else current["is_active"], - new_version, - new_checksum, - utc_now(), - policy_id, - ), - ) - self._audit(conn, policy_id, "update", actor, details={"activate": activate, "status": new_status}) - return self.get_policy(policy_id) - - def activate_policy(self, policy_id: int, actor: str | None) -> dict[str, Any] | None: - current = self.get_policy(policy_id) - if not current: - return None - if current.get("status") != "approved": - raise ValueError("policy must be approved before deploy") - - with self.connect() as conn: - conn.execute("UPDATE policies SET is_active = 0") - conn.execute( - "UPDATE policies SET status = 'deployed', is_active = 1, updated_at = ? WHERE id = ?", - (utc_now(), policy_id), - ) - self._audit(conn, policy_id, "deploy", actor) - return self.get_policy(policy_id) - - def rollback_active_policy(self, actor: str | None) -> dict[str, Any] | None: - active = self.get_active_policy() - if not active: - return None - - with self.connect() as conn: - rows = conn.execute( - """ - SELECT version, policy_json - FROM policy_versions - WHERE policy_id = ? - ORDER BY version DESC - LIMIT 2 - """, - (active["id"],), - ).fetchall() - if len(rows) < 2: - return active - - previous_version = int(rows[1]["version"]) - previous_policy = json.loads(rows[1]["policy_json"]) - rollback_version = int(active["current_version"]) + 1 - rollback_checksum = checksum_policy(previous_policy) - now = utc_now() - - conn.execute( - """ - INSERT INTO policy_versions(policy_id, version, policy_json, checksum, created_at, created_by, rollback_of_version) - VALUES(?, ?, ?, ?, ?, ?, ?) - """, - ( - active["id"], - rollback_version, - canonical_policy_json(previous_policy), - rollback_checksum, - now, - actor, - previous_version, - ), - ) - conn.execute( - """ - UPDATE policies - SET status = 'draft', current_version = ?, checksum = ?, updated_at = ? - WHERE id = ? - """, - (rollback_version, rollback_checksum, now, active["id"]), - ) - self._audit(conn, int(active["id"]), "rollback", actor, details={"rollback_to": previous_version}) - return self.get_policy(int(active["id"])) - - def delete_policy(self, policy_id: int) -> bool: - current = self.get_policy(policy_id) - if not current: - return False - if current["is_active"]: - raise ValueError("cannot delete active policy") - - with self.connect() as conn: - self._audit(conn, policy_id, "delete", None) - conn.execute("DELETE FROM policy_versions WHERE policy_id = ?", (policy_id,)) - conn.execute("DELETE FROM policies WHERE id = ?", (policy_id,)) - return True - - def set_policy_status(self, policy_id: int, status: str, actor: str | None, comment: str | None = None) -> dict[str, Any] | None: - current = self.get_policy(policy_id) - if not current: - return None - allowed = {"draft", "pending_approval", "approved", "deployed"} - if status not in allowed: - raise ValueError(f"unsupported status: {status}") - with self.connect() as conn: - conn.execute( - "UPDATE policies SET status = ?, updated_at = ? WHERE id = ?", - (status, utc_now(), policy_id), - ) - self._audit(conn, policy_id, "status_change", actor, comment=comment, details={"status": status}) - return self.get_policy(policy_id) - - def list_audit(self, policy_id: int | None = None, limit: int = 200) -> list[dict[str, Any]]: - query = """ - SELECT id, policy_id, action, actor, comment, details_json, created_at - FROM policy_audit - """ - params: tuple[Any, ...] - if policy_id is None: - query += " ORDER BY id DESC LIMIT ?" - params = (limit,) - else: - query += " WHERE policy_id = ? ORDER BY id DESC LIMIT ?" - params = (policy_id, limit) - with self.connect() as conn: - rows = conn.execute(query, params).fetchall() - items: list[dict[str, Any]] = [] - for row in rows: - d = dict(row) - if d.get("details_json"): - try: - d["details"] = json.loads(d["details_json"]) - except Exception: - d["details"] = None - else: - d["details"] = None - d.pop("details_json", None) - items.append(d) - return items diff --git a/aw-server/ensure-reliability.sh b/aw-server/ensure-reliability.sh index 8b2c6ce..54c40d7 100644 --- a/aw-server/ensure-reliability.sh +++ b/aw-server/ensure-reliability.sh @@ -18,7 +18,7 @@ else "$ROOT_DIR/adk-rust/target/release/aw-ensure-reliability" \ "/usr/local/bin/aw-ensure-reliability"; do if [[ -n "$candidate" && -x "$candidate" ]]; then - exec "$candidate" --health-script-source "$SCRIPT_DIR/health-check.sh" "$@" + exec "$candidate" "$@" fi done cat >&2 <<'EOF' @@ -115,10 +115,12 @@ setup_health_check() { local health_service="/etc/systemd/system/aw-health-check.service" if [[ ! -f "$health_script" ]]; then - log "Setting up health check..." - cp "$SCRIPT_DIR/health-check.sh" "$health_script" - chmod +x "$health_script" - + log "ERROR: Rust-first health check wrapper not found: $health_script" + return 1 + fi + + if [[ ! -f "$health_timer" || ! -f "$health_service" ]]; then + log "Setting up health check timer..." # Create systemd timer for health checks cat > "$health_timer" << 'EOF' [Unit] diff --git a/aw-server/hayabusa/aw-hayabusa-autoprocess.py b/aw-server/hayabusa/aw-hayabusa-autoprocess.py deleted file mode 100644 index d42b733..0000000 --- a/aw-server/hayabusa/aw-hayabusa-autoprocess.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import fcntl -import json -import pathlib -import shutil -import subprocess -import sys -from typing import Optional - -DROP_DIR = pathlib.Path('/opt/activitywatch/aw-rus-ops/drop') -LOCK_PATH = pathlib.Path('/opt/hayabusa/state/aw-hayabusa-autoprocess.lock') -WRAPPER = pathlib.Path('/usr/local/bin/aw-hayabusa') -LINKER = pathlib.Path('/usr/local/bin/aw-hayabusa-link-case') -CASE_ALERT = pathlib.Path('/usr/local/bin/aw-hayabusa-case-alert') - - -def run(cmd): - print('RUN', ' '.join(str(x) for x in cmd), flush=True) - subprocess.run(cmd, check=True) - - -def run_capture(cmd): - print('RUN', ' '.join(str(x) for x in cmd), flush=True) - return subprocess.run(cmd, check=False, text=True, capture_output=True) - - -def read_latest_intake(): - return json.loads(pathlib.Path('/opt/hayabusa/state/latest-intake.json').read_text(encoding='utf-8')) - - -def load_sidecars(zip_path: pathlib.Path): - base = zip_path.with_suffix('') - caseid_path = base.with_suffix('.caseid') - meta_path = base.with_suffix('.meta.json') - meta = {} - if meta_path.is_file(): - meta = json.loads(meta_path.read_text(encoding='utf-8')) - case_id = meta.get('case_id') - if case_id is None and caseid_path.is_file(): - raw = caseid_path.read_text(encoding='utf-8').strip() - if raw: - case_id = int(raw) - return { - 'base': base, - 'case_id': case_id, - 'host': meta.get('host'), - 'mode': meta.get('mode', 'incident'), - 'link_source': meta.get('link_source', 'aw-rus-drop-autoprocess'), - 'caseid_path': caseid_path, - 'meta_path': meta_path, - } - - -def archive_sidecars(report_dir: pathlib.Path, sidecars): - target_dir = report_dir / 'input-sidecars' - target_dir.mkdir(parents=True, exist_ok=True) - for path in (sidecars['caseid_path'], sidecars['meta_path']): - if path.is_file(): - shutil.move(str(path), str(target_dir / path.name)) - - -def archive_drop_package(report_dir: pathlib.Path, zip_path: pathlib.Path): - target_dir = report_dir / 'input-drop' - target_dir.mkdir(parents=True, exist_ok=True) - target_path = target_dir / zip_path.name - if target_path.exists(): - target_path.unlink() - shutil.move(str(zip_path), str(target_path)) - - -def guess_host(zip_path: pathlib.Path, sidecars) -> Optional[str]: - if sidecars['host']: - return str(sidecars['host']) - name = zip_path.stem - if '-' in name: - return name.split('-', 1)[0] - return name or None - - -def process_one(zip_path: pathlib.Path): - sidecars = load_sidecars(zip_path) - host = guess_host(zip_path, sidecars) - mode = sidecars['mode'] or 'incident' - accept_cmd = [str(WRAPPER), 'accept', '--package', str(zip_path)] - if host: - accept_cmd += ['--host', host] - run(accept_cmd) - run([str(WRAPPER), 'process-inbox', '--mode', mode, '--limit', '1']) - latest = read_latest_intake() - report_dir = pathlib.Path(latest['report_dir']) - case_alert = None - if CASE_ALERT.is_file(): - alert_cmd = [str(CASE_ALERT), '--mode', mode, '--link-source', sidecars['link_source']] - if sidecars['case_id'] is not None: - alert_cmd += ['--case-id', str(sidecars['case_id'])] - result = run_capture(alert_cmd) - case_alert = { - 'returncode': result.returncode, - 'stdout': result.stdout.strip(), - 'stderr': result.stderr.strip(), - } - archive_sidecars(report_dir, sidecars) - archive_drop_package(report_dir, zip_path) - if sidecars['case_id'] is not None and not CASE_ALERT.is_file(): - run([str(LINKER), '--case-id', str(sidecars['case_id']), '--mode', mode, '--link-source', sidecars['link_source']]) - return {'latest_intake': latest, 'case_alert': case_alert} - - -def main(): - p = argparse.ArgumentParser(description='Auto-process Hayabusa zip packages dropped onto aw-rus server') - p.add_argument('--drop-dir', default=str(DROP_DIR)) - p.add_argument('--once', action='store_true', default=True) - args = p.parse_args() - - drop_dir = pathlib.Path(args.drop_dir) - drop_dir.mkdir(parents=True, exist_ok=True) - LOCK_PATH.parent.mkdir(parents=True, exist_ok=True) - with LOCK_PATH.open('w') as lock_fh: - try: - fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - print('autoprocess already running', file=sys.stderr) - return 0 - zips = sorted(drop_dir.glob('*.zip')) - if not zips: - print('no zip packages in drop dir') - return 0 - for zip_path in zips: - result = process_one(zip_path) - print(json.dumps({'processed': str(zip_path), **result}, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == '__main__': - raise SystemExit(main()) diff --git a/aw-server/hayabusa/aw-hayabusa-case-alert.py b/aw-server/hayabusa/aw-hayabusa-case-alert.py deleted file mode 100644 index fe0fa01..0000000 --- a/aw-server/hayabusa/aw-hayabusa-case-alert.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import csv -import json -import os -import pathlib -import urllib.error -import urllib.parse -import urllib.request -from collections import Counter -from datetime import datetime, timezone -from typing import Any - - -SEVERITY_ORDER = {"low": 1, "medium": 2, "high": 3, "critical": 4} -LEVEL_NORMALIZATION = { - "informational": "info", - "info": "info", - "low": "low", - "med": "med", - "medium": "med", - "high": "high", - "crit": "crit", - "critical": "crit", -} -LEVEL_WEIGHTS = { - "info": 1, - "low": 4, - "med": 12, - "high": 40, - "crit": 100, -} - - -def env_bool(name: str, default: bool) -> bool: - value = os.environ.get(name) - if value is None: - return default - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def normalize_level(level: str | None) -> str: - if not level: - return "info" - return LEVEL_NORMALIZATION.get(level.strip().lower(), level.strip().lower()) - - -def severity_meets(actual: str, threshold: str) -> bool: - return SEVERITY_ORDER.get(actual, 0) >= SEVERITY_ORDER.get(threshold, 0) - - -def build_hayabusa_payload(intake: dict[str, Any], mode: str, link_source: str) -> dict[str, Any]: - report_dir = pathlib.Path(intake["report_dir"]) - return { - "tool": "hayabusa", - "host": intake["host"], - "mode": mode, - "status": intake["status"], - "intake_id": intake["intake_id"], - "package_path": intake["package_path"], - "sha256": intake["sha256"], - "report_dir": intake["report_dir"], - "summary_html": str(report_dir / "summary.html"), - "timeline_path": str(report_dir / "timeline.jsonl"), - "manifest_path": str(report_dir / "manifest.json"), - "link_source": link_source, - } - - -def post_json(url: str, payload: dict[str, Any]) -> dict[str, Any]: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - req = urllib.request.Request(url, data=data, method="POST", headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req) as resp: - body = resp.read().decode("utf-8") - return json.loads(body) if body else {} - - -def patch_json(url: str, payload: dict[str, Any]) -> dict[str, Any]: - data = json.dumps(payload, ensure_ascii=False).encode("utf-8") - req = urllib.request.Request(url, data=data, method="PATCH", headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req) as resp: - body = resp.read().decode("utf-8") - return json.loads(body) if body else {} - - -def get_json(url: str) -> dict[str, Any]: - with urllib.request.urlopen(url) as resp: - return json.loads(resp.read().decode("utf-8")) - - -def parse_timestamp(value: str | None) -> datetime | None: - if not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - -def read_csv_rows(path: pathlib.Path) -> int: - if not path.is_file(): - return 0 - with path.open("r", encoding="utf-8-sig", newline="") as fh: - reader = csv.reader(fh) - rows = list(reader) - if not rows: - return 0 - return max(0, len(rows) - 1) - - -def analyze_report(report_dir: pathlib.Path) -> dict[str, Any]: - timeline_path = report_dir / "timeline.jsonl" - level_counts: Counter[str] = Counter() - title_counts: Counter[str] = Counter() - first_ts: datetime | None = None - last_ts: datetime | None = None - total_events = 0 - - if timeline_path.is_file(): - with timeline_path.open("r", encoding="utf-8") as fh: - for raw_line in fh: - line = raw_line.strip() - if not line: - continue - try: - event = json.loads(line) - except json.JSONDecodeError: - continue - total_events += 1 - level = normalize_level(str(event.get("Level", ""))) - level_counts[level] += 1 - title = str(event.get("RuleTitle") or "").strip() or "Unknown rule" - title_counts[title] += 1 - ts = parse_timestamp(event.get("Timestamp")) - if ts is not None: - first_ts = ts if first_ts is None or ts < first_ts else first_ts - last_ts = ts if last_ts is None or ts > last_ts else last_ts - - failed_logons = read_csv_rows(report_dir / "logon-summary-failed.csv") - successful_logons = read_csv_rows(report_dir / "logon-summary-successful.csv") - - suspicious_pwsh = sum( - count - for title, count in title_counts.items() - if "pwsh" in title.lower() or "powershell" in title.lower() or "obfuscation" in title.lower() - ) - credential_events = sum(count for title, count in title_counts.items() if "credential" in title.lower()) - timestomp_events = sum(count for title, count in title_counts.items() if "timestomp" in title.lower()) - logon_failure_events = sum(count for title, count in title_counts.items() if "logon failure" in title.lower()) - - score = ( - sum(LEVEL_WEIGHTS.get(level, 0) * count for level, count in level_counts.items()) - + min(failed_logons, 200) * 2 - + suspicious_pwsh * 6 - + credential_events * 8 - + timestomp_events * 12 - + logon_failure_events * 2 - ) - - crit_count = level_counts.get("crit", 0) - high_count = level_counts.get("high", 0) - med_count = level_counts.get("med", 0) - - if crit_count >= 1 or score >= 240 or (high_count >= 4 and suspicious_pwsh >= 4): - severity = "critical" - elif high_count >= 1 or score >= 120 or suspicious_pwsh >= 8 or credential_events >= 5: - severity = "high" - elif med_count >= 1 or score >= 40 or failed_logons >= 10: - severity = "medium" - else: - severity = "low" - - return { - "severity": severity, - "score": score, - "events_total": total_events, - "level_counts": dict(level_counts), - "top_rules": [{"title": title, "count": count} for title, count in title_counts.most_common(5)], - "first_timestamp": first_ts.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if first_ts else None, - "last_timestamp": last_ts.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if last_ts else None, - "failed_logon_rows": failed_logons, - "successful_logon_rows": successful_logons, - "suspicious_pwsh": suspicious_pwsh, - "credential_events": credential_events, - "timestomp_events": timestomp_events, - "logon_failure_events": logon_failure_events, - } - - -def build_case_title(host: str, summary: dict[str, Any]) -> str: - top_rule = summary.get("top_rules") or [] - suffix = top_rule[0]["title"] if top_rule else "No dominant rule" - return f"Hayabusa {summary['severity'].upper()} · {host} · {suffix}" - - -def build_case_payload(intake: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]: - return { - "incident_id": f"hayabusa:{intake['host']}:{intake['intake_id']}", - "host": intake["host"], - "title": build_case_title(intake["host"], summary), - "severity": summary["severity"], - "evidence": { - "hayabusa": { - "intake_id": intake["intake_id"], - "package_path": intake["package_path"], - "sha256": intake["sha256"], - "report_dir": intake["report_dir"], - "summary": summary, - } - }, - } - - -def build_comment(summary: dict[str, Any], intake: dict[str, Any]) -> str: - top = ", ".join(f"{item['title']} ({item['count']})" for item in summary.get("top_rules", [])[:3]) or "n/a" - return ( - f"Hayabusa auto-summary\n" - f"Severity: {summary['severity']} (score={summary['score']})\n" - f"Host: {intake['host']}\n" - f"Intake: {intake['intake_id']}\n" - f"Events: {summary['events_total']}, failed_logons={summary['failed_logon_rows']}, " - f"suspicious_pwsh={summary['suspicious_pwsh']}, credential_events={summary['credential_events']}\n" - f"Top rules: {top}\n" - f"Report: {intake['report_dir']}" - ) - - -def send_telegram(bot_token: str, chat_ids: list[str], text: str) -> list[dict[str, Any]]: - results = [] - for chat_id in chat_ids: - payload = urllib.parse.urlencode({"chat_id": chat_id, "text": text}).encode("utf-8") - url = f"https://api.telegram.org/bot{bot_token}/sendMessage" - req = urllib.request.Request(url, data=payload, method="POST") - try: - with urllib.request.urlopen(req) as resp: - body = json.loads(resp.read().decode("utf-8")) - results.append({"chat_id": chat_id, "ok": True, "response": body}) - except Exception as exc: # noqa: BLE001 - results.append({"chat_id": chat_id, "ok": False, "error": str(exc)}) - return results - - -def build_telegram_text(case_id: int | None, intake: dict[str, Any], summary: dict[str, Any]) -> str: - severity_label = { - "critical": "критичное событие", - "high": "опасное событие", - "medium": "подозрительное событие", - "low": "слабый сигнал", - }.get(summary["severity"], summary["severity"]) - top_rules = summary.get("top_rules", []) - top_rule = top_rules[0]["title"] if top_rules else "нет явного доминирующего правила" - lines = [ - f"Hayabusa: {severity_label}", - "", - f"Хост: {intake['host']}", - ] - if case_id is not None: - lines.append(f"Кейс: {case_id}") - lines.extend( - [ - f"Уровень: {summary['severity']}", - "", - "Что найдено:", - f"- {top_rule}: {top_rules[0]['count'] if top_rules else 0}", - f"- подозрительный PowerShell: {summary['suspicious_pwsh']}", - f"- ошибок входа: {summary['logon_failure_events']}", - f"- событий по учётным данным: {summary['credential_events']}", - ] - ) - if summary.get("timestomp_events", 0) > 0: - lines.append(f"- timestomp-подобных событий: {summary['timestomp_events']}") - lines.extend( - [ - "", - "Главный риск:", - "возможная активность вокруг учётных данных и PowerShell", - "", - "Отчёт:", - f"{intake['report_dir']}", - ] - ) - return "\n".join(lines) - - -def main() -> int: - p = argparse.ArgumentParser(description="Auto-create/update AW-rus case, compute Hayabusa severity, and send Telegram alerts") - p.add_argument("--case-id", type=int) - p.add_argument("--intake-json", default="/opt/hayabusa/state/latest-intake.json") - p.add_argument("--case-api-base", default=os.environ.get("AW_HAYABUSA_CASE_API_BASE", "http://127.0.0.1:5602")) - p.add_argument("--mode", default="incident") - p.add_argument("--link-source", default="aw-rus-drop-autoprocess") - p.add_argument("--auto-create", action="store_true", default=env_bool("AW_HAYABUSA_AUTO_CASE_ENABLED", True)) - p.add_argument("--auto-create-min-severity", default=os.environ.get("AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY", "medium")) - p.add_argument("--telegram-enabled", action="store_true", default=env_bool("AW_HAYABUSA_TELEGRAM_ENABLED", False)) - p.add_argument("--telegram-min-severity", default=os.environ.get("AW_HAYABUSA_TELEGRAM_MIN_SEVERITY", "high")) - p.add_argument("--telegram-bot-token", default=os.environ.get("AW_HAYABUSA_TELEGRAM_BOT_TOKEN", "")) - p.add_argument("--telegram-chat-ids", default=os.environ.get("AW_HAYABUSA_TELEGRAM_CHAT_IDS", "")) - args = p.parse_args() - - intake_path = pathlib.Path(args.intake_json) - intake = json.loads(intake_path.read_text(encoding="utf-8")) - summary = analyze_report(pathlib.Path(intake["report_dir"])) - case_api_base = args.case_api_base.rstrip("/") - if case_api_base.endswith("/api/0/dlp/cases"): - case_api_base = case_api_base[: -len("/api/0/dlp/cases")] - case_id = args.case_id - created_case = None - case_error = None - linked = False - comment_added = False - - try: - if case_id is None and args.auto_create and severity_meets(summary["severity"], args.auto_create_min_severity): - created_case = post_json(f"{case_api_base}/api/0/dlp/cases", build_case_payload(intake, summary)) - case_id = int(created_case["id"]) - - if case_id is not None: - patch_json( - f"{case_api_base}/api/0/dlp/cases/{case_id}", - {"severity": summary["severity"]}, - ) - post_json( - f"{case_api_base}/api/0/dlp/cases/{case_id}/forensics/hayabusa", - build_hayabusa_payload(intake, args.mode, args.link_source), - ) - linked = True - post_json( - f"{case_api_base}/api/0/dlp/cases/{case_id}/comments", - {"comment": build_comment(summary, intake), "author": "aw-hayabusa-auto"}, - ) - comment_added = True - except Exception as exc: # noqa: BLE001 - case_error = str(exc) - - telegram_results: list[dict[str, Any]] = [] - if args.telegram_enabled and args.telegram_bot_token and severity_meets(summary["severity"], args.telegram_min_severity): - chat_ids = [item.strip() for item in args.telegram_chat_ids.split(",") if item.strip()] - if chat_ids: - telegram_results = send_telegram( - bot_token=args.telegram_bot_token, - chat_ids=chat_ids, - text=build_telegram_text(case_id, intake, summary), - ) - - result = { - "summary": summary, - "case_id": case_id, - "case_created": created_case, - "case_linked": linked, - "case_comment_added": comment_added, - "case_error": case_error, - "telegram_results": telegram_results, - } - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 if case_error is None else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/aw-server/hayabusa/aw-hayabusa-from-windows.py b/aw-server/hayabusa/aw-hayabusa-from-windows.py deleted file mode 100644 index 6d157ee..0000000 --- a/aw-server/hayabusa/aw-hayabusa-from-windows.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import pathlib -import re -import subprocess -import sys - -OPS_ROOT = pathlib.Path('/opt/activitywatch/aw-rus-ops') -DEFAULT_INVENTORY = OPS_ROOT / 'ansible' / 'inventory.ini' -DEFAULT_DROP = OPS_ROOT / 'drop' -DEFAULT_ANSIBLE = OPS_ROOT / 'venv' / 'bin' / 'ansible' -DEFAULT_WRAPPER = pathlib.Path('/usr/local/bin/aw-hayabusa') -DEFAULT_LINKER = pathlib.Path('/usr/local/bin/aw-hayabusa-link-case') -WINDOWS_EXPORT_CMD = r"powershell.exe -ExecutionPolicy Bypass -File C:\ProgramData\AWatch-rus\export-evtx-for-hayabusa.ps1 -DaysBack {days_back} | ConvertTo-Json -Depth 8 -Compress" -WINDOWS_LATEST_ZIP_CMD = r"Get-ChildItem 'C:\ProgramData\AWatch-rus\forensics\evtx-exports' -File -Filter '*.zip' | Sort-Object LastWriteTime -Descending | Select-Object -First 1 FullName,Length,LastWriteTime | ConvertTo-Json -Compress" - - -def run(cmd): - proc = subprocess.run(cmd, text=True, capture_output=True) - if proc.returncode != 0: - sys.stderr.write(proc.stdout) - sys.stderr.write(proc.stderr) - raise SystemExit(proc.returncode) - return proc.stdout - - -def extract_json_blob(text): - matches = re.findall(r'(\{.*\}|\[.*\])', text, re.S) - for candidate in reversed(matches): - try: - return json.loads(candidate) - except Exception: - continue - raise SystemExit('cannot parse JSON from ansible output:\n' + text) - - -def main(): - p = argparse.ArgumentParser(description='Run Windows EVTX export and Hayabusa intake directly from aw-server, without the laptop') - p.add_argument('--inventory', default=str(DEFAULT_INVENTORY)) - p.add_argument('--ansible-bin', default=str(DEFAULT_ANSIBLE)) - p.add_argument('--drop-dir', default=str(DEFAULT_DROP)) - p.add_argument('--days-back', type=int, default=1) - p.add_argument('--mode', default='incident', choices=['quick', 'incident', 'full']) - p.add_argument('--case-id', type=int) - p.add_argument('--link-source', default='aw-rus-ops-from-windows') - p.add_argument('--windows-group', default='aw_windows') - p.add_argument('--wrapper', default=str(DEFAULT_WRAPPER)) - p.add_argument('--linker', default=str(DEFAULT_LINKER)) - args = p.parse_args() - - inventory = pathlib.Path(args.inventory) - ansible_bin = pathlib.Path(args.ansible_bin) - drop_dir = pathlib.Path(args.drop_dir) - wrapper = pathlib.Path(args.wrapper) - linker = pathlib.Path(args.linker) - if not inventory.is_file(): - raise SystemExit(f'inventory not found: {inventory}') - if not ansible_bin.is_file(): - raise SystemExit(f'ansible binary not found: {ansible_bin}') - if not wrapper.is_file(): - raise SystemExit(f'wrapper not found: {wrapper}') - drop_dir.mkdir(parents=True, exist_ok=True) - - export_cmd = [str(ansible_bin), args.windows_group, '-i', str(inventory), '-m', 'win_shell', '-a', WINDOWS_EXPORT_CMD.format(days_back=args.days_back)] - print('RUN_EXPORT', ' '.join(export_cmd)) - export_out = run(export_cmd) - export_json = extract_json_blob(export_out) - - list_cmd = [str(ansible_bin), args.windows_group, '-i', str(inventory), '-m', 'win_shell', '-a', WINDOWS_LATEST_ZIP_CMD] - print('RUN_LIST', ' '.join(list_cmd)) - latest_out = run(list_cmd) - latest = extract_json_blob(latest_out) - if isinstance(latest, list): - latest = latest[0] - remote_zip = latest['FullName'] - filename = pathlib.PureWindowsPath(remote_zip).name - local_zip = drop_dir / filename - - remote_zip_posix = remote_zip.replace('\\', '/') - fetch_cmd = [str(ansible_bin), args.windows_group, '-i', str(inventory), '-m', 'fetch', '-a', f'src={remote_zip_posix} dest={drop_dir}/ flat=yes'] - print('RUN_FETCH', ' '.join(fetch_cmd)) - run(fetch_cmd) - if not local_zip.is_file(): - raise SystemExit(f'fetched zip not found: {local_zip}') - - accept_cmd = [str(wrapper), 'accept', '--package', str(local_zip)] - host = export_json.get('hostname') or pathlib.Path(filename).stem.split('-')[0] - if host: - accept_cmd += ['--host', str(host)] - print('RUN_ACCEPT', ' '.join(accept_cmd)) - subprocess.run(accept_cmd, check=True) - - process_cmd = [str(wrapper), 'process-inbox', '--mode', args.mode, '--limit', '1'] - print('RUN_PROCESS', ' '.join(process_cmd)) - subprocess.run(process_cmd, check=True) - - if args.case_id is not None: - if not linker.is_file(): - raise SystemExit(f'linker not found: {linker}') - link_cmd = [str(linker), '--case-id', str(args.case_id), '--mode', args.mode, '--link-source', args.link_source] - print('RUN_LINK', ' '.join(link_cmd)) - subprocess.run(link_cmd, check=True) - - latest_intake = pathlib.Path('/opt/hayabusa/state/latest-intake.json') - print('LATEST_INTAKE') - print(latest_intake.read_text(encoding='utf-8')) - - -if __name__ == '__main__': - main() diff --git a/aw-server/hayabusa/aw-hayabusa-link-case.py b/aw-server/hayabusa/aw-hayabusa-link-case.py deleted file mode 100644 index 1cb8d08..0000000 --- a/aw-server/hayabusa/aw-hayabusa-link-case.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import pathlib -import sys -import urllib.error -import urllib.request - - -def normalize_host(value): - return str(value or '').strip().lower() - - -def build_payload(intake, mode, link_source): - report_dir = pathlib.Path(intake['report_dir']) - return { - 'tool': 'hayabusa', - 'host': intake['host'], - 'mode': mode, - 'status': intake['status'], - 'intake_id': intake['intake_id'], - 'package_path': intake['package_path'], - 'sha256': intake['sha256'], - 'report_dir': intake['report_dir'], - 'summary_html': str(report_dir / 'summary.html'), - 'timeline_path': str(report_dir / 'timeline.jsonl'), - 'manifest_path': str(report_dir / 'manifest.json'), - 'link_source': link_source, - } - - -def post_json(url, payload): - data = json.dumps(payload, ensure_ascii=False).encode('utf-8') - req = urllib.request.Request(url, data=data, method='POST', headers={'Content-Type': 'application/json'}) - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read().decode('utf-8')) - - -def get_json(url): - with urllib.request.urlopen(url) as resp: - return json.loads(resp.read().decode('utf-8')) - - -def main(): - p = argparse.ArgumentParser(description='Link latest or specified Hayabusa intake metadata to AW-rus case management') - p.add_argument('--case-id', type=int, required=True) - p.add_argument('--intake-json', default='/opt/hayabusa/state/latest-intake.json') - p.add_argument('--case-api-base', default='http://127.0.0.1:5602') - p.add_argument('--mode', default='incident') - p.add_argument('--link-source', default='aw-rus-ops') - args = p.parse_args() - - intake_path = pathlib.Path(args.intake_json) - if not intake_path.is_file(): - raise SystemExit(f'intake json not found: {intake_path}') - intake = json.loads(intake_path.read_text(encoding='utf-8')) - case = get_json(f"{args.case_api_base.rstrip('/')}/api/0/dlp/cases/{args.case_id}") - case_host = normalize_host(case.get('host')) - intake_host = normalize_host(intake.get('host')) - if case_host and intake_host and case_host != intake_host: - raise SystemExit( - f"hayabusa host mismatch: case host={case.get('host')} intake host={intake.get('host')}" - ) - payload = build_payload(intake, args.mode, args.link_source) - case_url = f"{args.case_api_base.rstrip('/')}/api/0/dlp/cases/{args.case_id}/forensics/hayabusa" - try: - post_json(case_url, payload) - except urllib.error.HTTPError as exc: - body = exc.read().decode('utf-8', errors='replace') - raise SystemExit(f'case API POST failed: HTTP {exc.code}: {body}') - case = get_json(f"{args.case_api_base.rstrip('/')}/api/0/dlp/cases/{args.case_id}") - print(json.dumps({'case_id': args.case_id, 'intake': intake, 'forensics': case.get('forensics')}, ensure_ascii=False, indent=2)) - - -if __name__ == '__main__': - main() diff --git a/aw-server/hayabusa/aw-hayabusa.sh b/aw-server/hayabusa/aw-hayabusa.sh index b82a326..974f5d2 100644 --- a/aw-server/hayabusa/aw-hayabusa.sh +++ b/aw-server/hayabusa/aw-hayabusa.sh @@ -96,19 +96,8 @@ EOF json_field() { local json_path="$1" local field_name="$2" - python3 - "$json_path" "$field_name" <<'PY' -import json, sys -path, field = sys.argv[1], sys.argv[2] -try: - with open(path, 'r', encoding='utf-8') as fh: - obj = json.load(fh) -except Exception: - sys.exit(0) -value = obj.get(field) -if value is None: - sys.exit(0) -print(str(value)) -PY + command -v jq >/dev/null 2>&1 || fail "jq is required to read ${json_path}" + jq -er --arg field "${field_name}" '.[$field] // empty | tostring' "${json_path}" 2>/dev/null || true } detect_host_from_manifest() { @@ -124,34 +113,19 @@ detect_host_from_manifest() { extract_zip_normalized() { local package_path="$1" local dest_dir="$2" - python3 - "${package_path}" "${dest_dir}" <<'PY' -import pathlib -import shutil -import sys -import zipfile - -zip_path = pathlib.Path(sys.argv[1]) -dest_dir = pathlib.Path(sys.argv[2]) -dest_dir.mkdir(parents=True, exist_ok=True) - -with zipfile.ZipFile(zip_path) as zf: - for info in zf.infolist(): - raw_name = info.filename.replace('\\', '/') - normalized = pathlib.PurePosixPath(raw_name) - parts = [part for part in normalized.parts if part not in ('', '.')] - if any(part == '..' for part in parts): - raise SystemExit(f'unsafe zip entry: {info.filename}') - if not parts: - continue - target = dest_dir.joinpath(*parts) - is_dir = info.is_dir() or raw_name.endswith('/') - if is_dir: - target.mkdir(parents=True, exist_ok=True) - continue - target.parent.mkdir(parents=True, exist_ok=True) - with zf.open(info) as src, target.open('wb') as dst: - shutil.copyfileobj(src, dst) -PY + command -v zipinfo >/dev/null 2>&1 || fail "zipinfo is required to inspect ${package_path}" + command -v unzip >/dev/null 2>&1 || fail "unzip is required to extract ${package_path}" + mkdir -p "${dest_dir}" + local entry normalized + while IFS= read -r entry; do + normalized="${entry//\\//}" + case "${normalized}" in + ""|.|/*|*"/../"*|../*|*".."|*"/..") + fail "unsafe zip entry: ${entry}" + ;; + esac + done < <(zipinfo -1 "${package_path}") + unzip -q "${package_path}" -d "${dest_dir}" } write_package_manifest() { diff --git a/aw-server/health-check.sh b/aw-server/health-check.sh deleted file mode 100644 index 7df5693..0000000 --- a/aw-server/health-check.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Health check script for AW services -# Returns 0 if all services are healthy, 1 otherwise - -SERVICES=("activitywatch-server" "aw-worktime-api" "aw-worktime-ui-bridge") -UNHEALTHY_SERVICES=() -WARNINGS=() - -if [[ -f /etc/activitywatch/aw-server.env ]]; then - # shellcheck disable=SC1091 - source /etc/activitywatch/aw-server.env -fi - -WORKTIME_HEALTH_BASE="${AW_RUS_HEALTH_WORKTIME_API:-${AW_WORKTIME_REPORT_BASE:-http://127.0.0.1:5610}}" -WORKTIME_HEALTH_URL="${WORKTIME_HEALTH_BASE%/}/health" -WORKTIME_HEALTH_TIMEOUT_SECONDS="${AW_RUS_HEALTH_WORKTIME_TIMEOUT_SECONDS:-15}" -AW_HEALTH_API_TIMEOUT_SECONDS="${AW_RUS_HEALTH_AW_TIMEOUT_SECONDS:-15}" -AW_HEALTH_API_ATTEMPTS="${AW_RUS_HEALTH_AW_ATTEMPTS:-3}" -SETTINGS_TIMEOUT_SECONDS="${AW_RUS_HEALTH_SETTINGS_TIMEOUT_SECONDS:-15}" -SETTINGS_ATTEMPTS="${AW_RUS_HEALTH_SETTINGS_ATTEMPTS:-3}" - -check_service() { - local service=$1 - if [[ "$service" == "aw-worktime-ui-bridge" ]]; then - if systemctl is-active --quiet aw-worktime-ui-bridge.timer && systemctl is-enabled --quiet aw-worktime-ui-bridge.timer; then - echo "✓ aw-worktime-ui-bridge.timer is running and enabled" - else - echo "✗ aw-worktime-ui-bridge.timer is not active/enabled" - UNHEALTHY_SERVICES+=("aw-worktime-ui-bridge.timer") - fi - return - fi - if systemctl is-active --quiet "$service"; then - echo "✓ $service is running" - else - echo "✗ $service is not running" - UNHEALTHY_SERVICES+=("$service") - fi -} - -check_api_endpoint() { - local url=$1 - local service_name=$2 - local timeout_seconds=${3:-5} - local attempts=${4:-1} - local attempt - - for ((attempt=1; attempt<=attempts; attempt++)); do - if curl -fsS --connect-timeout 3 --max-time "$timeout_seconds" "$url" >/dev/null 2>&1; then - echo "✓ $service_name API endpoint is responding" - return - fi - if (( attempt < attempts )); then - sleep 1 - fi - done - - echo "✗ $service_name API endpoint is not responding" - UNHEALTHY_SERVICES+=("$service_name-api") -} - -read_setting_value() { - local key=$1 - local attempt - for ((attempt=1; attempt<=SETTINGS_ATTEMPTS; attempt++)); do - if curl -fsS --connect-timeout 3 --max-time "$SETTINGS_TIMEOUT_SECONDS" "http://127.0.0.1:5600/api/0/settings/${key}" 2>/dev/null | \ - python3 -c 'import json,sys; print(json.load(sys.stdin))'; then - return 0 - fi - if (( attempt < SETTINGS_ATTEMPTS )); then - sleep 1 - fi - done - return 1 -} - -check_expected_setting() { - local key=$1 - local expected=$2 - local label=${3:-$1} - - if [[ -z "$expected" ]]; then - echo "⚠ expected value for ${label} is not configured, skipping drift check" - WARNINGS+=("${key}-expected-missing") - return - fi - - local actual - if ! actual="$(read_setting_value "$key" 2>/dev/null)"; then - echo "✗ failed to read setting ${label}" - UNHEALTHY_SERVICES+=("setting-${key}") - return - fi - - if [[ "$actual" == "$expected" ]]; then - echo "✓ ${label} matches expected value (${expected})" - else - echo "✗ ${label} drift detected: actual='${actual}' expected='${expected}'" - UNHEALTHY_SERVICES+=("setting-${key}") - fi -} - -check_dlp_transport_freshness() { - local dlp_health="${DLP_HEALTH_BIN:-/usr/local/bin/dlp-health-check}" - local result - - if [[ ! -x "$dlp_health" ]]; then - echo "⚠ dlp-health-check is not available, skipping DLP transport freshness checks" - WARNINGS+=("dlp-health-check-missing") - return - fi - - result="$("$dlp_health" --json 2>/dev/null || true)" - if [[ -z "$result" ]]; then - echo "⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks" - WARNINGS+=("dlp-health-check-empty") - return - fi - - local ok - ok="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); names={r["name"]:r for r in data.get("results", [])}; checks=["buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics"]; bad=[n for n in checks if names.get(n,{}).get("status")=="fail"]; print("1" if not bad else "0")' 2>/dev/null || echo "0")" - if [[ "$ok" == "1" ]]; then - echo "✓ DLP transport freshness check passed" - else - echo "✗ DLP transport freshness check failed" - UNHEALTHY_SERVICES+=("dlp-transport") - fi - - local errors warnings - errors="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="fail" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" - warnings="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="warn" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" - if [[ -n "$errors" ]]; then - echo " errors: $errors" - fi - if [[ -n "$warnings" ]]; then - echo " warnings: $warnings" - WARNINGS+=("$warnings") - fi -} - -echo "=== AW Services Health Check ===" -echo "Timestamp: $(date)" -echo - -# Check systemd services -for service in "${SERVICES[@]}"; do - check_service "$service" -done - -echo - -# Check API endpoints -check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" "$AW_HEALTH_API_TIMEOUT_SECONDS" "$AW_HEALTH_API_ATTEMPTS" -check_api_endpoint "$WORKTIME_HEALTH_URL" "aw-worktime-api" "$WORKTIME_HEALTH_TIMEOUT_SECONDS" 2 -check_dlp_transport_freshness "http://127.0.0.1:5600/api/0" "900" "${AW_HEALTH_STRICT_FILEOPS:-0}" -check_expected_setting "startOfDay" "${AW_EXPECT_START_OF_DAY:-}" "startOfDay" -check_expected_setting "always_active_pattern" "${AW_EXPECT_ALWAYS_ACTIVE_PATTERN:-}" "always_active_pattern" -check_expected_setting "landingpage" "${AW_EXPECT_LANDINGPAGE:-}" "landingpage" - -echo - -if [ ${#UNHEALTHY_SERVICES[@]} -eq 0 ]; then - echo "✓ All services are healthy" - if [ ${#WARNINGS[@]} -gt 0 ]; then - echo "⚠ Warnings: ${WARNINGS[*]}" - fi - exit 0 -else - echo "✗ Unhealthy services: ${UNHEALTHY_SERVICES[*]}" - exit 1 -fi diff --git a/aw-server/install_aw_server.sh b/aw-server/install_aw_server.sh index 34930d6..cdc1241 100755 --- a/aw-server/install_aw_server.sh +++ b/aw-server/install_aw_server.sh @@ -24,13 +24,13 @@ required_vars=( BOOTSTRAP_DIR="/root/bootstrap" VIEWS_JSON="$BOOTSTRAP_DIR/settings/views-default.json" CLASSES_JSON="$BOOTSTRAP_DIR/settings/classes-worktime.json" -WORKTIME_API_SRC="$BOOTSTRAP_DIR/aw-worktime-api.py" +WORKTIME_API_RUST_SRC="$BOOTSTRAP_DIR/worktime-api" WORKTIME_API_SERVICE_SRC="$BOOTSTRAP_DIR/aw-worktime-api.service" WORKTIME_ALIASES_SRC="$BOOTSTRAP_DIR/worktime-manager-aliases.example.json" -WORKTIME_UI_BRIDGE_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.py" +WORKTIME_UI_BRIDGE_RUST_SRC="$BOOTSTRAP_DIR/worktime-ui-bridge" WORKTIME_UI_BRIDGE_SERVICE_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.service" WORKTIME_UI_BRIDGE_TIMER_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.timer" -HEALTHD_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.py" +HEALTHD_RUST_SRC="$BOOTSTRAP_DIR/aw-rus-healthd" HEALTHD_SERVICE_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.service" HEALTHD_TIMER_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.timer" @@ -99,11 +99,11 @@ systemctl enable activitywatch-server.service systemctl restart activitywatch-server.service systemctl --no-pager --full status activitywatch-server.service || true -if [[ -f "$WORKTIME_API_SRC" ]]; then - install -m 0755 "$WORKTIME_API_SRC" /usr/local/bin/aw-worktime-api.py +if [[ -f "$WORKTIME_API_RUST_SRC" ]]; then + install -m 0755 "$WORKTIME_API_RUST_SRC" /usr/local/bin/aw-worktime-api-rust fi -if [[ -f "$WORKTIME_API_SERVICE_SRC" ]]; then +if [[ -f "$WORKTIME_API_SERVICE_SRC" && -f /usr/local/bin/aw-worktime-api-rust ]]; then install -m 0644 "$WORKTIME_API_SERVICE_SRC" /etc/systemd/system/aw-worktime-api.service systemctl daemon-reload systemctl enable aw-worktime-api.service @@ -119,15 +119,15 @@ if [[ -f "$WORKTIME_ALIASES_SRC" ]]; then fi fi -if [[ -f "$WORKTIME_UI_BRIDGE_SRC" ]]; then - install -m 0755 "$WORKTIME_UI_BRIDGE_SRC" /usr/local/bin/aw-worktime-ui-bridge.py +if [[ -f "$WORKTIME_UI_BRIDGE_RUST_SRC" ]]; then + install -m 0755 "$WORKTIME_UI_BRIDGE_RUST_SRC" /usr/local/bin/aw-worktime-ui-bridge-rust fi -if [[ -f "$WORKTIME_UI_BRIDGE_SERVICE_SRC" ]]; then +if [[ -f "$WORKTIME_UI_BRIDGE_SERVICE_SRC" && -f /usr/local/bin/aw-worktime-ui-bridge-rust ]]; then install -m 0644 "$WORKTIME_UI_BRIDGE_SERVICE_SRC" /etc/systemd/system/aw-worktime-ui-bridge.service fi -if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" ]]; then +if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" && -f /usr/local/bin/aw-worktime-ui-bridge-rust ]]; then install -m 0644 "$WORKTIME_UI_BRIDGE_TIMER_SRC" /etc/systemd/system/aw-worktime-ui-bridge.timer systemctl daemon-reload systemctl disable --now aw-worktime-afk-bridge.timer >/dev/null 2>&1 || true @@ -137,15 +137,15 @@ if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" ]]; then systemctl --no-pager --full status aw-worktime-ui-bridge.timer || true fi -if [[ -f "$HEALTHD_SRC" ]]; then - install -m 0755 "$HEALTHD_SRC" /usr/local/bin/aw-rus-healthd.py +if [[ -f "$HEALTHD_RUST_SRC" ]]; then + install -m 0755 "$HEALTHD_RUST_SRC" /usr/local/bin/aw-rus-healthd-rust fi -if [[ -f "$HEALTHD_SERVICE_SRC" ]]; then +if [[ -f "$HEALTHD_SERVICE_SRC" && -f /usr/local/bin/aw-rus-healthd-rust ]]; then install -m 0644 "$HEALTHD_SERVICE_SRC" /etc/systemd/system/aw-rus-healthd.service fi -if [[ -f "$HEALTHD_TIMER_SRC" ]]; then +if [[ -f "$HEALTHD_TIMER_SRC" && -f /usr/local/bin/aw-rus-healthd-rust ]]; then install -m 0644 "$HEALTHD_TIMER_SRC" /etc/systemd/system/aw-rus-healthd.timer systemctl daemon-reload systemctl enable aw-rus-healthd.timer diff --git a/aw-server/test_aw_dlp_influx_exporter.py b/aw-server/test_aw_dlp_influx_exporter.py deleted file mode 100644 index a41c062..0000000 --- a/aw-server/test_aw_dlp_influx_exporter.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -from datetime import UTC, datetime -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-dlp-influx-exporter.py") -SPEC = importlib.util.spec_from_file_location("aw_dlp_influx_exporter", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -def test_build_endpoint_lines_emits_self_test_and_signal(): - events = [ - { - "id": 10, - "timestamp": "2026-05-15T10:00:00Z", - "data": { - "hostname": "SHARKON2025", - "username": "Администратор", - "signalType": "self_test", - "policyMode": "server", - "policySource": "local-fallback", - "queueDepth": 2, - "eventsEnqueued": 100, - "eventsFlushed": 99, - "sendFailures": 1, - "policyEnabled": True, - }, - }, - { - "id": 11, - "timestamp": "2026-05-15T10:01:00Z", - "data": { - "hostname": "SHARKON2025", - "username": "Администратор", - "signalType": "print_job", - "source": "endpoint-signals-phase2", - "documentName": "Документ.docx", - "printerName": "HP LaserJet", - }, - }, - ] - lines = MODULE.build_endpoint_lines("SHARKON2025", events) - assert any(line.startswith("aw_dlp_endpoint_self_test,") for line in lines) - assert any(line.startswith("aw_dlp_signal,") for line in lines) - - -def test_normalize_incident_handles_nested_source_event(): - item = { - "timestamp": "2026-05-15T10:02:00Z", - "data": { - "host": "SHARKON2025", - "incident": {"status": "open", "verdict": "incident"}, - "sourceBucket": "aw-dlp-endpoint-signals_SHARKON2025", - "sourceEvent": { - "data": { - "signalType": "print_job", - "hostname": "SHARKON2025", - "username": "Администратор", - "documentName": "Письмо", - "printerName": "HP", - "source": "endpoint-signals-phase2", - } - }, - }, - } - normalized = MODULE.normalize_incident(item, "SHARKON2025") - assert normalized["signal_type"] == "print_job" - assert normalized["username"] == "Администратор" - assert normalized["action"] == "incident" - assert normalized["incident_status"] == "open" - - -def test_build_case_lines_emits_case_state(): - cases = [ - { - "id": 28, - "host": "SHARKON2025", - "status": "open", - "severity": "medium", - "assignee": None, - "title": "DLP print_job · Администратор", - "incident_id": "case-1", - "evidence": {"items": [1], "chain_length": 1}, - "forensics": None, - "updated_at": "2026-05-15T10:03:00+00:00", - } - ] - lines = MODULE.build_case_lines("SHARKON2025", cases) - assert len(lines) == 1 - assert lines[0].startswith("aw_dlp_case,") - - -def test_timestamp_parser_accepts_zulu(): - assert MODULE.pts("2026-05-15T10:00:00Z") == datetime(2026, 5, 15, 10, 0, 0, tzinfo=UTC) diff --git a/aw-server/test_aw_dlp_report_generator.py b/aw-server/test_aw_dlp_report_generator.py deleted file mode 100644 index 214abaf..0000000 --- a/aw-server/test_aw_dlp_report_generator.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import os -import sys -from pathlib import Path - - -MODULE_PATH = Path(__file__).parent / "dlp-compliance" / "report_generator.py" - - -def load_module(name: str): - spec = importlib.util.spec_from_file_location(name, MODULE_PATH) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -MODULE = load_module("aw_dlp_report_generator") - - -def test_build_aw_api_base_accepts_root_and_api_urls(): - assert MODULE.build_aw_api_base("http://127.0.0.1:5600") == "http://127.0.0.1:5600/api/0" - assert MODULE.build_aw_api_base("http://127.0.0.1:5600/") == "http://127.0.0.1:5600/api/0" - assert MODULE.build_aw_api_base("http://127.0.0.1:5600/api/0") == "http://127.0.0.1:5600/api/0" - - -def test_aw_dlp_api_base_env_takes_precedence(monkeypatch): - monkeypatch.setenv("AW_SERVER_URL", "http://127.0.0.1:5600") - monkeypatch.setenv("AW_DLP_AW_API_BASE", "http://127.0.0.1:5600/api/0") - module = load_module("aw_dlp_report_generator_env_precedence") - assert module.AW_API_BASE == "http://127.0.0.1:5600/api/0" - - -def test_aw_server_url_is_normalized_when_dlp_api_base_is_missing(monkeypatch): - monkeypatch.setenv("AW_SERVER_URL", "http://127.0.0.1:5600") - monkeypatch.delenv("AW_DLP_AW_API_BASE", raising=False) - module = load_module("aw_dlp_report_generator_server_url_fallback") - assert module.AW_API_BASE == "http://127.0.0.1:5600/api/0" diff --git a/aw-server/test_aw_rus_healthd.py b/aw-server/test_aw_rus_healthd.py deleted file mode 100644 index 72fad82..0000000 --- a/aw-server/test_aw_rus_healthd.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import sys -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-rus-healthd.py") -SPEC = importlib.util.spec_from_file_location("aw_rus_healthd", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -def test_host_activity_marks_recent_active_session(): - MODULE.now_utc = lambda: MODULE.parse_ts("2026-05-18T10:05:00Z") - activity = MODULE.host_activity_from_worktime( - { - "timestamp": "2026-05-18T10:00:00Z", - "data": {"active": True}, - }, - max_age_seconds=900, - ) - assert activity["active"] is True - assert activity["fresh"] is True - - -def test_bucket_health_reports_missing_events_without_crash(monkeypatch): - monkeypatch.setattr(MODULE, "latest_bucket_event", lambda api_base, bucket_id: None) - status, summary, details = MODULE.bucket_health( - "http://127.0.0.1:5600/api/0", - "aw-watcher-window_SHARKON2025", - 900, - missing_status="warn", - stale_status="fail", - ) - assert status == "warn" - assert summary == "no events" - assert details["bucket"] == "aw-watcher-window_SHARKON2025" - - -def test_inactive_interactive_bucket_can_be_downgraded_to_ok(monkeypatch): - monkeypatch.setattr( - MODULE, - "latest_bucket_event", - lambda api_base, bucket_id: {"timestamp": "2026-05-30T07:00:00Z"}, - ) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T12:00:00Z")) - activity = {"active": False, "fresh": True, "age_seconds": 5} - status, summary, details = MODULE.bucket_health( - "http://127.0.0.1:5600/api/0", - "aw-watcher-window_SHARKON2025", - 900, - missing_status="warn", - stale_status="warn", - ) - details["interactive_required"] = bool(activity["active"]) - details["host_activity"] = activity - if not activity["active"] and status != "ok": - details["inactive_summary"] = summary - status = "ok" - summary = "inactive: no active interactive users" - - assert status == "ok" - assert summary == "inactive: no active interactive users" - assert details["inactive_summary"] == "stale (18000s)" - - -def test_bucket_health_marks_old_session_events_as_event_driven(monkeypatch): - monkeypatch.setattr( - MODULE, - "latest_bucket_event", - lambda api_base, bucket_id: {"timestamp": "2026-05-22T15:53:08Z"}, - ) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-24T01:46:25Z")) - status, summary, details = MODULE.bucket_health( - "http://127.0.0.1:5600/api/0", - "aw-session-events_SHARKON2025", - 86400, - missing_status="fail", - stale_status="warn", - ) - if status == "warn" and details.get("age_seconds") is not None: - status = "ok" - summary = f"event-driven ({details['age_seconds']}s since last logon marker)" - assert status == "ok" - assert summary.startswith("event-driven (") - assert details["age_seconds"] > 86400 - - -def test_bucket_timestamp_health_uses_bucket_metadata_without_event_query(monkeypatch): - def fail_if_called(api_base, bucket_id): - raise AssertionError("latest event query should not run when bucket metadata has end timestamp") - - monkeypatch.setattr(MODULE, "latest_bucket_event", fail_if_called) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T17:05:00Z")) - - status, summary, details = MODULE.bucket_timestamp_health( - "http://127.0.0.1:5600/api/0", - {"aw-session-events_SHARKON2025": {"metadata": {"end": "2026-05-30T17:00:00Z"}}}, - "aw-session-events_SHARKON2025", - 86400, - missing_status="fail", - stale_status="warn", - ) - - assert status == "ok" - assert summary == "fresh (300s)" - assert details["timestamp_source"] == "bucket_metadata.end" - assert details["age_seconds"] == 300 - - -def test_bucket_timestamp_health_falls_back_to_event_query_without_metadata(monkeypatch): - monkeypatch.setattr( - MODULE, - "latest_bucket_event", - lambda api_base, bucket_id: {"timestamp": "2026-05-30T17:00:00Z"}, - ) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T17:05:00Z")) - - status, summary, details = MODULE.bucket_timestamp_health( - "http://127.0.0.1:5600/api/0", - {"aw-session-events_SHARKON2025": {}}, - "aw-session-events_SHARKON2025", - 86400, - missing_status="fail", - stale_status="warn", - ) - - assert status == "ok" - assert summary == "fresh (300s)" - assert "timestamp_source" not in details - - -def test_guard_bucket_missing_is_warn_until_required(monkeypatch): - monkeypatch.setattr(MODULE, "latest_bucket_event", lambda api_base, bucket_id: None) - status, summary, details = MODULE.guard_bucket_health( - "http://127.0.0.1:5600/api/0", - "SHARKON2025", - 300, - required=False, - ) - assert status == "warn" - assert summary == "no guard heartbeat" - assert details["required"] is False - - -def test_guard_bucket_required_stale_is_fail(monkeypatch): - monkeypatch.setattr( - MODULE, - "latest_bucket_event", - lambda api_base, bucket_id: { - "timestamp": "2026-05-30T10:00:00Z", - "data": {"status": "ok", "mode": "enforce"}, - }, - ) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T10:10:00Z")) - status, summary, details = MODULE.guard_bucket_health( - "http://127.0.0.1:5600/api/0", - "SHARKON2025", - 300, - required=True, - ) - assert status == "fail" - assert summary == "guard stale (600s)" - assert details["mode"] == "enforce" - - -def test_aw_wrapper_failure_can_be_advisory_warn(tmp_path): - report = MODULE.Report() - missing = tmp_path / "missing-wrapper" - MODULE.check_wrapper(report, "wrapper:test", [str(missing)], failure_status="warn") - assert report.results[0].status == "warn" - - -def test_dlp_wrapper_failure_can_be_advisory_warn(monkeypatch, tmp_path): - wrapper = tmp_path / "dlp-health-check" - wrapper.write_text("#!/bin/sh\nprintf '{\"ok\":false}'\nexit 1\n", encoding="utf-8") - wrapper.chmod(0o755) - monkeypatch.setattr(MODULE, "run_command", lambda cmd: (1, '{"ok": false}')) - - report = MODULE.Report() - MODULE.check_wrapper( - report, - "wrapper:dlp-health-check", - [str(wrapper), "--json"], - json_mode=True, - failure_status="warn", - ) - - assert report.results[0].status == "warn" - assert report.results[0].details["payload"] == {"ok": False} - - -def test_load_env_file_ignores_permission_denied(): - class DeniedPath: - def exists(self): - return True - - def read_text(self, encoding="utf-8"): - raise PermissionError("denied") - - MODULE.load_env_file(DeniedPath()) - - -def test_chmod_if_possible_sets_readable_mode(tmp_path): - path = tmp_path / "aw-rus-health.json" - path.write_text("{}", encoding="utf-8") - - MODULE.chmod_if_possible(path, 0o644) - - assert oct(path.stat().st_mode & 0o777) == "0o644" diff --git a/aw-server/test_aw_slo_monitor.py b/aw-server/test_aw_slo_monitor.py deleted file mode 100644 index 00b55ee..0000000 --- a/aw-server/test_aw_slo_monitor.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import sys -import tempfile -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-slo-monitor.py") -SPEC = importlib.util.spec_from_file_location("aw_slo_monitor", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class FakeHeaders: - def __init__(self, values): - self.values = values - - def get(self, key, default=""): - return self.values.get(key, default) - - -class FakeResponse: - def __init__(self, body, status=200, content_type="text/plain"): - self.body = body - self.status = status - self.headers = FakeHeaders({"Content-Type": content_type}) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - return False - - def read(self): - return self.body - - -def test_summarize_window_calculates_9997_budget(): - now = MODULE.parse_ts("2026-05-30T12:00:00Z") - samples = [ - {"ts": "2026-05-30T11:58:00Z", "ok": True}, - {"ts": "2026-05-30T11:59:00Z", "ok": False}, - {"ts": "2026-05-30T12:00:00Z", "ok": True}, - ] - - summary = MODULE.summarize_window( - samples, - now=now, - window_seconds=86400, - sample_interval_seconds=60, - target_percent=99.97, - ) - - assert summary["samples"] == 3 - assert summary["good_samples"] == 2 - assert summary["bad_samples"] == 1 - assert summary["availability_percent"] == 66.66667 - assert summary["budget_seconds"] == 25 - assert summary["budget_remaining_seconds"] == -35 - assert summary["status"] == "burning" - - -def test_summarize_window_status_uses_remaining_error_budget_for_partial_window(): - now = MODULE.parse_ts("2026-05-30T12:00:00Z") - samples = [ - {"ts": "2026-05-30T11:59:30Z", "ok": False}, - {"ts": "2026-05-30T11:59:45Z", "ok": True}, - {"ts": "2026-05-30T12:00:00Z", "ok": True}, - ] - - summary = MODULE.summarize_window( - samples, - now=now, - window_seconds=86400, - sample_interval_seconds=15, - target_percent=99.97, - ) - - assert summary["availability_percent"] == 66.66667 - assert summary["budget_seconds"] == 25 - assert summary["budget_remaining_seconds"] == 10 - assert summary["status"] == "ok" - - -def test_append_and_trim_sample_keeps_retention_window(): - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "samples.jsonl" - path.write_text( - '{"ts":"2026-05-01T00:00:00Z","ok":true}\n' - '{"ts":"2026-05-30T11:59:00Z","ok":true}\n', - encoding="utf-8", - ) - sample = {"ts": "2026-05-30T12:00:00Z", "ok": False} - - samples = MODULE.append_and_trim_sample(path, sample, retention_seconds=3600) - - assert [item["ts"] for item in samples] == [ - "2026-05-30T11:59:00Z", - "2026-05-30T12:00:00Z", - ] - assert "2026-05-01T00:00:00Z" not in path.read_text(encoding="utf-8") - - -def test_load_samples_treats_unreadable_file_as_empty(monkeypatch, tmp_path): - path = tmp_path / "samples.jsonl" - path.write_text('{"ts":"2026-05-30T12:00:00Z","ok":true}\n', encoding="utf-8") - original_read_text = MODULE.Path.read_text - - def denied(self, *args, **kwargs): - if self == path: - raise PermissionError("denied") - return original_read_text(self, *args, **kwargs) - - monkeypatch.setattr(MODULE.Path, "read_text", denied) - - assert MODULE.load_samples(path, MODULE.parse_ts("2026-05-30T11:00:00Z")) == [] - - -def test_render_summary_text_includes_budget_remaining(): - summary = { - "generated_at_utc": "2026-05-30T12:00:00Z", - "target_percent": 99.97, - "current_sample": { - "ok": True, - "probes": {"worktime_today_csv": {"ok": True, "status": 200}}, - }, - "windows": { - "24h": { - "status": "ok", - "availability_percent": 100.0, - "samples": 10, - "bad_samples": 0, - "observed_bad_seconds": 0, - "budget_remaining_seconds": 25, - } - }, - } - - text = MODULE.render_summary_text(summary) - - assert "Target: 99.97%" in text - assert "24h: ok availability=100.00000%" in text - assert "budget_remaining_seconds=25" in text - assert "- worktime_today_csv: OK 200" in text - - -def test_html_probe_requires_non_empty_html_with_markers(monkeypatch): - body = b'ActivityWatch
    ' - monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="text/html; charset=utf-8")) - - result = MODULE.html_probe( - "http://127.0.0.1:5600/", - 5, - min_bytes=100, - required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), - ) - - assert result["ok"] is True - assert result["status"] == 200 - assert result["body_bytes"] == len(body) - assert "body" not in result - - -def test_html_probe_fails_on_missing_marker(monkeypatch): - monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(b"", content_type="text/html")) - - result = MODULE.html_probe( - "http://127.0.0.1:5600/", - 5, - min_bytes=1, - required_markers=("ActivityWatch",), - ) - - assert result["ok"] is False - assert result["missing_markers"] == ["ActivityWatch"] - - -def test_html_probe_retries_transient_fetch_error(monkeypatch): - calls = {"count": 0} - body = b"ActivityWatch" - - def fake_urlopen(req, timeout): - calls["count"] += 1 - if calls["count"] == 1: - raise TimeoutError("timed out") - return FakeResponse(body, content_type="text/html") - - monkeypatch.setattr(MODULE.request, "urlopen", fake_urlopen) - monkeypatch.setattr(MODULE.time, "sleep", lambda seconds: None) - - result = MODULE.html_probe( - "http://127.0.0.1:5600/", - 5, - min_bytes=1, - required_markers=("ActivityWatch",), - ) - - assert result["ok"] is True - assert result["attempts"] == 2 - assert calls["count"] == 2 - - -def test_json_probe_validates_shape_and_expected_values(monkeypatch): - body = b'{"generated_at_utc":"2026-05-30T12:00:00Z","host":"SHARKON2025","summary":{},"rows":[],"workday":{}}' - monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="application/json")) - - result = MODULE.json_probe( - "http://127.0.0.1:5610/reports/worktime/management?format=json", - 5, - expected_values={"host": "SHARKON2025"}, - required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), - ) - - assert result["ok"] is True - assert result["status"] == 200 - assert "body" not in result - - -def test_json_probe_fails_on_wrong_host(monkeypatch): - body = b'{"generated_at_utc":"2026-05-30T12:00:00Z","host":"OTHER","summary":{},"rows":[],"workday":{}}' - monkeypatch.setattr(MODULE.request, "urlopen", lambda req, timeout: FakeResponse(body, content_type="application/json")) - - result = MODULE.json_probe( - "http://127.0.0.1:5610/reports/worktime/management?format=json", - 5, - expected_values={"host": "SHARKON2025"}, - required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), - ) - - assert result["ok"] is False - assert result["mismatched_values"]["host"]["actual"] == "OTHER" - - -def test_load_env_file_ignores_permission_denied(monkeypatch): - class DeniedPath: - def exists(self): - return True - - def read_text(self, encoding="utf-8"): - raise PermissionError("denied") - - MODULE.load_env_file(DeniedPath()) - - -def test_read_healthd_state_requires_fresh_ok_snapshot(monkeypatch, tmp_path): - path = tmp_path / "aw-rus-health.json" - path.write_text( - '{"generated_at_utc":"2026-05-30T12:00:00Z","ok":true,"counts":{"ok":1,"warn":0,"fail":0}}', - encoding="utf-8", - ) - monkeypatch.setattr(MODULE, "now_utc", lambda: MODULE.parse_ts("2026-05-30T12:01:00Z")) - - fresh = MODULE.read_healthd_state(path, max_age_seconds=180) - stale = MODULE.read_healthd_state(path, max_age_seconds=30) - - assert fresh["ok"] is True - assert fresh["age_seconds"] == 60 - assert stale["ok"] is False - - -def test_chmod_if_possible_sets_readable_mode(tmp_path): - path = tmp_path / "summary.json" - path.write_text("{}", encoding="utf-8") - - MODULE.chmod_if_possible(path, 0o644) - - assert oct(path.stat().st_mode & 0o777) == "0o644" diff --git a/aw-server/test_aw_worktime_api.py b/aw-server/test_aw_worktime_api.py deleted file mode 100644 index 02acff8..0000000 --- a/aw-server/test_aw_worktime_api.py +++ /dev/null @@ -1,1078 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import json -import tempfile -from datetime import datetime, timezone -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-worktime-api.py") -SPEC = importlib.util.spec_from_file_location("aw_worktime_api", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -class _FakeWriter: - def __init__(self, exc=None): - self.exc = exc - self.data = b"" - - def write(self, data): - if self.exc is not None: - raise self.exc - self.data += data - - -class _FakeHandler: - def __init__(self, exc=None): - self.wfile = _FakeWriter(exc=exc) - self.status = None - self.headers = {} - self.ended = False - - def send_response(self, status): - self.status = status - - def send_header(self, key, value): - self.headers[key] = value - - def end_headers(self): - self.ended = True - - -def _event(ts, username, session_id, active, **extra): - data = { - "username": username, - "userId": f"WORKGROUP\\{username}", - "sessionId": session_id, - "state": "Активно" if active else "Диск", - "active": active, - } - data.update(extra) - return {"timestamp": ts, "duration": 0.0, "data": data} - - -def test_aggregate_rows_uses_sample_seconds_and_merges_overlap(): - start = datetime(2026, 5, 14, 6, 0, 0, tzinfo=timezone.utc) - end = datetime(2026, 5, 14, 6, 59, 59, tzinfo=timezone.utc) - rows = MODULE.aggregate_rows( - [ - _event("2026-05-14T06:00:00Z", "user5", 4, True, sampleSeconds=30), - _event("2026-05-14T06:00:30Z", "user5", 4, True, sampleSeconds=30), - _event("2026-05-14T06:00:15Z", "user5", 5, True, sampleSeconds=30), - _event("2026-05-14T06:01:00Z", "user5", 4, False, sampleSeconds=30), - ], - start, - end, - "SHARKON2025", - ) - assert len(rows) == 1 - row = rows[0] - assert row["user"] == "user5" - assert row["user_id"] == "SHARKON2025\\user5" - assert row["active_seconds"] == 60 - assert row["active_hhmm"] == "00:01" - assert row["sessions_count"] == 2 - assert row["samples_count"] == 4 - assert row["active_samples"] == 3 - assert row["first_activity"] == "2026-05-14T06:00:00Z" - assert row["last_activity"] == "2026-05-14T06:01:00Z" - - -def test_aggregate_rows_falls_back_to_next_sample_delta(): - start = datetime(2026, 5, 14, 7, 0, 0, tzinfo=timezone.utc) - end = datetime(2026, 5, 14, 7, 59, 59, tzinfo=timezone.utc) - rows = MODULE.aggregate_rows( - [ - _event("2026-05-14T07:00:00Z", "user1", 3, True), - _event("2026-05-14T07:00:05Z", "user1", 3, True), - _event("2026-05-14T07:00:10Z", "user1", 3, False), - ], - start, - end, - "SHARKON2025", - ) - row = rows[0] - assert row["active_seconds"] == 10 - assert row["active_hhmm"] == "00:00" - - -def test_build_aw_api_base_accepts_root_and_api_urls(): - assert MODULE.build_aw_api_base("http://127.0.0.1:5600") == "http://127.0.0.1:5600/api/0" - assert MODULE.build_aw_api_base("http://127.0.0.1:5600/") == "http://127.0.0.1:5600/api/0" - assert MODULE.build_aw_api_base("http://127.0.0.1:5600/api/0") == "http://127.0.0.1:5600/api/0" - - -def test_worktime_health_payload_is_lightweight_and_ok(): - payload = MODULE.worktime_health_payload() - assert payload["ok"] is True - assert payload["default_host"] == MODULE.DEFAULT_HOST - assert payload["aw_api_base"] == MODULE.AW - assert payload["report_timezone"] == str(MODULE.REPORT_TZ) - assert payload["generated_at_utc"].endswith("Z") - - -def test_aggregate_hourly_rows_splits_interval_by_local_hour(): - start = datetime(2026, 5, 14, 6, 0, 0, tzinfo=timezone.utc) - end = datetime(2026, 5, 14, 8, 59, 59, tzinfo=timezone.utc) - rows = MODULE.aggregate_hourly_rows( - [ - _event("2026-05-14T06:50:00Z", "user5", 4, True, sampleSeconds=1800), - _event("2026-05-14T07:20:00Z", "user5", 4, True, sampleSeconds=1800), - ], - start, - end, - "SHARKON2025", - ) - assert [row["hour_local"] for row in rows] == ["09:00", "10:00"] - assert [row["active_seconds"] for row in rows] == [300, 300] - - -def test_build_true_active_apps_requires_foreground_not_afk_and_evidence(): - start = datetime(2026, 5, 14, 6, 0, 0, tzinfo=timezone.utc) - end = datetime(2026, 5, 14, 7, 59, 59, tzinfo=timezone.utc) - window_events = [ - {"timestamp": "2026-05-14T06:00:00Z", "duration": 120, "data": {"app": "1cv8.exe", "title": "ИНФОВЕСТ"}}, - {"timestamp": "2026-05-14T06:02:00Z", "duration": 180, "data": {"app": "1cv8.exe", "title": "Счета учета: Материалы"}}, - {"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"app": "totalcmd.exe", "title": "Total Commander"}}, - ] - afk_events = [ - {"timestamp": "2026-05-14T06:00:00Z", "duration": 600, "data": {"status": "not-afk"}}, - {"timestamp": "2026-05-14T07:00:00Z", "duration": 600, "data": {"status": "not-afk"}}, - ] - evidence_events_by_bucket = { - "aw-file-operations_SHARKON2025": [ - { - "timestamp": "2026-05-14T07:05:00Z", - "duration": 0, - "data": {"signalType": "file_write", "path": "C:\\data\\report.xlsx"}, - } - ], - "aw-dlp-endpoint-signals_SHARKON2025": [ - { - "timestamp": "2026-05-14T07:06:00Z", - "duration": 0, - "data": {"signalType": "collector_health", "eventsFlushed": 100}, - } - ], - } - - rows = MODULE.build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end) - - by_app = {row["application"]: row for row in rows} - assert "1С" in by_app - assert "Total Commander" in by_app - assert by_app["1С"]["proved_work_seconds"] == 300 - assert by_app["1С"]["last_action"] == "Счета учета: Материалы" - assert by_app["Total Commander"]["proved_work_seconds"] == 480 - assert by_app["Total Commander"]["last_action"] == "C:\\data\\report.xlsx" - - -def test_latest_bucket_event_falls_back_to_bucket_metadata_end(monkeypatch): - def fake_get(url): - if url.endswith("/events?limit=1"): - raise RuntimeError("events timeout") - if url.endswith("/buckets/aw-worktime-sessions_SHARKON2025"): - return {"metadata": {"end": "2026-05-30T10:38:07.530Z"}} - raise AssertionError(url) - - monkeypatch.setattr(MODULE, "get", fake_get) - - event = MODULE.latest_bucket_event("aw-worktime-sessions_SHARKON2025") - - assert event["timestamp"] == "2026-05-30T10:38:07.530Z" - assert event["data"]["source"] == "bucket_metadata" - - -def test_fetch_events_for_date_queries_aw_with_report_bounds(monkeypatch): - urls = [] - - def fake_get(url): - urls.append(url) - if url.endswith("/buckets/aw-worktime-sessions_SHARKON2025"): - return {"metadata": {"end": "2026-05-30T10:38:07.530Z"}} - if "/events?" in url: - return [] - raise AssertionError(url) - - monkeypatch.setattr(MODULE, "get", fake_get) - monkeypatch.setattr(MODULE, "REPORT_TZ", timezone.utc) - original_cache = MODULE._EVENTS_CACHE - try: - MODULE._EVENTS_CACHE = {} - MODULE.fetch_events_for_date("SHARKON2025", datetime(2026, 5, 30, tzinfo=timezone.utc).date()) - finally: - MODULE._EVENTS_CACHE = original_cache - - event_urls = [url for url in urls if "/events?" in url] - assert len(event_urls) == 1 - assert "start=2026-05-30T00%3A00%3A00Z" in event_urls[0] - assert "end=2026-05-30T23%3A59%3A59" in event_urls[0] - - -def test_render_html_contains_true_active_apps_table(): - html = MODULE.render_html( - [], - "SHARKON2025", - datetime(2026, 5, 14, tzinfo=timezone.utc).date(), - selected_day="today", - true_active_apps=[ - { - "application": "1С", - "proved_work_human": "34 мин", - "proved_work_hhmm": "00:34", - "last_action_local": "15:31", - "last_action": "Счета учета: Материалы", - } - ], - ) - assert "Доказанная работа по приложениям" in html - assert "Приложение" in html - assert "Доказанная работа" in html - assert "Последнее действие" in html - assert "Счета учета: Материалы" in html - - -def test_build_management_payload_creates_actions_for_missing_and_late_users(): - rows = [ - { - "user": "user1", - "user_id": "SHARKON2025\\user1", - "active_seconds": 0, - "active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "idle_seconds": 86400, - "sessions_count": 1, - "samples_count": 10, - "active_samples": 0, - "_intervals": [], - }, - { - "user": "user5", - "user_id": "SHARKON2025\\user5", - "active_seconds": 5400, - "active_hhmm": "01:30", - "first_activity": "2026-05-14T08:30:00Z", - "last_activity": "2026-05-14T10:00:00Z", - "idle_seconds": 81000, - "sessions_count": 1, - "samples_count": 180, - "active_samples": 180, - "_intervals": [ - ( - datetime(2026, 5, 14, 8, 30, 0, tzinfo=timezone.utc), - datetime(2026, 5, 14, 10, 0, 0, tzinfo=timezone.utc), - ) - ], - }, - ] - payload = MODULE.build_management_payload(rows, "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) - assert payload["summary"]["users_count"] == 2 - assert payload["summary"]["inactive_users"] == 1 - assert payload["summary"]["actions_count"] >= 2 - action_ids = {action["action_id"] for action in payload["actions"]} - assert "missing_activity" in action_ids - assert "late_start_review" in action_ids - assert payload["actions"][0]["priority"] == "critical" - - -def test_build_management_payload_uses_workday_window_not_midnight_activity(): - rows = [ - { - "user": "администратор", - "user_id": "SHARKON2025\\Администратор", - "active_seconds": 39600, - "active_hhmm": "11:00", - "first_activity": "2026-05-14T00:00:00Z", - "last_activity": "2026-05-14T11:00:00Z", - "idle_seconds": 0, - "sessions_count": 1, - "samples_count": 100, - "active_samples": 100, - "_intervals": [ - ( - datetime(2026, 5, 14, 0, 0, 0, tzinfo=timezone.utc), - datetime(2026, 5, 14, 11, 0, 0, tzinfo=timezone.utc), - ) - ], - } - ] - payload = MODULE.build_management_payload(rows, "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) - roster = payload["rows"][0] - assert roster["calendar_active_hhmm"] == "11:00" - assert roster["workday_active_hhmm"] == "05:00" - assert roster["coverage_pct"] == 55.56 - assert payload["summary"]["calendar_total_active_hhmm"] == "11:00" - assert payload["summary"]["workday_total_active_hhmm"] == "05:00" - - -def test_build_management_payload_applies_alias_and_executive_summary(): - original = MODULE.load_manager_aliases - original_owners = MODULE.load_manager_owners - original_sources = MODULE.build_source_freshness - original_trend = MODULE.build_management_trend - try: - MODULE.load_manager_aliases = lambda: { - "sharkon2025\\user1": { - "display_name": "Иван Петров", - "manager": "Руководитель смены", - "department": "Бухгалтерия", - "role": "Оператор 1С", - } - } - MODULE.load_manager_owners = lambda: { - "руководитель смены": { - "display_name": "Сменный руководитель", - "title": "Руководитель смены 1С", - "department": "Операторы 1С", - "contact": "@shift-lead", - "escalation_to": "Финансовый директор", - "notes": "Дневной контур", - } - } - MODULE.build_source_freshness = lambda host, **kwargs: ([], []) - MODULE.build_management_trend = lambda host, report_date, owner_filter="", department_filter="", **kwargs: [] - rows = [ - { - "user": "user1", - "user_id": "SHARKON2025\\user1", - "active_seconds": 0, - "active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "idle_seconds": 86400, - "sessions_count": 1, - "samples_count": 10, - "active_samples": 0, - "_intervals": [], - } - ] - payload = MODULE.build_management_payload(rows, "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) - row = payload["rows"][0] - assert row["user"] == "Иван Петров" - assert row["manager_owner"] == "Руководитель смены" - assert row["department"] == "Бухгалтерия" - assert payload["actions"][0]["owner"] == "Руководитель смены" - assert payload["executive"]["portfolio_state"] == "critical" - assert payload["executive"]["focus_items"][0]["owner"] == "Руководитель смены" - assert payload["owner_rollups"][0]["name"] == "Руководитель смены" - assert payload["owner_rollups"][0]["critical_actions_count"] >= 1 - assert payload["department_rollups"][0]["name"] == "Бухгалтерия" - assert payload["owner_roster"][0]["display_name"] == "Сменный руководитель" - assert payload["owner_roster"][0]["contact"] == "@shift-lead" - finally: - MODULE.load_manager_aliases = original - MODULE.load_manager_owners = original_owners - MODULE.build_source_freshness = original_sources - MODULE.build_management_trend = original_trend - - -def test_build_management_payload_filters_by_owner(): - original_aliases = MODULE.load_manager_aliases - original_owners = MODULE.load_manager_owners - original_trend = MODULE.build_management_trend - original_sources = MODULE.build_source_freshness - try: - MODULE.load_manager_aliases = lambda: { - "sharkon2025\\user1": { - "display_name": "Иван Петров", - "manager": "Руководитель смены", - "department": "Бухгалтерия", - "role": "Оператор 1С", - }, - "sharkon2025\\user2": { - "display_name": "Мария Соколова", - "manager": "Финансовый директор", - "department": "Финансы", - "role": "Главбух", - }, - } - MODULE.load_manager_owners = lambda: { - "руководитель смены": {"display_name": "Сменный руководитель", "title": "Руководитель смены 1С"} - } - MODULE.build_management_trend = lambda *args, **kwargs: [] - MODULE.build_source_freshness = lambda host, **kwargs: ([], []) - rows = [ - { - "user": "user1", - "user_id": "SHARKON2025\\user1", - "active_seconds": 0, - "active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "idle_seconds": 86400, - "sessions_count": 1, - "samples_count": 10, - "active_samples": 0, - "_intervals": [], - }, - { - "user": "user2", - "user_id": "SHARKON2025\\user2", - "active_seconds": 3600, - "active_hhmm": "01:00", - "first_activity": "2026-05-14T07:00:00Z", - "last_activity": "2026-05-14T08:00:00Z", - "idle_seconds": 82800, - "sessions_count": 1, - "samples_count": 120, - "active_samples": 120, - "_intervals": [ - ( - datetime(2026, 5, 14, 7, 0, 0, tzinfo=timezone.utc), - datetime(2026, 5, 14, 8, 0, 0, tzinfo=timezone.utc), - ) - ], - }, - ] - payload = MODULE.build_management_payload( - rows, - "SHARKON2025", - datetime(2026, 5, 14, tzinfo=timezone.utc).date(), - owner_filter="Руководитель смены", - ) - assert payload["filters"]["owner"] == "Руководитель смены" - assert payload["filters"]["department"] == "" - assert payload["summary"]["users_count"] == 1 - assert [row["user"] for row in payload["rows"]] == ["Иван Петров"] - assert len(payload["owner_rollups"]) == 1 - assert payload["owner_rollups"][0]["name"] == "Руководитель смены" - assert len(payload["department_rollups"]) == 1 - assert payload["department_rollups"][0]["name"] == "Бухгалтерия" - assert payload["owner_roster"][0]["display_name"] == "Сменный руководитель" - assert all(action["owner"] == "Руководитель смены" for action in payload["actions"]) - finally: - MODULE.load_manager_aliases = original_aliases - MODULE.load_manager_owners = original_owners - MODULE.build_management_trend = original_trend - MODULE.build_source_freshness = original_sources - - -def _minimal_management_payload(report_date, users_count=1): - return { - "summary": { - "users_count": users_count, - "active_users": users_count, - "inactive_users": 0, - "workday_total_active_seconds": 3600 * users_count, - "workday_total_active_hhmm": f"0{users_count}:00", - "portfolio_coverage_pct": 100.0, - "actions_count": 0, - "critical_actions_count": 0, - }, - "rows": [], - "actions": [], - "sources": [], - "filters": {"owner": "", "department": ""}, - } - - -def test_load_management_cache_keeps_historical_reports_after_ttl(): - original_dir = MODULE.MANAGER_CACHE_DIR - original_ttl = MODULE.MANAGER_CACHE_TTL_SECONDS - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.MANAGER_CACHE_DIR = Path(tmp) - MODULE.MANAGER_CACHE_TTL_SECONDS = 1 - report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() - payload = _minimal_management_payload(report_date) - path = MODULE.management_cache_path("SHARKON2025", report_date) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload), encoding="utf-8") - loaded = MODULE.load_management_cache("SHARKON2025", report_date) - assert loaded["summary"]["users_count"] == 1 - finally: - MODULE.MANAGER_CACHE_DIR = original_dir - MODULE.MANAGER_CACHE_TTL_SECONDS = original_ttl - - -def test_build_management_trend_reuses_precomputed_anchor_payload(): - original_trend_days = MODULE.MANAGER_TREND_DAYS - original_load = MODULE.load_management_cache - original_fetch = MODULE.fetch_events_for_date - original_aggregate = MODULE.aggregate_rows_with_intervals - fetch_dates = [] - try: - MODULE.MANAGER_TREND_DAYS = 3 - anchor = datetime(2026, 5, 14, tzinfo=timezone.utc).date() - - def fake_load(host, report_date): - if report_date < anchor: - return _minimal_management_payload(report_date) - return None - - def fake_fetch(host, report_date): - fetch_dates.append(report_date) - return MODULE.get_report_bounds(report_date), [] - - MODULE.load_management_cache = fake_load - MODULE.fetch_events_for_date = fake_fetch - MODULE.aggregate_rows_with_intervals = lambda events, start, end, host: [] - trend = MODULE.build_management_trend( - "SHARKON2025", - anchor, - precomputed_payloads={anchor: _minimal_management_payload(anchor, users_count=2)}, - ) - assert [item["report_date"] for item in trend] == ["2026-05-12", "2026-05-13", "2026-05-14"] - assert trend[-1]["users_count"] == 2 - assert fetch_dates == [] - finally: - MODULE.MANAGER_TREND_DAYS = original_trend_days - MODULE.load_management_cache = original_load - MODULE.fetch_events_for_date = original_fetch - MODULE.aggregate_rows_with_intervals = original_aggregate - - -def test_render_management_html_contains_action_queue(): - payload = { - "generated_at_utc": "2026-05-14T12:00:00Z", - "host": "SHARKON2025", - "report_date": "2026-05-14", - "report_timezone": "Europe/Moscow", - "filters": { - "owner": "Руководитель смены", - "department": "Бухгалтерия", - }, - "workday": { - "start_local": "2026-05-14T09:00:00+03:00", - "end_local": "2026-05-14T18:00:00+03:00", - "expected_seconds_per_user": 32400, - "expected_hhmm_per_user": "09:00", - "target_coverage_pct": 75, - "low_coverage_pct": 35, - }, - "summary": { - "users_count": 1, - "active_users": 0, - "inactive_users": 1, - "on_target_users": 0, - "below_target_users": 0, - "portfolio_coverage_pct": 0.0, - "actions_count": 1, - "critical_actions_count": 1, - "high_actions_count": 0, - "total_active_seconds": 0, - "total_active_hhmm": "00:00", - "calendar_total_active_hhmm": "00:00", - "workday_total_active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "top_user": "", - "top_user_active_hhmm": "00:00", - }, - "actions": [ - { - "action_id": "missing_activity", - "priority": "critical", - "owner": "Руководитель смены", - "user_id": "SHARKON2025\\user1", - "deadline_hint": "today", - "reason": "Нет активности", - "recommended_action": "Проверить сотрудника", - "evidence": {}, - } - ], - "rows": [ - { - "user": "Иван Петров", - "user_id": "SHARKON2025\\user1", - "canonical_user_id": "SHARKON2025\\user1", - "manager_owner": "Руководитель смены", - "department": "Бухгалтерия", - "active_hhmm": "00:00", - "calendar_active_hhmm": "00:00", - "workday_active_hhmm": "00:00", - "coverage_pct": 0.0, - "status": "inactive", - "first_activity_local": "", - "last_activity_local": "", - "workday_first_activity_local": "", - "workday_last_activity_local": "", - "sessions_count": 1, - } - ], - "executive": { - "portfolio_state": "critical", - "headline": "Есть 1 критичный вопрос, требующий решения сегодня.", - "message": "Активны 0 из 1 сотрудников. Покрытие рабочего окна 0.0%.", - "focus_items": [ - { - "priority": "critical", - "owner": "Руководитель смены", - "title": "missing_activity", - "reason": "Нет активности", - "recommended_action": "Проверить сотрудника", - } - ], - "stale_sources": [], - }, - "owner_rollups": [ - { - "name": "Руководитель смены", - "users_count": 1, - "active_users": 0, - "inactive_users": 1, - "below_target_users": 0, - "workday_total_active_seconds": 0, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 1, - "critical_actions_count": 1, - "high_actions_count": 0, - "medium_actions_count": 0, - "low_actions_count": 0, - "users": ["Иван Петров"], - } - ], - "department_rollups": [ - { - "name": "Бухгалтерия", - "users_count": 1, - "active_users": 0, - "inactive_users": 1, - "below_target_users": 0, - "workday_total_active_seconds": 0, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 1, - "critical_actions_count": 1, - "high_actions_count": 0, - "medium_actions_count": 0, - "low_actions_count": 0, - "users": ["Иван Петров"], - } - ], - "owner_roster": [ - { - "name": "Руководитель смены", - "display_name": "Сменный руководитель", - "title": "Руководитель смены 1С", - "department": "Операторы 1С", - "contact": "@shift-lead", - "escalation_to": "Финансовый директор", - "notes": "Дневной контур", - "users_count": 1, - "active_users": 0, - "inactive_users": 1, - "below_target_users": 0, - "workday_total_active_seconds": 0, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 1, - "critical_actions_count": 1, - "high_actions_count": 0, - "medium_actions_count": 0, - "low_actions_count": 0, - "users": ["Иван Петров"] - } - ], - "trend": [ - { - "report_date": "2026-05-14", - "users_count": 1, - "active_users": 0, - "inactive_users": 1, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 1, - "critical_actions_count": 1, - } - ], - "sources": [ - { - "label": "RDP worktime sessions", - "status": "ok", - "status_label": "fresh", - "bucket_id": "aw-worktime-sessions_SHARKON2025", - "timestamp": "2026-05-14T12:00:00Z", - "age_seconds": 30, - "required": True, - "summary": "fresh (30s)", - "event_summary": "queue=0 failures=0 flushed=10", - } - ], - "bucket_id": "aw-worktime-sessions_SHARKON2025", - "report_bounds": { - "start_utc": "2026-05-13T21:00:00Z", - "end_utc": "2026-05-14T20:59:59Z", - }, - } - html = MODULE.render_management_html(payload, selected_day="today") - assert "Управленческий отчёт по работе в RDP" in html - assert "Очередь действий руководителя" in html - assert "missing_activity" in html - assert "Тренд за" in html - assert "По ответственным" in html - assert "Ответственные и эскалация" in html - assert "По подразделениям" in html - assert "Свежесть источников данных" in html - assert "Что делать сегодня" in html - assert "Фильтр:" in html - assert "Сбросить" in html - assert "@shift-lead" in html - assert "Иван Петров" in html - assert "Руководитель смены" in html - - -def test_build_source_freshness_uses_host_fileops_bucket_only(): - original = MODULE.latest_bucket_event - original_now_utc = MODULE.now_utc - try: - def fake_latest(bucket_id): - if bucket_id == "aw-file-operations_SHARKON2025": - return { - "timestamp": "2026-05-23T12:00:00Z", - "data": {"signalType": "collector_health", "queueDepth": 0, "sendFailures": 0, "eventsFlushed": 50}, - } - return None - - MODULE.latest_bucket_event = fake_latest - MODULE.now_utc = lambda: datetime(2026, 5, 23, 12, 5, 0, tzinfo=timezone.utc) - sources, actions = MODULE.build_source_freshness("SHARKON2025") - file_source = next(source for source in sources if source["source_id"] == "file_operations") - assert file_source["bucket_id"] == "aw-file-operations_SHARKON2025" - assert file_source["status"] == "ok" - assert not any(action["evidence"].get("source_id") == "file_operations" for action in actions) - finally: - MODULE.latest_bucket_event = original - MODULE.now_utc = original_now_utc - - -def test_build_source_freshness_marks_desktop_sources_inactive_without_active_users(): - original = MODULE.latest_bucket_event - original_now_utc = MODULE.now_utc - try: - def fake_latest(bucket_id): - if bucket_id in {"aw-watcher-window_SHARKON2025", "aw-file-operations_SHARKON2025"}: - return { - "timestamp": "2026-05-23T11:00:00Z", - "data": {"signalType": "collector_health", "queueDepth": 0, "sendFailures": 0, "eventsFlushed": 50}, - } - if bucket_id.startswith("aw-worktime-sessions_") or bucket_id.startswith("aw-rdp-window_") or bucket_id.startswith("aw-rdp-afk_"): - return { - "timestamp": "2026-05-23T12:00:00Z", - "data": {"status": "ok"}, - } - return None - - MODULE.latest_bucket_event = fake_latest - MODULE.now_utc = lambda: datetime(2026, 5, 23, 12, 5, 0, tzinfo=timezone.utc) - sources, actions = MODULE.build_source_freshness("SHARKON2025", interactive_required=False) - source_by_id = {source["source_id"]: source for source in sources} - assert source_by_id["watcher_window"]["status"] == "inactive" - assert source_by_id["watcher_window"]["status_label"] == "inactive" - assert source_by_id["file_operations"]["status"] == "inactive" - assert source_by_id["file_operations"]["status_label"] == "inactive" - assert not any(action["evidence"].get("source_id") in {"watcher_window", "file_operations"} for action in actions) - finally: - MODULE.latest_bucket_event = original - MODULE.now_utc = original_now_utc - - -def test_resolve_report_format_uses_html_for_browser_accept_without_breaking_api_default(): - assert MODULE.resolve_report_format({}) == "json" - assert MODULE.resolve_report_format({}, "text/html,application/xhtml+xml") == "html" - assert MODULE.resolve_report_format({"format": ["json"]}, "text/html") == "json" - assert MODULE.resolve_report_format({"format": ["csv"]}, "text/html") == "csv" - - -def test_send_bytes_returns_false_on_broken_pipe(): - handler = _FakeHandler(exc=BrokenPipeError()) - ok = MODULE.send_bytes(handler, b"{}", "application/json; charset=utf-8") - assert ok is False - assert handler.status == 200 - assert handler.headers["Content-Type"] == "application/json; charset=utf-8" - assert handler.headers["Content-Length"] == "2" - assert handler.ended is True - - -def test_send_bytes_writes_payload_when_client_is_connected(): - handler = _FakeHandler() - ok = MODULE.send_bytes(handler, b"payload", "text/plain; charset=utf-8", status=201) - assert ok is True - assert handler.status == 201 - assert handler.headers["Content-Length"] == "7" - assert handler.wfile.data == b"payload" - - -def test_report_response_cache_returns_fresh_and_stale_entries(): - original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS - original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS - original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR - original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS - original_cache = MODULE._REPORT_RESPONSE_CACHE - original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.REPORT_CACHE_TTL_SECONDS = 1 - MODULE.REPORT_STALE_TTL_SECONDS = 60 - MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) - MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 - MODULE.REPORT_BUSY_WAIT_SECONDS = 0 - MODULE._REPORT_RESPONSE_CACHE = {} - key = MODULE.make_report_cache_key("/reports/worktime/today", "json", "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) - MODULE.save_report_response_cache(key, b"{}", "application/json; charset=utf-8") - fresh = MODULE.get_report_response_cache(key) - assert fresh is not None - assert fresh["stale"] is False - assert fresh["cache_source"] == "memory" - MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 - assert MODULE.get_report_response_cache(key) is None - stale = MODULE.get_report_response_cache(key, allow_stale=True) - assert stale is not None - assert stale["stale"] is True - finally: - MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl - MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl - MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir - MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl - MODULE._REPORT_RESPONSE_CACHE = original_cache - MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait - - -def test_report_response_cache_uses_disk_stale_after_memory_loss(): - original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS - original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS - original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR - original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS - original_cache = MODULE._REPORT_RESPONSE_CACHE - original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.REPORT_CACHE_TTL_SECONDS = 1 - MODULE.REPORT_STALE_TTL_SECONDS = 60 - MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) - MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 - MODULE.REPORT_BUSY_WAIT_SECONDS = 0 - MODULE._REPORT_RESPONSE_CACHE = {} - key = MODULE.make_report_cache_key("/reports/worktime/today", "csv", "SHARKON2025", datetime(2026, 5, 14, tzinfo=timezone.utc).date()) - MODULE.save_report_response_cache(key, b"user,active_seconds\nuser1,60\n", "text/csv; charset=utf-8") - MODULE._REPORT_RESPONSE_CACHE = {} - - cached = MODULE.get_report_response_cache(key, allow_stale=True) - - assert cached is not None - assert cached["data"].startswith(b"user,active_seconds") - assert cached["content_type"] == "text/csv; charset=utf-8" - assert cached["stale"] is True - assert cached["cache_source"] == "disk" - finally: - MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl - MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl - MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir - MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl - MODULE._REPORT_RESPONSE_CACHE = original_cache - MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait - - -def test_send_cached_report_marks_cache_reason_and_age(): - handler = _FakeHandler() - cached = { - "data": b"cached", - "content_type": "application/json; charset=utf-8", - "age_seconds": 42, - "stale": True, - } - ok = MODULE.send_cached_report(handler, cached, reason="busy") - assert ok is True - assert handler.status == 200 - assert handler.headers["X-AW-Worktime-Cache"] == "stale" - assert handler.headers["X-AW-Worktime-Cache-Reason"] == "busy" - assert handler.headers["X-AW-Worktime-Cache-Age"] == "42" - assert handler.headers["X-AW-Worktime-Cache-Source"] == "memory" - assert handler.wfile.data == b"cached" - - -def test_send_report_unavailable_returns_503_json(): - handler = _FakeHandler() - MODULE.send_report_unavailable(handler, "busy") - payload = json.loads(handler.wfile.data.decode("utf-8")) - assert handler.status == 503 - assert payload["ok"] is False - assert payload["error"] == "report_unavailable" - - -def test_send_worktime_report_rebuilds_stale_cache_when_builder_is_available(monkeypatch): - original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS - original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS - original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR - original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS - original_cache = MODULE._REPORT_RESPONSE_CACHE - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.REPORT_CACHE_TTL_SECONDS = 1 - MODULE.REPORT_STALE_TTL_SECONDS = 60 - MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) - MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 - MODULE._REPORT_RESPONSE_CACHE = {} - report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() - key = MODULE.make_report_cache_key("/reports/worktime/today", "html", "SHARKON2025", report_date) - MODULE.save_report_response_cache(key, b"stale", "text/html; charset=utf-8") - MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 - - calls = [] - - def fake_build(path, params): - calls.append((path, params)) - return b"fresh", "text/html; charset=utf-8" - - monkeypatch.setattr(MODULE, "build_worktime_report_response", fake_build) - handler = _FakeHandler() - - MODULE.send_worktime_report( - handler, - "/reports/worktime/today", - {"format": ["html"]}, - key, - host="SHARKON2025", - report_date=report_date, - ) - - assert calls == [("/reports/worktime/today", {"format": ["html"]})] - assert handler.status == 200 - assert handler.headers["Content-Type"] == "text/html; charset=utf-8" - assert "X-AW-Worktime-Cache" not in handler.headers - assert handler.wfile.data == b"fresh" - finally: - MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl - MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl - MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir - MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl - MODULE._REPORT_RESPONSE_CACHE = original_cache - - -def test_send_worktime_report_serves_stale_cache_only_when_builder_is_busy(monkeypatch): - original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS - original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS - original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR - original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS - original_cache = MODULE._REPORT_RESPONSE_CACHE - original_busy_wait = MODULE.REPORT_BUSY_WAIT_SECONDS - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.REPORT_CACHE_TTL_SECONDS = 1 - MODULE.REPORT_STALE_TTL_SECONDS = 60 - MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) - MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 - MODULE.REPORT_BUSY_WAIT_SECONDS = 0 - MODULE._REPORT_RESPONSE_CACHE = {} - report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() - key = MODULE.make_report_cache_key("/reports/worktime/today", "html", "SHARKON2025", report_date) - MODULE.save_report_response_cache(key, b"stale", "text/html; charset=utf-8") - MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 - - semaphore = MODULE.threading.BoundedSemaphore(1) - assert semaphore.acquire(blocking=False) is True - monkeypatch.setattr(MODULE, "_REPORT_BUILD_SEMAPHORE", semaphore) - monkeypatch.setattr( - MODULE, - "build_worktime_report_response", - lambda path, params: (_ for _ in ()).throw(AssertionError("builder must not run while busy")), - ) - handler = _FakeHandler() - - MODULE.send_worktime_report( - handler, - "/reports/worktime/today", - {"format": ["html"]}, - key, - host="SHARKON2025", - report_date=report_date, - ) - - assert handler.status == 200 - assert handler.headers["X-AW-Worktime-Cache"] == "stale" - assert handler.headers["X-AW-Worktime-Cache-Reason"] == "busy" - assert handler.wfile.data == b"stale" - semaphore.release() - finally: - MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl - MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl - MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir - MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl - MODULE._REPORT_RESPONSE_CACHE = original_cache - MODULE.REPORT_BUSY_WAIT_SECONDS = original_busy_wait - - -def test_send_worktime_report_allows_explicit_machine_stale_without_blocking(monkeypatch): - original_ttl = MODULE.REPORT_CACHE_TTL_SECONDS - original_stale_ttl = MODULE.REPORT_STALE_TTL_SECONDS - original_disk_dir = MODULE.REPORT_DISK_CACHE_DIR - original_disk_ttl = MODULE.REPORT_DISK_STALE_TTL_SECONDS - original_cache = MODULE._REPORT_RESPONSE_CACHE - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.REPORT_CACHE_TTL_SECONDS = 1 - MODULE.REPORT_STALE_TTL_SECONDS = 60 - MODULE.REPORT_DISK_CACHE_DIR = Path(tmp) - MODULE.REPORT_DISK_STALE_TTL_SECONDS = 3600 - MODULE._REPORT_RESPONSE_CACHE = {} - report_date = datetime(2026, 5, 14, tzinfo=timezone.utc).date() - key = MODULE.make_report_cache_key("/reports/worktime/today", "csv", "SHARKON2025", report_date) - MODULE.save_report_response_cache(key, b"stale", "text/csv; charset=utf-8") - MODULE._REPORT_RESPONSE_CACHE[key]["stored_monotonic"] -= 10 - - refreshes = [] - monkeypatch.setattr( - MODULE, - "trigger_report_background_refresh", - lambda key, path, params: refreshes.append((key, path, params)) or True, - ) - monkeypatch.setattr( - MODULE, - "build_worktime_report_response", - lambda path, params: (_ for _ in ()).throw(AssertionError("explicit stale request must not block on build")), - ) - handler = _FakeHandler() - - MODULE.send_worktime_report( - handler, - "/reports/worktime/today", - {"format": ["csv"], "allow_stale": ["1"]}, - key, - host="SHARKON2025", - report_date=report_date, - ) - - assert refreshes == [] - assert handler.status == 200 - assert handler.headers["X-AW-Worktime-Cache"] == "stale" - assert handler.headers["X-AW-Worktime-Cache-Reason"] == "stale-allowed" - assert handler.wfile.data == b"stale" - finally: - MODULE.REPORT_CACHE_TTL_SECONDS = original_ttl - MODULE.REPORT_STALE_TTL_SECONDS = original_stale_ttl - MODULE.REPORT_DISK_CACHE_DIR = original_disk_dir - MODULE.REPORT_DISK_STALE_TTL_SECONDS = original_disk_ttl - MODULE._REPORT_RESPONSE_CACHE = original_cache - - -def test_management_report_for_date_bypasses_today_payload_cache_for_human_requests(monkeypatch): - original_ttl = MODULE.MANAGER_CACHE_TTL_SECONDS - original_dir = MODULE.MANAGER_CACHE_DIR - try: - with tempfile.TemporaryDirectory() as tmp: - MODULE.MANAGER_CACHE_TTL_SECONDS = 300 - MODULE.MANAGER_CACHE_DIR = Path(tmp) - report_date = datetime.now(timezone.utc).date() - monkeypatch.setattr(MODULE, "REPORT_TZ", timezone.utc) - monkeypatch.setattr(MODULE, "fetch_events_for_date", lambda host, date: ({"start": datetime.combine(date, datetime.min.time(), tzinfo=timezone.utc), "end": datetime.combine(date, datetime.max.time(), tzinfo=timezone.utc)}, [])) - monkeypatch.setattr(MODULE, "aggregate_rows_with_intervals", lambda events, start, end, host: []) - - generated = iter(["old", "fresh"]) - monkeypatch.setattr( - MODULE, - "build_management_payload", - lambda rows, host, date: {"generated_at_utc": next(generated), "rows": rows}, - ) - - cached = MODULE.management_report_for_date("SHARKON2025", report_date, use_cache=True) - fresh = MODULE.management_report_for_date("SHARKON2025", report_date, use_cache=False) - - assert cached["generated_at_utc"] == "old" - assert fresh["generated_at_utc"] == "fresh" - assert MODULE.load_management_cache("SHARKON2025", report_date)["generated_at_utc"] == "fresh" - finally: - MODULE.MANAGER_CACHE_TTL_SECONDS = original_ttl - MODULE.MANAGER_CACHE_DIR = original_dir diff --git a/aw-server/test_aw_worktime_influx_exporter.py b/aw-server/test_aw_worktime_influx_exporter.py deleted file mode 100644 index c4ee32e..0000000 --- a/aw-server/test_aw_worktime_influx_exporter.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -from datetime import date -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("aw-worktime-influx-exporter.py") -SPEC = importlib.util.spec_from_file_location("aw_worktime_influx_exporter", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -def test_build_lines_for_day_emits_daily_hourly_and_summary(monkeypatch): - bounds = MODULE.WORKTIME.get_report_bounds(date(2026, 5, 14)) - events = [ - { - "timestamp": "2026-05-14T06:00:00Z", - "duration": 0.0, - "data": { - "username": "user5", - "userId": "WORKGROUP\\user5", - "sessionId": 4, - "state": "Активно", - "active": True, - "sampleSeconds": 1800, - }, - } - ] - - monkeypatch.setattr(MODULE.WORKTIME, "fetch_events_for_date", lambda host, report_date: (bounds, events)) - monkeypatch.setattr(MODULE.WORKTIME, "build_true_active_apps", lambda host, report_date: []) - lines = MODULE.build_lines_for_day("SHARKON2025", date(2026, 5, 14)) - - assert any(line.startswith("aw_rdp_worktime_daily,") for line in lines) - assert any(line.startswith("aw_rdp_worktime_hourly,") for line in lines) - assert any(line.startswith("aw_rdp_worktime_summary_daily,") for line in lines) - - -def test_build_lines_for_day_emits_true_active_app_daily(monkeypatch): - bounds = MODULE.WORKTIME.get_report_bounds(date(2026, 5, 14)) - - monkeypatch.setattr(MODULE.WORKTIME, "fetch_events_for_date", lambda host, report_date: (bounds, [])) - monkeypatch.setattr( - MODULE.WORKTIME, - "build_true_active_apps", - lambda host, report_date: [ - { - "application": "1С", - "proved_work_seconds": 2040, - "proved_work_human": "34 мин", - "proved_work_hhmm": "00:34", - "last_action": "1С:Предприятие", - "last_action_local": "09:42", - "last_action_utc": "2026-05-14T06:42:00Z", - "evidence_events": 7, - } - ], - ) - - lines = MODULE.build_lines_for_day("SHARKON2025", date(2026, 5, 14)) - - true_active_line = next(line for line in lines if line.startswith("aw_true_active_app_daily,")) - assert "application=1С" in true_active_line - assert "report_date=2026-05-14" in true_active_line - assert "proved_work_seconds=2040i" in true_active_line - assert "evidence_events=7i" in true_active_line - assert 'last_action="1С:Предприятие"' in true_active_line diff --git a/aw-server/test_aw_worktime_ui_bridge.py b/aw-server/test_aw_worktime_ui_bridge.py deleted file mode 100644 index 3becf60..0000000 --- a/aw-server/test_aw_worktime_ui_bridge.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import unittest -from datetime import datetime, timedelta, timezone -from pathlib import Path -from unittest import mock - - -MODULE_PATH = Path(__file__).with_name("aw-worktime-ui-bridge.py") -SPEC = importlib.util.spec_from_file_location("aw_worktime_ui_bridge", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -class WorktimeUiBridgeTests(unittest.TestCase): - def test_bucket_needs_fallback_when_bucket_is_missing(self): - now = datetime(2026, 5, 22, 16, 0, 0, tzinfo=timezone.utc) - with mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=None): - self.assertTrue(MODULE.bucket_needs_fallback("aw-watcher-window_SHARKON2025", now, 600)) - - def test_bucket_needs_fallback_when_bucket_is_recent(self): - now = datetime(2026, 5, 22, 16, 0, 0, tzinfo=timezone.utc) - recent = now - timedelta(seconds=120) - with mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=recent): - self.assertFalse(MODULE.bucket_needs_fallback("aw-watcher-window_SHARKON2025", now, 600)) - - def test_bucket_needs_fallback_when_bucket_is_stale(self): - now = datetime(2026, 5, 22, 16, 0, 0, tzinfo=timezone.utc) - stale = now - timedelta(seconds=601) - with mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=stale): - self.assertTrue(MODULE.bucket_needs_fallback("aw-watcher-afk_SHARKON2025", now, 600)) - - def test_get_latest_foreground_context_uses_recent_collector_health(self): - now = datetime(2026, 5, 27, 8, 0, 0, tzinfo=timezone.utc) - recent_events = [ - { - "timestamp": "2026-05-27T07:59:30Z", - "data": { - "signalType": "collector_health", - "foregroundProcess": "totalcmd", - "foregroundTitle": "Total Commander 6.01 - HARVEST", - }, - } - ] - with mock.patch.object(MODULE, "_req", return_value=recent_events): - ctx = MODULE.get_latest_foreground_context(now) - self.assertEqual(ctx["app"], "totalcmd.exe") - self.assertEqual(ctx["title"], "Total Commander 6.01 - HARVEST") - - def test_get_latest_active_session_ids_uses_latest_timestamp_group(self): - events = [ - { - "timestamp": "2026-05-27T07:59:25Z", - "data": {"sessionId": 2, "state": "Активно", "username": "администратор", "sessionName": "console"}, - }, - { - "timestamp": "2026-05-27T07:59:30Z", - "data": {"sessionId": 2, "state": "Активно", "username": "администратор", "sessionName": "console"}, - }, - { - "timestamp": "2026-05-27T07:59:30Z", - "data": {"sessionId": 3, "state": "Активно", "username": "user5", "sessionName": "rdp-tcp#0"}, - }, - { - "timestamp": "2026-05-27T07:59:30Z", - "data": {"sessionId": 4, "state": "Диск", "username": "user1", "sessionName": ""}, - }, - ] - self.assertEqual(MODULE.get_latest_active_session_ids(events), {2, 3}) - - def test_get_latest_foreground_context_prefers_active_session(self): - now = datetime(2026, 5, 27, 8, 0, 0, tzinfo=timezone.utc) - recent_events = [ - { - "timestamp": "2026-05-27T07:59:40Z", - "data": { - "signalType": "collector_health", - "foregroundProcess": "explorer", - "foregroundTitle": "Explorer", - "sessionId": 9, - }, - }, - { - "timestamp": "2026-05-27T07:59:45Z", - "data": { - "signalType": "collector_health", - "foregroundProcess": "totalcmd", - "foregroundTitle": "Total Commander 6.01 - HARVEST", - "sessionId": 2, - }, - }, - ] - with mock.patch.object(MODULE, "_req", return_value=recent_events): - ctx = MODULE.get_latest_foreground_context(now, active_session_ids={2}) - self.assertEqual(ctx["app"], "totalcmd.exe") - self.assertEqual(ctx["title"], "Total Commander 6.01 - HARVEST") - - def test_get_latest_foreground_context_uses_cached_context_when_recent_events_are_blank(self): - now = datetime(2026, 5, 27, 8, 5, 0, tzinfo=timezone.utc) - recent_events = [ - { - "timestamp": "2026-05-27T08:04:50Z", - "data": { - "signalType": "collector_health", - "foregroundProcess": "", - "foregroundTitle": "", - "sessionId": 3, - }, - } - ] - state = { - "last_foreground_context": { - "app": "totalcmd.exe", - "title": "Total Commander 6.01 - HARVEST", - "timestamp": "2026-05-27T08:03:30Z", - } - } - with mock.patch.object(MODULE, "_req", return_value=recent_events): - ctx = MODULE.get_latest_foreground_context(now, active_session_ids={2, 3}, state=state) - self.assertEqual(ctx["app"], "totalcmd.exe") - self.assertEqual(ctx["title"], "Total Commander 6.01 - HARVEST") - - def test_transform_uses_foreground_context_for_active_sessions(self): - events = [ - { - "timestamp": "2026-05-27T07:59:30Z", - "duration": 0, - "data": { - "username": "user5", - "state": "Активно", - "sessionId": 3, - "sessionName": "rdp-tcp#0", - }, - } - ] - afk_events, win_events, last_ts = MODULE.transform( - events, - foreground_context={"app": "totalcmd.exe", "title": "Total Commander 6.01 - HARVEST"}, - ) - self.assertEqual(afk_events[0]["data"]["status"], "not-afk") - self.assertEqual(win_events[0]["data"]["app"], "totalcmd.exe") - self.assertEqual(win_events[0]["data"]["title"], "Total Commander 6.01 - HARVEST") - self.assertEqual(last_ts, "2026-05-27T07:59:30Z") - - def test_transform_caps_duration_at_next_timestamp_gap(self): - events = [ - { - "timestamp": "2026-05-27T07:59:30Z", - "duration": 5, - "data": { - "username": "user5", - "state": "Активно", - "sessionId": 3, - "sessionName": "rdp-tcp#0", - }, - }, - { - "timestamp": "2026-05-27T07:59:31Z", - "duration": 5, - "data": { - "username": "user5", - "state": "Активно", - "sessionId": 3, - "sessionName": "rdp-tcp#0", - }, - }, - ] - _, win_events, _ = MODULE.transform( - events, - foreground_context={"app": "totalcmd.exe", "title": "Total Commander 6.01 - HARVEST"}, - ) - self.assertEqual(win_events[0]["duration"], 1.0) - self.assertEqual(win_events[1]["duration"], 5) - - def test_normalize_watcher_window_events_strips_rdp_suffix_for_real_apps(self): - events = [ - { - "timestamp": "2026-05-27T08:10:00Z", - "duration": 5, - "data": { - "app": "totalcmd.exe", - "title": "Total Commander 6.01 - HARVEST | RDP active (2): user5, администратор", - "source": "aw-worktime-ui-bridge", - }, - }, - { - "timestamp": "2026-05-27T08:10:05Z", - "duration": 5, - "data": { - "app": "RDP", - "title": "RDP active (2): user5, администратор", - "source": "aw-worktime-ui-bridge", - }, - }, - ] - normalized = MODULE.normalize_watcher_window_events(events) - self.assertEqual(normalized[0]["data"]["title"], "Total Commander 6.01 - HARVEST") - self.assertEqual(len(normalized), 1) - - def test_watcher_window_needs_bridge_sync_for_generic_bridge_rdp(self): - now = datetime(2026, 5, 27, 8, 5, 0, tzinfo=timezone.utc) - latest_event = { - "timestamp": "2026-05-27T08:04:40Z", - "data": { - "app": "RDP", - "title": "RDP active (2): user5, администратор", - "source": "aw-worktime-ui-bridge", - }, - } - with mock.patch.object(MODULE, "get_latest_bucket_event", return_value=latest_event), \ - mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=datetime(2026, 5, 27, 8, 4, 40, tzinfo=timezone.utc)): - self.assertTrue(MODULE.watcher_window_needs_bridge_sync(now)) - - def test_watcher_window_needs_bridge_sync_for_non_generic_bridge_event(self): - now = datetime(2026, 5, 27, 8, 5, 0, tzinfo=timezone.utc) - latest_event = { - "timestamp": "2026-05-27T08:04:40Z", - "data": { - "app": "totalcmd.exe", - "title": "Total Commander 6.01 - HARVEST", - "source": "aw-worktime-ui-bridge", - }, - } - with mock.patch.object(MODULE, "get_latest_bucket_event", return_value=latest_event), \ - mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=datetime(2026, 5, 27, 8, 4, 40, tzinfo=timezone.utc)): - self.assertTrue(MODULE.watcher_window_needs_bridge_sync(now)) - - def test_watcher_window_does_not_override_real_watcher_stream(self): - now = datetime(2026, 5, 27, 8, 5, 0, tzinfo=timezone.utc) - latest_event = { - "timestamp": "2026-05-27T08:04:40Z", - "data": { - "app": "notepad.exe", - "title": "Безымянный — Блокнот", - "source": "aw-watcher-window", - }, - } - with mock.patch.object(MODULE, "get_latest_bucket_event", return_value=latest_event), \ - mock.patch.object(MODULE, "get_latest_bucket_event_ts", return_value=datetime(2026, 5, 27, 8, 4, 40, tzinfo=timezone.utc)): - self.assertFalse(MODULE.watcher_window_needs_bridge_sync(now)) - - -if __name__ == "__main__": - unittest.main() diff --git a/docs/DETMIR_PYTHON_RETIREMENT_RU.md b/docs/DETMIR_PYTHON_RETIREMENT_RU.md new file mode 100644 index 0000000..1bb453a --- /dev/null +++ b/docs/DETMIR_PYTHON_RETIREMENT_RU.md @@ -0,0 +1,54 @@ +# DetMir: статус вывода Python из runtime + +Дата фиксации: 2026-06-03. + +## Решение + +Боевой контур DetMir/AWatch-rus переводится на Rust-first runtime. Python entrypoints удаляются из серверных DLP, health, SLO, worktime, Hayabusa/offline tooling, backup/merge и deploy путей, если для них уже есть Rust-замена. + +## Удалено из repo/runtime path + +- AW health/SLO/worktime Python services. +- DLP policy engine, case management, compliance, integrations Python services. +- DLP aggregator, health check, admin CLI, IOC extractor, AW DB merge Python scripts. +- Hayabusa case/offline helper Python scripts. +- Legacy `aw-health-check.sh` server-side fallback. +- Active `/usr/local/bin/*.py` entrypoints and old DLP service virtualenvs on AW server. +- Python-era DLP Ansible roles now fail fast and point to `ansible/deploy_aw_server.yml`. +- Python-era Proxmox CT bootstrap playbooks now fail fast until rebuilt as Rust bootstrap. + +## Оставленные исключения + +- Telegram bot runtime on Proxmox remains Python by product decision. +- pfSense tooling is frozen/no-touch in this migration track. +- OCR/content-analysis keeps Python dependencies for image/OCR path; Rust handles text/dictionary/regex path. +- `clickhouse-1c/ai` and `clickhouse-1c/etl` remain a separate 1C/business-data track. +- `detmir-mcp/main.py` remains a separate MCP/runtime track. + +## Проверка + +```bash +rg -n '\.py\b|python3' ansible aw-server scripts adk-rust \ + --glob '!adk-rust/target/**' + +ansible-playbook --syntax-check -i ansible/inventory.ini ansible/deploy_aw_server.yml + +cd adk-rust +CARGO_TARGET_DIR=/tmp/detmir-adk-rust-target cargo test --workspace +CARGO_TARGET_DIR=/tmp/detmir-adk-rust-target cargo clippy --workspace --all-targets -- -D warnings +``` + +Ожидаемые оставшиеся Python-ссылки после очистки: только перечисленные исключения, историческая документация или legacy cleanup patterns. + +Production AW server scan excludes the agreed OCR/content-analysis path, +temporary external Hayabusa rules under `dlp-ioc/tmp`, and rollback backups: + +```bash +find /usr/local/bin /opt/activitywatch \ + -path "/opt/activitywatch/dlp-content-analysis" -prune -o \ + -path "/opt/activitywatch/dlp-ioc/tmp" -prune -o \ + -path "*/switch-backups" -prune -o \ + \( -name "*.py" -o -name "health-check.sh" -o -path "*/.venv" \) -print +``` + +Ожидаемый результат для active server runtime: пустой вывод. diff --git a/install-kit-awindows-20260427-211240.tar.gz b/install-kit-awindows-20260427-211240.tar.gz index 72b91d6..6a74deb 100644 Binary files a/install-kit-awindows-20260427-211240.tar.gz and b/install-kit-awindows-20260427-211240.tar.gz differ diff --git a/install-kit-awindows-20260427-211240.zip b/install-kit-awindows-20260427-211240.zip index 8dce547..9a613cb 100644 Binary files a/install-kit-awindows-20260427-211240.zip and b/install-kit-awindows-20260427-211240.zip differ diff --git a/install-kit-awindows-20260427-211240/MANIFEST.txt b/install-kit-awindows-20260427-211240/MANIFEST.txt index 35550c9..e1c4ced 100644 --- a/install-kit-awindows-20260427-211240/MANIFEST.txt +++ b/install-kit-awindows-20260427-211240/MANIFEST.txt @@ -1,8 +1,8 @@ 1b776081c24ad8413c2f3e24aff3952949e7786a108a08ef45121536bcae9482 install-kit-awindows-20260427-211240/README-INSTALL-KIT.txt e973ad9aa3824a6588ce4ef1b80ce3caec0085434a2755f9b657f09fb7435632 install-kit-awindows-20260427-211240/ansible/README.md 412bb766bbf0791c3593f38daa771d5d0aa58cc1f2d3c9010fcd4588d0fe87df install-kit-awindows-20260427-211240/ansible/deploy_aw_pfsense_poller.yml -951b6e22c285a95a3f45ead81fe98069eb48405368617a600f86240ccad65b70 install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml -7c1afd6088e7ec26842be4b4b8a0530780d1d7075f19f80d63eb08445857823f install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml +ac091618a6793a9be9b52eeb78267068df16d4f57e0051b9d868813c4a641f35 install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml +cd09574fd0d55bf62d9d7df757bb206b5184ba9ffdcc200e755700e50cfa41f3 install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml d666eac9d94ad71842e21aaac5d23968a0c2213bb4edab899bf432d7ad417ddf install-kit-awindows-20260427-211240/ansible/group_vars/all.example.yml a896676303be9a47ed6b0c8cc07deb5a5432aebc97231e801fda985121c539eb install-kit-awindows-20260427-211240/ansible/group_vars/pfsense-poller.example.yml c5cab36645065815571c99f6d360f910dcccbb54b780c8bfd526a6cdc3684e19 install-kit-awindows-20260427-211240/ansible/group_vars/proxmox-matrix.example.yml @@ -10,8 +10,8 @@ c5cab36645065815571c99f6d360f910dcccbb54b780c8bfd526a6cdc3684e19 install-kit-aw 9e4939d2d951870cacab6e42c5bf0ccd03350089dc0d32b9a681948ff96754d2 install-kit-awindows-20260427-211240/ansible/group_vars/windows.example.yml 7189b5205bd25313db54e5be027b0d066199e6ae34ad74be2095a1691adaf5e3 install-kit-awindows-20260427-211240/ansible/install_full_stack.yml fea0574d7eb98ce24a1e7025afb9837c6241180095d21eaa74892225305d05a3 install-kit-awindows-20260427-211240/ansible/inventory.example.ini -8a5e4923c0f581dd4fbb32549ee7ab45ba506260056da923ba86d9f1b1081714 install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml -18928adcaef5d01b4c621b48f5559383c8b749ef182fd3710f10b222a164f8b7 install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml +3adda7422ffbbb76be50858f2f1ed6073b6bf9ee0b57283c41d0fabd3e7c7f9e install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml +234932632092226526173b9ef906de2cce961bc95be0f37c878e48d9779e1de7 install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml a74a49371e889dc3ea404534a939f32f2dac940d8902d20770590951ab67d532 install-kit-awindows-20260427-211240/ansible/tasks/provision_ct_and_deploy_aw.yml ba16fe9e267194459a6082045a387acb828a1d39a98e66b401ece5069ea62e64 install-kit-awindows-20260427-211240/aw-server/activitywatch-server.service fea5734c99b516b01bfe8ebbef59972d0ef553c09d7691790b08472ecfb9602d install-kit-awindows-20260427-211240/aw-server/apply_webui_ru_patch.sh @@ -19,30 +19,25 @@ fabcdbb933b1d3b605c5ac1234af4e10495c09f18bd50931bf4a92081fcb5023 install-kit-aw e59d58d4cc6b2b0dceef5b1da14ca0a1b24a202ce576ee43028f44ffe43c5ef4 install-kit-awindows-20260427-211240/aw-server/aw-browser-smoke.timer ba88cc284d047f521427ac038c266624a6fe8493ce3e79bc27c172a2e70ac54a install-kit-awindows-20260427-211240/aw-server/aw-host-groups.json 509759461ce0918a2dc839832812cf5f4c77b1cea1e6ffdcab0146e02598df79 install-kit-awindows-20260427-211240/aw-server/aw-ru-patch.js -6072757b685430962f61f97f6de4b0ed6f6653454559974a6ee87936fc6c7571 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py -05b04b5f49e9c7783917e0861e3edd63f5d6db8638d6b33dbfd4dc0f1c16040f install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service +699f697143f6be588c698bad9a14a45c736fe74f5e4b0e0ea0f48b569c7df396 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service 8ae02b42f4bc33d95268624bae79c2101298b7ec6644553a1e81f7841dda74d8 install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.timer 28950acbfeed3f3e5729e0227fd96785dd840330ed9b94a3d6ab0a6a855ca5ec install-kit-awindows-20260427-211240/aw-server/aw-server.env.example -bc8f88fa615a74fc69f80c0cc614ddea323156e5e49d3de28a9958295737afab install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py -6db182e8e5237b1daaafdcd1003fccf7e5aaaabb7be8ab4226cb513f7b1c739b install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service +18d038e67b926eca04bec725ed5e6de34b93f8330c3b30a0b07d52eab414bd03 install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service 257cdc18f088e8848b1bee69e7f8b99aa860a0f600eebd74365a40ed94aa5ecb install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.timer 98c0bed353bbda0fa7a69df23f3b008cb0e8e70cdff6cc63330d4caf79fd3280 install-kit-awindows-20260427-211240/aw-server/aw-sw-cleanup.js -9597e8e12a3472b86b222f10c2e0a71c653efc41415771aeaab2e830c9424d3c install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py -f764e566d70952acc1b4f2baeccd6b7888905bc6b98b36333336c0dc77c66694 install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service +0e555853f1cb498e63b70d305ce5facfaa626a2ceee6f7edb854cf37c4371f65 install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service afbb4be301b4940ed9b7671be3441a48d53bee968d8870749442f81c6d066650 install-kit-awindows-20260427-211240/aw-server/aw-worktime-panel.js -c9169d01469b2f49eab0f0e9a483f883d4c367719fde51fb262d74c35e798c44 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service -1733c8c84b17ad89e0f3e5b27f649b968dedd52d9048f9bc2d92e7cc10daebd1 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh +7e37dd8bbfdbbe661defb9ce69284db8ad339a49b34f8c507060d0dbc75b7aa8 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service 0cd38bae7ad1b4718d7decc8ef3ac932939a6e5f8c25e9e27d1d0ecf8cc328b9 install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.timer -2e3a0667f8a9eddcb4803c0ba274615092cdd3eec23c2dba3f494715dac91423 install-kit-awindows-20260427-211240/aw-server/health-check.sh -ca7d1ed7665e225824d4ed7ef8456c81a6d97ef2c90cb3aa799d72e07b6bcbb8 install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh +6de9b272ad732ad174c32398ab8baf5cc8c83be47efe0d6df46c3190b4010680 install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh aedffecfa24834968742cb2477faef80bf794345275a9679ac12c5a1f609acc2 install-kit-awindows-20260427-211240/aw-server/settings/classes-worktime.json 47c50054515506b72af2d6bca0ae959ca57861803e528991c230611e6a8893d6 install-kit-awindows-20260427-211240/aw-server/settings/views-default.json 4e5b23300ba5c9878b2c8ce6eea65043b7a4c5cb2d14619243cfca6d202aec9f install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.mjs -a851fba13a14a529f95eeb054c2f64dc108cf7a379c4ea35402e84c0b4a2733d install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh -da178cb5dfff3ae8465a6139048adc6757cb1dc23edb30d747d09a05e6b07aeb install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh +62980eeebe01d7da243be8535a48f5d41d4eead83cf94015df807111c3d920ab install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh +6770275fac17607770653a522a64cf5ab31a7a11993b68e7e80a2858c3b8930e install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh fe8d2b846a64f1357dda48b4c817c7919a3c7398921ddca2adf9a99711ac2613 install-kit-awindows-20260427-211240/scripts/quality-gate.sh -f634ea60040153fafcab6363bea8c442d05c1734e6a9d2e16844cf63a741df24 install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh -b58c572edd12e4c3620c045f3382e7b9f74664ef5550302772a828a023f8f3fe install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh +db1a9b0ccd21aec78d8e2aa08dfa28368e5ba6209c57361e6824e85c1af9f6ca install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh +38ad2e112796c2fa9bdc1cc2403c679bb896f90ae2b51d6ccc9a9ec103b4dea6 install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh 0b49e6db51d5abcecaeee0b85f186efd4f10d3360cf04e8aea2e4c7d4465dbda install-kit-awindows-20260427-211240/scripts/verify_innosetup_installer.sh dd2389e9cb199ed86d219120294fdaa64415cfbc683004d5f5db5d52aa758a92 install-kit-awindows-20260427-211240/server-configs-192.168.100.18/awatch-rus-admin.deployment-config.json 5279f9d677faed76a5f0248f9217ecc29eac977ab752552cab852f5b4b6715df install-kit-awindows-20260427-211240/server-configs-192.168.100.18/awatch-rus-u2u5.deployment-config.json diff --git a/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml b/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml index 7b68704..6a81578 100644 --- a/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml +++ b/install-kit-awindows-20260427-211240/ansible/deploy_aw_server.yml @@ -27,6 +27,7 @@ - tesseract-ocr - tesseract-ocr-rus - unzip + - jq state: present update_cache: true @@ -109,6 +110,64 @@ - "{{ aw_rus_health_validation_dir }}" - "{{ aw_server_log_dir }}" + - name: Проверить существующие health state files + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ aw_rus_health_state_dir }}/aw-rus-health.json" + - "{{ aw_rus_health_state_dir }}/aw-rus-health.txt" + - "{{ aw_rus_health_state_dir }}/dlp-health-check-counters.json" + register: aw_rus_health_state_files + + - name: Нормализовать владельца существующих health state files + ansible.builtin.file: + path: "{{ item.stat.path }}" + owner: "{{ aw_server_user }}" + group: "{{ aw_server_group }}" + mode: "0644" + loop: "{{ aw_rus_health_state_files.results }}" + when: item.stat.exists | default(false) + + - name: Удалить устаревшие Python entrypoints после Rust migration + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /opt/activitywatch/aw-rus-ops/health-check.sh + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py + - /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py + - /opt/activitywatch/dlp-case-management/.venv + - /opt/activitywatch/dlp-case-management/case_rules.py + - /opt/activitywatch/dlp-case-management/case_schema.py + - /opt/activitywatch/dlp-case-management/case_service.py + - /opt/activitywatch/dlp-case-management/case_storage.py + - /opt/activitywatch/dlp-case-management/evidence_chain.py + - /opt/activitywatch/dlp-compliance/compliance_scheduler.py + - /opt/activitywatch/dlp-compliance/report_generator.py + - /opt/activitywatch/dlp-compliance/.venv + - /opt/activitywatch/dlp-integrations/cef_exporter.py + - /opt/activitywatch/dlp-integrations/syslog_forwarder.py + - /opt/activitywatch/dlp-integrations/webhook_sender.py + - /opt/activitywatch/dlp-integrations/.venv + - /opt/activitywatch/dlp-policy-engine/.venv + - /opt/activitywatch/dlp-policy-engine/policy_distributor.py + - /opt/activitywatch/dlp-policy-engine/policy_schema.py + - /opt/activitywatch/dlp-policy-engine/policy_service.py + - /opt/activitywatch/dlp-policy-engine/policy_storage.py + - /opt/activitywatch/scripts/aggregate_dlp_events.py + - /usr/local/bin/aw-dlp-influx-exporter.py + - /usr/local/bin/aw-extract-ioc-from-sigma.py + - /usr/local/bin/aw-rus-healthd.py + - /usr/local/bin/aw-slo-monitor.py + - /usr/local/bin/aw-worktime-afk-bridge.py + - /usr/local/bin/aw-worktime-api.py + - /usr/local/bin/aw-worktime-influx-exporter.py + - /usr/local/bin/aw-worktime-ui-bridge.py + - /usr/local/bin/dlp-admin-cli.py + - /usr/local/bin/merge_aw_server_dbs.py + - name: Установить prune script для локального state ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-prune-local-state.sh" @@ -431,14 +490,6 @@ group: root mode: "0755" - - name: Положить legacy aw-health-check до Influx проверок - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" - dest: /opt/activitywatch/aw-rus-ops/health-check.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust aw-health-check до Influx проверок ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-health-check" @@ -455,6 +506,12 @@ mode: "0755" when: aw_health_check_rust_binary_early.stat.exists | default(false) + - name: Требовать Rust aw-health-check до Influx проверок + ansible.builtin.assert: + that: + - aw_health_check_rust_binary_early.stat.exists | default(false) + fail_msg: "Rust aw-health-check artifact is required at {{ aw_rust_release_dir }}/aw-health-check" + - name: Проверить локальный Rust aw-db-health до Influx проверок ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-db-health" @@ -471,6 +528,64 @@ mode: "0755" when: aw_db_health_rust_binary_early.stat.exists | default(false) + - name: Проверить локальный Rust aw-ensure-reliability до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-ensure-reliability" + delegate_to: localhost + register: aw_ensure_reliability_rust_binary_early + become: false + + - name: Установить Rust aw-ensure-reliability до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-ensure-reliability" + dest: /usr/local/bin/aw-ensure-reliability + owner: root + group: root + mode: "0755" + when: aw_ensure_reliability_rust_binary_early.stat.exists | default(false) + + - name: Проверить локальный Rust aw-db-maintenance до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-db-maintenance" + delegate_to: localhost + register: aw_db_maintenance_rust_binary_early + become: false + + - name: Установить Rust aw-db-maintenance до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-db-maintenance" + dest: /usr/local/bin/aw-db-maintenance + owner: root + group: root + mode: "0755" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Установить aw-db-maintenance service до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-db-maintenance.service" + dest: /etc/systemd/system/aw-db-maintenance.service + owner: root + group: root + mode: "0644" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Установить aw-db-maintenance timer до Influx проверок + ansible.builtin.copy: + src: "{{ aw_repo_root }}/aw-server/aw-db-maintenance.timer" + dest: /etc/systemd/system/aw-db-maintenance.timer + owner: root + group: root + mode: "0644" + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + + - name: Включить weekly aw-db-maintenance timer до Influx проверок + ansible.builtin.systemd: + name: aw-db-maintenance.timer + enabled: true + state: started + daemon_reload: true + when: aw_db_maintenance_rust_binary_early.stat.exists | default(false) + - name: Прочитать текущий aw-server.env для сохранения Influx token ansible.builtin.slurp: path: /etc/activitywatch/aw-server.env @@ -592,34 +707,6 @@ mode: "0755" when: aw_dlp_policy_engine_enabled | default(false) | bool - - name: Скопировать файлы DLP policy engine - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/{{ item }}" - dest: "/opt/activitywatch/dlp-policy-engine/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0644" - loop: - - policy_service.py - - policy_schema.py - - policy_storage.py - - policy_distributor.py - - requirements.txt - when: aw_dlp_policy_engine_enabled | default(false) | bool - - - name: Создать virtualenv DLP policy engine - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-policy-engine/.venv - args: - creates: /opt/activitywatch/dlp-policy-engine/.venv/bin/python - when: aw_dlp_policy_engine_enabled | default(false) | bool - - - name: Установить зависимости DLP policy engine - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-policy-engine/requirements.txt - virtualenv: /opt/activitywatch/dlp-policy-engine/.venv - when: aw_dlp_policy_engine_enabled | default(false) | bool - - name: Установить systemd unit DLP policy engine ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-policy-engine/dlp-policy-engine.service" @@ -637,6 +724,13 @@ become: false when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Требовать Rust DLP policy engine artifact + ansible.builtin.assert: + that: + - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-policy-engine" + when: aw_dlp_policy_engine_enabled | default(false) | bool + - name: Установить Rust DLP policy engine ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-policy-engine" @@ -648,31 +742,6 @@ - aw_dlp_policy_engine_enabled | default(false) | bool - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP policy engine - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-policy-engine.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_policy_engine_enabled | default(false) | bool - - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - - name: Переключить DLP policy engine на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-policy-engine.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-policy-engine-rust - when: - - aw_dlp_policy_engine_enabled | default(false) | bool - - aw_dlp_policy_engine_rust_binary.stat.exists | default(false) - - name: Создать каталог DLP content analysis ansible.builtin.file: path: /opt/activitywatch/dlp-content-analysis @@ -749,38 +818,9 @@ group: "{{ aw_server_group }}" mode: "0644" loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - cef-config.yaml - syslog-forwarder-config.yaml - webhook-config.yaml - - requirements.txt - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Выставить executable для python scripts DLP integrations - ansible.builtin.file: - path: "/opt/activitywatch/dlp-integrations/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0755" - loop: - - cef_exporter.py - - syslog_forwarder.py - - webhook_sender.py - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Создать virtualenv DLP integrations - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-integrations/.venv - args: - creates: /opt/activitywatch/dlp-integrations/.venv/bin/python - when: aw_dlp_integrations_enabled | default(true) | bool - - - name: Установить зависимости DLP integrations - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-integrations/requirements.txt - virtualenv: /opt/activitywatch/dlp-integrations/.venv when: aw_dlp_integrations_enabled | default(true) | bool - name: Создать state каталог DLP integrations @@ -818,6 +858,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust CEF exporter artifact + ansible.builtin.assert: + that: + - dlp_cef_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-cef-exporter" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust CEF exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-cef-exporter" @@ -829,31 +876,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог CEF exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-cef-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить CEF exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-cef-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-cef-exporter-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_cef_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd unit syslog forwarder ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/syslog-forwarder.service" @@ -880,6 +902,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust syslog forwarder artifact + ansible.builtin.assert: + that: + - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-syslog-forwarder" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust syslog forwarder ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-syslog-forwarder" @@ -891,31 +920,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог syslog forwarder - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - - name: Переключить syslog forwarder на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-syslog-forwarder.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-syslog-forwarder-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_syslog_forwarder_rust_binary.stat.exists | default(false) - - name: Установить systemd unit webhook sender ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-integrations/webhook-sender.service" @@ -942,6 +946,13 @@ become: false when: aw_dlp_integrations_enabled | default(true) | bool + - name: Требовать Rust webhook sender artifact + ansible.builtin.assert: + that: + - dlp_webhook_sender_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-webhook-sender" + when: aw_dlp_integrations_enabled | default(true) | bool + - name: Установить Rust webhook sender ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-webhook-sender" @@ -953,31 +964,6 @@ - aw_dlp_integrations_enabled | default(true) | bool - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог webhook sender - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-webhook-sender.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - - name: Переключить webhook sender на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-webhook-sender.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/dlp-webhook-sender-rust - when: - - aw_dlp_integrations_enabled | default(true) | bool - - dlp_webhook_sender_rust_binary.stat.exists | default(false) - - name: Создать каталог DLP case management ansible.builtin.file: path: /opt/activitywatch/dlp-case-management @@ -987,43 +973,6 @@ mode: "0755" when: aw_dlp_case_management_enabled | default(true) | bool - - name: Скопировать файлы DLP case management - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/dlp-case-management/{{ item }}" - dest: "/opt/activitywatch/dlp-case-management/{{ item }}" - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0644" - loop: - - case_service.py - - case_rules.py - - case_schema.py - - case_storage.py - - evidence_chain.py - - requirements.txt - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Выставить executable для case_service - ansible.builtin.file: - path: /opt/activitywatch/dlp-case-management/case_service.py - owner: "{{ aw_server_user }}" - group: "{{ aw_server_group }}" - mode: "0755" - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Создать virtualenv DLP case management - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-case-management/.venv - args: - creates: /opt/activitywatch/dlp-case-management/.venv/bin/python - when: aw_dlp_case_management_enabled | default(true) | bool - - - name: Установить зависимости DLP case management - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-case-management/requirements.txt - virtualenv: /opt/activitywatch/dlp-case-management/.venv - when: aw_dlp_case_management_enabled | default(true) | bool - - name: Установить systemd unit DLP case management ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/dlp-case-management/case-service.service" @@ -1041,6 +990,13 @@ become: false when: aw_dlp_case_management_enabled | default(true) | bool + - name: Требовать Rust DLP case management artifact + ansible.builtin.assert: + that: + - aw_dlp_case_management_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-case-management" + when: aw_dlp_case_management_enabled | default(true) | bool + - name: Установить Rust DLP case management ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-case-management" @@ -1052,31 +1008,6 @@ - aw_dlp_case_management_enabled | default(true) | bool - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP case management - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-case-management.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_case_management_enabled | default(true) | bool - - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - - name: Переключить DLP case management на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-case-management.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-case-management-rust - when: - - aw_dlp_case_management_enabled | default(true) | bool - - aw_dlp_case_management_rust_binary.stat.exists | default(false) - - name: Создать каталоги DLP compliance ansible.builtin.file: path: "{{ item }}" @@ -1098,28 +1029,12 @@ group: "{{ aw_server_group }}" mode: "{{ item.mode }}" loop: - - { src: "report_generator.py", dest: "/opt/activitywatch/dlp-compliance/report_generator.py", mode: "0755" } - - { src: "compliance_scheduler.py", dest: "/opt/activitywatch/dlp-compliance/compliance_scheduler.py", mode: "0755" } - - { src: "requirements.txt", dest: "/opt/activitywatch/dlp-compliance/requirements.txt", mode: "0644" } - { src: "templates/152-fz-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html", mode: "0644" } - { src: "templates/pci-dss-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/pci-dss-report.html", mode: "0644" } - { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" } - { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" } when: aw_dlp_compliance_enabled | default(true) | bool - - name: Создать virtualenv DLP compliance - ansible.builtin.command: - cmd: python3 -m venv /opt/activitywatch/dlp-compliance/.venv - args: - creates: /opt/activitywatch/dlp-compliance/.venv/bin/python - when: aw_dlp_compliance_enabled | default(true) | bool - - - name: Установить зависимости DLP compliance - ansible.builtin.pip: - requirements: /opt/activitywatch/dlp-compliance/requirements.txt - virtualenv: /opt/activitywatch/dlp-compliance/.venv - when: aw_dlp_compliance_enabled | default(true) | bool - - name: Проверить локальный Rust DLP compliance ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-compliance" @@ -1128,6 +1043,13 @@ become: false when: aw_dlp_compliance_enabled | default(true) | bool + - name: Требовать Rust DLP compliance artifact + ansible.builtin.assert: + that: + - aw_dlp_compliance_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-compliance" + when: aw_dlp_compliance_enabled | default(true) | bool + - name: Установить Rust DLP compliance ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-compliance" @@ -1139,45 +1061,18 @@ - aw_dlp_compliance_enabled | default(true) | bool - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP compliance report scheduler - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-report-scheduler.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_compliance_enabled | default(true) | bool - - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - - name: Переключить DLP compliance report scheduler на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-report-scheduler.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-compliance-rust - when: - - aw_dlp_compliance_enabled | default(true) | bool - - aw_dlp_compliance_rust_binary.stat.exists | default(false) - - - name: Установить dlp-admin-cli - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/dlp-admin-cli.py" - dest: /usr/local/bin/dlp-admin-cli.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust dlp-admin-cli ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-admin-cli" delegate_to: localhost register: aw_dlp_admin_cli_rust_binary + - name: Требовать Rust dlp-admin-cli artifact + ansible.builtin.assert: + that: + - aw_dlp_admin_cli_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-admin-cli" + - name: Установить Rust dlp-admin-cli ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-admin-cli" @@ -1187,14 +1082,6 @@ mode: "0755" when: aw_dlp_admin_cli_rust_binary.stat.exists - - name: Установить скрипт AW worktime API - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-api.py" - dest: /usr/local/bin/aw-worktime-api.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime API ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-api" @@ -1202,6 +1089,12 @@ register: aw_worktime_api_rust_binary become: false + - name: Требовать Rust AW worktime API artifact + ansible.builtin.assert: + that: + - aw_worktime_api_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-api" + - name: Установить Rust AW worktime API ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-api" @@ -1219,35 +1112,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime API - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-api.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_api_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime API на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-api.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-api-rust - when: aw_worktime_api_rust_binary.stat.exists | default(false) - - - name: Установить скрипт AW worktime UI bridge - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.py" - dest: /usr/local/bin/aw-worktime-ui-bridge.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime UI bridge ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-ui-bridge" @@ -1255,6 +1119,12 @@ register: aw_worktime_ui_bridge_rust_binary become: false + - name: Требовать Rust AW worktime UI bridge artifact + ansible.builtin.assert: + that: + - aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-ui-bridge" + - name: Установить Rust AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-ui-bridge" @@ -1264,14 +1134,6 @@ mode: "0755" when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime autoheal - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.sh" - dest: /usr/local/bin/aw-worktime-autoheal.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime autoheal ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-autoheal" @@ -1279,6 +1141,12 @@ register: aw_worktime_autoheal_rust_binary become: false + - name: Требовать Rust AW worktime autoheal artifact + ansible.builtin.assert: + that: + - aw_worktime_autoheal_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-autoheal" + - name: Установить Rust AW worktime autoheal ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-autoheal" @@ -1288,14 +1156,6 @@ mode: "0755" when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime prewarm - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.sh" - dest: /usr/local/bin/aw-worktime-prewarm.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust AW worktime prewarm ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/worktime-prewarm" @@ -1303,6 +1163,12 @@ register: aw_worktime_prewarm_rust_binary become: false + - name: Требовать Rust AW worktime prewarm artifact + ansible.builtin.assert: + that: + - aw_worktime_prewarm_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-prewarm" + - name: Установить Rust AW worktime prewarm ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-prewarm" @@ -1312,25 +1178,7 @@ mode: "0755" when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - name: Установить скрипт AW worktime Influx exporter - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.py" - dest: /usr/local/bin/aw-worktime-influx-exporter.py - owner: root - group: root - mode: "0755" - when: aw_worktime_influx_enabled | default(false) | bool - - - name: Установить скрипт AW DLP Influx exporter - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.py" - dest: /usr/local/bin/aw-dlp-influx-exporter.py - owner: root - group: root - mode: "0755" - when: aw_dlp_influx_enabled | default(false) | bool - - - name: Создать server-side ops bundle для legacy aw-health-check + - name: Создать server-side ops bundle для AW ops helpers ansible.builtin.file: path: /opt/activitywatch/aw-rus-ops state: directory @@ -1346,14 +1194,6 @@ group: root mode: "0755" - - name: Положить legacy aw-health-check в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/health-check.sh" - dest: /opt/activitywatch/aw-rus-ops/health-check.sh - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust aw-health-check ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-health-check" @@ -1370,6 +1210,12 @@ mode: "0755" when: aw_health_check_rust_binary.stat.exists | default(false) + - name: Требовать Rust aw-health-check artifact + ansible.builtin.assert: + that: + - aw_health_check_rust_binary.stat.exists | default(false) + fail_msg: "Rust aw-health-check artifact is required at {{ aw_rust_release_dir }}/aw-health-check" + - name: Проверить локальный Rust aw-db-health ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-db-health" @@ -1434,30 +1280,12 @@ mode: "0755" when: dlp_health_check_rust_binary.stat.exists | default(false) - - name: Установить legacy Python dlp-health-check - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/dlp-health-check.py" - dest: /usr/local/bin/dlp-health-check - owner: root - group: root - mode: "0755" - when: not (dlp_health_check_rust_binary.stat.exists | default(false)) - - - name: Установить скрипт aw-rus-healthd - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-rus-healthd.py" - dest: /usr/local/bin/aw-rus-healthd.py - owner: root - group: root - mode: "0755" - - - name: Установить скрипт aw-slo-monitor - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.py" - dest: /usr/local/bin/aw-slo-monitor.py - owner: root - group: root - mode: "0755" + - name: Проверить локальный Rust AW-RUS healthd + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-rus-healthd" + delegate_to: localhost + register: aw_rus_healthd_rust_binary + become: false - name: Проверить локальный Rust AW-RUS SLO monitor ansible.builtin.stat: @@ -1466,6 +1294,32 @@ register: aw_slo_monitor_rust_binary become: false + - name: Требовать Rust dlp-health-check artifact + ansible.builtin.assert: + that: + - dlp_health_check_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-health-check" + + - name: Требовать Rust AW-RUS healthd artifact + ansible.builtin.assert: + that: + - aw_rus_healthd_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-rus-healthd" + + - name: Требовать Rust AW-RUS SLO monitor artifact + ansible.builtin.assert: + that: + - aw_slo_monitor_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-slo-monitor" + + - name: Установить Rust AW-RUS healthd + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-rus-healthd" + dest: /usr/local/bin/aw-rus-healthd-rust + owner: root + group: root + mode: "0755" + - name: Установить Rust AW-RUS SLO monitor ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/aw-slo-monitor" @@ -1492,27 +1346,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime UI bridge - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-ui-bridge.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime UI bridge на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-ui-bridge.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-ui-bridge-rust - when: aw_worktime_ui_bridge_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime UI bridge ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-ui-bridge.timer" @@ -1529,27 +1362,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime autoheal - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-autoheal.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime autoheal на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-autoheal.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-autoheal-rust - when: aw_worktime_autoheal_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime autoheal ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-autoheal.timer" @@ -1566,27 +1378,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW worktime prewarm - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-prewarm.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime prewarm на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-prewarm.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-prewarm-rust - when: aw_worktime_prewarm_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime prewarm ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-prewarm.timer" @@ -1619,24 +1410,6 @@ group: root mode: "0644" - - name: Создать drop-in каталог AW-RUS SLO monitor - ansible.builtin.file: - path: /etc/systemd/system/aw-slo-monitor.service.d - state: directory - owner: root - group: root - mode: "0755" - when: aw_slo_monitor_rust_binary.stat.exists | default(false) - - - name: Переключить AW-RUS SLO monitor на Rust - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor-rust-switch.conf" - dest: /etc/systemd/system/aw-slo-monitor.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - when: aw_slo_monitor_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW-RUS SLO monitor ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-slo-monitor.timer" @@ -1680,6 +1453,13 @@ become: false when: aw_worktime_influx_enabled | default(false) | bool + - name: Требовать Rust AW worktime Influx exporter artifact + ansible.builtin.assert: + that: + - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/worktime-influx-exporter" + when: aw_worktime_influx_enabled | default(false) | bool + - name: Установить Rust AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/worktime-influx-exporter" @@ -1691,31 +1471,6 @@ - aw_worktime_influx_enabled | default(false) | bool - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог AW worktime Influx exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-worktime-influx-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_worktime_influx_enabled | default(false) | bool - - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить AW worktime Influx exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-worktime-influx-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-worktime-influx-exporter-rust - when: - - aw_worktime_influx_enabled | default(false) | bool - - aw_worktime_influx_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW worktime Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-worktime-influx-exporter.timer" @@ -1742,6 +1497,13 @@ become: false when: aw_dlp_influx_enabled | default(false) | bool + - name: Требовать Rust AW DLP Influx exporter artifact + ansible.builtin.assert: + that: + - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-influx-exporter" + when: aw_dlp_influx_enabled | default(false) | bool + - name: Установить Rust AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-influx-exporter" @@ -1753,31 +1515,6 @@ - aw_dlp_influx_enabled | default(false) | bool - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог AW DLP Influx exporter - ansible.builtin.file: - path: /etc/systemd/system/aw-dlp-influx-exporter.service.d - state: directory - owner: root - group: root - mode: "0755" - when: - - aw_dlp_influx_enabled | default(false) | bool - - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - - name: Переключить AW DLP Influx exporter на Rust - ansible.builtin.copy: - dest: /etc/systemd/system/aw-dlp-influx-exporter.service.d/20-rust-switch.conf - owner: root - group: root - mode: "0644" - content: | - [Service] - ExecStart= - ExecStart=/usr/local/bin/aw-dlp-influx-exporter-rust - when: - - aw_dlp_influx_enabled | default(false) | bool - - aw_dlp_influx_exporter_rust_binary.stat.exists | default(false) - - name: Установить systemd timer AW DLP Influx exporter ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-dlp-influx-exporter.timer" @@ -1986,13 +1723,28 @@ regexp: '' replace: '' - - name: Скопировать merge script AW DB на сервер + - name: Check Rust merge AW DB binary artifact + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" + register: aw_merge_db_rust_binary + delegate_to: localhost + become: false + + - name: Требовать Rust merge AW DB artifact + ansible.builtin.assert: + that: + - aw_merge_db_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/merge-aw-server-dbs" + when: aw_legacy_db_merge_enabled | default(false) | bool + + - name: Install Rust merge AW DB binary ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/merge_aw_server_dbs.py" - dest: /usr/local/bin/merge_aw_server_dbs.py + src: "{{ aw_rust_release_dir }}/merge-aw-server-dbs" + dest: /usr/local/bin/merge-aw-server-dbs owner: root group: root mode: "0755" + when: aw_merge_db_rust_binary.stat.exists | default(false) - name: Проверить наличие legacy root DB ansible.builtin.stat: @@ -2051,8 +1803,7 @@ - name: Merge legacy root DB в target DB ansible.builtin.command: argv: - - python3 - - /usr/local/bin/merge_aw_server_dbs.py + - /usr/local/bin/merge-aw-server-dbs - --base - /root/.local/share/activitywatch/aw-server-rust/sqlite.db - --overlay @@ -2182,34 +1933,6 @@ group: root mode: "0755" - - name: Скопировать агрегатор событий DLP - ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/aggregate_dlp_events.py" - dest: "/opt/activitywatch/scripts/aggregate_dlp_events.py" - owner: root - group: root - mode: "0755" - - - name: Установить systemd unit для агрегатора - ansible.builtin.copy: - dest: /etc/systemd/system/activitywatch-dlp-aggregator.service - content: | - [Unit] - Description=ActivityWatch DLP Event Aggregator - After=activitywatch-server.service - - [Service] - Type=oneshot - User={{ aw_server_user }} - WorkingDirectory={{ aw_server_data_dir }} - ExecStart=/usr/bin/python3 /opt/activitywatch/scripts/aggregate_dlp_events.py \ - --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ - --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ - --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json - - [Install] - WantedBy=multi-user.target - - name: Проверить локальный Rust DLP aggregator ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/dlp-aggregator" @@ -2217,6 +1940,12 @@ register: dlp_aggregator_rust_binary become: false + - name: Требовать Rust DLP aggregator artifact + ansible.builtin.assert: + that: + - dlp_aggregator_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-aggregator" + - name: Установить Rust DLP aggregator ansible.builtin.copy: src: "{{ aw_rust_release_dir }}/dlp-aggregator" @@ -2226,29 +1955,28 @@ mode: "0755" when: dlp_aggregator_rust_binary.stat.exists | default(false) - - name: Создать drop-in каталог DLP aggregator - ansible.builtin.file: - path: /etc/systemd/system/activitywatch-dlp-aggregator.service.d - state: directory - owner: root - group: root - mode: "0755" - when: dlp_aggregator_rust_binary.stat.exists | default(false) - - - name: Переключить DLP aggregator на Rust + - name: Установить systemd unit для агрегатора ansible.builtin.copy: - dest: /etc/systemd/system/activitywatch-dlp-aggregator.service.d/20-rust-switch.conf + dest: /etc/systemd/system/activitywatch-dlp-aggregator.service owner: root group: root mode: "0644" content: | + [Unit] + Description=ActivityWatch DLP Event Aggregator + After=activitywatch-server.service + [Service] - ExecStart= + Type=oneshot + User={{ aw_server_user }} + WorkingDirectory={{ aw_server_data_dir }} ExecStart=/usr/local/bin/dlp-aggregator-rust \ --aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \ --sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \ --state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json - when: dlp_aggregator_rust_binary.stat.exists | default(false) + + [Install] + WantedBy=multi-user.target - name: Установить systemd timer для агрегатора ansible.builtin.copy: @@ -2276,13 +2004,6 @@ - name: Настроить IOC enrichment из Hayabusa Sigma when: aw_dlp_ioc_enabled | default(false) | bool block: - - name: Установить зависимости IOC refresh - ansible.builtin.apt: - name: - - python3-yaml - state: present - update_cache: true - - name: Создать каталог IOC enrichment ansible.builtin.file: path: "{{ aw_dlp_ioc_workdir }}/{{ item }}" @@ -2295,10 +2016,23 @@ - output - tmp - - name: Установить экстрактор IOC из Sigma + - name: Проверить локальный Rust extractor IOC из Sigma + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/extract-ioc-from-sigma" + delegate_to: localhost + register: aw_extract_ioc_from_sigma_rust_binary + become: false + + - name: Требовать Rust extractor IOC из Sigma + ansible.builtin.assert: + that: + - aw_extract_ioc_from_sigma_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/extract-ioc-from-sigma" + + - name: Установить Rust extractor IOC из Sigma ansible.builtin.copy: - src: "{{ aw_repo_root }}/scripts/extract_ioc_from_sigma.py" - dest: /usr/local/bin/aw-extract-ioc-from-sigma.py + src: "{{ aw_rust_release_dir }}/extract-ioc-from-sigma" + dest: /usr/local/bin/aw-extract-ioc-from-sigma owner: root group: root mode: "0755" @@ -2330,7 +2064,7 @@ fi mv "${EXTRACTED_DIR}" "${RULES_DIR}" - /usr/bin/python3 /usr/local/bin/aw-extract-ioc-from-sigma.py \ + /usr/local/bin/aw-extract-ioc-from-sigma \ --rules-root "${RULES_DIR}" \ --out-dir "${OUTDIR}" @@ -2642,28 +2376,23 @@ mode: "0755" when: aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper link-case для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_link_case_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-link-case-rust" + - name: Установить helper link-case для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-link-case-rust ]]; then - exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py "$@" + exec /usr/local/bin/aw-hayabusa-link-case-rust "$@" dest: /usr/local/bin/aw-hayabusa-link-case owner: root group: root mode: "0755" - - name: Положить helper link-case в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-link-case.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-link-case.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust helper from-windows для Hayabusa ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" @@ -2680,28 +2409,23 @@ mode: "0755" when: aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper from-windows для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_from_windows_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-from-windows-rust" + - name: Установить helper from-windows для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-from-windows-rust ]]; then - exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py "$@" + exec /usr/local/bin/aw-hayabusa-from-windows-rust "$@" dest: /usr/local/bin/aw-hayabusa-from-windows owner: root group: root mode: "0755" - - name: Положить helper from-windows в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-from-windows.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-from-windows.py - owner: root - group: root - mode: "0755" - - name: Положить README Hayabusa в server-side ops bundle ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/hayabusa/README.md" @@ -2726,28 +2450,23 @@ mode: "0755" when: aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper autoprocess для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_autoprocess_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-autoprocess-rust" + - name: Установить helper autoprocess для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-autoprocess-rust ]]; then - exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py "$@" + exec /usr/local/bin/aw-hayabusa-autoprocess-rust "$@" dest: /usr/local/bin/aw-hayabusa-autoprocess owner: root group: root mode: "0755" - - name: Положить helper autoprocess в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-autoprocess.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-autoprocess.py - owner: root - group: root - mode: "0755" - - name: Проверить локальный Rust helper case-alert для Hayabusa ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" @@ -2764,28 +2483,23 @@ mode: "0755" when: aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + - name: Требовать Rust helper case-alert для Hayabusa + ansible.builtin.assert: + that: + - aw_hayabusa_case_alert_rust_binary.stat.exists | default(false) + fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/aw-hayabusa-case-alert-rust" + - name: Установить helper case-alert для Hayabusa ansible.builtin.copy: content: | #!/usr/bin/env bash set -euo pipefail - if [[ -x /usr/local/bin/aw-hayabusa-case-alert-rust ]]; then - exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" - fi - exec /usr/bin/python3 /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py "$@" + exec /usr/local/bin/aw-hayabusa-case-alert-rust "$@" dest: /usr/local/bin/aw-hayabusa-case-alert owner: root group: root mode: "0755" - - name: Положить helper case-alert в server-side ops bundle - ansible.builtin.copy: - src: "{{ aw_repo_root }}/aw-server/hayabusa/aw-hayabusa-case-alert.py" - dest: /opt/activitywatch/aw-rus-ops/hayabusa/aw-hayabusa-case-alert.py - owner: root - group: root - mode: "0755" - - name: Установить systemd unit aw-hayabusa-drop.service ansible.builtin.copy: src: "{{ aw_repo_root }}/aw-server/aw-hayabusa-drop.service" @@ -2833,9 +2547,9 @@ - not ansible_check_mode - aw_server_post_deploy_health_check_enabled | default(true) | bool block: - - name: Запустить /usr/local/bin/aw-rus-healthd.py --json + - name: Запустить /usr/local/bin/aw-rus-healthd-rust --json ansible.builtin.command: - cmd: /usr/local/bin/aw-rus-healthd.py --json + cmd: /usr/local/bin/aw-rus-healthd-rust --json register: aw_post_deploy_health changed_when: false failed_when: false diff --git a/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml b/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml index e71a3a0..4a1734c 100644 --- a/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml +++ b/install-kit-awindows-20260427-211240/ansible/deploy_aw_windows.yml @@ -181,6 +181,7 @@ - export-evtx-for-hayabusa.ps1 - export-upload-hayabusa-to-aw-server.ps1 - export-upload-file-1c-telemetry.ps1 + - sync-dlp-evidence-artifacts.ps1 - migrate-awatch-rus-paths.ps1 - deploy-domain-users.ps1 - deploy-ensemble.ps1 diff --git a/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml b/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml index 1ac786e..c6318db 100644 --- a/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml +++ b/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_and_deploy_aw.yml @@ -1,46 +1,14 @@ --- -- name: Создать один Proxmox CT и развернуть AWatch-rus +- name: Deprecated Proxmox CT bootstrap path hosts: proxmox gather_facts: false - vars: - proxmox_bootstrap_dir: "/tmp/aw-rus-bootstrap" - aw_bootstrap_files: - - install_aw_server.sh - - apply_webui_ru_patch.sh - - activitywatch-server.service - - aw-worktime-api.py - - aw-worktime-api.service - - aw-worktime-ui-bridge.py - - aw-worktime-ui-bridge.service - - aw-worktime-ui-bridge.timer - - aw-worktime-panel.js - - aw-server.env.example - - aw-ru-patch.js - - aw-sw-cleanup.js - - aw-host-groups.json - - settings/classes-worktime.json - - settings/views-default.json - tasks: - - name: Выполнить workflow создания одного CT - ansible.builtin.include_tasks: tasks/provision_ct_and_deploy_aw.yml - vars: - ct_id: "{{ proxmox_ct_id }}" - ct_hostname: "{{ proxmox_ct_hostname }}" - ct_storage: "{{ proxmox_ct_storage }}" - ct_template: "{{ proxmox_ct_template }}" - ct_rootfs_size: "{{ proxmox_ct_rootfs_size }}" - ct_cores: "{{ proxmox_ct_cores }}" - ct_memory: "{{ proxmox_ct_memory }}" - ct_swap: "{{ proxmox_ct_swap }}" - ct_bridge: "{{ proxmox_ct_bridge }}" - ct_ip: "{{ proxmox_ct_ip }}" - ct_gw: "{{ proxmox_ct_gw }}" - ct_vlan: "{{ proxmox_ct_vlan | default('') }}" - ct_nameserver: "{{ proxmox_ct_nameserver | default('') }}" - ct_searchdomain: "{{ proxmox_ct_searchdomain | default('') }}" - ct_password: "{{ proxmox_ct_password }}" - ct_unprivileged: "{{ proxmox_ct_unprivileged }}" - ct_onboot: "{{ proxmox_ct_onboot }}" - ct_features: "{{ proxmox_ct_features }}" + - name: Stop deprecated Python-era CT bootstrap + ansible.builtin.fail: + msg: >- + ansible/provision_proxmox_ct_and_deploy_aw.yml is deprecated because + its bootstrap path copied removed Python service entrypoints. Use the + existing DetMir AW server and ansible/deploy_aw_server.yml for the + Rust application deploy. Rebuild this CT bootstrap path separately + before using it for new infrastructure. diff --git a/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml b/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml index e0ff9bc..3819d24 100644 --- a/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml +++ b/install-kit-awindows-20260427-211240/ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml @@ -1,56 +1,14 @@ --- -- name: Создать матрицу Proxmox CT и развернуть AWatch-rus с RU patch +- name: Deprecated Proxmox CT matrix bootstrap path hosts: proxmox gather_facts: false - vars: - proxmox_bootstrap_dir: "/tmp/aw-rus-bootstrap" - aw_bootstrap_files: - - install_aw_server.sh - - apply_webui_ru_patch.sh - - activitywatch-server.service - - aw-worktime-api.py - - aw-worktime-api.service - - aw-worktime-ui-bridge.py - - aw-worktime-ui-bridge.service - - aw-worktime-ui-bridge.timer - - aw-worktime-panel.js - - aw-server.env.example - - aw-ru-patch.js - - aw-sw-cleanup.js - - aw-host-groups.json - - settings/classes-worktime.json - - settings/views-default.json - tasks: - - name: Проверить, что матрица CT задана - ansible.builtin.assert: - that: - - proxmox_ct_matrix is defined - - proxmox_ct_matrix | length > 0 - fail_msg: "Задайте proxmox_ct_matrix в group_vars/proxmox-matrix.yml" - - - name: Выполнить workflow создания для каждого CT - ansible.builtin.include_tasks: tasks/provision_ct_and_deploy_aw.yml - vars: - ct_id: "{{ item.id }}" - ct_hostname: "{{ item.hostname }}" - ct_storage: "{{ item.storage }}" - ct_template: "{{ item.template }}" - ct_rootfs_size: "{{ item.rootfs_size }}" - ct_cores: "{{ item.cores }}" - ct_memory: "{{ item.memory }}" - ct_swap: "{{ item.swap }}" - ct_bridge: "{{ item.bridge }}" - ct_ip: "{{ item.ip }}" - ct_gw: "{{ item.gw }}" - ct_vlan: "{{ item.vlan | default('') }}" - ct_nameserver: "{{ item.nameserver | default('') }}" - ct_searchdomain: "{{ item.searchdomain | default('') }}" - ct_password: "{{ item.password }}" - ct_unprivileged: "{{ item.unprivileged }}" - ct_onboot: "{{ item.onboot }}" - ct_features: "{{ item.features }}" - loop: "{{ proxmox_ct_matrix }}" - loop_control: - label: "ct={{ item.id }} host={{ item.hostname }} ip={{ item.ip }}" + - name: Stop deprecated Python-era CT matrix bootstrap + ansible.builtin.fail: + msg: >- + ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml is deprecated + because its bootstrap path copied removed Python service entrypoints. + Use the existing DetMir AW server and ansible/deploy_aw_server.yml for + the Rust application deploy. Rebuild this CT bootstrap path separately + before using it for new infrastructure. diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py deleted file mode 100644 index 07d0538..0000000 --- a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.py +++ /dev/null @@ -1,512 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import socket -import subprocess -import sys -import tempfile -import time -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -from urllib import request - - -ENV_FILE = Path("/etc/activitywatch/aw-server.env") - - -def load_env_file(path: Path) -> None: - if not path.exists(): - return - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - # systemd EnvironmentFile has already injected the variables for service runs. - return - for raw_line in lines: - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - key = key.strip() - value = value.strip().strip("'").strip('"') - os.environ.setdefault(key, value) - - -def env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def now_utc() -> datetime: - return datetime.now(UTC) - - -def parse_ts(value: str | None) -> datetime | None: - if not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) - except ValueError: - return None - - -def age_seconds(ts: datetime | None, now: datetime) -> int | None: - if ts is None: - return None - return max(0, int((now - ts).total_seconds())) - - -def http_json(url: str, timeout: int = 20, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: - last_exc: Exception | None = None - for attempt in range(max(1, attempts)): - try: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) - except Exception as exc: - last_exc = exc - if attempt + 1 >= max(1, attempts): - break - time.sleep(backoff_seconds * (2**attempt)) - assert last_exc is not None - raise last_exc - - -def run_command(cmd: list[str]) -> tuple[int, str]: - proc = subprocess.run( - cmd, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - return proc.returncode, proc.stdout.strip() - - -def tcp_connect(host: str, port: int, timeout: float) -> tuple[bool, str]: - try: - with socket.create_connection((host, port), timeout=timeout): - return True, "connected" - except OSError as exc: - return False, str(exc) - - -@dataclass -class CheckResult: - name: str - status: str - summary: str - details: dict[str, Any] - - -class Report: - def __init__(self) -> None: - self.results: list[CheckResult] = [] - - def add(self, name: str, status: str, summary: str, **details: Any) -> None: - self.results.append(CheckResult(name=name, status=status, summary=summary, details=details)) - - @property - def ok(self) -> bool: - return not any(item.status == "fail" for item in self.results) - - def as_dict(self) -> dict[str, Any]: - counts = {"ok": 0, "warn": 0, "fail": 0} - for item in self.results: - counts[item.status] = counts.get(item.status, 0) + 1 - return { - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "ok": self.ok, - "counts": counts, - "results": [ - { - "name": item.name, - "status": item.status, - "summary": item.summary, - "details": item.details, - } - for item in self.results - ], - } - - def render_text(self) -> str: - icon = {"ok": "OK", "warn": "WARN", "fail": "FAIL"} - lines = ["=== AW-RUS Health ===", f"Timestamp: {now_utc().isoformat().replace('+00:00', 'Z')}", ""] - for item in self.results: - lines.append(f"[{icon.get(item.status, item.status.upper())}] {item.name}: {item.summary}") - lines.append("") - payload = self.as_dict() - lines.append( - "Counts: ok={ok} warn={warn} fail={fail}".format( - ok=payload["counts"]["ok"], - warn=payload["counts"]["warn"], - fail=payload["counts"]["fail"], - ) - ) - lines.append(f"Overall: {'OK' if payload['ok'] else 'FAIL'}") - return "\n".join(lines) - - -def latest_bucket_event(api_base: str, bucket_id: str) -> dict[str, Any] | None: - events = http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1", timeout=25) - if isinstance(events, list) and events: - events = [item for item in events if isinstance(item, dict)] - if not events: - return None - events.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return events[0] - return None - - -def bucket_metadata_ts(buckets: dict[str, Any], bucket_id: str) -> datetime | None: - bucket = buckets.get(bucket_id) if isinstance(buckets, dict) else None - if not isinstance(bucket, dict): - return None - metadata = bucket.get("metadata") - if not isinstance(metadata, dict): - return None - return parse_ts(metadata.get("end")) - - -def host_activity_from_worktime(event: dict[str, Any] | None, max_age_seconds: int) -> dict[str, Any]: - now = now_utc() - if not event: - return {"fresh": False, "active": False, "age_seconds": None, "timestamp": None} - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now) - data = event.get("data") or {} - is_fresh = age is not None and age <= max_age_seconds - is_active = bool(is_fresh and data.get("active")) - return { - "fresh": bool(is_fresh), - "active": bool(is_active), - "age_seconds": age, - "timestamp": event.get("timestamp"), - "data": data, - } - - -def bucket_health( - api_base: str, - bucket_id: str, - max_age_seconds: int, - missing_status: str, - stale_status: str, -) -> tuple[str, str, dict[str, Any]]: - try: - event = latest_bucket_event(api_base, bucket_id) - except Exception as exc: - return "fail", f"bucket query failed: {exc}", {"bucket": bucket_id} - - if not event: - return missing_status, "no events", {"bucket": bucket_id} - - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now_utc()) - details = {"bucket": bucket_id, "timestamp": event.get("timestamp"), "age_seconds": age} - if age is None: - return "warn", "timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"stale ({age}s)", details - return "ok", f"fresh ({age}s)", details - - -def bucket_timestamp_health( - api_base: str, - buckets: dict[str, Any], - bucket_id: str, - max_age_seconds: int, - missing_status: str, - stale_status: str, -) -> tuple[str, str, dict[str, Any]]: - metadata_ts = bucket_metadata_ts(buckets, bucket_id) - if metadata_ts is not None: - age = age_seconds(metadata_ts, now_utc()) - timestamp = metadata_ts.isoformat().replace("+00:00", "Z") - details = { - "bucket": bucket_id, - "timestamp": timestamp, - "age_seconds": age, - "timestamp_source": "bucket_metadata.end", - } - if age is None: - return "warn", "timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"stale ({age}s)", details - return "ok", f"fresh ({age}s)", details - - return bucket_health(api_base, bucket_id, max_age_seconds, missing_status, stale_status) - - -def guard_bucket_health( - api_base: str, - host: str, - max_age_seconds: int, - required: bool, -) -> tuple[str, str, dict[str, Any]]: - bucket_id = f"aw-rus-collector-guard_{host}" - try: - event = latest_bucket_event(api_base, bucket_id) - except Exception as exc: - status = "fail" if required else "warn" - return status, f"guard bucket query failed: {exc}", {"bucket": bucket_id} - - if not event: - status = "fail" if required else "warn" - return status, "no guard heartbeat", {"bucket": bucket_id, "required": required} - - ts = parse_ts(event.get("timestamp")) - age = age_seconds(ts, now_utc()) - data = event.get("data") if isinstance(event.get("data"), dict) else {} - guard_status = str(data.get("status") or "unknown").lower() - details = { - "bucket": bucket_id, - "timestamp": event.get("timestamp"), - "age_seconds": age, - "required": required, - "guard_status": guard_status, - "mode": data.get("mode"), - "live_session_count": data.get("liveSessionCount"), - "problems": data.get("problems", []), - "actions": data.get("actions", []), - } - - stale_status = "fail" if required else "warn" - if age is None: - return "warn", "guard timestamp parse failed", details - if age > max_age_seconds: - return stale_status, f"guard stale ({age}s)", details - if guard_status in ("fail", "error"): - return "fail" if required else "warn", f"guard reports {guard_status}", details - if guard_status == "warn": - return "warn", "guard reports warn", details - return "ok", f"guard fresh ({age}s)", details - - -def latest_validation_report(validation_dir: Path) -> Path | None: - candidates = sorted( - (path for path in validation_dir.glob("*-aw_validate_ansible.json") if path.is_file()), - key=lambda item: item.stat().st_mtime, - reverse=True, - ) - return candidates[0] if candidates else None - - -def write_atomic(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - handle.write(content) - tmp_name = handle.name - os.replace(tmp_name, path) - - -def chmod_if_possible(path: Path, mode: int) -> None: - try: - path.chmod(mode) - except OSError: - pass - - -def check_wrapper( - report: Report, - name: str, - cmd: list[str], - json_mode: bool = False, - failure_status: str = "fail", -) -> None: - if not Path(cmd[0]).exists(): - report.add(name, "warn", "binary missing", command=cmd) - return - rc, output = run_command(cmd) - details: dict[str, Any] = {"command": cmd, "returncode": rc} - if json_mode: - try: - details["payload"] = json.loads(output) if output else {} - except json.JSONDecodeError: - details["raw_output"] = output - report.add(name, "fail", "invalid JSON output", **details) - return - else: - details["output"] = output - report.add(name, "ok" if rc == 0 else failure_status, "passed" if rc == 0 else "failed", **details) - - -def main() -> int: - load_env_file(ENV_FILE) - - parser = argparse.ArgumentParser(description="Unified AW-RUS health orchestrator") - parser.add_argument("--aw-server", default=env("AW_SERVER_URL", "http://127.0.0.1:5600")) - parser.add_argument( - "--worktime-api", - default=env("AW_RUS_HEALTH_WORKTIME_API", env("AW_WORKTIME_REPORT_BASE", "http://127.0.0.1:5610")), - ) - parser.add_argument("--rdp-host", default=env("AW_MONITORED_WINDOWS_HOST", "192.168.100.18")) - parser.add_argument("--rdp-hostname", default=env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025")) - parser.add_argument("--state-dir", default=env("AW_RUS_HEALTH_STATE_DIR", "/var/lib/activitywatch/health")) - parser.add_argument("--validation-dir", default=env("AW_RUS_HEALTH_VALIDATION_DIR", "/var/lib/activitywatch/health/windows-validation")) - parser.add_argument("--session-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_MAX_AGE_SECONDS", "900"))) - parser.add_argument("--interactive-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_INTERACTIVE_MAX_AGE_SECONDS", "900"))) - parser.add_argument("--session-events-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS", "86400"))) - parser.add_argument("--guard-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS", "300"))) - parser.add_argument("--guard-required", action="store_true", default=env("AW_RUS_HEALTH_GUARD_REQUIRED", "0").lower() in ("1", "true", "yes")) - parser.add_argument("--validation-max-age-seconds", type=int, default=int(env("AW_RUS_HEALTH_VALIDATION_MAX_AGE_SECONDS", "259200"))) - parser.add_argument("--tcp-timeout-seconds", type=float, default=float(env("AW_RUS_HEALTH_TCP_TIMEOUT_SECONDS", "3"))) - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - - report = Report() - aw_api_base = args.aw_server.rstrip("/") - if not aw_api_base.endswith("/api/0"): - aw_api_base = aw_api_base.rstrip("/") + "/api/0" - - check_wrapper(report, "wrapper:aw-health-check", ["/usr/local/bin/aw-health-check"], failure_status="warn") - check_wrapper( - report, - "wrapper:dlp-health-check", - ["/usr/local/bin/dlp-health-check", "--json"], - json_mode=True, - failure_status="warn", - ) - - try: - info = http_json(f"{aw_api_base}/info") - report.add("http:aw-server", "ok", "activitywatch API responded", version=info.get("version")) - except Exception as exc: - report.add("http:aw-server", "fail", f"activitywatch API failed: {exc}", url=f"{aw_api_base}/info") - - try: - payload = http_json(args.worktime_api.rstrip("/") + "/health") - report.add("http:worktime-api", "ok", "worktime API responded", payload=payload if isinstance(payload, dict) else {}) - except Exception as exc: - report.add("http:worktime-api", "fail", f"worktime API failed: {exc}", url=args.worktime_api) - - for port, label in ((5985, "winrm"), (3389, "rdp")): - ok, message = tcp_connect(args.rdp_host, port, args.tcp_timeout_seconds) - report.add(f"tcp:{label}", "ok" if ok else "fail", message if ok else f"unreachable: {message}", host=args.rdp_host, port=port) - - try: - buckets = http_json(f"{aw_api_base}/buckets") - if not isinstance(buckets, dict): - raise RuntimeError("bucket index is not a dict") - report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets)) - except Exception as exc: - report.add("aw:buckets-index", "fail", f"failed to load bucket index: {exc}") - buckets = {} - - host = args.rdp_hostname - guard_status, guard_summary, guard_details = guard_bucket_health( - aw_api_base, - host, - args.guard_max_age_seconds, - args.guard_required, - ) - report.add("bucket:collector-guard", guard_status, guard_summary, **guard_details) - - worktime_bucket = f"aw-worktime-sessions_{host}" - worktime_event = None - if buckets: - try: - worktime_event = latest_bucket_event(aw_api_base, worktime_bucket) - except Exception: - worktime_event = None - activity = host_activity_from_worktime(worktime_event, args.session_max_age_seconds) - if worktime_event: - status, summary, details = bucket_health( - aw_api_base, - worktime_bucket, - args.session_max_age_seconds, - missing_status="fail", - stale_status="fail", - ) - details["host_activity"] = activity - report.add("bucket:worktime-sessions", status, summary, **details) - else: - report.add("bucket:worktime-sessions", "fail", "no events", bucket=worktime_bucket, host_activity=activity) - - interactive_required = bool(activity["active"]) - for bucket_name, label in ( - ("aw-watcher-afk", "bucket:afk"), - ("aw-watcher-window", "bucket:window"), - ("aw-dlp-endpoint-signals", "bucket:endpoint-signals"), - ): - status, summary, details = bucket_health( - aw_api_base, - f"{bucket_name}_{host}", - args.interactive_max_age_seconds, - missing_status="fail" if interactive_required else "warn", - stale_status="fail" if interactive_required else "warn", - ) - details["interactive_required"] = interactive_required - details["host_activity"] = activity - if not interactive_required and status != "ok": - details["inactive_summary"] = summary - status = "ok" - summary = "inactive: no active interactive users" - report.add(label, status, summary, **details) - - session_status, session_summary, session_details = bucket_timestamp_health( - aw_api_base, - buckets, - f"aw-session-events_{host}", - args.session_events_max_age_seconds, - missing_status="fail", - stale_status="warn", - ) - if session_status == "warn" and session_details.get("age_seconds") is not None: - session_status = "ok" - session_summary = f"event-driven ({session_details['age_seconds']}s since last logon marker)" - report.add("bucket:session-events", session_status, session_summary, **session_details) - - validation_dir = Path(args.validation_dir) - validation_report = latest_validation_report(validation_dir) - if validation_report is None: - report.add("validation:windows", "warn", "no validation report snapshot", directory=str(validation_dir)) - else: - try: - payload = json.loads(validation_report.read_text(encoding="utf-8-sig")) - age = age_seconds(datetime.fromtimestamp(validation_report.stat().st_mtime, tz=UTC), now_utc()) - if age is not None and age > args.validation_max_age_seconds: - report.add( - "validation:windows", - "warn", - f"validation snapshot is stale ({age}s)", - path=str(validation_report), - overall_ok=payload.get("overallOk"), - failed_sections=payload.get("summary", {}).get("failedSections", []), - ) - elif payload.get("overallOk") is True: - report.add("validation:windows", "ok", "validation snapshot OK", path=str(validation_report), age_seconds=age) - else: - report.add( - "validation:windows", - "fail", - "validation snapshot reports failure", - path=str(validation_report), - age_seconds=age, - failed_sections=payload.get("summary", {}).get("failedSections", []), - ) - except Exception as exc: - report.add("validation:windows", "fail", f"invalid validation snapshot: {exc}", path=str(validation_report)) - - payload = report.as_dict() - state_dir = Path(args.state_dir) - health_json_path = state_dir / "aw-rus-health.json" - health_txt_path = state_dir / "aw-rus-health.txt" - write_atomic(health_json_path, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") - write_atomic(health_txt_path, report.render_text() + "\n") - chmod_if_possible(health_json_path, 0o644) - chmod_if_possible(health_txt_path, 0o644) - - if args.json: - print(json.dumps(payload, ensure_ascii=False, indent=2)) - else: - print(report.render_text()) - return 0 if payload["ok"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service index 015a906..b5ba629 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service +++ b/install-kit-awindows-20260427-211240/aw-server/aw-rus-healthd.service @@ -6,7 +6,7 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-rus-healthd.py +ExecStart=/usr/local/bin/aw-rus-healthd-rust User=root Group=root StandardOutput=journal diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py deleted file mode 100644 index 854d6ff..0000000 --- a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.py +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import tempfile -import time -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any -from urllib import request - - -ENV_FILE = Path("/etc/activitywatch/aw-server.env") -DEFAULT_STATE_DIR = Path("/var/lib/activitywatch/slo") -DEFAULT_HEALTHD_CMD = "/usr/local/bin/aw-rus-healthd.py --json" -DEFAULT_HEALTHD_STATE_FILE = Path("/var/lib/activitywatch/health/aw-rus-health.json") - - -def load_env_file(path: Path) -> None: - if not path.exists(): - return - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - # systemd EnvironmentFile has already loaded the variables for the service. - return - for raw_line in lines: - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - os.environ.setdefault(key.strip(), value.strip().strip("'").strip('"')) - - -def env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def now_utc() -> datetime: - return datetime.now(UTC) - - -def parse_ts(value: str) -> datetime: - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) - - -def iso(dt: datetime) -> str: - return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -def write_atomic(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - handle.write(content) - tmp_name = handle.name - os.replace(tmp_name, path) - - -def chmod_if_possible(path: Path, mode: int) -> None: - try: - path.chmod(mode) - except OSError: - pass - - -def chown_like_parent_if_possible(path: Path) -> None: - try: - parent_stat = path.parent.stat() - os.chown(path, parent_stat.st_uid, parent_stat.st_gid) - except OSError: - pass - - -def fetch_url(url: str, timeout_seconds: float, *, accept: str | None = None, attempts: int = 1) -> dict[str, Any]: - started = now_utc() - last_error = "" - for attempt in range(1, max(1, attempts) + 1): - try: - req = request.Request(url, headers={"Accept": accept}) if accept else url - with request.urlopen(req, timeout=timeout_seconds) as resp: - body = resp.read() - status = int(resp.status) - content_type = resp.headers.get("Content-Type", "") - ok = 200 <= status < 300 - return { - "ok": ok, - "status": status, - "body_bytes": len(body), - "content_type": content_type, - "body": body, - "attempts": attempt, - "latency_ms": int((now_utc() - started).total_seconds() * 1000), - "url": url, - } - except Exception as exc: - last_error = str(exc) - if attempt < max(1, attempts): - time.sleep(1) - return { - "ok": False, - "error": last_error, - "body_bytes": 0, - "attempts": max(1, attempts), - "latency_ms": int((now_utc() - started).total_seconds() * 1000), - "url": url, - } - - -def public_probe_result(result: dict[str, Any]) -> dict[str, Any]: - return {key: value for key, value in result.items() if key != "body"} - - -def http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: - return public_probe_result(fetch_url(url, timeout_seconds, attempts=2)) - - -def html_probe( - url: str, - timeout_seconds: float, - *, - min_bytes: int, - required_markers: tuple[str, ...], -) -> dict[str, Any]: - result = fetch_url(url, timeout_seconds, accept="text/html", attempts=2) - if not result.get("ok"): - return public_probe_result(result) - - body = result.get("body", b"") - text = body.decode("utf-8", errors="replace") if isinstance(body, bytes) else str(body) - missing = [marker for marker in required_markers if marker not in text] - content_type = str(result.get("content_type") or "") - if len(body) < min_bytes: - result["ok"] = False - result["error"] = f"body too small: {len(body)} < {min_bytes}" - elif "text/html" not in content_type.lower(): - result["ok"] = False - result["error"] = f"unexpected content-type: {content_type or 'unknown'}" - elif missing: - result["ok"] = False - result["error"] = "missing markers: " + ", ".join(missing) - result["missing_markers"] = missing - return public_probe_result(result) - - -def json_probe( - url: str, - timeout_seconds: float, - *, - expected_values: dict[str, Any], - required_keys: tuple[str, ...], -) -> dict[str, Any]: - result = fetch_url(url, timeout_seconds, accept="application/json", attempts=2) - if not result.get("ok"): - return public_probe_result(result) - - body = result.get("body", b"") - try: - payload = json.loads(body.decode("utf-8") if isinstance(body, bytes) else str(body)) - except Exception as exc: - result["ok"] = False - result["error"] = f"invalid json: {exc}" - return public_probe_result(result) - - if not isinstance(payload, dict): - result["ok"] = False - result["error"] = "json root is not object" - return public_probe_result(result) - - missing_keys = [key for key in required_keys if key not in payload] - mismatched = { - key: {"expected": expected, "actual": payload.get(key)} - for key, expected in expected_values.items() - if payload.get(key) != expected - } - if missing_keys: - result["ok"] = False - result["error"] = "missing json keys: " + ", ".join(missing_keys) - result["missing_keys"] = missing_keys - elif mismatched: - result["ok"] = False - result["error"] = "unexpected json values" - result["mismatched_values"] = mismatched - else: - result["json_keys"] = sorted(payload.keys()) - return public_probe_result(result) - - -def run_healthd(command: str, timeout_seconds: int) -> dict[str, Any]: - try: - proc = subprocess.run( - command, - shell=True, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as exc: - return { - "ok": False, - "returncode": None, - "error": f"timeout after {timeout_seconds}s", - "output_tail": (exc.stdout or "")[-1000:] if isinstance(exc.stdout, str) else "", - } - - payload: dict[str, Any] = {} - try: - payload = json.loads(proc.stdout or "{}") - except json.JSONDecodeError: - pass - return { - "ok": proc.returncode == 0 and bool(payload.get("ok")), - "returncode": proc.returncode, - "counts": payload.get("counts", {}), - "payload": payload if isinstance(payload, dict) else {}, - "output_tail": (proc.stdout or "")[-1000:], - } - - -def read_healthd_state(path: Path, max_age_seconds: int) -> dict[str, Any]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - generated = parse_ts(str(payload["generated_at_utc"])) - age = max(0, int((now_utc() - generated).total_seconds())) - except Exception as exc: - return {"ok": False, "error": str(exc), "path": str(path)} - return { - "ok": bool(payload.get("ok")) and age <= max_age_seconds, - "counts": payload.get("counts", {}), - "age_seconds": age, - "path": str(path), - "payload": payload if isinstance(payload, dict) else {}, - } - - -def load_samples(path: Path, cutoff: datetime) -> list[dict[str, Any]]: - samples: list[dict[str, Any]] = [] - if not path.exists(): - return samples - try: - lines = path.read_text(encoding="utf-8").splitlines() - except PermissionError: - return samples - for line in lines: - if not line.strip(): - continue - try: - item = json.loads(line) - ts = parse_ts(str(item["ts"])) - except Exception: - continue - if ts >= cutoff: - samples.append(item) - return samples - - -def append_and_trim_sample(path: Path, sample: dict[str, Any], retention_seconds: int) -> list[dict[str, Any]]: - cutoff = parse_ts(sample["ts"]) - timedelta(seconds=retention_seconds) - samples = load_samples(path, cutoff) - samples.append(sample) - content = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n" for item in samples) - write_atomic(path, content) - chmod_if_possible(path, 0o644) - chown_like_parent_if_possible(path) - return samples - - -def summarize_window( - samples: list[dict[str, Any]], - *, - now: datetime, - window_seconds: int, - sample_interval_seconds: int, - target_percent: float, -) -> dict[str, Any]: - cutoff = now - timedelta(seconds=window_seconds) - window = [] - for item in samples: - try: - if parse_ts(str(item["ts"])) >= cutoff: - window.append(item) - except Exception: - continue - - total = len(window) - good = sum(1 for item in window if item.get("ok") is True) - bad = total - good - availability = round((good / total) * 100, 5) if total else None - observed_bad_seconds = bad * sample_interval_seconds - budget_seconds = int(window_seconds * ((100.0 - target_percent) / 100.0)) - budget_remaining_seconds = budget_seconds - observed_bad_seconds - status = "unknown" - if total: - status = "ok" if budget_remaining_seconds >= 0 else "burning" - return { - "window_seconds": window_seconds, - "samples": total, - "good_samples": good, - "bad_samples": bad, - "availability_percent": availability, - "target_percent": target_percent, - "observed_bad_seconds": observed_bad_seconds, - "budget_seconds": budget_seconds, - "budget_remaining_seconds": budget_remaining_seconds, - "status": status, - } - - -def render_summary_text(summary: dict[str, Any]) -> str: - lines = [ - "=== AW-RUS SLO ===", - f"Timestamp: {summary['generated_at_utc']}", - f"Target: {summary['target_percent']}%", - "", - ] - for name, data in summary["windows"].items(): - availability = data["availability_percent"] - availability_text = "n/a" if availability is None else f"{availability:.5f}%" - remaining = data["budget_remaining_seconds"] - lines.append( - f"{name}: {data['status']} availability={availability_text} " - f"samples={data['samples']} bad={data['bad_samples']} " - f"bad_seconds={data['observed_bad_seconds']} budget_remaining_seconds={remaining}" - ) - lines.append("") - lines.append(f"Current sample: {'OK' if summary['current_sample']['ok'] else 'FAIL'}") - for name, probe in summary["current_sample"].get("probes", {}).items(): - marker = "OK" if probe.get("ok") else "FAIL" - detail = probe.get("status", probe.get("error", "")) - lines.append(f"- {name}: {marker} {detail}") - return "\n".join(lines) - - -def build_sample(args: argparse.Namespace) -> dict[str, Any]: - ts = iso(now_utc()) - if args.healthd_mode == "run": - health = run_healthd(args.healthd_cmd, args.health_timeout_seconds) - else: - health = read_healthd_state(Path(args.healthd_state_file), args.healthd_state_max_age_seconds) - probes = { - "aw_webui_index": html_probe( - args.aw_webui_url, - args.http_timeout_seconds, - min_bytes=1000, - required_markers=("ActivityWatch", 'id="app"', "ru-patch-v5.js"), - ), - "worktime_today_html": html_probe( - args.worktime_today_html_url, - args.http_timeout_seconds, - min_bytes=5000, - required_markers=("AW-rus", ""), - ), - "worktime_management_html": html_probe( - args.worktime_management_html_url, - args.http_timeout_seconds, - min_bytes=5000, - required_markers=("AW-rus", ""), - ), - "worktime_today_csv": http_probe(args.worktime_today_csv_url, args.http_timeout_seconds), - "worktime_management_json": json_probe( - args.worktime_management_json_url, - args.http_timeout_seconds, - expected_values={"host": args.host}, - required_keys=("generated_at_utc", "host", "summary", "rows", "workday"), - ), - } - ok = bool(health.get("ok")) and all(probe.get("ok") for probe in probes.values()) - return { - "ts": ts, - "ok": ok, - "healthd_ok": bool(health.get("ok")), - "healthd_counts": health.get("counts", {}), - "probes": probes, - } - - -def main() -> int: - load_env_file(ENV_FILE) - parser = argparse.ArgumentParser(description="AW-RUS rolling SLO sampler") - parser.add_argument("--state-dir", default=env("AW_RUS_SLO_STATE_DIR", str(DEFAULT_STATE_DIR))) - parser.add_argument("--healthd-cmd", default=env("AW_RUS_SLO_HEALTHD_CMD", DEFAULT_HEALTHD_CMD)) - parser.add_argument("--healthd-mode", choices=["state", "run"], default=env("AW_RUS_SLO_HEALTHD_MODE", "state")) - parser.add_argument("--healthd-state-file", default=env("AW_RUS_SLO_HEALTHD_STATE_FILE", str(DEFAULT_HEALTHD_STATE_FILE))) - parser.add_argument("--healthd-state-max-age-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTHD_STATE_MAX_AGE_SECONDS", "180"))) - parser.add_argument("--aw-base", default=env("AW_RUS_SLO_AW_BASE", env("AW_SERVER_URL", "http://127.0.0.1:5600"))) - parser.add_argument("--worktime-base", default=env("AW_RUS_SLO_WORKTIME_BASE", env("AW_RUS_HEALTH_WORKTIME_API", "http://127.0.0.1:5610"))) - parser.add_argument("--host", default=env("AW_RUS_SLO_HOST", env("AW_MONITORED_WINDOWS_HOSTNAME", "SHARKON2025"))) - parser.add_argument("--target-percent", type=float, default=float(env("AW_RUS_SLO_TARGET_PERCENT", "99.97"))) - parser.add_argument("--sample-interval-seconds", type=int, default=int(env("AW_RUS_SLO_SAMPLE_INTERVAL_SECONDS", "15"))) - parser.add_argument("--retention-days", type=int, default=int(env("AW_RUS_SLO_RETENTION_DAYS", "35"))) - parser.add_argument("--http-timeout-seconds", type=float, default=float(env("AW_RUS_SLO_HTTP_TIMEOUT_SECONDS", "15"))) - parser.add_argument("--health-timeout-seconds", type=int, default=int(env("AW_RUS_SLO_HEALTH_TIMEOUT_SECONDS", "90"))) - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - - aw_base = args.aw_base.rstrip("/") - worktime_base = args.worktime_base.rstrip("/") - args.aw_webui_url = f"{aw_base}/" - args.worktime_today_html_url = f"{worktime_base}/reports/worktime/today?format=html&day=today&host={args.host}&allow_stale=1" - args.worktime_management_html_url = f"{worktime_base}/reports/worktime/management?format=html&day=today&host={args.host}&allow_stale=1" - args.worktime_today_csv_url = f"{worktime_base}/reports/worktime/today?format=csv&day=today&host={args.host}&allow_stale=1" - args.worktime_management_json_url = f"{worktime_base}/reports/worktime/management?format=json&day=today&host={args.host}&allow_stale=1" - - state_dir = Path(args.state_dir) - sample_path = state_dir / "aw-slo-samples.jsonl" - sample = build_sample(args) - samples = append_and_trim_sample( - sample_path, - sample, - retention_seconds=max(1, args.retention_days) * 86400, - ) - - generated_at = now_utc() - summary = { - "generated_at_utc": iso(generated_at), - "target_percent": args.target_percent, - "sample_interval_seconds": args.sample_interval_seconds, - "current_sample": sample, - "windows": { - "24h": summarize_window( - samples, - now=generated_at, - window_seconds=86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - "7d": summarize_window( - samples, - now=generated_at, - window_seconds=7 * 86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - "30d": summarize_window( - samples, - now=generated_at, - window_seconds=30 * 86400, - sample_interval_seconds=args.sample_interval_seconds, - target_percent=args.target_percent, - ), - }, - } - summary_json_path = state_dir / "aw-slo-summary.json" - summary_txt_path = state_dir / "aw-slo-summary.txt" - write_atomic(summary_json_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") - text = render_summary_text(summary) - write_atomic(summary_txt_path, text + "\n") - chmod_if_possible(summary_json_path, 0o644) - chmod_if_possible(summary_txt_path, 0o644) - chown_like_parent_if_possible(summary_json_path) - chown_like_parent_if_possible(summary_txt_path) - if args.json: - print(json.dumps(summary, ensure_ascii=False, indent=2)) - else: - print(text) - return 0 if sample["ok"] else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service index a1bd724..0fbba5e 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service +++ b/install-kit-awindows-20260427-211240/aw-server/aw-slo-monitor.service @@ -6,7 +6,7 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-slo-monitor.py +ExecStart=/usr/local/bin/aw-slo-monitor-rust SuccessExitStatus=1 User=activitywatch Group=activitywatch diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py deleted file mode 100644 index caba965..0000000 --- a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.py +++ /dev/null @@ -1,3039 +0,0 @@ -#!/usr/bin/env python3 -import base64 -import csv -import hashlib -import html -import io -import importlib.util -import json -import os -import sys -import tempfile -import threading -import time -import urllib.request -from datetime import datetime, timezone, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from urllib.parse import parse_qs, urlencode, urlparse -from zoneinfo import ZoneInfo - - -def build_aw_api_base(raw_url): - url = (raw_url or "http://127.0.0.1:5600").strip().rstrip("/") - if url.endswith("/api/0"): - return url - return url + "/api/0" - - -AW_SERVER_URL = os.environ.get("AW_SERVER_URL", "http://127.0.0.1:5600") -AW = build_aw_api_base(AW_SERVER_URL) -REPORT_TZ = ZoneInfo(os.environ.get("AW_WORKTIME_TZ", "Europe/Moscow")) -IOC_DIR = os.environ.get("AW_DLP_IOC_DIR", "/opt/activitywatch/dlp-ioc/output") -DEFAULT_HOST = os.environ.get("AW_WORKTIME_HOST", "SHARKON2025").strip() or "SHARKON2025" -DEFAULT_SAMPLE_SECONDS = max(1.0, float(os.environ.get("AW_WORKTIME_DEFAULT_SAMPLE_SECONDS", "30"))) -MAX_SAMPLE_SECONDS = max(DEFAULT_SAMPLE_SECONDS, float(os.environ.get("AW_WORKTIME_MAX_SAMPLE_SECONDS", "300"))) -LISTEN_HOST = os.environ.get("AW_WORKTIME_LISTEN_HOST", "0.0.0.0") -LISTEN_PORT = int(os.environ.get("AW_WORKTIME_PORT", "5610")) -WORKDAY_START_HOUR = int(os.environ.get("AW_WORKTIME_MANAGER_START_HOUR", "9")) -WORKDAY_END_HOUR = int(os.environ.get("AW_WORKTIME_MANAGER_END_HOUR", "18")) -MANAGER_TARGET_COVERAGE_PCT = max(1, min(100, int(os.environ.get("AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT", "75")))) -MANAGER_LOW_COVERAGE_PCT = max(1, min(100, int(os.environ.get("AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT", "35")))) -MANAGER_LATE_START_GRACE_MINUTES = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_LATE_START_GRACE_MINUTES", "60"))) -MANAGER_EARLY_FINISH_GRACE_MINUTES = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_EARLY_FINISH_GRACE_MINUTES", "90"))) -MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS = max(60, int(os.environ.get("AW_WORKTIME_MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS", "900"))) -MANAGER_WEB_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_WEB_SOURCE_MAX_AGE_SECONDS", "259200"))) -MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS", "604800"))) -MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS = max(3600, int(os.environ.get("AW_WORKTIME_MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS", "172800"))) -MANAGER_TREND_DAYS = max(3, min(31, int(os.environ.get("AW_WORKTIME_MANAGER_TREND_DAYS", "7")))) -MANAGER_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_MANAGER_CACHE_TTL_SECONDS", "300"))) -MANAGER_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_MANAGER_CACHE_DIR", "/var/lib/activitywatch/worktime-cache")) -MANAGER_ALIASES_JSON = Path(os.environ.get("AW_WORKTIME_MANAGER_ALIASES_JSON", "/etc/activitywatch/worktime-manager-aliases.json")) -MANAGER_EXCLUDE_USERS = {item.strip().lower() for item in os.environ.get("AW_WORKTIME_MANAGER_EXCLUDE_USERS", "").split(",") if item.strip()} -EVENTS_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS", "30"))) -WORKTIME_EVENTS_LIMIT = max(1000, int(os.environ.get("AW_WORKTIME_EVENTS_LIMIT", "50000"))) -AW_HTTP_TIMEOUT_SECONDS = max(0.5, float(os.environ.get("AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS", "5"))) -REPORT_CACHE_TTL_SECONDS = max(0, int(os.environ.get("AW_WORKTIME_REPORT_CACHE_TTL_SECONDS", "60"))) -REPORT_STALE_TTL_SECONDS = max(REPORT_CACHE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_STALE_TTL_SECONDS", "900"))) -REPORT_DISK_CACHE_DIR = Path(os.environ.get("AW_WORKTIME_REPORT_DISK_CACHE_DIR", "/var/lib/activitywatch/worktime-report-cache")) -REPORT_DISK_STALE_TTL_SECONDS = max(REPORT_STALE_TTL_SECONDS, int(os.environ.get("AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS", "86400"))) -REPORT_BUSY_WAIT_SECONDS = max(0.0, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_SECONDS", "8"))) -REPORT_BUSY_WAIT_INTERVAL_SECONDS = max(0.05, float(os.environ.get("AW_WORKTIME_REPORT_BUSY_WAIT_INTERVAL_SECONDS", "0.2"))) -REPORT_MAX_CONCURRENT = max(1, int(os.environ.get("AW_WORKTIME_REPORT_MAX_CONCURRENT", "1"))) -TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS", "180"))) -TRUE_ACTIVE_MAX_EVENT_SECONDS = max(30, int(os.environ.get("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", "600"))) -MODULE_PATH = Path(__file__).resolve() -_ALIASES_CACHE = {"mtime": None, "users": {}, "owners": {}, "raw": {}} -_EVENTS_CACHE_LOCK = threading.Lock() -_EVENTS_CACHE = {} -_REPORT_CACHE_LOCK = threading.Lock() -_REPORT_RESPONSE_CACHE = {} -_REPORT_REFRESHING = set() -_REPORT_BUILD_SEMAPHORE = threading.BoundedSemaphore(REPORT_MAX_CONCURRENT) -_MANAGEMENT_BUILD_LOCKS_LOCK = threading.Lock() -_MANAGEMENT_BUILD_LOCKS = {} - - -def get(u): - with urllib.request.urlopen(u, timeout=AW_HTTP_TIMEOUT_SECONDS) as r: - return json.loads(r.read().decode()) - - -def log_warning(message): - print(f"[aw-worktime-api] {message}", file=sys.stderr, flush=True) - - -def pts(s): - return datetime.fromisoformat(s.replace("Z", "+00:00")).astimezone(timezone.utc) - - -def to_iso_utc(dt): - return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") - - -def parse_iso_utc(value): - if not value: - return None - return pts(value) - - -def hhmm(total_seconds): - total_seconds = max(0, int(total_seconds)) - return "%02d:%02d" % (total_seconds // 3600, (total_seconds % 3600) // 60) - - -def human_duration_ru(total_seconds): - total_seconds = max(0, int(total_seconds)) - hours = total_seconds // 3600 - minutes = (total_seconds % 3600) // 60 - if hours and minutes: - return f"{hours} ч {minutes} мин" - if hours: - return f"{hours} ч" - if minutes: - return f"{minutes} мин" - return f"{total_seconds} сек" - - -def now_utc(): - return datetime.now(timezone.utc) - - -def worktime_health_payload(): - return { - "ok": True, - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "default_host": DEFAULT_HOST, - "aw_api_base": AW, - } - - -def age_seconds(ts, now=None): - if ts is None: - return None - ref = now or now_utc() - return max(0, int((ref - ts).total_seconds())) - - -def write_atomic_json(path, payload): - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as handle: - json.dump(payload, handle, ensure_ascii=False, indent=2) - handle.write("\n") - tmp_name = handle.name - os.replace(tmp_name, path) - - -def safe_slug(value): - text = str(value or "").strip().lower() - slug = [] - for char in text: - if char.isalnum(): - slug.append(char) - else: - slug.append("-") - normalized = "".join(slug).strip("-") - while "--" in normalized: - normalized = normalized.replace("--", "-") - return normalized or "user" - - -def _normalize_identity_key(value): - return str(value or "").strip().lower() - - -def _default_display_name(user, user_id): - base = str(user_id or user or "").strip() - if "\\" in base: - base = base.split("\\", 1)[1] - if not base: - base = str(user or "").strip() - if base and base.isascii() and base.lower() == base and any(ch.isalpha() for ch in base): - return base.upper() - return base or str(user or "").strip() - - -def _load_manager_directory(): - try: - stat = MANAGER_ALIASES_JSON.stat() - except FileNotFoundError: - _ALIASES_CACHE["mtime"] = None - _ALIASES_CACHE["users"] = {} - _ALIASES_CACHE["owners"] = {} - _ALIASES_CACHE["raw"] = {} - return {"users": {}, "owners": {}, "raw": {}} - mtime = stat.st_mtime - if _ALIASES_CACHE["mtime"] == mtime: - return { - "users": _ALIASES_CACHE["users"], - "owners": _ALIASES_CACHE["owners"], - "raw": _ALIASES_CACHE["raw"], - } - try: - raw = json.loads(MANAGER_ALIASES_JSON.read_text(encoding="utf-8")) - except Exception as exc: - log_warning(f"failed to load manager aliases from {MANAGER_ALIASES_JSON}: {exc}") - raw = {} - users_payload = {} - owners_payload = {} - if isinstance(raw, dict): - users = raw.get("users", raw) - if isinstance(users, dict): - for key, value in users.items(): - norm_key = _normalize_identity_key(key) - if not norm_key: - continue - if isinstance(value, str): - users_payload[norm_key] = {"display_name": value} - elif isinstance(value, dict): - users_payload[norm_key] = dict(value) - owners = raw.get("owners", {}) - if isinstance(owners, dict): - for key, value in owners.items(): - norm_key = _normalize_identity_key(key) - if not norm_key: - continue - if isinstance(value, str): - owners_payload[norm_key] = {"display_name": value} - elif isinstance(value, dict): - owners_payload[norm_key] = dict(value) - _ALIASES_CACHE["mtime"] = mtime - _ALIASES_CACHE["users"] = users_payload - _ALIASES_CACHE["owners"] = owners_payload - _ALIASES_CACHE["raw"] = raw if isinstance(raw, dict) else {} - return { - "users": users_payload, - "owners": owners_payload, - "raw": _ALIASES_CACHE["raw"], - } - - -def load_manager_aliases(): - return _load_manager_directory()["users"] - - -def load_manager_owners(): - return _load_manager_directory()["owners"] - - -def resolve_user_alias(user, user_id, host): - aliases = load_manager_aliases() - candidates = [ - _normalize_identity_key(user_id), - _normalize_identity_key(f"{resolve_host(host)}\\{user}"), - _normalize_identity_key(user), - ] - alias = {} - for candidate in candidates: - if candidate and candidate in aliases: - alias = dict(aliases[candidate]) - break - display_name = str(alias.get("display_name") or alias.get("name") or _default_display_name(user, user_id)).strip() - manager_owner = str(alias.get("manager") or alias.get("owner") or display_name).strip() or display_name - department = str(alias.get("department") or "").strip() - role = str(alias.get("role") or "").strip() - notes = str(alias.get("notes") or "").strip() - canonical_user_id = str(alias.get("canonical_user_id") or user_id or "").strip() - exclude = bool(alias.get("exclude")) or _normalize_identity_key(user) in MANAGER_EXCLUDE_USERS or _normalize_identity_key(display_name) in MANAGER_EXCLUDE_USERS - return { - "display_name": display_name, - "manager_owner": manager_owner, - "department": department, - "role": role, - "notes": notes, - "canonical_user_id": canonical_user_id, - "exclude": exclude, - } - - -def resolve_owner_profile(owner_name): - owner_name = normalize_management_filter(owner_name) - if not owner_name: - owner_name = "unassigned" - owners = load_manager_owners() - profile = dict(owners.get(_normalize_identity_key(owner_name), {})) - display_name = str(profile.get("display_name") or profile.get("name") or owner_name).strip() or owner_name - title = str(profile.get("title") or profile.get("role") or "").strip() - department = str(profile.get("department") or "").strip() - escalation_to = str(profile.get("escalation_to") or profile.get("escalate_to") or "").strip() - contact = str(profile.get("contact") or profile.get("telegram") or profile.get("email") or "").strip() - notes = str(profile.get("notes") or "").strip() - return { - "owner_name": owner_name, - "display_name": display_name, - "title": title, - "department": department, - "escalation_to": escalation_to, - "contact": contact, - "notes": notes, - } - - -def clamp_seconds(value, fallback=DEFAULT_SAMPLE_SECONDS): - try: - seconds = float(value) - except Exception: - seconds = float(fallback) - if seconds <= 0: - seconds = float(fallback) - return min(seconds, MAX_SAMPLE_SECONDS) - - -def resolve_host(request_host=None): - host = (request_host or DEFAULT_HOST).strip() - if not host: - host = DEFAULT_HOST - return host - - -def get_sessions_bucket_id(host): - return f"aw-worktime-sessions_{resolve_host(host)}" - - -def get_management_build_lock(host, report_date): - key = (resolve_host(host), report_date.isoformat()) - with _MANAGEMENT_BUILD_LOCKS_LOCK: - lock = _MANAGEMENT_BUILD_LOCKS.get(key) - if lock is None: - lock = threading.Lock() - _MANAGEMENT_BUILD_LOCKS[key] = lock - return lock - - -def resolve_report_date(day=None, date_text=None): - now_local = datetime.now(REPORT_TZ) - if date_text: - return datetime.strptime(date_text, "%Y-%m-%d").date() - if day == "yesterday": - return (now_local - timedelta(days=1)).date() - return now_local.date() - - -def get_report_bounds(report_date): - start_local = datetime(report_date.year, report_date.month, report_date.day, tzinfo=REPORT_TZ) - end_local = start_local + timedelta(days=1) - timedelta(seconds=1) - start = start_local.astimezone(timezone.utc) - end = end_local.astimezone(timezone.utc) - end_exclusive = end + timedelta(seconds=1) - return { - "start_local": start_local, - "end_local": end_local, - "start": start, - "end": end, - "end_exclusive": end_exclusive, - } - - -def get_workday_bounds(report_date): - start_local = datetime(report_date.year, report_date.month, report_date.day, WORKDAY_START_HOUR, 0, 0, tzinfo=REPORT_TZ) - end_local = datetime(report_date.year, report_date.month, report_date.day, WORKDAY_END_HOUR, 0, 0, tzinfo=REPORT_TZ) - if end_local <= start_local: - end_local = start_local + timedelta(hours=8) - return { - "start_local": start_local, - "end_local": end_local, - "duration_seconds": int((end_local - start_local).total_seconds()), - } - - -def _is_machine_user(user: str): - u = (user or "").strip().lower() - return u.endswith("$") or u in {"system", "localservice", "networkservice"} - - -def _is_active_sample(data: dict): - state = str(data.get("state") or "").strip().lower() - if isinstance(data.get("active"), bool) and data.get("active"): - return True - if ("актив" in state) or (state == "active"): - return True - if state == "unknown": - try: - sid = int(data.get("sessionId")) - except Exception: - sid = -1 - user = str(data.get("username") or "").strip() - session_name = str(data.get("sessionName") or "").strip().lower() - if sid > 0 and user and (not _is_machine_user(user)) and (session_name.startswith("rdp-") or session_name == "console"): - return True - return False - - -def _normalize_user_id(data, host, username): - user_id = str(data.get("userId") or "").strip() - if user_id: - left, sep, right = user_id.partition("\\") - if sep and right: - return f"{resolve_host(host)}\\{right}" - return user_id - return f"{resolve_host(host)}\\{username}" - - -def _event_sample_seconds(event, next_same_session_ts=None): - data = event.get("data") or {} - for key in ("sampleSeconds", "pollSeconds"): - value = data.get(key) - try: - if float(value) > 0: - return clamp_seconds(value) - except Exception: - pass - try: - duration = float(event.get("duration") or 0.0) - except Exception: - duration = 0.0 - if duration > 0: - return clamp_seconds(duration) - if next_same_session_ts is not None: - delta = (next_same_session_ts - event["_ts"]).total_seconds() - if delta > 0: - return clamp_seconds(delta) - return clamp_seconds(DEFAULT_SAMPLE_SECONDS) - - -def _merge_intervals(intervals): - if not intervals: - return [] - ordered = sorted(intervals, key=lambda item: item[0]) - merged = [ordered[0]] - for start, end in ordered[1:]: - last_start, last_end = merged[-1] - if start <= last_end: - if end > last_end: - merged[-1] = (last_start, end) - continue - merged.append((start, end)) - return merged - - -def _overlap_interval(left, right): - start = max(left[0], right[0]) - end = min(left[1], right[1]) - if end <= start: - return None - return start, end - - -def _interval_contains(interval, ts): - return interval[0] <= ts < interval[1] - - -def _event_timestamp(event): - try: - return pts(event.get("timestamp")) - except Exception: - return None - - -def _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS): - try: - duration = float(event.get("duration") or 0.0) - except Exception: - duration = 0.0 - if duration <= 0: - duration = default_seconds - return max(1.0, min(float(TRUE_ACTIVE_MAX_EVENT_SECONDS), duration)) - - -def _normalize_app_name(app, title=""): - app_raw = str(app or "").strip() - app_l = app_raw.lower() - title_raw = str(title or "").strip() - if app_l.startswith(("1cv8", "1cestart")): - return "1С" - if app_l in {"chrome.exe", "google chrome"} or "google chrome" in title_raw.lower(): - return "Chrome" - if app_l in {"msedge.exe", "microsoft edge"} or "microsoft edge" in title_raw.lower(): - return "Edge" - if app_l in {"browser.exe", "browser"} or "яндекс" in title_raw.lower(): - return "Яндекс Браузер" - if app_l in {"excel.exe"}: - return "Excel" - if app_l in {"winword.exe"}: - return "Word" - if app_l in {"powerpnt.exe"}: - return "PowerPoint" - if app_l in {"outlook.exe"}: - return "Outlook" - if app_l in {"explorer.exe"}: - return "Проводник" - if app_l in {"totalcmd.exe", "totalcmd64.exe"}: - return "Total Commander" - if app_l in {"cmd.exe"}: - return "Command Prompt" - if app_l in {"powershell.exe", "pwsh.exe"}: - return "PowerShell" - if app_l in {"acrord32.exe", "acrobat.exe"}: - return "Adobe Acrobat Reader" - if app_l in {"windowsterminal.exe", "windowsterminal"}: - return "Windows Terminal" - if app_raw: - return app_raw[:-4] if app_l.endswith(".exe") else app_raw - if title_raw: - return title_raw - return "Неизвестное приложение" - - -def _event_context(event): - data = event.get("data") or {} - for key in ("title", "url", "path", "filePath", "targetPath", "windowTitle", "foregroundTitle", "signalType"): - value = str(data.get(key) or "").strip() - if value: - return value - return "активность" - - -def _is_not_afk_event(event): - data = event.get("data") or {} - status = str(data.get("status") or data.get("state") or "").strip().lower() - return status in {"not-afk", "not_afk", "active", "активно"} - - -def _is_real_evidence_event(event): - data = event.get("data") or {} - signal_type = str(data.get("signalType") or data.get("type") or "").strip().lower() - if signal_type in {"collector_health", "self_test", "heartbeat", "health"}: - return False - if data.get("url") or data.get("title") or data.get("path") or data.get("filePath") or data.get("targetPath"): - return True - if signal_type: - return True - return False - - -def _events_for_bounds(events, start, end): - result = [] - for event in events: - ts = _event_timestamp(event) - if ts is None or ts < start or ts > end: - continue - result.append((ts, event)) - result.sort(key=lambda item: item[0]) - return result - - -def _build_window_intervals(window_events, start, end): - intervals = [] - previous_key = None - for ts, event in _events_for_bounds(window_events, start, end): - data = event.get("data") or {} - app = str(data.get("app") or data.get("process") or data.get("processName") or "").strip() - title = str(data.get("title") or data.get("windowTitle") or "").strip() - if not app and not title: - continue - duration = _event_duration_seconds(event, default_seconds=DEFAULT_SAMPLE_SECONDS) - interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) - if interval[1] <= interval[0]: - continue - app_name = _normalize_app_name(app, title) - current_key = (app_name, title) - title_changed = previous_key is not None and current_key != previous_key - previous_key = current_key - intervals.append( - { - "app": app_name, - "raw_app": app, - "title": title, - "start": interval[0], - "end": interval[1], - "title_changed": title_changed, - "timestamp": ts, - } - ) - return intervals - - -def _build_not_afk_intervals(afk_events, start, end): - intervals = [] - for ts, event in _events_for_bounds(afk_events, start, end): - if not _is_not_afk_event(event): - continue - duration = _event_duration_seconds(event, default_seconds=5) - interval = (max(ts, start), min(ts + timedelta(seconds=duration), end + timedelta(seconds=1))) - if interval[1] > interval[0]: - intervals.append(interval) - return _merge_intervals(intervals) - - -def _find_window_at(window_intervals, ts): - for item in window_intervals: - if item["start"] <= ts < item["end"]: - return item - return None - - -def _add_app_evidence(evidence_by_app, app, ts, context): - evidence_by_app.setdefault(app, []).append((ts, str(context or "").strip() or "активность")) - - -def build_true_active_apps_from_events(window_events, afk_events, evidence_events_by_bucket, start, end): - window_intervals = _build_window_intervals(window_events, start, end) - not_afk_intervals = _build_not_afk_intervals(afk_events, start, end) - evidence_by_app = {} - - for item in window_intervals: - if item["title_changed"]: - _add_app_evidence(evidence_by_app, item["app"], item["timestamp"], item["title"] or item["raw_app"]) - - for events in evidence_events_by_bucket.values(): - for ts, event in _events_for_bounds(events, start, end): - if not _is_real_evidence_event(event): - continue - window = _find_window_at(window_intervals, ts) - if window is None: - continue - _add_app_evidence(evidence_by_app, window["app"], ts, _event_context(event)) - - rows = [] - evidence_delta = timedelta(seconds=TRUE_ACTIVE_EVIDENCE_WINDOW_SECONDS) - for app in sorted({item["app"] for item in window_intervals} | set(evidence_by_app)): - app_evidence = sorted(evidence_by_app.get(app, []), key=lambda item: item[0]) - if not app_evidence: - continue - evidence_windows = _merge_intervals([(ts - evidence_delta, ts + evidence_delta) for ts, _context in app_evidence]) - proved_intervals = [] - for window in [item for item in window_intervals if item["app"] == app]: - base = (window["start"], window["end"]) - for afk_interval in not_afk_intervals: - active_overlap = _overlap_interval(base, afk_interval) - if active_overlap is None: - continue - for evidence_interval in evidence_windows: - proved = _overlap_interval(active_overlap, evidence_interval) - if proved is not None: - proved_intervals.append(proved) - proved_intervals = _merge_intervals(proved_intervals) - proved_seconds = int(sum((right - left).total_seconds() for left, right in proved_intervals)) - if proved_seconds <= 0: - continue - last_ts, last_context = app_evidence[-1] - rows.append( - { - "application": app, - "proved_work_seconds": proved_seconds, - "proved_work_hhmm": hhmm(proved_seconds), - "proved_work_human": human_duration_ru(proved_seconds), - "last_action_utc": to_iso_utc(last_ts), - "last_action_local": last_ts.astimezone(REPORT_TZ).strftime("%H:%M"), - "last_action": last_context, - "evidence_events": len(app_evidence), - } - ) - rows.sort(key=lambda item: (-item["proved_work_seconds"], item["application"].lower())) - return rows - - -def build_true_active_apps(host, report_date): - bounds = get_report_bounds(report_date) - host = resolve_host(host) - window_events = fetch_bucket_events(f"aw-watcher-window_{host}", host) or fetch_bucket_events(f"aw-rdp-window_{host}", host) - afk_events = fetch_bucket_events(f"aw-watcher-afk_{host}", host) or fetch_bucket_events(f"aw-rdp-afk_{host}", host) - evidence_events_by_bucket = {} - for bucket_id in ( - f"aw-file-operations_{host}", - f"aw-dlp-endpoint-signals_{host}", - f"aw-watcher-web-chrome_{host}", - f"aw-watcher-web-edge_{host}", - f"aw-detmir-web-category_{host}", - ): - evidence_events_by_bucket[bucket_id] = fetch_bucket_events(bucket_id, host) - return build_true_active_apps_from_events( - window_events, - afk_events, - evidence_events_by_bucket, - bounds["start"], - bounds["end"], - ) - - -def _collect_user_rows(events, start, end, host): - end_exclusive = end + timedelta(seconds=1) - by_user = {} - by_identity = {} - - for event in events: - ts = pts(event.get("timestamp")) - if ts < start or ts > end: - continue - data = event.get("data") or {} - username = str(data.get("username") or "").strip() - if not username: - continue - session_id = str(data.get("sessionId") or "").strip() or "unknown" - event_copy = { - "_ts": ts, - "data": data, - "duration": event.get("duration"), - } - by_identity.setdefault((username, session_id), []).append(event_copy) - - for (username, session_id), samples in by_identity.items(): - ordered = sorted(samples, key=lambda item: item["_ts"]) - for idx, sample in enumerate(ordered): - data = sample["data"] - active = _is_active_sample(data) - next_ts = ordered[idx + 1]["_ts"] if idx + 1 < len(ordered) else None - sample_seconds = _event_sample_seconds(sample, next_ts) - row = by_user.setdefault( - username, - { - "user": username, - "user_id": _normalize_user_id(data, host, username), - "samples_count": 0, - "active_samples": 0, - "session_ids": set(), - "intervals": [], - }, - ) - row["samples_count"] += 1 - row["session_ids"].add(session_id) - if active: - row["active_samples"] += 1 - interval_start = sample["_ts"] - interval_end = min(sample["_ts"] + timedelta(seconds=sample_seconds), end_exclusive) - if interval_end > interval_start: - row["intervals"].append((interval_start, interval_end)) - return by_user - - -def aggregate_rows(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - return _build_rows_from_user_map(by_user, start, end, include_intervals=False) - - -def aggregate_rows_with_intervals(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - return _build_rows_from_user_map(by_user, start, end, include_intervals=True) - - -def _build_rows_from_user_map(by_user, start, end, include_intervals): - rows = [] - full_range = int((end - start).total_seconds()) + 1 - for username in sorted(by_user): - row = by_user[username] - merged = _merge_intervals(row["intervals"]) - active_seconds = int(sum((end_dt - start_dt).total_seconds() for start_dt, end_dt in merged)) - active_seconds = min(active_seconds, full_range) - first_activity = to_iso_utc(merged[0][0]) if merged else "" - last_activity = to_iso_utc(merged[-1][1]) if merged else "" - rows.append( - { - "user": row["user"], - "user_id": row["user_id"], - "active_seconds": active_seconds, - "active_hhmm": hhmm(active_seconds), - "first_activity": first_activity, - "last_activity": last_activity, - "idle_seconds": max(0, full_range - active_seconds), - "sessions_count": len(row["session_ids"]), - "samples_count": row["samples_count"], - "active_samples": row["active_samples"], - } - ) - if include_intervals: - rows[-1]["_intervals"] = merged - return rows - - -def aggregate_hourly_rows(events, start, end, host): - by_user = _collect_user_rows(events, start, end, host) - rows = [] - for username in sorted(by_user): - row = by_user[username] - merged = _merge_intervals(row["intervals"]) - per_bucket = {} - for interval_start, interval_end in merged: - cursor = interval_start - while cursor < interval_end: - bucket_local = cursor.astimezone(REPORT_TZ).replace(minute=0, second=0, microsecond=0) - bucket_start = bucket_local.astimezone(timezone.utc) - bucket_end = (bucket_local + timedelta(hours=1)).astimezone(timezone.utc) - overlap_start = max(interval_start, bucket_start) - overlap_end = min(interval_end, bucket_end) - if overlap_end > overlap_start: - key = bucket_start - per_bucket[key] = per_bucket.get(key, 0) + int((overlap_end - overlap_start).total_seconds()) - cursor = bucket_end - - for bucket_start in sorted(per_bucket): - active_seconds = per_bucket[bucket_start] - if active_seconds <= 0: - continue - bucket_local = bucket_start.astimezone(REPORT_TZ) - rows.append( - { - "user": row["user"], - "user_id": row["user_id"], - "bucket_start_utc": to_iso_utc(bucket_start), - "bucket_start_local": bucket_local.isoformat(), - "report_date": bucket_local.date().isoformat(), - "hour_local": bucket_local.strftime("%H:00"), - "active_seconds": active_seconds, - "active_hhmm": hhmm(active_seconds), - } - ) - return rows - - -def fetch_events_for_date(host, report_date): - bounds = get_report_bounds(report_date) - bucket_id = get_sessions_bucket_id(host) - events = fetch_bucket_events(bucket_id, host, start=bounds["start"], end=bounds["end"]) - return bounds, events - - -def fetch_bucket_events(bucket_id, host, start=None, end=None): - now = now_utc() - cache_key = bucket_id - params = {"limit": WORKTIME_EVENTS_LIMIT} - if start is not None and end is not None: - params["start"] = to_iso_utc(start) - params["end"] = to_iso_utc(end) - cache_key = f"{bucket_id}|{params['start']}|{params['end']}" - if EVENTS_CACHE_TTL_SECONDS > 0: - with _EVENTS_CACHE_LOCK: - cached = _EVENTS_CACHE.get(cache_key) - if cached is not None and age_seconds(cached["stored_at"], now=now) <= EVENTS_CACHE_TTL_SECONDS: - return cached["events"] - try: - get(f"{AW}/buckets/{bucket_id}") - except Exception: - log_warning(f"bucket lookup failed for host={host} bucket={bucket_id} aw_base={AW}") - return [] - try: - events = get(f"{AW}/buckets/{bucket_id}/events?{urlencode(params)}") - except Exception: - log_warning(f"events fetch failed for host={host} bucket={bucket_id} aw_base={AW}") - return [] - if EVENTS_CACHE_TTL_SECONDS > 0: - with _EVENTS_CACHE_LOCK: - _EVENTS_CACHE[cache_key] = {"stored_at": now, "events": events} - return events - - -def build_report_summary(rows): - if not rows: - return { - "users_count": 0, - "total_active_seconds": 0, - "total_active_hhmm": "00:00", - "first_activity": "", - "last_activity": "", - "top_user": "", - "top_user_active_hhmm": "00:00", - } - - total_active_seconds = sum(int(row.get("active_seconds", 0) or 0) for row in rows) - first_values = [row.get("first_activity") for row in rows if row.get("first_activity")] - last_values = [row.get("last_activity") for row in rows if row.get("last_activity")] - top_row = max(rows, key=lambda row: int(row.get("active_seconds", 0) or 0)) - return { - "users_count": len(rows), - "total_active_seconds": total_active_seconds, - "total_active_hhmm": hhmm(total_active_seconds), - "first_activity": min(first_values) if first_values else "", - "last_activity": max(last_values) if last_values else "", - "top_user": top_row.get("user", ""), - "top_user_active_hhmm": top_row.get("active_hhmm", "00:00"), - } - - -def latest_bucket_event(bucket_id): - try: - events = get(f"{AW}/buckets/{bucket_id}/events?limit=1") - except Exception: - events = None - if isinstance(events, list) and events: - valid = [item for item in events if isinstance(item, dict)] - if valid: - valid.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - return valid[0] - try: - bucket = get(f"{AW}/buckets/{bucket_id}") - except Exception: - return None - end = ((bucket or {}).get("metadata") or {}).get("end") - if end: - return {"timestamp": end, "duration": 0, "data": {"source": "bucket_metadata"}} - return None - - -def _priority_rank(priority): - return {"critical": 0, "high": 1, "medium": 2, "low": 3}.get(priority, 9) - - -def _clamp_pct(value): - return round(min(100.0, max(0.0, float(value))), 2) - - -def _action(action_id, priority, owner, deadline_hint, reason, recommended_action, *, user_id="", evidence=None): - return { - "action_id": action_id, - "priority": priority, - "owner": owner, - "user_id": user_id, - "deadline_hint": deadline_hint, - "reason": reason, - "recommended_action": recommended_action, - "evidence": evidence or {}, - } - - -def build_executive_summary(summary, actions, sources): - critical = [action for action in actions if action.get("priority") == "critical"] - high = [action for action in actions if action.get("priority") == "high"] - stale_sources = [source for source in sources if source.get("status") != "ok"] - if critical: - portfolio_state = "critical" - headline = f"Есть {len(critical)} критичных вопроса, требующих решения сегодня." - elif high: - portfolio_state = "attention" - headline = f"Критичных провалов нет, но есть {len(high)} вопроса повышенного внимания." - elif summary.get("portfolio_coverage_pct", 0.0) < MANAGER_TARGET_COVERAGE_PCT: - portfolio_state = "attention" - headline = "Покрытие ниже целевого порога, но явных критичных кейсов не найдено." - else: - portfolio_state = "stable" - headline = "Критичных отклонений не найдено, рабочий день идёт в пределах нормы." - - message_parts = [ - f"Активны {summary.get('active_users', 0)} из {summary.get('users_count', 0)} сотрудников.", - f"Покрытие рабочего окна {summary.get('portfolio_coverage_pct', 0.0)}%.", - ] - if stale_sources: - message_parts.append(f"Есть {len(stale_sources)} проблем(ы) со свежестью источников.") - message = " ".join(message_parts) - - focus_items = [] - for action in actions[:5]: - focus_items.append( - { - "priority": action["priority"], - "owner": action["owner"], - "title": action["action_id"], - "reason": action["reason"], - "recommended_action": action["recommended_action"], - } - ) - stale_items = [] - for source in stale_sources[:3]: - stale_items.append( - { - "source_id": source["source_id"], - "label": source["label"], - "status": source["status_label"], - "summary": source["summary"], - } - ) - return { - "portfolio_state": portfolio_state, - "headline": headline, - "message": message, - "focus_items": focus_items, - "stale_sources": stale_items, - } - - -def _empty_rollup(name): - return { - "name": name or "unassigned", - "users_count": 0, - "active_users": 0, - "inactive_users": 0, - "below_target_users": 0, - "workday_total_active_seconds": 0, - "workday_total_active_hhmm": "00:00", - "portfolio_coverage_pct": 0.0, - "actions_count": 0, - "critical_actions_count": 0, - "high_actions_count": 0, - "medium_actions_count": 0, - "low_actions_count": 0, - "users": [], - } - - -def normalize_management_filter(value): - if value is None: - return "" - return " ".join(str(value).split()).strip() - - -def _filter_key(value): - return normalize_management_filter(value).casefold() - - -def _matches_management_filters(alias, owner_filter="", department_filter=""): - if owner_filter and _filter_key(alias.get("manager_owner")) != _filter_key(owner_filter): - return False - if department_filter and _filter_key(alias.get("department")) != _filter_key(department_filter): - return False - return True - - -def _action_department(action): - evidence = action.get("evidence") or {} - department = normalize_management_filter(evidence.get("department")) - if department: - return department - if normalize_management_filter(action.get("owner")) == "ops": - return "Инфраструктура" - return "" - - -def filter_management_actions(actions, rows, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - if not owner_filter and not department_filter: - return list(actions) - row_user_ids = { - normalize_management_filter(row.get("canonical_user_id") or row.get("user_id")) - for row in rows - if normalize_management_filter(row.get("canonical_user_id") or row.get("user_id")) - } - filtered = [] - for action in actions: - if owner_filter and _filter_key(action.get("owner")) != _filter_key(owner_filter): - continue - if department_filter: - action_department = _action_department(action) - if action_department: - if _filter_key(action_department) != _filter_key(department_filter): - continue - else: - user_id = normalize_management_filter(action.get("user_id")) - if not user_id or user_id not in row_user_ids: - continue - filtered.append(action) - return filtered - - -def _row_matches_payload_filters(row, owner_filter="", department_filter=""): - if owner_filter and _filter_key(row.get("manager_owner")) != _filter_key(owner_filter): - return False - if department_filter and _filter_key(row.get("department")) != _filter_key(department_filter): - return False - return True - - -def summarize_management_rows(rows, actions, workday): - expected_seconds_per_user = int(workday.get("expected_seconds_per_user", 0) or 0) - workday_total_active_seconds = sum(int(row.get("workday_active_seconds", 0) or 0) for row in rows) - calendar_total_active_seconds = sum(int(row.get("calendar_active_seconds", 0) or 0) for row in rows) - active_users = sum(1 for row in rows if row.get("status") != "inactive") - inactive_users = sum(1 for row in rows if row.get("status") == "inactive") - below_target_users = sum(1 for row in rows if row.get("status") == "below_target") - on_target_users = sum(1 for row in rows if row.get("status") == "ok") - workday_first_values = sorted(str(row.get("workday_first_activity_local") or "") for row in rows if row.get("workday_first_activity_local")) - workday_last_values = sorted(str(row.get("workday_last_activity_local") or "") for row in rows if row.get("workday_last_activity_local")) - calendar_first_values = sorted(str(row.get("first_activity_local") or "") for row in rows if row.get("first_activity_local")) - calendar_last_values = sorted(str(row.get("last_activity_local") or "") for row in rows if row.get("last_activity_local")) - top_row = max(rows, key=lambda row: int(row.get("workday_active_seconds", 0) or 0), default=None) - portfolio_coverage_pct = _clamp_pct((workday_total_active_seconds / (expected_seconds_per_user * len(rows))) * 100.0) if rows and expected_seconds_per_user > 0 else 0.0 - return { - "users_count": len(rows), - "active_users": active_users, - "inactive_users": inactive_users, - "on_target_users": on_target_users, - "below_target_users": below_target_users, - "portfolio_coverage_pct": portfolio_coverage_pct, - "actions_count": len(actions), - "critical_actions_count": sum(1 for action in actions if action.get("priority") == "critical"), - "high_actions_count": sum(1 for action in actions if action.get("priority") == "high"), - "calendar_total_active_seconds": calendar_total_active_seconds, - "calendar_total_active_hhmm": hhmm(calendar_total_active_seconds), - "calendar_first_activity": calendar_first_values[0] if calendar_first_values else "", - "calendar_last_activity": calendar_last_values[-1] if calendar_last_values else "", - "workday_total_active_seconds": workday_total_active_seconds, - "workday_total_active_hhmm": hhmm(workday_total_active_seconds), - "workday_first_activity": workday_first_values[0] if workday_first_values else "", - "workday_last_activity": workday_last_values[-1] if workday_last_values else "", - "total_active_seconds": workday_total_active_seconds, - "total_active_hhmm": hhmm(workday_total_active_seconds), - "first_activity": workday_first_values[0] if workday_first_values else "", - "last_activity": workday_last_values[-1] if workday_last_values else "", - "top_user": str((top_row or {}).get("user") or ""), - "top_user_active_hhmm": hhmm(int((top_row or {}).get("workday_active_seconds", 0) or 0)), - } - - -def apply_management_filters_to_payload(payload, owner_filter="", department_filter="", include_sources=True, include_source_actions=True): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - filtered_rows = [ - dict(row) - for row in payload.get("rows", []) - if _row_matches_payload_filters(row, owner_filter=owner_filter, department_filter=department_filter) - ] - base_actions = list(payload.get("actions", [])) - if not include_source_actions: - base_actions = [action for action in base_actions if action.get("action_id") != "source_freshness_review"] - filtered_actions = filter_management_actions(base_actions, filtered_rows, owner_filter=owner_filter, department_filter=department_filter) - filtered_payload = dict(payload) - filtered_payload["filters"] = { - "owner": owner_filter, - "department": department_filter, - } - filtered_payload["rows"] = filtered_rows - filtered_payload["actions"] = filtered_actions - filtered_payload["summary"] = summarize_management_rows(filtered_rows, filtered_actions, payload.get("workday") or {}) - filtered_payload["owner_rollups"] = build_owner_rollups(filtered_rows, filtered_actions) - filtered_payload["department_rollups"] = build_department_rollups(filtered_rows, filtered_actions) - filtered_payload["owner_roster"] = build_owner_roster(filtered_rows, filtered_actions) - if include_sources: - filtered_payload["sources"] = list(payload.get("sources", [])) - else: - filtered_payload["sources"] = [] - filtered_payload["executive"] = build_executive_summary(filtered_payload["summary"], filtered_actions, filtered_payload["sources"]) - return filtered_payload - - -def build_owner_rollups(rows, actions): - groups = {} - for row in rows: - owner = str(row.get("manager_owner") or row.get("user") or "unassigned").strip() or "unassigned" - group = groups.setdefault(owner, _empty_rollup(owner)) - group["users_count"] += 1 - if row.get("status") == "inactive": - group["inactive_users"] += 1 - else: - group["active_users"] += 1 - if row.get("status") == "below_target": - group["below_target_users"] += 1 - group["workday_total_active_seconds"] += int(row.get("workday_active_seconds", 0) or 0) - group["users"].append(row.get("user", "unknown")) - - for action in actions: - owner = str(action.get("owner") or "unassigned").strip() or "unassigned" - group = groups.setdefault(owner, _empty_rollup(owner)) - group["actions_count"] += 1 - prio = str(action.get("priority") or "").lower() - if prio == "critical": - group["critical_actions_count"] += 1 - elif prio == "high": - group["high_actions_count"] += 1 - elif prio == "medium": - group["medium_actions_count"] += 1 - elif prio == "low": - group["low_actions_count"] += 1 - - for group in groups.values(): - expected = group["users_count"] * max(1, WORKDAY_END_HOUR - WORKDAY_START_HOUR) * 3600 - group["workday_total_active_hhmm"] = hhmm(group["workday_total_active_seconds"]) - group["portfolio_coverage_pct"] = _clamp_pct((group["workday_total_active_seconds"] / expected) * 100.0) if expected > 0 else 0.0 - group["users"] = sorted(set(str(user or "").strip() for user in group["users"] if str(user or "").strip())) - - return sorted( - groups.values(), - key=lambda item: ( - -item["critical_actions_count"], - -item["high_actions_count"], - -item["inactive_users"], - item["name"].lower(), - ), - ) - - -def build_department_rollups(rows, actions): - groups = {} - for row in rows: - department = str(row.get("department") or "Без подразделения").strip() or "Без подразделения" - group = groups.setdefault(department, _empty_rollup(department)) - group["users_count"] += 1 - if row.get("status") == "inactive": - group["inactive_users"] += 1 - else: - group["active_users"] += 1 - if row.get("status") == "below_target": - group["below_target_users"] += 1 - group["workday_total_active_seconds"] += int(row.get("workday_active_seconds", 0) or 0) - group["users"].append(row.get("user", "unknown")) - - for action in actions: - evidence = action.get("evidence") or {} - department = str(evidence.get("department") or ("Инфраструктура" if str(action.get("owner") or "").strip().lower() == "ops" else "Без подразделения")).strip() - group = groups.setdefault(department, _empty_rollup(department)) - group["actions_count"] += 1 - prio = str(action.get("priority") or "").lower() - if prio == "critical": - group["critical_actions_count"] += 1 - elif prio == "high": - group["high_actions_count"] += 1 - elif prio == "medium": - group["medium_actions_count"] += 1 - elif prio == "low": - group["low_actions_count"] += 1 - - for group in groups.values(): - expected = group["users_count"] * max(1, WORKDAY_END_HOUR - WORKDAY_START_HOUR) * 3600 - group["workday_total_active_hhmm"] = hhmm(group["workday_total_active_seconds"]) - group["portfolio_coverage_pct"] = _clamp_pct((group["workday_total_active_seconds"] / expected) * 100.0) if expected > 0 else 0.0 - group["users"] = sorted(set(str(user or "").strip() for user in group["users"] if str(user or "").strip())) - - return sorted( - groups.values(), - key=lambda item: ( - -item["critical_actions_count"], - -item["high_actions_count"], - -item["inactive_users"], - item["name"].lower(), - ), - ) - - -def build_owner_roster(rows, actions): - owner_rollups = build_owner_rollups(rows, actions) - roster = [] - for item in owner_rollups: - profile = resolve_owner_profile(item["name"]) - roster.append( - { - **item, - "display_name": profile["display_name"], - "title": profile["title"], - "department": profile["department"], - "contact": profile["contact"], - "escalation_to": profile["escalation_to"], - "notes": profile["notes"], - } - ) - return roster - - -def _interval_overlap_seconds(intervals, start, end): - total = 0 - first = None - last = None - for interval_start, interval_end in intervals or []: - overlap_start = max(interval_start, start) - overlap_end = min(interval_end, end) - if overlap_end <= overlap_start: - continue - seconds = int((overlap_end - overlap_start).total_seconds()) - if seconds <= 0: - continue - total += seconds - if first is None or overlap_start < first: - first = overlap_start - if last is None or overlap_end > last: - last = overlap_end - return total, first, last - - -def _source_status_label(status): - return { - "ok": "fresh", - "warn": "stale", - "fail": "missing", - "inactive": "inactive", - }.get(status, status) - - -def _source_summary(event): - data = (event or {}).get("data") or {} - signal_type = str(data.get("signalType") or "").strip() - if signal_type == "collector_health": - return ( - f"queue={data.get('queueDepth', 0)} " - f"failures={data.get('sendFailures', 0)} " - f"flushed={data.get('eventsFlushed', 0)}" - ) - if data.get("domain"): - return f"{data.get('domain')} ({data.get('category', 'uncategorized')})" - if data.get("eventType"): - return f"{data.get('eventType')} {data.get('username', '')}".strip() - if data.get("action"): - return f"{data.get('action')} {data.get('result', '')}".strip() - if data.get("title"): - return str(data.get("title"))[:120] - if data.get("status"): - return str(data.get("status")) - if data.get("app"): - return str(data.get("app")) - return "" - - -def management_cache_path(host, report_date): - host_slug = safe_slug(host) - return MANAGER_CACHE_DIR / f"{host_slug}-{report_date.isoformat()}.json" - - -def load_management_cache(host, report_date): - if MANAGER_CACHE_TTL_SECONDS <= 0: - return None - path = management_cache_path(host, report_date) - if not path.exists(): - return None - if report_date >= datetime.now(REPORT_TZ).date(): - age = age_seconds(datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)) - if age is None or age > MANAGER_CACHE_TTL_SECONDS: - return None - try: - return json.loads(path.read_text(encoding="utf-8")) - except Exception: - return None - - -def save_management_cache(host, report_date, payload): - if MANAGER_CACHE_TTL_SECONDS <= 0: - return - write_atomic_json(management_cache_path(host, report_date), payload) - - -def make_report_cache_key(path, fmt, host, report_date, selected_day="", owner_filter="", department_filter=""): - return "|".join( - [ - str(path), - str(fmt), - resolve_host(host), - report_date.isoformat(), - str(selected_day or ""), - normalize_management_filter(owner_filter), - normalize_management_filter(department_filter), - ] - ) - - -def resolve_report_format(params, accept_header=""): - requested = str(params.get("format", [""])[0] or "").lower() - if requested in {"csv", "html", "json"}: - return requested - accept = str(accept_header or "").lower() - if "text/html" in accept and "application/json" not in accept: - return "html" - return "json" - - -def report_disk_cache_path(key): - digest = hashlib.sha256(key.encode("utf-8")).hexdigest() - return REPORT_DISK_CACHE_DIR / f"{digest}.json" - - -def load_report_response_disk_cache(key): - if REPORT_DISK_STALE_TTL_SECONDS <= 0: - return None - path = report_disk_cache_path(key) - if not path.exists(): - return None - try: - payload = json.loads(path.read_text(encoding="utf-8")) - stored_epoch = float(payload["stored_epoch"]) - age = max(0, int(time.time() - stored_epoch)) - if age > REPORT_DISK_STALE_TTL_SECONDS: - return None - data = base64.b64decode(payload["data_b64"].encode("ascii")) - content_type = str(payload["content_type"]) - except Exception as exc: - log_warning(f"failed to load report disk cache key={key}: {exc}") - return None - return { - "stored_monotonic": time.monotonic() - age, - "generated_at_utc": payload.get("generated_at_utc", ""), - "data": data, - "content_type": content_type, - "age_seconds": age, - "stale": True, - "cache_source": "disk", - } - - -def save_report_response_disk_cache(key, data, content_type, generated_at_utc): - if REPORT_DISK_STALE_TTL_SECONDS <= 0: - return - payload = { - "key": key, - "stored_epoch": time.time(), - "generated_at_utc": generated_at_utc, - "content_type": content_type, - "data_b64": base64.b64encode(data).decode("ascii"), - } - try: - write_atomic_json(report_disk_cache_path(key), payload) - except Exception as exc: - log_warning(f"failed to save report disk cache key={key}: {exc}") - - -def get_report_response_cache(key, *, allow_stale=False): - if REPORT_CACHE_TTL_SECONDS <= 0 and not allow_stale: - return None - now = time.monotonic() - with _REPORT_CACHE_LOCK: - cached = _REPORT_RESPONSE_CACHE.get(key) - if cached is not None: - age = now - cached["stored_monotonic"] - max_age = REPORT_STALE_TTL_SECONDS if allow_stale else REPORT_CACHE_TTL_SECONDS - if max_age > 0 and age <= max_age: - result = dict(cached) - result["age_seconds"] = int(age) - result["stale"] = age > REPORT_CACHE_TTL_SECONDS - result["cache_source"] = "memory" - return result - if allow_stale: - return load_report_response_disk_cache(key) - return None - - -def save_report_response_cache(key, data, content_type): - if REPORT_STALE_TTL_SECONDS <= 0: - return - generated_at_utc = now_utc().isoformat().replace("+00:00", "Z") - with _REPORT_CACHE_LOCK: - _REPORT_RESPONSE_CACHE[key] = { - "stored_monotonic": time.monotonic(), - "generated_at_utc": generated_at_utc, - "data": data, - "content_type": content_type, - } - save_report_response_disk_cache(key, data, content_type, generated_at_utc) - - -def trigger_report_background_refresh(key, path, params): - refresh_params = {name: list(values) for name, values in params.items()} - with _REPORT_CACHE_LOCK: - if key in _REPORT_REFRESHING: - return False - _REPORT_REFRESHING.add(key) - - def refresh(): - try: - if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): - return - try: - data, content_type = build_worktime_report_response(path, refresh_params) - save_report_response_cache(key, data, content_type) - except Exception as exc: - log_warning(f"background report refresh failed path={path}: {exc}") - finally: - _REPORT_BUILD_SEMAPHORE.release() - finally: - with _REPORT_CACHE_LOCK: - _REPORT_REFRESHING.discard(key) - - thread = threading.Thread(target=refresh, name="worktime-report-refresh", daemon=True) - thread.start() - return True - - -def send_cached_report(handler, cached, *, reason): - handler.send_response(200) - handler.send_header("Content-Type", cached["content_type"]) - handler.send_header("Content-Length", str(len(cached["data"]))) - handler.send_header("X-AW-Worktime-Cache", "stale" if cached.get("stale") else "fresh") - handler.send_header("X-AW-Worktime-Cache-Reason", reason) - handler.send_header("X-AW-Worktime-Cache-Age", str(cached.get("age_seconds", 0))) - handler.send_header("X-AW-Worktime-Cache-Source", cached.get("cache_source", "memory")) - handler.end_headers() - try: - handler.wfile.write(cached["data"]) - except (BrokenPipeError, ConnectionResetError): - return False - return True - - -def send_report_unavailable(handler, message): - payload = { - "ok": False, - "error": "report_unavailable", - "message": message, - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - } - data = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(handler, data, "application/json; charset=utf-8", status=503) - - -def send_worktime_report(handler, path, params, cache_key, *, host, report_date): - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl") - return - - allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} - if allow_immediate_stale: - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - refresh_stale = str(params.get("refresh_stale", [""])[0]).lower() in {"1", "true", "yes"} - if refresh_stale: - trigger_report_background_refresh(cache_key, path, params) - send_cached_report(handler, cached, reason="stale-allowed") - return - - if not _REPORT_BUILD_SEMAPHORE.acquire(blocking=False): - if not allow_immediate_stale and REPORT_BUSY_WAIT_SECONDS > 0: - deadline = time.monotonic() + REPORT_BUSY_WAIT_SECONDS - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - time.sleep(min(REPORT_BUSY_WAIT_INTERVAL_SECONDS, remaining)) - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl-after-wait") - return - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - send_cached_report(handler, cached, reason="busy") - return - send_report_unavailable(handler, "report builder is busy and no cached response is available") - return - - try: - cached = get_report_response_cache(cache_key) - if cached is not None: - send_cached_report(handler, cached, reason="ttl-after-lock") - return - try: - data, content_type = build_worktime_report_response(path, params) - except Exception as exc: - log_warning(f"report build failed path={path} host={host} date={report_date}: {exc}") - cached = get_report_response_cache(cache_key, allow_stale=True) - if cached is not None: - send_cached_report(handler, cached, reason="build-error") - return - send_report_unavailable(handler, "report build failed and no cached response is available") - return - save_report_response_cache(cache_key, data, content_type) - send_bytes(handler, data, content_type) - finally: - _REPORT_BUILD_SEMAPHORE.release() - - -def build_source_freshness(host, interactive_required=True): - source_specs = [ - { - "source_id": "worktime_sessions", - "label": "RDP worktime sessions", - "bucket_candidates": [f"aw-worktime-sessions_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "rdp_window", - "label": "RDP current window", - "bucket_candidates": [f"aw-rdp-window_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "rdp_afk", - "label": "RDP AFK", - "bucket_candidates": [f"aw-rdp-afk_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "watcher_window", - "label": "Local watcher window", - "bucket_candidates": [f"aw-watcher-window_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "interactive_only": True, - "owner": "ops", - }, - { - "source_id": "watcher_afk", - "label": "Local watcher AFK", - "bucket_candidates": [f"aw-watcher-afk_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "owner": "ops", - }, - { - "source_id": "file_operations", - "label": "File operations collector", - "bucket_candidates": [f"aw-file-operations_{host}"], - "max_age_seconds": MANAGER_CRITICAL_SOURCE_MAX_AGE_SECONDS, - "required": True, - "interactive_only": True, - "owner": "ops", - }, - { - "source_id": "web_categories", - "label": "Browser/web categories", - "bucket_candidates": [f"aw-detmir-web-category_{host}"], - "max_age_seconds": MANAGER_WEB_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - { - "source_id": "session_events", - "label": "Windows session events", - "bucket_candidates": [f"aw-session-events_{host}"], - "max_age_seconds": MANAGER_SESSION_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - { - "source_id": "pve_tasks", - "label": "PVE task feed", - "bucket_candidates": ["aw-pve-task-events_pve-detmir"], - "max_age_seconds": MANAGER_INFRA_SOURCE_MAX_AGE_SECONDS, - "required": False, - "owner": "ops", - }, - ] - now = now_utc() - sources = [] - actions = [] - for spec in source_specs: - matched_bucket = "" - matched_event = None - matched_age = None - for candidate in spec["bucket_candidates"]: - event = latest_bucket_event(candidate) - if not event: - continue - ts = parse_iso_utc(event.get("timestamp")) - candidate_age = age_seconds(ts, now=now) - if matched_event is None or (candidate_age is not None and (matched_age is None or candidate_age < matched_age)): - matched_bucket = candidate - matched_event = event - matched_age = candidate_age - ts = parse_iso_utc((matched_event or {}).get("timestamp")) - age = matched_age if matched_event is not None else age_seconds(ts, now=now) - if matched_event is None: - status = "fail" if spec["required"] else "warn" - summary = "bucket missing or empty" - elif age is None: - status = "warn" - summary = "timestamp parse failed" - elif age > spec["max_age_seconds"]: - status = "fail" if spec["required"] else "warn" - summary = f"stale ({age}s)" - else: - status = "ok" - summary = f"fresh ({age}s)" - if spec.get("interactive_only") and not interactive_required and status != "ok": - status = "inactive" - summary = "inactive: no active interactive users" - detail = { - "source_id": spec["source_id"], - "label": spec["label"], - "status": status, - "status_label": _source_status_label(status), - "bucket_id": matched_bucket or spec["bucket_candidates"][0], - "timestamp": (matched_event or {}).get("timestamp", ""), - "age_seconds": age, - "required": spec["required"], - "interactive_only": bool(spec.get("interactive_only")), - "interactive_required": bool(interactive_required), - "max_age_seconds": spec["max_age_seconds"], - "summary": summary, - "event_summary": _source_summary(matched_event), - } - sources.append(detail) - if status in {"ok", "inactive"}: - continue - priority = "critical" if spec["required"] else "medium" - actions.append( - _action( - "source_freshness_review", - priority, - spec["owner"], - "today" if spec["required"] else "3d", - f"Источник '{spec['label']}' в состоянии {detail['status_label']}: {summary}.", - "Проверить collector/service, причину отставания и подтвердить, что управленческие выводы по данным ещё надёжны.", - evidence={ - "source_id": spec["source_id"], - "bucket_id": detail["bucket_id"], - "age_seconds": age, - "required": spec["required"], - }, - ) - ) - return sources, actions - - -def _build_management_core(rows, host, report_date, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - report_bounds = get_report_bounds(report_date) - workday = get_workday_bounds(report_date) - now_local = datetime.now(REPORT_TZ) - is_today = report_date == now_local.date() - effective_end_local = min(now_local, workday["end_local"]) if is_today else workday["end_local"] - elapsed_seconds = max(0, int((effective_end_local - workday["start_local"]).total_seconds())) - if not is_today: - elapsed_seconds = workday["duration_seconds"] - expected_seconds_per_user = min(workday["duration_seconds"], max(0, elapsed_seconds)) - target_seconds = int(expected_seconds_per_user * (MANAGER_TARGET_COVERAGE_PCT / 100.0)) - low_seconds = int(expected_seconds_per_user * (MANAGER_LOW_COVERAGE_PCT / 100.0)) - late_start_local = workday["start_local"] + timedelta(minutes=MANAGER_LATE_START_GRACE_MINUTES) - early_finish_local = workday["end_local"] - timedelta(minutes=MANAGER_EARLY_FINISH_GRACE_MINUTES) - - roster = [] - actions = [] - active_users = 0 - on_target_users = 0 - below_target_users = 0 - workday_total_active_seconds = 0 - workday_first_values = [] - workday_last_values = [] - top_workday_user = "" - top_workday_seconds = -1 - filtered_rows = [] - - for row in rows: - alias = resolve_user_alias(row.get("user", ""), row.get("user_id", ""), host) - if alias["exclude"]: - continue - if not _matches_management_filters(alias, owner_filter=owner_filter, department_filter=department_filter): - continue - filtered_rows.append(row) - public_row = {key: value for key, value in row.items() if key != "_intervals"} - calendar_active_seconds = int(row.get("active_seconds", 0) or 0) - intervals = row.get("_intervals") or [] - workday_active_seconds, workday_first, workday_last = _interval_overlap_seconds( - intervals, - workday["start_local"].astimezone(timezone.utc), - effective_end_local.astimezone(timezone.utc), - ) - first_activity = parse_iso_utc(row.get("first_activity")) - last_activity = parse_iso_utc(row.get("last_activity")) - first_local = first_activity.astimezone(REPORT_TZ) if first_activity else None - last_local = last_activity.astimezone(REPORT_TZ) if last_activity else None - workday_first_local = workday_first.astimezone(REPORT_TZ) if workday_first else None - workday_last_local = workday_last.astimezone(REPORT_TZ) if workday_last else None - coverage_pct = _clamp_pct((workday_active_seconds / expected_seconds_per_user) * 100.0) if expected_seconds_per_user > 0 else 0.0 - status = "ok" - if workday_active_seconds <= 0: - status = "inactive" - elif workday_active_seconds < target_seconds: - status = "below_target" - if workday_active_seconds > 0: - active_users += 1 - workday_total_active_seconds += workday_active_seconds - if workday_first_local: - workday_first_values.append(workday_first_local.isoformat()) - if workday_last_local: - workday_last_values.append(workday_last_local.isoformat()) - if workday_active_seconds >= target_seconds and workday_active_seconds > 0: - on_target_users += 1 - elif workday_active_seconds > 0: - below_target_users += 1 - if workday_active_seconds > top_workday_seconds: - top_workday_seconds = workday_active_seconds - top_workday_user = alias["display_name"] - - roster.append( - { - **public_row, - "user": alias["display_name"], - "user_original": row.get("user", ""), - "manager_owner": alias["manager_owner"], - "department": alias["department"], - "role": alias["role"], - "notes": alias["notes"], - "canonical_user_id": alias["canonical_user_id"] or row.get("user_id", ""), - "calendar_active_seconds": calendar_active_seconds, - "calendar_active_hhmm": row.get("active_hhmm", "00:00"), - "workday_active_seconds": workday_active_seconds, - "workday_active_hhmm": hhmm(workday_active_seconds), - "coverage_pct": coverage_pct, - "status": status, - "first_activity_local": first_local.isoformat() if first_local else "", - "last_activity_local": last_local.isoformat() if last_local else "", - "workday_first_activity_local": workday_first_local.isoformat() if workday_first_local else "", - "workday_last_activity_local": workday_last_local.isoformat() if workday_last_local else "", - } - ) - - owner = alias["display_name"] - user_id = alias["canonical_user_id"] or row.get("user_id", "") - evidence = { - "calendar_active_hhmm": row.get("active_hhmm", "00:00"), - "workday_active_hhmm": hhmm(workday_active_seconds), - "coverage_pct": coverage_pct, - "first_activity": row.get("first_activity", ""), - "last_activity": row.get("last_activity", ""), - "sessions_count": row.get("sessions_count", 0), - "manager_owner": alias["manager_owner"], - "department": alias["department"], - "role": alias["role"], - } - if workday_active_seconds <= 0: - actions.append( - _action( - "missing_activity", - "critical", - alias["manager_owner"], - "today", - f"За {report_date.isoformat()} у сотрудника {owner} нет подтверждённой активности в рабочем окне RDP.", - f"Проверить сотрудника {owner}: работал ли он в рабочее время, была ли потеря сбора данных или отсутствие входа в систему.", - user_id=user_id, - evidence=evidence, - ) - ) - continue - if expected_seconds_per_user > 0 and workday_active_seconds < low_seconds: - actions.append( - _action( - "low_activity_review", - "high", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} активное время в рабочем окне {hhmm(workday_active_seconds)} ниже {MANAGER_LOW_COVERAGE_PCT}% от ожидаемого окна.", - f"Проверить загрузку сотрудника {owner}, задачи и фактическое присутствие в рабочем процессе.", - user_id=user_id, - evidence=evidence, - ) - ) - elif expected_seconds_per_user > 0 and workday_active_seconds < target_seconds: - actions.append( - _action( - "target_gap_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} активное время в рабочем окне {hhmm(workday_active_seconds)} ниже управленческого целевого порога {MANAGER_TARGET_COVERAGE_PCT}%.", - f"Уточнить причину отклонения по сотруднику {owner} и подтвердить план работ.", - user_id=user_id, - evidence=evidence, - ) - ) - if workday_first_local and workday_first_local > late_start_local: - actions.append( - _action( - "late_start_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} первая активность в рабочем окне зафиксирована поздно: {workday_first_local.strftime('%H:%M')}.", - f"Проверить причину позднего старта сотрудника {owner} и подтвердить, что это не проблема доступа или дисциплины.", - user_id=user_id, - evidence=evidence, - ) - ) - if (not is_today) and workday_last_local and workday_last_local < early_finish_local: - actions.append( - _action( - "early_finish_review", - "medium", - alias["manager_owner"], - "24h", - f"У сотрудника {owner} последняя активность в рабочем окне завершилась рано: {workday_last_local.strftime('%H:%M')}.", - f"Проверить, было ли досрочное завершение рабочего дня сотрудника {owner} согласовано и чем оно объясняется.", - user_id=user_id, - evidence=evidence, - ) - ) - - actions.sort(key=lambda item: (_priority_rank(item["priority"]), item["owner"].lower(), item["action_id"])) - calendar_summary = build_report_summary(filtered_rows) - inactive_users = sum(1 for row in roster if row["status"] == "inactive") - portfolio_coverage_pct = _clamp_pct((workday_total_active_seconds / (expected_seconds_per_user * len(roster))) * 100.0) if roster and expected_seconds_per_user > 0 else 0.0 - calendar_first = calendar_summary.get("first_activity", "") - calendar_last = calendar_summary.get("last_activity", "") - return { - "generated_at_utc": now_utc().isoformat().replace("+00:00", "Z"), - "host": resolve_host(host), - "report_date": report_date.isoformat(), - "report_timezone": str(REPORT_TZ), - "filters": { - "owner": owner_filter, - "department": department_filter, - }, - "workday": { - "start_local": workday["start_local"].isoformat(), - "end_local": workday["end_local"].isoformat(), - "expected_seconds_per_user": expected_seconds_per_user, - "expected_hhmm_per_user": hhmm(expected_seconds_per_user), - "target_coverage_pct": MANAGER_TARGET_COVERAGE_PCT, - "low_coverage_pct": MANAGER_LOW_COVERAGE_PCT, - }, - "summary": { - **calendar_summary, - "calendar_total_active_seconds": calendar_summary["total_active_seconds"], - "calendar_total_active_hhmm": calendar_summary["total_active_hhmm"], - "calendar_first_activity": calendar_first, - "calendar_last_activity": calendar_last, - "workday_total_active_seconds": workday_total_active_seconds, - "workday_total_active_hhmm": hhmm(workday_total_active_seconds), - "workday_first_activity": min(workday_first_values) if workday_first_values else "", - "workday_last_activity": max(workday_last_values) if workday_last_values else "", - "total_active_seconds": workday_total_active_seconds, - "total_active_hhmm": hhmm(workday_total_active_seconds), - "first_activity": min(workday_first_values) if workday_first_values else "", - "last_activity": max(workday_last_values) if workday_last_values else "", - "top_user": top_workday_user, - "top_user_active_hhmm": hhmm(top_workday_seconds if top_workday_seconds > 0 else 0), - "active_users": active_users, - "inactive_users": inactive_users, - "on_target_users": on_target_users, - "below_target_users": below_target_users, - "portfolio_coverage_pct": portfolio_coverage_pct, - "actions_count": len(actions), - "critical_actions_count": sum(1 for action in actions if action["priority"] == "critical"), - "high_actions_count": sum(1 for action in actions if action["priority"] == "high"), - }, - "actions": actions, - "rows": roster, - "bucket_id": get_sessions_bucket_id(host), - "report_bounds": { - "start_utc": to_iso_utc(report_bounds["start"]), - "end_utc": to_iso_utc(report_bounds["end"]), - }, - } - - -def _management_trend_item(payload, report_date): - summary = payload["summary"] - return { - "report_date": report_date.isoformat(), - "users_count": summary["users_count"], - "active_users": summary["active_users"], - "inactive_users": summary["inactive_users"], - "workday_total_active_seconds": summary["workday_total_active_seconds"], - "workday_total_active_hhmm": summary["workday_total_active_hhmm"], - "portfolio_coverage_pct": summary["portfolio_coverage_pct"], - "actions_count": summary["actions_count"], - "critical_actions_count": summary["critical_actions_count"], - } - - -def build_management_trend(host, anchor_date, owner_filter="", department_filter="", precomputed_payloads=None): - trend = [] - precomputed_payloads = precomputed_payloads or {} - for offset in range(MANAGER_TREND_DAYS - 1, -1, -1): - current_date = anchor_date - timedelta(days=offset) - payload = precomputed_payloads.get(current_date) - if payload is None: - payload = load_management_cache(host, current_date) - if payload is None: - bounds, events = fetch_events_for_date(host, current_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - payload = _build_management_core(rows, host, current_date, owner_filter=owner_filter, department_filter=department_filter) - elif owner_filter or department_filter: - payload = apply_management_filters_to_payload( - payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=False, - include_source_actions=False, - ) - trend.append(_management_trend_item(payload, current_date)) - return trend - - -def build_filtered_management_trend(host, anchor_date, owner_filter="", department_filter=""): - trend = [] - for offset in range(MANAGER_TREND_DAYS - 1, -1, -1): - current_date = anchor_date - timedelta(days=offset) - base_payload = load_management_cache(host, current_date) - if base_payload is None: - bounds, events = fetch_events_for_date(host, current_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - base_payload = _build_management_core(rows, host, current_date) - filtered_payload = apply_management_filters_to_payload( - base_payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=False, - include_source_actions=False, - ) - summary = filtered_payload["summary"] - trend.append( - { - "report_date": current_date.isoformat(), - "users_count": summary["users_count"], - "active_users": summary["active_users"], - "inactive_users": summary["inactive_users"], - "workday_total_active_seconds": summary["workday_total_active_seconds"], - "workday_total_active_hhmm": summary["workday_total_active_hhmm"], - "portfolio_coverage_pct": summary["portfolio_coverage_pct"], - "actions_count": summary["actions_count"], - "critical_actions_count": summary["critical_actions_count"], - } - ) - return trend - - -def build_management_payload(rows, host, report_date, owner_filter="", department_filter=""): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - payload = _build_management_core(rows, host, report_date, owner_filter=owner_filter, department_filter=department_filter) - interactive_required = int(payload.get("summary", {}).get("active_users", 0) or 0) > 0 - source_freshness, source_actions = build_source_freshness(resolve_host(host), interactive_required=interactive_required) - payload["sources"] = source_freshness - payload["trend"] = build_management_trend( - resolve_host(host), - report_date, - owner_filter=owner_filter, - department_filter=department_filter, - precomputed_payloads={report_date: payload}, - ) - payload["trend_scope"] = "portfolio" - if source_actions: - payload["actions"].extend(filter_management_actions(source_actions, payload["rows"], owner_filter=owner_filter, department_filter=department_filter)) - payload["actions"].sort(key=lambda item: (_priority_rank(item["priority"]), item["owner"].lower(), item["action_id"])) - payload["summary"]["actions_count"] = len(payload["actions"]) - payload["summary"]["critical_actions_count"] = sum(1 for action in payload["actions"] if action["priority"] == "critical") - payload["summary"]["high_actions_count"] = sum(1 for action in payload["actions"] if action["priority"] == "high") - payload["executive"] = build_executive_summary(payload["summary"], payload["actions"], payload["sources"]) - payload["owner_rollups"] = build_owner_rollups(payload["rows"], payload["actions"]) - payload["department_rollups"] = build_department_rollups(payload["rows"], payload["actions"]) - payload["owner_roster"] = build_owner_roster(payload["rows"], payload["actions"]) - return payload - - -def report_for_date(host, report_date): - bounds, events = fetch_events_for_date(host, report_date) - return aggregate_rows(events, bounds["start"], bounds["end"], host) - - -def management_report_for_date(host, report_date, owner_filter="", department_filter="", use_cache=True): - owner_filter = normalize_management_filter(owner_filter) - department_filter = normalize_management_filter(department_filter) - if owner_filter or department_filter: - base_payload = management_report_for_date(host, report_date, use_cache=use_cache) - filtered_payload = apply_management_filters_to_payload( - base_payload, - owner_filter=owner_filter, - department_filter=department_filter, - include_sources=True, - include_source_actions=True, - ) - filtered_payload["trend"] = [] - filtered_payload["trend_scope"] = "filtered_current_only" - return filtered_payload - cache_allowed = use_cache or report_date < datetime.now(REPORT_TZ).date() - if cache_allowed: - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - lock = get_management_build_lock(host, report_date) - with lock: - if cache_allowed: - cached = load_management_cache(host, report_date) - if cached is not None: - return cached - bounds, events = fetch_events_for_date(host, report_date) - rows = aggregate_rows_with_intervals(events, bounds["start"], bounds["end"], host) - payload = build_management_payload(rows, host, report_date) - save_management_cache(host, report_date, payload) - return payload - - -def report_today(host): - return report_for_date(host, resolve_report_date()) - - -def report_for_date_fresh(host, report_date): - spec = importlib.util.spec_from_file_location("aw_worktime_runtime", MODULE_PATH) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module.report_for_date(host, report_date) - - -def render_html(rows, host, report_date, selected_day=None, true_active_apps=None): - generated = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - date_local = report_date.strftime("%Y-%m-%d") - day_query = f"&day={selected_day}" if selected_day in {"today", "yesterday"} else "" - date_query = f"&date={date_local}" if not day_query else "" - summary = build_report_summary(rows) - today_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": resolve_host(host), "day": "today"}) - yesterday_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": resolve_host(host), "day": "yesterday"}) - csv_url = "/reports/worktime/today?" + urlencode({"format": "csv", "host": resolve_host(host), **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": date_local})}) - json_url = "/reports/worktime/today?" + urlencode({"host": resolve_host(host), **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": date_local})}) - form_action = "/reports/worktime/today" - cards = [ - ("Пользователи", str(summary["users_count"])), - ("Активное время", summary["total_active_hhmm"]), - ("Лидер дня", f"{summary['top_user']} · {summary['top_user_active_hhmm']}" if summary["top_user"] else "н/д"), - ("Диапазон", f"{summary['first_activity']} -> {summary['last_activity']}" if summary["first_activity"] else "нет активности"), - ] - trs = [] - detail_cards = [] - true_active_apps = true_active_apps or [] - true_active_rows = [] - for app_row in true_active_apps: - last_action = app_row.get("last_action") or "-" - last_time = app_row.get("last_action_local") or "-" - true_active_rows.append( - "" - f"{html.escape(app_row.get('application') or '-')}" - f"{html.escape(app_row.get('proved_work_human') or app_row.get('proved_work_hhmm') or '0 сек')}" - f"{html.escape(last_time)} · {html.escape(last_action)}" - "" - ) - if not true_active_rows: - true_active_rows.append("Пока нет доказанной активной работы по приложениям за выбранную дату.") - for row in rows: - user_slug = safe_slug(row["user"]) - active_seconds = int(row.get("active_seconds", 0) or 0) - utilization = 0.0 - day_total = 24 * 3600 - if day_total > 0: - utilization = round((active_seconds / day_total) * 100.0, 2) - trs.append( - "" - f"{html.escape(row['user'])}" - f"{html.escape(row['user_id'])}" - f"{row['active_hhmm']}" - f"{row['active_seconds']}" - f"{html.escape(row['first_activity'])}" - f"{html.escape(row['last_activity'])}" - f"{row['idle_seconds']}" - f"{row['sessions_count']}" - f"{row['samples_count']}" - "" - ) - detail_cards.append( - "
    " - "
    " - "

    {user}

    " - "{active}" - "
    " - "
    " - "
    Пользователь{user_id}
    " - "
    Загрузка{utilization}%
    " - "
    Начало активности{first_activity}
    " - "
    Конец активности{last_activity}
    " - "
    Сессии{sessions}
    " - "
    Активные сэмплы{active_samples} / {samples}
    " - "
    " - "
    " - .format( - slug=user_slug, - user=html.escape(row["user"]), - active=html.escape(row["active_hhmm"]), - user_id=html.escape(row["user_id"]), - utilization=utilization, - first_activity=html.escape(row["first_activity"] or "н/д"), - last_activity=html.escape(row["last_activity"] or "н/д"), - sessions=row["sessions_count"], - active_samples=row["active_samples"], - samples=row["samples_count"], - )) - if not trs: - trs.append('За выбранную дату данных пока нет.') - detail_cards.append("

    За выбранную дату нет активности пользователей.

    ") - return f""" - - - - - AW-rus Отчёт по работе в RDP - - - -
    -
    -

    Отчёт по работе в RDP

    -
    Хост: {resolve_host(host)} · Дата: {date_local} · Часовой пояс: {REPORT_TZ} · Сформировано UTC: {generated}
    -
    - Сегодня - Вчера - Скачать CSV - Открыть JSON -
    -
    -
    - - - - -
    -
    -
    - {''.join(f"
    {html.escape(label)}{html.escape(value)}
    " for label, value in cards)} -
    -
    -
    -

    Доказанная работа по приложениям

    - - - - - - - - - - {''.join(true_active_rows)} - -
    ПриложениеДоказанная работаПоследнее действие
    -
    -
    -

    Таблица по пользователям

    - - - - - - - - - - - - - - - - {''.join(trs)} - -
    ПользовательУчётная записьАктивноАктивно, секНачало активностиКонец активностиПростой, секСессииСэмплы
    -
    -
    -

    Детали по пользователям

    -
    - {''.join(detail_cards)} -
    -
    -
    - -""" - - -def build_management_report_url(host, report_date, selected_day=None, fmt=None, owner_filter="", department_filter=""): - params = {"host": resolve_host(host)} - if fmt: - params["format"] = fmt - if selected_day in {"today", "yesterday"}: - params["day"] = selected_day - else: - params["date"] = report_date - if normalize_management_filter(owner_filter): - params["owner"] = normalize_management_filter(owner_filter) - if normalize_management_filter(department_filter): - params["department"] = normalize_management_filter(department_filter) - return "/reports/worktime/management?" + urlencode(params) - - -def render_management_html(payload, selected_day=None): - summary = payload["summary"] - workday = payload["workday"] - report_date = payload["report_date"] - host = payload["host"] - executive = payload.get("executive") or {} - active_filters = payload.get("filters") or {} - owner_filter = normalize_management_filter(active_filters.get("owner")) - department_filter = normalize_management_filter(active_filters.get("department")) - today_url = build_management_report_url(host, report_date, selected_day="today", fmt="html", owner_filter=owner_filter, department_filter=department_filter) - yesterday_url = build_management_report_url(host, report_date, selected_day="yesterday", fmt="html", owner_filter=owner_filter, department_filter=department_filter) - json_url = build_management_report_url(host, report_date, selected_day=selected_day, owner_filter=owner_filter, department_filter=department_filter) - classic_url = "/reports/worktime/today?" + urlencode({"format": "html", "host": host, **({"day": selected_day} if selected_day in {"today", "yesterday"} else {"date": report_date})}) - reset_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html") - actions_html = [] - for action in payload["actions"]: - actions_html.append( - "" - f"{html.escape(action['priority'])}" - f"{html.escape(action['owner'])}" - f"{html.escape(action['action_id'])}" - f"{html.escape(action['deadline_hint'])}" - f"{html.escape(action['reason'])}" - f"{html.escape(action['recommended_action'])}" - "" - ) - if not actions_html: - actions_html.append("Отклонений по текущим правилам не найдено.") - - roster_html = [] - for row in payload["rows"]: - roster_html.append( - "" - f"{html.escape(row['user'])}" - f"{html.escape(row.get('canonical_user_id') or row['user_id'])}" - f"{html.escape(row.get('manager_owner') or row['user'])}" - f"{html.escape(row.get('department') or '-')}" - f"{html.escape(row['workday_active_hhmm'])}" - f"{html.escape(row['calendar_active_hhmm'])}" - f"{row['coverage_pct']}" - f"{html.escape(row['status'])}" - f"{html.escape(row.get('workday_first_activity_local') or '-')}" - f"{html.escape(row.get('workday_last_activity_local') or '-')}" - f"{row['sessions_count']}" - "" - ) - if not roster_html: - roster_html.append("За выбранную дату данных нет.") - - trend_html = [] - for row in payload.get("trend", []): - trend_html.append( - "" - f"{html.escape(row['report_date'])}" - f"{row['users_count']}" - f"{row['active_users']}" - f"{row['inactive_users']}" - f"{html.escape(row['workday_total_active_hhmm'])}" - f"{row['portfolio_coverage_pct']}" - f"{row['actions_count']}" - f"{row['critical_actions_count']}" - "" - ) - if not trend_html: - trend_message = "Тренд по выбранному фильтру пока отключён, чтобы current-scope отчёт отвечал быстро." if (owner_filter or department_filter) else "Тренд пока недоступен." - trend_html.append(f"{html.escape(trend_message)}") - - owner_rollup_html = [] - for item in payload.get("owner_rollups", []): - owner_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html", owner_filter=item["name"], department_filter=department_filter) - owner_rollup_html.append( - "" - f"{html.escape(item['name'])}" - f"{item['users_count']}" - f"{item['inactive_users']}" - f"{item['below_target_users']}" - f"{item['critical_actions_count']}" - f"{item['high_actions_count']}" - f"{html.escape(item['workday_total_active_hhmm'])}" - f"{item['portfolio_coverage_pct']}" - f"{html.escape(', '.join(item['users']) if item['users'] else '-')}" - "" - ) - if not owner_rollup_html: - owner_rollup_html.append("Нет данных по ответственным.") - - department_rollup_html = [] - for item in payload.get("department_rollups", []): - department_url = build_management_report_url(host, report_date, selected_day=selected_day, fmt="html", owner_filter=owner_filter, department_filter=item["name"]) - department_rollup_html.append( - "" - f"{html.escape(item['name'])}" - f"{item['users_count']}" - f"{item['inactive_users']}" - f"{item['below_target_users']}" - f"{item['critical_actions_count']}" - f"{item['high_actions_count']}" - f"{html.escape(item['workday_total_active_hhmm'])}" - f"{item['portfolio_coverage_pct']}" - f"{html.escape(', '.join(item['users']) if item['users'] else '-')}" - "" - ) - if not department_rollup_html: - department_rollup_html.append("Нет данных по подразделениям.") - - owner_profile_html = [] - for item in payload.get("owner_roster", []): - owner_profile_html.append( - "
    " - f"
    0 else ('high' if item['high_actions_count'] > 0 else 'low')}'>{html.escape(item['display_name'])}
    " - f"

    {html.escape(item['title'] or item['name'])}

    " - f"
    Подразделение: {html.escape(item['department'] or '-')}
    " - f"

    Пользователи: {item['users_count']} · inactive: {item['inactive_users']} · actions: {item['actions_count']}

    " - f"

    Контакт: {html.escape(item['contact'] or '-')}

    " - f"

    Эскалация: {html.escape(item['escalation_to'] or '-')}

    " - f"

    {html.escape(item['notes'] or 'Без дополнительных заметок.')}

    " - "
    " - ) - if not owner_profile_html: - owner_profile_html.append("

    Каталог ответственных пуст

    Добавьте блок owners в worktime-manager-aliases.json, чтобы отчёт показывал роли, контакты и эскалацию.

    ") - - sources_html = [] - for source in payload.get("sources", []): - source_priority_class = "low" if source["status"] in {"ok", "inactive"} else ("critical" if source["required"] else "medium") - sources_html.append( - "" - f"{html.escape(source['label'])}" - f"{html.escape(source['status_label'])}" - f"{html.escape(source['bucket_id'])}" - f"{html.escape(source.get('timestamp') or '-')}" - f"{html.escape(str(source.get('age_seconds')) if source.get('age_seconds') is not None else '-')}" - f"{html.escape(source.get('event_summary') or source.get('summary') or '-')}" - "" - ) - if not sources_html: - sources_html.append("Статусы источников недоступны.") - - focus_html = [] - for item in executive.get("focus_items", []): - focus_html.append( - "
    " - f"
    {html.escape(item['priority'])}
    " - f"

    {html.escape(item['title'])}

    " - f"
    Ответственный: {html.escape(item['owner'])}
    " - f"

    {html.escape(item['reason'])}

    " - f"{html.escape(item['recommended_action'])}" - "
    " - ) - if not focus_html: - focus_html.append("

    Критичных действий нет

    На текущий момент менеджерских отклонений по активным правилам не найдено.

    ") - - stale_html = [] - for item in executive.get("stale_sources", []): - stale_html.append( - "
  • " - f"{html.escape(item['label'])}: {html.escape(item['status'])} · {html.escape(item['summary'])}" - "
    • " - ) - stale_block = ( - "
    Проблемы со свежестью источников:
      " - + "".join(stale_html) - + "
    " - ) if stale_html else "" - filter_parts = [] - if owner_filter: - filter_parts.append(f"ответственный: {html.escape(owner_filter)}") - if department_filter: - filter_parts.append(f"подразделение: {html.escape(department_filter)}") - filter_block = ( - "
    Фильтр: " - + " · ".join(filter_parts) - + f" Сбросить
    " - ) if filter_parts else "" - - cards = [ - ("Пользователи", str(summary["users_count"])), - ("Активны", str(summary["active_users"])), - ("Без активности", str(summary["inactive_users"])), - ("Ниже цели", str(summary["below_target_users"])), - ("Покрытие", f"{summary['portfolio_coverage_pct']}%"), - ("Действия", str(summary["actions_count"])), - ("Рабочее окно", summary["workday_total_active_hhmm"]), - ("Календарный день", summary["calendar_total_active_hhmm"]), - ] - - return f""" - - - - - AW-rus Управленческий отчёт по работе в RDP - - - -
    -
    -

    Управленческий отчёт по работе в RDP

    -
    Хост: {html.escape(host)} · Дата: {html.escape(report_date)} · Рабочее окно: {html.escape(workday['start_local'])} -> {html.escape(workday['end_local'])} · Сформировано UTC: {html.escape(payload['generated_at_utc'])}
    -
    - Сегодня - Вчера - Открыть JSON - Классический RDP отчёт -
    -
    - {''.join(f"
    {html.escape(label)}{html.escape(value)}
    " for label, value in cards)} -
    -
    - Целевое покрытие: {MANAGER_TARGET_COVERAGE_PCT}% от ожидаемого рабочего окна на пользователя. - Критический провал: ниже {MANAGER_LOW_COVERAGE_PCT}% или полное отсутствие активности. - Рабочее окно считается отдельно от календарной активности, чтобы ночная работа не маскировала дневной провал. -
    -
    -
    -

    Что делать сегодня

    -
    - {html.escape(executive.get('headline') or 'Сводка недоступна')} -

    {html.escape(executive.get('message') or '')}

    - {filter_block} - {stale_block} -
    -
    - {''.join(focus_html)} -
    -
    -
    -

    Тренд за {MANAGER_TREND_DAYS} дней

    - - - - - - - - - - - - - - - {''.join(trend_html)} - -
    ДатаПользователиАктивныБез активностиРабочее окноПокрытие, %ДействияCritical
    -
    -
    -

    По ответственным

    - - - - - - - - - - - - - - - - {''.join(owner_rollup_html)} - -
    ОтветственныйСотрудникиБез активностиНиже целиCriticalHighРабочее окноПокрытие, %Кого затрагивает
    -
    -
    -

    Ответственные и эскалация

    -
    - {''.join(owner_profile_html)} -
    -
    -
    -

    По подразделениям

    - - - - - - - - - - - - - - - - {''.join(department_rollup_html)} - -
    ПодразделениеСотрудникиБез активностиНиже целиCriticalHighРабочее окноПокрытие, %Кого затрагивает
    -
    -
    -

    Очередь действий руководителя

    - - - - - - - - - - - - - {''.join(actions_html)} - -
    ПриоритетСотрудникТипСрокПочему это важноЧто сделать
    -
    -
    -

    Покрытие по сотрудникам

    - - - - - - - - - - - - - - - - - - {''.join(roster_html)} - -
    СотрудникУчётная записьОтветственныйПодразделениеАктивно в окнеАктивно за деньПокрытие, %СтатусПервая активность в окнеПоследняя активность в окнеСессии
    -
    -
    -

    Свежесть источников данных

    - - - - - - - - - - - - - {''.join(sources_html)} - -
    ИсточникСтатусБакетПоследнее событие UTCВозраст, секКонтекст
    -
    -
    - -""" - - -def build_worktime_report_response(path, params): - fmt = resolve_report_format(params) - host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) - day = params.get("day", ["today"])[0] - date_text = params.get("date", [None])[0] - owner_filter = normalize_management_filter(params.get("owner", [""])[0]) - department_filter = normalize_management_filter(params.get("department", [""])[0]) - report_date = resolve_report_date(day=day, date_text=date_text) - is_management = path == "/reports/worktime/management" - allow_immediate_stale = str(params.get("allow_stale", [""])[0]).lower() in {"1", "true", "yes"} - management_payload = ( - management_report_for_date( - host, - report_date, - owner_filter=owner_filter, - department_filter=department_filter, - use_cache=allow_immediate_stale, - ) - if is_management - else None - ) - rows = report_for_date_fresh(host, report_date) if not is_management else management_payload["rows"] - true_active_apps = [] if is_management else build_true_active_apps(host, report_date) - - if fmt == "csv": - if is_management: - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=["priority", "owner", "user_id", "action_id", "deadline_hint", "reason", "recommended_action"], - extrasaction="ignore", - ) - writer.writeheader() - writer.writerows(management_payload["actions"]) - return out.getvalue().encode(), "text/csv; charset=utf-8" - out = io.StringIO() - writer = csv.DictWriter( - out, - fieldnames=[ - "user", - "user_id", - "active_seconds", - "active_hhmm", - "first_activity", - "last_activity", - "idle_seconds", - "sessions_count", - "samples_count", - "active_samples", - ], - ) - writer.writeheader() - writer.writerows(rows) - return out.getvalue().encode(), "text/csv; charset=utf-8" - - if fmt == "html": - if is_management: - data = render_management_html(management_payload, selected_day=day if day in {"today", "yesterday"} else None).encode("utf-8") - else: - data = render_html(rows, host, report_date, selected_day=day if day in {"today", "yesterday"} else None, true_active_apps=true_active_apps).encode("utf-8") - return data, "text/html; charset=utf-8" - - if is_management: - data = json.dumps(management_payload, ensure_ascii=False, indent=2).encode("utf-8") - return data, "application/json; charset=utf-8" - - obj = { - "generated_at_utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "report_timezone": str(REPORT_TZ), - "host": host, - "report_date": report_date.isoformat(), - "bucket_id": get_sessions_bucket_id(host), - "rows": rows, - "true_active_apps": true_active_apps, - } - data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") - return data, "application/json; charset=utf-8" - - -def send_bytes(handler, data, content_type, status=200): - handler.send_response(status) - handler.send_header("Content-Type", content_type) - handler.send_header("Content-Length", str(len(data))) - handler.end_headers() - try: - handler.wfile.write(data) - except (BrokenPipeError, ConnectionResetError): - return False - return True - - -class H(BaseHTTPRequestHandler): - def do_GET(self): - parsed = urlparse(self.path) - if parsed.path in {"/health", "/api/health"}: - data = json.dumps(worktime_health_payload(), ensure_ascii=False, indent=2).encode("utf-8") - send_bytes(self, data, "application/json; charset=utf-8") - return - - if parsed.path.startswith("/dlp-ioc/"): - name = parsed.path.rsplit("/", 1)[-1] - if name not in {"ioc_blacklist.json", "ioc_blacklist.csv", "ioc_blacklist.sql"}: - self.send_response(404) - self.end_headers() - return - path = os.path.join(IOC_DIR, name) - if not os.path.isfile(path): - self.send_response(404) - self.end_headers() - return - with open(path, "rb") as f: - data = f.read() - if name.endswith(".json"): - ctype = "application/json; charset=utf-8" - elif name.endswith(".csv"): - ctype = "text/csv; charset=utf-8" - else: - ctype = "text/plain; charset=utf-8" - send_bytes(self, data, ctype) - return - - if parsed.path not in {"/reports/worktime/today", "/reports/worktime/management"}: - self.send_response(404) - self.end_headers() - return - - params = parse_qs(parsed.query, keep_blank_values=False) - fmt = resolve_report_format(params, self.headers.get("Accept", "")) - if not params.get("format") and fmt != "json": - params = {name: list(values) for name, values in params.items()} - params["format"] = [fmt] - host = resolve_host(params.get("host", [DEFAULT_HOST])[0]) - day = params.get("day", ["today"])[0] - date_text = params.get("date", [None])[0] - owner_filter = normalize_management_filter(params.get("owner", [""])[0]) - department_filter = normalize_management_filter(params.get("department", [""])[0]) - report_date = resolve_report_date(day=day, date_text=date_text) - cache_key = make_report_cache_key( - parsed.path, - fmt, - host, - report_date, - selected_day=day if day in {"today", "yesterday"} else "", - owner_filter=owner_filter, - department_filter=department_filter, - ) - - send_worktime_report(self, parsed.path, params, cache_key, host=host, report_date=report_date) - - -class WorktimeHTTPServer(ThreadingHTTPServer): - daemon_threads = True - request_queue_size = 64 - - -def main(): - WorktimeHTTPServer((LISTEN_HOST, LISTEN_PORT), H).serve_forever(poll_interval=0.5) - - -if __name__ == "__main__": - main() diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service index e85d3bf..07e32d8 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-api.service @@ -8,7 +8,7 @@ StartLimitIntervalSec=60 [Service] Type=simple EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/bin/python3 /usr/local/bin/aw-worktime-api.py +ExecStart=/usr/local/bin/aw-worktime-api-rust Restart=on-failure RestartSec=5 User=activitywatch diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service index a013770..6f3cb29 100644 --- a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service +++ b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.service @@ -6,10 +6,9 @@ Wants=network-online.target activitywatch-server.service aw-worktime-api.service [Service] Type=oneshot EnvironmentFile=/etc/activitywatch/aw-server.env -ExecStart=/usr/local/bin/aw-worktime-prewarm.sh +ExecStart=/usr/local/bin/aw-worktime-prewarm-rust User=activitywatch Group=activitywatch StandardOutput=journal StandardError=journal SyslogIdentifier=aw-worktime-prewarm - diff --git a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh b/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh deleted file mode 100644 index 7ae33e9..0000000 --- a/install-kit-awindows-20260427-211240/aw-server/aw-worktime-prewarm.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKTIME_BASE_URL="${WORKTIME_BASE_URL:-http://127.0.0.1:5610}" -WORKTIME_PREWARM_TIMEOUT_SECONDS="${WORKTIME_PREWARM_TIMEOUT_SECONDS:-45}" -WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS="${WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS:-10}" -WORKTIME_PREWARM_READY_TIMEOUT_SECONDS="${WORKTIME_PREWARM_READY_TIMEOUT_SECONDS:-60}" -WORKTIME_PREWARM_READY_INTERVAL_SECONDS="${WORKTIME_PREWARM_READY_INTERVAL_SECONDS:-2}" -WORKTIME_PREWARM_HOST="${WORKTIME_PREWARM_HOST:-${AW_WORKTIME_HOST:-SHARKON2025}}" -WORKTIME_PREWARM_PROFILE="${WORKTIME_PREWARM_PROFILE:-full}" - -log() { - printf '%s %s\n' "$(date '+%F %T')" "$*" -} - -probe() { - local url="$1" - local timeout="$2" - local tmp - tmp="$(mktemp)" - local code - code="$(curl -sS --max-time "$timeout" -o /dev/null -D "$tmp" -w '%{http_code}' "$url" 2>/dev/null || true)" - if [[ "$code" =~ ^2 ]]; then - local cache - cache="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache:/ {print $2; exit}' || true)" - local reason - reason="$(tr '\r' '\n' < "$tmp" | awk 'tolower($0) ~ /^x-aw-worktime-cache-reason:/ {print $2; exit}' || true)" - rm -f "$tmp" - log "ok code=$code cache=${cache:-none} reason=${reason:-none} url=$url" - return 0 - fi - rm -f "$tmp" - log "warn code=${code:-000} url=$url" - return 1 -} - -wait_until_ready() { - local deadline - deadline=$(( $(date +%s) + WORKTIME_PREWARM_READY_TIMEOUT_SECONDS )) - while true; do - if probe "$WORKTIME_BASE_URL/health" "$WORKTIME_PREWARM_HEALTH_TIMEOUT_SECONDS"; then - return 0 - fi - if [[ "$(date +%s)" -ge "$deadline" ]]; then - return 1 - fi - sleep "$WORKTIME_PREWARM_READY_INTERVAL_SECONDS" - done -} - -if ! wait_until_ready; then - log "health readiness timed out; skip prewarm" - exit 0 -fi - -full_urls=( - "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=html&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=html&host=$WORKTIME_PREWARM_HOST" -) - -startup_urls=( - "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/today?day=today&format=json&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=json&host=$WORKTIME_PREWARM_HOST" - "$WORKTIME_BASE_URL/reports/worktime/management?day=today&format=csv&host=$WORKTIME_PREWARM_HOST" -) - -case "$WORKTIME_PREWARM_PROFILE" in - full) - urls=("${full_urls[@]}") - ;; - startup) - urls=("${startup_urls[@]}") - ;; - *) - log "unknown profile=$WORKTIME_PREWARM_PROFILE" - exit 0 - ;; -esac - -failures=0 -for url in "${urls[@]}"; do - if ! probe "$url" "$WORKTIME_PREWARM_TIMEOUT_SECONDS"; then - failures=$((failures + 1)) - fi -done - -if [[ "$failures" -gt 0 ]]; then - log "completed profile=$WORKTIME_PREWARM_PROFILE with failures=$failures" -else - log "completed profile=$WORKTIME_PREWARM_PROFILE successfully" -fi - -exit 0 diff --git a/install-kit-awindows-20260427-211240/aw-server/health-check.sh b/install-kit-awindows-20260427-211240/aw-server/health-check.sh deleted file mode 100644 index 7df5693..0000000 --- a/install-kit-awindows-20260427-211240/aw-server/health-check.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Health check script for AW services -# Returns 0 if all services are healthy, 1 otherwise - -SERVICES=("activitywatch-server" "aw-worktime-api" "aw-worktime-ui-bridge") -UNHEALTHY_SERVICES=() -WARNINGS=() - -if [[ -f /etc/activitywatch/aw-server.env ]]; then - # shellcheck disable=SC1091 - source /etc/activitywatch/aw-server.env -fi - -WORKTIME_HEALTH_BASE="${AW_RUS_HEALTH_WORKTIME_API:-${AW_WORKTIME_REPORT_BASE:-http://127.0.0.1:5610}}" -WORKTIME_HEALTH_URL="${WORKTIME_HEALTH_BASE%/}/health" -WORKTIME_HEALTH_TIMEOUT_SECONDS="${AW_RUS_HEALTH_WORKTIME_TIMEOUT_SECONDS:-15}" -AW_HEALTH_API_TIMEOUT_SECONDS="${AW_RUS_HEALTH_AW_TIMEOUT_SECONDS:-15}" -AW_HEALTH_API_ATTEMPTS="${AW_RUS_HEALTH_AW_ATTEMPTS:-3}" -SETTINGS_TIMEOUT_SECONDS="${AW_RUS_HEALTH_SETTINGS_TIMEOUT_SECONDS:-15}" -SETTINGS_ATTEMPTS="${AW_RUS_HEALTH_SETTINGS_ATTEMPTS:-3}" - -check_service() { - local service=$1 - if [[ "$service" == "aw-worktime-ui-bridge" ]]; then - if systemctl is-active --quiet aw-worktime-ui-bridge.timer && systemctl is-enabled --quiet aw-worktime-ui-bridge.timer; then - echo "✓ aw-worktime-ui-bridge.timer is running and enabled" - else - echo "✗ aw-worktime-ui-bridge.timer is not active/enabled" - UNHEALTHY_SERVICES+=("aw-worktime-ui-bridge.timer") - fi - return - fi - if systemctl is-active --quiet "$service"; then - echo "✓ $service is running" - else - echo "✗ $service is not running" - UNHEALTHY_SERVICES+=("$service") - fi -} - -check_api_endpoint() { - local url=$1 - local service_name=$2 - local timeout_seconds=${3:-5} - local attempts=${4:-1} - local attempt - - for ((attempt=1; attempt<=attempts; attempt++)); do - if curl -fsS --connect-timeout 3 --max-time "$timeout_seconds" "$url" >/dev/null 2>&1; then - echo "✓ $service_name API endpoint is responding" - return - fi - if (( attempt < attempts )); then - sleep 1 - fi - done - - echo "✗ $service_name API endpoint is not responding" - UNHEALTHY_SERVICES+=("$service_name-api") -} - -read_setting_value() { - local key=$1 - local attempt - for ((attempt=1; attempt<=SETTINGS_ATTEMPTS; attempt++)); do - if curl -fsS --connect-timeout 3 --max-time "$SETTINGS_TIMEOUT_SECONDS" "http://127.0.0.1:5600/api/0/settings/${key}" 2>/dev/null | \ - python3 -c 'import json,sys; print(json.load(sys.stdin))'; then - return 0 - fi - if (( attempt < SETTINGS_ATTEMPTS )); then - sleep 1 - fi - done - return 1 -} - -check_expected_setting() { - local key=$1 - local expected=$2 - local label=${3:-$1} - - if [[ -z "$expected" ]]; then - echo "⚠ expected value for ${label} is not configured, skipping drift check" - WARNINGS+=("${key}-expected-missing") - return - fi - - local actual - if ! actual="$(read_setting_value "$key" 2>/dev/null)"; then - echo "✗ failed to read setting ${label}" - UNHEALTHY_SERVICES+=("setting-${key}") - return - fi - - if [[ "$actual" == "$expected" ]]; then - echo "✓ ${label} matches expected value (${expected})" - else - echo "✗ ${label} drift detected: actual='${actual}' expected='${expected}'" - UNHEALTHY_SERVICES+=("setting-${key}") - fi -} - -check_dlp_transport_freshness() { - local dlp_health="${DLP_HEALTH_BIN:-/usr/local/bin/dlp-health-check}" - local result - - if [[ ! -x "$dlp_health" ]]; then - echo "⚠ dlp-health-check is not available, skipping DLP transport freshness checks" - WARNINGS+=("dlp-health-check-missing") - return - fi - - result="$("$dlp_health" --json 2>/dev/null || true)" - if [[ -z "$result" ]]; then - echo "⚠ dlp-health-check did not return JSON, skipping DLP transport freshness checks" - WARNINGS+=("dlp-health-check-empty") - return - fi - - local ok - ok="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); names={r["name"]:r for r in data.get("results", [])}; checks=["buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics"]; bad=[n for n in checks if names.get(n,{}).get("status")=="fail"]; print("1" if not bad else "0")' 2>/dev/null || echo "0")" - if [[ "$ok" == "1" ]]; then - echo "✓ DLP transport freshness check passed" - else - echo "✗ DLP transport freshness check failed" - UNHEALTHY_SERVICES+=("dlp-transport") - fi - - local errors warnings - errors="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="fail" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" - warnings="$(printf '%s' "$result" | python3 -c 'import json,sys; data=json.load(sys.stdin); out=[]; [out.append(f"{r.get(\"name\")}:{r.get(\"summary\")}") for r in data.get("results", []) if r.get("status")=="warn" and r.get("name") in ("buckets:endpoint-signals","buckets:file-operations","endpoint-self-test-metrics")]; print(", ".join(out))' 2>/dev/null || true)" - if [[ -n "$errors" ]]; then - echo " errors: $errors" - fi - if [[ -n "$warnings" ]]; then - echo " warnings: $warnings" - WARNINGS+=("$warnings") - fi -} - -echo "=== AW Services Health Check ===" -echo "Timestamp: $(date)" -echo - -# Check systemd services -for service in "${SERVICES[@]}"; do - check_service "$service" -done - -echo - -# Check API endpoints -check_api_endpoint "http://127.0.0.1:5600/api/0/info" "activitywatch-server" "$AW_HEALTH_API_TIMEOUT_SECONDS" "$AW_HEALTH_API_ATTEMPTS" -check_api_endpoint "$WORKTIME_HEALTH_URL" "aw-worktime-api" "$WORKTIME_HEALTH_TIMEOUT_SECONDS" 2 -check_dlp_transport_freshness "http://127.0.0.1:5600/api/0" "900" "${AW_HEALTH_STRICT_FILEOPS:-0}" -check_expected_setting "startOfDay" "${AW_EXPECT_START_OF_DAY:-}" "startOfDay" -check_expected_setting "always_active_pattern" "${AW_EXPECT_ALWAYS_ACTIVE_PATTERN:-}" "always_active_pattern" -check_expected_setting "landingpage" "${AW_EXPECT_LANDINGPAGE:-}" "landingpage" - -echo - -if [ ${#UNHEALTHY_SERVICES[@]} -eq 0 ]; then - echo "✓ All services are healthy" - if [ ${#WARNINGS[@]} -gt 0 ]; then - echo "⚠ Warnings: ${WARNINGS[*]}" - fi - exit 0 -else - echo "✗ Unhealthy services: ${UNHEALTHY_SERVICES[*]}" - exit 1 -fi diff --git a/install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh b/install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh index 34930d6..cdc1241 100755 --- a/install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh +++ b/install-kit-awindows-20260427-211240/aw-server/install_aw_server.sh @@ -24,13 +24,13 @@ required_vars=( BOOTSTRAP_DIR="/root/bootstrap" VIEWS_JSON="$BOOTSTRAP_DIR/settings/views-default.json" CLASSES_JSON="$BOOTSTRAP_DIR/settings/classes-worktime.json" -WORKTIME_API_SRC="$BOOTSTRAP_DIR/aw-worktime-api.py" +WORKTIME_API_RUST_SRC="$BOOTSTRAP_DIR/worktime-api" WORKTIME_API_SERVICE_SRC="$BOOTSTRAP_DIR/aw-worktime-api.service" WORKTIME_ALIASES_SRC="$BOOTSTRAP_DIR/worktime-manager-aliases.example.json" -WORKTIME_UI_BRIDGE_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.py" +WORKTIME_UI_BRIDGE_RUST_SRC="$BOOTSTRAP_DIR/worktime-ui-bridge" WORKTIME_UI_BRIDGE_SERVICE_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.service" WORKTIME_UI_BRIDGE_TIMER_SRC="$BOOTSTRAP_DIR/aw-worktime-ui-bridge.timer" -HEALTHD_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.py" +HEALTHD_RUST_SRC="$BOOTSTRAP_DIR/aw-rus-healthd" HEALTHD_SERVICE_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.service" HEALTHD_TIMER_SRC="$BOOTSTRAP_DIR/aw-rus-healthd.timer" @@ -99,11 +99,11 @@ systemctl enable activitywatch-server.service systemctl restart activitywatch-server.service systemctl --no-pager --full status activitywatch-server.service || true -if [[ -f "$WORKTIME_API_SRC" ]]; then - install -m 0755 "$WORKTIME_API_SRC" /usr/local/bin/aw-worktime-api.py +if [[ -f "$WORKTIME_API_RUST_SRC" ]]; then + install -m 0755 "$WORKTIME_API_RUST_SRC" /usr/local/bin/aw-worktime-api-rust fi -if [[ -f "$WORKTIME_API_SERVICE_SRC" ]]; then +if [[ -f "$WORKTIME_API_SERVICE_SRC" && -f /usr/local/bin/aw-worktime-api-rust ]]; then install -m 0644 "$WORKTIME_API_SERVICE_SRC" /etc/systemd/system/aw-worktime-api.service systemctl daemon-reload systemctl enable aw-worktime-api.service @@ -119,15 +119,15 @@ if [[ -f "$WORKTIME_ALIASES_SRC" ]]; then fi fi -if [[ -f "$WORKTIME_UI_BRIDGE_SRC" ]]; then - install -m 0755 "$WORKTIME_UI_BRIDGE_SRC" /usr/local/bin/aw-worktime-ui-bridge.py +if [[ -f "$WORKTIME_UI_BRIDGE_RUST_SRC" ]]; then + install -m 0755 "$WORKTIME_UI_BRIDGE_RUST_SRC" /usr/local/bin/aw-worktime-ui-bridge-rust fi -if [[ -f "$WORKTIME_UI_BRIDGE_SERVICE_SRC" ]]; then +if [[ -f "$WORKTIME_UI_BRIDGE_SERVICE_SRC" && -f /usr/local/bin/aw-worktime-ui-bridge-rust ]]; then install -m 0644 "$WORKTIME_UI_BRIDGE_SERVICE_SRC" /etc/systemd/system/aw-worktime-ui-bridge.service fi -if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" ]]; then +if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" && -f /usr/local/bin/aw-worktime-ui-bridge-rust ]]; then install -m 0644 "$WORKTIME_UI_BRIDGE_TIMER_SRC" /etc/systemd/system/aw-worktime-ui-bridge.timer systemctl daemon-reload systemctl disable --now aw-worktime-afk-bridge.timer >/dev/null 2>&1 || true @@ -137,15 +137,15 @@ if [[ -f "$WORKTIME_UI_BRIDGE_TIMER_SRC" ]]; then systemctl --no-pager --full status aw-worktime-ui-bridge.timer || true fi -if [[ -f "$HEALTHD_SRC" ]]; then - install -m 0755 "$HEALTHD_SRC" /usr/local/bin/aw-rus-healthd.py +if [[ -f "$HEALTHD_RUST_SRC" ]]; then + install -m 0755 "$HEALTHD_RUST_SRC" /usr/local/bin/aw-rus-healthd-rust fi -if [[ -f "$HEALTHD_SERVICE_SRC" ]]; then +if [[ -f "$HEALTHD_SERVICE_SRC" && -f /usr/local/bin/aw-rus-healthd-rust ]]; then install -m 0644 "$HEALTHD_SERVICE_SRC" /etc/systemd/system/aw-rus-healthd.service fi -if [[ -f "$HEALTHD_TIMER_SRC" ]]; then +if [[ -f "$HEALTHD_TIMER_SRC" && -f /usr/local/bin/aw-rus-healthd-rust ]]; then install -m 0644 "$HEALTHD_TIMER_SRC" /etc/systemd/system/aw-rus-healthd.timer systemctl daemon-reload systemctl enable aw-rus-healthd.timer diff --git a/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh index ebc8624..66d2670 100644 --- a/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh +++ b/install-kit-awindows-20260427-211240/scripts/aw-webui-browser-smoke.sh @@ -3,6 +3,19 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +rust_candidates=( + "${AW_BROWSER_SMOKE_RUST:-}" + "${CARGO_TARGET_DIR:-}/release/aw-browser-smoke" + "$ROOT_DIR/adk-rust/target/release/aw-browser-smoke" + "/usr/local/bin/aw-browser-smoke" +) + +for rust_bin in "${rust_candidates[@]}"; do + if [[ -n "$rust_bin" && -x "$rust_bin" ]]; then + exec "$rust_bin" --root "$ROOT_DIR" -- "$@" + fi +done + if [[ -z "${NODE_PATH:-}" && -d /home/igor/.agents/skills/playwright/node_modules ]]; then export NODE_PATH=/home/igor/.agents/skills/playwright/node_modules fi diff --git a/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh b/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh index ff2600b..a6db8d7 100644 --- a/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh +++ b/install-kit-awindows-20260427-211240/scripts/check_install_kit_vs_repo.sh @@ -24,59 +24,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -python3 - <<'PY' -from pathlib import Path -import hashlib -import sys - -root=Path('.') -kit=Path('install-kit-awindows-20260427-211240') -if not kit.exists(): - raise SystemExit('Install kit directory not found') - -def sha(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - -all_compared=[] -mismatches=[] -missing_in_repo=[] -allowed_kit_only_prefixes=('server-configs-' ,) -allowed_kit_only_files={'README-INSTALL-KIT.txt'} - -for kp in sorted(p for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'): - rel=kp.relative_to(kit) - rp=root/rel - if not rp.exists(): - rel_str=str(rel) - if rel_str in allowed_kit_only_files or rel_str.startswith(allowed_kit_only_prefixes): - continue - missing_in_repo.append(str(rel)) - continue - all_compared.append(str(rel)) - if sha(kp)!=sha(rp): - mismatches.append(str(rel)) - -ps_mismatches=[ - p for p in mismatches - if p.startswith('windows/') and p.endswith(('.ps1', '.psm1', '.psd1')) -] - -print(f'Compared files: {len(all_compared)}') -print(f'Missing in repo: {len(missing_in_repo)}') -print(f'Mismatched content: {len(mismatches)}') -if missing_in_repo: - print('--- Missing in repo ---') - for p in missing_in_repo: - print(p) -if mismatches: - print('--- Mismatches ---') - for p in mismatches: - print(p) -print(f'PowerShell mismatches: {len(ps_mismatches)}') -if ps_mismatches: - print('--- PowerShell mismatches ---') - for p in ps_mismatches: - print(p) -if missing_in_repo or mismatches: - sys.exit(1) -PY +echo "ERROR: Rust checker not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p check-install-kit-vs-repo" >&2 +exit 127 diff --git a/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh b/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh index 13936f0..001c39e 100644 --- a/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh +++ b/install-kit-awindows-20260427-211240/scripts/rebuild_install_kit.sh @@ -23,150 +23,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -KIT_DIR="install-kit-awindows-20260427-211240" -ZIP_ARCHIVE="${KIT_DIR}.zip" -TAR_ARCHIVE="${KIT_DIR}.tar.gz" -SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.18" -OLD_SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.21" -TMP_SERVER_CONFIG_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_SERVER_CONFIG_DIR"' EXIT - -copy_file() { - local src="$1" - local dest="$2" - mkdir -p "$(dirname "$dest")" - cp "$src" "$dest" -} - -sync_tree() { - local base="$1" - shift - for rel in "$@"; do - copy_file "$rel" "${KIT_DIR}/${rel}" - done -} - -if [[ -d "$OLD_SERVER_CONFIG_DIR" ]]; then - cp "$OLD_SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ -fi -if [[ -d "$SERVER_CONFIG_DIR" ]]; then - cp "$SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ -fi - -rm -rf "${KIT_DIR}/ansible" "${KIT_DIR}/aw-server" "${KIT_DIR}/windows" "${KIT_DIR}/scripts" "${KIT_DIR}/server-configs-"* - -ansible_files=( - ansible/README.md - ansible/deploy_aw_pfsense_poller.yml - ansible/deploy_aw_server.yml - ansible/deploy_aw_windows.yml - ansible/group_vars/all.example.yml - ansible/group_vars/pfsense-poller.example.yml - ansible/group_vars/proxmox-matrix.example.yml - ansible/group_vars/proxmox.example.yml - ansible/group_vars/windows.example.yml - ansible/install_full_stack.yml - ansible/inventory.example.ini - ansible/provision_proxmox_ct_and_deploy_aw.yml - ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml - ansible/tasks/provision_ct_and_deploy_aw.yml -) - -aw_server_files=( - aw-server/activitywatch-server.service - aw-server/apply_webui_ru_patch.sh - aw-server/aw-host-groups.json - aw-server/aw-ru-patch.js - aw-server/aw-rus-healthd.py - aw-server/aw-rus-healthd.service - aw-server/aw-rus-healthd.timer - aw-server/aw-browser-smoke.service - aw-server/aw-browser-smoke.timer - aw-server/aw-slo-monitor.py - aw-server/aw-slo-monitor.service - aw-server/aw-slo-monitor.timer - aw-server/aw-server.env.example - aw-server/aw-sw-cleanup.js - aw-server/aw-worktime-api.py - aw-server/aw-worktime-api.service - aw-server/aw-worktime-prewarm.sh - aw-server/aw-worktime-prewarm.service - aw-server/aw-worktime-prewarm.timer - aw-server/aw-worktime-panel.js - aw-server/health-check.sh - aw-server/install_aw_server.sh - aw-server/settings/classes-worktime.json - aw-server/settings/views-default.json -) - -windows_files=( - windows/ActivityWatch.Windows.Common.psd1 - windows/ActivityWatch.Windows.Common.psm1 - windows/browser-domains-native-collector.ps1 - windows/deploy-domain-users.ps1 - windows/deploy-ensemble.ps1 - windows/deploy-single-user.ps1 - windows/AWatchRusCollectorGuardService.cs - windows/aw-collector-guard.ps1 - windows/install-collector-guard-service.ps1 - windows/dlp-endpoint-signals-collector.ps1 - windows/dlp-policy.example.json - windows/dlp-policy.native-cross-os.example.json - windows/email-outbound-collector.ps1 - windows/hardening-recovery.ps1 - windows/migrate-awatch-rus-paths.ps1 - windows/validate-deployment.ps1 - windows/web-category-rules.example.json - windows/worktime-session-collector.ps1 -) - -scripts_files=( - scripts/aw-webui-browser-smoke.mjs - scripts/aw-webui-browser-smoke.sh - scripts/check_install_kit_vs_repo.sh - scripts/quality-gate.sh - scripts/rebuild_install_kit.sh - scripts/validate_install_kit.sh - scripts/verify_innosetup_installer.sh -) - -sync_tree ansible "${ansible_files[@]}" -sync_tree aw-server "${aw_server_files[@]}" -sync_tree windows "${windows_files[@]}" -sync_tree scripts "${scripts_files[@]}" - -mkdir -p "$SERVER_CONFIG_DIR" -if compgen -G "$TMP_SERVER_CONFIG_DIR/*.deployment-config.json" >/dev/null; then - cp "$TMP_SERVER_CONFIG_DIR"/*.deployment-config.json "$SERVER_CONFIG_DIR"/ -fi - -cat > "${KIT_DIR}/README-INSTALL-KIT.txt" <<'EOF' -ActivityWatch DetMir Windows Install Kit - -Includes: -- windows/* (deploy scripts, collectors, common module, configs/examples) -- ansible/* (Windows and AW server playbooks, examples, inventory, tasks) -- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) -- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks) -- server-configs-192.168.100.18/* (working Windows/RDP config snapshots) - -Source: -- Local project snapshot at build time. -EOF - -python3 - <<'PY' -from pathlib import Path -import hashlib - -root = Path('install-kit-awindows-20260427-211240') -manifest = root / 'MANIFEST.txt' -files = sorted(p for p in root.rglob('*') if p.is_file() and p.name != 'MANIFEST.txt') -with manifest.open('w', encoding='utf-8') as handle: - for path in files: - digest = hashlib.sha256(path.read_bytes()).hexdigest() - handle.write(f"{digest} {path.as_posix()}\n") -PY - -rm -f "$ZIP_ARCHIVE" "$TAR_ARCHIVE" -zip -rq "$ZIP_ARCHIVE" "$KIT_DIR" -tar -czf "$TAR_ARCHIVE" "$KIT_DIR" +echo "ERROR: Rust install-kit builder not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p rebuild-install-kit" >&2 +exit 127 diff --git a/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh b/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh index 94b924a..c209127 100644 --- a/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh +++ b/install-kit-awindows-20260427-211240/scripts/validate_install_kit.sh @@ -32,75 +32,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -required_files=( - "$MANIFEST" - "$KIT_DIR/README-INSTALL-KIT.txt" - "$KIT_DIR/windows/deploy-ensemble.ps1" - "$KIT_DIR/windows/validate-deployment.ps1" - "$KIT_DIR/ansible/deploy_aw_windows.yml" - "$KIT_DIR/aw-server/install_aw_server.sh" - "$KIT_DIR/scripts/rebuild_install_kit.sh" - "$KIT_DIR/scripts/validate_install_kit.sh" - "$KIT_DIR/scripts/check_install_kit_vs_repo.sh" - "$KIT_DIR/scripts/quality-gate.sh" -) - -echo "[1/4] Required files presence" -for file in "${required_files[@]}"; do - [[ -f "$file" ]] || { echo "Missing required file: $file"; exit 1; } -done - -echo "[2/4] Manifest checksum verification" -sha256sum -c "$MANIFEST" >/dev/null - -echo "[3/4] Manifest completeness" -python3 - <<'PY' -from pathlib import Path -import sys -kit=Path('install-kit-awindows-20260427-211240') -manifest=kit/'MANIFEST.txt' -listed=[] -for line in manifest.read_text().splitlines(): - line=line.strip() - if not line: - continue - parts=line.split(' ',1) - if len(parts)!=2: - print(f'Invalid MANIFEST line: {line}') - sys.exit(1) - listed.append(parts[1]) -listed_set=set(listed) -actual_set={str(p) for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'} -missing=sorted(listed_set-actual_set) -extra=sorted(actual_set-listed_set) -if missing or extra: - print('Missing files listed in MANIFEST:', missing) - print('Files not listed in MANIFEST:', extra) - sys.exit(1) -print(f'MANIFEST complete: {len(actual_set)} files tracked') -PY - -echo "[4/4] Archive composition check" -python3 - <<'PY' -from pathlib import Path -import tarfile, zipfile, sys -kit_prefix='install-kit-awindows-20260427-211240/' -zip_path=Path('install-kit-awindows-20260427-211240.zip') -tar_path=Path('install-kit-awindows-20260427-211240.tar.gz') -if not zip_path.exists() or not tar_path.exists(): - print('Archives not found') - sys.exit(1) -with zipfile.ZipFile(zip_path) as z: - zip_files=sorted(i for i in z.namelist() if not i.endswith('/')) -with tarfile.open(tar_path, 'r:gz') as t: - tar_files=sorted(m.name for m in t.getmembers() if m.isfile()) -if zip_files != tar_files: - print('ZIP and TAR contents differ') - sys.exit(1) -if not all(f.startswith(kit_prefix) for f in zip_files): - print('Unexpected archive prefix layout') - sys.exit(1) -print(f'Archives match: {len(zip_files)} files') -PY - -echo "validate_install_kit: OK" +echo "ERROR: Rust validator not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p validate-install-kit" >&2 +exit 127 diff --git a/scripts/aggregate_dlp_events.py b/scripts/aggregate_dlp_events.py deleted file mode 100755 index 7be7760..0000000 --- a/scripts/aggregate_dlp_events.py +++ /dev/null @@ -1,517 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import sqlite3 -import sys -import urllib.error -import urllib.parse -import urllib.request -from collections.abc import Iterable -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Protocol, TypeAlias - - -JsonScalar: TypeAlias = str | int | float | bool | None -JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] - - -DEFAULT_BUCKET_PREFIXES = ("aw-file-operations_", "aw-dlp-incidents_") -DEFAULT_SQLITE_PATH = "data/dlp-events.sqlite3" -EVENT_COLUMNS = ( - "bucket_id", - "event_id", - "stream_type", - "hostname", - "username", - "event_ts", - "duration", - "operation", - "file_path", - "old_file_path", - "extension", - "archive_hint", - "rule_id", - "action", - "severity", - "signal_type", - "message", - "source", - "screenshot_path", - "raw_json", - "ingested_at", -) - - -@dataclass(frozen=True) -class Bucket: - id: str - type: str - client: str - hostname: str - - -@dataclass(frozen=True) -class AwEvent: - bucket_id: str - hostname: str - stream_type: str - event_id: str - timestamp: str - duration: float - data: dict[str, JsonValue] - - -class PsycopgConnection(Protocol): - def cursor(self): - ... - - def commit(self) -> None: - ... - - -def utc_now() -> datetime: - return datetime.now(tz=UTC) - - -def parse_timestamp(value: str) -> datetime: - normalized = value.replace("Z", "+00:00") - parsed = datetime.fromisoformat(normalized) - if parsed.tzinfo is None: - return parsed.replace(tzinfo=UTC) - return parsed.astimezone(UTC) - - -def format_aw_timestamp(value: datetime) -> str: - return value.astimezone(UTC).isoformat().replace("+00:00", "Z") - - -def load_state(path: Path) -> dict[str, str]: - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) - - -def save_state(path: Path, state: dict[str, str]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def normalize_base_url(base_url: str) -> str: - return base_url.rstrip("/") - - -def aw_get_json(base_url: str, path: str, timeout: int) -> JsonValue: - url = normalize_base_url(base_url) + path - request = urllib.request.Request(url, headers={"Accept": "application/json"}) - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.loads(response.read().decode("utf-8")) - - -def list_buckets(base_url: str, timeout: int) -> list[Bucket]: - payload = aw_get_json(base_url, "/buckets", timeout) - if not isinstance(payload, dict): - raise ValueError("ActivityWatch /buckets response must be a JSON object") - buckets: list[Bucket] = [] - for bucket_id, bucket_data in payload.items(): - if not isinstance(bucket_data, dict): - continue - buckets.append( - Bucket( - id=str(bucket_id), - type=str(bucket_data.get("type", "")), - client=str(bucket_data.get("client", "")), - hostname=str(bucket_data.get("hostname", "")), - ) - ) - return buckets - - -def bucket_stream_type(bucket: Bucket) -> str | None: - if bucket.id.startswith("aw-file-operations_") or bucket.type == "aw.file.operation": - return "file_operation" - if bucket.id.startswith("aw-dlp-incidents_") or bucket.type == "aw.dlp.incident": - return "dlp_incident" - return None - - -def select_buckets(buckets: Iterable[Bucket], prefixes: tuple[str, ...]) -> list[tuple[Bucket, str]]: - selected: list[tuple[Bucket, str]] = [] - for bucket in buckets: - stream_type = bucket_stream_type(bucket) - if stream_type and any(bucket.id.startswith(prefix) for prefix in prefixes): - selected.append((bucket, stream_type)) - return selected - - -def build_events_path(bucket_id: str, start: datetime, end: datetime, limit: int) -> str: - query = urllib.parse.urlencode( - { - "start": format_aw_timestamp(start), - "end": format_aw_timestamp(end), - "limit": str(limit), - } - ) - return f"/buckets/{urllib.parse.quote(bucket_id, safe='')}/events?{query}" - - -def event_key(bucket_id: str, timestamp: str, duration: float, data: dict[str, JsonValue]) -> str: - payload = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - return f"{bucket_id}|{timestamp}|{duration}|{payload}" - - -def fetch_bucket_events( - base_url: str, - bucket: Bucket, - stream_type: str, - start: datetime, - end: datetime, - limit: int, - timeout: int, -) -> list[AwEvent]: - payload = aw_get_json(base_url, build_events_path(bucket.id, start, end, limit), timeout) - if not isinstance(payload, list): - raise ValueError(f"ActivityWatch events response for {bucket.id} must be a JSON array") - events: list[AwEvent] = [] - for item in payload: - if not isinstance(item, dict): - continue - timestamp = str(item["timestamp"]) - duration = float(item.get("duration", 0) or 0) - data = item.get("data") or {} - if not isinstance(data, dict): - data = {"raw": data} - item_id = str(item.get("id") or event_key(bucket.id, timestamp, duration, data)) - events.append( - AwEvent( - bucket_id=bucket.id, - hostname=bucket.hostname or str(data.get("hostname") or ""), - stream_type=stream_type, - event_id=item_id, - timestamp=timestamp, - duration=duration, - data=data, - ) - ) - return events - - -def connect_sqlite(path: Path) -> sqlite3.Connection: - path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect(str(path)) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA synchronous=NORMAL") - connection.execute("PRAGMA foreign_keys=ON") - return connection - - -def ensure_schema(connection: sqlite3.Connection) -> None: - connection.executescript( - """ - create table if not exists dlp_events ( - id integer primary key autoincrement, - bucket_id text not null, - event_id text not null, - stream_type text not null, - hostname text not null, - username text, - event_ts text not null, - duration real not null default 0, - operation text, - file_path text, - old_file_path text, - extension text, - archive_hint integer not null default 0, - rule_id text, - action text, - severity text, - signal_type text, - message text, - source text, - screenshot_path text, - raw_json text not null, - ingested_at text not null, - unique (bucket_id, event_id) - ); - create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts); - create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts); - create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts); - create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts); - create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts); - - create view if not exists dlp_file_operations as - select * - from dlp_events - where stream_type = 'file_operation'; - - create view if not exists dlp_incidents as - select * - from dlp_events - where stream_type = 'dlp_incident'; - """ - ) - connection.commit() - - -def ensure_postgres_schema(connection: PsycopgConnection) -> None: - with connection.cursor() as cursor: - cursor.execute( - """ - create table if not exists dlp_events ( - id bigserial primary key, - bucket_id text not null, - event_id text not null, - stream_type text not null, - hostname text not null, - username text, - event_ts timestamptz not null, - duration double precision not null default 0, - operation text, - file_path text, - old_file_path text, - extension text, - archive_hint boolean not null default false, - rule_id text, - action text, - severity text, - signal_type text, - message text, - source text, - screenshot_path text, - raw_json jsonb not null, - ingested_at timestamptz not null, - unique (bucket_id, event_id) - ); - create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts); - create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts); - create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts); - create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts); - create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts); - - create or replace view dlp_file_operations as - select * - from dlp_events - where stream_type = 'file_operation'; - - create or replace view dlp_incidents as - select * - from dlp_events - where stream_type = 'dlp_incident'; - """ - ) - connection.commit() - - -def first_string(data: dict[str, JsonValue], keys: tuple[str, ...]) -> str | None: - for key in keys: - value = data.get(key) - if value is not None and str(value) != "": - return str(value) - return None - - -def bool_as_int(value: JsonValue) -> int: - if isinstance(value, bool): - return int(value) - if isinstance(value, str): - return int(value.lower() in {"1", "true", "yes", "y"}) - return int(bool(value)) - - -def event_row(event: AwEvent, ingested_at: str) -> tuple[JsonValue, ...]: - data = event.data - event_id = event.event_id or event_key(event.bucket_id, event.timestamp, event.duration, data) - return ( - event.bucket_id, - event_id, - event.stream_type, - event.hostname, - first_string(data, ("username", "user")), - event.timestamp, - event.duration, - first_string(data, ("operation",)), - first_string(data, ("path", "filePath")), - first_string(data, ("oldPath", "oldFilePath")), - first_string(data, ("extension",)), - bool_as_int(data.get("archiveHint")), - first_string(data, ("ruleId", "rule")), - first_string(data, ("action",)), - first_string(data, ("severity",)), - first_string(data, ("signalType",)), - first_string(data, ("message",)), - first_string(data, ("source",)), - first_string(data, ("screenshotPath", "capturePath", "artifactPath")), - json.dumps(data, ensure_ascii=False, sort_keys=True), - ingested_at, - ) - - -def insert_events(connection: sqlite3.Connection, events: Iterable[AwEvent]) -> int: - inserted = 0 - now = format_aw_timestamp(utc_now()) - for event in events: - cursor = connection.execute( - """ - insert or ignore into dlp_events ( - bucket_id, - event_id, - stream_type, - hostname, - username, - event_ts, - duration, - operation, - file_path, - old_file_path, - extension, - archive_hint, - rule_id, - action, - severity, - signal_type, - message, - source, - screenshot_path, - raw_json, - ingested_at - ) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - event_row(event, now), - ) - inserted += int(cursor.rowcount > 0) - connection.commit() - return inserted - - -def insert_postgres_events(dsn: str, events: Iterable[AwEvent]) -> int: - try: - import psycopg - except ImportError as exc: - raise SystemExit("PostgreSQL mode requires psycopg: python3 -m pip install 'psycopg[binary]'") from exc - - inserted = 0 - now = format_aw_timestamp(utc_now()) - columns = ", ".join(EVENT_COLUMNS) - placeholders = ", ".join(["%s"] * len(EVENT_COLUMNS)) - sql = f""" - insert into dlp_events ({columns}) - values ({placeholders}) - on conflict (bucket_id, event_id) do nothing - """ - with psycopg.connect(dsn) as connection: - ensure_postgres_schema(connection) - with connection.cursor() as cursor: - for event in events: - row = list(event_row(event, now)) - row[EVENT_COLUMNS.index("archive_hint")] = bool(row[EVENT_COLUMNS.index("archive_hint")]) - cursor.execute(sql, row) - inserted += int(cursor.rowcount > 0) - connection.commit() - return inserted - - -def get_start_time(args: argparse.Namespace, state: dict[str, str]) -> datetime: - if args.since: - return parse_timestamp(args.since) - if state.get("last_end"): - return parse_timestamp(state["last_end"]) - timedelta(seconds=args.overlap_seconds) - return utc_now() - timedelta(hours=args.lookback_hours) - - -def parse_prefixes(value: str) -> tuple[str, ...]: - prefixes = tuple(item.strip() for item in value.split(",") if item.strip()) - if not prefixes: - raise argparse.ArgumentTypeError("at least one bucket prefix is required") - return prefixes - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Aggregate AWatch-rus DLP buckets into a local warehouse database.") - parser.add_argument("--aw-url", default=os.environ.get("AW_URL", "http://127.0.0.1:5600/api/0")) - parser.add_argument("--postgres-dsn", default=os.environ.get("DLP_AGGREGATOR_POSTGRES_DSN")) - parser.add_argument("--sqlite-path", default=os.environ.get("DLP_AGGREGATOR_SQLITE_PATH", DEFAULT_SQLITE_PATH)) - parser.add_argument("--state-path", default=os.environ.get("DLP_AGGREGATOR_STATE_PATH", "data/dlp-aggregator-state.json")) - parser.add_argument("--bucket-prefixes", type=parse_prefixes, default=DEFAULT_BUCKET_PREFIXES) - parser.add_argument("--since", help="UTC ISO timestamp. Overrides saved state, for example 2026-05-02T00:00:00Z.") - parser.add_argument("--lookback-hours", type=int, default=24) - parser.add_argument("--overlap-seconds", type=int, default=60) - parser.add_argument("--limit", type=int, default=10000) - parser.add_argument("--timeout", type=int, default=15) - parser.add_argument("--dry-run", action="store_true") - return parser - - -def main() -> int: - args = build_parser().parse_args() - state_path = Path(args.state_path) - state = load_state(state_path) - start = get_start_time(args, state) - end = utc_now() - - buckets = select_buckets(list_buckets(args.aw_url, args.timeout), args.bucket_prefixes) - all_events: list[AwEvent] = [] - for bucket, stream_type in buckets: - all_events.extend(fetch_bucket_events(args.aw_url, bucket, stream_type, start, end, args.limit, args.timeout)) - - if args.dry_run: - print( - json.dumps( - { - "aw_url": args.aw_url, - "start": format_aw_timestamp(start), - "end": format_aw_timestamp(end), - "selected_buckets": [bucket.id for bucket, _stream_type in buckets], - "fetched_events": len(all_events), - }, - ensure_ascii=False, - indent=2, - ) - ) - return 0 - - if args.postgres_dsn: - target = "postgres" - target_path = args.postgres_dsn.split("@")[-1] - inserted = insert_postgres_events(args.postgres_dsn, all_events) - else: - target = "sqlite" - sqlite_path = Path(args.sqlite_path) - target_path = str(sqlite_path) - connection = connect_sqlite(sqlite_path) - try: - ensure_schema(connection) - inserted = insert_events(connection, all_events) - finally: - connection.close() - - state["last_end"] = format_aw_timestamp(end) - save_state(state_path, state) - print( - json.dumps( - { - "aw_url": args.aw_url, - "target": target, - "target_path": target_path, - "state_path": str(state_path), - "start": format_aw_timestamp(start), - "end": format_aw_timestamp(end), - "selected_buckets": len(buckets), - "fetched_events": len(all_events), - "inserted_events": inserted, - }, - ensure_ascii=False, - indent=2, - ) - ) - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except urllib.error.URLError as exc: - print(f"ActivityWatch API request failed: {exc}", file=sys.stderr) - raise SystemExit(2) diff --git a/scripts/build_dlp_ioc_from_hayabusa.sh b/scripts/build_dlp_ioc_from_hayabusa.sh index 21e120e..f73f729 100644 --- a/scripts/build_dlp_ioc_from_hayabusa.sh +++ b/scripts/build_dlp_ioc_from_hayabusa.sh @@ -43,8 +43,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -python3 "$REPO_ROOT/scripts/extract_ioc_from_sigma.py" \ - --rules-root "$RULES_ROOT" \ - --out-dir "$OUT_DIR" - -echo "IOC artifacts generated in: $OUT_DIR" +echo "ERROR: Rust extractor not found. Build it with: cd '$REPO_ROOT/adk-rust' && cargo build --release -p extract-ioc-from-sigma" >&2 +exit 127 diff --git a/scripts/check_install_kit_vs_repo.sh b/scripts/check_install_kit_vs_repo.sh index ff2600b..a6db8d7 100755 --- a/scripts/check_install_kit_vs_repo.sh +++ b/scripts/check_install_kit_vs_repo.sh @@ -24,59 +24,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -python3 - <<'PY' -from pathlib import Path -import hashlib -import sys - -root=Path('.') -kit=Path('install-kit-awindows-20260427-211240') -if not kit.exists(): - raise SystemExit('Install kit directory not found') - -def sha(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - -all_compared=[] -mismatches=[] -missing_in_repo=[] -allowed_kit_only_prefixes=('server-configs-' ,) -allowed_kit_only_files={'README-INSTALL-KIT.txt'} - -for kp in sorted(p for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'): - rel=kp.relative_to(kit) - rp=root/rel - if not rp.exists(): - rel_str=str(rel) - if rel_str in allowed_kit_only_files or rel_str.startswith(allowed_kit_only_prefixes): - continue - missing_in_repo.append(str(rel)) - continue - all_compared.append(str(rel)) - if sha(kp)!=sha(rp): - mismatches.append(str(rel)) - -ps_mismatches=[ - p for p in mismatches - if p.startswith('windows/') and p.endswith(('.ps1', '.psm1', '.psd1')) -] - -print(f'Compared files: {len(all_compared)}') -print(f'Missing in repo: {len(missing_in_repo)}') -print(f'Mismatched content: {len(mismatches)}') -if missing_in_repo: - print('--- Missing in repo ---') - for p in missing_in_repo: - print(p) -if mismatches: - print('--- Mismatches ---') - for p in mismatches: - print(p) -print(f'PowerShell mismatches: {len(ps_mismatches)}') -if ps_mismatches: - print('--- PowerShell mismatches ---') - for p in ps_mismatches: - print(p) -if missing_in_repo or mismatches: - sys.exit(1) -PY +echo "ERROR: Rust checker not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p check-install-kit-vs-repo" >&2 +exit 127 diff --git a/scripts/dlp-admin-cli.py b/scripts/dlp-admin-cli.py deleted file mode 100644 index 46900f6..0000000 --- a/scripts/dlp-admin-cli.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -from datetime import UTC, datetime, timedelta -from urllib import request -from urllib.parse import quote - - -def get_json(url: str): - with request.urlopen(url, timeout=30) as r: - return json.loads(r.read().decode("utf-8")) - - -def send_json(url: str, method: str, payload: dict | None = None): - body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8") - req = request.Request(url, data=body, method=method, headers={"Content-Type": "application/json"}) - with request.urlopen(req, timeout=30) as r: - raw = r.read().decode("utf-8") - return json.loads(raw) if raw else {} - - -def parse_iso(ts: str | None) -> datetime | None: - if not ts: - return None - try: - return datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone(UTC) - except ValueError: - return None - - -def main() -> None: - p = argparse.ArgumentParser(description="AWatch DLP admin CLI") - p.add_argument("--policy-server", default="http://127.0.0.1:5601") - p.add_argument("--case-server", default="http://127.0.0.1:5602") - p.add_argument("--aw-server", default="http://127.0.0.1:5600") - sub = p.add_subparsers(dest="cmd", required=True) - - policies = sub.add_parser("policies") - policies_sub = policies.add_subparsers(dest="policies_cmd", required=True) - policies_sub.add_parser("list") - policies_sub.add_parser("active") - - incidents = sub.add_parser("incidents") - incidents_sub = incidents.add_subparsers(dest="incidents_cmd", required=True) - incidents_list = incidents_sub.add_parser("list") - incidents_list.add_argument("--host") - incidents_list.add_argument("--severity") - incidents_list.add_argument("--limit", type=int, default=100) - incidents_list.add_argument("--since-hours", type=int, default=24) - - cases = sub.add_parser("cases") - cases_sub = cases.add_subparsers(dest="cases_cmd", required=True) - cases_list = cases_sub.add_parser("list") - cases_list.add_argument("--host") - cases_list.add_argument("--status") - cases_list.add_argument("--limit", type=int, default=100) - cases_create = cases_sub.add_parser("create") - cases_create.add_argument("--incident-id", required=True) - cases_create.add_argument("--title", required=True) - cases_create.add_argument("--host") - cases_create.add_argument("--severity", default="medium") - - health = sub.add_parser("health") - health_sub = health.add_subparsers(dest="health_cmd", required=True) - health_sub.add_parser("check") - - args = p.parse_args() - - if args.cmd == "policies" and args.policies_cmd == "list": - data = get_json(f"{args.policy_server}/api/0/dlp/policies") - print(json.dumps(data, ensure_ascii=False, indent=2)) - return - - if args.cmd == "policies" and args.policies_cmd == "active": - data = get_json(f"{args.policy_server}/api/0/dlp/policies/active") - print(json.dumps(data, ensure_ascii=False, indent=2)) - return - - if args.cmd == "incidents" and args.incidents_cmd == "list": - bucket_map = get_json(f"{args.aw_server}/api/0/buckets") - if not isinstance(bucket_map, dict): - print("[]") - return - - bucket_ids = [x for x in bucket_map.keys() if str(x).startswith("aw-dlp-incidents_")] - if args.host: - bucket_ids = [x for x in bucket_ids if str(x).endswith("_" + args.host)] - - after = datetime.now(UTC) - timedelta(hours=max(1, args.since_hours)) - rows = [] - for bucket_id in sorted(bucket_ids): - encoded = quote(str(bucket_id), safe="") - events = get_json(f"{args.aw_server}/api/0/buckets/{encoded}/events?limit={max(1, args.limit)}") - if not isinstance(events, list): - continue - for ev in events: - if not isinstance(ev, dict): - continue - ts = parse_iso(ev.get("timestamp")) - if ts is None or ts < after: - continue - data = ev.get("data") or {} - if args.severity and str((data or {}).get("severity", "")).lower() != args.severity.lower(): - continue - rows.append(ev) - print(json.dumps(rows, ensure_ascii=False, indent=2)) - return - - if args.cmd == "cases" and args.cases_cmd == "list": - query = [] - if args.host: - query.append(f"host={quote(args.host, safe='')}") - if args.status: - query.append(f"status={quote(args.status, safe='')}") - query.append(f"limit={max(1, args.limit)}") - data = get_json(f"{args.case_server}/api/0/dlp/cases?{'&'.join(query)}") - print(json.dumps(data, ensure_ascii=False, indent=2)) - return - - if args.cmd == "cases" and args.cases_cmd == "create": - payload = { - "incident_id": args.incident_id, - "title": args.title, - "host": args.host, - "severity": args.severity, - "evidence": {"source": "dlp-admin-cli"}, - } - data = send_json(f"{args.case_server}/api/0/dlp/cases", "POST", payload) - print(json.dumps(data, ensure_ascii=False, indent=2)) - return - - if args.cmd == "health" and args.health_cmd == "check": - out = {} - try: - out["policy"] = get_json(f"{args.policy_server}/healthz") - except Exception as exc: - out["policy"] = {"status": "error", "error": str(exc)} - - try: - out["cases"] = get_json(f"{args.case_server}/health") - except Exception as exc: - out["cases"] = {"status": "error", "error": str(exc)} - - try: - out["aw"] = get_json(f"{args.aw_server}/api/0/info") - except Exception as exc: - out["aw"] = {"status": "error", "error": str(exc)} - - print(json.dumps(out, ensure_ascii=False, indent=2)) - return - - raise SystemExit("unsupported command") - - -if __name__ == "__main__": - main() diff --git a/scripts/dlp-health-check.py b/scripts/dlp-health-check.py deleted file mode 100644 index 260b364..0000000 --- a/scripts/dlp-health-check.py +++ /dev/null @@ -1,1023 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -import time -from dataclasses import dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any -from urllib import request - - -def _env(name: str, default: str) -> str: - value = os.environ.get(name) - return value if value not in (None, "") else default - - -def _http_json(url: str, timeout: int = 15, attempts: int = 2, backoff_seconds: float = 0.5) -> Any: - last_exc: Exception | None = None - for attempt in range(max(1, attempts)): - try: - with request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode("utf-8")) - except Exception as exc: - last_exc = exc - if attempt + 1 >= max(1, attempts): - break - time.sleep(backoff_seconds * (2**attempt)) - assert last_exc is not None - raise last_exc - - -def _parse_ts(value: str | None) -> datetime | None: - if not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) - except ValueError: - return None - - -def _now_utc() -> datetime: - return datetime.now(UTC) - - -def _age_seconds(ts: datetime | None, now: datetime) -> int | None: - if ts is None: - return None - return max(0, int((now - ts).total_seconds())) - - -def _int_or_zero(value: Any) -> int: - try: - return int(value) - except (TypeError, ValueError): - return 0 - - -def _path_tail(value: Any, parts: int = 2) -> str: - text = str(value or "").replace("\\", "/").strip() - if not text: - return "" - tokens = [item for item in text.split("/") if item] - return "/".join(tokens[-parts:]) if tokens else text - - -def _text_excerpt(value: Any, limit: int = 120) -> str: - text = " ".join(str(value or "").split()) - if len(text) <= limit: - return text - return text[: max(0, limit - 1)].rstrip() + "…" - - -def _load_counter_state(path: Path) -> dict[str, Any]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except FileNotFoundError: - return {"counters": {}} - except Exception: - return {"counters": {}} - if not isinstance(payload, dict): - return {"counters": {}} - counters = payload.get("counters") - if not isinstance(counters, dict): - payload["counters"] = {} - return payload - - -def _save_counter_state(path: Path, state: dict[str, Any]) -> str | None: - try: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.parent / f".{path.name}.{os.getpid()}.tmp" - tmp_path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - os.replace(tmp_path, path) - return None - except Exception as exc: - try: - path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return None - except Exception: - return str(exc) - - -def _counter_delta(counter_state: dict[str, Any] | None, key: str, current_value: int) -> tuple[int | None, int]: - if counter_state is None: - return None, current_value - counters = counter_state.setdefault("counters", {}) - if not isinstance(counters, dict): - counters = {} - counter_state["counters"] = counters - previous: int | None = None - try: - previous = int(counters[key]) - except (KeyError, TypeError, ValueError): - previous = None - counters[key] = current_value - if previous is None or current_value < previous: - return previous, 0 - return previous, current_value - previous - - -def _run_systemctl(*args: str) -> tuple[int, str]: - proc = subprocess.run( - ["systemctl", *args], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - return proc.returncode, proc.stdout.strip() - - -@dataclass -class CheckResult: - name: str - status: str - summary: str - details: dict[str, Any] - - -class HealthReport: - def __init__(self) -> None: - self.results: list[CheckResult] = [] - - def add(self, name: str, status: str, summary: str, **details: Any) -> None: - self.results.append(CheckResult(name=name, status=status, summary=summary, details=details)) - - @property - def ok(self) -> bool: - return not any(item.status == "fail" for item in self.results) - - def as_dict(self) -> dict[str, Any]: - counts = {"ok": 0, "warn": 0, "fail": 0} - for item in self.results: - counts[item.status] = counts.get(item.status, 0) + 1 - return { - "ok": self.ok, - "counts": counts, - "results": [ - { - "name": item.name, - "status": item.status, - "summary": item.summary, - "details": item.details, - } - for item in self.results - ], - } - - def render_text(self) -> str: - icon = {"ok": "OK", "warn": "WARN", "fail": "FAIL"} - lines = ["=== DLP Health Check ===", f"Timestamp: {_now_utc().isoformat().replace('+00:00', 'Z')}", ""] - for item in self.results: - lines.append(f"[{icon.get(item.status, item.status.upper())}] {item.name}: {item.summary}") - if item.details: - lines.append(f" details: {json.dumps(item.details, ensure_ascii=False, sort_keys=True)}") - lines.append("") - lines.append(f"Overall: {'OK' if self.ok else 'FAIL'}") - return "\n".join(lines) - - -def check_http_endpoint(report: HealthReport, name: str, url: str) -> None: - try: - payload = _http_json(url) - report.add(name, "ok", f"HTTP endpoint responded", url=url, payload=payload) - except Exception as exc: - report.add(name, "fail", f"HTTP endpoint failed: {exc}", url=url) - - -def check_systemd_unit(report: HealthReport, unit: str, kind: str) -> None: - active_rc, active_out = _run_systemctl("is-active", unit) - enabled_rc, enabled_out = _run_systemctl("is-enabled", unit) - exists_rc, _ = _run_systemctl("status", unit) - if exists_rc != 0 and active_rc != 0 and enabled_rc != 0: - report.add(f"systemd:{unit}", "warn", "unit not installed", kind=kind) - return - - if active_rc == 0 and enabled_rc == 0: - report.add(f"systemd:{unit}", "ok", "active and enabled", kind=kind) - return - - report.add( - f"systemd:{unit}", - "fail", - "unit is not active/enabled", - kind=kind, - active=active_out or str(active_rc), - enabled=enabled_out or str(enabled_rc), - ) - - -def _latest_bucket_ts(api_base: str, bucket_id: str, bucket_meta: dict[str, Any]) -> datetime | None: - meta = bucket_meta.get("metadata") or {} - ts = _parse_ts(meta.get("end")) - if ts is not None: - return ts - try: - events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit=1") - except Exception: - return None - if isinstance(events, list) and events: - return _parse_ts(events[0].get("timestamp")) - return None - - -def _bucket_suffix(bucket_id: str, prefix: str) -> str: - return bucket_id[len(prefix):] if bucket_id.startswith(prefix) else bucket_id - - -def check_bucket_group( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - name: str, - prefix: str, - max_age_seconds: int, - severity_if_missing: str = "fail", - severity_if_stale: str = "fail", -) -> None: - now = _now_utc() - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - if not matched: - report.add( - f"buckets:{name}", - severity_if_missing, - f"no buckets matched prefix {prefix}", - prefix=prefix, - ) - return - - stale: list[dict[str, Any]] = [] - unknown: list[str] = [] - ages: dict[str, int] = {} - for bucket_id in matched: - ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) - age = _age_seconds(ts, now) - if age is None: - unknown.append(bucket_id) - continue - ages[bucket_id] = age - if age > max_age_seconds: - stale.append({"bucket": bucket_id, "age_seconds": age}) - - status = "ok" - summary = f"{len(matched)} buckets, freshest ok" - if stale: - status = severity_if_stale - summary = f"{len(stale)} stale buckets" - elif unknown: - status = "warn" - summary = f"{len(unknown)} buckets without timestamp" - - report.add( - f"buckets:{name}", - status, - summary, - prefix=prefix, - max_age_seconds=max_age_seconds, - bucket_count=len(matched), - max_observed_age_seconds=max(ages.values()) if ages else None, - stale=stale, - unknown=unknown, - ) - - -def check_incident_buckets( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - max_age_seconds: int, -) -> None: - now = _now_utc() - prefix = "aw-dlp-incidents_" - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - - if not matched: - report.add( - "buckets:incidents", - "ok", - "no incident buckets yet", - prefix=prefix, - bucket_count=0, - ) - return - - ages: dict[str, int] = {} - unknown: list[str] = [] - stale: list[dict[str, Any]] = [] - for bucket_id in matched: - ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) - age = _age_seconds(ts, now) - if age is None: - unknown.append(bucket_id) - continue - ages[bucket_id] = age - if age > max_age_seconds: - stale.append({"bucket": bucket_id, "age_seconds": age}) - - if stale and not unknown: - report.add( - "buckets:incidents", - "ok", - "no recent incidents", - prefix=prefix, - bucket_count=len(matched), - max_age_seconds=max_age_seconds, - max_observed_age_seconds=max(ages.values()) if ages else None, - stale=stale, - unknown=[], - ) - return - - status = "ok" if not unknown else "warn" - summary = "incident buckets healthy" if not unknown else f"{len(unknown)} incident buckets without timestamp" - report.add( - "buckets:incidents", - status, - summary, - prefix=prefix, - bucket_count=len(matched), - max_age_seconds=max_age_seconds, - max_observed_age_seconds=max(ages.values()) if ages else None, - stale=stale, - unknown=unknown, - ) - - -def check_incident_runtime( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - sample_limit: int = 20, -) -> None: - now = _now_utc() - prefix = "aw-dlp-incidents_" - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - if not matched: - report.add("incident-runtime", "ok", "no incident buckets to sample", bucket_count=0) - return - - if sample_limit <= 0: - metadata = [] - for bucket_id in matched: - ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) - metadata.append( - { - "bucket": bucket_id, - "end": ts.isoformat().replace("+00:00", "Z") if ts else None, - "age_seconds": _age_seconds(ts, now), - } - ) - report.add( - "incident-runtime", - "ok", - "incident event sampling disabled", - bucket_count=len(matched), - sample_limit=sample_limit, - metadata=metadata, - ) - return - - sampled: list[dict[str, Any]] = [] - latest_incidents: list[dict[str, Any]] = [] - read_failed: list[dict[str, str]] = [] - totals = { - "sampled_events": 0, - "real_incidents": 0, - "self_tests": 0, - } - severity_counts: dict[str, int] = {} - action_counts: dict[str, int] = {} - rule_counts: dict[str, int] = {} - - for bucket_id in matched: - try: - events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}", timeout=5, attempts=1) - except Exception as exc: - read_failed.append({"bucket": bucket_id, "error": str(exc)}) - continue - if not isinstance(events, list): - read_failed.append({"bucket": bucket_id, "error": "events response is not a list"}) - continue - - bucket_summary = { - "bucket": bucket_id, - "sampled_events": len(events), - "real_incidents": 0, - "self_tests": 0, - } - sampled.append(bucket_summary) - totals["sampled_events"] += len(events) - - for event in events: - data = event.get("data") or {} - if not isinstance(data, dict): - continue - - signal_type = str(data.get("signalType") or "").strip() - source = str(data.get("source") or "").strip() - rule_id = str(data.get("ruleId") or data.get("rule_id") or "").strip() - is_self_test = signal_type == "self_test" or source == "self-test" or rule_id.startswith("selftest-") - if is_self_test: - bucket_summary["self_tests"] += 1 - totals["self_tests"] += 1 - continue - - ts = _parse_ts(event.get("timestamp")) - severity = str(data.get("severity") or "unknown").strip().lower() or "unknown" - action = str(data.get("action") or "unknown").strip().lower() or "unknown" - rule_key = rule_id or "unknown" - severity_counts[severity] = severity_counts.get(severity, 0) + 1 - action_counts[action] = action_counts.get(action, 0) + 1 - rule_counts[rule_key] = rule_counts.get(rule_key, 0) + 1 - bucket_summary["real_incidents"] += 1 - totals["real_incidents"] += 1 - latest_incidents.append( - { - "bucket": bucket_id, - "timestamp": event.get("timestamp"), - "age_seconds": _age_seconds(ts, now), - "ruleId": rule_id, - "severity": severity, - "action": action, - "username": str(data.get("username") or ""), - "hostname": str(data.get("hostname") or ""), - "source": source, - "message_excerpt": _text_excerpt(data.get("message")), - } - ) - - latest_incidents.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - status = "ok" - summary = f"{totals['real_incidents']} real incidents in sampled events" - if read_failed: - status = "warn" - summary = f"{len(read_failed)} incident buckets failed to sample" - elif totals["real_incidents"] == 0: - summary = "no real incidents in sampled events" - - report.add( - "incident-runtime", - status, - summary, - bucket_count=len(matched), - sample_limit=sample_limit, - totals=totals, - sampled=sampled, - severity_counts=severity_counts, - action_counts=action_counts, - rule_counts=dict(sorted(rule_counts.items(), key=lambda item: (-item[1], item[0]))[:10]), - latest_incidents=latest_incidents[:5], - read_failed=read_failed, - ) - - -def _worktime_activity_map(api_base: str, buckets: dict[str, Any], max_age_seconds: int) -> dict[str, dict[str, Any]]: - now = _now_utc() - activity: dict[str, dict[str, Any]] = {} - prefix = "aw-worktime-sessions_" - for bucket_id in sorted(key for key in buckets if key.startswith(prefix)): - host = _bucket_suffix(bucket_id, prefix) - latest_ts: datetime | None = None - latest_active = False - try: - events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit=20") - except Exception: - activity[host] = {"active": False, "age_seconds": None, "bucket": bucket_id} - continue - if isinstance(events, list): - for event in events: - ts = _parse_ts(event.get("timestamp")) - if ts is None: - continue - if latest_ts is None or ts > latest_ts: - latest_ts = ts - latest_active = bool((event.get("data") or {}).get("active")) - activity[host] = { - "active": bool(latest_ts and latest_active and (_age_seconds(latest_ts, now) or 0) <= max_age_seconds), - "age_seconds": _age_seconds(latest_ts, now), - "bucket": bucket_id, - } - return activity - - -def check_file_operations_buckets( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - max_age_seconds: int, - strict: bool, -) -> None: - now = _now_utc() - prefix = "aw-file-operations_" - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - worktime = _worktime_activity_map(api_base, buckets, max_age_seconds) - active_hosts = sorted(host for host, meta in worktime.items() if meta.get("active")) - matched_by_host = {_bucket_suffix(bucket_id, prefix): bucket_id for bucket_id in matched} - - ignored_unmanaged: list[str] = [] - ignored_inactive: list[str] = [] - missing_active: list[str] = [] - stale: list[dict[str, Any]] = [] - unknown: list[str] = [] - fresh: list[str] = [] - - for host, bucket_id in matched_by_host.items(): - if host not in worktime: - ignored_unmanaged.append(bucket_id) - continue - if host not in active_hosts: - ignored_inactive.append(bucket_id) - continue - ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) - age = _age_seconds(ts, now) - if age is None: - unknown.append(bucket_id) - continue - if age > max_age_seconds: - stale.append({"bucket": bucket_id, "age_seconds": age}) - else: - fresh.append(bucket_id) - - for host in active_hosts: - if host not in matched_by_host: - missing_active.append(host) - - if not active_hosts: - report.add( - "buckets:file-operations", - "ok", - "no active managed hosts require file-operations freshness", - active_hosts=[], - ignored_unmanaged=ignored_unmanaged, - ignored_inactive=ignored_inactive, - worktime_hosts=sorted(worktime), - ) - return - - status = "ok" - summary = f"{len(fresh)} active host buckets fresh" - if missing_active: - status = "fail" if strict else "warn" - summary = f"{len(missing_active)} active hosts missing file-operations buckets" - elif stale: - status = "fail" if strict else "warn" - summary = f"{len(stale)} active host buckets stale" - elif unknown: - status = "warn" - summary = f"{len(unknown)} active host buckets without timestamp" - - report.add( - "buckets:file-operations", - status, - summary, - active_hosts=active_hosts, - fresh=fresh, - stale=stale, - missing_active=missing_active, - unknown=unknown, - ignored_unmanaged=ignored_unmanaged, - ignored_inactive=ignored_inactive, - ) - - -def check_file_operations_runtime( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - sample_limit: int = 20, - queue_warn_depth: int = 100, - send_failure_warn_count: int = 1, - counter_state: dict[str, Any] | None = None, -) -> None: - now = _now_utc() - prefix = "aw-file-operations_" - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - if not matched: - report.add("file-operations-runtime", "warn", "no file-operations buckets to sample", bucket_count=0) - return - - sampled: list[dict[str, Any]] = [] - latest_operations: list[dict[str, Any]] = [] - latest_health: list[dict[str, Any]] = [] - warnings: list[dict[str, Any]] = [] - read_failed: list[dict[str, str]] = [] - - for bucket_id in matched: - try: - events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit={sample_limit}") - except Exception as exc: - read_failed.append({"bucket": bucket_id, "error": str(exc)}) - continue - if not isinstance(events, list): - read_failed.append({"bucket": bucket_id, "error": "events response is not a list"}) - continue - - operation_counts: dict[str, int] = {} - latest_health_event: dict[str, Any] | None = None - latest_health_ts: datetime | None = None - - for event in events: - data = event.get("data") or {} - if not isinstance(data, dict): - continue - ts = _parse_ts(event.get("timestamp")) - signal_type = str(data.get("signalType") or "") - operation = str(data.get("operation") or "") - - if signal_type == "collector_health": - if latest_health_event is None or (ts is not None and (latest_health_ts is None or ts > latest_health_ts)): - latest_health_event = event - latest_health_ts = ts - continue - - if operation: - operation_counts[operation] = operation_counts.get(operation, 0) + 1 - latest_operations.append( - { - "bucket": bucket_id, - "timestamp": event.get("timestamp"), - "age_seconds": _age_seconds(ts, now), - "operation": operation, - "username": str(data.get("username") or ""), - "hostname": str(data.get("hostname") or ""), - "extension": str(data.get("extension") or ""), - "archiveHint": bool(data.get("archiveHint")), - "path_tail": _path_tail(data.get("path")), - "size": _int_or_zero(data.get("size")), - } - ) - - bucket_sample = { - "bucket": bucket_id, - "sampled_events": len(events), - "operation_counts": operation_counts, - } - sampled.append(bucket_sample) - - if latest_health_event is None: - warnings.append({"bucket": bucket_id, "metric": "collector_health", "value": "missing_in_sample"}) - continue - - health_data = latest_health_event.get("data") or {} - send_failures = _int_or_zero(health_data.get("sendFailures")) - previous_send_failures, send_failures_delta = _counter_delta( - counter_state, - f"file-operations:{bucket_id}:sendFailures", - send_failures, - ) - health_item = { - "bucket": bucket_id, - "timestamp": latest_health_event.get("timestamp"), - "age_seconds": _age_seconds(latest_health_ts, now), - "queueDepth": _int_or_zero(health_data.get("queueDepth")), - "eventsEnqueued": _int_or_zero(health_data.get("eventsEnqueued")), - "eventsFlushed": _int_or_zero(health_data.get("eventsFlushed")), - "sendFailures": send_failures, - "sendFailuresPrevious": previous_send_failures, - "sendFailuresDelta": send_failures_delta, - "username": str(health_data.get("username") or ""), - "hostname": str(health_data.get("hostname") or ""), - "sessionId": _int_or_zero(health_data.get("sessionId")), - } - latest_health.append(health_item) - if health_item["queueDepth"] > queue_warn_depth: - warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": health_item["queueDepth"], "threshold": queue_warn_depth}) - if send_failure_warn_count > 0 and health_item["sendFailuresDelta"] >= send_failure_warn_count: - warnings.append( - { - "bucket": bucket_id, - "metric": "sendFailuresDelta", - "value": health_item["sendFailuresDelta"], - "current": health_item["sendFailures"], - "previous": health_item["sendFailuresPrevious"], - "threshold": send_failure_warn_count, - } - ) - - latest_operations.sort(key=lambda item: item.get("timestamp") or "", reverse=True) - status = "ok" - summary = f"{len(matched)} file-operations buckets sampled" - if read_failed: - status = "warn" - summary = f"{len(read_failed)} file-operations buckets failed to sample" - elif warnings: - status = "warn" - summary = "file-operations runtime counters outside expectations" - - report.add( - "file-operations-runtime", - status, - summary, - bucket_count=len(matched), - sample_limit=sample_limit, - sampled=sampled, - latest_health=latest_health, - latest_operations=latest_operations[:5], - warnings=warnings, - read_failed=read_failed, - thresholds={ - "queueDepth": queue_warn_depth, - "sendFailures": send_failure_warn_count, - }, - ) - - -def check_endpoint_signal_buckets( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - max_age_seconds: int, -) -> None: - now = _now_utc() - prefix = "aw-dlp-endpoint-signals_" - matched = sorted(bucket_id for bucket_id in buckets if bucket_id.startswith(prefix)) - worktime = _worktime_activity_map(api_base, buckets, max_age_seconds) - active_hosts = sorted(host for host, meta in worktime.items() if meta.get("active")) - matched_by_host = {_bucket_suffix(bucket_id, prefix): bucket_id for bucket_id in matched} - - ignored_unmanaged: list[str] = [] - ignored_inactive: list[str] = [] - missing_active: list[str] = [] - stale: list[dict[str, Any]] = [] - unknown: list[str] = [] - fresh: list[str] = [] - - for host, bucket_id in matched_by_host.items(): - if host not in worktime: - ignored_unmanaged.append(bucket_id) - continue - if host not in active_hosts: - ignored_inactive.append(bucket_id) - continue - ts = _latest_bucket_ts(api_base, bucket_id, buckets.get(bucket_id, {})) - age = _age_seconds(ts, now) - if age is None: - unknown.append(bucket_id) - continue - if age > max_age_seconds: - stale.append({"bucket": bucket_id, "age_seconds": age}) - else: - fresh.append(bucket_id) - - for host in active_hosts: - if host not in matched_by_host: - missing_active.append(host) - - if not active_hosts: - report.add( - "buckets:endpoint-signals", - "ok", - "no active managed hosts require endpoint-signals freshness", - active_hosts=[], - ignored_unmanaged=ignored_unmanaged, - ignored_inactive=ignored_inactive, - worktime_hosts=sorted(worktime), - ) - return - - status = "ok" - summary = f"{len(fresh)} active host buckets fresh" - if missing_active: - status = "fail" - summary = f"{len(missing_active)} active hosts missing endpoint-signals buckets" - elif stale: - status = "fail" - summary = f"{len(stale)} active host buckets stale" - elif unknown: - status = "warn" - summary = f"{len(unknown)} active host buckets without timestamp" - - report.add( - "buckets:endpoint-signals", - status, - summary, - active_hosts=active_hosts, - fresh=fresh, - stale=stale, - missing_active=missing_active, - unknown=unknown, - ignored_unmanaged=ignored_unmanaged, - ignored_inactive=ignored_inactive, - ) - - -def check_endpoint_self_test_metrics( - report: HealthReport, - api_base: str, - buckets: dict[str, Any], - queue_warn_depth: int = 100, - send_failure_warn_count: int = 1, - counter_state: dict[str, Any] | None = None, -) -> None: - now = _now_utc() - missing: list[str] = [] - latest_self_tests: list[dict[str, Any]] = [] - warnings: list[dict[str, Any]] = [] - expected = ("queueDepth", "eventsEnqueued", "eventsFlushed", "sendFailures") - for bucket_id in sorted(k for k in buckets if k.startswith("aw-dlp-endpoint-signals_")): - try: - events = _http_json(f"{api_base}/buckets/{bucket_id}/events?limit=20") - except Exception as exc: - report.add(f"endpoint-self-test:{bucket_id}", "warn", f"failed to read events: {exc}", bucket=bucket_id) - continue - latest_event: dict[str, Any] | None = None - latest_ts: datetime | None = None - if isinstance(events, list): - for event in events: - data = event.get("data") or {} - if data.get("signalType") != "self_test" or not all(key in data for key in expected): - continue - ts = _parse_ts(event.get("timestamp")) - if latest_event is None or (ts is not None and (latest_ts is None or ts > latest_ts)): - latest_event = event - latest_ts = ts - if latest_event is None: - missing.append(bucket_id) - continue - - data = latest_event.get("data") or {} - send_failures = _int_or_zero(data.get("sendFailures")) - previous_send_failures, send_failures_delta = _counter_delta( - counter_state, - f"endpoint-self-test:{bucket_id}:sendFailures", - send_failures, - ) - item = { - "bucket": bucket_id, - "timestamp": latest_event.get("timestamp"), - "age_seconds": _age_seconds(latest_ts, now), - "queueDepth": _int_or_zero(data.get("queueDepth")), - "eventsEnqueued": _int_or_zero(data.get("eventsEnqueued")), - "eventsFlushed": _int_or_zero(data.get("eventsFlushed")), - "sendFailures": send_failures, - "sendFailuresPrevious": previous_send_failures, - "sendFailuresDelta": send_failures_delta, - } - latest_self_tests.append(item) - if item["queueDepth"] > queue_warn_depth: - warnings.append({"bucket": bucket_id, "metric": "queueDepth", "value": item["queueDepth"], "threshold": queue_warn_depth}) - if send_failure_warn_count > 0 and item["sendFailuresDelta"] >= send_failure_warn_count: - warnings.append( - { - "bucket": bucket_id, - "metric": "sendFailuresDelta", - "value": item["sendFailuresDelta"], - "current": item["sendFailures"], - "previous": item["sendFailuresPrevious"], - "threshold": send_failure_warn_count, - } - ) - - if missing: - report.add( - "endpoint-self-test-metrics", - "warn", - "missing transport metrics in sampled self_test events", - buckets=missing, - latest_self_tests=latest_self_tests, - thresholds={ - "queueDepth": queue_warn_depth, - "sendFailures": send_failure_warn_count, - }, - ) - elif warnings: - report.add( - "endpoint-self-test-metrics", - "warn", - "endpoint transport counters outside thresholds", - latest_self_tests=latest_self_tests, - warnings=warnings, - thresholds={ - "queueDepth": queue_warn_depth, - "sendFailures": send_failure_warn_count, - }, - ) - else: - report.add( - "endpoint-self-test-metrics", - "ok", - "self_test transport metrics present", - latest_self_tests=latest_self_tests, - thresholds={ - "queueDepth": queue_warn_depth, - "sendFailures": send_failure_warn_count, - }, - ) - - -def check_compliance_reports(report: HealthReport, report_dir: Path, profiles: list[str], month: str) -> None: - missing: list[str] = [] - present: list[str] = [] - for profile in profiles: - for suffix in ("html", "json"): - path = report_dir / f"{profile}-{month}.{suffix}" - if path.exists(): - present.append(str(path)) - else: - missing.append(str(path)) - if missing: - report.add("compliance-reports", "fail", "missing expected compliance report artifacts", present=present, missing=missing) - else: - report.add("compliance-reports", "ok", "all expected compliance artifacts exist", present=present) - - -def main() -> int: - parser = argparse.ArgumentParser(description="AWatch DLP health check") - parser.add_argument("--aw-server", default=_env("AW_HEALTH_AW_SERVER", "http://127.0.0.1:5600")) - parser.add_argument("--policy-server", default=_env("AW_HEALTH_POLICY_SERVER", "http://127.0.0.1:5601")) - parser.add_argument("--case-server", default=_env("AW_HEALTH_CASE_SERVER", "http://127.0.0.1:5602")) - parser.add_argument("--max-age-seconds", type=int, default=int(_env("AW_HEALTH_MAX_AGE_SECONDS", "900"))) - parser.add_argument("--strict-fileops", action="store_true", default=_env("AW_HEALTH_STRICT_FILEOPS", "0").lower() in {"1", "true", "yes", "on"}) - parser.add_argument("--endpoint-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_QUEUE_WARN_DEPTH", "100"))) - parser.add_argument("--endpoint-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_ENDPOINT_SEND_FAILURE_WARN_COUNT", "1"))) - parser.add_argument("--fileops-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SAMPLE_LIMIT", "20"))) - parser.add_argument("--fileops-queue-warn-depth", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_QUEUE_WARN_DEPTH", "100"))) - parser.add_argument("--fileops-send-failure-warn-count", type=int, default=int(_env("AW_DLP_HEALTH_FILEOPS_SEND_FAILURE_WARN_COUNT", "1"))) - parser.add_argument("--incident-sample-limit", type=int, default=int(_env("AW_DLP_HEALTH_INCIDENT_SAMPLE_LIMIT", "0"))) - parser.add_argument("--state-dir", default=_env("AW_DLP_HEALTH_STATE_DIR", "/var/lib/activitywatch/health")) - parser.add_argument("--report-dir", default=_env("AW_DLP_COMPLIANCE_REPORT_DIR", "/opt/activitywatch/dlp-compliance/reports")) - parser.add_argument("--profiles", default=_env("AW_DLP_COMPLIANCE_PROFILES", "152-fz,pci-dss")) - parser.add_argument("--json", action="store_true") - args = parser.parse_args() - - report = HealthReport() - aw_api_base = args.aw_server.rstrip("/") + "/api/0" - counter_state_path = Path(args.state_dir) / "dlp-health-check-counters.json" - counter_state = _load_counter_state(counter_state_path) - - check_http_endpoint(report, "http:aw", f"{aw_api_base}/info") - check_http_endpoint(report, "http:policy", args.policy_server.rstrip("/") + "/healthz") - check_http_endpoint(report, "http:cases", args.case_server.rstrip("/") + "/health") - - for unit in ( - "activitywatch-server", - "aw-dlp-policy-engine.service", - "aw-dlp-case-management.service", - "aw-worktime-api.service", - ): - check_systemd_unit(report, unit, "service") - - for unit in ( - "aw-dlp-report-scheduler.timer", - "aw-dlp-syslog-forwarder.timer", - "aw-dlp-webhook-sender.timer", - "aw-dlp-cef-exporter.timer", - "activitywatch-dlp-aggregator.timer", - "aw-dlp-ioc-refresh.timer", - "aw-worktime-ui-bridge.timer", - ): - check_systemd_unit(report, unit, "timer") - - try: - buckets = _http_json(f"{aw_api_base}/buckets") - if not isinstance(buckets, dict): - raise RuntimeError("bucket list is not a dict") - report.add("aw:buckets-index", "ok", "bucket index loaded", total=len(buckets)) - check_endpoint_signal_buckets(report, aw_api_base, buckets, args.max_age_seconds) - check_file_operations_buckets(report, aw_api_base, buckets, args.max_age_seconds, args.strict_fileops) - check_file_operations_runtime( - report, - aw_api_base, - buckets, - args.fileops_sample_limit, - args.fileops_queue_warn_depth, - args.fileops_send_failure_warn_count, - counter_state, - ) - check_incident_buckets(report, aw_api_base, buckets, args.max_age_seconds * 24) - check_incident_runtime(report, aw_api_base, buckets, args.incident_sample_limit) - check_endpoint_self_test_metrics( - report, - aw_api_base, - buckets, - args.endpoint_queue_warn_depth, - args.endpoint_send_failure_warn_count, - counter_state, - ) - except Exception as exc: - report.add("aw:buckets-index", "fail", f"failed to inspect bucket index: {exc}") - - state_error = _save_counter_state(counter_state_path, counter_state) - if state_error: - report.add("state:counters", "warn", f"failed to save counter baseline: {state_error}", path=str(counter_state_path)) - - month = _now_utc().strftime("%Y-%m") - profiles = [x.strip() for x in args.profiles.split(",") if x.strip()] - check_compliance_reports(report, Path(args.report_dir), profiles, month) - - payload = report.as_dict() - if args.json: - print(json.dumps(payload, ensure_ascii=False, indent=2)) - else: - print(report.render_text()) - return 0 if payload["ok"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/extract_ioc_from_sigma.py b/scripts/extract_ioc_from_sigma.py deleted file mode 100644 index a9c27c2..0000000 --- a/scripts/extract_ioc_from_sigma.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -""" -Extract IOC-like indicators from Sigma YAML rules for DLP preload. - -Extracted fields: - - Image|endswith - - CommandLine|contains - - OriginalFileName - - Hashes|SHA256 (plus SHA256 values embedded in Hashes strings) -""" - -from __future__ import annotations - -import argparse -import csv -import json -import re -from pathlib import Path -from typing import Any - -import yaml - - -BASE_FIELDS = {"image", "commandline", "originalfilename", "hashes"} -SHA256_RE = re.compile(r"\b[a-fA-F0-9]{64}\b") - - -def split_key(key: str) -> tuple[str, list[str]]: - parts = [p.strip() for p in str(key).split("|") if p.strip()] - if not parts: - return "", [] - return parts[0].lower(), [p.lower() for p in parts[1:]] - - -def flatten_values(value: Any) -> list[str]: - if value is None: - return [] - if isinstance(value, str): - v = value.strip() - return [v] if v else [] - if isinstance(value, (int, float, bool)): - return [str(value)] - if isinstance(value, list): - out: list[str] = [] - for item in value: - out.extend(flatten_values(item)) - return out - if isinstance(value, dict): - out: list[str] = [] - for k, v in value.items(): - vals = flatten_values(v) - for vv in vals: - out.append(f"{k}:{vv}") - return out - return [] - - -def detect_ioc_type(base: str, ops: list[str], raw: str) -> str | None: - if base == "image" and "endswith" in ops: - return "process_image_endswith" - if base == "commandline" and "contains" in ops: - return "commandline_contains" - if base == "originalfilename": - return "original_filename" - if base == "hashes" and ("sha256" in ops or SHA256_RE.search(raw)): - return "sha256" - return None - - -def parse_sha256(raw: str) -> list[str]: - vals = SHA256_RE.findall(raw) - seen = set() - out = [] - for v in vals: - lv = v.lower() - if lv in seen: - continue - seen.add(lv) - out.append(lv) - return out - - -def walk(node: Any, *, rule_id: str, rule_title: str, source_file: str, out: list[dict[str, str]]) -> None: - if isinstance(node, dict): - for k, v in node.items(): - base, ops = split_key(str(k)) - if base in BASE_FIELDS: - for raw in flatten_values(v): - ioc_type = detect_ioc_type(base, ops, raw) - if not ioc_type: - continue - if ioc_type == "sha256": - for h in parse_sha256(raw): - out.append( - { - "ioc_type": "sha256", - "ioc_value": h, - "field": str(k), - "rule_id": rule_id, - "rule_title": rule_title, - "source_file": source_file, - } - ) - else: - out.append( - { - "ioc_type": ioc_type, - "ioc_value": raw, - "field": str(k), - "rule_id": rule_id, - "rule_title": rule_title, - "source_file": source_file, - } - ) - walk(v, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out) - elif isinstance(node, list): - for item in node: - walk(item, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out) - - -def extract_from_yaml(path: Path) -> list[dict[str, str]]: - try: - doc = yaml.safe_load(path.read_text(encoding="utf-8", errors="ignore")) - except Exception: - return [] - if not isinstance(doc, dict): - return [] - detection = doc.get("detection") - if detection is None: - return [] - rid = str(doc.get("id") or "") - title = str(doc.get("title") or "") - rows: list[dict[str, str]] = [] - walk(detection, rule_id=rid, rule_title=title, source_file=str(path), out=rows) - return rows - - -def dedupe(rows: list[dict[str, str]]) -> list[dict[str, str]]: - seen = set() - out = [] - for r in rows: - key = (r["ioc_type"], r["ioc_value"].lower(), r["field"]) - if key in seen: - continue - seen.add(key) - out.append(r) - return out - - -def write_json(path: Path, rows: list[dict[str, str]]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8") - - -def write_csv(path: Path, rows: list[dict[str, str]]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - fields = ["ioc_type", "ioc_value", "field", "rule_id", "rule_title", "source_file"] - with path.open("w", encoding="utf-8", newline="") as f: - w = csv.DictWriter(f, fieldnames=fields) - w.writeheader() - for row in rows: - w.writerow(row) - - -def sql_escape(s: str) -> str: - return s.replace("'", "''") - - -def write_sql(path: Path, rows: list[dict[str, str]], table_name: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as f: - f.write( - f"CREATE TABLE IF NOT EXISTS {table_name} (\n" - " id INTEGER PRIMARY KEY AUTOINCREMENT,\n" - " ioc_type TEXT NOT NULL,\n" - " ioc_value TEXT NOT NULL,\n" - " field TEXT,\n" - " rule_id TEXT,\n" - " rule_title TEXT,\n" - " source_file TEXT\n" - ");\n\n" - ) - for r in rows: - f.write( - f"INSERT INTO {table_name} (ioc_type, ioc_value, field, rule_id, rule_title, source_file) VALUES " - f"('{sql_escape(r['ioc_type'])}'," - f"'{sql_escape(r['ioc_value'])}'," - f"'{sql_escape(r['field'])}'," - f"'{sql_escape(r['rule_id'])}'," - f"'{sql_escape(r['rule_title'])}'," - f"'{sql_escape(r['source_file'])}');\n" - ) - - -def main() -> int: - ap = argparse.ArgumentParser(description="Extract IOC-like Sigma values for DLP preload.") - ap.add_argument("--rules-root", default="rules", help="Path to hayabusa-rules root") - ap.add_argument("--out-dir", default="ioc_export", help="Output directory") - ap.add_argument("--table-name", default="dlp_blacklist_ioc", help="SQL table name") - args = ap.parse_args() - - rules_root = Path(args.rules_root) - if not rules_root.exists(): - raise SystemExit(f"rules root not found: {rules_root}") - - yaml_files = [p for p in rules_root.rglob("*") if p.is_file() and p.suffix.lower() in {".yml", ".yaml"}] - all_rows: list[dict[str, str]] = [] - for yp in yaml_files: - all_rows.extend(extract_from_yaml(yp)) - - rows = dedupe(all_rows) - rows.sort(key=lambda r: (r["ioc_type"], r["ioc_value"].lower())) - - out_dir = Path(args.out_dir) - write_json(out_dir / "ioc_blacklist.json", rows) - write_csv(out_dir / "ioc_blacklist.csv", rows) - write_sql(out_dir / "ioc_blacklist.sql", rows, args.table_name) - - counts: dict[str, int] = {} - for r in rows: - counts[r["ioc_type"]] = counts.get(r["ioc_type"], 0) + 1 - - print(f"rules_scanned={len(yaml_files)}") - print(f"iocs_extracted={len(rows)}") - for k in sorted(counts): - print(f"{k}={counts[k]}") - print(f"json={out_dir / 'ioc_blacklist.json'}") - print(f"csv={out_dir / 'ioc_blacklist.csv'}") - print(f"sql={out_dir / 'ioc_blacklist.sql'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) - diff --git a/scripts/merge_aw_server_dbs.py b/scripts/merge_aw_server_dbs.py deleted file mode 100644 index 9f7408f..0000000 --- a/scripts/merge_aw_server_dbs.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import sqlite3 -import sys -from pathlib import Path - - -def maybe_exec_rust() -> None: - if os.environ.get("MERGE_AW_SERVER_DBS_FORCE_LEGACY") == "1": - return - script_path = Path(__file__).resolve() - repo_root = script_path.parent.parent if script_path.parent.name == "scripts" else None - candidates = [ - os.environ.get("MERGE_AW_SERVER_DBS_RUST"), - str(Path(os.environ.get("CARGO_TARGET_DIR", "")) / "release" / "merge-aw-server-dbs") - if os.environ.get("CARGO_TARGET_DIR") - else None, - str(repo_root / "adk-rust" / "target" / "release" / "merge-aw-server-dbs") - if repo_root - else None, - "/usr/local/bin/merge-aw-server-dbs", - ] - for candidate in candidates: - if candidate and os.path.isfile(candidate) and os.access(candidate, os.X_OK): - os.execv(candidate, [candidate, *sys.argv[1:]]) - - -maybe_exec_rust() - - -def connect(path: Path) -> sqlite3.Connection: - connection = sqlite3.connect(str(path)) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA synchronous=NORMAL") - return connection - - -def bucket_key(row: sqlite3.Row) -> tuple[str, str, str, str]: - return ( - str(row["name"]), - str(row["type"]), - str(row["client"]), - str(row["hostname"]), - ) - -def find_bucket_by_name(connection: sqlite3.Connection, name: str) -> int | None: - row = connection.execute( - "select rowid as bucketrow from buckets where name = ? order by rowid limit 1", - (name,), - ).fetchone() - return int(row["bucketrow"]) if row else None - - -def find_bucket_by_key(connection: sqlite3.Connection, name: str, type_: str, client: str, hostname: str) -> int | None: - row = connection.execute( - "select rowid as bucketrow from buckets where name = ? and type = ? and client = ? and hostname = ? order by rowid limit 1", - (name, type_, client, hostname), - ).fetchone() - return int(row["bucketrow"]) if row else None - - -def copy_sqlite_via_backup(src: Path, dst: Path) -> None: - """Create a consistent copy of an sqlite DB using the sqlite backup API. - - This avoids corrupt/inconsistent files if the source DB is live. - """ - # ensure parent exists - dst.parent.mkdir(parents=True, exist_ok=True) - # remove any existing tmp file - if dst.exists(): - dst.unlink() - src_conn = sqlite3.connect(str(src)) - dst_conn = sqlite3.connect(str(dst)) - try: - # perform online backup - src_conn.backup(dst_conn) - dst_conn.commit() - finally: - try: - src_conn.close() - finally: - dst_conn.close() - - -def ensure_parent(path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - - -def load_existing_events(connection: sqlite3.Connection, bucketrow: int) -> set[tuple[int, int, str]]: - cursor = connection.execute( - "select starttime, endtime, data from events where bucketrow = ?", - (bucketrow,), - ) - return {(int(start), int(end), str(data)) for start, end, data in cursor.fetchall()} - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--base", required=True) - parser.add_argument("--output", required=True) - parser.add_argument("--overlay") - args = parser.parse_args() - - base = Path(args.base) - output = Path(args.output) - overlay = Path(args.overlay) if args.overlay else None - - if not base.exists(): - raise SystemExit(f"Base DB not found: {base}") - - ensure_parent(output) - tmp_output = output.with_suffix(output.suffix + ".tmp") - # make a consistent copy of the base DB into tmp_output - copy_sqlite_via_backup(base, tmp_output) - - dest = connect(tmp_output) - dest.row_factory = sqlite3.Row - - inserted_buckets = 0 - inserted_events = 0 - - if overlay and overlay.exists(): - source = connect(overlay) - source.row_factory = sqlite3.Row - try: - source_buckets = source.execute( - "select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid" - ).fetchall() - - dest_bucket_map = { - bucket_key(row): row["bucketrow"] - for row in dest.execute( - "select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid" - ).fetchall() - } - dest_id_map = { - row["id"]: row["bucketrow"] - for row in dest.execute( - "select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid" - ).fetchall() - if row["id"] - } - - for src_bucket in source_buckets: - key = bucket_key(src_bucket) - src_id = src_bucket["id"] if "id" in src_bucket.keys() else None - dest_rowid = None - # Prefer exact id match if available - if src_id: - dest_rowid = dest_id_map.get(src_id) - if dest_rowid is None: - dest_rowid = dest_bucket_map.get(key) - if dest_rowid is None: - dest_rowid = find_bucket_by_name(dest, str(src_bucket["name"])) - if dest_rowid is not None: - dest.execute( - """ - UPDATE buckets - SET type = ?, client = ?, hostname = ?, created = ?, - data_deprecated = ?, data = ? - WHERE rowid = ? - """, - ( - src_bucket["type"], - src_bucket["client"], - src_bucket["hostname"], - src_bucket["created"], - src_bucket["data_deprecated"], - src_bucket["data"], - dest_rowid, - ), - ) - else: - cursor = dest.execute( - """ - INSERT INTO buckets (id, name, type, client, hostname, created, data_deprecated, data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - src_bucket["id"], - src_bucket["name"], - src_bucket["type"], - src_bucket["client"], - src_bucket["hostname"], - src_bucket["created"], - src_bucket["data_deprecated"], - src_bucket["data"], - ), - ) - dest_rowid = int(cursor.lastrowid) - inserted_buckets += 1 - - dest_bucket_map[key] = dest_rowid - if src_id: - dest_id_map[str(src_id)] = dest_rowid - - existing_events = load_existing_events(dest, dest_rowid) - for starttime, endtime, data in source.execute( - "select starttime, endtime, data from events where bucketrow = ? order by id", - (src_bucket["bucketrow"],), - ).fetchall(): - event_key = (int(starttime), int(endtime), str(data)) - if event_key in existing_events: - continue - dest.execute( - "insert into events (bucketrow, starttime, endtime, data) values (?, ?, ?, ?)", - (dest_rowid, int(starttime), int(endtime), str(data)), - ) - existing_events.add(event_key) - inserted_events += 1 - - dest.commit() - finally: - source.close() - - dest.close() - os.replace(tmp_output, output) - print( - json.dumps( - { - "base": str(base), - "overlay": str(overlay) if overlay else None, - "output": str(output), - "inserted_buckets": inserted_buckets, - "inserted_events": inserted_events, - }, - ensure_ascii=False, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/prod_backup_restore.sh b/scripts/prod_backup_restore.sh index 9780df2..bc3f50e 100644 --- a/scripts/prod_backup_restore.sh +++ b/scripts/prod_backup_restore.sh @@ -11,84 +11,19 @@ if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then set +a fi -if [[ "${1:-}" == "--apply-legacy" ]]; then - shift -else - for candidate in \ - "${PROD_BACKUP_RESTORE_RUST:-}" \ - "${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \ - "$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \ - "/usr/local/bin/prod-backup-restore"; do - if [[ -n "$candidate" && -x "$candidate" ]]; then - exec "$candidate" --root "$ROOT_DIR" "$@" - fi - done - cat >&2 <<'EOF' -prod_backup_restore.sh is destructive and now requires the Rust planner. +for candidate in \ + "${PROD_BACKUP_RESTORE_RUST:-}" \ + "${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \ + "$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \ + "/usr/local/bin/prod-backup-restore"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + exec "$candidate" --root "$ROOT_DIR" "$@" + fi +done + +cat >&2 </dev/null 2>&1 || { echo "missing sshpass" >&2; exit 127; } -command -v ansible-playbook >/dev/null 2>&1 || { echo "missing ansible-playbook" >&2; exit 127; } - -SERVER_HOST="${AW_SERVER_HOST:-10.10.10.13}" -SERVER_USER="${AW_SERVER_USER:-igor}" -TIMESTAMP="$(date +%Y%m%d-%H%M%S)" -REMOTE_BACKUP_DIR="/var/lib/activitywatch/backups/prod-restore-${TIMESTAMP}" -LEGACY_DB="/root/.local/share/activitywatch/aw-server-rust/sqlite.db" -TARGET_DB="/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db" -REMOTE_MERGE_SCRIPT="/tmp/merge_aw_server_dbs.py" - -ssh_remote() { - sshpass -p "$AW_SSH_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${SERVER_USER}@${SERVER_HOST}" "$@" -} - -scp_remote() { - sshpass -p "$AW_SSH_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@" -} - -scp_remote "${ROOT_DIR}/scripts/merge_aw_server_dbs.py" "${SERVER_USER}@${SERVER_HOST}:${REMOTE_MERGE_SCRIPT}" - -ssh_remote "sudo mkdir -p '${REMOTE_BACKUP_DIR}' && sudo chown root:root '${REMOTE_BACKUP_DIR}'" -ssh_remote "sudo test -f '${LEGACY_DB}'" -ssh_remote "sudo test -f '${TARGET_DB}'" -ssh_remote "sudo cp -a '${LEGACY_DB}' '${REMOTE_BACKUP_DIR}/legacy-root-sqlite.db' && sudo cp -a '${TARGET_DB}' '${REMOTE_BACKUP_DIR}/target-before-merge-sqlite.db'" -ssh_remote "sudo systemctl stop activitywatch-server.service || true" -ssh_remote "sudo python3 '${REMOTE_MERGE_SCRIPT}' --base '${LEGACY_DB}' --overlay '${TARGET_DB}' --output '${REMOTE_BACKUP_DIR}/sqlite.merged.db'" -ssh_remote "sudo install -o activitywatch -g activitywatch -m 0644 '${REMOTE_BACKUP_DIR}/sqlite.merged.db' '${TARGET_DB}'" - -ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml -ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_windows.yml -ansible-playbook -i ansible/inventory.ini ansible/post_validate_aw_windows.yml - -python3 - <<'PY' -import json, urllib.request -base = 'http://10.10.10.13:5600' -window_payload = { - 'timeperiods': ['2026-04-29T00:00:00+03:00/2026-04-29T23:59:59+03:00'], - 'query': [ - 'window_events = query_bucket(find_bucket("aw-watcher-window_SHARKON2025"));', - 'RETURN = window_events;' - ] -} -req = urllib.request.Request(base + '/api/0/query/', data=json.dumps(window_payload).encode(), method='POST', headers={'Content-Type': 'application/json', 'Origin': 'http://10.10.10.13:5600'}) -with urllib.request.urlopen(req) as response: - data = json.loads(response.read().decode()) -window_count = len(data[0]) if isinstance(data, list) and data else 0 -if window_count <= 0: - raise SystemExit('no historical window data restored for 2026-04-29') -with urllib.request.urlopen(base + '/api/0/settings/') as response: - settings = json.loads(response.read().decode()) -if settings.get('always_active_pattern') != 'aw-watcher-window': - raise SystemExit('always_active_pattern is not configured') -print(json.dumps({'restored_window_events_2026_04_29': window_count, 'always_active_pattern': settings.get('always_active_pattern')}, ensure_ascii=False)) -PY +exit 127 diff --git a/scripts/rebuild_install_kit.sh b/scripts/rebuild_install_kit.sh index 13936f0..001c39e 100644 --- a/scripts/rebuild_install_kit.sh +++ b/scripts/rebuild_install_kit.sh @@ -23,150 +23,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -KIT_DIR="install-kit-awindows-20260427-211240" -ZIP_ARCHIVE="${KIT_DIR}.zip" -TAR_ARCHIVE="${KIT_DIR}.tar.gz" -SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.18" -OLD_SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.21" -TMP_SERVER_CONFIG_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_SERVER_CONFIG_DIR"' EXIT - -copy_file() { - local src="$1" - local dest="$2" - mkdir -p "$(dirname "$dest")" - cp "$src" "$dest" -} - -sync_tree() { - local base="$1" - shift - for rel in "$@"; do - copy_file "$rel" "${KIT_DIR}/${rel}" - done -} - -if [[ -d "$OLD_SERVER_CONFIG_DIR" ]]; then - cp "$OLD_SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ -fi -if [[ -d "$SERVER_CONFIG_DIR" ]]; then - cp "$SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/ -fi - -rm -rf "${KIT_DIR}/ansible" "${KIT_DIR}/aw-server" "${KIT_DIR}/windows" "${KIT_DIR}/scripts" "${KIT_DIR}/server-configs-"* - -ansible_files=( - ansible/README.md - ansible/deploy_aw_pfsense_poller.yml - ansible/deploy_aw_server.yml - ansible/deploy_aw_windows.yml - ansible/group_vars/all.example.yml - ansible/group_vars/pfsense-poller.example.yml - ansible/group_vars/proxmox-matrix.example.yml - ansible/group_vars/proxmox.example.yml - ansible/group_vars/windows.example.yml - ansible/install_full_stack.yml - ansible/inventory.example.ini - ansible/provision_proxmox_ct_and_deploy_aw.yml - ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml - ansible/tasks/provision_ct_and_deploy_aw.yml -) - -aw_server_files=( - aw-server/activitywatch-server.service - aw-server/apply_webui_ru_patch.sh - aw-server/aw-host-groups.json - aw-server/aw-ru-patch.js - aw-server/aw-rus-healthd.py - aw-server/aw-rus-healthd.service - aw-server/aw-rus-healthd.timer - aw-server/aw-browser-smoke.service - aw-server/aw-browser-smoke.timer - aw-server/aw-slo-monitor.py - aw-server/aw-slo-monitor.service - aw-server/aw-slo-monitor.timer - aw-server/aw-server.env.example - aw-server/aw-sw-cleanup.js - aw-server/aw-worktime-api.py - aw-server/aw-worktime-api.service - aw-server/aw-worktime-prewarm.sh - aw-server/aw-worktime-prewarm.service - aw-server/aw-worktime-prewarm.timer - aw-server/aw-worktime-panel.js - aw-server/health-check.sh - aw-server/install_aw_server.sh - aw-server/settings/classes-worktime.json - aw-server/settings/views-default.json -) - -windows_files=( - windows/ActivityWatch.Windows.Common.psd1 - windows/ActivityWatch.Windows.Common.psm1 - windows/browser-domains-native-collector.ps1 - windows/deploy-domain-users.ps1 - windows/deploy-ensemble.ps1 - windows/deploy-single-user.ps1 - windows/AWatchRusCollectorGuardService.cs - windows/aw-collector-guard.ps1 - windows/install-collector-guard-service.ps1 - windows/dlp-endpoint-signals-collector.ps1 - windows/dlp-policy.example.json - windows/dlp-policy.native-cross-os.example.json - windows/email-outbound-collector.ps1 - windows/hardening-recovery.ps1 - windows/migrate-awatch-rus-paths.ps1 - windows/validate-deployment.ps1 - windows/web-category-rules.example.json - windows/worktime-session-collector.ps1 -) - -scripts_files=( - scripts/aw-webui-browser-smoke.mjs - scripts/aw-webui-browser-smoke.sh - scripts/check_install_kit_vs_repo.sh - scripts/quality-gate.sh - scripts/rebuild_install_kit.sh - scripts/validate_install_kit.sh - scripts/verify_innosetup_installer.sh -) - -sync_tree ansible "${ansible_files[@]}" -sync_tree aw-server "${aw_server_files[@]}" -sync_tree windows "${windows_files[@]}" -sync_tree scripts "${scripts_files[@]}" - -mkdir -p "$SERVER_CONFIG_DIR" -if compgen -G "$TMP_SERVER_CONFIG_DIR/*.deployment-config.json" >/dev/null; then - cp "$TMP_SERVER_CONFIG_DIR"/*.deployment-config.json "$SERVER_CONFIG_DIR"/ -fi - -cat > "${KIT_DIR}/README-INSTALL-KIT.txt" <<'EOF' -ActivityWatch DetMir Windows Install Kit - -Includes: -- windows/* (deploy scripts, collectors, common module, configs/examples) -- ansible/* (Windows and AW server playbooks, examples, inventory, tasks) -- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings) -- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks) -- server-configs-192.168.100.18/* (working Windows/RDP config snapshots) - -Source: -- Local project snapshot at build time. -EOF - -python3 - <<'PY' -from pathlib import Path -import hashlib - -root = Path('install-kit-awindows-20260427-211240') -manifest = root / 'MANIFEST.txt' -files = sorted(p for p in root.rglob('*') if p.is_file() and p.name != 'MANIFEST.txt') -with manifest.open('w', encoding='utf-8') as handle: - for path in files: - digest = hashlib.sha256(path.read_bytes()).hexdigest() - handle.write(f"{digest} {path.as_posix()}\n") -PY - -rm -f "$ZIP_ARCHIVE" "$TAR_ARCHIVE" -zip -rq "$ZIP_ARCHIVE" "$KIT_DIR" -tar -czf "$TAR_ARCHIVE" "$KIT_DIR" +echo "ERROR: Rust install-kit builder not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p rebuild-install-kit" >&2 +exit 127 diff --git a/scripts/test_dlp_health_check.py b/scripts/test_dlp_health_check.py deleted file mode 100644 index 13ad931..0000000 --- a/scripts/test_dlp_health_check.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -import importlib.util -import sys -from pathlib import Path - - -MODULE_PATH = Path(__file__).with_name("dlp-health-check.py") -SPEC = importlib.util.spec_from_file_location("dlp_health_check", MODULE_PATH) -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -def test_endpoint_signals_ok_when_no_active_managed_hosts(monkeypatch): - buckets = { - "aw-worktime-sessions_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}, - "aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T07:00:00Z"}}, - } - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:00:10Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url: [{"timestamp": "2026-05-30T12:00:00Z", "data": {"active": False}}] - if "aw-worktime-sessions_SHARKON2025/events" in url - else [], - ) - report = MODULE.HealthReport() - - MODULE.check_endpoint_signal_buckets(report, "http://127.0.0.1:5600/api/0", buckets, 900) - - result = report.results[0] - assert result.name == "buckets:endpoint-signals" - assert result.status == "ok" - assert result.summary == "no active managed hosts require endpoint-signals freshness" - assert result.details["ignored_inactive"] == ["aw-dlp-endpoint-signals_SHARKON2025"] - - -def test_endpoint_self_test_metrics_reports_transport_counters(monkeypatch): - buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url, **kwargs: [ - { - "timestamp": "2026-05-30T11:59:00Z", - "data": { - "signalType": "self_test", - "queueDepth": 8, - "eventsEnqueued": 100, - "eventsFlushed": 92, - "sendFailures": 0, - }, - }, - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "self_test", - "queueDepth": "3", - "eventsEnqueued": "120", - "eventsFlushed": "117", - "sendFailures": "0", - }, - }, - ], - ) - report = MODULE.HealthReport() - - MODULE.check_endpoint_self_test_metrics(report, "http://127.0.0.1:5600/api/0", buckets) - - result = report.results[0] - assert result.name == "endpoint-self-test-metrics" - assert result.status == "ok" - latest = result.details["latest_self_tests"][0] - assert latest["bucket"] == "aw-dlp-endpoint-signals_SHARKON2025" - assert latest["timestamp"] == "2026-05-30T12:00:30Z" - assert latest["age_seconds"] == 30 - assert latest["queueDepth"] == 3 - assert latest["eventsEnqueued"] == 120 - assert latest["eventsFlushed"] == 117 - assert latest["sendFailures"] == 0 - assert latest["sendFailuresDelta"] == 0 - - -def test_endpoint_self_test_metrics_warns_on_transport_counters(monkeypatch): - buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url, **kwargs: [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "self_test", - "queueDepth": 101, - "eventsEnqueued": 120, - "eventsFlushed": 19, - "sendFailures": 1, - }, - } - ], - ) - report = MODULE.HealthReport() - - MODULE.check_endpoint_self_test_metrics( - report, - "http://127.0.0.1:5600/api/0", - buckets, - queue_warn_depth=100, - send_failure_warn_count=1, - ) - - result = report.results[0] - assert result.status == "warn" - assert result.summary == "endpoint transport counters outside thresholds" - assert result.details["warnings"] == [ - {"bucket": "aw-dlp-endpoint-signals_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100}, - { - "bucket": "aw-dlp-endpoint-signals_SHARKON2025", - "metric": "sendFailuresDelta", - "value": 1, - "current": 1, - "previous": None, - "threshold": 1, - }, - ] - - -def test_file_operations_runtime_reports_health_and_latest_operations(monkeypatch): - buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url: [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "collector_health", - "queueDepth": "0", - "eventsEnqueued": "5", - "eventsFlushed": "5", - "sendFailures": "0", - "username": "USER1", - "hostname": "SHARKON2025", - "sessionId": "3", - }, - }, - { - "timestamp": "2026-05-30T12:00:20Z", - "data": { - "operation": "Created", - "path": "C:\\Users\\USER1\\Downloads\\report.zip", - "extension": ".zip", - "archiveHint": True, - "username": "USER1", - "hostname": "SHARKON2025", - "size": "42", - }, - }, - ], - ) - report = MODULE.HealthReport() - - MODULE.check_file_operations_runtime(report, "http://127.0.0.1:5600/api/0", buckets) - - result = report.results[0] - assert result.name == "file-operations-runtime" - assert result.status == "ok" - assert result.details["latest_health"][0]["queueDepth"] == 0 - assert result.details["latest_health"][0]["eventsEnqueued"] == 5 - assert result.details["latest_health"][0]["sendFailuresDelta"] == 0 - latest_op = result.details["latest_operations"][0] - assert latest_op["operation"] == "Created" - assert latest_op["extension"] == ".zip" - assert latest_op["archiveHint"] is True - assert latest_op["path_tail"] == "Downloads/report.zip" - assert latest_op["size"] == 42 - assert result.details["sampled"][0]["operation_counts"] == {"Created": 1} - - -def test_file_operations_runtime_warns_on_transport_counters(monkeypatch): - buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url: [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "collector_health", - "queueDepth": 101, - "eventsEnqueued": 5, - "eventsFlushed": 2, - "sendFailures": 1, - "username": "USER1", - "hostname": "SHARKON2025", - "sessionId": 3, - }, - } - ], - ) - report = MODULE.HealthReport() - - MODULE.check_file_operations_runtime( - report, - "http://127.0.0.1:5600/api/0", - buckets, - queue_warn_depth=100, - send_failure_warn_count=1, - ) - - result = report.results[0] - assert result.status == "warn" - assert result.summary == "file-operations runtime counters outside expectations" - assert result.details["warnings"] == [ - {"bucket": "aw-file-operations_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100}, - { - "bucket": "aw-file-operations_SHARKON2025", - "metric": "sendFailuresDelta", - "value": 1, - "current": 1, - "previous": None, - "threshold": 1, - }, - ] - - -def test_endpoint_send_failures_uses_delta_baseline(monkeypatch): - buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - state = {"counters": {}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - values = [12, 12, 13] - - def fake_http(url, **kwargs): - value = values.pop(0) - return [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "self_test", - "queueDepth": 0, - "eventsEnqueued": 120, - "eventsFlushed": 117, - "sendFailures": value, - }, - } - ] - - monkeypatch.setattr(MODULE, "_http_json", fake_http) - - first = MODULE.HealthReport() - MODULE.check_endpoint_self_test_metrics(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert first.results[0].status == "ok" - assert first.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] is None - assert first.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0 - - second = MODULE.HealthReport() - MODULE.check_endpoint_self_test_metrics(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert second.results[0].status == "ok" - assert second.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] == 12 - assert second.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0 - - third = MODULE.HealthReport() - MODULE.check_endpoint_self_test_metrics(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert third.results[0].status == "warn" - assert third.results[0].details["warnings"] == [ - { - "bucket": "aw-dlp-endpoint-signals_SHARKON2025", - "metric": "sendFailuresDelta", - "value": 1, - "current": 13, - "previous": 12, - "threshold": 1, - } - ] - - -def test_file_operations_send_failures_uses_delta_baseline(monkeypatch): - buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - state = {"counters": {}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - values = [28, 28, 29] - - def fake_http(url, **kwargs): - value = values.pop(0) - return [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "signalType": "collector_health", - "queueDepth": 0, - "eventsEnqueued": 120, - "eventsFlushed": 117, - "sendFailures": value, - }, - } - ] - - monkeypatch.setattr(MODULE, "_http_json", fake_http) - - first = MODULE.HealthReport() - MODULE.check_file_operations_runtime(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert first.results[0].status == "ok" - assert first.results[0].details["latest_health"][0]["sendFailuresPrevious"] is None - assert first.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0 - - second = MODULE.HealthReport() - MODULE.check_file_operations_runtime(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert second.results[0].status == "ok" - assert second.results[0].details["latest_health"][0]["sendFailuresPrevious"] == 28 - assert second.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0 - - third = MODULE.HealthReport() - MODULE.check_file_operations_runtime(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state) - assert third.results[0].status == "warn" - assert third.results[0].details["warnings"] == [ - { - "bucket": "aw-file-operations_SHARKON2025", - "metric": "sendFailuresDelta", - "value": 1, - "current": 29, - "previous": 28, - "threshold": 1, - } - ] - - -def test_incident_runtime_reports_counts_and_latest_real_incidents(monkeypatch): - buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - monkeypatch.setattr( - MODULE, - "_http_json", - lambda url, **kwargs: [ - { - "timestamp": "2026-05-30T12:00:30Z", - "data": { - "ruleId": "usb-archive-copy", - "severity": "high", - "action": "alert", - "message": "Archive copied to removable device with a long message that should not leak full raw content.", - "username": "USER1", - "hostname": "SHARKON2025", - "source": "endpoint", - }, - }, - { - "timestamp": "2026-05-30T12:00:00Z", - "data": { - "ruleId": "selftest-dlp-incident", - "severity": "low", - "action": "alert", - "message": "Self-test DLP incident from validation", - "signalType": "self_test", - "source": "self-test", - }, - }, - ], - ) - report = MODULE.HealthReport() - - MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20) - - result = report.results[0] - assert result.name == "incident-runtime" - assert result.status == "ok" - assert result.summary == "1 real incidents in sampled events" - assert result.details["totals"] == {"sampled_events": 2, "real_incidents": 1, "self_tests": 1} - assert result.details["severity_counts"] == {"high": 1} - assert result.details["action_counts"] == {"alert": 1} - assert result.details["rule_counts"] == {"usb-archive-copy": 1} - latest = result.details["latest_incidents"][0] - assert latest["age_seconds"] == 30 - assert latest["ruleId"] == "usb-archive-copy" - assert latest["message_excerpt"].endswith(".") - - -def test_incident_runtime_defaults_to_metadata_without_event_sampling(monkeypatch): - buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z")) - - def fail_if_called(url, **kwargs): - raise AssertionError("event sampling should stay disabled") - - monkeypatch.setattr(MODULE, "_http_json", fail_if_called) - report = MODULE.HealthReport() - - MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=0) - - result = report.results[0] - assert result.name == "incident-runtime" - assert result.status == "ok" - assert result.summary == "incident event sampling disabled" - assert result.details["metadata"] == [ - { - "bucket": "aw-dlp-incidents_SHARKON2025", - "end": "2026-05-30T12:00:00Z", - "age_seconds": 60, - } - ] - - -def test_incident_runtime_warns_on_read_failure(monkeypatch): - buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}} - - def fail_http(url, **kwargs): - raise RuntimeError("timeout") - - monkeypatch.setattr(MODULE, "_http_json", fail_http) - report = MODULE.HealthReport() - - MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20) - - result = report.results[0] - assert result.name == "incident-runtime" - assert result.status == "warn" - assert result.summary == "1 incident buckets failed to sample" - assert result.details["read_failed"] == [{"bucket": "aw-dlp-incidents_SHARKON2025", "error": "timeout"}] diff --git a/scripts/validate_install_kit.sh b/scripts/validate_install_kit.sh index 94b924a..c209127 100755 --- a/scripts/validate_install_kit.sh +++ b/scripts/validate_install_kit.sh @@ -32,75 +32,5 @@ for candidate in "${rust_candidates[@]}"; do fi done -required_files=( - "$MANIFEST" - "$KIT_DIR/README-INSTALL-KIT.txt" - "$KIT_DIR/windows/deploy-ensemble.ps1" - "$KIT_DIR/windows/validate-deployment.ps1" - "$KIT_DIR/ansible/deploy_aw_windows.yml" - "$KIT_DIR/aw-server/install_aw_server.sh" - "$KIT_DIR/scripts/rebuild_install_kit.sh" - "$KIT_DIR/scripts/validate_install_kit.sh" - "$KIT_DIR/scripts/check_install_kit_vs_repo.sh" - "$KIT_DIR/scripts/quality-gate.sh" -) - -echo "[1/4] Required files presence" -for file in "${required_files[@]}"; do - [[ -f "$file" ]] || { echo "Missing required file: $file"; exit 1; } -done - -echo "[2/4] Manifest checksum verification" -sha256sum -c "$MANIFEST" >/dev/null - -echo "[3/4] Manifest completeness" -python3 - <<'PY' -from pathlib import Path -import sys -kit=Path('install-kit-awindows-20260427-211240') -manifest=kit/'MANIFEST.txt' -listed=[] -for line in manifest.read_text().splitlines(): - line=line.strip() - if not line: - continue - parts=line.split(' ',1) - if len(parts)!=2: - print(f'Invalid MANIFEST line: {line}') - sys.exit(1) - listed.append(parts[1]) -listed_set=set(listed) -actual_set={str(p) for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'} -missing=sorted(listed_set-actual_set) -extra=sorted(actual_set-listed_set) -if missing or extra: - print('Missing files listed in MANIFEST:', missing) - print('Files not listed in MANIFEST:', extra) - sys.exit(1) -print(f'MANIFEST complete: {len(actual_set)} files tracked') -PY - -echo "[4/4] Archive composition check" -python3 - <<'PY' -from pathlib import Path -import tarfile, zipfile, sys -kit_prefix='install-kit-awindows-20260427-211240/' -zip_path=Path('install-kit-awindows-20260427-211240.zip') -tar_path=Path('install-kit-awindows-20260427-211240.tar.gz') -if not zip_path.exists() or not tar_path.exists(): - print('Archives not found') - sys.exit(1) -with zipfile.ZipFile(zip_path) as z: - zip_files=sorted(i for i in z.namelist() if not i.endswith('/')) -with tarfile.open(tar_path, 'r:gz') as t: - tar_files=sorted(m.name for m in t.getmembers() if m.isfile()) -if zip_files != tar_files: - print('ZIP and TAR contents differ') - sys.exit(1) -if not all(f.startswith(kit_prefix) for f in zip_files): - print('Unexpected archive prefix layout') - sys.exit(1) -print(f'Archives match: {len(zip_files)} files') -PY - -echo "validate_install_kit: OK" +echo "ERROR: Rust validator not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p validate-install-kit" >&2 +exit 127