From c017cb08a96ffbfde1f82c013854442a43ffbe4a Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Wed, 1 Jul 2026 06:06:01 +0300 Subject: [PATCH] Harden DetMir runtime hot paths --- adk-rust/crates/check-aw-data/src/main.rs | 32 +- adk-rust/crates/detmir-auto/src/main.rs | 18 +- adk-rust/crates/detmir-check/src/main.rs | 163 +++++- adk-rust/crates/detmir-dlp/Cargo.toml | 1 + adk-rust/crates/detmir-dlp/src/main.rs | 90 ++- adk-rust/crates/dlp-health-check/src/main.rs | 238 +++++++- adk-rust/crates/worktime-api/src/main.rs | 527 +++++++++++++++++- adk-rust/crates/worktime-autoheal/src/main.rs | 12 +- ...orktime-interpretation-policy.example.json | 2 +- docs/CONTOUR_CHECK_MATRIX_RU.md | 53 +- docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md | 161 ++++++ docs/WORKFORCE_OPERATIONS_MODEL_RU.md | 239 ++++++++ .../awatch-contour-daily-check.service | 1 + .../awatch-contour-weekly-check.service | 1 + scripts/detmir_resilience_check.sh | 471 ++++++++++++++++ 15 files changed, 1923 insertions(+), 86 deletions(-) create mode 100644 docs/WORKFORCE_OPERATIONS_MODEL_RU.md create mode 100644 scripts/detmir_resilience_check.sh diff --git a/adk-rust/crates/check-aw-data/src/main.rs b/adk-rust/crates/check-aw-data/src/main.rs index adb44ac..17b51c6 100644 --- a/adk-rust/crates/check-aw-data/src/main.rs +++ b/adk-rust/crates/check-aw-data/src/main.rs @@ -43,6 +43,9 @@ struct Cli { #[arg(long, default_value_t = false)] no_color: bool, + + #[arg(long, default_value_t = true)] + dlp_enabled: bool, } #[derive(Debug, Clone)] @@ -109,7 +112,12 @@ fn main() { } fn run() -> Result { - let cli = Cli::parse(); + let mut cli = Cli::parse(); + if let Some(value) = + env_nonempty("AW_DLP_ENABLED").or_else(|| env_nonempty("DETMIR_DLP_ENABLED")) + { + cli.dlp_enabled = parse_env_flag(&value); + } let server = cli .server .or_else(|| env_nonempty("AW_CHECK_SERVER")) @@ -172,7 +180,11 @@ fn run() -> Result { "---------------------------------------------", "--------", "----------------------" ); - for bucket in BUCKETS { + for bucket in BUCKETS + .iter() + .copied() + .filter(|bucket| cli.dlp_enabled || !bucket.starts_with("aw-dlp-")) + { let bucket_full = format!("{bucket}_{host}"); let event = bucket_event( &server, @@ -190,6 +202,15 @@ fn run() -> Result { render_status(&colors, status) ); } + if !cli.dlp_enabled { + println!( + "{:<45} {:<8} {:<22} {}", + "aw-dlp-*", + "-", + "disabled", + colors.paint(colors.cyan, "SKIPPED") + ); + } println!(); println!("--- CORS Check ---"); @@ -493,6 +514,13 @@ fn env_nonempty(name: &str) -> Option { .filter(|value| !value.is_empty()) } +fn parse_env_flag(value: &str) -> bool { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/adk-rust/crates/detmir-auto/src/main.rs b/adk-rust/crates/detmir-auto/src/main.rs index bb52ee9..0e55587 100644 --- a/adk-rust/crates/detmir-auto/src/main.rs +++ b/adk-rust/crates/detmir-auto/src/main.rs @@ -1,6 +1,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::symlink; +use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::time::{Duration, SystemTime}; @@ -198,6 +199,7 @@ fn run_to_file( let mut child = Command::new(command) .args(args) + .process_group(0) .stdout(Stdio::from(stdout)) .stderr(Stdio::from(stderr)) .spawn() @@ -209,7 +211,7 @@ fn run_to_file( break status.code().unwrap_or(1); } if started.elapsed() >= timeout { - let _ = child.kill(); + terminate_process_group(child.id()); let _ = child.wait(); let mut stderr = OpenOptions::new().append(true).open(&stderr_path)?; writeln!( @@ -234,6 +236,17 @@ fn run_to_file( Ok(rc) } +fn terminate_process_group(child_pid: u32) { + let process_group = format!("-{child_pid}"); + let _ = Command::new("/bin/kill") + .args(["-TERM", "--", &process_group]) + .status(); + std::thread::sleep(Duration::from_secs(2)); + let _ = Command::new("/bin/kill") + .args(["-KILL", "--", &process_group]) + .status(); +} + fn read_rc(path: &Path) -> i32 { fs::read_to_string(path) .ok() @@ -411,6 +424,7 @@ fn write_report( fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Result { let mut child = Command::new(polli_bin) .args(["--model", "text.daily", "--max-tokens", "900"]) + .process_group(0) .stdin(Stdio::from(bundle)) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -425,7 +439,7 @@ fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Resul .with_context(|| format!("failed to collect {polli_bin} output")); } if started.elapsed() >= timeout { - let _ = child.kill(); + terminate_process_group(child.id()); let _ = child.wait(); anyhow::bail!( "Pollinations report timed out after {} seconds", diff --git a/adk-rust/crates/detmir-check/src/main.rs b/adk-rust/crates/detmir-check/src/main.rs index 8850798..cce85e4 100644 --- a/adk-rust/crates/detmir-check/src/main.rs +++ b/adk-rust/crates/detmir-check/src/main.rs @@ -1,5 +1,6 @@ use std::io::Read; use std::net::{SocketAddr, TcpStream}; +use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; @@ -89,9 +90,15 @@ struct Cli { #[arg(long, default_value_t = 45)] dlp_timeout_seconds: u64, + #[arg(long, default_value_t = 150)] + overall_timeout_seconds: u64, + #[arg(long, default_value_t = false)] disable_dlp_health_check: bool, + #[arg(long, default_value_t = true)] + dlp_enabled: bool, + #[arg(long, default_value_t = false)] disable_portal_check: bool, } @@ -193,8 +200,8 @@ fn parse_env_flag(value: &str) -> bool { ) } -fn bucket_specs(hostname: &str) -> Vec { - vec![ +fn bucket_specs(hostname: &str, dlp_enabled: bool) -> Vec { + let mut specs = vec![ BucketSpec { label: "AFK watcher", bucket: format!("aw-watcher-afk_{hostname}"), @@ -219,31 +226,36 @@ fn bucket_specs(hostname: &str) -> Vec { max_age_seconds: None, mode: BucketMode::EventDriven, }, - BucketSpec { - label: "DLP signals", - bucket: format!("aw-dlp-endpoint-signals_{hostname}"), - max_age_seconds: Some(10 * 60), - mode: BucketMode::InteractiveFresh, - }, - BucketSpec { - label: "DLP incidents", - bucket: format!("aw-dlp-incidents_{hostname}"), - max_age_seconds: None, - mode: BucketMode::EventDriven, - }, - BucketSpec { - label: "DLP review", - bucket: format!("aw-dlp-review_{hostname}"), - max_age_seconds: None, - mode: BucketMode::EventDriven, - }, - BucketSpec { - label: "DLP rules", - bucket: format!("aw-dlp-rules_{hostname}"), - max_age_seconds: None, - mode: BucketMode::EventDriven, - }, - ] + ]; + if dlp_enabled { + specs.extend([ + BucketSpec { + label: "DLP signals", + bucket: format!("aw-dlp-endpoint-signals_{hostname}"), + max_age_seconds: Some(10 * 60), + mode: BucketMode::InteractiveFresh, + }, + BucketSpec { + label: "DLP incidents", + bucket: format!("aw-dlp-incidents_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + BucketSpec { + label: "DLP review", + bucket: format!("aw-dlp-review_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + BucketSpec { + label: "DLP rules", + bucket: format!("aw-dlp-rules_{hostname}"), + max_age_seconds: None, + mode: BucketMode::EventDriven, + }, + ]); + } + specs } fn build_headers(items: &[(&str, &str)]) -> Result { @@ -387,7 +399,20 @@ fn service_checks(args: &Cli) -> Vec { if security_events_clickhouse_enabled(args) { checks.push(clickhouse_security_events_check(args)); } - if !args.disable_dlp_health_check { + if !args.dlp_enabled { + checks.push(ServiceCheck { + name: "aw-dlp-mode".to_string(), + required: false, + ok: true, + url: None, + payload: Some(serde_json::json!({ + "mode": "disabled", + "reason": "DETMIR_DLP_ENABLED=false", + "checks_skipped": ["DLP health command", "DLP buckets"] + })), + error: None, + }); + } else if !args.disable_dlp_health_check { checks.push(dlp_health_check(args)); } checks @@ -500,6 +525,7 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result Result= timeout { - let _ = child.kill(); + terminate_process_group(child.id()); let _ = child.wait(); return read_command_output(child, None, true); } @@ -518,6 +544,17 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result, @@ -796,7 +833,7 @@ fn bucket_health(args: &Cli) -> Result> { let interactive_required = interactive_required(&client, &args.hostname, now); let mut out = Vec::new(); - for spec in bucket_specs(&args.hostname) { + for spec in bucket_specs(&args.hostname, args.dlp_enabled) { if matches!(spec.mode, BucketMode::EventDriven) { out.push(BucketCheck { label: spec.label.to_string(), @@ -993,6 +1030,19 @@ fn render_text(report: &CheckReport) -> String { fn main() -> Result<()> { let mut args = Cli::parse(); + args.service_timeout_seconds = env_u64( + "DETMIR_SERVICE_TIMEOUT_SECONDS", + args.service_timeout_seconds, + ); + args.bucket_timeout_seconds = + env_u64("DETMIR_BUCKET_TIMEOUT_SECONDS", args.bucket_timeout_seconds); + args.tcp_timeout_seconds = env_f64("DETMIR_TCP_TIMEOUT_SECONDS", args.tcp_timeout_seconds); + args.dlp_timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", args.dlp_timeout_seconds); + args.overall_timeout_seconds = env_u64( + "DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS", + args.overall_timeout_seconds, + ); + start_overall_timeout_watchdog(args.overall_timeout_seconds); args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api); args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url); args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url); @@ -1011,6 +1061,12 @@ fn main() -> Result<()> { if env_flag_enabled("DETMIR_DISABLE_DLP_HEALTH_CHECK") { args.disable_dlp_health_check = true; } + if let Some(value) = std::env::var("DETMIR_DLP_ENABLED") + .ok() + .filter(|value| !value.is_empty()) + { + args.dlp_enabled = parse_env_flag(&value); + } if env_flag_enabled("DETMIR_DISABLE_PORTAL_CHECK") { args.disable_portal_check = true; } @@ -1028,6 +1084,31 @@ fn main() -> Result<()> { }); } +fn env_u64(name: &str, fallback: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_f64(name: &str, fallback: f64) -> f64 { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn start_overall_timeout_watchdog(seconds: u64) { + if seconds == 0 { + return; + } + std::thread::spawn(move || { + std::thread::sleep(Duration::from_secs(seconds)); + eprintln!("detmir-check timed out after {seconds} seconds"); + std::process::exit(124); + }); +} + #[cfg(test)] mod tests { use super::*; @@ -1095,6 +1176,28 @@ mod tests { assert!(!parse_env_flag("false")); } + #[test] + fn dlp_disabled_removes_dlp_bucket_specs() { + let enabled = bucket_specs("HOST-EXAMPLE", true); + assert!( + enabled + .iter() + .any(|spec| spec.bucket.starts_with("aw-dlp-")) + ); + + let disabled = bucket_specs("HOST-EXAMPLE", false); + assert!( + disabled + .iter() + .all(|spec| !spec.bucket.starts_with("aw-dlp-")) + ); + assert!( + disabled + .iter() + .any(|spec| spec.bucket.starts_with("aw-worktime-sessions_")) + ); + } + #[test] fn clickhouse_database_identifier_rejects_injection() { assert_eq!( diff --git a/adk-rust/crates/detmir-dlp/Cargo.toml b/adk-rust/crates/detmir-dlp/Cargo.toml index a8c03bf..018c61e 100644 --- a/adk-rust/crates/detmir-dlp/Cargo.toml +++ b/adk-rust/crates/detmir-dlp/Cargo.toml @@ -9,3 +9,4 @@ publish.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +serde_json.workspace = true diff --git a/adk-rust/crates/detmir-dlp/src/main.rs b/adk-rust/crates/detmir-dlp/src/main.rs index 270e430..37bac8f 100644 --- a/adk-rust/crates/detmir-dlp/src/main.rs +++ b/adk-rust/crates/detmir-dlp/src/main.rs @@ -1,5 +1,6 @@ use std::io::{self, Write}; -use std::process::Command; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use clap::Parser; @@ -19,8 +20,14 @@ struct Cli { #[arg(long, default_value_t = 10)] connect_timeout_seconds: u64, + #[arg(long, default_value_t = 90)] + timeout_seconds: u64, + #[arg(long, default_value = DEFAULT_REMOTE_COMMAND)] remote_command: String, + + #[arg(long, default_value_t = true)] + enabled: bool, } impl Cli { @@ -31,6 +38,8 @@ impl Cli { ); self.remote_command = env_first(&["DETMIR_DLP_REMOTE_COMMAND"], &self.remote_command); self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin); + self.timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", self.timeout_seconds); + self.enabled = env_bool("DETMIR_DLP_ENABLED", self.enabled); self } } @@ -42,6 +51,25 @@ fn env_first(names: &[&str], fallback: &str) -> String { .unwrap_or_else(|| fallback.to_string()) } +fn env_u64(name: &str, fallback: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(fallback) +} + +fn env_bool(name: &str, fallback: bool) -> bool { + std::env::var(name) + .ok() + .map(|value| { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) + .unwrap_or(fallback) +} + fn ssh_args(cli: &Cli) -> Vec { vec![ "-o".to_string(), @@ -56,22 +84,76 @@ fn ssh_args(cli: &Cli) -> Vec { } fn run(cli: Cli) -> Result { + if !cli.enabled { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "ok": true, + "counts": {"ok": 1, "warn": 0, "fail": 0}, + "results": [{ + "name": "dlp:mode", + "status": "ok", + "summary": "DLP health check disabled by DETMIR_DLP_ENABLED=false", + "details": { + "mode": "disabled", + "load_reduction": ["aw-dlp health ssh probe skipped"] + } + }] + }))? + ); + return Ok(0); + } + let args = ssh_args(&cli); - let output = Command::new(&cli.ssh_bin) + let mut child = Command::new(&cli.ssh_bin) .args(&args) - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .with_context(|| format!("failed to execute {}", cli.ssh_bin))?; + let started = Instant::now(); + let mut timed_out = false; + loop { + if child.try_wait()?.is_some() { + break; + } + if started.elapsed() >= Duration::from_secs(cli.timeout_seconds) { + timed_out = true; + terminate_child(&mut child); + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + + let output = child + .wait_with_output() + .context("failed to collect SSH output")?; io::stdout() .write_all(&output.stdout) .context("failed to write DLP stdout")?; io::stderr() .write_all(&output.stderr) .context("failed to write DLP stderr")?; + if timed_out { + writeln!( + io::stderr(), + "detmir-dlp timed out after {} seconds", + cli.timeout_seconds + ) + .context("failed to write timeout message")?; + return Ok(124); + } Ok(output.status.code().unwrap_or(1)) } +fn terminate_child(child: &mut std::process::Child) { + let _ = child.kill(); + std::thread::sleep(Duration::from_secs(2)); + let _ = child.kill(); +} + fn main() -> Result<()> { let cli = Cli::parse().apply_env(); let code = run(cli)?; @@ -88,7 +170,9 @@ mod tests { ssh_bin: "ssh".to_string(), ssh_target: DEFAULT_SSH_TARGET.to_string(), connect_timeout_seconds: 10, + timeout_seconds: 90, remote_command: DEFAULT_REMOTE_COMMAND.to_string(), + enabled: true, }; assert_eq!( ssh_args(&cli), diff --git a/adk-rust/crates/dlp-health-check/src/main.rs b/adk-rust/crates/dlp-health-check/src/main.rs index 9fbf799..f5e1bd0 100644 --- a/adk-rust/crates/dlp-health-check/src/main.rs +++ b/adk-rust/crates/dlp-health-check/src/main.rs @@ -62,6 +62,21 @@ struct Cli { #[arg(long)] json: bool, + + #[arg(long, default_value_t = 120)] + overall_timeout_seconds: u64, + + #[arg(long, default_value = "full")] + profile: String, + + #[arg(long, default_value_t = true)] + enabled: bool, + + #[arg(long, default_value = "operator_disabled")] + disabled_reason: String, + + #[arg(long, default_value = "")] + disabled_since: String, } impl Cli { @@ -129,6 +144,28 @@ impl Cli { if !cli_arg_present("--profiles") { self.profiles = env_string("AW_DLP_COMPLIANCE_PROFILES").unwrap_or(self.profiles); } + if !cli_arg_present("--overall-timeout-seconds") { + self.overall_timeout_seconds = env_u64( + "AW_DLP_HEALTH_OVERALL_TIMEOUT_SECONDS", + self.overall_timeout_seconds, + ); + } + if !cli_arg_present("--profile") { + self.profile = env_string("AW_DLP_PROFILE") + .or_else(|| env_string("DETMIR_PORTAL_DLP_PROFILE")) + .unwrap_or(self.profile); + } + if !cli_arg_present("--enabled") { + self.enabled = env_bool_default("AW_DLP_ENABLED", self.enabled); + } + if !cli_arg_present("--disabled-reason") { + self.disabled_reason = + env_string("AW_DLP_DISABLED_REASON").unwrap_or(self.disabled_reason); + } + if !cli_arg_present("--disabled-since") { + self.disabled_since = + env_string("AW_DLP_DISABLED_SINCE").unwrap_or(self.disabled_since); + } self } } @@ -1270,8 +1307,66 @@ fn check_compliance_reports( } } +fn normalized_profile(profile: &str) -> String { + match profile.trim().to_ascii_lowercase().as_str() { + "disabled" | "off" => "core_only".to_string(), + "core-only" | "core_only" => "core_only".to_string(), + "light" | "lite" => "light".to_string(), + "on-demand" | "on_demand" => "on_demand".to_string(), + "enabled" | "on" | "full" => "full".to_string(), + "" => "full".to_string(), + other => other.to_string(), + } +} + +fn profile_is_disabled(profile: &str) -> bool { + matches!(normalized_profile(profile).as_str(), "core_only") +} + +fn profile_checks_heavy_services(profile: &str) -> bool { + matches!(normalized_profile(profile).as_str(), "full" | "on_demand") +} + fn build_report(cli: &Cli, client: &Client) -> HealthReport { let mut report = HealthReport::default(); + let profile = normalized_profile(&cli.profile); + if !cli.enabled || profile_is_disabled(&profile) { + report.add( + "dlp:mode", + "ok", + "DLP runtime disabled by production profile", + json!({ + "mode": "disabled", + "profile": profile, + "reason": &cli.disabled_reason, + "disabled_since": empty_string_as_null(&cli.disabled_since), + "checks_skipped": [ + "policy API", + "case API", + "DLP systemd units", + "DLP ActivityWatch buckets", + "DLP compliance reports" + ], + "load_reduction": [ + "no DLP bucket freshness reads", + "no DLP case/policy HTTP checks", + "no DLP compliance filesystem scan" + ] + }), + ); + return report; + } + report.add( + "dlp:mode", + "ok", + format!("DLP runtime profile {profile}"), + json!({ + "mode": if profile == "light" { "light" } else { "enabled" }, + "profile": profile, + "heavy_services_checked": profile_checks_heavy_services(&cli.profile) + }), + ); + let aw_api_base = format!("{}/api/0", cli.aw_server.trim_end_matches('/')); let counter_state_path = cli.state_dir.join("dlp-health-check-counters.json"); let mut counter_state = load_counter_state(&counter_state_path); @@ -1282,36 +1377,70 @@ fn build_report(cli: &Cli, client: &Client) -> HealthReport { "http:aw", &format!("{aw_api_base}/info"), ); - check_http_endpoint( - &mut report, - client, - "http:policy", - &format!("{}/healthz", cli.policy_server.trim_end_matches('/')), - ); - check_http_endpoint( - &mut report, - client, - "http:cases", - &format!("{}/health", cli.case_server.trim_end_matches('/')), - ); + if profile_checks_heavy_services(&cli.profile) { + check_http_endpoint( + &mut report, + client, + "http:policy", + &format!("{}/healthz", cli.policy_server.trim_end_matches('/')), + ); + check_http_endpoint( + &mut report, + client, + "http:cases", + &format!("{}/health", cli.case_server.trim_end_matches('/')), + ); + } else { + report.add( + "http:heavy-dlp", + "ok", + "heavy DLP policy/case HTTP checks skipped for lightweight profile", + json!({ + "profile": profile, + "skipped": ["policy API", "case API"] + }), + ); + } - for unit in [ - "activitywatch-server", - "aw-dlp-policy-engine.service", - "aw-dlp-case-management.service", - "aw-worktime-api.service", - ] { + for unit in ["activitywatch-server", "aw-worktime-api.service"] { check_systemd_unit(&mut report, unit, "service"); } - for unit in [ - "aw-dlp-report-scheduler.timer", - "aw-dlp-syslog-forwarder.timer", - "aw-dlp-webhook-sender.timer", - "aw-dlp-cef-exporter.timer", - "activitywatch-dlp-aggregator.timer", - "aw-dlp-ioc-refresh.timer", - "aw-worktime-ui-bridge.timer", - ] { + if profile_checks_heavy_services(&cli.profile) { + for unit in [ + "aw-dlp-policy-engine.service", + "aw-dlp-case-management.service", + ] { + check_systemd_unit(&mut report, unit, "service"); + } + for unit in [ + "aw-dlp-report-scheduler.timer", + "aw-dlp-syslog-forwarder.timer", + "aw-dlp-webhook-sender.timer", + "aw-dlp-cef-exporter.timer", + "activitywatch-dlp-aggregator.timer", + "aw-dlp-ioc-refresh.timer", + ] { + check_systemd_unit(&mut report, unit, "timer"); + } + } else { + report.add( + "systemd:heavy-dlp", + "ok", + "heavy DLP systemd checks skipped for lightweight profile", + json!({ + "profile": profile, + "skipped": [ + "aw-dlp-policy-engine.service", + "aw-dlp-case-management.service", + "aw-dlp-report-scheduler.timer", + "aw-dlp-syslog-forwarder.timer", + "aw-dlp-webhook-sender.timer", + "aw-dlp-cef-exporter.timer" + ] + }), + ); + } + for unit in ["aw-worktime-ui-bridge.timer"] { check_systemd_unit(&mut report, unit, "timer"); } @@ -1462,6 +1591,12 @@ fn env_i64(name: &str, default: i64) -> i64 { .unwrap_or(default) } +fn env_u64(name: &str, default: u64) -> u64 { + env_string(name) + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + fn env_bool(name: &str) -> bool { env_string(name) .map(|value| { @@ -1473,8 +1608,39 @@ fn env_bool(name: &str) -> bool { .unwrap_or(false) } +fn env_bool_default(name: &str, default: bool) -> bool { + env_string(name) + .map(|value| { + matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) + }) + .unwrap_or(default) +} + +fn empty_string_as_null(value: &str) -> Value { + if value.trim().is_empty() { + Value::Null + } else { + json!(value) + } +} + +fn start_overall_timeout_watchdog(seconds: u64) { + if seconds == 0 { + return; + } + std::thread::spawn(move || { + sleep(Duration::from_secs(seconds)); + eprintln!("dlp-health-check timed out after {seconds} seconds"); + std::process::exit(124); + }); +} + fn main() -> Result<()> { let cli = Cli::parse().apply_env(); + start_overall_timeout_watchdog(cli.overall_timeout_seconds); let client = Client::builder() .no_proxy() .build() @@ -1547,4 +1713,22 @@ mod tests { assert_eq!(payload.counts.warn, 1); assert_eq!(payload.counts.fail, 0); } + + #[test] + fn dlp_profile_normalization_matches_runtime_control_names() { + assert_eq!(normalized_profile("disabled"), "core_only"); + assert_eq!(normalized_profile("core-only"), "core_only"); + assert_eq!(normalized_profile("light"), "light"); + assert_eq!(normalized_profile("lite"), "light"); + assert_eq!(normalized_profile("on-demand"), "on_demand"); + assert_eq!(normalized_profile("enabled"), "full"); + } + + #[test] + fn light_profile_does_not_require_heavy_services() { + assert!(!profile_checks_heavy_services("light")); + assert!(!profile_checks_heavy_services("core_only")); + assert!(profile_checks_heavy_services("on_demand")); + assert!(profile_checks_heavy_services("full")); + } } diff --git a/adk-rust/crates/worktime-api/src/main.rs b/adk-rust/crates/worktime-api/src/main.rs index de446d6..cd69a97 100644 --- a/adk-rust/crates/worktime-api/src/main.rs +++ b/adk-rust/crates/worktime-api/src/main.rs @@ -77,6 +77,7 @@ struct Config { management_history_retention_days: i64, true_active_evidence_window_seconds: i64, true_active_max_event_seconds: i64, + dlp_evidence_enabled: bool, offset: FixedOffset, } @@ -227,6 +228,14 @@ fn threshold_to_pct(value: f64) -> f64 { } } +fn overload_threshold_to_pct(value: f64) -> f64 { + if (0.0..=3.0).contains(&value) { + value * 100.0 + } else { + value + } +} + fn parse_hhmm(value: &str) -> Option { NaiveTime::parse_from_str(value.trim(), "%H:%M").ok() } @@ -262,7 +271,7 @@ fn load_config() -> Config { } if let Some(value) = policy.overload_threshold { manager_overload_coverage_pct = - threshold_to_pct(value).round().clamp(1.0, 300.0) as i64; + overload_threshold_to_pct(value).round().clamp(100.0, 300.0) as i64; } if let Some(value) = policy.drop_threshold_pct { manager_trend_delta_pct = threshold_to_pct(value).clamp(1.0, 100.0); @@ -371,6 +380,10 @@ fn load_config() -> Config { .max(30), true_active_max_event_seconds: env_i64("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", 600) .max(30), + dlp_evidence_enabled: env_bool( + "AW_WORKTIME_DLP_EVIDENCE_ENABLED", + env_bool("AW_DLP_ENABLED", true), + ), offset: FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"), } } @@ -1010,13 +1023,7 @@ impl App { } }; let mut evidence = HashMap::new(); - for bucket in [ - format!("aw-file-operations_{host}"), - format!("aw-dlp-endpoint-signals_{host}"), - format!("aw-watcher-web-chrome_{host}"), - format!("aw-watcher-web-edge_{host}"), - format!("aw-detmir-web-category_{host}"), - ] { + for bucket in evidence_bucket_ids(&self.config, host) { evidence.insert( bucket.clone(), self.fetch_bucket_events(&bucket, Some(bounds.0), Some(bounds.1)), @@ -1156,6 +1163,19 @@ fn sanitize_bucket_for_log(bucket_id: &str) -> String { bucket_id.to_string() } +fn evidence_bucket_ids(config: &Config, host: &str) -> Vec { + let mut buckets = vec![format!("aw-file-operations_{host}")]; + if config.dlp_evidence_enabled { + buckets.push(format!("aw-dlp-endpoint-signals_{host}")); + } + buckets.extend([ + format!("aw-watcher-web-chrome_{host}"), + format!("aw-watcher-web-edge_{host}"), + format!("aw-detmir-web-category_{host}"), + ]); + buckets +} + fn sanitize_error_for_log(value: &str) -> String { static IP_RE: OnceLock = OnceLock::new(); static BUCKET_HOST_RE: OnceLock = OnceLock::new(); @@ -1605,6 +1625,156 @@ fn interval_overlap_seconds( (total, first, last) } +#[derive(Clone, Copy)] +struct EmployeeOperationsInput { + expected_seconds: i64, + low_seconds: i64, + target_seconds: i64, + overload_seconds: i64, + work_secs: i64, + calendar_secs: i64, + work_first: Option>, + work_last: Option>, + is_today: bool, + late_start: DateTime, + early_finish: DateTime, + samples_count: i64, + active_samples: i64, + sessions_count: i64, +} + +fn build_employee_operations_status(config: &Config, input: EmployeeOperationsInput) -> Value { + let workday_idle_seconds = (input.expected_seconds - input.work_secs).max(0); + let off_hours_seconds = (input.calendar_secs - input.work_secs).max(0); + let coverage = if input.expected_seconds > 0 { + clamp_pct(input.work_secs as f64 / input.expected_seconds as f64 * 100.0) + } else { + 0.0 + }; + let load_status = if input.expected_seconds <= 0 { + "insufficient_data" + } else if input.sessions_count <= 0 || input.samples_count <= 0 { + "no_data" + } else if input.work_secs <= 0 { + "no_activity" + } else if input.work_secs < input.low_seconds { + "underloaded" + } else if input.work_secs >= input.overload_seconds { + "overloaded" + } else if input.work_secs < input.target_seconds { + "below_target" + } else { + "normal" + }; + let idle_status = if input.expected_seconds <= 0 { + "not_applicable" + } else if input.sessions_count <= 0 || input.samples_count <= 0 { + "unknown" + } else if input.work_secs <= 0 { + "full_workday_idle_or_absent" + } else if workday_idle_seconds >= config.manager_off_hours_threshold_seconds { + "idle_detected" + } else { + "no_significant_idle" + }; + let mut discipline_flags = Vec::new(); + if off_hours_seconds >= config.manager_off_hours_threshold_seconds { + discipline_flags.push("off_hours"); + } + if input + .work_first + .is_some_and(|dt| dt.with_timezone(&config.offset) > input.late_start) + { + discipline_flags.push("late_start"); + } + if !input.is_today + && input + .work_last + .is_some_and(|dt| dt.with_timezone(&config.offset) < input.early_finish) + { + discipline_flags.push("early_finish"); + } + let discipline_status = match discipline_flags.as_slice() { + [] => "ok", + [single] => single, + _ => "multiple_flags", + }; + let mut confidence_reasons = Vec::new(); + if input.sessions_count <= 0 { + confidence_reasons.push("missing_session_samples"); + } + if input.samples_count <= 0 { + confidence_reasons.push("missing_worktime_samples"); + } else if input.samples_count < 3 { + confidence_reasons.push("few_worktime_samples"); + } else { + confidence_reasons.push("worktime_samples_present"); + } + if input.active_samples <= 0 { + confidence_reasons.push("missing_active_samples"); + } else { + confidence_reasons.push("active_samples_present"); + } + if input.expected_seconds <= 0 { + confidence_reasons.push("workday_window_not_started_or_empty"); + } + let data_confidence = + if input.expected_seconds <= 0 || input.sessions_count <= 0 || input.samples_count <= 0 { + "low" + } else if input.samples_count < 3 || input.active_samples <= 0 { + "medium" + } else { + "high" + }; + let recommended_action = match (data_confidence, load_status, discipline_status, idle_status) { + ("low", _, _, _) => { + "Сначала проверить свежесть источников и наличие сессии; вывод по сотруднику не использовать как дисциплинарный." + } + (_, "overloaded", _, _) => { + "Проверить переработку, перераспределение задач и риск аврального процесса." + } + (_, "underloaded" | "below_target" | "no_activity", _, _) => { + "Проверить фактическую загрузку, задачи, доступ к рабочим системам и отсутствие сбоя сбора." + } + (_, _, "off_hours" | "late_start" | "early_finish" | "multiple_flags", _) => { + "Проверить согласование рабочего графика и причину отклонения от процесса." + } + (_, _, _, "idle_detected" | "full_workday_idle_or_absent") => { + "Проверить простой: отсутствие задач, ожидание внешнего процесса или техническую проблему." + } + _ => "Наблюдать; отклонений, требующих немедленного действия, не выявлено.", + }; + json!({ + "load_status": load_status, + "idle_status": idle_status, + "discipline_status": discipline_status, + "discipline_flags": discipline_flags, + "data_confidence": data_confidence, + "confidence_reasons": confidence_reasons, + "workday_idle_seconds": workday_idle_seconds, + "workday_idle_hhmm": hhmm(workday_idle_seconds), + "off_hours_seconds": off_hours_seconds, + "off_hours_hhmm": hhmm(off_hours_seconds), + "coverage_pct": coverage, + "evidence": { + "expected_hhmm": hhmm(input.expected_seconds), + "workday_active_hhmm": hhmm(input.work_secs), + "calendar_active_hhmm": hhmm(input.calendar_secs), + "sessions_count": input.sessions_count, + "samples_count": input.samples_count, + "active_samples": input.active_samples, + "first_workday_activity_local": input.work_first.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default(), + "last_workday_activity_local": input.work_last.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default() + }, + "guardrail": if data_confidence == "low" { + "low_confidence_not_for_discipline" + } else { + "evidence_backed_manual_review_only" + }, + "recommended_action": recommended_action, + }) +} + impl App { fn build_management_payload( &self, @@ -1634,6 +1804,7 @@ impl App { }; let target_seconds = expected_seconds * self.config.manager_target_coverage_pct / 100; let low_seconds = expected_seconds * self.config.manager_low_coverage_pct / 100; + let overload_seconds = expected_seconds * self.config.manager_overload_coverage_pct / 100; let late_start = work_start_local + TimeDelta::minutes(self.config.manager_late_start_grace_minutes); let early_finish = @@ -1706,6 +1877,37 @@ impl App { .map(|dt| dt.with_timezone(&self.config.offset)) .map(|dt| dt.to_rfc3339()) .unwrap_or_default(); + let samples_count = row + .get("samples_count") + .and_then(Value::as_i64) + .unwrap_or(0); + let active_samples = row + .get("active_samples") + .and_then(Value::as_i64) + .unwrap_or(0); + let sessions_count = row + .get("sessions_count") + .and_then(Value::as_i64) + .unwrap_or(0); + let operations = build_employee_operations_status( + &self.config, + EmployeeOperationsInput { + expected_seconds, + low_seconds, + target_seconds, + overload_seconds, + work_secs, + calendar_secs, + work_first, + work_last, + is_today, + late_start, + early_finish, + samples_count, + active_samples, + sessions_count, + }, + ); let mut public = row.as_object().cloned().unwrap_or_default(); public.remove("_intervals"); public.insert("user".into(), json!(alias.display_name)); @@ -1719,8 +1921,52 @@ impl App { public.insert("calendar_active_hhmm".into(), json!(hhmm(calendar_secs))); public.insert("workday_active_seconds".into(), json!(work_secs)); public.insert("workday_active_hhmm".into(), json!(hhmm(work_secs))); + public.insert( + "workday_idle_seconds".into(), + json!((expected_seconds - work_secs).max(0)), + ); + public.insert( + "workday_idle_hhmm".into(), + json!(hhmm((expected_seconds - work_secs).max(0))), + ); public.insert("coverage_pct".into(), json!(coverage)); public.insert("status".into(), json!(status)); + public.insert( + "load_status".into(), + operations + .get("load_status") + .cloned() + .unwrap_or_else(|| json!("unknown")), + ); + public.insert( + "idle_status".into(), + operations + .get("idle_status") + .cloned() + .unwrap_or_else(|| json!("unknown")), + ); + public.insert( + "discipline_status".into(), + operations + .get("discipline_status") + .cloned() + .unwrap_or_else(|| json!("unknown")), + ); + public.insert( + "data_confidence".into(), + operations + .get("data_confidence") + .cloned() + .unwrap_or_else(|| json!("low")), + ); + public.insert( + "operations_recommended_action".into(), + operations + .get("recommended_action") + .cloned() + .unwrap_or_else(|| json!("Проверить первичные источники.")), + ); + public.insert("operations".into(), operations.clone()); public.insert("first_activity_local".into(), json!(first_local)); public.insert("last_activity_local".into(), json!(last_local)); public.insert( @@ -1739,6 +1985,10 @@ impl App { "manager_owner": public.get("manager_owner").cloned().unwrap_or(json!("")), "department": public.get("department").cloned().unwrap_or(json!("")), "role": public.get("role").cloned().unwrap_or(json!("")), + "load_status": public.get("load_status").cloned().unwrap_or(json!("unknown")), + "idle_status": public.get("idle_status").cloned().unwrap_or(json!("unknown")), + "discipline_status": public.get("discipline_status").cloned().unwrap_or(json!("unknown")), + "data_confidence": public.get("data_confidence").cloned().unwrap_or(json!("low")), }); let owner = public .get("manager_owner") @@ -1763,6 +2013,9 @@ impl App { } else if expected_seconds > 0 && work_secs < target_seconds { actions.push(action("target_gap_review", "medium", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже управленческого целевого порога {}%.", hhmm(work_secs), self.config.manager_target_coverage_pct), &format!("Уточнить причину отклонения по сотруднику {display} и подтвердить план работ."), &canonical, evidence.clone())); } + if expected_seconds > 0 && work_secs >= overload_seconds { + actions.push(action("overload_review", "high", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} выше порога перегруза {}%.", hhmm(work_secs), self.config.manager_overload_coverage_pct), &format!("Проверить переработку сотрудника {display}, распределение задач и риск аврального процесса."), &canonical, evidence.clone())); + } if work_first.is_some_and(|dt| dt.with_timezone(&self.config.offset) > late_start) { actions.push(action("late_start_review", "medium", &owner, "24h", &format!("У сотрудника {display} первая активность в рабочем окне зафиксирована поздно."), &format!("Проверить причину позднего старта сотрудника {display} и подтвердить, что это не проблема доступа или дисциплины."), &canonical, evidence.clone())); } @@ -1810,6 +2063,37 @@ impl App { ) }); let summary = summarize_management_rows(&roster, &actions, expected_seconds); + let workforce_operations = json!({ + "status": summary.pointer("/workforce_operations/status").cloned().unwrap_or_else(|| json!("LOW_CONFIDENCE")), + "summary": summary.pointer("/workforce_operations").cloned().unwrap_or_else(|| json!({})), + "rows": roster.iter().map(|row| { + json!({ + "user": row.get("user").cloned().unwrap_or(json!("")), + "manager_owner": row.get("manager_owner").cloned().unwrap_or(json!("")), + "department": row.get("department").cloned().unwrap_or(json!("")), + "role": row.get("role").cloned().unwrap_or(json!("")), + "load_status": row.get("load_status").cloned().unwrap_or(json!("unknown")), + "idle_status": row.get("idle_status").cloned().unwrap_or(json!("unknown")), + "discipline_status": row.get("discipline_status").cloned().unwrap_or(json!("unknown")), + "data_confidence": row.get("data_confidence").cloned().unwrap_or(json!("low")), + "coverage_pct": row.get("coverage_pct").cloned().unwrap_or(json!(0.0)), + "workday_active_hhmm": row.get("workday_active_hhmm").cloned().unwrap_or(json!("00:00")), + "workday_idle_hhmm": row.get("workday_idle_hhmm").cloned().unwrap_or(json!("00:00")), + "recommended_action": row.get("operations_recommended_action").cloned().unwrap_or(json!("Проверить первичные источники.")) + }) + }).collect::>(), + "model": { + "type": "rule_based", + "ml": false, + "llm": false, + "version": "workforce-operations-v1" + }, + "guardrails": [ + "Строки low confidence требуют проверки источников до персонального вывода", + "Отсутствие данных не считается простоем", + "Все статусы предназначены только для ручного операционного разбора" + ] + }); let owner_rollups = build_rollups(&roster, &actions, "manager_owner"); let department_rollups = build_rollups(&roster, &actions, "department"); let owner_roster = owner_rollups.clone(); @@ -1865,8 +2149,10 @@ impl App { "expected_hhmm_per_user": hhmm(expected_seconds), "target_coverage_pct": self.config.manager_target_coverage_pct, "low_coverage_pct": self.config.manager_low_coverage_pct, + "overload_coverage_pct": self.config.manager_overload_coverage_pct, }, "summary": summary, + "workforce_operations": workforce_operations, "actions": actions, "rows": roster, "sources": sources, @@ -2329,6 +2615,7 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds .and_then(Value::as_i64) .unwrap_or(0) }); + let operations_summary = summarize_operations(rows); json!({ "users_count": users_count, "active_users": active_users, @@ -2349,6 +2636,85 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds "last_activity": rows.iter().filter_map(|r| r.get("workday_last_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).max().unwrap_or(""), "top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""), "top_user_active_hhmm": top.and_then(|r| r.get("workday_active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"), + "workforce_operations": operations_summary, + }) +} + +fn row_str<'a>(row: &'a Value, key: &str) -> &'a str { + row.get(key).and_then(Value::as_str).unwrap_or("") +} + +fn count_rows_by(rows: &[Value], key: &str, expected: &[&str]) -> i64 { + rows.iter() + .filter(|row| expected.contains(&row_str(row, key))) + .count() as i64 +} + +fn summarize_operations(rows: &[Value]) -> Value { + let users_count = rows.len() as i64; + let low_confidence_users = count_rows_by(rows, "data_confidence", &["low"]); + let medium_confidence_users = count_rows_by(rows, "data_confidence", &["medium"]); + let high_confidence_users = count_rows_by(rows, "data_confidence", &["high"]); + let unknown_or_no_data_users = count_rows_by( + rows, + "load_status", + &["no_data", "insufficient_data", "no_activity"], + ); + let underloaded_users = count_rows_by(rows, "load_status", &["underloaded", "below_target"]); + let normal_users = count_rows_by(rows, "load_status", &["normal"]); + let overloaded_users = count_rows_by(rows, "load_status", &["overloaded"]); + let idle_users = count_rows_by( + rows, + "idle_status", + &["idle_detected", "full_workday_idle_or_absent"], + ); + let discipline_review_users = rows + .iter() + .filter(|row| !matches!(row_str(row, "discipline_status"), "" | "ok")) + .count() as i64; + let action_required_users = rows + .iter() + .filter(|row| { + !matches!(row_str(row, "load_status"), "normal" | "") + || !matches!(row_str(row, "discipline_status"), "ok" | "") + || matches!( + row_str(row, "idle_status"), + "idle_detected" | "full_workday_idle_or_absent" + ) + || row_str(row, "data_confidence") == "low" + }) + .count() as i64; + let status = if users_count == 0 || low_confidence_users == users_count { + "LOW_CONFIDENCE" + } else if overloaded_users > 0 || discipline_review_users > 0 || idle_users > 0 { + "ATTENTION" + } else if underloaded_users > 0 || unknown_or_no_data_users > 0 || medium_confidence_users > 0 { + "WATCH" + } else { + "OK" + }; + json!({ + "status": status, + "users_count": users_count, + "action_required_users": action_required_users, + "load": { + "unknown_or_no_data_users": unknown_or_no_data_users, + "underloaded_users": underloaded_users, + "normal_users": normal_users, + "overloaded_users": overloaded_users + }, + "idle": { + "idle_users": idle_users + }, + "discipline": { + "review_users": discipline_review_users + }, + "confidence": { + "low_users": low_confidence_users, + "medium_users": medium_confidence_users, + "high_users": high_confidence_users + }, + "guardrail": "Строки low confidence требуют проверки источников до персонального вывода" }) } @@ -2369,6 +2735,11 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec { "active_users", "inactive_users", "below_target_users", + "underloaded_users", + "overloaded_users", + "idle_users", + "discipline_review_users", + "low_confidence_users", "workday_total_active_seconds", "actions_count", "critical_actions_count", @@ -2390,6 +2761,24 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec { if row.get("status").and_then(Value::as_str) == Some("below_target") { inc(group, "below_target_users", 1); } + if matches!(row_str(row, "load_status"), "underloaded" | "below_target") { + inc(group, "underloaded_users", 1); + } + if row_str(row, "load_status") == "overloaded" { + inc(group, "overloaded_users", 1); + } + if matches!( + row_str(row, "idle_status"), + "idle_detected" | "full_workday_idle_or_absent" + ) { + inc(group, "idle_users", 1); + } + if !matches!(row_str(row, "discipline_status"), "" | "ok") { + inc(group, "discipline_review_users", 1); + } + if row_str(row, "data_confidence") == "low" { + inc(group, "low_confidence_users", 1); + } inc( group, "workday_total_active_seconds", @@ -3350,9 +3739,9 @@ fn render_management_html(payload: &Value) -> String { .collect() }; let user_rows = if rows.is_empty() { - "Нет сотрудников в выборке.".to_string() + "Нет сотрудников в выборке.".to_string() } else { - rows.iter().map(|r| format!("{}{}{}{}{}{}%{}{}", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["workday_first_activity_local"].as_str().unwrap_or("")), esc(r["workday_last_activity_local"].as_str().unwrap_or("")))).collect() + rows.iter().map(|r| format!("{}{}{}{}{}{}{}%{}{}{}{}{}", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), esc(r["workday_idle_hhmm"].as_str().unwrap_or("00:00")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["load_status"].as_str().unwrap_or("unknown")), esc(r["idle_status"].as_str().unwrap_or("unknown")), esc(r["discipline_status"].as_str().unwrap_or("unknown")), esc(r["data_confidence"].as_str().unwrap_or("low")), esc(r["operations_recommended_action"].as_str().unwrap_or("")))).collect() }; let source_rows = sources .iter() @@ -3369,7 +3758,7 @@ fn render_management_html(payload: &Value) -> String { }) .collect::(); format!( - r#"AW-rus Управленческий отчёт по работе в RDP

AW-rus Управленческий отчёт по работе в RDP

{} · {} · {}

Что делать сегодня

{}

Очередь действий руководителя

{}

Сотрудники

{}

Тренд за период

Тренд за {} дней

По ответственным

{}

Ответственные и эскалация

{}

По подразделениям

{}

Свежесть источников данных

{}

Фильтр: {}

"#, + r#"AW-rus Управленческий отчёт по работе в RDP

AW-rus Управленческий отчёт по работе в RDP

{} · {} · {}

Что делать сегодня

{}

Очередь действий руководителя

{}

Рабочая активность сотрудников

Статусы загрузки, простоя, дисциплины процесса и достоверности. Low confidence означает: сначала проверить источники, не делать персональный вывод.

{}
СотрудникОтветственныйПодразделениеСтатусАктивноПростойCoverageЗагрузкаПростойДисциплинаConfidenceДействие

Тренд за период

Тренд за {} дней

По ответственным

{}

Ответственные и эскалация

{}

По подразделениям

{}

Свежесть источников данных

{}

Фильтр: {}

"#, base_css(), esc(payload["host"].as_str().unwrap_or("")), esc(payload["report_date"].as_str().unwrap_or("")), @@ -3639,6 +4028,36 @@ mod tests { ); } + #[test] + fn dlp_evidence_bucket_is_optional_for_true_active_hot_path() { + let mut cfg = test_config(); + cfg.dlp_evidence_enabled = false; + let buckets = evidence_bucket_ids(&cfg, "SHARKON2025"); + assert!( + buckets + .iter() + .any(|bucket| bucket == "aw-file-operations_SHARKON2025") + ); + assert!( + buckets + .iter() + .any(|bucket| bucket == "aw-watcher-web-chrome_SHARKON2025") + ); + assert!( + !buckets + .iter() + .any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025") + ); + + cfg.dlp_evidence_enabled = true; + let buckets = evidence_bucket_ids(&cfg, "SHARKON2025"); + assert!( + buckets + .iter() + .any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025") + ); + } + #[test] fn management_insights_detect_falling_portfolio_trend() { let cfg = test_config(); @@ -3758,6 +4177,85 @@ mod tests { assert_eq!(rollups[0]["actions_count"], 1); } + #[test] + fn employee_operations_detects_overload_and_off_hours() { + let cfg = test_config(); + let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap(); + let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date); + let input = EmployeeOperationsInput { + expected_seconds, + low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100, + target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100, + overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100, + work_secs: expected_seconds * 2, + calendar_secs: expected_seconds * 2 + 3600, + work_first: Some(work_start.with_timezone(&Utc)), + work_last: Some(work_end.with_timezone(&Utc)), + is_today: false, + late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes), + early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes), + samples_count: 20, + active_samples: 18, + sessions_count: 1, + }; + let status = build_employee_operations_status(&cfg, input); + assert_eq!(status["load_status"], "overloaded"); + assert_eq!(status["discipline_status"], "off_hours"); + assert_eq!(status["data_confidence"], "high"); + assert_eq!(status["guardrail"], "evidence_backed_manual_review_only"); + } + + #[test] + fn employee_operations_low_confidence_blocks_personnel_conclusion() { + let cfg = test_config(); + let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap(); + let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date); + let input = EmployeeOperationsInput { + expected_seconds, + low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100, + target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100, + overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100, + work_secs: 0, + calendar_secs: 0, + work_first: None, + work_last: None, + is_today: false, + late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes), + early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes), + samples_count: 0, + active_samples: 0, + sessions_count: 0, + }; + let status = build_employee_operations_status(&cfg, input); + assert_eq!(status["load_status"], "no_data"); + assert_eq!(status["idle_status"], "unknown"); + assert_eq!(status["data_confidence"], "low"); + assert_eq!(status["guardrail"], "low_confidence_not_for_discipline"); + assert!( + status["recommended_action"] + .as_str() + .unwrap() + .contains("не использовать как дисциплинарный") + ); + } + + #[test] + fn operations_summary_counts_attention_groups() { + let rows = vec![ + json!({"load_status":"overloaded","idle_status":"no_significant_idle","discipline_status":"ok","data_confidence":"high"}), + json!({"load_status":"underloaded","idle_status":"idle_detected","discipline_status":"late_start","data_confidence":"medium"}), + json!({"load_status":"no_data","idle_status":"unknown","discipline_status":"ok","data_confidence":"low"}), + ]; + let summary = summarize_operations(&rows); + assert_eq!(summary["status"], "ATTENTION"); + assert_eq!(summary["load"]["overloaded_users"], 1); + assert_eq!(summary["load"]["underloaded_users"], 1); + assert_eq!(summary["idle"]["idle_users"], 1); + assert_eq!(summary["discipline"]["review_users"], 1); + assert_eq!(summary["confidence"]["low_users"], 1); + assert_eq!(summary["action_required_users"], 3); + } + #[test] fn stale_legacy_rdp_sources_are_covered_by_fresh_rust_sources() { assert!(legacy_rdp_covered_by_rust_sources( @@ -3888,14 +4386,17 @@ mod tests { #[test] fn interpretation_policy_accepts_fraction_thresholds() { let policy: InterpretationPolicy = serde_json::from_value(json!({ - "overload_threshold": 0.92, + "overload_threshold": 1.15, "underload_threshold": 0.45, "drop_threshold_pct": 20, "night_work_after": "20:00", "weekend_work": true })) .unwrap(); - assert_eq!(threshold_to_pct(policy.overload_threshold.unwrap()), 92.0); + assert_eq!( + overload_threshold_to_pct(policy.overload_threshold.unwrap()).round(), + 115.0 + ); assert_eq!(threshold_to_pct(policy.underload_threshold.unwrap()), 45.0); assert_eq!(threshold_to_pct(policy.drop_threshold_pct.unwrap()), 20.0); assert_eq!( diff --git a/adk-rust/crates/worktime-autoheal/src/main.rs b/adk-rust/crates/worktime-autoheal/src/main.rs index e4644a1..902e1a7 100644 --- a/adk-rust/crates/worktime-autoheal/src/main.rs +++ b/adk-rust/crates/worktime-autoheal/src/main.rs @@ -275,10 +275,20 @@ impl AwClient { ) -> Result<()> { for chunk in events.chunks(chunk_size.max(1)) { let path = format!("/api/0/buckets/{bucket_id}/events"); - self.request_json(Method::POST, &path, Some(json!(chunk)), false)?; + self.request_status(Method::POST, &path, Some(json!(chunk)))?; } Ok(()) } + + fn request_status(&self, method: Method, path: &str, payload: Option) -> Result<()> { + let response = self.send_retry(method, path, payload)?; + let status = response.status(); + if status.is_success() { + Ok(()) + } else { + Err(anyhow!("ActivityWatch {path} returned HTTP {status}")) + } + } } fn log(message: &str) { diff --git a/configs/worktime-interpretation-policy.example.json b/configs/worktime-interpretation-policy.example.json index f9b5fa0..92cdfbe 100644 --- a/configs/worktime-interpretation-policy.example.json +++ b/configs/worktime-interpretation-policy.example.json @@ -1,5 +1,5 @@ { - "overload_threshold": 0.92, + "overload_threshold": 1.15, "underload_threshold": 0.45, "drop_threshold_pct": 20, "night_work_after": "20:00", diff --git a/docs/CONTOUR_CHECK_MATRIX_RU.md b/docs/CONTOUR_CHECK_MATRIX_RU.md index cbe8c86..a033f90 100644 --- a/docs/CONTOUR_CHECK_MATRIX_RU.md +++ b/docs/CONTOUR_CHECK_MATRIX_RU.md @@ -55,6 +55,21 @@ Live endpoints, hostnames, tokens and passwords must be supplied through `/etc/detmir/detmir-check.env` (systemd units) or another private environment file outside the public repository. +Production note checked on 2026-06-24: + +- `DETMIR_PORTAL_URL` must point to the local DetMir portal listener, + currently `http://127.0.0.1:8720`, for server-side health checks. If it is + omitted, `detmir-check` falls back to the public HTTPS gateway and protected + `/readyz`, `/version` and `/metrics` can correctly return `401`, producing a + false operational failure. +- `DETMIR_GATEWAY_HOST=127.0.0.1` is used with the local listener so the Host + header does not accidentally select the public protected gateway path. +- Cold `/api/reports` builds can take more than 60 seconds on the live contour + when cache is empty or concurrent checks are active. The production + `detmir-portal-prewarm.service` therefore uses `curl --max-time 180` and + `TimeoutStartSec=210`. Shorter 45-60 second limits caused false failed + systemd states while the portal eventually returned HTTP 200. + ## Текущий планировщик Proxmox, проверено 2026-06-21 На Proxmox уже присутствуют следующие регулярные проверки: @@ -75,10 +90,10 @@ file outside the public repository. Наблюдение: отдельный ежедневный полный gate по всей матрице AWatch-rus отсутствует. Его роль должен закрыть `awatch-contour-daily-check.timer`. -Наблюдение: последняя проверка `detmir-portal-prewarm.service` на момент осмотра -имела `Result=exit-code` и `ExecMainStatus=28`. Это не надо маскировать: -канонический check должен показывать такой сбой как fail/warn в зависимости от -политики эксплуатации. +Историческое наблюдение 2026-06-24: `detmir-portal-prewarm.service` был найден +в failed state из-за устаревшего `curl --max-time 60` для холодной сборки +`/api/reports`. В текущей ветке это оформлено как отдельный prewarm/resilience +пакет, а не как обязательная часть DLP production hot path. ## Матрица требований и проверок @@ -91,14 +106,36 @@ file outside the public repository. | Portal hardening | `/healthz`, `/readyz`, `/version`, `/metrics` | `detmir-check` | да | да | | Windows/RDP | TCP 5985 и 22 | `detmir-check` | да | да | | ActivityWatch buckets | AFK/window/worktime/session events | `detmir-check` | да | да | -| AWatch DLP buckets | endpoint signals/incidents/review/rules | `detmir-check` | да | да | -| AWatch DLP health | remote `dlp-health-check --json` через `detmir-dlp` | `detmir-check` | да | да | +| AWatch DLP buckets | endpoint signals/incidents/review/rules, только если DLP включен | `detmir-check` | условно | условно | +| AWatch DLP health | disabled/core_only должен быть SKIPPED/WARN, `light/full` проверяются через `detmir-dlp` | `detmir-check` | да | да | | Grafana evidence | свежий JSON артефакт Grafana check | `detmir-check` | да | да | | Security events backend | ClickHouse events, если включено | `detmir-check` | да | да | | Portal contract | role/API smoke | `scripts/awatch-production-hardening-smoke.mjs` | нет | да | | Pilot contract | demo/API smoke | `scripts/detmir-pilot-demo-smoke.mjs` | нет | да | | Registry/readiness docs | registry readiness check | `scripts/registry_readiness_check.sh` | опционально | да | +## Timeout/fail-closed параметры live contour + +После ручного live-прогона 2026-06-24 production +`/etc/detmir/detmir-check.env` должен содержать bounded timeouts, соответствующие +фактической latency AW datastore: + +```env +DETMIR_SERVICE_TIMEOUT_SECONDS=35 +DETMIR_BUCKET_TIMEOUT_SECONDS=35 +DETMIR_DLP_TIMEOUT_SECONDS=120 +DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS=300 +``` + +Назначение: + +- не считать bucket `DEAD` только из-за штатной 15-30 секундной latency + большого SQLite datastore; +- не оставлять `detmir-check`, `detmir-dlp`, `ssh` и remote + `dlp-health-check` хвосты при timeout; +- сохранять красный non-zero результат при реальной недоступности, но + завершать проверку bounded. + ## Fail-closed политика Ежедневный check должен завершаться non-zero, если падает обязательная область: @@ -108,7 +145,9 @@ file outside the public repository. - Gateway/Portal health; - RDP/Windows reachability; - свежесть обязательных bucket streams; -- AWatch DLP health; +- AWatch DLP health только если DLP runtime включен; при штатном + `AW_DLP_ENABLED=false`/`core_only` disabled-state не является отказом + Workforce/Worktime core; - Grafana evidence freshness. Event-driven buckets не должны считаться stale только из-за отсутствия новых diff --git a/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md b/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md index 70ea766..3bac9be 100644 --- a/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md +++ b/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md @@ -7,6 +7,20 @@ ClickHouse не является обязательной зависимость перезапускайте ClickHouse для восстановления отчетов рабочего времени, если нет отдельного подтвержденного отказа ClickHouse. +## Stable host id + +Worktime reports используют stable logical host id, а не обязательно текущее +Windows `COMPUTERNAME`. Для DetMir production текущий logical id: + +```text +SHARKON2025 +``` + +При переименовании RDP-сервера не меняйте `awHostname` автоматически. Сначала +обновите Windows account domain для задач, затем проверьте, что collectors +продолжают писать в bucket-и `*_SHARKON2025`. Подробный порядок: +`docs/WINDOWS_LOGICAL_HOST_ID_RU.md`. + ## Симптомы перегруза - `/portal/api/reports?role=executive` открывается медленно или отвечает @@ -226,6 +240,153 @@ curl -sS --max-time 8 http:///portal/api/health | jq curl -sS --max-time 12 "http:///portal/api/reports?role=executive" | jq '.status' ``` +## Production repair: AW SQLite hot path, 2026-06-30 + +Симптомы: + +- `activitywatch-server` отвечает `503` на bucket API; +- журнал содержит `poisoned lock` / `database is locked`; +- `aw-worktime-api` уходит в bounded `DEGRADED`; +- `/buckets/aw-worktime-sessions_/events?limit=...` тайм-аутится даже + при малом лимите; +- RDP browser/category collector пишет `bucket create failed` или timeout. + +Порядок безопасного восстановления: + +1. Остановить RDP guard и процессы `aw-windows-telemetry`, чтобы не продолжать + штурмовать AW API. +2. Остановить `aw-worktime-*` timers/services и другие локальные потребители AW + API. +3. Перезапустить `activitywatch-server` отдельно и проверить `/api/0/info`. +4. Если bucket metadata отвечает, но `/events` медленный, проверить SQLite plan: + +```sql +EXPLAIN QUERY PLAN +SELECT id,starttime,endtime,data +FROM events +WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_') +ORDER BY starttime DESC +LIMIT 100; +``` + +Если план строит `TEMP B-TREE FOR ORDER BY`, нужен составной индекс: + +```sql +CREATE INDEX IF NOT EXISTS events_bucketrow_starttime_desc_index +ON events(bucketrow, starttime DESC); +ANALYZE; +PRAGMA optimize; +PRAGMA integrity_check; +``` + +Индекс добавлять только в controlled window: + +- остановить `activitywatch-server`; +- сделать rollback backup SQLite DB; +- создать индекс; +- проверить `PRAGMA integrity_check = ok`; +- запустить `activitywatch-server`; +- проверить, что `/events?limit=100` больше не тайм-аутится. + +Production DetMir repair 2026-06-30: + +- оставлены две свежие ежедневные SQLite VACUUM backup-копии, старые backup-и + ротированы для освобождения места; +- создан rollback backup: + `/var/lib/activitywatch/backups/db/aw-sqlite-before-hotpath-index-20260630T035032Z.db`; +- добавлен индекс `events_bucketrow_starttime_desc_index`; +- `ROCKET_WORKERS=8` добавлен в `/etc/activitywatch/aw-server.env`; +- для `aw-worktime-api.service` добавлен stabilization drop-in: + `AW_WORKTIME_EVENTS_LIMIT=100`, + `AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS=25`, + `AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS=600`, + `AW_WORKTIME_REPORT_STALE_TTL_SECONDS=7200`. + +После ремонта проверить: + +```bash +curl -sS --max-time 10 http://127.0.0.1:5600/api/0/info +curl -sS --max-time 15 \ + 'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100' +curl -sS --max-time 15 \ + 'http://127.0.0.1:5610/reports/worktime/today?format=json' | jq '{rows:(.rows|length),degraded,runtime}' +``` + +Также проверить отсутствие новых `poisoned lock` после финального старта: + +```bash +journalctl -u activitywatch-server --since '' --no-pager | + grep -E 'poisoned lock|Taking datastore lock failed|database is locked' +``` + +## Crash/readiness test after AW repair + +Цель: проверить, что контур выдерживает restart и короткую параллельную +нагрузку, а проверки не путают `systemctl active` с готовым API. + +Порядок: + +1. Зафиксировать baseline: + +```bash +./check-aw-full.sh +``` + +2. На AW server проверить readiness, hot-path и Worktime API: + +```bash +AW_API=http://127.0.0.1:5600 \ +AW_WORKTIME_API=http://127.0.0.1:5610 \ +AW_LOGICAL_HOST_ID=SHARKON2025 \ +scripts/detmir_resilience_check.sh --live +``` + +Если скрипт запускается с ноутбука, live-mode нужно выполнять на самом +AW-сервере через SSH/Ansible, потому что он проверяет local systemd и SQLite. + +3. Controlled restart: + +```bash +systemctl restart aw-worktime-api +curl -sS --max-time 12 \ + 'http://127.0.0.1:5610/reports/worktime/today?format=json&host=SHARKON2025&allow_stale=1' | + jq '{rows:(.rows|length),degraded}' + +systemctl restart activitywatch-server +# Не считать "active" готовностью: дождаться HTTP readiness. +timeout 90 bash -c 'until curl -fsS --max-time 8 http://127.0.0.1:5600/api/0/info >/dev/null; do sleep 2; done' +curl -sS --max-time 15 \ + 'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100' >/dev/null +``` + +4. Проверить, что после рестарта нет новых lock/503: + +```bash +journalctl -u activitywatch-server --since '' --no-pager | + grep -E 'poisoned lock|Taking datastore lock failed|database is locked|503' +``` + +5. Проверить RDP guard restart отдельно: + +```powershell +Restart-Service AWatchRusCollectorGuard -Force +Start-Sleep -Seconds 75 +Get-Service AWatchRusCollectorGuard +Get-Process aw-windows-telemetry -ErrorAction SilentlyContinue | Measure-Object +``` + +Ожидаемый результат для DetMir после ремонта 2026-06-30: + +- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`; +- `/events?limit=100` отвечает за bounded time и использует + `events_bucketrow_starttime_desc_index`; +- `aw-rus-healthd.service` завершается `status=0/SUCCESS`; +- server-side TCP до RDP может быть `warn`, если + `AW_RUS_HEALTH_RDP_TCP_REQUIRED=false`, но bucket freshness и WinRM/SSH + через admin path должны оставаться зелёными; +- optional DLP/Loki heavy runtime units должны быть inactive в экономном + production profile. + ## Rollback Rollback нужен, если после обновления бинарника или env-настроек: diff --git a/docs/WORKFORCE_OPERATIONS_MODEL_RU.md b/docs/WORKFORCE_OPERATIONS_MODEL_RU.md new file mode 100644 index 0000000..51a7c20 --- /dev/null +++ b/docs/WORKFORCE_OPERATIONS_MODEL_RU.md @@ -0,0 +1,239 @@ +# Workforce Operations Model + +Статус: implemented in Worktime API and DetMir portal. + +Модель отвечает на главный управленческий вопрос AWatch-rus Workforce: +рабочая активность сотрудников, загрузка, простои, перегруз и дисциплина +рабочего процесса. Это rule-based слой операционного контроля. Он не является +HR-оценкой, не использует ML/LLM и не выполняет автоматических санкций. + +## Где смотреть + +Основные точки: + +- Worktime API: + `GET /reports/worktime/management?format=json`; +- Worktime HTML: + `GET /reports/worktime/management?format=html`; +- DetMir portal: + `/api/reports`, блок `workforce_operations`; +- UI портала: + роли `Руководитель` и вкладка `Отчеты`, блок `Операционная загрузка`. + +## Источники + +Модель использует только подтвержденные рабочие источники: + +- ActivityWatch worktime rows; +- bucket рабочих сессий RDP; +- интервалы активности в рабочем окне; +- configured owner/department aliases; +- freshness/coverage metadata, которые уже возвращает Worktime API. + +Отсутствие данных не считается простоем. При пропусках источников строка +получает `data_confidence=low` и guardrail +`low_confidence_not_for_discipline`. + +## Runtime-настройки + +Основной файл политики: + +- пример: `configs/worktime-interpretation-policy.example.json`; +- runtime: `/etc/activitywatch/worktime-interpretation-policy.json`; +- env path: `AW_WORKTIME_MANAGER_INTERPRETATION_POLICY`. + +Поля policy: + +| Поле | Смысл | Рекомендуемое значение | +| --- | --- | --- | +| `underload_threshold` | порог недогруза от рабочего окна | `0.35..0.45` | +| `overload_threshold` | порог перегруза от рабочего окна | `1.10..1.25` | +| `drop_threshold_pct` | порог просадки тренда | `10..25` | +| `night_work_after` | начало вечернего/ночного отклонения | `20:00` | +| `weekend_work` | учитывать выходные отклонения | `true` | +| `min_trend_points` | минимум daily points для тренда | `3..7` | +| `off_hours_threshold_seconds` | минимум внерабочей активности для флага | `1800` | + +`underload_threshold` и `overload_threshold` можно задавать дробью или +процентом: `0.45` равно `45`, `1.15` равно `115`. +Для перегруза effective threshold fail-closed зажат в диапазон `100..300`, чтобы +значение ниже 100% не создавало ложный статус перегруза. + +Env fallback: + +- `AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT`; +- `AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT`; +- `AW_WORKTIME_MANAGER_OVERLOAD_COVERAGE_PCT`; +- `AW_WORKTIME_MANAGER_TREND_MIN_POINTS`; +- `AW_WORKTIME_MANAGER_TREND_DELTA_PCT`; +- `AW_WORKTIME_MANAGER_OFF_HOURS_THRESHOLD_SECONDS`; +- `AW_WORKTIME_MANAGER_NIGHT_WORK_AFTER`; +- `AW_WORKTIME_MANAGER_WEEKEND_WORK_ENABLED`. + +Веса приложений остаются отдельной политикой: + +- пример: `configs/detmir-workforce-policy.example.json`; +- runtime: `/etc/detmir-portal-workforce-policy.json`. + +Она влияет на explainable KPI и weighted activity, но не подменяет +операционные статусы загрузки/простоя. + +## API contract + +`/reports/worktime/management?format=json` содержит: + +```json +{ + "workday": { + "target_coverage_pct": 75, + "low_coverage_pct": 35, + "overload_coverage_pct": 115 + }, + "workforce_operations": { + "status": "ATTENTION", + "summary": {}, + "rows": [], + "model": { + "type": "rule_based", + "ml": false, + "llm": false, + "version": "workforce-operations-v1" + } + } +} +``` + +Каждая строка сотрудника содержит: + +- `workday_active_seconds`, `workday_active_hhmm`; +- `workday_idle_seconds`, `workday_idle_hhmm`; +- `coverage_pct`; +- `load_status`; +- `idle_status`; +- `discipline_status`; +- `data_confidence`; +- `recommended_action`. + +Полный roster в `rows[]` дополнительно содержит `operations`, +`operations.evidence`, `operations.guardrail` и +`operations_recommended_action`. + +## Статусы загрузки + +| Status | Значение | Действие | +| --- | --- | --- | +| `insufficient_data` | рабочее окно еще не началось или равно нулю | не делать вывод | +| `no_data` | нет сессий или worktime samples | проверить источники | +| `no_activity` | сессия/данные есть, активности в окне нет | проверить присутствие и задачи | +| `underloaded` | ниже low threshold | проверить загрузку и доступ к процессам | +| `below_target` | ниже target threshold | уточнить причину отклонения | +| `normal` | в рабочем диапазоне | наблюдать | +| `overloaded` | выше overload threshold | проверить переработку и риск аврала | + +## Статусы простоя + +| Status | Значение | +| --- | --- | +| `not_applicable` | нет рабочего окна | +| `unknown` | нет достаточных источников | +| `full_workday_idle_or_absent` | активность в рабочем окне отсутствует | +| `idle_detected` | простой выше порога | +| `no_significant_idle` | существенный простой не найден | + +## Дисциплина процесса + +`discipline_status` показывает отклонение от рабочего процесса, а не +автоматическое нарушение: + +- `ok`; +- `off_hours`; +- `late_start`; +- `early_finish`; +- `multiple_flags`. + +Для текущего дня `early_finish` не выставляется до завершения рабочего окна. + +## Достоверность + +`data_confidence`: + +- `high`: есть session samples, worktime samples и active samples; +- `medium`: данных мало или нет active samples; +- `low`: нет сессий/worktime samples или рабочее окно невалидно. + +Правило: low confidence строки сначала проверяются как проблема источников. +Их нельзя использовать как персональный дисциплинарный вывод. + +## Summary + +`workforce_operations.summary` содержит: + +- `users_count`; +- `action_required_users`; +- `load.unknown_or_no_data_users`; +- `load.underloaded_users`; +- `load.normal_users`; +- `load.overloaded_users`; +- `idle.idle_users`; +- `discipline.review_users`; +- `confidence.low_users`; +- `confidence.medium_users`; +- `confidence.high_users`; +- `guardrail`. + +Summary status: + +- `LOW_CONFIDENCE`: нет строк или все строки low confidence; +- `ATTENTION`: есть перегруз, простой или дисциплинарные флаги; +- `WATCH`: есть недогруз, нет данных или low confidence; +- `OK`: отклонений нет. + +## UI contract + +Портал показывает отдельный блок `Операционная загрузка`: + +- сводка: требуют разбора, недогруз, перегруз, простой, дисциплина, low + confidence; +- таблица сотрудников: active/idle/coverage/load/idle/discipline/confidence; +- рекомендуемое действие; +- guardrail и версию rule-based модели. + +Это отдельный блок от `Почему такой индекс активности?`: explainable KPI +отвечает на вопрос "почему такой процент", а Workforce Operations отвечает +"кого и почему нужно разобрать". + +## Ограничения + +- Не утверждать автоматическую оценку эффективности сотрудника. +- Не считать missing data простоем. +- Не смешивать Security/Forensics claims с Workforce Operations. +- Не заявлять ML/LLM detection. +- Не выполнять автоматическое remediation/action. +- Не использовать GitHub Actions или демо-данные как registry release evidence. + +## Проверка после изменения + +Минимальный локальный контур: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust +export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target +cargo fmt --all --check +cargo test -p worktime-api -p detmir-portal --locked +cargo clippy -p worktime-api -p detmir-portal --all-targets --locked -- -D warnings +``` + +Минимальный live smoke: + +```bash +curl -fsS 'http://10.10.10.13:5610/reports/worktime/management?format=json' \ + | jq '.workforce_operations.summary' + +curl -fsS 'http://10.10.10.2:8720/api/reports?role=manager' \ + | jq '{status: .workforce_operations.summary.status, rows: (.workforce_operations.rows | length)}' +``` + +Браузерный smoke: открыть `http://10.10.10.2:8720/`, выбрать представление +менеджера и проверить блок `Операционная загрузка`. В рабочем состоянии должны +быть видны summary-карточки, таблица сотрудников, `workforce-operations-v1` и +guardrail про `low confidence`. diff --git a/ops/systemd/awatch-contour-daily-check.service b/ops/systemd/awatch-contour-daily-check.service index a7fbfb2..dd27036 100644 --- a/ops/systemd/awatch-contour-daily-check.service +++ b/ops/systemd/awatch-contour-daily-check.service @@ -12,6 +12,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 +Environment=DETMIR_DLP_ENABLED=false EnvironmentFile=-/etc/detmir/detmir-check.env EnvironmentFile=-/etc/awatch-rus/contour-check.env ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check diff --git a/ops/systemd/awatch-contour-weekly-check.service b/ops/systemd/awatch-contour-weekly-check.service index 06b75d4..a0f3cdc 100644 --- a/ops/systemd/awatch-contour-weekly-check.service +++ b/ops/systemd/awatch-contour-weekly-check.service @@ -13,6 +13,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 +Environment=DETMIR_DLP_ENABLED=false EnvironmentFile=-/etc/detmir/detmir-check.env EnvironmentFile=-/etc/awatch-rus/contour-check.env ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check diff --git a/scripts/detmir_resilience_check.sh b/scripts/detmir_resilience_check.sh new file mode 100644 index 0000000..1e563a4 --- /dev/null +++ b/scripts/detmir_resilience_check.sh @@ -0,0 +1,471 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODE="repo" +AW_API="${AW_API:-http://127.0.0.1:5600}" +AW_WORKTIME_API="${AW_WORKTIME_API:-http://127.0.0.1:5610}" +AW_LOGICAL_HOST_ID="${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-SHARKON2025}}" +AW_READINESS_TIMEOUT_SECONDS="${AW_READINESS_TIMEOUT_SECONDS:-60}" +AW_READINESS_INTERVAL_SECONDS="${AW_READINESS_INTERVAL_SECONDS:-2}" +AW_EVENTS_LIMIT="${AW_EVENTS_LIMIT:-100}" +AW_EVENTS_MAX_SECONDS="${AW_EVENTS_MAX_SECONDS:-15}" +HAYA_ROOT="${AW_HAYABUSA_ROOT:-/opt/hayabusa}" +HAYA_DROP_DIR="${AW_HAYABUSA_DROP_DIR:-/opt/activitywatch/aw-rus-ops/drop}" +SQLITE_DB="${AW_SQLITE_DB:-/var/lib/activitywatch/aw-server-rust/sqlite.db}" +MAX_INCOMING_AGE_SECONDS="${MAX_HAYABUSA_INCOMING_AGE_SECONDS:-900}" +STRICT_SECRETS="${DETMIR_RESILIENCE_STRICT_SECRETS:-0}" +EXPECT_DLP_PROFILE="${DETMIR_RESILIENCE_EXPECT_DLP_PROFILE:-light}" +EXPECT_OPTIONAL_DLP_OFF="${DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF:-0}" +EXPECT_LOKI_OFF="${DETMIR_RESILIENCE_EXPECT_LOKI_OFF:-1}" + +DLP_RUNTIME_UNITS=( + aw-dlp-influx-exporter.timer + aw-dlp-influx-exporter.service + activitywatch-dlp-aggregator.timer + activitywatch-dlp-aggregator.service + aw-dlp-report-scheduler.timer + aw-dlp-report-scheduler.service + aw-dlp-syslog-forwarder.timer + aw-dlp-syslog-forwarder.service + aw-dlp-webhook-sender.timer + aw-dlp-webhook-sender.service + aw-dlp-cef-exporter.timer + aw-dlp-cef-exporter.service + aw-dlp-ioc-refresh.timer + aw-dlp-ioc-refresh.service + aw-dlp-policy-engine.service + aw-dlp-case-management.service + detmir-portal-evidence.service +) + +DLP_LIGHT_ALLOWED_UNITS=( + activitywatch-dlp-aggregator.timer + activitywatch-dlp-aggregator.service + aw-dlp-ioc-refresh.timer + aw-dlp-ioc-refresh.service + detmir-dlp-load-guard.timer + detmir-dlp-load-guard.service +) + +DLP_HEAVY_RUNTIME_UNITS=( + aw-dlp-influx-exporter.timer + aw-dlp-influx-exporter.service + aw-dlp-report-scheduler.timer + aw-dlp-report-scheduler.service + aw-dlp-syslog-forwarder.timer + aw-dlp-syslog-forwarder.service + aw-dlp-webhook-sender.timer + aw-dlp-webhook-sender.service + aw-dlp-cef-exporter.timer + aw-dlp-cef-exporter.service + aw-dlp-policy-engine.service + aw-dlp-case-management.service + detmir-portal-evidence.service +) + +LOKI_RUNTIME_UNITS=( + loki.service + promtail.service +) + +OK_COUNT=0 +WARN_COUNT=0 +FAIL_COUNT=0 + +usage() { + cat <<'EOF' +Usage: + scripts/detmir_resilience_check.sh [--repo|--live|--all] + +Modes: + --repo check repository hardening and docs only (default, CI-safe) + --live read-only checks for the local AW server/Hayabusa host + --all repo + live + +Environment: + AW_API=http://127.0.0.1:5600 + AW_WORKTIME_API=http://127.0.0.1:5610 + AW_LOGICAL_HOST_ID=SHARKON2025 + AW_READINESS_TIMEOUT_SECONDS=60 + AW_EVENTS_MAX_SECONDS=15 + AW_HAYABUSA_ROOT=/opt/hayabusa + AW_HAYABUSA_DROP_DIR=/opt/activitywatch/aw-rus-ops/drop + AW_SQLITE_DB=/var/lib/activitywatch/aw-server-rust/sqlite.db + DETMIR_RESILIENCE_EXPECT_DLP_PROFILE=light + DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF=0 + DETMIR_RESILIENCE_EXPECT_LOKI_OFF=1 + DETMIR_RESILIENCE_STRICT_SECRETS=1 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) MODE="repo"; shift ;; + --live) MODE="live"; shift ;; + --all) MODE="all"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +ok() { + OK_COUNT=$((OK_COUNT + 1)) + printf '[OK] %s\n' "$*" +} + +warn() { + WARN_COUNT=$((WARN_COUNT + 1)) + printf '[WARN] %s\n' "$*" +} + +fail() { + FAIL_COUNT=$((FAIL_COUNT + 1)) + printf '[FAIL] %s\n' "$*" +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +require_file() { + local path="$1" + if [[ -f "$ROOT_DIR/$path" ]]; then + ok "file exists: $path" + else + fail "missing file: $path" + fi +} + +require_pattern() { + local path="$1" + local pattern="$2" + local label="$3" + if grep -Eq "$pattern" "$ROOT_DIR/$path"; then + ok "$label" + else + fail "$label" + fi +} + +check_repo() { + printf '== repo resilience checks ==\n' + + require_file "scripts/detmir_resilience_check.sh" + require_file "docs/DETMIR_RESILIENCE_HARDENING_RU.md" + require_file "docs/DLP_RESOURCE_PROFILES_RU.md" + require_file "docs/DLP_OPTIONAL_RUNTIME_RU.md" + require_file "scripts/detmir_dlp_load_guard.sh" + require_file "scripts/detmir_dlp_warehouse_sync.sh" + require_file "aw-server/hayabusa/aw-hayabusa.sh" + require_file "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" + require_file "windows/AWatchRusCollectorGuardService.cs" + require_file "windows/install-collector-guard-service.ps1" + + require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "HAYA_QUARANTINE_DIR" "Hayabusa wrapper has quarantine root" + require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "quarantine_incoming_package" "Hayabusa wrapper isolates incoming poison packages" + require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "validate_drop_inputs" "Hayabusa autoprocess validates drop package before accept" + require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "quarantine_drop_package" "Hayabusa autoprocess quarantines bad drop package" + require_pattern "windows/AWatchRusCollectorGuardService.cs" "Process\\.Exited|ChildExited" "Collector guard service watches child exit" + require_pattern "windows/AWatchRusCollectorGuardService.cs" "MaxChildRestartsInWindow" "Collector guard service has bounded child restart budget" + require_pattern "windows/install-collector-guard-service.ps1" "failureflag" "Collector guard installer enables SCM failureflag" + require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Hayabusa poison-package isolation" "Resilience doc records Hayabusa hardening" + require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Windows collector guard service child watchdog" "Resilience doc records guard child watchdog" + require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "core_only" "DLP resource profiles document core_only" + require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "auto.?disable|автоотключ" "DLP resource profiles document auto-disable guard" + require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "rollback" "DLP resource profiles document rollback" + require_pattern "docs/DETMIR_CURRENT_STATE_RU.md" "AW_DLP_PROFILE=light" "Current state records DLP light profile" + require_pattern "scripts/detmir_dlp_runtime_control.sh" "set-profile" "DLP runtime control supports profile switching" + require_pattern "scripts/detmir_dlp_runtime_control.sh" "rollback_dlp" "DLP runtime control supports rollback" + require_pattern "scripts/detmir_dlp_load_guard.sh" "set-profile core_only" "DLP load guard can auto-disable DLP to core_only" + require_pattern "scripts/detmir_dlp_load_guard.sh" "STRIKES_REQUIRED" "DLP load guard requires consecutive overload checks" + require_pattern "scripts/detmir_dlp_warehouse_sync.sh" "sqlite3 .*\\.backup" "DLP warehouse sync uses SQLite backup" + require_pattern "ansible/group_vars/all.yml" 'aw_dlp_profile: "light"' "Production defaults keep DLP profile light" + require_pattern "ansible/group_vars/all.yml" 'aw_dlp_enabled: true' "Production defaults enable lightweight DLP" + require_pattern "ansible/group_vars/all.yml" 'aw_dlp_influx_enabled: false' "Production defaults keep DLP Influx disabled" + require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_collector_enabled: true' "Production defaults enable lightweight DLP collector" + require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_guard_enabled: true' "Production defaults enable DLP load guard" + require_pattern "ansible/group_vars/all.yml" 'detmir_portal_dlp_module_enabled_override: true' "Production defaults expose DLP light status to portal" + require_pattern "ansible/deploy_aw_server.yml" "detmir-dlp-load-guard.service" "AW server deploy installs DLP load guard service" + require_pattern "ansible/deploy_aw_server.yml" "CPUQuota=.*aw_dlp_aggregator_cpu_quota" "DLP aggregator has systemd CPU quota" + require_pattern "ansible/deploy_detmir_portal.yml" "detmir-dlp-warehouse-sync.service" "Portal deploy installs DLP warehouse sync service" + require_pattern "ansible/deploy_detmir_portal.yml" "detmir_portal_dlp_module_enabled_override \\| default\\(false\\)" "Portal deploy defaults DLP module to disabled" + + if [[ -f "$ROOT_DIR/ansible/inventory.ini" ]] && grep -Eq '(^|[[:space:]])ansible_(become_)?password[[:space:]]*=[[:space:]]*[^<{]' "$ROOT_DIR/ansible/inventory.ini"; then + if [[ "$STRICT_SECRETS" == "1" ]]; then + fail "ansible/inventory.ini appears to contain literal password assignments; move them to vault/env" + else + warn "ansible/inventory.ini appears to contain literal password assignments; strict mode would fail" + fi + else + ok "no literal ansible password assignments detected in ansible/inventory.ini" + fi +} + +check_systemd_unit() { + local unit="$1" + if ! have systemctl; then + warn "systemctl unavailable; skipping $unit" + return + fi + if systemctl is-active --quiet "$unit"; then + ok "systemd active: $unit" + else + fail "systemd not active: $unit" + fi +} + +check_aw_api() { + if ! have curl; then + warn "curl unavailable; skipping AW API check" + return + fi + local deadline last_code elapsed + deadline=$((SECONDS + AW_READINESS_TIMEOUT_SECONDS)) + last_code="" + while (( SECONDS <= deadline )); do + elapsed="$( + curl -sS --connect-timeout 3 --max-time 8 -o /dev/null -w '%{http_code} %{time_total}' \ + "$AW_API/api/0/info" 2>/dev/null || true + )" + last_code="${elapsed%% *}" + if [[ "$last_code" == "200" ]]; then + ok "ActivityWatch API readiness /api/0/info returns 200 (${elapsed#* }s)" + return + fi + sleep "$AW_READINESS_INTERVAL_SECONDS" + done + case "$last_code" in + 503) fail "ActivityWatch API readiness ended on 503; possible datastore lock poisoning" ;; + ""|000) fail "ActivityWatch API did not become ready within ${AW_READINESS_TIMEOUT_SECONDS}s" ;; + *) fail "ActivityWatch API readiness unexpected final HTTP status: $last_code" ;; + esac +} + +check_aw_hot_path() { + if ! have curl; then + warn "curl unavailable; skipping AW hot-path event check" + return + fi + local url result code elapsed + url="${AW_API%/}/api/0/buckets/aw-worktime-sessions_${AW_LOGICAL_HOST_ID}/events?limit=${AW_EVENTS_LIMIT}" + result="$(curl -sS --connect-timeout 3 --max-time "$AW_EVENTS_MAX_SECONDS" -o /dev/null -w '%{http_code} %{time_total}' "$url" 2>/dev/null || true)" + code="${result%% *}" + elapsed="${result#* }" + if [[ "$code" != "200" ]]; then + fail "ActivityWatch worktime events hot path returned HTTP ${code:-none}" + return + fi + if awk -v t="$elapsed" -v max="$AW_EVENTS_MAX_SECONDS" 'BEGIN { exit !(t <= max) }'; then + ok "ActivityWatch worktime events hot path returns 200 (${elapsed}s, limit=${AW_EVENTS_LIMIT})" + else + fail "ActivityWatch worktime events hot path exceeded ${AW_EVENTS_MAX_SECONDS}s (${elapsed}s)" + fi +} + +check_worktime_api() { + if ! have curl; then + warn "curl unavailable; skipping Worktime API check" + return + fi + local tmp code + tmp="$(mktemp)" + code="$(curl -sS --connect-timeout 3 --max-time 12 -o "$tmp" -w '%{http_code}' \ + "${AW_WORKTIME_API%/}/reports/worktime/today?format=json&host=${AW_LOGICAL_HOST_ID}&allow_stale=1" 2>/dev/null || true)" + if [[ "$code" != "200" ]]; then + fail "Worktime API today report returned HTTP ${code:-none}" + rm -f "$tmp" + return + fi + if have jq; then + if jq -e '(.degraded // false) == false' "$tmp" >/dev/null 2>&1; then + ok "Worktime API today report is not degraded" + else + fail "Worktime API today report is degraded" + fi + if jq -e '((.rows // .users // []) | length) > 0' "$tmp" >/dev/null 2>&1; then + ok "Worktime API today report has employee rows" + else + fail "Worktime API today report has no employee rows" + fi + else + ok "Worktime API today report returns 200" + fi + rm -f "$tmp" +} + +check_failed_units() { + if ! have systemctl; then + return + fi + local failed + failed="$(systemctl --failed --no-legend --plain 2>/dev/null | wc -l | tr -d ' ')" + if [[ "$failed" == "0" ]]; then + ok "systemd failed units count is 0" + else + fail "systemd failed units count is $failed" + fi +} + +count_files() { + local dir="$1" + local pattern="$2" + if [[ ! -d "$dir" ]]; then + printf '0\n' + return + fi + find "$dir" -maxdepth 1 -type f -name "$pattern" | wc -l | tr -d ' ' +} + +check_hayabusa_queues() { + local incoming_dir="$HAYA_ROOT/inbox/incoming" + local quarantine_dir="$HAYA_ROOT/quarantine" + local incoming_count drop_count old_count quarantine_count + incoming_count="$(count_files "$incoming_dir" '*.zip')" + drop_count="$(count_files "$HAYA_DROP_DIR" '*.zip')" + old_count="0" + if [[ -d "$incoming_dir" ]]; then + old_count="$(find "$incoming_dir" -maxdepth 1 -type f -name '*.zip' -mmin "+$((MAX_INCOMING_AGE_SECONDS / 60))" | wc -l | tr -d ' ')" + fi + if [[ "$incoming_count" == "0" ]]; then + ok "Hayabusa incoming zip count is 0" + else + warn "Hayabusa incoming zip count is $incoming_count" + fi + if [[ "$drop_count" == "0" ]]; then + ok "Hayabusa drop zip count is 0" + else + warn "Hayabusa drop zip count is $drop_count" + fi + if [[ "$old_count" == "0" ]]; then + ok "Hayabusa incoming has no stale zip older than ${MAX_INCOMING_AGE_SECONDS}s" + else + fail "Hayabusa incoming has $old_count stale zip package(s)" + fi + if [[ -d "$quarantine_dir" ]]; then + quarantine_count="$(find "$quarantine_dir" -type f -name reason.json | wc -l | tr -d ' ')" + if [[ "$quarantine_count" == "0" ]]; then + ok "Hayabusa quarantine reason count is 0" + else + warn "Hayabusa quarantine reason count is $quarantine_count; review/replay policy required" + fi + else + warn "Hayabusa quarantine root not found yet: $quarantine_dir" + fi +} + +check_sqlite_files() { + if [[ ! -f "$SQLITE_DB" ]]; then + warn "AW SQLite DB not found at $SQLITE_DB; skipping local DB size check" + return + fi + local db_size wal_size + db_size="$(stat -c '%s' "$SQLITE_DB")" + wal_size="0" + [[ -f "$SQLITE_DB-wal" ]] && wal_size="$(stat -c '%s' "$SQLITE_DB-wal")" + if (( db_size > 5 * 1024 * 1024 * 1024 )); then + fail "AW SQLite DB exceeds 5GiB" + elif (( db_size > 2 * 1024 * 1024 * 1024 )); then + warn "AW SQLite DB exceeds 2GiB" + else + ok "AW SQLite DB size below 2GiB" + fi + if (( wal_size > 1024 * 1024 * 1024 )); then + fail "AW SQLite WAL exceeds 1GiB" + elif (( wal_size > 256 * 1024 * 1024 )); then + warn "AW SQLite WAL exceeds 256MiB" + else + ok "AW SQLite WAL size below 256MiB" + fi +} + +check_sqlite_hot_path_index() { + if [[ ! -f "$SQLITE_DB" ]]; then + warn "AW SQLite DB not found at $SQLITE_DB; skipping hot-path index check" + return + fi + if ! have sqlite3; then + warn "sqlite3 unavailable; skipping hot-path index check" + return + fi + local index_exists plan + index_exists="$(sqlite3 "$SQLITE_DB" "SELECT name FROM sqlite_master WHERE type='index' AND name='events_bucketrow_starttime_desc_index';" 2>/dev/null || true)" + if [[ "$index_exists" == "events_bucketrow_starttime_desc_index" ]]; then + ok "AW SQLite hot-path index exists" + else + fail "AW SQLite hot-path index events_bucketrow_starttime_desc_index is missing" + return + fi + plan="$( + sqlite3 "$SQLITE_DB" "EXPLAIN QUERY PLAN SELECT id,starttime,endtime,data FROM events WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_${AW_LOGICAL_HOST_ID}') ORDER BY starttime DESC LIMIT ${AW_EVENTS_LIMIT};" 2>/dev/null || true + )" + if printf '%s' "$plan" | grep -q 'events_bucketrow_starttime_desc_index'; then + ok "AW SQLite worktime event query uses hot-path index" + else + fail "AW SQLite worktime event query does not use hot-path index" + fi + if printf '%s' "$plan" | grep -q 'TEMP B-TREE'; then + fail "AW SQLite worktime event query still builds TEMP B-TREE" + else + ok "AW SQLite worktime event query avoids TEMP B-TREE" + fi +} + +check_units_inactive() { + local label="$1" + shift + if ! have systemctl; then + warn "systemctl unavailable; skipping $label runtime check" + return + fi + local active_units=() + local unit active + for unit in "$@"; do + active="$(systemctl is-active "$unit" 2>/dev/null || true)" + if [[ "$active" == "active" || "$active" == "activating" ]]; then + active_units+=("$unit:$active") + fi + done + if [[ "${#active_units[@]}" -eq 0 ]]; then + ok "$label runtime units are inactive" + else + fail "$label runtime units active: ${active_units[*]}" + fi +} + +check_live() { + printf '== live resilience checks ==\n' + check_systemd_unit "activitywatch-server" + check_systemd_unit "aw-worktime-api" + check_aw_api + check_aw_hot_path + check_worktime_api + check_failed_units + check_hayabusa_queues + check_sqlite_files + check_sqlite_hot_path_index + if [[ "$EXPECT_OPTIONAL_DLP_OFF" == "1" || "$EXPECT_DLP_PROFILE" == "core_only" ]]; then + check_units_inactive "optional DLP" "${DLP_RUNTIME_UNITS[@]}" + elif [[ "$EXPECT_DLP_PROFILE" == "light" ]]; then + check_units_inactive "heavy DLP" "${DLP_HEAVY_RUNTIME_UNITS[@]}" + else + warn "optional DLP runtime inactive check skipped for profile=$EXPECT_DLP_PROFILE" + fi + if [[ "$EXPECT_LOKI_OFF" == "1" ]]; then + check_units_inactive "Loki" "${LOKI_RUNTIME_UNITS[@]}" + else + warn "Loki inactive check skipped by env" + fi +} + +case "$MODE" in + repo) check_repo ;; + live) check_live ;; + all) check_repo; check_live ;; + *) fail "invalid mode: $MODE" ;; +esac + +printf 'summary: ok=%s warn=%s fail=%s\n' "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT" +if (( FAIL_COUNT > 0 )); then + exit 1 +fi