Compare commits

...
Author SHA1 Message Date
igor04091968 812851063e Fix dlp health clippy warning
CI / Rust checks (push) Canceled after 0s
CI / Docs and registry checks (push) Canceled after 0s
CI / Smoke checks (push) Canceled after 0s
Coverage / Coverage baseline (push) Canceled after 0s
Security / Cargo audit (push) Canceled after 0s
Security / Cargo deny (push) Canceled after 0s
Security / Secret pattern check (push) Canceled after 0s
Security / Dependency review (push) Canceled after 0s
2026-07-01 06:18:23 +03:00
igor04091968 c017cb08a9 Harden DetMir runtime hot paths 2026-07-01 06:06:01 +03:00
15 changed files with 1922 additions and 87 deletions
+30 -2
View File
@@ -43,6 +43,9 @@ struct Cli {
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
no_color: bool, no_color: bool,
#[arg(long, default_value_t = true)]
dlp_enabled: bool,
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -109,7 +112,12 @@ fn main() {
} }
fn run() -> Result<i32> { fn run() -> Result<i32> {
let cli = Cli::parse(); let mut cli = Cli::parse();
if let Some(value) =
env_nonempty("AW_DLP_ENABLED").or_else(|| env_nonempty("DETMIR_DLP_ENABLED"))
{
cli.dlp_enabled = parse_env_flag(&value);
}
let server = cli let server = cli
.server .server
.or_else(|| env_nonempty("AW_CHECK_SERVER")) .or_else(|| env_nonempty("AW_CHECK_SERVER"))
@@ -172,7 +180,11 @@ fn run() -> Result<i32> {
"---------------------------------------------", "--------", "----------------------" "---------------------------------------------", "--------", "----------------------"
); );
for bucket in BUCKETS { for bucket in BUCKETS
.iter()
.copied()
.filter(|bucket| cli.dlp_enabled || !bucket.starts_with("aw-dlp-"))
{
let bucket_full = format!("{bucket}_{host}"); let bucket_full = format!("{bucket}_{host}");
let event = bucket_event( let event = bucket_event(
&server, &server,
@@ -190,6 +202,15 @@ fn run() -> Result<i32> {
render_status(&colors, status) render_status(&colors, status)
); );
} }
if !cli.dlp_enabled {
println!(
"{:<45} {:<8} {:<22} {}",
"aw-dlp-*",
"-",
"disabled",
colors.paint(colors.cyan, "SKIPPED")
);
}
println!(); println!();
println!("--- CORS Check ---"); println!("--- CORS Check ---");
@@ -493,6 +514,13 @@ fn env_nonempty(name: &str) -> Option<String> {
.filter(|value| !value.is_empty()) .filter(|value| !value.is_empty())
} }
fn parse_env_flag(value: &str) -> bool {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
+16 -2
View File
@@ -1,6 +1,7 @@
use std::fs::{self, File, OpenOptions}; use std::fs::{self, File, OpenOptions};
use std::io::Write; use std::io::Write;
use std::os::unix::fs::symlink; use std::os::unix::fs::symlink;
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio}; use std::process::{Command, Output, Stdio};
use std::time::{Duration, SystemTime}; use std::time::{Duration, SystemTime};
@@ -198,6 +199,7 @@ fn run_to_file(
let mut child = Command::new(command) let mut child = Command::new(command)
.args(args) .args(args)
.process_group(0)
.stdout(Stdio::from(stdout)) .stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr)) .stderr(Stdio::from(stderr))
.spawn() .spawn()
@@ -209,7 +211,7 @@ fn run_to_file(
break status.code().unwrap_or(1); break status.code().unwrap_or(1);
} }
if started.elapsed() >= timeout { if started.elapsed() >= timeout {
let _ = child.kill(); terminate_process_group(child.id());
let _ = child.wait(); let _ = child.wait();
let mut stderr = OpenOptions::new().append(true).open(&stderr_path)?; let mut stderr = OpenOptions::new().append(true).open(&stderr_path)?;
writeln!( writeln!(
@@ -234,6 +236,17 @@ fn run_to_file(
Ok(rc) Ok(rc)
} }
fn terminate_process_group(child_pid: u32) {
let process_group = format!("-{child_pid}");
let _ = Command::new("/bin/kill")
.args(["-TERM", "--", &process_group])
.status();
std::thread::sleep(Duration::from_secs(2));
let _ = Command::new("/bin/kill")
.args(["-KILL", "--", &process_group])
.status();
}
fn read_rc(path: &Path) -> i32 { fn read_rc(path: &Path) -> i32 {
fs::read_to_string(path) fs::read_to_string(path)
.ok() .ok()
@@ -411,6 +424,7 @@ fn write_report(
fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Result<Output> { fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Result<Output> {
let mut child = Command::new(polli_bin) let mut child = Command::new(polli_bin)
.args(["--model", "text.daily", "--max-tokens", "900"]) .args(["--model", "text.daily", "--max-tokens", "900"])
.process_group(0)
.stdin(Stdio::from(bundle)) .stdin(Stdio::from(bundle))
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
@@ -425,7 +439,7 @@ fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Resul
.with_context(|| format!("failed to collect {polli_bin} output")); .with_context(|| format!("failed to collect {polli_bin} output"));
} }
if started.elapsed() >= timeout { if started.elapsed() >= timeout {
let _ = child.kill(); terminate_process_group(child.id());
let _ = child.wait(); let _ = child.wait();
anyhow::bail!( anyhow::bail!(
"Pollinations report timed out after {} seconds", "Pollinations report timed out after {} seconds",
+133 -30
View File
@@ -1,5 +1,6 @@
use std::io::Read; use std::io::Read;
use std::net::{SocketAddr, TcpStream}; use std::net::{SocketAddr, TcpStream};
use std::os::unix::process::CommandExt;
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
@@ -89,9 +90,15 @@ struct Cli {
#[arg(long, default_value_t = 45)] #[arg(long, default_value_t = 45)]
dlp_timeout_seconds: u64, dlp_timeout_seconds: u64,
#[arg(long, default_value_t = 150)]
overall_timeout_seconds: u64,
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
disable_dlp_health_check: bool, disable_dlp_health_check: bool,
#[arg(long, default_value_t = true)]
dlp_enabled: bool,
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
disable_portal_check: bool, disable_portal_check: bool,
} }
@@ -193,8 +200,8 @@ fn parse_env_flag(value: &str) -> bool {
) )
} }
fn bucket_specs(hostname: &str) -> Vec<BucketSpec> { fn bucket_specs(hostname: &str, dlp_enabled: bool) -> Vec<BucketSpec> {
vec![ let mut specs = vec![
BucketSpec { BucketSpec {
label: "AFK watcher", label: "AFK watcher",
bucket: format!("aw-watcher-afk_{hostname}"), bucket: format!("aw-watcher-afk_{hostname}"),
@@ -219,31 +226,36 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
max_age_seconds: None, max_age_seconds: None,
mode: BucketMode::EventDriven, mode: BucketMode::EventDriven,
}, },
BucketSpec { ];
label: "DLP signals", if dlp_enabled {
bucket: format!("aw-dlp-endpoint-signals_{hostname}"), specs.extend([
max_age_seconds: Some(10 * 60), BucketSpec {
mode: BucketMode::InteractiveFresh, label: "DLP signals",
}, bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
BucketSpec { max_age_seconds: Some(10 * 60),
label: "DLP incidents", mode: BucketMode::InteractiveFresh,
bucket: format!("aw-dlp-incidents_{hostname}"), },
max_age_seconds: None, BucketSpec {
mode: BucketMode::EventDriven, label: "DLP incidents",
}, bucket: format!("aw-dlp-incidents_{hostname}"),
BucketSpec { max_age_seconds: None,
label: "DLP review", mode: BucketMode::EventDriven,
bucket: format!("aw-dlp-review_{hostname}"), },
max_age_seconds: None, BucketSpec {
mode: BucketMode::EventDriven, label: "DLP review",
}, bucket: format!("aw-dlp-review_{hostname}"),
BucketSpec { max_age_seconds: None,
label: "DLP rules", mode: BucketMode::EventDriven,
bucket: format!("aw-dlp-rules_{hostname}"), },
max_age_seconds: None, BucketSpec {
mode: BucketMode::EventDriven, label: "DLP rules",
}, bucket: format!("aw-dlp-rules_{hostname}"),
] max_age_seconds: None,
mode: BucketMode::EventDriven,
},
]);
}
specs
} }
fn build_headers(items: &[(&str, &str)]) -> Result<HeaderMap> { fn build_headers(items: &[(&str, &str)]) -> Result<HeaderMap> {
@@ -387,7 +399,20 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
if security_events_clickhouse_enabled(args) { if security_events_clickhouse_enabled(args) {
checks.push(clickhouse_security_events_check(args)); checks.push(clickhouse_security_events_check(args));
} }
if !args.disable_dlp_health_check { if !args.dlp_enabled {
checks.push(ServiceCheck {
name: "aw-dlp-mode".to_string(),
required: false,
ok: true,
url: None,
payload: Some(serde_json::json!({
"mode": "disabled",
"reason": "DETMIR_DLP_ENABLED=false",
"checks_skipped": ["DLP health command", "DLP buckets"]
})),
error: None,
});
} else if !args.disable_dlp_health_check {
checks.push(dlp_health_check(args)); checks.push(dlp_health_check(args));
} }
checks checks
@@ -500,6 +525,7 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
let mut child = Command::new("/bin/sh") let mut child = Command::new("/bin/sh")
.arg("-lc") .arg("-lc")
.arg(command) .arg(command)
.process_group(0)
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::piped()) .stderr(Stdio::piped())
.spawn() .spawn()
@@ -510,7 +536,7 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
return read_command_output(child, status.code(), false); return read_command_output(child, status.code(), false);
} }
if started.elapsed() >= timeout { if started.elapsed() >= timeout {
let _ = child.kill(); terminate_process_group(child.id());
let _ = child.wait(); let _ = child.wait();
return read_command_output(child, None, true); return read_command_output(child, None, true);
} }
@@ -518,6 +544,17 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
} }
} }
fn terminate_process_group(child_pid: u32) {
let process_group = format!("-{child_pid}");
let _ = Command::new("/bin/kill")
.args(["-TERM", "--", &process_group])
.status();
std::thread::sleep(Duration::from_secs(2));
let _ = Command::new("/bin/kill")
.args(["-KILL", "--", &process_group])
.status();
}
fn read_command_output( fn read_command_output(
mut child: std::process::Child, mut child: std::process::Child,
code: Option<i32>, code: Option<i32>,
@@ -796,7 +833,7 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
let interactive_required = interactive_required(&client, &args.hostname, now); let interactive_required = interactive_required(&client, &args.hostname, now);
let mut out = Vec::new(); let mut out = Vec::new();
for spec in bucket_specs(&args.hostname) { for spec in bucket_specs(&args.hostname, args.dlp_enabled) {
if matches!(spec.mode, BucketMode::EventDriven) { if matches!(spec.mode, BucketMode::EventDriven) {
out.push(BucketCheck { out.push(BucketCheck {
label: spec.label.to_string(), label: spec.label.to_string(),
@@ -993,6 +1030,19 @@ fn render_text(report: &CheckReport) -> String {
fn main() -> Result<()> { fn main() -> Result<()> {
let mut args = Cli::parse(); let mut args = Cli::parse();
args.service_timeout_seconds = env_u64(
"DETMIR_SERVICE_TIMEOUT_SECONDS",
args.service_timeout_seconds,
);
args.bucket_timeout_seconds =
env_u64("DETMIR_BUCKET_TIMEOUT_SECONDS", args.bucket_timeout_seconds);
args.tcp_timeout_seconds = env_f64("DETMIR_TCP_TIMEOUT_SECONDS", args.tcp_timeout_seconds);
args.dlp_timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", args.dlp_timeout_seconds);
args.overall_timeout_seconds = env_u64(
"DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS",
args.overall_timeout_seconds,
);
start_overall_timeout_watchdog(args.overall_timeout_seconds);
args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api); args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api);
args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url); args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url); args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
@@ -1011,6 +1061,12 @@ fn main() -> Result<()> {
if env_flag_enabled("DETMIR_DISABLE_DLP_HEALTH_CHECK") { if env_flag_enabled("DETMIR_DISABLE_DLP_HEALTH_CHECK") {
args.disable_dlp_health_check = true; args.disable_dlp_health_check = true;
} }
if let Some(value) = std::env::var("DETMIR_DLP_ENABLED")
.ok()
.filter(|value| !value.is_empty())
{
args.dlp_enabled = parse_env_flag(&value);
}
if env_flag_enabled("DETMIR_DISABLE_PORTAL_CHECK") { if env_flag_enabled("DETMIR_DISABLE_PORTAL_CHECK") {
args.disable_portal_check = true; args.disable_portal_check = true;
} }
@@ -1028,6 +1084,31 @@ fn main() -> Result<()> {
}); });
} }
fn env_u64(name: &str, fallback: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(fallback)
}
fn env_f64(name: &str, fallback: f64) -> f64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(fallback)
}
fn start_overall_timeout_watchdog(seconds: u64) {
if seconds == 0 {
return;
}
std::thread::spawn(move || {
std::thread::sleep(Duration::from_secs(seconds));
eprintln!("detmir-check timed out after {seconds} seconds");
std::process::exit(124);
});
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -1095,6 +1176,28 @@ mod tests {
assert!(!parse_env_flag("false")); assert!(!parse_env_flag("false"));
} }
#[test]
fn dlp_disabled_removes_dlp_bucket_specs() {
let enabled = bucket_specs("HOST-EXAMPLE", true);
assert!(
enabled
.iter()
.any(|spec| spec.bucket.starts_with("aw-dlp-"))
);
let disabled = bucket_specs("HOST-EXAMPLE", false);
assert!(
disabled
.iter()
.all(|spec| !spec.bucket.starts_with("aw-dlp-"))
);
assert!(
disabled
.iter()
.any(|spec| spec.bucket.starts_with("aw-worktime-sessions_"))
);
}
#[test] #[test]
fn clickhouse_database_identifier_rejects_injection() { fn clickhouse_database_identifier_rejects_injection() {
assert_eq!( assert_eq!(
+1
View File
@@ -9,3 +9,4 @@ publish.workspace = true
[dependencies] [dependencies]
anyhow.workspace = true anyhow.workspace = true
clap.workspace = true clap.workspace = true
serde_json.workspace = true
+87 -3
View File
@@ -1,5 +1,6 @@
use std::io::{self, Write}; use std::io::{self, Write};
use std::process::Command; use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
@@ -19,8 +20,14 @@ struct Cli {
#[arg(long, default_value_t = 10)] #[arg(long, default_value_t = 10)]
connect_timeout_seconds: u64, connect_timeout_seconds: u64,
#[arg(long, default_value_t = 90)]
timeout_seconds: u64,
#[arg(long, default_value = DEFAULT_REMOTE_COMMAND)] #[arg(long, default_value = DEFAULT_REMOTE_COMMAND)]
remote_command: String, remote_command: String,
#[arg(long, default_value_t = true)]
enabled: bool,
} }
impl Cli { impl Cli {
@@ -31,6 +38,8 @@ impl Cli {
); );
self.remote_command = env_first(&["DETMIR_DLP_REMOTE_COMMAND"], &self.remote_command); self.remote_command = env_first(&["DETMIR_DLP_REMOTE_COMMAND"], &self.remote_command);
self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin); self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin);
self.timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", self.timeout_seconds);
self.enabled = env_bool("DETMIR_DLP_ENABLED", self.enabled);
self self
} }
} }
@@ -42,6 +51,25 @@ fn env_first(names: &[&str], fallback: &str) -> String {
.unwrap_or_else(|| fallback.to_string()) .unwrap_or_else(|| fallback.to_string())
} }
fn env_u64(name: &str, fallback: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(fallback)
}
fn env_bool(name: &str, fallback: bool) -> bool {
std::env::var(name)
.ok()
.map(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
.unwrap_or(fallback)
}
fn ssh_args(cli: &Cli) -> Vec<String> { fn ssh_args(cli: &Cli) -> Vec<String> {
vec![ vec![
"-o".to_string(), "-o".to_string(),
@@ -56,22 +84,76 @@ fn ssh_args(cli: &Cli) -> Vec<String> {
} }
fn run(cli: Cli) -> Result<i32> { fn run(cli: Cli) -> Result<i32> {
if !cli.enabled {
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"ok": true,
"counts": {"ok": 1, "warn": 0, "fail": 0},
"results": [{
"name": "dlp:mode",
"status": "ok",
"summary": "DLP health check disabled by DETMIR_DLP_ENABLED=false",
"details": {
"mode": "disabled",
"load_reduction": ["aw-dlp health ssh probe skipped"]
}
}]
}))?
);
return Ok(0);
}
let args = ssh_args(&cli); let args = ssh_args(&cli);
let output = Command::new(&cli.ssh_bin) let mut child = Command::new(&cli.ssh_bin)
.args(&args) .args(&args)
.output() .stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.with_context(|| format!("failed to execute {}", cli.ssh_bin))?; .with_context(|| format!("failed to execute {}", cli.ssh_bin))?;
let started = Instant::now();
let mut timed_out = false;
loop {
if child.try_wait()?.is_some() {
break;
}
if started.elapsed() >= Duration::from_secs(cli.timeout_seconds) {
timed_out = true;
terminate_child(&mut child);
break;
}
std::thread::sleep(Duration::from_millis(100));
}
let output = child
.wait_with_output()
.context("failed to collect SSH output")?;
io::stdout() io::stdout()
.write_all(&output.stdout) .write_all(&output.stdout)
.context("failed to write DLP stdout")?; .context("failed to write DLP stdout")?;
io::stderr() io::stderr()
.write_all(&output.stderr) .write_all(&output.stderr)
.context("failed to write DLP stderr")?; .context("failed to write DLP stderr")?;
if timed_out {
writeln!(
io::stderr(),
"detmir-dlp timed out after {} seconds",
cli.timeout_seconds
)
.context("failed to write timeout message")?;
return Ok(124);
}
Ok(output.status.code().unwrap_or(1)) Ok(output.status.code().unwrap_or(1))
} }
fn terminate_child(child: &mut std::process::Child) {
let _ = child.kill();
std::thread::sleep(Duration::from_secs(2));
let _ = child.kill();
}
fn main() -> Result<()> { fn main() -> Result<()> {
let cli = Cli::parse().apply_env(); let cli = Cli::parse().apply_env();
let code = run(cli)?; let code = run(cli)?;
@@ -88,7 +170,9 @@ mod tests {
ssh_bin: "ssh".to_string(), ssh_bin: "ssh".to_string(),
ssh_target: DEFAULT_SSH_TARGET.to_string(), ssh_target: DEFAULT_SSH_TARGET.to_string(),
connect_timeout_seconds: 10, connect_timeout_seconds: 10,
timeout_seconds: 90,
remote_command: DEFAULT_REMOTE_COMMAND.to_string(), remote_command: DEFAULT_REMOTE_COMMAND.to_string(),
enabled: true,
}; };
assert_eq!( assert_eq!(
ssh_args(&cli), ssh_args(&cli),
+210 -28
View File
@@ -62,6 +62,21 @@ struct Cli {
#[arg(long)] #[arg(long)]
json: bool, json: bool,
#[arg(long, default_value_t = 120)]
overall_timeout_seconds: u64,
#[arg(long, default_value = "full")]
profile: String,
#[arg(long, default_value_t = true)]
enabled: bool,
#[arg(long, default_value = "operator_disabled")]
disabled_reason: String,
#[arg(long, default_value = "")]
disabled_since: String,
} }
impl Cli { impl Cli {
@@ -129,6 +144,28 @@ impl Cli {
if !cli_arg_present("--profiles") { if !cli_arg_present("--profiles") {
self.profiles = env_string("AW_DLP_COMPLIANCE_PROFILES").unwrap_or(self.profiles); self.profiles = env_string("AW_DLP_COMPLIANCE_PROFILES").unwrap_or(self.profiles);
} }
if !cli_arg_present("--overall-timeout-seconds") {
self.overall_timeout_seconds = env_u64(
"AW_DLP_HEALTH_OVERALL_TIMEOUT_SECONDS",
self.overall_timeout_seconds,
);
}
if !cli_arg_present("--profile") {
self.profile = env_string("AW_DLP_PROFILE")
.or_else(|| env_string("DETMIR_PORTAL_DLP_PROFILE"))
.unwrap_or(self.profile);
}
if !cli_arg_present("--enabled") {
self.enabled = env_bool_default("AW_DLP_ENABLED", self.enabled);
}
if !cli_arg_present("--disabled-reason") {
self.disabled_reason =
env_string("AW_DLP_DISABLED_REASON").unwrap_or(self.disabled_reason);
}
if !cli_arg_present("--disabled-since") {
self.disabled_since =
env_string("AW_DLP_DISABLED_SINCE").unwrap_or(self.disabled_since);
}
self self
} }
} }
@@ -1270,8 +1307,66 @@ fn check_compliance_reports(
} }
} }
fn normalized_profile(profile: &str) -> String {
match profile.trim().to_ascii_lowercase().as_str() {
"disabled" | "off" => "core_only".to_string(),
"core-only" | "core_only" => "core_only".to_string(),
"light" | "lite" => "light".to_string(),
"on-demand" | "on_demand" => "on_demand".to_string(),
"enabled" | "on" | "full" => "full".to_string(),
"" => "full".to_string(),
other => other.to_string(),
}
}
fn profile_is_disabled(profile: &str) -> bool {
matches!(normalized_profile(profile).as_str(), "core_only")
}
fn profile_checks_heavy_services(profile: &str) -> bool {
matches!(normalized_profile(profile).as_str(), "full" | "on_demand")
}
fn build_report(cli: &Cli, client: &Client) -> HealthReport { fn build_report(cli: &Cli, client: &Client) -> HealthReport {
let mut report = HealthReport::default(); let mut report = HealthReport::default();
let profile = normalized_profile(&cli.profile);
if !cli.enabled || profile_is_disabled(&profile) {
report.add(
"dlp:mode",
"ok",
"DLP runtime disabled by production profile",
json!({
"mode": "disabled",
"profile": profile,
"reason": &cli.disabled_reason,
"disabled_since": empty_string_as_null(&cli.disabled_since),
"checks_skipped": [
"policy API",
"case API",
"DLP systemd units",
"DLP ActivityWatch buckets",
"DLP compliance reports"
],
"load_reduction": [
"no DLP bucket freshness reads",
"no DLP case/policy HTTP checks",
"no DLP compliance filesystem scan"
]
}),
);
return report;
}
report.add(
"dlp:mode",
"ok",
format!("DLP runtime profile {profile}"),
json!({
"mode": if profile == "light" { "light" } else { "enabled" },
"profile": profile,
"heavy_services_checked": profile_checks_heavy_services(&cli.profile)
}),
);
let aw_api_base = format!("{}/api/0", cli.aw_server.trim_end_matches('/')); let aw_api_base = format!("{}/api/0", cli.aw_server.trim_end_matches('/'));
let counter_state_path = cli.state_dir.join("dlp-health-check-counters.json"); let counter_state_path = cli.state_dir.join("dlp-health-check-counters.json");
let mut counter_state = load_counter_state(&counter_state_path); let mut counter_state = load_counter_state(&counter_state_path);
@@ -1282,38 +1377,70 @@ fn build_report(cli: &Cli, client: &Client) -> HealthReport {
"http:aw", "http:aw",
&format!("{aw_api_base}/info"), &format!("{aw_api_base}/info"),
); );
check_http_endpoint( if profile_checks_heavy_services(&cli.profile) {
&mut report, check_http_endpoint(
client, &mut report,
"http:policy", client,
&format!("{}/healthz", cli.policy_server.trim_end_matches('/')), "http:policy",
); &format!("{}/healthz", cli.policy_server.trim_end_matches('/')),
check_http_endpoint( );
&mut report, check_http_endpoint(
client, &mut report,
"http:cases", client,
&format!("{}/health", cli.case_server.trim_end_matches('/')), "http:cases",
); &format!("{}/health", cli.case_server.trim_end_matches('/')),
);
} else {
report.add(
"http:heavy-dlp",
"ok",
"heavy DLP policy/case HTTP checks skipped for lightweight profile",
json!({
"profile": profile,
"skipped": ["policy API", "case API"]
}),
);
}
for unit in [ for unit in ["activitywatch-server", "aw-worktime-api.service"] {
"activitywatch-server",
"aw-dlp-policy-engine.service",
"aw-dlp-case-management.service",
"aw-worktime-api.service",
] {
check_systemd_unit(&mut report, unit, "service"); check_systemd_unit(&mut report, unit, "service");
} }
for unit in [ if profile_checks_heavy_services(&cli.profile) {
"aw-dlp-report-scheduler.timer", for unit in [
"aw-dlp-syslog-forwarder.timer", "aw-dlp-policy-engine.service",
"aw-dlp-webhook-sender.timer", "aw-dlp-case-management.service",
"aw-dlp-cef-exporter.timer", ] {
"activitywatch-dlp-aggregator.timer", check_systemd_unit(&mut report, unit, "service");
"aw-dlp-ioc-refresh.timer", }
"aw-worktime-ui-bridge.timer", for unit in [
] { "aw-dlp-report-scheduler.timer",
check_systemd_unit(&mut report, unit, "timer"); "aw-dlp-syslog-forwarder.timer",
"aw-dlp-webhook-sender.timer",
"aw-dlp-cef-exporter.timer",
"activitywatch-dlp-aggregator.timer",
"aw-dlp-ioc-refresh.timer",
] {
check_systemd_unit(&mut report, unit, "timer");
}
} else {
report.add(
"systemd:heavy-dlp",
"ok",
"heavy DLP systemd checks skipped for lightweight profile",
json!({
"profile": profile,
"skipped": [
"aw-dlp-policy-engine.service",
"aw-dlp-case-management.service",
"aw-dlp-report-scheduler.timer",
"aw-dlp-syslog-forwarder.timer",
"aw-dlp-webhook-sender.timer",
"aw-dlp-cef-exporter.timer"
]
}),
);
} }
check_systemd_unit(&mut report, "aw-worktime-ui-bridge.timer", "timer");
match http_json(client, &format!("{aw_api_base}/buckets"), 15, 2) { match http_json(client, &format!("{aw_api_base}/buckets"), 15, 2) {
Ok(Value::Object(map)) => { Ok(Value::Object(map)) => {
@@ -1462,6 +1589,12 @@ fn env_i64(name: &str, default: i64) -> i64 {
.unwrap_or(default) .unwrap_or(default)
} }
fn env_u64(name: &str, default: u64) -> u64 {
env_string(name)
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(default)
}
fn env_bool(name: &str) -> bool { fn env_bool(name: &str) -> bool {
env_string(name) env_string(name)
.map(|value| { .map(|value| {
@@ -1473,8 +1606,39 @@ fn env_bool(name: &str) -> bool {
.unwrap_or(false) .unwrap_or(false)
} }
fn env_bool_default(name: &str, default: bool) -> bool {
env_string(name)
.map(|value| {
matches!(
value.to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
.unwrap_or(default)
}
fn empty_string_as_null(value: &str) -> Value {
if value.trim().is_empty() {
Value::Null
} else {
json!(value)
}
}
fn start_overall_timeout_watchdog(seconds: u64) {
if seconds == 0 {
return;
}
std::thread::spawn(move || {
sleep(Duration::from_secs(seconds));
eprintln!("dlp-health-check timed out after {seconds} seconds");
std::process::exit(124);
});
}
fn main() -> Result<()> { fn main() -> Result<()> {
let cli = Cli::parse().apply_env(); let cli = Cli::parse().apply_env();
start_overall_timeout_watchdog(cli.overall_timeout_seconds);
let client = Client::builder() let client = Client::builder()
.no_proxy() .no_proxy()
.build() .build()
@@ -1547,4 +1711,22 @@ mod tests {
assert_eq!(payload.counts.warn, 1); assert_eq!(payload.counts.warn, 1);
assert_eq!(payload.counts.fail, 0); assert_eq!(payload.counts.fail, 0);
} }
#[test]
fn dlp_profile_normalization_matches_runtime_control_names() {
assert_eq!(normalized_profile("disabled"), "core_only");
assert_eq!(normalized_profile("core-only"), "core_only");
assert_eq!(normalized_profile("light"), "light");
assert_eq!(normalized_profile("lite"), "light");
assert_eq!(normalized_profile("on-demand"), "on_demand");
assert_eq!(normalized_profile("enabled"), "full");
}
#[test]
fn light_profile_does_not_require_heavy_services() {
assert!(!profile_checks_heavy_services("light"));
assert!(!profile_checks_heavy_services("core_only"));
assert!(profile_checks_heavy_services("on_demand"));
assert!(profile_checks_heavy_services("full"));
}
} }
+514 -13
View File
@@ -77,6 +77,7 @@ struct Config {
management_history_retention_days: i64, management_history_retention_days: i64,
true_active_evidence_window_seconds: i64, true_active_evidence_window_seconds: i64,
true_active_max_event_seconds: i64, true_active_max_event_seconds: i64,
dlp_evidence_enabled: bool,
offset: FixedOffset, offset: FixedOffset,
} }
@@ -227,6 +228,14 @@ fn threshold_to_pct(value: f64) -> f64 {
} }
} }
fn overload_threshold_to_pct(value: f64) -> f64 {
if (0.0..=3.0).contains(&value) {
value * 100.0
} else {
value
}
}
fn parse_hhmm(value: &str) -> Option<NaiveTime> { fn parse_hhmm(value: &str) -> Option<NaiveTime> {
NaiveTime::parse_from_str(value.trim(), "%H:%M").ok() NaiveTime::parse_from_str(value.trim(), "%H:%M").ok()
} }
@@ -262,7 +271,7 @@ fn load_config() -> Config {
} }
if let Some(value) = policy.overload_threshold { if let Some(value) = policy.overload_threshold {
manager_overload_coverage_pct = manager_overload_coverage_pct =
threshold_to_pct(value).round().clamp(1.0, 300.0) as i64; overload_threshold_to_pct(value).round().clamp(100.0, 300.0) as i64;
} }
if let Some(value) = policy.drop_threshold_pct { if let Some(value) = policy.drop_threshold_pct {
manager_trend_delta_pct = threshold_to_pct(value).clamp(1.0, 100.0); manager_trend_delta_pct = threshold_to_pct(value).clamp(1.0, 100.0);
@@ -371,6 +380,10 @@ fn load_config() -> Config {
.max(30), .max(30),
true_active_max_event_seconds: env_i64("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", 600) true_active_max_event_seconds: env_i64("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", 600)
.max(30), .max(30),
dlp_evidence_enabled: env_bool(
"AW_WORKTIME_DLP_EVIDENCE_ENABLED",
env_bool("AW_DLP_ENABLED", true),
),
offset: FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"), offset: FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"),
} }
} }
@@ -1010,13 +1023,7 @@ impl App {
} }
}; };
let mut evidence = HashMap::new(); let mut evidence = HashMap::new();
for bucket in [ for bucket in evidence_bucket_ids(&self.config, host) {
format!("aw-file-operations_{host}"),
format!("aw-dlp-endpoint-signals_{host}"),
format!("aw-watcher-web-chrome_{host}"),
format!("aw-watcher-web-edge_{host}"),
format!("aw-detmir-web-category_{host}"),
] {
evidence.insert( evidence.insert(
bucket.clone(), bucket.clone(),
self.fetch_bucket_events(&bucket, Some(bounds.0), Some(bounds.1)), self.fetch_bucket_events(&bucket, Some(bounds.0), Some(bounds.1)),
@@ -1156,6 +1163,19 @@ fn sanitize_bucket_for_log(bucket_id: &str) -> String {
bucket_id.to_string() bucket_id.to_string()
} }
fn evidence_bucket_ids(config: &Config, host: &str) -> Vec<String> {
let mut buckets = vec![format!("aw-file-operations_{host}")];
if config.dlp_evidence_enabled {
buckets.push(format!("aw-dlp-endpoint-signals_{host}"));
}
buckets.extend([
format!("aw-watcher-web-chrome_{host}"),
format!("aw-watcher-web-edge_{host}"),
format!("aw-detmir-web-category_{host}"),
]);
buckets
}
fn sanitize_error_for_log(value: &str) -> String { fn sanitize_error_for_log(value: &str) -> String {
static IP_RE: OnceLock<Regex> = OnceLock::new(); static IP_RE: OnceLock<Regex> = OnceLock::new();
static BUCKET_HOST_RE: OnceLock<Regex> = OnceLock::new(); static BUCKET_HOST_RE: OnceLock<Regex> = OnceLock::new();
@@ -1605,6 +1625,156 @@ fn interval_overlap_seconds(
(total, first, last) (total, first, last)
} }
#[derive(Clone, Copy)]
struct EmployeeOperationsInput {
expected_seconds: i64,
low_seconds: i64,
target_seconds: i64,
overload_seconds: i64,
work_secs: i64,
calendar_secs: i64,
work_first: Option<DateTime<Utc>>,
work_last: Option<DateTime<Utc>>,
is_today: bool,
late_start: DateTime<FixedOffset>,
early_finish: DateTime<FixedOffset>,
samples_count: i64,
active_samples: i64,
sessions_count: i64,
}
fn build_employee_operations_status(config: &Config, input: EmployeeOperationsInput) -> Value {
let workday_idle_seconds = (input.expected_seconds - input.work_secs).max(0);
let off_hours_seconds = (input.calendar_secs - input.work_secs).max(0);
let coverage = if input.expected_seconds > 0 {
clamp_pct(input.work_secs as f64 / input.expected_seconds as f64 * 100.0)
} else {
0.0
};
let load_status = if input.expected_seconds <= 0 {
"insufficient_data"
} else if input.sessions_count <= 0 || input.samples_count <= 0 {
"no_data"
} else if input.work_secs <= 0 {
"no_activity"
} else if input.work_secs < input.low_seconds {
"underloaded"
} else if input.work_secs >= input.overload_seconds {
"overloaded"
} else if input.work_secs < input.target_seconds {
"below_target"
} else {
"normal"
};
let idle_status = if input.expected_seconds <= 0 {
"not_applicable"
} else if input.sessions_count <= 0 || input.samples_count <= 0 {
"unknown"
} else if input.work_secs <= 0 {
"full_workday_idle_or_absent"
} else if workday_idle_seconds >= config.manager_off_hours_threshold_seconds {
"idle_detected"
} else {
"no_significant_idle"
};
let mut discipline_flags = Vec::new();
if off_hours_seconds >= config.manager_off_hours_threshold_seconds {
discipline_flags.push("off_hours");
}
if input
.work_first
.is_some_and(|dt| dt.with_timezone(&config.offset) > input.late_start)
{
discipline_flags.push("late_start");
}
if !input.is_today
&& input
.work_last
.is_some_and(|dt| dt.with_timezone(&config.offset) < input.early_finish)
{
discipline_flags.push("early_finish");
}
let discipline_status = match discipline_flags.as_slice() {
[] => "ok",
[single] => single,
_ => "multiple_flags",
};
let mut confidence_reasons = Vec::new();
if input.sessions_count <= 0 {
confidence_reasons.push("missing_session_samples");
}
if input.samples_count <= 0 {
confidence_reasons.push("missing_worktime_samples");
} else if input.samples_count < 3 {
confidence_reasons.push("few_worktime_samples");
} else {
confidence_reasons.push("worktime_samples_present");
}
if input.active_samples <= 0 {
confidence_reasons.push("missing_active_samples");
} else {
confidence_reasons.push("active_samples_present");
}
if input.expected_seconds <= 0 {
confidence_reasons.push("workday_window_not_started_or_empty");
}
let data_confidence =
if input.expected_seconds <= 0 || input.sessions_count <= 0 || input.samples_count <= 0 {
"low"
} else if input.samples_count < 3 || input.active_samples <= 0 {
"medium"
} else {
"high"
};
let recommended_action = match (data_confidence, load_status, discipline_status, idle_status) {
("low", _, _, _) => {
"Сначала проверить свежесть источников и наличие сессии; вывод по сотруднику не использовать как дисциплинарный."
}
(_, "overloaded", _, _) => {
"Проверить переработку, перераспределение задач и риск аврального процесса."
}
(_, "underloaded" | "below_target" | "no_activity", _, _) => {
"Проверить фактическую загрузку, задачи, доступ к рабочим системам и отсутствие сбоя сбора."
}
(_, _, "off_hours" | "late_start" | "early_finish" | "multiple_flags", _) => {
"Проверить согласование рабочего графика и причину отклонения от процесса."
}
(_, _, _, "idle_detected" | "full_workday_idle_or_absent") => {
"Проверить простой: отсутствие задач, ожидание внешнего процесса или техническую проблему."
}
_ => "Наблюдать; отклонений, требующих немедленного действия, не выявлено.",
};
json!({
"load_status": load_status,
"idle_status": idle_status,
"discipline_status": discipline_status,
"discipline_flags": discipline_flags,
"data_confidence": data_confidence,
"confidence_reasons": confidence_reasons,
"workday_idle_seconds": workday_idle_seconds,
"workday_idle_hhmm": hhmm(workday_idle_seconds),
"off_hours_seconds": off_hours_seconds,
"off_hours_hhmm": hhmm(off_hours_seconds),
"coverage_pct": coverage,
"evidence": {
"expected_hhmm": hhmm(input.expected_seconds),
"workday_active_hhmm": hhmm(input.work_secs),
"calendar_active_hhmm": hhmm(input.calendar_secs),
"sessions_count": input.sessions_count,
"samples_count": input.samples_count,
"active_samples": input.active_samples,
"first_workday_activity_local": input.work_first.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default(),
"last_workday_activity_local": input.work_last.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default()
},
"guardrail": if data_confidence == "low" {
"low_confidence_not_for_discipline"
} else {
"evidence_backed_manual_review_only"
},
"recommended_action": recommended_action,
})
}
impl App { impl App {
fn build_management_payload( fn build_management_payload(
&self, &self,
@@ -1634,6 +1804,7 @@ impl App {
}; };
let target_seconds = expected_seconds * self.config.manager_target_coverage_pct / 100; let target_seconds = expected_seconds * self.config.manager_target_coverage_pct / 100;
let low_seconds = expected_seconds * self.config.manager_low_coverage_pct / 100; let low_seconds = expected_seconds * self.config.manager_low_coverage_pct / 100;
let overload_seconds = expected_seconds * self.config.manager_overload_coverage_pct / 100;
let late_start = let late_start =
work_start_local + TimeDelta::minutes(self.config.manager_late_start_grace_minutes); work_start_local + TimeDelta::minutes(self.config.manager_late_start_grace_minutes);
let early_finish = let early_finish =
@@ -1706,6 +1877,37 @@ impl App {
.map(|dt| dt.with_timezone(&self.config.offset)) .map(|dt| dt.with_timezone(&self.config.offset))
.map(|dt| dt.to_rfc3339()) .map(|dt| dt.to_rfc3339())
.unwrap_or_default(); .unwrap_or_default();
let samples_count = row
.get("samples_count")
.and_then(Value::as_i64)
.unwrap_or(0);
let active_samples = row
.get("active_samples")
.and_then(Value::as_i64)
.unwrap_or(0);
let sessions_count = row
.get("sessions_count")
.and_then(Value::as_i64)
.unwrap_or(0);
let operations = build_employee_operations_status(
&self.config,
EmployeeOperationsInput {
expected_seconds,
low_seconds,
target_seconds,
overload_seconds,
work_secs,
calendar_secs,
work_first,
work_last,
is_today,
late_start,
early_finish,
samples_count,
active_samples,
sessions_count,
},
);
let mut public = row.as_object().cloned().unwrap_or_default(); let mut public = row.as_object().cloned().unwrap_or_default();
public.remove("_intervals"); public.remove("_intervals");
public.insert("user".into(), json!(alias.display_name)); public.insert("user".into(), json!(alias.display_name));
@@ -1719,8 +1921,52 @@ impl App {
public.insert("calendar_active_hhmm".into(), json!(hhmm(calendar_secs))); public.insert("calendar_active_hhmm".into(), json!(hhmm(calendar_secs)));
public.insert("workday_active_seconds".into(), json!(work_secs)); public.insert("workday_active_seconds".into(), json!(work_secs));
public.insert("workday_active_hhmm".into(), json!(hhmm(work_secs))); public.insert("workday_active_hhmm".into(), json!(hhmm(work_secs)));
public.insert(
"workday_idle_seconds".into(),
json!((expected_seconds - work_secs).max(0)),
);
public.insert(
"workday_idle_hhmm".into(),
json!(hhmm((expected_seconds - work_secs).max(0))),
);
public.insert("coverage_pct".into(), json!(coverage)); public.insert("coverage_pct".into(), json!(coverage));
public.insert("status".into(), json!(status)); public.insert("status".into(), json!(status));
public.insert(
"load_status".into(),
operations
.get("load_status")
.cloned()
.unwrap_or_else(|| json!("unknown")),
);
public.insert(
"idle_status".into(),
operations
.get("idle_status")
.cloned()
.unwrap_or_else(|| json!("unknown")),
);
public.insert(
"discipline_status".into(),
operations
.get("discipline_status")
.cloned()
.unwrap_or_else(|| json!("unknown")),
);
public.insert(
"data_confidence".into(),
operations
.get("data_confidence")
.cloned()
.unwrap_or_else(|| json!("low")),
);
public.insert(
"operations_recommended_action".into(),
operations
.get("recommended_action")
.cloned()
.unwrap_or_else(|| json!("Проверить первичные источники.")),
);
public.insert("operations".into(), operations.clone());
public.insert("first_activity_local".into(), json!(first_local)); public.insert("first_activity_local".into(), json!(first_local));
public.insert("last_activity_local".into(), json!(last_local)); public.insert("last_activity_local".into(), json!(last_local));
public.insert( public.insert(
@@ -1739,6 +1985,10 @@ impl App {
"manager_owner": public.get("manager_owner").cloned().unwrap_or(json!("")), "manager_owner": public.get("manager_owner").cloned().unwrap_or(json!("")),
"department": public.get("department").cloned().unwrap_or(json!("")), "department": public.get("department").cloned().unwrap_or(json!("")),
"role": public.get("role").cloned().unwrap_or(json!("")), "role": public.get("role").cloned().unwrap_or(json!("")),
"load_status": public.get("load_status").cloned().unwrap_or(json!("unknown")),
"idle_status": public.get("idle_status").cloned().unwrap_or(json!("unknown")),
"discipline_status": public.get("discipline_status").cloned().unwrap_or(json!("unknown")),
"data_confidence": public.get("data_confidence").cloned().unwrap_or(json!("low")),
}); });
let owner = public let owner = public
.get("manager_owner") .get("manager_owner")
@@ -1763,6 +2013,9 @@ impl App {
} else if expected_seconds > 0 && work_secs < target_seconds { } else if expected_seconds > 0 && work_secs < target_seconds {
actions.push(action("target_gap_review", "medium", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже управленческого целевого порога {}%.", hhmm(work_secs), self.config.manager_target_coverage_pct), &format!("Уточнить причину отклонения по сотруднику {display} и подтвердить план работ."), &canonical, evidence.clone())); actions.push(action("target_gap_review", "medium", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже управленческого целевого порога {}%.", hhmm(work_secs), self.config.manager_target_coverage_pct), &format!("Уточнить причину отклонения по сотруднику {display} и подтвердить план работ."), &canonical, evidence.clone()));
} }
if expected_seconds > 0 && work_secs >= overload_seconds {
actions.push(action("overload_review", "high", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} выше порога перегруза {}%.", hhmm(work_secs), self.config.manager_overload_coverage_pct), &format!("Проверить переработку сотрудника {display}, распределение задач и риск аврального процесса."), &canonical, evidence.clone()));
}
if work_first.is_some_and(|dt| dt.with_timezone(&self.config.offset) > late_start) { if work_first.is_some_and(|dt| dt.with_timezone(&self.config.offset) > late_start) {
actions.push(action("late_start_review", "medium", &owner, "24h", &format!("У сотрудника {display} первая активность в рабочем окне зафиксирована поздно."), &format!("Проверить причину позднего старта сотрудника {display} и подтвердить, что это не проблема доступа или дисциплины."), &canonical, evidence.clone())); actions.push(action("late_start_review", "medium", &owner, "24h", &format!("У сотрудника {display} первая активность в рабочем окне зафиксирована поздно."), &format!("Проверить причину позднего старта сотрудника {display} и подтвердить, что это не проблема доступа или дисциплины."), &canonical, evidence.clone()));
} }
@@ -1810,6 +2063,37 @@ impl App {
) )
}); });
let summary = summarize_management_rows(&roster, &actions, expected_seconds); let summary = summarize_management_rows(&roster, &actions, expected_seconds);
let workforce_operations = json!({
"status": summary.pointer("/workforce_operations/status").cloned().unwrap_or_else(|| json!("LOW_CONFIDENCE")),
"summary": summary.pointer("/workforce_operations").cloned().unwrap_or_else(|| json!({})),
"rows": roster.iter().map(|row| {
json!({
"user": row.get("user").cloned().unwrap_or(json!("")),
"manager_owner": row.get("manager_owner").cloned().unwrap_or(json!("")),
"department": row.get("department").cloned().unwrap_or(json!("")),
"role": row.get("role").cloned().unwrap_or(json!("")),
"load_status": row.get("load_status").cloned().unwrap_or(json!("unknown")),
"idle_status": row.get("idle_status").cloned().unwrap_or(json!("unknown")),
"discipline_status": row.get("discipline_status").cloned().unwrap_or(json!("unknown")),
"data_confidence": row.get("data_confidence").cloned().unwrap_or(json!("low")),
"coverage_pct": row.get("coverage_pct").cloned().unwrap_or(json!(0.0)),
"workday_active_hhmm": row.get("workday_active_hhmm").cloned().unwrap_or(json!("00:00")),
"workday_idle_hhmm": row.get("workday_idle_hhmm").cloned().unwrap_or(json!("00:00")),
"recommended_action": row.get("operations_recommended_action").cloned().unwrap_or(json!("Проверить первичные источники."))
})
}).collect::<Vec<_>>(),
"model": {
"type": "rule_based",
"ml": false,
"llm": false,
"version": "workforce-operations-v1"
},
"guardrails": [
"Строки low confidence требуют проверки источников до персонального вывода",
"Отсутствие данных не считается простоем",
"Все статусы предназначены только для ручного операционного разбора"
]
});
let owner_rollups = build_rollups(&roster, &actions, "manager_owner"); let owner_rollups = build_rollups(&roster, &actions, "manager_owner");
let department_rollups = build_rollups(&roster, &actions, "department"); let department_rollups = build_rollups(&roster, &actions, "department");
let owner_roster = owner_rollups.clone(); let owner_roster = owner_rollups.clone();
@@ -1865,8 +2149,10 @@ impl App {
"expected_hhmm_per_user": hhmm(expected_seconds), "expected_hhmm_per_user": hhmm(expected_seconds),
"target_coverage_pct": self.config.manager_target_coverage_pct, "target_coverage_pct": self.config.manager_target_coverage_pct,
"low_coverage_pct": self.config.manager_low_coverage_pct, "low_coverage_pct": self.config.manager_low_coverage_pct,
"overload_coverage_pct": self.config.manager_overload_coverage_pct,
}, },
"summary": summary, "summary": summary,
"workforce_operations": workforce_operations,
"actions": actions, "actions": actions,
"rows": roster, "rows": roster,
"sources": sources, "sources": sources,
@@ -2329,6 +2615,7 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds
.and_then(Value::as_i64) .and_then(Value::as_i64)
.unwrap_or(0) .unwrap_or(0)
}); });
let operations_summary = summarize_operations(rows);
json!({ json!({
"users_count": users_count, "users_count": users_count,
"active_users": active_users, "active_users": active_users,
@@ -2349,6 +2636,85 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds
"last_activity": rows.iter().filter_map(|r| r.get("workday_last_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).max().unwrap_or(""), "last_activity": rows.iter().filter_map(|r| r.get("workday_last_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).max().unwrap_or(""),
"top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""), "top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""),
"top_user_active_hhmm": top.and_then(|r| r.get("workday_active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"), "top_user_active_hhmm": top.and_then(|r| r.get("workday_active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"),
"workforce_operations": operations_summary,
})
}
fn row_str<'a>(row: &'a Value, key: &str) -> &'a str {
row.get(key).and_then(Value::as_str).unwrap_or("")
}
fn count_rows_by(rows: &[Value], key: &str, expected: &[&str]) -> i64 {
rows.iter()
.filter(|row| expected.contains(&row_str(row, key)))
.count() as i64
}
fn summarize_operations(rows: &[Value]) -> Value {
let users_count = rows.len() as i64;
let low_confidence_users = count_rows_by(rows, "data_confidence", &["low"]);
let medium_confidence_users = count_rows_by(rows, "data_confidence", &["medium"]);
let high_confidence_users = count_rows_by(rows, "data_confidence", &["high"]);
let unknown_or_no_data_users = count_rows_by(
rows,
"load_status",
&["no_data", "insufficient_data", "no_activity"],
);
let underloaded_users = count_rows_by(rows, "load_status", &["underloaded", "below_target"]);
let normal_users = count_rows_by(rows, "load_status", &["normal"]);
let overloaded_users = count_rows_by(rows, "load_status", &["overloaded"]);
let idle_users = count_rows_by(
rows,
"idle_status",
&["idle_detected", "full_workday_idle_or_absent"],
);
let discipline_review_users = rows
.iter()
.filter(|row| !matches!(row_str(row, "discipline_status"), "" | "ok"))
.count() as i64;
let action_required_users = rows
.iter()
.filter(|row| {
!matches!(row_str(row, "load_status"), "normal" | "")
|| !matches!(row_str(row, "discipline_status"), "ok" | "")
|| matches!(
row_str(row, "idle_status"),
"idle_detected" | "full_workday_idle_or_absent"
)
|| row_str(row, "data_confidence") == "low"
})
.count() as i64;
let status = if users_count == 0 || low_confidence_users == users_count {
"LOW_CONFIDENCE"
} else if overloaded_users > 0 || discipline_review_users > 0 || idle_users > 0 {
"ATTENTION"
} else if underloaded_users > 0 || unknown_or_no_data_users > 0 || medium_confidence_users > 0 {
"WATCH"
} else {
"OK"
};
json!({
"status": status,
"users_count": users_count,
"action_required_users": action_required_users,
"load": {
"unknown_or_no_data_users": unknown_or_no_data_users,
"underloaded_users": underloaded_users,
"normal_users": normal_users,
"overloaded_users": overloaded_users
},
"idle": {
"idle_users": idle_users
},
"discipline": {
"review_users": discipline_review_users
},
"confidence": {
"low_users": low_confidence_users,
"medium_users": medium_confidence_users,
"high_users": high_confidence_users
},
"guardrail": "Строки low confidence требуют проверки источников до персонального вывода"
}) })
} }
@@ -2369,6 +2735,11 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec<Value> {
"active_users", "active_users",
"inactive_users", "inactive_users",
"below_target_users", "below_target_users",
"underloaded_users",
"overloaded_users",
"idle_users",
"discipline_review_users",
"low_confidence_users",
"workday_total_active_seconds", "workday_total_active_seconds",
"actions_count", "actions_count",
"critical_actions_count", "critical_actions_count",
@@ -2390,6 +2761,24 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec<Value> {
if row.get("status").and_then(Value::as_str) == Some("below_target") { if row.get("status").and_then(Value::as_str) == Some("below_target") {
inc(group, "below_target_users", 1); inc(group, "below_target_users", 1);
} }
if matches!(row_str(row, "load_status"), "underloaded" | "below_target") {
inc(group, "underloaded_users", 1);
}
if row_str(row, "load_status") == "overloaded" {
inc(group, "overloaded_users", 1);
}
if matches!(
row_str(row, "idle_status"),
"idle_detected" | "full_workday_idle_or_absent"
) {
inc(group, "idle_users", 1);
}
if !matches!(row_str(row, "discipline_status"), "" | "ok") {
inc(group, "discipline_review_users", 1);
}
if row_str(row, "data_confidence") == "low" {
inc(group, "low_confidence_users", 1);
}
inc( inc(
group, group,
"workday_total_active_seconds", "workday_total_active_seconds",
@@ -3350,9 +3739,9 @@ fn render_management_html(payload: &Value) -> String {
.collect() .collect()
}; };
let user_rows = if rows.is_empty() { let user_rows = if rows.is_empty() {
"<tr><td colspan='8'>Нет сотрудников в выборке.</td></tr>".to_string() "<tr><td colspan='12'>Нет сотрудников в выборке.</td></tr>".to_string()
} else { } else {
rows.iter().map(|r| format!("<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td class='good'>{}</td><td>{}%</td><td>{}</td><td>{}</td></tr>", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["workday_first_activity_local"].as_str().unwrap_or("")), esc(r["workday_last_activity_local"].as_str().unwrap_or("")))).collect() rows.iter().map(|r| format!("<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td class='good'>{}</td><td>{}</td><td>{}%</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), esc(r["workday_idle_hhmm"].as_str().unwrap_or("00:00")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["load_status"].as_str().unwrap_or("unknown")), esc(r["idle_status"].as_str().unwrap_or("unknown")), esc(r["discipline_status"].as_str().unwrap_or("unknown")), esc(r["data_confidence"].as_str().unwrap_or("low")), esc(r["operations_recommended_action"].as_str().unwrap_or("")))).collect()
}; };
let source_rows = sources let source_rows = sources
.iter() .iter()
@@ -3369,7 +3758,7 @@ fn render_management_html(payload: &Value) -> String {
}) })
.collect::<String>(); .collect::<String>();
format!( format!(
r#"<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AW-rus Управленческий отчёт по работе в RDP</title><style>{}</style></head><body><main><section class="hero"><h1>AW-rus Управленческий отчёт по работе в RDP</h1><p>{} · {} · {}</p><h2>Что делать сегодня</h2><p>{}</p><nav><a href="/reports/worktime/management?format=json&host={}">JSON</a><a href="/reports/worktime/management?format=csv&host={}">CSV</a><a href="/reports/worktime/today?format=html&host={}">Классический отчёт</a><a href="/reports/worktime/management?format=html&host={}">Сбросить</a></nav></section><section><h2>Очередь действий руководителя</h2><table><tbody>{}</tbody></table></section><section><h2>Сотрудники</h2><table><tbody>{}</tbody></table></section><section><h2>Тренд за период</h2><p>Тренд за {} дней</p></section><section><h2>По ответственным</h2><pre>{}</pre></section><section><h2>Ответственные и эскалация</h2><pre>{}</pre></section><section><h2>По подразделениям</h2><pre>{}</pre></section><section><h2>Свежесть источников данных</h2><table><tbody>{}</tbody></table></section><p>Фильтр: {}</p></main></body></html>"#, r#"<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AW-rus Управленческий отчёт по работе в RDP</title><style>{}</style></head><body><main><section class="hero"><h1>AW-rus Управленческий отчёт по работе в RDP</h1><p>{} · {} · {}</p><h2>Что делать сегодня</h2><p>{}</p><nav><a href="/reports/worktime/management?format=json&host={}">JSON</a><a href="/reports/worktime/management?format=csv&host={}">CSV</a><a href="/reports/worktime/today?format=html&host={}">Классический отчёт</a><a href="/reports/worktime/management?format=html&host={}">Сбросить</a></nav></section><section><h2>Очередь действий руководителя</h2><table><tbody>{}</tbody></table></section><section><h2>Рабочая активность сотрудников</h2><p>Статусы загрузки, простоя, дисциплины процесса и достоверности. Low confidence означает: сначала проверить источники, не делать персональный вывод.</p><table><thead><tr><th>Сотрудник</th><th>Ответственный</th><th>Подразделение</th><th>Статус</th><th>Активно</th><th>Простой</th><th>Coverage</th><th>Загрузка</th><th>Простой</th><th>Дисциплина</th><th>Confidence</th><th>Действие</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Тренд за период</h2><p>Тренд за {} дней</p></section><section><h2>По ответственным</h2><pre>{}</pre></section><section><h2>Ответственные и эскалация</h2><pre>{}</pre></section><section><h2>По подразделениям</h2><pre>{}</pre></section><section><h2>Свежесть источников данных</h2><table><tbody>{}</tbody></table></section><p>Фильтр: {}</p></main></body></html>"#,
base_css(), base_css(),
esc(payload["host"].as_str().unwrap_or("")), esc(payload["host"].as_str().unwrap_or("")),
esc(payload["report_date"].as_str().unwrap_or("")), esc(payload["report_date"].as_str().unwrap_or("")),
@@ -3639,6 +4028,36 @@ mod tests {
); );
} }
#[test]
fn dlp_evidence_bucket_is_optional_for_true_active_hot_path() {
let mut cfg = test_config();
cfg.dlp_evidence_enabled = false;
let buckets = evidence_bucket_ids(&cfg, "SHARKON2025");
assert!(
buckets
.iter()
.any(|bucket| bucket == "aw-file-operations_SHARKON2025")
);
assert!(
buckets
.iter()
.any(|bucket| bucket == "aw-watcher-web-chrome_SHARKON2025")
);
assert!(
!buckets
.iter()
.any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025")
);
cfg.dlp_evidence_enabled = true;
let buckets = evidence_bucket_ids(&cfg, "SHARKON2025");
assert!(
buckets
.iter()
.any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025")
);
}
#[test] #[test]
fn management_insights_detect_falling_portfolio_trend() { fn management_insights_detect_falling_portfolio_trend() {
let cfg = test_config(); let cfg = test_config();
@@ -3758,6 +4177,85 @@ mod tests {
assert_eq!(rollups[0]["actions_count"], 1); assert_eq!(rollups[0]["actions_count"], 1);
} }
#[test]
fn employee_operations_detects_overload_and_off_hours() {
let cfg = test_config();
let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap();
let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date);
let input = EmployeeOperationsInput {
expected_seconds,
low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100,
target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100,
overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100,
work_secs: expected_seconds * 2,
calendar_secs: expected_seconds * 2 + 3600,
work_first: Some(work_start.with_timezone(&Utc)),
work_last: Some(work_end.with_timezone(&Utc)),
is_today: false,
late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes),
early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes),
samples_count: 20,
active_samples: 18,
sessions_count: 1,
};
let status = build_employee_operations_status(&cfg, input);
assert_eq!(status["load_status"], "overloaded");
assert_eq!(status["discipline_status"], "off_hours");
assert_eq!(status["data_confidence"], "high");
assert_eq!(status["guardrail"], "evidence_backed_manual_review_only");
}
#[test]
fn employee_operations_low_confidence_blocks_personnel_conclusion() {
let cfg = test_config();
let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap();
let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date);
let input = EmployeeOperationsInput {
expected_seconds,
low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100,
target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100,
overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100,
work_secs: 0,
calendar_secs: 0,
work_first: None,
work_last: None,
is_today: false,
late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes),
early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes),
samples_count: 0,
active_samples: 0,
sessions_count: 0,
};
let status = build_employee_operations_status(&cfg, input);
assert_eq!(status["load_status"], "no_data");
assert_eq!(status["idle_status"], "unknown");
assert_eq!(status["data_confidence"], "low");
assert_eq!(status["guardrail"], "low_confidence_not_for_discipline");
assert!(
status["recommended_action"]
.as_str()
.unwrap()
.contains("не использовать как дисциплинарный")
);
}
#[test]
fn operations_summary_counts_attention_groups() {
let rows = vec![
json!({"load_status":"overloaded","idle_status":"no_significant_idle","discipline_status":"ok","data_confidence":"high"}),
json!({"load_status":"underloaded","idle_status":"idle_detected","discipline_status":"late_start","data_confidence":"medium"}),
json!({"load_status":"no_data","idle_status":"unknown","discipline_status":"ok","data_confidence":"low"}),
];
let summary = summarize_operations(&rows);
assert_eq!(summary["status"], "ATTENTION");
assert_eq!(summary["load"]["overloaded_users"], 1);
assert_eq!(summary["load"]["underloaded_users"], 1);
assert_eq!(summary["idle"]["idle_users"], 1);
assert_eq!(summary["discipline"]["review_users"], 1);
assert_eq!(summary["confidence"]["low_users"], 1);
assert_eq!(summary["action_required_users"], 3);
}
#[test] #[test]
fn stale_legacy_rdp_sources_are_covered_by_fresh_rust_sources() { fn stale_legacy_rdp_sources_are_covered_by_fresh_rust_sources() {
assert!(legacy_rdp_covered_by_rust_sources( assert!(legacy_rdp_covered_by_rust_sources(
@@ -3888,14 +4386,17 @@ mod tests {
#[test] #[test]
fn interpretation_policy_accepts_fraction_thresholds() { fn interpretation_policy_accepts_fraction_thresholds() {
let policy: InterpretationPolicy = serde_json::from_value(json!({ let policy: InterpretationPolicy = serde_json::from_value(json!({
"overload_threshold": 0.92, "overload_threshold": 1.15,
"underload_threshold": 0.45, "underload_threshold": 0.45,
"drop_threshold_pct": 20, "drop_threshold_pct": 20,
"night_work_after": "20:00", "night_work_after": "20:00",
"weekend_work": true "weekend_work": true
})) }))
.unwrap(); .unwrap();
assert_eq!(threshold_to_pct(policy.overload_threshold.unwrap()), 92.0); assert_eq!(
overload_threshold_to_pct(policy.overload_threshold.unwrap()).round(),
115.0
);
assert_eq!(threshold_to_pct(policy.underload_threshold.unwrap()), 45.0); assert_eq!(threshold_to_pct(policy.underload_threshold.unwrap()), 45.0);
assert_eq!(threshold_to_pct(policy.drop_threshold_pct.unwrap()), 20.0); assert_eq!(threshold_to_pct(policy.drop_threshold_pct.unwrap()), 20.0);
assert_eq!( assert_eq!(
+11 -1
View File
@@ -275,10 +275,20 @@ impl AwClient {
) -> Result<()> { ) -> Result<()> {
for chunk in events.chunks(chunk_size.max(1)) { for chunk in events.chunks(chunk_size.max(1)) {
let path = format!("/api/0/buckets/{bucket_id}/events"); let path = format!("/api/0/buckets/{bucket_id}/events");
self.request_json(Method::POST, &path, Some(json!(chunk)), false)?; self.request_status(Method::POST, &path, Some(json!(chunk)))?;
} }
Ok(()) Ok(())
} }
fn request_status(&self, method: Method, path: &str, payload: Option<Value>) -> Result<()> {
let response = self.send_retry(method, path, payload)?;
let status = response.status();
if status.is_success() {
Ok(())
} else {
Err(anyhow!("ActivityWatch {path} returned HTTP {status}"))
}
}
} }
fn log(message: &str) { fn log(message: &str) {
@@ -1,5 +1,5 @@
{ {
"overload_threshold": 0.92, "overload_threshold": 1.15,
"underload_threshold": 0.45, "underload_threshold": 0.45,
"drop_threshold_pct": 20, "drop_threshold_pct": 20,
"night_work_after": "20:00", "night_work_after": "20:00",
+46 -7
View File
@@ -55,6 +55,21 @@ Live endpoints, hostnames, tokens and passwords must be supplied through
`/etc/detmir/detmir-check.env` (systemd units) or another private environment `/etc/detmir/detmir-check.env` (systemd units) or another private environment
file outside the public repository. file outside the public repository.
Production note checked on 2026-06-24:
- `DETMIR_PORTAL_URL` must point to the local DetMir portal listener,
currently `http://127.0.0.1:8720`, for server-side health checks. If it is
omitted, `detmir-check` falls back to the public HTTPS gateway and protected
`/readyz`, `/version` and `/metrics` can correctly return `401`, producing a
false operational failure.
- `DETMIR_GATEWAY_HOST=127.0.0.1` is used with the local listener so the Host
header does not accidentally select the public protected gateway path.
- Cold `/api/reports` builds can take more than 60 seconds on the live contour
when cache is empty or concurrent checks are active. The production
`detmir-portal-prewarm.service` therefore uses `curl --max-time 180` and
`TimeoutStartSec=210`. Shorter 45-60 second limits caused false failed
systemd states while the portal eventually returned HTTP 200.
## Текущий планировщик Proxmox, проверено 2026-06-21 ## Текущий планировщик Proxmox, проверено 2026-06-21
На Proxmox уже присутствуют следующие регулярные проверки: На Proxmox уже присутствуют следующие регулярные проверки:
@@ -75,10 +90,10 @@ file outside the public repository.
Наблюдение: отдельный ежедневный полный gate по всей матрице AWatch-rus Наблюдение: отдельный ежедневный полный gate по всей матрице AWatch-rus
отсутствует. Его роль должен закрыть `awatch-contour-daily-check.timer`. отсутствует. Его роль должен закрыть `awatch-contour-daily-check.timer`.
Наблюдение: последняя проверка `detmir-portal-prewarm.service` на момент осмотра Историческое наблюдение 2026-06-24: `detmir-portal-prewarm.service` был найден
имела `Result=exit-code` и `ExecMainStatus=28`. Это не надо маскировать: в failed state из-за устаревшего `curl --max-time 60` для холодной сборки
канонический check должен показывать такой сбой как fail/warn в зависимости от `/api/reports`. В текущей ветке это оформлено как отдельный prewarm/resilience
политики эксплуатации. пакет, а не как обязательная часть DLP production hot path.
## Матрица требований и проверок ## Матрица требований и проверок
@@ -91,14 +106,36 @@ file outside the public repository.
| Portal hardening | `/healthz`, `/readyz`, `/version`, `/metrics` | `detmir-check` | да | да | | Portal hardening | `/healthz`, `/readyz`, `/version`, `/metrics` | `detmir-check` | да | да |
| Windows/RDP | TCP 5985 и 22 | `detmir-check` | да | да | | Windows/RDP | TCP 5985 и 22 | `detmir-check` | да | да |
| ActivityWatch buckets | AFK/window/worktime/session events | `detmir-check` | да | да | | ActivityWatch buckets | AFK/window/worktime/session events | `detmir-check` | да | да |
| AWatch DLP buckets | endpoint signals/incidents/review/rules | `detmir-check` | да | да | | AWatch DLP buckets | endpoint signals/incidents/review/rules, только если DLP включен | `detmir-check` | условно | условно |
| AWatch DLP health | remote `dlp-health-check --json` через `detmir-dlp` | `detmir-check` | да | да | | AWatch DLP health | disabled/core_only должен быть SKIPPED/WARN, `light/full` проверяются через `detmir-dlp` | `detmir-check` | да | да |
| Grafana evidence | свежий JSON артефакт Grafana check | `detmir-check` | да | да | | Grafana evidence | свежий JSON артефакт Grafana check | `detmir-check` | да | да |
| Security events backend | ClickHouse events, если включено | `detmir-check` | да | да | | Security events backend | ClickHouse events, если включено | `detmir-check` | да | да |
| Portal contract | role/API smoke | `scripts/awatch-production-hardening-smoke.mjs` | нет | да | | Portal contract | role/API smoke | `scripts/awatch-production-hardening-smoke.mjs` | нет | да |
| Pilot contract | demo/API smoke | `scripts/detmir-pilot-demo-smoke.mjs` | нет | да | | Pilot contract | demo/API smoke | `scripts/detmir-pilot-demo-smoke.mjs` | нет | да |
| Registry/readiness docs | registry readiness check | `scripts/registry_readiness_check.sh` | опционально | да | | Registry/readiness docs | registry readiness check | `scripts/registry_readiness_check.sh` | опционально | да |
## Timeout/fail-closed параметры live contour
После ручного live-прогона 2026-06-24 production
`/etc/detmir/detmir-check.env` должен содержать bounded timeouts, соответствующие
фактической latency AW datastore:
```env
DETMIR_SERVICE_TIMEOUT_SECONDS=35
DETMIR_BUCKET_TIMEOUT_SECONDS=35
DETMIR_DLP_TIMEOUT_SECONDS=120
DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS=300
```
Назначение:
- не считать bucket `DEAD` только из-за штатной 15-30 секундной latency
большого SQLite datastore;
- не оставлять `detmir-check`, `detmir-dlp`, `ssh` и remote
`dlp-health-check` хвосты при timeout;
- сохранять красный non-zero результат при реальной недоступности, но
завершать проверку bounded.
## Fail-closed политика ## Fail-closed политика
Ежедневный check должен завершаться non-zero, если падает обязательная область: Ежедневный check должен завершаться non-zero, если падает обязательная область:
@@ -108,7 +145,9 @@ file outside the public repository.
- Gateway/Portal health; - Gateway/Portal health;
- RDP/Windows reachability; - RDP/Windows reachability;
- свежесть обязательных bucket streams; - свежесть обязательных bucket streams;
- AWatch DLP health; - AWatch DLP health только если DLP runtime включен; при штатном
`AW_DLP_ENABLED=false`/`core_only` disabled-state не является отказом
Workforce/Worktime core;
- Grafana evidence freshness. - Grafana evidence freshness.
Event-driven buckets не должны считаться stale только из-за отсутствия новых Event-driven buckets не должны считаться stale только из-за отсутствия новых
+161
View File
@@ -7,6 +7,20 @@ ClickHouse не является обязательной зависимость
перезапускайте ClickHouse для восстановления отчетов рабочего времени, если нет перезапускайте ClickHouse для восстановления отчетов рабочего времени, если нет
отдельного подтвержденного отказа ClickHouse. отдельного подтвержденного отказа ClickHouse.
## Stable host id
Worktime reports используют stable logical host id, а не обязательно текущее
Windows `COMPUTERNAME`. Для DetMir production текущий logical id:
```text
SHARKON2025
```
При переименовании RDP-сервера не меняйте `awHostname` автоматически. Сначала
обновите Windows account domain для задач, затем проверьте, что collectors
продолжают писать в bucket-и `*_SHARKON2025`. Подробный порядок:
`docs/WINDOWS_LOGICAL_HOST_ID_RU.md`.
## Симптомы перегруза ## Симптомы перегруза
- `/portal/api/reports?role=executive` открывается медленно или отвечает - `/portal/api/reports?role=executive` открывается медленно или отвечает
@@ -226,6 +240,153 @@ curl -sS --max-time 8 http://<PORTAL_HOST>/portal/api/health | jq
curl -sS --max-time 12 "http://<PORTAL_HOST>/portal/api/reports?role=executive" | jq '.status' curl -sS --max-time 12 "http://<PORTAL_HOST>/portal/api/reports?role=executive" | jq '.status'
``` ```
## Production repair: AW SQLite hot path, 2026-06-30
Симптомы:
- `activitywatch-server` отвечает `503` на bucket API;
- журнал содержит `poisoned lock` / `database is locked`;
- `aw-worktime-api` уходит в bounded `DEGRADED`;
- `/buckets/aw-worktime-sessions_<HOST>/events?limit=...` тайм-аутится даже
при малом лимите;
- RDP browser/category collector пишет `bucket create failed` или timeout.
Порядок безопасного восстановления:
1. Остановить RDP guard и процессы `aw-windows-telemetry`, чтобы не продолжать
штурмовать AW API.
2. Остановить `aw-worktime-*` timers/services и другие локальные потребители AW
API.
3. Перезапустить `activitywatch-server` отдельно и проверить `/api/0/info`.
4. Если bucket metadata отвечает, но `/events` медленный, проверить SQLite plan:
```sql
EXPLAIN QUERY PLAN
SELECT id,starttime,endtime,data
FROM events
WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_<HOST>')
ORDER BY starttime DESC
LIMIT 100;
```
Если план строит `TEMP B-TREE FOR ORDER BY`, нужен составной индекс:
```sql
CREATE INDEX IF NOT EXISTS events_bucketrow_starttime_desc_index
ON events(bucketrow, starttime DESC);
ANALYZE;
PRAGMA optimize;
PRAGMA integrity_check;
```
Индекс добавлять только в controlled window:
- остановить `activitywatch-server`;
- сделать rollback backup SQLite DB;
- создать индекс;
- проверить `PRAGMA integrity_check = ok`;
- запустить `activitywatch-server`;
- проверить, что `/events?limit=100` больше не тайм-аутится.
Production DetMir repair 2026-06-30:
- оставлены две свежие ежедневные SQLite VACUUM backup-копии, старые backup-и
ротированы для освобождения места;
- создан rollback backup:
`/var/lib/activitywatch/backups/db/aw-sqlite-before-hotpath-index-20260630T035032Z.db`;
- добавлен индекс `events_bucketrow_starttime_desc_index`;
- `ROCKET_WORKERS=8` добавлен в `/etc/activitywatch/aw-server.env`;
- для `aw-worktime-api.service` добавлен stabilization drop-in:
`AW_WORKTIME_EVENTS_LIMIT=100`,
`AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS=25`,
`AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS=600`,
`AW_WORKTIME_REPORT_STALE_TTL_SECONDS=7200`.
После ремонта проверить:
```bash
curl -sS --max-time 10 http://127.0.0.1:5600/api/0/info
curl -sS --max-time 15 \
'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100'
curl -sS --max-time 15 \
'http://127.0.0.1:5610/reports/worktime/today?format=json' | jq '{rows:(.rows|length),degraded,runtime}'
```
Также проверить отсутствие новых `poisoned lock` после финального старта:
```bash
journalctl -u activitywatch-server --since '<FINAL_START_TIME>' --no-pager |
grep -E 'poisoned lock|Taking datastore lock failed|database is locked'
```
## Crash/readiness test after AW repair
Цель: проверить, что контур выдерживает restart и короткую параллельную
нагрузку, а проверки не путают `systemctl active` с готовым API.
Порядок:
1. Зафиксировать baseline:
```bash
./check-aw-full.sh
```
2. На AW server проверить readiness, hot-path и Worktime API:
```bash
AW_API=http://127.0.0.1:5600 \
AW_WORKTIME_API=http://127.0.0.1:5610 \
AW_LOGICAL_HOST_ID=SHARKON2025 \
scripts/detmir_resilience_check.sh --live
```
Если скрипт запускается с ноутбука, live-mode нужно выполнять на самом
AW-сервере через SSH/Ansible, потому что он проверяет local systemd и SQLite.
3. Controlled restart:
```bash
systemctl restart aw-worktime-api
curl -sS --max-time 12 \
'http://127.0.0.1:5610/reports/worktime/today?format=json&host=SHARKON2025&allow_stale=1' |
jq '{rows:(.rows|length),degraded}'
systemctl restart activitywatch-server
# Не считать "active" готовностью: дождаться HTTP readiness.
timeout 90 bash -c 'until curl -fsS --max-time 8 http://127.0.0.1:5600/api/0/info >/dev/null; do sleep 2; done'
curl -sS --max-time 15 \
'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100' >/dev/null
```
4. Проверить, что после рестарта нет новых lock/503:
```bash
journalctl -u activitywatch-server --since '<RESTART_TIME>' --no-pager |
grep -E 'poisoned lock|Taking datastore lock failed|database is locked|503'
```
5. Проверить RDP guard restart отдельно:
```powershell
Restart-Service AWatchRusCollectorGuard -Force
Start-Sleep -Seconds 75
Get-Service AWatchRusCollectorGuard
Get-Process aw-windows-telemetry -ErrorAction SilentlyContinue | Measure-Object
```
Ожидаемый результат для DetMir после ремонта 2026-06-30:
- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`;
- `/events?limit=100` отвечает за bounded time и использует
`events_bucketrow_starttime_desc_index`;
- `aw-rus-healthd.service` завершается `status=0/SUCCESS`;
- server-side TCP до RDP может быть `warn`, если
`AW_RUS_HEALTH_RDP_TCP_REQUIRED=false`, но bucket freshness и WinRM/SSH
через admin path должны оставаться зелёными;
- optional DLP/Loki heavy runtime units должны быть inactive в экономном
production profile.
## Rollback ## Rollback
Rollback нужен, если после обновления бинарника или env-настроек: Rollback нужен, если после обновления бинарника или env-настроек:
+239
View File
@@ -0,0 +1,239 @@
# Workforce Operations Model
Статус: implemented in Worktime API and DetMir portal.
Модель отвечает на главный управленческий вопрос AWatch-rus Workforce:
рабочая активность сотрудников, загрузка, простои, перегруз и дисциплина
рабочего процесса. Это rule-based слой операционного контроля. Он не является
HR-оценкой, не использует ML/LLM и не выполняет автоматических санкций.
## Где смотреть
Основные точки:
- Worktime API:
`GET /reports/worktime/management?format=json`;
- Worktime HTML:
`GET /reports/worktime/management?format=html`;
- DetMir portal:
`/api/reports`, блок `workforce_operations`;
- UI портала:
роли `Руководитель` и вкладка `Отчеты`, блок `Операционная загрузка`.
## Источники
Модель использует только подтвержденные рабочие источники:
- ActivityWatch worktime rows;
- bucket рабочих сессий RDP;
- интервалы активности в рабочем окне;
- configured owner/department aliases;
- freshness/coverage metadata, которые уже возвращает Worktime API.
Отсутствие данных не считается простоем. При пропусках источников строка
получает `data_confidence=low` и guardrail
`low_confidence_not_for_discipline`.
## Runtime-настройки
Основной файл политики:
- пример: `configs/worktime-interpretation-policy.example.json`;
- runtime: `/etc/activitywatch/worktime-interpretation-policy.json`;
- env path: `AW_WORKTIME_MANAGER_INTERPRETATION_POLICY`.
Поля policy:
| Поле | Смысл | Рекомендуемое значение |
| --- | --- | --- |
| `underload_threshold` | порог недогруза от рабочего окна | `0.35..0.45` |
| `overload_threshold` | порог перегруза от рабочего окна | `1.10..1.25` |
| `drop_threshold_pct` | порог просадки тренда | `10..25` |
| `night_work_after` | начало вечернего/ночного отклонения | `20:00` |
| `weekend_work` | учитывать выходные отклонения | `true` |
| `min_trend_points` | минимум daily points для тренда | `3..7` |
| `off_hours_threshold_seconds` | минимум внерабочей активности для флага | `1800` |
`underload_threshold` и `overload_threshold` можно задавать дробью или
процентом: `0.45` равно `45`, `1.15` равно `115`.
Для перегруза effective threshold fail-closed зажат в диапазон `100..300`, чтобы
значение ниже 100% не создавало ложный статус перегруза.
Env fallback:
- `AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT`;
- `AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT`;
- `AW_WORKTIME_MANAGER_OVERLOAD_COVERAGE_PCT`;
- `AW_WORKTIME_MANAGER_TREND_MIN_POINTS`;
- `AW_WORKTIME_MANAGER_TREND_DELTA_PCT`;
- `AW_WORKTIME_MANAGER_OFF_HOURS_THRESHOLD_SECONDS`;
- `AW_WORKTIME_MANAGER_NIGHT_WORK_AFTER`;
- `AW_WORKTIME_MANAGER_WEEKEND_WORK_ENABLED`.
Веса приложений остаются отдельной политикой:
- пример: `configs/detmir-workforce-policy.example.json`;
- runtime: `/etc/detmir-portal-workforce-policy.json`.
Она влияет на explainable KPI и weighted activity, но не подменяет
операционные статусы загрузки/простоя.
## API contract
`/reports/worktime/management?format=json` содержит:
```json
{
"workday": {
"target_coverage_pct": 75,
"low_coverage_pct": 35,
"overload_coverage_pct": 115
},
"workforce_operations": {
"status": "ATTENTION",
"summary": {},
"rows": [],
"model": {
"type": "rule_based",
"ml": false,
"llm": false,
"version": "workforce-operations-v1"
}
}
}
```
Каждая строка сотрудника содержит:
- `workday_active_seconds`, `workday_active_hhmm`;
- `workday_idle_seconds`, `workday_idle_hhmm`;
- `coverage_pct`;
- `load_status`;
- `idle_status`;
- `discipline_status`;
- `data_confidence`;
- `recommended_action`.
Полный roster в `rows[]` дополнительно содержит `operations`,
`operations.evidence`, `operations.guardrail` и
`operations_recommended_action`.
## Статусы загрузки
| Status | Значение | Действие |
| --- | --- | --- |
| `insufficient_data` | рабочее окно еще не началось или равно нулю | не делать вывод |
| `no_data` | нет сессий или worktime samples | проверить источники |
| `no_activity` | сессия/данные есть, активности в окне нет | проверить присутствие и задачи |
| `underloaded` | ниже low threshold | проверить загрузку и доступ к процессам |
| `below_target` | ниже target threshold | уточнить причину отклонения |
| `normal` | в рабочем диапазоне | наблюдать |
| `overloaded` | выше overload threshold | проверить переработку и риск аврала |
## Статусы простоя
| Status | Значение |
| --- | --- |
| `not_applicable` | нет рабочего окна |
| `unknown` | нет достаточных источников |
| `full_workday_idle_or_absent` | активность в рабочем окне отсутствует |
| `idle_detected` | простой выше порога |
| `no_significant_idle` | существенный простой не найден |
## Дисциплина процесса
`discipline_status` показывает отклонение от рабочего процесса, а не
автоматическое нарушение:
- `ok`;
- `off_hours`;
- `late_start`;
- `early_finish`;
- `multiple_flags`.
Для текущего дня `early_finish` не выставляется до завершения рабочего окна.
## Достоверность
`data_confidence`:
- `high`: есть session samples, worktime samples и active samples;
- `medium`: данных мало или нет active samples;
- `low`: нет сессий/worktime samples или рабочее окно невалидно.
Правило: low confidence строки сначала проверяются как проблема источников.
Их нельзя использовать как персональный дисциплинарный вывод.
## Summary
`workforce_operations.summary` содержит:
- `users_count`;
- `action_required_users`;
- `load.unknown_or_no_data_users`;
- `load.underloaded_users`;
- `load.normal_users`;
- `load.overloaded_users`;
- `idle.idle_users`;
- `discipline.review_users`;
- `confidence.low_users`;
- `confidence.medium_users`;
- `confidence.high_users`;
- `guardrail`.
Summary status:
- `LOW_CONFIDENCE`: нет строк или все строки low confidence;
- `ATTENTION`: есть перегруз, простой или дисциплинарные флаги;
- `WATCH`: есть недогруз, нет данных или low confidence;
- `OK`: отклонений нет.
## UI contract
Портал показывает отдельный блок `Операционная загрузка`:
- сводка: требуют разбора, недогруз, перегруз, простой, дисциплина, low
confidence;
- таблица сотрудников: active/idle/coverage/load/idle/discipline/confidence;
- рекомендуемое действие;
- guardrail и версию rule-based модели.
Это отдельный блок от `Почему такой индекс активности?`: explainable KPI
отвечает на вопрос "почему такой процент", а Workforce Operations отвечает
"кого и почему нужно разобрать".
## Ограничения
- Не утверждать автоматическую оценку эффективности сотрудника.
- Не считать missing data простоем.
- Не смешивать Security/Forensics claims с Workforce Operations.
- Не заявлять ML/LLM detection.
- Не выполнять автоматическое remediation/action.
- Не использовать GitHub Actions или демо-данные как registry release evidence.
## Проверка после изменения
Минимальный локальный контур:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust
export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
cargo fmt --all --check
cargo test -p worktime-api -p detmir-portal --locked
cargo clippy -p worktime-api -p detmir-portal --all-targets --locked -- -D warnings
```
Минимальный live smoke:
```bash
curl -fsS 'http://10.10.10.13:5610/reports/worktime/management?format=json' \
| jq '.workforce_operations.summary'
curl -fsS 'http://10.10.10.2:8720/api/reports?role=manager' \
| jq '{status: .workforce_operations.summary.status, rows: (.workforce_operations.rows | length)}'
```
Браузерный smoke: открыть `http://10.10.10.2:8720/`, выбрать представление
менеджера и проверить блок `Операционная загрузка`. В рабочем состоянии должны
быть видны summary-карточки, таблица сотрудников, `workforce-operations-v1` и
guardrail про `low confidence`.
@@ -12,6 +12,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp
Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs
Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env
Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720
Environment=DETMIR_DLP_ENABLED=false
EnvironmentFile=-/etc/detmir/detmir-check.env EnvironmentFile=-/etc/detmir/detmir-check.env
EnvironmentFile=-/etc/awatch-rus/contour-check.env EnvironmentFile=-/etc/awatch-rus/contour-check.env
ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check
@@ -13,6 +13,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp
Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs
Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env
Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720 Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720
Environment=DETMIR_DLP_ENABLED=false
EnvironmentFile=-/etc/detmir/detmir-check.env EnvironmentFile=-/etc/detmir/detmir-check.env
EnvironmentFile=-/etc/awatch-rus/contour-check.env EnvironmentFile=-/etc/awatch-rus/contour-check.env
ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check
+471
View File
@@ -0,0 +1,471 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODE="repo"
AW_API="${AW_API:-http://127.0.0.1:5600}"
AW_WORKTIME_API="${AW_WORKTIME_API:-http://127.0.0.1:5610}"
AW_LOGICAL_HOST_ID="${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-SHARKON2025}}"
AW_READINESS_TIMEOUT_SECONDS="${AW_READINESS_TIMEOUT_SECONDS:-60}"
AW_READINESS_INTERVAL_SECONDS="${AW_READINESS_INTERVAL_SECONDS:-2}"
AW_EVENTS_LIMIT="${AW_EVENTS_LIMIT:-100}"
AW_EVENTS_MAX_SECONDS="${AW_EVENTS_MAX_SECONDS:-15}"
HAYA_ROOT="${AW_HAYABUSA_ROOT:-/opt/hayabusa}"
HAYA_DROP_DIR="${AW_HAYABUSA_DROP_DIR:-/opt/activitywatch/aw-rus-ops/drop}"
SQLITE_DB="${AW_SQLITE_DB:-/var/lib/activitywatch/aw-server-rust/sqlite.db}"
MAX_INCOMING_AGE_SECONDS="${MAX_HAYABUSA_INCOMING_AGE_SECONDS:-900}"
STRICT_SECRETS="${DETMIR_RESILIENCE_STRICT_SECRETS:-0}"
EXPECT_DLP_PROFILE="${DETMIR_RESILIENCE_EXPECT_DLP_PROFILE:-light}"
EXPECT_OPTIONAL_DLP_OFF="${DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF:-0}"
EXPECT_LOKI_OFF="${DETMIR_RESILIENCE_EXPECT_LOKI_OFF:-1}"
DLP_RUNTIME_UNITS=(
aw-dlp-influx-exporter.timer
aw-dlp-influx-exporter.service
activitywatch-dlp-aggregator.timer
activitywatch-dlp-aggregator.service
aw-dlp-report-scheduler.timer
aw-dlp-report-scheduler.service
aw-dlp-syslog-forwarder.timer
aw-dlp-syslog-forwarder.service
aw-dlp-webhook-sender.timer
aw-dlp-webhook-sender.service
aw-dlp-cef-exporter.timer
aw-dlp-cef-exporter.service
aw-dlp-ioc-refresh.timer
aw-dlp-ioc-refresh.service
aw-dlp-policy-engine.service
aw-dlp-case-management.service
detmir-portal-evidence.service
)
DLP_LIGHT_ALLOWED_UNITS=(
activitywatch-dlp-aggregator.timer
activitywatch-dlp-aggregator.service
aw-dlp-ioc-refresh.timer
aw-dlp-ioc-refresh.service
detmir-dlp-load-guard.timer
detmir-dlp-load-guard.service
)
DLP_HEAVY_RUNTIME_UNITS=(
aw-dlp-influx-exporter.timer
aw-dlp-influx-exporter.service
aw-dlp-report-scheduler.timer
aw-dlp-report-scheduler.service
aw-dlp-syslog-forwarder.timer
aw-dlp-syslog-forwarder.service
aw-dlp-webhook-sender.timer
aw-dlp-webhook-sender.service
aw-dlp-cef-exporter.timer
aw-dlp-cef-exporter.service
aw-dlp-policy-engine.service
aw-dlp-case-management.service
detmir-portal-evidence.service
)
LOKI_RUNTIME_UNITS=(
loki.service
promtail.service
)
OK_COUNT=0
WARN_COUNT=0
FAIL_COUNT=0
usage() {
cat <<'EOF'
Usage:
scripts/detmir_resilience_check.sh [--repo|--live|--all]
Modes:
--repo check repository hardening and docs only (default, CI-safe)
--live read-only checks for the local AW server/Hayabusa host
--all repo + live
Environment:
AW_API=http://127.0.0.1:5600
AW_WORKTIME_API=http://127.0.0.1:5610
AW_LOGICAL_HOST_ID=SHARKON2025
AW_READINESS_TIMEOUT_SECONDS=60
AW_EVENTS_MAX_SECONDS=15
AW_HAYABUSA_ROOT=/opt/hayabusa
AW_HAYABUSA_DROP_DIR=/opt/activitywatch/aw-rus-ops/drop
AW_SQLITE_DB=/var/lib/activitywatch/aw-server-rust/sqlite.db
DETMIR_RESILIENCE_EXPECT_DLP_PROFILE=light
DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF=0
DETMIR_RESILIENCE_EXPECT_LOKI_OFF=1
DETMIR_RESILIENCE_STRICT_SECRETS=1
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
--repo) MODE="repo"; shift ;;
--live) MODE="live"; shift ;;
--all) MODE="all"; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
esac
done
ok() {
OK_COUNT=$((OK_COUNT + 1))
printf '[OK] %s\n' "$*"
}
warn() {
WARN_COUNT=$((WARN_COUNT + 1))
printf '[WARN] %s\n' "$*"
}
fail() {
FAIL_COUNT=$((FAIL_COUNT + 1))
printf '[FAIL] %s\n' "$*"
}
have() {
command -v "$1" >/dev/null 2>&1
}
require_file() {
local path="$1"
if [[ -f "$ROOT_DIR/$path" ]]; then
ok "file exists: $path"
else
fail "missing file: $path"
fi
}
require_pattern() {
local path="$1"
local pattern="$2"
local label="$3"
if grep -Eq "$pattern" "$ROOT_DIR/$path"; then
ok "$label"
else
fail "$label"
fi
}
check_repo() {
printf '== repo resilience checks ==\n'
require_file "scripts/detmir_resilience_check.sh"
require_file "docs/DETMIR_RESILIENCE_HARDENING_RU.md"
require_file "docs/DLP_RESOURCE_PROFILES_RU.md"
require_file "docs/DLP_OPTIONAL_RUNTIME_RU.md"
require_file "scripts/detmir_dlp_load_guard.sh"
require_file "scripts/detmir_dlp_warehouse_sync.sh"
require_file "aw-server/hayabusa/aw-hayabusa.sh"
require_file "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs"
require_file "windows/AWatchRusCollectorGuardService.cs"
require_file "windows/install-collector-guard-service.ps1"
require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "HAYA_QUARANTINE_DIR" "Hayabusa wrapper has quarantine root"
require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "quarantine_incoming_package" "Hayabusa wrapper isolates incoming poison packages"
require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "validate_drop_inputs" "Hayabusa autoprocess validates drop package before accept"
require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "quarantine_drop_package" "Hayabusa autoprocess quarantines bad drop package"
require_pattern "windows/AWatchRusCollectorGuardService.cs" "Process\\.Exited|ChildExited" "Collector guard service watches child exit"
require_pattern "windows/AWatchRusCollectorGuardService.cs" "MaxChildRestartsInWindow" "Collector guard service has bounded child restart budget"
require_pattern "windows/install-collector-guard-service.ps1" "failureflag" "Collector guard installer enables SCM failureflag"
require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Hayabusa poison-package isolation" "Resilience doc records Hayabusa hardening"
require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Windows collector guard service child watchdog" "Resilience doc records guard child watchdog"
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "core_only" "DLP resource profiles document core_only"
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "auto.?disable|автоотключ" "DLP resource profiles document auto-disable guard"
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "rollback" "DLP resource profiles document rollback"
require_pattern "docs/DETMIR_CURRENT_STATE_RU.md" "AW_DLP_PROFILE=light" "Current state records DLP light profile"
require_pattern "scripts/detmir_dlp_runtime_control.sh" "set-profile" "DLP runtime control supports profile switching"
require_pattern "scripts/detmir_dlp_runtime_control.sh" "rollback_dlp" "DLP runtime control supports rollback"
require_pattern "scripts/detmir_dlp_load_guard.sh" "set-profile core_only" "DLP load guard can auto-disable DLP to core_only"
require_pattern "scripts/detmir_dlp_load_guard.sh" "STRIKES_REQUIRED" "DLP load guard requires consecutive overload checks"
require_pattern "scripts/detmir_dlp_warehouse_sync.sh" "sqlite3 .*\\.backup" "DLP warehouse sync uses SQLite backup"
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_profile: "light"' "Production defaults keep DLP profile light"
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_enabled: true' "Production defaults enable lightweight DLP"
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_influx_enabled: false' "Production defaults keep DLP Influx disabled"
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_collector_enabled: true' "Production defaults enable lightweight DLP collector"
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_guard_enabled: true' "Production defaults enable DLP load guard"
require_pattern "ansible/group_vars/all.yml" 'detmir_portal_dlp_module_enabled_override: true' "Production defaults expose DLP light status to portal"
require_pattern "ansible/deploy_aw_server.yml" "detmir-dlp-load-guard.service" "AW server deploy installs DLP load guard service"
require_pattern "ansible/deploy_aw_server.yml" "CPUQuota=.*aw_dlp_aggregator_cpu_quota" "DLP aggregator has systemd CPU quota"
require_pattern "ansible/deploy_detmir_portal.yml" "detmir-dlp-warehouse-sync.service" "Portal deploy installs DLP warehouse sync service"
require_pattern "ansible/deploy_detmir_portal.yml" "detmir_portal_dlp_module_enabled_override \\| default\\(false\\)" "Portal deploy defaults DLP module to disabled"
if [[ -f "$ROOT_DIR/ansible/inventory.ini" ]] && grep -Eq '(^|[[:space:]])ansible_(become_)?password[[:space:]]*=[[:space:]]*[^<{]' "$ROOT_DIR/ansible/inventory.ini"; then
if [[ "$STRICT_SECRETS" == "1" ]]; then
fail "ansible/inventory.ini appears to contain literal password assignments; move them to vault/env"
else
warn "ansible/inventory.ini appears to contain literal password assignments; strict mode would fail"
fi
else
ok "no literal ansible password assignments detected in ansible/inventory.ini"
fi
}
check_systemd_unit() {
local unit="$1"
if ! have systemctl; then
warn "systemctl unavailable; skipping $unit"
return
fi
if systemctl is-active --quiet "$unit"; then
ok "systemd active: $unit"
else
fail "systemd not active: $unit"
fi
}
check_aw_api() {
if ! have curl; then
warn "curl unavailable; skipping AW API check"
return
fi
local deadline last_code elapsed
deadline=$((SECONDS + AW_READINESS_TIMEOUT_SECONDS))
last_code=""
while (( SECONDS <= deadline )); do
elapsed="$(
curl -sS --connect-timeout 3 --max-time 8 -o /dev/null -w '%{http_code} %{time_total}' \
"$AW_API/api/0/info" 2>/dev/null || true
)"
last_code="${elapsed%% *}"
if [[ "$last_code" == "200" ]]; then
ok "ActivityWatch API readiness /api/0/info returns 200 (${elapsed#* }s)"
return
fi
sleep "$AW_READINESS_INTERVAL_SECONDS"
done
case "$last_code" in
503) fail "ActivityWatch API readiness ended on 503; possible datastore lock poisoning" ;;
""|000) fail "ActivityWatch API did not become ready within ${AW_READINESS_TIMEOUT_SECONDS}s" ;;
*) fail "ActivityWatch API readiness unexpected final HTTP status: $last_code" ;;
esac
}
check_aw_hot_path() {
if ! have curl; then
warn "curl unavailable; skipping AW hot-path event check"
return
fi
local url result code elapsed
url="${AW_API%/}/api/0/buckets/aw-worktime-sessions_${AW_LOGICAL_HOST_ID}/events?limit=${AW_EVENTS_LIMIT}"
result="$(curl -sS --connect-timeout 3 --max-time "$AW_EVENTS_MAX_SECONDS" -o /dev/null -w '%{http_code} %{time_total}' "$url" 2>/dev/null || true)"
code="${result%% *}"
elapsed="${result#* }"
if [[ "$code" != "200" ]]; then
fail "ActivityWatch worktime events hot path returned HTTP ${code:-none}"
return
fi
if awk -v t="$elapsed" -v max="$AW_EVENTS_MAX_SECONDS" 'BEGIN { exit !(t <= max) }'; then
ok "ActivityWatch worktime events hot path returns 200 (${elapsed}s, limit=${AW_EVENTS_LIMIT})"
else
fail "ActivityWatch worktime events hot path exceeded ${AW_EVENTS_MAX_SECONDS}s (${elapsed}s)"
fi
}
check_worktime_api() {
if ! have curl; then
warn "curl unavailable; skipping Worktime API check"
return
fi
local tmp code
tmp="$(mktemp)"
code="$(curl -sS --connect-timeout 3 --max-time 12 -o "$tmp" -w '%{http_code}' \
"${AW_WORKTIME_API%/}/reports/worktime/today?format=json&host=${AW_LOGICAL_HOST_ID}&allow_stale=1" 2>/dev/null || true)"
if [[ "$code" != "200" ]]; then
fail "Worktime API today report returned HTTP ${code:-none}"
rm -f "$tmp"
return
fi
if have jq; then
if jq -e '(.degraded // false) == false' "$tmp" >/dev/null 2>&1; then
ok "Worktime API today report is not degraded"
else
fail "Worktime API today report is degraded"
fi
if jq -e '((.rows // .users // []) | length) > 0' "$tmp" >/dev/null 2>&1; then
ok "Worktime API today report has employee rows"
else
fail "Worktime API today report has no employee rows"
fi
else
ok "Worktime API today report returns 200"
fi
rm -f "$tmp"
}
check_failed_units() {
if ! have systemctl; then
return
fi
local failed
failed="$(systemctl --failed --no-legend --plain 2>/dev/null | wc -l | tr -d ' ')"
if [[ "$failed" == "0" ]]; then
ok "systemd failed units count is 0"
else
fail "systemd failed units count is $failed"
fi
}
count_files() {
local dir="$1"
local pattern="$2"
if [[ ! -d "$dir" ]]; then
printf '0\n'
return
fi
find "$dir" -maxdepth 1 -type f -name "$pattern" | wc -l | tr -d ' '
}
check_hayabusa_queues() {
local incoming_dir="$HAYA_ROOT/inbox/incoming"
local quarantine_dir="$HAYA_ROOT/quarantine"
local incoming_count drop_count old_count quarantine_count
incoming_count="$(count_files "$incoming_dir" '*.zip')"
drop_count="$(count_files "$HAYA_DROP_DIR" '*.zip')"
old_count="0"
if [[ -d "$incoming_dir" ]]; then
old_count="$(find "$incoming_dir" -maxdepth 1 -type f -name '*.zip' -mmin "+$((MAX_INCOMING_AGE_SECONDS / 60))" | wc -l | tr -d ' ')"
fi
if [[ "$incoming_count" == "0" ]]; then
ok "Hayabusa incoming zip count is 0"
else
warn "Hayabusa incoming zip count is $incoming_count"
fi
if [[ "$drop_count" == "0" ]]; then
ok "Hayabusa drop zip count is 0"
else
warn "Hayabusa drop zip count is $drop_count"
fi
if [[ "$old_count" == "0" ]]; then
ok "Hayabusa incoming has no stale zip older than ${MAX_INCOMING_AGE_SECONDS}s"
else
fail "Hayabusa incoming has $old_count stale zip package(s)"
fi
if [[ -d "$quarantine_dir" ]]; then
quarantine_count="$(find "$quarantine_dir" -type f -name reason.json | wc -l | tr -d ' ')"
if [[ "$quarantine_count" == "0" ]]; then
ok "Hayabusa quarantine reason count is 0"
else
warn "Hayabusa quarantine reason count is $quarantine_count; review/replay policy required"
fi
else
warn "Hayabusa quarantine root not found yet: $quarantine_dir"
fi
}
check_sqlite_files() {
if [[ ! -f "$SQLITE_DB" ]]; then
warn "AW SQLite DB not found at $SQLITE_DB; skipping local DB size check"
return
fi
local db_size wal_size
db_size="$(stat -c '%s' "$SQLITE_DB")"
wal_size="0"
[[ -f "$SQLITE_DB-wal" ]] && wal_size="$(stat -c '%s' "$SQLITE_DB-wal")"
if (( db_size > 5 * 1024 * 1024 * 1024 )); then
fail "AW SQLite DB exceeds 5GiB"
elif (( db_size > 2 * 1024 * 1024 * 1024 )); then
warn "AW SQLite DB exceeds 2GiB"
else
ok "AW SQLite DB size below 2GiB"
fi
if (( wal_size > 1024 * 1024 * 1024 )); then
fail "AW SQLite WAL exceeds 1GiB"
elif (( wal_size > 256 * 1024 * 1024 )); then
warn "AW SQLite WAL exceeds 256MiB"
else
ok "AW SQLite WAL size below 256MiB"
fi
}
check_sqlite_hot_path_index() {
if [[ ! -f "$SQLITE_DB" ]]; then
warn "AW SQLite DB not found at $SQLITE_DB; skipping hot-path index check"
return
fi
if ! have sqlite3; then
warn "sqlite3 unavailable; skipping hot-path index check"
return
fi
local index_exists plan
index_exists="$(sqlite3 "$SQLITE_DB" "SELECT name FROM sqlite_master WHERE type='index' AND name='events_bucketrow_starttime_desc_index';" 2>/dev/null || true)"
if [[ "$index_exists" == "events_bucketrow_starttime_desc_index" ]]; then
ok "AW SQLite hot-path index exists"
else
fail "AW SQLite hot-path index events_bucketrow_starttime_desc_index is missing"
return
fi
plan="$(
sqlite3 "$SQLITE_DB" "EXPLAIN QUERY PLAN SELECT id,starttime,endtime,data FROM events WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_${AW_LOGICAL_HOST_ID}') ORDER BY starttime DESC LIMIT ${AW_EVENTS_LIMIT};" 2>/dev/null || true
)"
if printf '%s' "$plan" | grep -q 'events_bucketrow_starttime_desc_index'; then
ok "AW SQLite worktime event query uses hot-path index"
else
fail "AW SQLite worktime event query does not use hot-path index"
fi
if printf '%s' "$plan" | grep -q 'TEMP B-TREE'; then
fail "AW SQLite worktime event query still builds TEMP B-TREE"
else
ok "AW SQLite worktime event query avoids TEMP B-TREE"
fi
}
check_units_inactive() {
local label="$1"
shift
if ! have systemctl; then
warn "systemctl unavailable; skipping $label runtime check"
return
fi
local active_units=()
local unit active
for unit in "$@"; do
active="$(systemctl is-active "$unit" 2>/dev/null || true)"
if [[ "$active" == "active" || "$active" == "activating" ]]; then
active_units+=("$unit:$active")
fi
done
if [[ "${#active_units[@]}" -eq 0 ]]; then
ok "$label runtime units are inactive"
else
fail "$label runtime units active: ${active_units[*]}"
fi
}
check_live() {
printf '== live resilience checks ==\n'
check_systemd_unit "activitywatch-server"
check_systemd_unit "aw-worktime-api"
check_aw_api
check_aw_hot_path
check_worktime_api
check_failed_units
check_hayabusa_queues
check_sqlite_files
check_sqlite_hot_path_index
if [[ "$EXPECT_OPTIONAL_DLP_OFF" == "1" || "$EXPECT_DLP_PROFILE" == "core_only" ]]; then
check_units_inactive "optional DLP" "${DLP_RUNTIME_UNITS[@]}"
elif [[ "$EXPECT_DLP_PROFILE" == "light" ]]; then
check_units_inactive "heavy DLP" "${DLP_HEAVY_RUNTIME_UNITS[@]}"
else
warn "optional DLP runtime inactive check skipped for profile=$EXPECT_DLP_PROFILE"
fi
if [[ "$EXPECT_LOKI_OFF" == "1" ]]; then
check_units_inactive "Loki" "${LOKI_RUNTIME_UNITS[@]}"
else
warn "Loki inactive check skipped by env"
fi
}
case "$MODE" in
repo) check_repo ;;
live) check_live ;;
all) check_repo; check_live ;;
*) fail "invalid mode: $MODE" ;;
esac
printf 'summary: ok=%s warn=%s fail=%s\n' "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT"
if (( FAIL_COUNT > 0 )); then
exit 1
fi