Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 igor04091968
|
812851063e | ||
|
igor04091968
|
c017cb08a9 |
@@ -43,6 +43,9 @@ struct Cli {
|
||||
|
||||
#[arg(long, default_value_t = false)]
|
||||
no_color: bool,
|
||||
|
||||
#[arg(long, default_value_t = true)]
|
||||
dlp_enabled: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -109,7 +112,12 @@ fn main() {
|
||||
}
|
||||
|
||||
fn run() -> Result<i32> {
|
||||
let cli = Cli::parse();
|
||||
let mut cli = Cli::parse();
|
||||
if let Some(value) =
|
||||
env_nonempty("AW_DLP_ENABLED").or_else(|| env_nonempty("DETMIR_DLP_ENABLED"))
|
||||
{
|
||||
cli.dlp_enabled = parse_env_flag(&value);
|
||||
}
|
||||
let server = cli
|
||||
.server
|
||||
.or_else(|| env_nonempty("AW_CHECK_SERVER"))
|
||||
@@ -172,7 +180,11 @@ fn run() -> Result<i32> {
|
||||
"---------------------------------------------", "--------", "----------------------"
|
||||
);
|
||||
|
||||
for bucket in BUCKETS {
|
||||
for bucket in BUCKETS
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|bucket| cli.dlp_enabled || !bucket.starts_with("aw-dlp-"))
|
||||
{
|
||||
let bucket_full = format!("{bucket}_{host}");
|
||||
let event = bucket_event(
|
||||
&server,
|
||||
@@ -190,6 +202,15 @@ fn run() -> Result<i32> {
|
||||
render_status(&colors, status)
|
||||
);
|
||||
}
|
||||
if !cli.dlp_enabled {
|
||||
println!(
|
||||
"{:<45} {:<8} {:<22} {}",
|
||||
"aw-dlp-*",
|
||||
"-",
|
||||
"disabled",
|
||||
colors.paint(colors.cyan, "SKIPPED")
|
||||
);
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("--- CORS Check ---");
|
||||
@@ -493,6 +514,13 @@ fn env_nonempty(name: &str) -> Option<String> {
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn parse_env_flag(value: &str) -> bool {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::symlink;
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Output, Stdio};
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -198,6 +199,7 @@ fn run_to_file(
|
||||
|
||||
let mut child = Command::new(command)
|
||||
.args(args)
|
||||
.process_group(0)
|
||||
.stdout(Stdio::from(stdout))
|
||||
.stderr(Stdio::from(stderr))
|
||||
.spawn()
|
||||
@@ -209,7 +211,7 @@ fn run_to_file(
|
||||
break status.code().unwrap_or(1);
|
||||
}
|
||||
if started.elapsed() >= timeout {
|
||||
let _ = child.kill();
|
||||
terminate_process_group(child.id());
|
||||
let _ = child.wait();
|
||||
let mut stderr = OpenOptions::new().append(true).open(&stderr_path)?;
|
||||
writeln!(
|
||||
@@ -234,6 +236,17 @@ fn run_to_file(
|
||||
Ok(rc)
|
||||
}
|
||||
|
||||
fn terminate_process_group(child_pid: u32) {
|
||||
let process_group = format!("-{child_pid}");
|
||||
let _ = Command::new("/bin/kill")
|
||||
.args(["-TERM", "--", &process_group])
|
||||
.status();
|
||||
std::thread::sleep(Duration::from_secs(2));
|
||||
let _ = Command::new("/bin/kill")
|
||||
.args(["-KILL", "--", &process_group])
|
||||
.status();
|
||||
}
|
||||
|
||||
fn read_rc(path: &Path) -> i32 {
|
||||
fs::read_to_string(path)
|
||||
.ok()
|
||||
@@ -411,6 +424,7 @@ fn write_report(
|
||||
fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Result<Output> {
|
||||
let mut child = Command::new(polli_bin)
|
||||
.args(["--model", "text.daily", "--max-tokens", "900"])
|
||||
.process_group(0)
|
||||
.stdin(Stdio::from(bundle))
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
@@ -425,7 +439,7 @@ fn run_report_command(polli_bin: &str, bundle: File, timeout: Duration) -> Resul
|
||||
.with_context(|| format!("failed to collect {polli_bin} output"));
|
||||
}
|
||||
if started.elapsed() >= timeout {
|
||||
let _ = child.kill();
|
||||
terminate_process_group(child.id());
|
||||
let _ = child.wait();
|
||||
anyhow::bail!(
|
||||
"Pollinations report timed out after {} seconds",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::io::Read;
|
||||
use std::net::{SocketAddr, TcpStream};
|
||||
use std::os::unix::process::CommandExt;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -89,9 +90,15 @@ struct Cli {
|
||||
#[arg(long, default_value_t = 45)]
|
||||
dlp_timeout_seconds: u64,
|
||||
|
||||
#[arg(long, default_value_t = 150)]
|
||||
overall_timeout_seconds: u64,
|
||||
|
||||
#[arg(long, default_value_t = false)]
|
||||
disable_dlp_health_check: bool,
|
||||
|
||||
#[arg(long, default_value_t = true)]
|
||||
dlp_enabled: bool,
|
||||
|
||||
#[arg(long, default_value_t = false)]
|
||||
disable_portal_check: bool,
|
||||
}
|
||||
@@ -193,8 +200,8 @@ fn parse_env_flag(value: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
|
||||
vec![
|
||||
fn bucket_specs(hostname: &str, dlp_enabled: bool) -> Vec<BucketSpec> {
|
||||
let mut specs = vec![
|
||||
BucketSpec {
|
||||
label: "AFK watcher",
|
||||
bucket: format!("aw-watcher-afk_{hostname}"),
|
||||
@@ -219,31 +226,36 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP signals",
|
||||
bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
|
||||
max_age_seconds: Some(10 * 60),
|
||||
mode: BucketMode::InteractiveFresh,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP incidents",
|
||||
bucket: format!("aw-dlp-incidents_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP review",
|
||||
bucket: format!("aw-dlp-review_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP rules",
|
||||
bucket: format!("aw-dlp-rules_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
]
|
||||
];
|
||||
if dlp_enabled {
|
||||
specs.extend([
|
||||
BucketSpec {
|
||||
label: "DLP signals",
|
||||
bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
|
||||
max_age_seconds: Some(10 * 60),
|
||||
mode: BucketMode::InteractiveFresh,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP incidents",
|
||||
bucket: format!("aw-dlp-incidents_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP review",
|
||||
bucket: format!("aw-dlp-review_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
BucketSpec {
|
||||
label: "DLP rules",
|
||||
bucket: format!("aw-dlp-rules_{hostname}"),
|
||||
max_age_seconds: None,
|
||||
mode: BucketMode::EventDriven,
|
||||
},
|
||||
]);
|
||||
}
|
||||
specs
|
||||
}
|
||||
|
||||
fn build_headers(items: &[(&str, &str)]) -> Result<HeaderMap> {
|
||||
@@ -387,7 +399,20 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
|
||||
if security_events_clickhouse_enabled(args) {
|
||||
checks.push(clickhouse_security_events_check(args));
|
||||
}
|
||||
if !args.disable_dlp_health_check {
|
||||
if !args.dlp_enabled {
|
||||
checks.push(ServiceCheck {
|
||||
name: "aw-dlp-mode".to_string(),
|
||||
required: false,
|
||||
ok: true,
|
||||
url: None,
|
||||
payload: Some(serde_json::json!({
|
||||
"mode": "disabled",
|
||||
"reason": "DETMIR_DLP_ENABLED=false",
|
||||
"checks_skipped": ["DLP health command", "DLP buckets"]
|
||||
})),
|
||||
error: None,
|
||||
});
|
||||
} else if !args.disable_dlp_health_check {
|
||||
checks.push(dlp_health_check(args));
|
||||
}
|
||||
checks
|
||||
@@ -500,6 +525,7 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
|
||||
let mut child = Command::new("/bin/sh")
|
||||
.arg("-lc")
|
||||
.arg(command)
|
||||
.process_group(0)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
@@ -510,7 +536,7 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
|
||||
return read_command_output(child, status.code(), false);
|
||||
}
|
||||
if started.elapsed() >= timeout {
|
||||
let _ = child.kill();
|
||||
terminate_process_group(child.id());
|
||||
let _ = child.wait();
|
||||
return read_command_output(child, None, true);
|
||||
}
|
||||
@@ -518,6 +544,17 @@ fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<Command
|
||||
}
|
||||
}
|
||||
|
||||
fn terminate_process_group(child_pid: u32) {
|
||||
let process_group = format!("-{child_pid}");
|
||||
let _ = Command::new("/bin/kill")
|
||||
.args(["-TERM", "--", &process_group])
|
||||
.status();
|
||||
std::thread::sleep(Duration::from_secs(2));
|
||||
let _ = Command::new("/bin/kill")
|
||||
.args(["-KILL", "--", &process_group])
|
||||
.status();
|
||||
}
|
||||
|
||||
fn read_command_output(
|
||||
mut child: std::process::Child,
|
||||
code: Option<i32>,
|
||||
@@ -796,7 +833,7 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
|
||||
let interactive_required = interactive_required(&client, &args.hostname, now);
|
||||
let mut out = Vec::new();
|
||||
|
||||
for spec in bucket_specs(&args.hostname) {
|
||||
for spec in bucket_specs(&args.hostname, args.dlp_enabled) {
|
||||
if matches!(spec.mode, BucketMode::EventDriven) {
|
||||
out.push(BucketCheck {
|
||||
label: spec.label.to_string(),
|
||||
@@ -993,6 +1030,19 @@ fn render_text(report: &CheckReport) -> String {
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let mut args = Cli::parse();
|
||||
args.service_timeout_seconds = env_u64(
|
||||
"DETMIR_SERVICE_TIMEOUT_SECONDS",
|
||||
args.service_timeout_seconds,
|
||||
);
|
||||
args.bucket_timeout_seconds =
|
||||
env_u64("DETMIR_BUCKET_TIMEOUT_SECONDS", args.bucket_timeout_seconds);
|
||||
args.tcp_timeout_seconds = env_f64("DETMIR_TCP_TIMEOUT_SECONDS", args.tcp_timeout_seconds);
|
||||
args.dlp_timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", args.dlp_timeout_seconds);
|
||||
args.overall_timeout_seconds = env_u64(
|
||||
"DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS",
|
||||
args.overall_timeout_seconds,
|
||||
);
|
||||
start_overall_timeout_watchdog(args.overall_timeout_seconds);
|
||||
args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api);
|
||||
args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
|
||||
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
|
||||
@@ -1011,6 +1061,12 @@ fn main() -> Result<()> {
|
||||
if env_flag_enabled("DETMIR_DISABLE_DLP_HEALTH_CHECK") {
|
||||
args.disable_dlp_health_check = true;
|
||||
}
|
||||
if let Some(value) = std::env::var("DETMIR_DLP_ENABLED")
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
args.dlp_enabled = parse_env_flag(&value);
|
||||
}
|
||||
if env_flag_enabled("DETMIR_DISABLE_PORTAL_CHECK") {
|
||||
args.disable_portal_check = true;
|
||||
}
|
||||
@@ -1028,6 +1084,31 @@ fn main() -> Result<()> {
|
||||
});
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, fallback: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn env_f64(name: &str, fallback: f64) -> f64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn start_overall_timeout_watchdog(seconds: u64) {
|
||||
if seconds == 0 {
|
||||
return;
|
||||
}
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_secs(seconds));
|
||||
eprintln!("detmir-check timed out after {seconds} seconds");
|
||||
std::process::exit(124);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1095,6 +1176,28 @@ mod tests {
|
||||
assert!(!parse_env_flag("false"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dlp_disabled_removes_dlp_bucket_specs() {
|
||||
let enabled = bucket_specs("HOST-EXAMPLE", true);
|
||||
assert!(
|
||||
enabled
|
||||
.iter()
|
||||
.any(|spec| spec.bucket.starts_with("aw-dlp-"))
|
||||
);
|
||||
|
||||
let disabled = bucket_specs("HOST-EXAMPLE", false);
|
||||
assert!(
|
||||
disabled
|
||||
.iter()
|
||||
.all(|spec| !spec.bucket.starts_with("aw-dlp-"))
|
||||
);
|
||||
assert!(
|
||||
disabled
|
||||
.iter()
|
||||
.any(|spec| spec.bucket.starts_with("aw-worktime-sessions_"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clickhouse_database_identifier_rejects_injection() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -9,3 +9,4 @@ publish.workspace = true
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::io::{self, Write};
|
||||
use std::process::Command;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
@@ -19,8 +20,14 @@ struct Cli {
|
||||
#[arg(long, default_value_t = 10)]
|
||||
connect_timeout_seconds: u64,
|
||||
|
||||
#[arg(long, default_value_t = 90)]
|
||||
timeout_seconds: u64,
|
||||
|
||||
#[arg(long, default_value = DEFAULT_REMOTE_COMMAND)]
|
||||
remote_command: String,
|
||||
|
||||
#[arg(long, default_value_t = true)]
|
||||
enabled: bool,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
@@ -31,6 +38,8 @@ impl Cli {
|
||||
);
|
||||
self.remote_command = env_first(&["DETMIR_DLP_REMOTE_COMMAND"], &self.remote_command);
|
||||
self.ssh_bin = env_first(&["DETMIR_SSH_BIN"], &self.ssh_bin);
|
||||
self.timeout_seconds = env_u64("DETMIR_DLP_TIMEOUT_SECONDS", self.timeout_seconds);
|
||||
self.enabled = env_bool("DETMIR_DLP_ENABLED", self.enabled);
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -42,6 +51,25 @@ fn env_first(names: &[&str], fallback: &str) -> String {
|
||||
.unwrap_or_else(|| fallback.to_string())
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, fallback: u64) -> u64 {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn env_bool(name: &str, fallback: bool) -> bool {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.map(|value| {
|
||||
matches!(
|
||||
value.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn ssh_args(cli: &Cli) -> Vec<String> {
|
||||
vec![
|
||||
"-o".to_string(),
|
||||
@@ -56,22 +84,76 @@ fn ssh_args(cli: &Cli) -> Vec<String> {
|
||||
}
|
||||
|
||||
fn run(cli: Cli) -> Result<i32> {
|
||||
if !cli.enabled {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"ok": true,
|
||||
"counts": {"ok": 1, "warn": 0, "fail": 0},
|
||||
"results": [{
|
||||
"name": "dlp:mode",
|
||||
"status": "ok",
|
||||
"summary": "DLP health check disabled by DETMIR_DLP_ENABLED=false",
|
||||
"details": {
|
||||
"mode": "disabled",
|
||||
"load_reduction": ["aw-dlp health ssh probe skipped"]
|
||||
}
|
||||
}]
|
||||
}))?
|
||||
);
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let args = ssh_args(&cli);
|
||||
let output = Command::new(&cli.ssh_bin)
|
||||
let mut child = Command::new(&cli.ssh_bin)
|
||||
.args(&args)
|
||||
.output()
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("failed to execute {}", cli.ssh_bin))?;
|
||||
|
||||
let started = Instant::now();
|
||||
let mut timed_out = false;
|
||||
loop {
|
||||
if child.try_wait()?.is_some() {
|
||||
break;
|
||||
}
|
||||
if started.elapsed() >= Duration::from_secs(cli.timeout_seconds) {
|
||||
timed_out = true;
|
||||
terminate_child(&mut child);
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.context("failed to collect SSH output")?;
|
||||
io::stdout()
|
||||
.write_all(&output.stdout)
|
||||
.context("failed to write DLP stdout")?;
|
||||
io::stderr()
|
||||
.write_all(&output.stderr)
|
||||
.context("failed to write DLP stderr")?;
|
||||
if timed_out {
|
||||
writeln!(
|
||||
io::stderr(),
|
||||
"detmir-dlp timed out after {} seconds",
|
||||
cli.timeout_seconds
|
||||
)
|
||||
.context("failed to write timeout message")?;
|
||||
return Ok(124);
|
||||
}
|
||||
|
||||
Ok(output.status.code().unwrap_or(1))
|
||||
}
|
||||
|
||||
fn terminate_child(child: &mut std::process::Child) {
|
||||
let _ = child.kill();
|
||||
std::thread::sleep(Duration::from_secs(2));
|
||||
let _ = child.kill();
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse().apply_env();
|
||||
let code = run(cli)?;
|
||||
@@ -88,7 +170,9 @@ mod tests {
|
||||
ssh_bin: "ssh".to_string(),
|
||||
ssh_target: DEFAULT_SSH_TARGET.to_string(),
|
||||
connect_timeout_seconds: 10,
|
||||
timeout_seconds: 90,
|
||||
remote_command: DEFAULT_REMOTE_COMMAND.to_string(),
|
||||
enabled: true,
|
||||
};
|
||||
assert_eq!(
|
||||
ssh_args(&cli),
|
||||
|
||||
@@ -62,6 +62,21 @@ struct Cli {
|
||||
|
||||
#[arg(long)]
|
||||
json: bool,
|
||||
|
||||
#[arg(long, default_value_t = 120)]
|
||||
overall_timeout_seconds: u64,
|
||||
|
||||
#[arg(long, default_value = "full")]
|
||||
profile: String,
|
||||
|
||||
#[arg(long, default_value_t = true)]
|
||||
enabled: bool,
|
||||
|
||||
#[arg(long, default_value = "operator_disabled")]
|
||||
disabled_reason: String,
|
||||
|
||||
#[arg(long, default_value = "")]
|
||||
disabled_since: String,
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
@@ -129,6 +144,28 @@ impl Cli {
|
||||
if !cli_arg_present("--profiles") {
|
||||
self.profiles = env_string("AW_DLP_COMPLIANCE_PROFILES").unwrap_or(self.profiles);
|
||||
}
|
||||
if !cli_arg_present("--overall-timeout-seconds") {
|
||||
self.overall_timeout_seconds = env_u64(
|
||||
"AW_DLP_HEALTH_OVERALL_TIMEOUT_SECONDS",
|
||||
self.overall_timeout_seconds,
|
||||
);
|
||||
}
|
||||
if !cli_arg_present("--profile") {
|
||||
self.profile = env_string("AW_DLP_PROFILE")
|
||||
.or_else(|| env_string("DETMIR_PORTAL_DLP_PROFILE"))
|
||||
.unwrap_or(self.profile);
|
||||
}
|
||||
if !cli_arg_present("--enabled") {
|
||||
self.enabled = env_bool_default("AW_DLP_ENABLED", self.enabled);
|
||||
}
|
||||
if !cli_arg_present("--disabled-reason") {
|
||||
self.disabled_reason =
|
||||
env_string("AW_DLP_DISABLED_REASON").unwrap_or(self.disabled_reason);
|
||||
}
|
||||
if !cli_arg_present("--disabled-since") {
|
||||
self.disabled_since =
|
||||
env_string("AW_DLP_DISABLED_SINCE").unwrap_or(self.disabled_since);
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -1270,8 +1307,66 @@ fn check_compliance_reports(
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_profile(profile: &str) -> String {
|
||||
match profile.trim().to_ascii_lowercase().as_str() {
|
||||
"disabled" | "off" => "core_only".to_string(),
|
||||
"core-only" | "core_only" => "core_only".to_string(),
|
||||
"light" | "lite" => "light".to_string(),
|
||||
"on-demand" | "on_demand" => "on_demand".to_string(),
|
||||
"enabled" | "on" | "full" => "full".to_string(),
|
||||
"" => "full".to_string(),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn profile_is_disabled(profile: &str) -> bool {
|
||||
matches!(normalized_profile(profile).as_str(), "core_only")
|
||||
}
|
||||
|
||||
fn profile_checks_heavy_services(profile: &str) -> bool {
|
||||
matches!(normalized_profile(profile).as_str(), "full" | "on_demand")
|
||||
}
|
||||
|
||||
fn build_report(cli: &Cli, client: &Client) -> HealthReport {
|
||||
let mut report = HealthReport::default();
|
||||
let profile = normalized_profile(&cli.profile);
|
||||
if !cli.enabled || profile_is_disabled(&profile) {
|
||||
report.add(
|
||||
"dlp:mode",
|
||||
"ok",
|
||||
"DLP runtime disabled by production profile",
|
||||
json!({
|
||||
"mode": "disabled",
|
||||
"profile": profile,
|
||||
"reason": &cli.disabled_reason,
|
||||
"disabled_since": empty_string_as_null(&cli.disabled_since),
|
||||
"checks_skipped": [
|
||||
"policy API",
|
||||
"case API",
|
||||
"DLP systemd units",
|
||||
"DLP ActivityWatch buckets",
|
||||
"DLP compliance reports"
|
||||
],
|
||||
"load_reduction": [
|
||||
"no DLP bucket freshness reads",
|
||||
"no DLP case/policy HTTP checks",
|
||||
"no DLP compliance filesystem scan"
|
||||
]
|
||||
}),
|
||||
);
|
||||
return report;
|
||||
}
|
||||
report.add(
|
||||
"dlp:mode",
|
||||
"ok",
|
||||
format!("DLP runtime profile {profile}"),
|
||||
json!({
|
||||
"mode": if profile == "light" { "light" } else { "enabled" },
|
||||
"profile": profile,
|
||||
"heavy_services_checked": profile_checks_heavy_services(&cli.profile)
|
||||
}),
|
||||
);
|
||||
|
||||
let aw_api_base = format!("{}/api/0", cli.aw_server.trim_end_matches('/'));
|
||||
let counter_state_path = cli.state_dir.join("dlp-health-check-counters.json");
|
||||
let mut counter_state = load_counter_state(&counter_state_path);
|
||||
@@ -1282,38 +1377,70 @@ fn build_report(cli: &Cli, client: &Client) -> HealthReport {
|
||||
"http:aw",
|
||||
&format!("{aw_api_base}/info"),
|
||||
);
|
||||
check_http_endpoint(
|
||||
&mut report,
|
||||
client,
|
||||
"http:policy",
|
||||
&format!("{}/healthz", cli.policy_server.trim_end_matches('/')),
|
||||
);
|
||||
check_http_endpoint(
|
||||
&mut report,
|
||||
client,
|
||||
"http:cases",
|
||||
&format!("{}/health", cli.case_server.trim_end_matches('/')),
|
||||
);
|
||||
if profile_checks_heavy_services(&cli.profile) {
|
||||
check_http_endpoint(
|
||||
&mut report,
|
||||
client,
|
||||
"http:policy",
|
||||
&format!("{}/healthz", cli.policy_server.trim_end_matches('/')),
|
||||
);
|
||||
check_http_endpoint(
|
||||
&mut report,
|
||||
client,
|
||||
"http:cases",
|
||||
&format!("{}/health", cli.case_server.trim_end_matches('/')),
|
||||
);
|
||||
} else {
|
||||
report.add(
|
||||
"http:heavy-dlp",
|
||||
"ok",
|
||||
"heavy DLP policy/case HTTP checks skipped for lightweight profile",
|
||||
json!({
|
||||
"profile": profile,
|
||||
"skipped": ["policy API", "case API"]
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
for unit in [
|
||||
"activitywatch-server",
|
||||
"aw-dlp-policy-engine.service",
|
||||
"aw-dlp-case-management.service",
|
||||
"aw-worktime-api.service",
|
||||
] {
|
||||
for unit in ["activitywatch-server", "aw-worktime-api.service"] {
|
||||
check_systemd_unit(&mut report, unit, "service");
|
||||
}
|
||||
for unit in [
|
||||
"aw-dlp-report-scheduler.timer",
|
||||
"aw-dlp-syslog-forwarder.timer",
|
||||
"aw-dlp-webhook-sender.timer",
|
||||
"aw-dlp-cef-exporter.timer",
|
||||
"activitywatch-dlp-aggregator.timer",
|
||||
"aw-dlp-ioc-refresh.timer",
|
||||
"aw-worktime-ui-bridge.timer",
|
||||
] {
|
||||
check_systemd_unit(&mut report, unit, "timer");
|
||||
if profile_checks_heavy_services(&cli.profile) {
|
||||
for unit in [
|
||||
"aw-dlp-policy-engine.service",
|
||||
"aw-dlp-case-management.service",
|
||||
] {
|
||||
check_systemd_unit(&mut report, unit, "service");
|
||||
}
|
||||
for unit in [
|
||||
"aw-dlp-report-scheduler.timer",
|
||||
"aw-dlp-syslog-forwarder.timer",
|
||||
"aw-dlp-webhook-sender.timer",
|
||||
"aw-dlp-cef-exporter.timer",
|
||||
"activitywatch-dlp-aggregator.timer",
|
||||
"aw-dlp-ioc-refresh.timer",
|
||||
] {
|
||||
check_systemd_unit(&mut report, unit, "timer");
|
||||
}
|
||||
} else {
|
||||
report.add(
|
||||
"systemd:heavy-dlp",
|
||||
"ok",
|
||||
"heavy DLP systemd checks skipped for lightweight profile",
|
||||
json!({
|
||||
"profile": profile,
|
||||
"skipped": [
|
||||
"aw-dlp-policy-engine.service",
|
||||
"aw-dlp-case-management.service",
|
||||
"aw-dlp-report-scheduler.timer",
|
||||
"aw-dlp-syslog-forwarder.timer",
|
||||
"aw-dlp-webhook-sender.timer",
|
||||
"aw-dlp-cef-exporter.timer"
|
||||
]
|
||||
}),
|
||||
);
|
||||
}
|
||||
check_systemd_unit(&mut report, "aw-worktime-ui-bridge.timer", "timer");
|
||||
|
||||
match http_json(client, &format!("{aw_api_base}/buckets"), 15, 2) {
|
||||
Ok(Value::Object(map)) => {
|
||||
@@ -1462,6 +1589,12 @@ fn env_i64(name: &str, default: i64) -> i64 {
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn env_u64(name: &str, default: u64) -> u64 {
|
||||
env_string(name)
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn env_bool(name: &str) -> bool {
|
||||
env_string(name)
|
||||
.map(|value| {
|
||||
@@ -1473,8 +1606,39 @@ fn env_bool(name: &str) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn env_bool_default(name: &str, default: bool) -> bool {
|
||||
env_string(name)
|
||||
.map(|value| {
|
||||
matches!(
|
||||
value.to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
fn empty_string_as_null(value: &str) -> Value {
|
||||
if value.trim().is_empty() {
|
||||
Value::Null
|
||||
} else {
|
||||
json!(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn start_overall_timeout_watchdog(seconds: u64) {
|
||||
if seconds == 0 {
|
||||
return;
|
||||
}
|
||||
std::thread::spawn(move || {
|
||||
sleep(Duration::from_secs(seconds));
|
||||
eprintln!("dlp-health-check timed out after {seconds} seconds");
|
||||
std::process::exit(124);
|
||||
});
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse().apply_env();
|
||||
start_overall_timeout_watchdog(cli.overall_timeout_seconds);
|
||||
let client = Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
@@ -1547,4 +1711,22 @@ mod tests {
|
||||
assert_eq!(payload.counts.warn, 1);
|
||||
assert_eq!(payload.counts.fail, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dlp_profile_normalization_matches_runtime_control_names() {
|
||||
assert_eq!(normalized_profile("disabled"), "core_only");
|
||||
assert_eq!(normalized_profile("core-only"), "core_only");
|
||||
assert_eq!(normalized_profile("light"), "light");
|
||||
assert_eq!(normalized_profile("lite"), "light");
|
||||
assert_eq!(normalized_profile("on-demand"), "on_demand");
|
||||
assert_eq!(normalized_profile("enabled"), "full");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn light_profile_does_not_require_heavy_services() {
|
||||
assert!(!profile_checks_heavy_services("light"));
|
||||
assert!(!profile_checks_heavy_services("core_only"));
|
||||
assert!(profile_checks_heavy_services("on_demand"));
|
||||
assert!(profile_checks_heavy_services("full"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +77,7 @@ struct Config {
|
||||
management_history_retention_days: i64,
|
||||
true_active_evidence_window_seconds: i64,
|
||||
true_active_max_event_seconds: i64,
|
||||
dlp_evidence_enabled: bool,
|
||||
offset: FixedOffset,
|
||||
}
|
||||
|
||||
@@ -227,6 +228,14 @@ fn threshold_to_pct(value: f64) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn overload_threshold_to_pct(value: f64) -> f64 {
|
||||
if (0.0..=3.0).contains(&value) {
|
||||
value * 100.0
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hhmm(value: &str) -> Option<NaiveTime> {
|
||||
NaiveTime::parse_from_str(value.trim(), "%H:%M").ok()
|
||||
}
|
||||
@@ -262,7 +271,7 @@ fn load_config() -> Config {
|
||||
}
|
||||
if let Some(value) = policy.overload_threshold {
|
||||
manager_overload_coverage_pct =
|
||||
threshold_to_pct(value).round().clamp(1.0, 300.0) as i64;
|
||||
overload_threshold_to_pct(value).round().clamp(100.0, 300.0) as i64;
|
||||
}
|
||||
if let Some(value) = policy.drop_threshold_pct {
|
||||
manager_trend_delta_pct = threshold_to_pct(value).clamp(1.0, 100.0);
|
||||
@@ -371,6 +380,10 @@ fn load_config() -> Config {
|
||||
.max(30),
|
||||
true_active_max_event_seconds: env_i64("AW_WORKTIME_TRUE_ACTIVE_MAX_EVENT_SECONDS", 600)
|
||||
.max(30),
|
||||
dlp_evidence_enabled: env_bool(
|
||||
"AW_WORKTIME_DLP_EVIDENCE_ENABLED",
|
||||
env_bool("AW_DLP_ENABLED", true),
|
||||
),
|
||||
offset: FixedOffset::east_opt(3 * 3600).expect("valid Moscow offset"),
|
||||
}
|
||||
}
|
||||
@@ -1010,13 +1023,7 @@ impl App {
|
||||
}
|
||||
};
|
||||
let mut evidence = HashMap::new();
|
||||
for bucket in [
|
||||
format!("aw-file-operations_{host}"),
|
||||
format!("aw-dlp-endpoint-signals_{host}"),
|
||||
format!("aw-watcher-web-chrome_{host}"),
|
||||
format!("aw-watcher-web-edge_{host}"),
|
||||
format!("aw-detmir-web-category_{host}"),
|
||||
] {
|
||||
for bucket in evidence_bucket_ids(&self.config, host) {
|
||||
evidence.insert(
|
||||
bucket.clone(),
|
||||
self.fetch_bucket_events(&bucket, Some(bounds.0), Some(bounds.1)),
|
||||
@@ -1156,6 +1163,19 @@ fn sanitize_bucket_for_log(bucket_id: &str) -> String {
|
||||
bucket_id.to_string()
|
||||
}
|
||||
|
||||
fn evidence_bucket_ids(config: &Config, host: &str) -> Vec<String> {
|
||||
let mut buckets = vec![format!("aw-file-operations_{host}")];
|
||||
if config.dlp_evidence_enabled {
|
||||
buckets.push(format!("aw-dlp-endpoint-signals_{host}"));
|
||||
}
|
||||
buckets.extend([
|
||||
format!("aw-watcher-web-chrome_{host}"),
|
||||
format!("aw-watcher-web-edge_{host}"),
|
||||
format!("aw-detmir-web-category_{host}"),
|
||||
]);
|
||||
buckets
|
||||
}
|
||||
|
||||
fn sanitize_error_for_log(value: &str) -> String {
|
||||
static IP_RE: OnceLock<Regex> = OnceLock::new();
|
||||
static BUCKET_HOST_RE: OnceLock<Regex> = OnceLock::new();
|
||||
@@ -1605,6 +1625,156 @@ fn interval_overlap_seconds(
|
||||
(total, first, last)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct EmployeeOperationsInput {
|
||||
expected_seconds: i64,
|
||||
low_seconds: i64,
|
||||
target_seconds: i64,
|
||||
overload_seconds: i64,
|
||||
work_secs: i64,
|
||||
calendar_secs: i64,
|
||||
work_first: Option<DateTime<Utc>>,
|
||||
work_last: Option<DateTime<Utc>>,
|
||||
is_today: bool,
|
||||
late_start: DateTime<FixedOffset>,
|
||||
early_finish: DateTime<FixedOffset>,
|
||||
samples_count: i64,
|
||||
active_samples: i64,
|
||||
sessions_count: i64,
|
||||
}
|
||||
|
||||
fn build_employee_operations_status(config: &Config, input: EmployeeOperationsInput) -> Value {
|
||||
let workday_idle_seconds = (input.expected_seconds - input.work_secs).max(0);
|
||||
let off_hours_seconds = (input.calendar_secs - input.work_secs).max(0);
|
||||
let coverage = if input.expected_seconds > 0 {
|
||||
clamp_pct(input.work_secs as f64 / input.expected_seconds as f64 * 100.0)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let load_status = if input.expected_seconds <= 0 {
|
||||
"insufficient_data"
|
||||
} else if input.sessions_count <= 0 || input.samples_count <= 0 {
|
||||
"no_data"
|
||||
} else if input.work_secs <= 0 {
|
||||
"no_activity"
|
||||
} else if input.work_secs < input.low_seconds {
|
||||
"underloaded"
|
||||
} else if input.work_secs >= input.overload_seconds {
|
||||
"overloaded"
|
||||
} else if input.work_secs < input.target_seconds {
|
||||
"below_target"
|
||||
} else {
|
||||
"normal"
|
||||
};
|
||||
let idle_status = if input.expected_seconds <= 0 {
|
||||
"not_applicable"
|
||||
} else if input.sessions_count <= 0 || input.samples_count <= 0 {
|
||||
"unknown"
|
||||
} else if input.work_secs <= 0 {
|
||||
"full_workday_idle_or_absent"
|
||||
} else if workday_idle_seconds >= config.manager_off_hours_threshold_seconds {
|
||||
"idle_detected"
|
||||
} else {
|
||||
"no_significant_idle"
|
||||
};
|
||||
let mut discipline_flags = Vec::new();
|
||||
if off_hours_seconds >= config.manager_off_hours_threshold_seconds {
|
||||
discipline_flags.push("off_hours");
|
||||
}
|
||||
if input
|
||||
.work_first
|
||||
.is_some_and(|dt| dt.with_timezone(&config.offset) > input.late_start)
|
||||
{
|
||||
discipline_flags.push("late_start");
|
||||
}
|
||||
if !input.is_today
|
||||
&& input
|
||||
.work_last
|
||||
.is_some_and(|dt| dt.with_timezone(&config.offset) < input.early_finish)
|
||||
{
|
||||
discipline_flags.push("early_finish");
|
||||
}
|
||||
let discipline_status = match discipline_flags.as_slice() {
|
||||
[] => "ok",
|
||||
[single] => single,
|
||||
_ => "multiple_flags",
|
||||
};
|
||||
let mut confidence_reasons = Vec::new();
|
||||
if input.sessions_count <= 0 {
|
||||
confidence_reasons.push("missing_session_samples");
|
||||
}
|
||||
if input.samples_count <= 0 {
|
||||
confidence_reasons.push("missing_worktime_samples");
|
||||
} else if input.samples_count < 3 {
|
||||
confidence_reasons.push("few_worktime_samples");
|
||||
} else {
|
||||
confidence_reasons.push("worktime_samples_present");
|
||||
}
|
||||
if input.active_samples <= 0 {
|
||||
confidence_reasons.push("missing_active_samples");
|
||||
} else {
|
||||
confidence_reasons.push("active_samples_present");
|
||||
}
|
||||
if input.expected_seconds <= 0 {
|
||||
confidence_reasons.push("workday_window_not_started_or_empty");
|
||||
}
|
||||
let data_confidence =
|
||||
if input.expected_seconds <= 0 || input.sessions_count <= 0 || input.samples_count <= 0 {
|
||||
"low"
|
||||
} else if input.samples_count < 3 || input.active_samples <= 0 {
|
||||
"medium"
|
||||
} else {
|
||||
"high"
|
||||
};
|
||||
let recommended_action = match (data_confidence, load_status, discipline_status, idle_status) {
|
||||
("low", _, _, _) => {
|
||||
"Сначала проверить свежесть источников и наличие сессии; вывод по сотруднику не использовать как дисциплинарный."
|
||||
}
|
||||
(_, "overloaded", _, _) => {
|
||||
"Проверить переработку, перераспределение задач и риск аврального процесса."
|
||||
}
|
||||
(_, "underloaded" | "below_target" | "no_activity", _, _) => {
|
||||
"Проверить фактическую загрузку, задачи, доступ к рабочим системам и отсутствие сбоя сбора."
|
||||
}
|
||||
(_, _, "off_hours" | "late_start" | "early_finish" | "multiple_flags", _) => {
|
||||
"Проверить согласование рабочего графика и причину отклонения от процесса."
|
||||
}
|
||||
(_, _, _, "idle_detected" | "full_workday_idle_or_absent") => {
|
||||
"Проверить простой: отсутствие задач, ожидание внешнего процесса или техническую проблему."
|
||||
}
|
||||
_ => "Наблюдать; отклонений, требующих немедленного действия, не выявлено.",
|
||||
};
|
||||
json!({
|
||||
"load_status": load_status,
|
||||
"idle_status": idle_status,
|
||||
"discipline_status": discipline_status,
|
||||
"discipline_flags": discipline_flags,
|
||||
"data_confidence": data_confidence,
|
||||
"confidence_reasons": confidence_reasons,
|
||||
"workday_idle_seconds": workday_idle_seconds,
|
||||
"workday_idle_hhmm": hhmm(workday_idle_seconds),
|
||||
"off_hours_seconds": off_hours_seconds,
|
||||
"off_hours_hhmm": hhmm(off_hours_seconds),
|
||||
"coverage_pct": coverage,
|
||||
"evidence": {
|
||||
"expected_hhmm": hhmm(input.expected_seconds),
|
||||
"workday_active_hhmm": hhmm(input.work_secs),
|
||||
"calendar_active_hhmm": hhmm(input.calendar_secs),
|
||||
"sessions_count": input.sessions_count,
|
||||
"samples_count": input.samples_count,
|
||||
"active_samples": input.active_samples,
|
||||
"first_workday_activity_local": input.work_first.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default(),
|
||||
"last_workday_activity_local": input.work_last.map(|dt| dt.with_timezone(&config.offset).to_rfc3339()).unwrap_or_default()
|
||||
},
|
||||
"guardrail": if data_confidence == "low" {
|
||||
"low_confidence_not_for_discipline"
|
||||
} else {
|
||||
"evidence_backed_manual_review_only"
|
||||
},
|
||||
"recommended_action": recommended_action,
|
||||
})
|
||||
}
|
||||
|
||||
impl App {
|
||||
fn build_management_payload(
|
||||
&self,
|
||||
@@ -1634,6 +1804,7 @@ impl App {
|
||||
};
|
||||
let target_seconds = expected_seconds * self.config.manager_target_coverage_pct / 100;
|
||||
let low_seconds = expected_seconds * self.config.manager_low_coverage_pct / 100;
|
||||
let overload_seconds = expected_seconds * self.config.manager_overload_coverage_pct / 100;
|
||||
let late_start =
|
||||
work_start_local + TimeDelta::minutes(self.config.manager_late_start_grace_minutes);
|
||||
let early_finish =
|
||||
@@ -1706,6 +1877,37 @@ impl App {
|
||||
.map(|dt| dt.with_timezone(&self.config.offset))
|
||||
.map(|dt| dt.to_rfc3339())
|
||||
.unwrap_or_default();
|
||||
let samples_count = row
|
||||
.get("samples_count")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(0);
|
||||
let active_samples = row
|
||||
.get("active_samples")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(0);
|
||||
let sessions_count = row
|
||||
.get("sessions_count")
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(0);
|
||||
let operations = build_employee_operations_status(
|
||||
&self.config,
|
||||
EmployeeOperationsInput {
|
||||
expected_seconds,
|
||||
low_seconds,
|
||||
target_seconds,
|
||||
overload_seconds,
|
||||
work_secs,
|
||||
calendar_secs,
|
||||
work_first,
|
||||
work_last,
|
||||
is_today,
|
||||
late_start,
|
||||
early_finish,
|
||||
samples_count,
|
||||
active_samples,
|
||||
sessions_count,
|
||||
},
|
||||
);
|
||||
let mut public = row.as_object().cloned().unwrap_or_default();
|
||||
public.remove("_intervals");
|
||||
public.insert("user".into(), json!(alias.display_name));
|
||||
@@ -1719,8 +1921,52 @@ impl App {
|
||||
public.insert("calendar_active_hhmm".into(), json!(hhmm(calendar_secs)));
|
||||
public.insert("workday_active_seconds".into(), json!(work_secs));
|
||||
public.insert("workday_active_hhmm".into(), json!(hhmm(work_secs)));
|
||||
public.insert(
|
||||
"workday_idle_seconds".into(),
|
||||
json!((expected_seconds - work_secs).max(0)),
|
||||
);
|
||||
public.insert(
|
||||
"workday_idle_hhmm".into(),
|
||||
json!(hhmm((expected_seconds - work_secs).max(0))),
|
||||
);
|
||||
public.insert("coverage_pct".into(), json!(coverage));
|
||||
public.insert("status".into(), json!(status));
|
||||
public.insert(
|
||||
"load_status".into(),
|
||||
operations
|
||||
.get("load_status")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("unknown")),
|
||||
);
|
||||
public.insert(
|
||||
"idle_status".into(),
|
||||
operations
|
||||
.get("idle_status")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("unknown")),
|
||||
);
|
||||
public.insert(
|
||||
"discipline_status".into(),
|
||||
operations
|
||||
.get("discipline_status")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("unknown")),
|
||||
);
|
||||
public.insert(
|
||||
"data_confidence".into(),
|
||||
operations
|
||||
.get("data_confidence")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("low")),
|
||||
);
|
||||
public.insert(
|
||||
"operations_recommended_action".into(),
|
||||
operations
|
||||
.get("recommended_action")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!("Проверить первичные источники.")),
|
||||
);
|
||||
public.insert("operations".into(), operations.clone());
|
||||
public.insert("first_activity_local".into(), json!(first_local));
|
||||
public.insert("last_activity_local".into(), json!(last_local));
|
||||
public.insert(
|
||||
@@ -1739,6 +1985,10 @@ impl App {
|
||||
"manager_owner": public.get("manager_owner").cloned().unwrap_or(json!("")),
|
||||
"department": public.get("department").cloned().unwrap_or(json!("")),
|
||||
"role": public.get("role").cloned().unwrap_or(json!("")),
|
||||
"load_status": public.get("load_status").cloned().unwrap_or(json!("unknown")),
|
||||
"idle_status": public.get("idle_status").cloned().unwrap_or(json!("unknown")),
|
||||
"discipline_status": public.get("discipline_status").cloned().unwrap_or(json!("unknown")),
|
||||
"data_confidence": public.get("data_confidence").cloned().unwrap_or(json!("low")),
|
||||
});
|
||||
let owner = public
|
||||
.get("manager_owner")
|
||||
@@ -1763,6 +2013,9 @@ impl App {
|
||||
} else if expected_seconds > 0 && work_secs < target_seconds {
|
||||
actions.push(action("target_gap_review", "medium", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} ниже управленческого целевого порога {}%.", hhmm(work_secs), self.config.manager_target_coverage_pct), &format!("Уточнить причину отклонения по сотруднику {display} и подтвердить план работ."), &canonical, evidence.clone()));
|
||||
}
|
||||
if expected_seconds > 0 && work_secs >= overload_seconds {
|
||||
actions.push(action("overload_review", "high", &owner, "24h", &format!("У сотрудника {display} активное время в рабочем окне {} выше порога перегруза {}%.", hhmm(work_secs), self.config.manager_overload_coverage_pct), &format!("Проверить переработку сотрудника {display}, распределение задач и риск аврального процесса."), &canonical, evidence.clone()));
|
||||
}
|
||||
if work_first.is_some_and(|dt| dt.with_timezone(&self.config.offset) > late_start) {
|
||||
actions.push(action("late_start_review", "medium", &owner, "24h", &format!("У сотрудника {display} первая активность в рабочем окне зафиксирована поздно."), &format!("Проверить причину позднего старта сотрудника {display} и подтвердить, что это не проблема доступа или дисциплины."), &canonical, evidence.clone()));
|
||||
}
|
||||
@@ -1810,6 +2063,37 @@ impl App {
|
||||
)
|
||||
});
|
||||
let summary = summarize_management_rows(&roster, &actions, expected_seconds);
|
||||
let workforce_operations = json!({
|
||||
"status": summary.pointer("/workforce_operations/status").cloned().unwrap_or_else(|| json!("LOW_CONFIDENCE")),
|
||||
"summary": summary.pointer("/workforce_operations").cloned().unwrap_or_else(|| json!({})),
|
||||
"rows": roster.iter().map(|row| {
|
||||
json!({
|
||||
"user": row.get("user").cloned().unwrap_or(json!("")),
|
||||
"manager_owner": row.get("manager_owner").cloned().unwrap_or(json!("")),
|
||||
"department": row.get("department").cloned().unwrap_or(json!("")),
|
||||
"role": row.get("role").cloned().unwrap_or(json!("")),
|
||||
"load_status": row.get("load_status").cloned().unwrap_or(json!("unknown")),
|
||||
"idle_status": row.get("idle_status").cloned().unwrap_or(json!("unknown")),
|
||||
"discipline_status": row.get("discipline_status").cloned().unwrap_or(json!("unknown")),
|
||||
"data_confidence": row.get("data_confidence").cloned().unwrap_or(json!("low")),
|
||||
"coverage_pct": row.get("coverage_pct").cloned().unwrap_or(json!(0.0)),
|
||||
"workday_active_hhmm": row.get("workday_active_hhmm").cloned().unwrap_or(json!("00:00")),
|
||||
"workday_idle_hhmm": row.get("workday_idle_hhmm").cloned().unwrap_or(json!("00:00")),
|
||||
"recommended_action": row.get("operations_recommended_action").cloned().unwrap_or(json!("Проверить первичные источники."))
|
||||
})
|
||||
}).collect::<Vec<_>>(),
|
||||
"model": {
|
||||
"type": "rule_based",
|
||||
"ml": false,
|
||||
"llm": false,
|
||||
"version": "workforce-operations-v1"
|
||||
},
|
||||
"guardrails": [
|
||||
"Строки low confidence требуют проверки источников до персонального вывода",
|
||||
"Отсутствие данных не считается простоем",
|
||||
"Все статусы предназначены только для ручного операционного разбора"
|
||||
]
|
||||
});
|
||||
let owner_rollups = build_rollups(&roster, &actions, "manager_owner");
|
||||
let department_rollups = build_rollups(&roster, &actions, "department");
|
||||
let owner_roster = owner_rollups.clone();
|
||||
@@ -1865,8 +2149,10 @@ impl App {
|
||||
"expected_hhmm_per_user": hhmm(expected_seconds),
|
||||
"target_coverage_pct": self.config.manager_target_coverage_pct,
|
||||
"low_coverage_pct": self.config.manager_low_coverage_pct,
|
||||
"overload_coverage_pct": self.config.manager_overload_coverage_pct,
|
||||
},
|
||||
"summary": summary,
|
||||
"workforce_operations": workforce_operations,
|
||||
"actions": actions,
|
||||
"rows": roster,
|
||||
"sources": sources,
|
||||
@@ -2329,6 +2615,7 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds
|
||||
.and_then(Value::as_i64)
|
||||
.unwrap_or(0)
|
||||
});
|
||||
let operations_summary = summarize_operations(rows);
|
||||
json!({
|
||||
"users_count": users_count,
|
||||
"active_users": active_users,
|
||||
@@ -2349,6 +2636,85 @@ fn summarize_management_rows(rows: &[Value], actions: &[Value], expected_seconds
|
||||
"last_activity": rows.iter().filter_map(|r| r.get("workday_last_activity_local").and_then(Value::as_str)).filter(|s| !s.is_empty()).max().unwrap_or(""),
|
||||
"top_user": top.and_then(|r| r.get("user")).and_then(Value::as_str).unwrap_or(""),
|
||||
"top_user_active_hhmm": top.and_then(|r| r.get("workday_active_hhmm")).and_then(Value::as_str).unwrap_or("00:00"),
|
||||
"workforce_operations": operations_summary,
|
||||
})
|
||||
}
|
||||
|
||||
fn row_str<'a>(row: &'a Value, key: &str) -> &'a str {
|
||||
row.get(key).and_then(Value::as_str).unwrap_or("")
|
||||
}
|
||||
|
||||
fn count_rows_by(rows: &[Value], key: &str, expected: &[&str]) -> i64 {
|
||||
rows.iter()
|
||||
.filter(|row| expected.contains(&row_str(row, key)))
|
||||
.count() as i64
|
||||
}
|
||||
|
||||
fn summarize_operations(rows: &[Value]) -> Value {
|
||||
let users_count = rows.len() as i64;
|
||||
let low_confidence_users = count_rows_by(rows, "data_confidence", &["low"]);
|
||||
let medium_confidence_users = count_rows_by(rows, "data_confidence", &["medium"]);
|
||||
let high_confidence_users = count_rows_by(rows, "data_confidence", &["high"]);
|
||||
let unknown_or_no_data_users = count_rows_by(
|
||||
rows,
|
||||
"load_status",
|
||||
&["no_data", "insufficient_data", "no_activity"],
|
||||
);
|
||||
let underloaded_users = count_rows_by(rows, "load_status", &["underloaded", "below_target"]);
|
||||
let normal_users = count_rows_by(rows, "load_status", &["normal"]);
|
||||
let overloaded_users = count_rows_by(rows, "load_status", &["overloaded"]);
|
||||
let idle_users = count_rows_by(
|
||||
rows,
|
||||
"idle_status",
|
||||
&["idle_detected", "full_workday_idle_or_absent"],
|
||||
);
|
||||
let discipline_review_users = rows
|
||||
.iter()
|
||||
.filter(|row| !matches!(row_str(row, "discipline_status"), "" | "ok"))
|
||||
.count() as i64;
|
||||
let action_required_users = rows
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
!matches!(row_str(row, "load_status"), "normal" | "")
|
||||
|| !matches!(row_str(row, "discipline_status"), "ok" | "")
|
||||
|| matches!(
|
||||
row_str(row, "idle_status"),
|
||||
"idle_detected" | "full_workday_idle_or_absent"
|
||||
)
|
||||
|| row_str(row, "data_confidence") == "low"
|
||||
})
|
||||
.count() as i64;
|
||||
let status = if users_count == 0 || low_confidence_users == users_count {
|
||||
"LOW_CONFIDENCE"
|
||||
} else if overloaded_users > 0 || discipline_review_users > 0 || idle_users > 0 {
|
||||
"ATTENTION"
|
||||
} else if underloaded_users > 0 || unknown_or_no_data_users > 0 || medium_confidence_users > 0 {
|
||||
"WATCH"
|
||||
} else {
|
||||
"OK"
|
||||
};
|
||||
json!({
|
||||
"status": status,
|
||||
"users_count": users_count,
|
||||
"action_required_users": action_required_users,
|
||||
"load": {
|
||||
"unknown_or_no_data_users": unknown_or_no_data_users,
|
||||
"underloaded_users": underloaded_users,
|
||||
"normal_users": normal_users,
|
||||
"overloaded_users": overloaded_users
|
||||
},
|
||||
"idle": {
|
||||
"idle_users": idle_users
|
||||
},
|
||||
"discipline": {
|
||||
"review_users": discipline_review_users
|
||||
},
|
||||
"confidence": {
|
||||
"low_users": low_confidence_users,
|
||||
"medium_users": medium_confidence_users,
|
||||
"high_users": high_confidence_users
|
||||
},
|
||||
"guardrail": "Строки low confidence требуют проверки источников до персонального вывода"
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2369,6 +2735,11 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec<Value> {
|
||||
"active_users",
|
||||
"inactive_users",
|
||||
"below_target_users",
|
||||
"underloaded_users",
|
||||
"overloaded_users",
|
||||
"idle_users",
|
||||
"discipline_review_users",
|
||||
"low_confidence_users",
|
||||
"workday_total_active_seconds",
|
||||
"actions_count",
|
||||
"critical_actions_count",
|
||||
@@ -2390,6 +2761,24 @@ fn build_rollups(rows: &[Value], actions: &[Value], field: &str) -> Vec<Value> {
|
||||
if row.get("status").and_then(Value::as_str) == Some("below_target") {
|
||||
inc(group, "below_target_users", 1);
|
||||
}
|
||||
if matches!(row_str(row, "load_status"), "underloaded" | "below_target") {
|
||||
inc(group, "underloaded_users", 1);
|
||||
}
|
||||
if row_str(row, "load_status") == "overloaded" {
|
||||
inc(group, "overloaded_users", 1);
|
||||
}
|
||||
if matches!(
|
||||
row_str(row, "idle_status"),
|
||||
"idle_detected" | "full_workday_idle_or_absent"
|
||||
) {
|
||||
inc(group, "idle_users", 1);
|
||||
}
|
||||
if !matches!(row_str(row, "discipline_status"), "" | "ok") {
|
||||
inc(group, "discipline_review_users", 1);
|
||||
}
|
||||
if row_str(row, "data_confidence") == "low" {
|
||||
inc(group, "low_confidence_users", 1);
|
||||
}
|
||||
inc(
|
||||
group,
|
||||
"workday_total_active_seconds",
|
||||
@@ -3350,9 +3739,9 @@ fn render_management_html(payload: &Value) -> String {
|
||||
.collect()
|
||||
};
|
||||
let user_rows = if rows.is_empty() {
|
||||
"<tr><td colspan='8'>Нет сотрудников в выборке.</td></tr>".to_string()
|
||||
"<tr><td colspan='12'>Нет сотрудников в выборке.</td></tr>".to_string()
|
||||
} else {
|
||||
rows.iter().map(|r| format!("<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td class='good'>{}</td><td>{}%</td><td>{}</td><td>{}</td></tr>", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["workday_first_activity_local"].as_str().unwrap_or("")), esc(r["workday_last_activity_local"].as_str().unwrap_or("")))).collect()
|
||||
rows.iter().map(|r| format!("<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td class='good'>{}</td><td>{}</td><td>{}%</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>", esc(r["user"].as_str().unwrap_or("")), esc(r["manager_owner"].as_str().unwrap_or("")), esc(r["department"].as_str().unwrap_or("")), esc(r["status"].as_str().unwrap_or("")), esc(r["workday_active_hhmm"].as_str().unwrap_or("")), esc(r["workday_idle_hhmm"].as_str().unwrap_or("00:00")), r["coverage_pct"].as_f64().unwrap_or(0.0), esc(r["load_status"].as_str().unwrap_or("unknown")), esc(r["idle_status"].as_str().unwrap_or("unknown")), esc(r["discipline_status"].as_str().unwrap_or("unknown")), esc(r["data_confidence"].as_str().unwrap_or("low")), esc(r["operations_recommended_action"].as_str().unwrap_or("")))).collect()
|
||||
};
|
||||
let source_rows = sources
|
||||
.iter()
|
||||
@@ -3369,7 +3758,7 @@ fn render_management_html(payload: &Value) -> String {
|
||||
})
|
||||
.collect::<String>();
|
||||
format!(
|
||||
r#"<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AW-rus Управленческий отчёт по работе в RDP</title><style>{}</style></head><body><main><section class="hero"><h1>AW-rus Управленческий отчёт по работе в RDP</h1><p>{} · {} · {}</p><h2>Что делать сегодня</h2><p>{}</p><nav><a href="/reports/worktime/management?format=json&host={}">JSON</a><a href="/reports/worktime/management?format=csv&host={}">CSV</a><a href="/reports/worktime/today?format=html&host={}">Классический отчёт</a><a href="/reports/worktime/management?format=html&host={}">Сбросить</a></nav></section><section><h2>Очередь действий руководителя</h2><table><tbody>{}</tbody></table></section><section><h2>Сотрудники</h2><table><tbody>{}</tbody></table></section><section><h2>Тренд за период</h2><p>Тренд за {} дней</p></section><section><h2>По ответственным</h2><pre>{}</pre></section><section><h2>Ответственные и эскалация</h2><pre>{}</pre></section><section><h2>По подразделениям</h2><pre>{}</pre></section><section><h2>Свежесть источников данных</h2><table><tbody>{}</tbody></table></section><p>Фильтр: {}</p></main></body></html>"#,
|
||||
r#"<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AW-rus Управленческий отчёт по работе в RDP</title><style>{}</style></head><body><main><section class="hero"><h1>AW-rus Управленческий отчёт по работе в RDP</h1><p>{} · {} · {}</p><h2>Что делать сегодня</h2><p>{}</p><nav><a href="/reports/worktime/management?format=json&host={}">JSON</a><a href="/reports/worktime/management?format=csv&host={}">CSV</a><a href="/reports/worktime/today?format=html&host={}">Классический отчёт</a><a href="/reports/worktime/management?format=html&host={}">Сбросить</a></nav></section><section><h2>Очередь действий руководителя</h2><table><tbody>{}</tbody></table></section><section><h2>Рабочая активность сотрудников</h2><p>Статусы загрузки, простоя, дисциплины процесса и достоверности. Low confidence означает: сначала проверить источники, не делать персональный вывод.</p><table><thead><tr><th>Сотрудник</th><th>Ответственный</th><th>Подразделение</th><th>Статус</th><th>Активно</th><th>Простой</th><th>Coverage</th><th>Загрузка</th><th>Простой</th><th>Дисциплина</th><th>Confidence</th><th>Действие</th></tr></thead><tbody>{}</tbody></table></section><section><h2>Тренд за период</h2><p>Тренд за {} дней</p></section><section><h2>По ответственным</h2><pre>{}</pre></section><section><h2>Ответственные и эскалация</h2><pre>{}</pre></section><section><h2>По подразделениям</h2><pre>{}</pre></section><section><h2>Свежесть источников данных</h2><table><tbody>{}</tbody></table></section><p>Фильтр: {}</p></main></body></html>"#,
|
||||
base_css(),
|
||||
esc(payload["host"].as_str().unwrap_or("")),
|
||||
esc(payload["report_date"].as_str().unwrap_or("")),
|
||||
@@ -3639,6 +4028,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dlp_evidence_bucket_is_optional_for_true_active_hot_path() {
|
||||
let mut cfg = test_config();
|
||||
cfg.dlp_evidence_enabled = false;
|
||||
let buckets = evidence_bucket_ids(&cfg, "SHARKON2025");
|
||||
assert!(
|
||||
buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket == "aw-file-operations_SHARKON2025")
|
||||
);
|
||||
assert!(
|
||||
buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket == "aw-watcher-web-chrome_SHARKON2025")
|
||||
);
|
||||
assert!(
|
||||
!buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025")
|
||||
);
|
||||
|
||||
cfg.dlp_evidence_enabled = true;
|
||||
let buckets = evidence_bucket_ids(&cfg, "SHARKON2025");
|
||||
assert!(
|
||||
buckets
|
||||
.iter()
|
||||
.any(|bucket| bucket == "aw-dlp-endpoint-signals_SHARKON2025")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn management_insights_detect_falling_portfolio_trend() {
|
||||
let cfg = test_config();
|
||||
@@ -3758,6 +4177,85 @@ mod tests {
|
||||
assert_eq!(rollups[0]["actions_count"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn employee_operations_detects_overload_and_off_hours() {
|
||||
let cfg = test_config();
|
||||
let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap();
|
||||
let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date);
|
||||
let input = EmployeeOperationsInput {
|
||||
expected_seconds,
|
||||
low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100,
|
||||
target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100,
|
||||
overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100,
|
||||
work_secs: expected_seconds * 2,
|
||||
calendar_secs: expected_seconds * 2 + 3600,
|
||||
work_first: Some(work_start.with_timezone(&Utc)),
|
||||
work_last: Some(work_end.with_timezone(&Utc)),
|
||||
is_today: false,
|
||||
late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes),
|
||||
early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes),
|
||||
samples_count: 20,
|
||||
active_samples: 18,
|
||||
sessions_count: 1,
|
||||
};
|
||||
let status = build_employee_operations_status(&cfg, input);
|
||||
assert_eq!(status["load_status"], "overloaded");
|
||||
assert_eq!(status["discipline_status"], "off_hours");
|
||||
assert_eq!(status["data_confidence"], "high");
|
||||
assert_eq!(status["guardrail"], "evidence_backed_manual_review_only");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn employee_operations_low_confidence_blocks_personnel_conclusion() {
|
||||
let cfg = test_config();
|
||||
let report_date = NaiveDate::from_ymd_opt(2026, 5, 14).unwrap();
|
||||
let (work_start, work_end, expected_seconds) = workday_bounds(&cfg, report_date);
|
||||
let input = EmployeeOperationsInput {
|
||||
expected_seconds,
|
||||
low_seconds: expected_seconds * cfg.manager_low_coverage_pct / 100,
|
||||
target_seconds: expected_seconds * cfg.manager_target_coverage_pct / 100,
|
||||
overload_seconds: expected_seconds * cfg.manager_overload_coverage_pct / 100,
|
||||
work_secs: 0,
|
||||
calendar_secs: 0,
|
||||
work_first: None,
|
||||
work_last: None,
|
||||
is_today: false,
|
||||
late_start: work_start + TimeDelta::minutes(cfg.manager_late_start_grace_minutes),
|
||||
early_finish: work_end - TimeDelta::minutes(cfg.manager_early_finish_grace_minutes),
|
||||
samples_count: 0,
|
||||
active_samples: 0,
|
||||
sessions_count: 0,
|
||||
};
|
||||
let status = build_employee_operations_status(&cfg, input);
|
||||
assert_eq!(status["load_status"], "no_data");
|
||||
assert_eq!(status["idle_status"], "unknown");
|
||||
assert_eq!(status["data_confidence"], "low");
|
||||
assert_eq!(status["guardrail"], "low_confidence_not_for_discipline");
|
||||
assert!(
|
||||
status["recommended_action"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("не использовать как дисциплинарный")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operations_summary_counts_attention_groups() {
|
||||
let rows = vec![
|
||||
json!({"load_status":"overloaded","idle_status":"no_significant_idle","discipline_status":"ok","data_confidence":"high"}),
|
||||
json!({"load_status":"underloaded","idle_status":"idle_detected","discipline_status":"late_start","data_confidence":"medium"}),
|
||||
json!({"load_status":"no_data","idle_status":"unknown","discipline_status":"ok","data_confidence":"low"}),
|
||||
];
|
||||
let summary = summarize_operations(&rows);
|
||||
assert_eq!(summary["status"], "ATTENTION");
|
||||
assert_eq!(summary["load"]["overloaded_users"], 1);
|
||||
assert_eq!(summary["load"]["underloaded_users"], 1);
|
||||
assert_eq!(summary["idle"]["idle_users"], 1);
|
||||
assert_eq!(summary["discipline"]["review_users"], 1);
|
||||
assert_eq!(summary["confidence"]["low_users"], 1);
|
||||
assert_eq!(summary["action_required_users"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_legacy_rdp_sources_are_covered_by_fresh_rust_sources() {
|
||||
assert!(legacy_rdp_covered_by_rust_sources(
|
||||
@@ -3888,14 +4386,17 @@ mod tests {
|
||||
#[test]
|
||||
fn interpretation_policy_accepts_fraction_thresholds() {
|
||||
let policy: InterpretationPolicy = serde_json::from_value(json!({
|
||||
"overload_threshold": 0.92,
|
||||
"overload_threshold": 1.15,
|
||||
"underload_threshold": 0.45,
|
||||
"drop_threshold_pct": 20,
|
||||
"night_work_after": "20:00",
|
||||
"weekend_work": true
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(threshold_to_pct(policy.overload_threshold.unwrap()), 92.0);
|
||||
assert_eq!(
|
||||
overload_threshold_to_pct(policy.overload_threshold.unwrap()).round(),
|
||||
115.0
|
||||
);
|
||||
assert_eq!(threshold_to_pct(policy.underload_threshold.unwrap()), 45.0);
|
||||
assert_eq!(threshold_to_pct(policy.drop_threshold_pct.unwrap()), 20.0);
|
||||
assert_eq!(
|
||||
|
||||
@@ -275,10 +275,20 @@ impl AwClient {
|
||||
) -> Result<()> {
|
||||
for chunk in events.chunks(chunk_size.max(1)) {
|
||||
let path = format!("/api/0/buckets/{bucket_id}/events");
|
||||
self.request_json(Method::POST, &path, Some(json!(chunk)), false)?;
|
||||
self.request_status(Method::POST, &path, Some(json!(chunk)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn request_status(&self, method: Method, path: &str, payload: Option<Value>) -> Result<()> {
|
||||
let response = self.send_retry(method, path, payload)?;
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow!("ActivityWatch {path} returned HTTP {status}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn log(message: &str) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"overload_threshold": 0.92,
|
||||
"overload_threshold": 1.15,
|
||||
"underload_threshold": 0.45,
|
||||
"drop_threshold_pct": 20,
|
||||
"night_work_after": "20:00",
|
||||
|
||||
@@ -55,6 +55,21 @@ Live endpoints, hostnames, tokens and passwords must be supplied through
|
||||
`/etc/detmir/detmir-check.env` (systemd units) or another private environment
|
||||
file outside the public repository.
|
||||
|
||||
Production note checked on 2026-06-24:
|
||||
|
||||
- `DETMIR_PORTAL_URL` must point to the local DetMir portal listener,
|
||||
currently `http://127.0.0.1:8720`, for server-side health checks. If it is
|
||||
omitted, `detmir-check` falls back to the public HTTPS gateway and protected
|
||||
`/readyz`, `/version` and `/metrics` can correctly return `401`, producing a
|
||||
false operational failure.
|
||||
- `DETMIR_GATEWAY_HOST=127.0.0.1` is used with the local listener so the Host
|
||||
header does not accidentally select the public protected gateway path.
|
||||
- Cold `/api/reports` builds can take more than 60 seconds on the live contour
|
||||
when cache is empty or concurrent checks are active. The production
|
||||
`detmir-portal-prewarm.service` therefore uses `curl --max-time 180` and
|
||||
`TimeoutStartSec=210`. Shorter 45-60 second limits caused false failed
|
||||
systemd states while the portal eventually returned HTTP 200.
|
||||
|
||||
## Текущий планировщик Proxmox, проверено 2026-06-21
|
||||
|
||||
На Proxmox уже присутствуют следующие регулярные проверки:
|
||||
@@ -75,10 +90,10 @@ file outside the public repository.
|
||||
Наблюдение: отдельный ежедневный полный gate по всей матрице AWatch-rus
|
||||
отсутствует. Его роль должен закрыть `awatch-contour-daily-check.timer`.
|
||||
|
||||
Наблюдение: последняя проверка `detmir-portal-prewarm.service` на момент осмотра
|
||||
имела `Result=exit-code` и `ExecMainStatus=28`. Это не надо маскировать:
|
||||
канонический check должен показывать такой сбой как fail/warn в зависимости от
|
||||
политики эксплуатации.
|
||||
Историческое наблюдение 2026-06-24: `detmir-portal-prewarm.service` был найден
|
||||
в failed state из-за устаревшего `curl --max-time 60` для холодной сборки
|
||||
`/api/reports`. В текущей ветке это оформлено как отдельный prewarm/resilience
|
||||
пакет, а не как обязательная часть DLP production hot path.
|
||||
|
||||
## Матрица требований и проверок
|
||||
|
||||
@@ -91,14 +106,36 @@ file outside the public repository.
|
||||
| Portal hardening | `/healthz`, `/readyz`, `/version`, `/metrics` | `detmir-check` | да | да |
|
||||
| Windows/RDP | TCP 5985 и 22 | `detmir-check` | да | да |
|
||||
| ActivityWatch buckets | AFK/window/worktime/session events | `detmir-check` | да | да |
|
||||
| AWatch DLP buckets | endpoint signals/incidents/review/rules | `detmir-check` | да | да |
|
||||
| AWatch DLP health | remote `dlp-health-check --json` через `detmir-dlp` | `detmir-check` | да | да |
|
||||
| AWatch DLP buckets | endpoint signals/incidents/review/rules, только если DLP включен | `detmir-check` | условно | условно |
|
||||
| AWatch DLP health | disabled/core_only должен быть SKIPPED/WARN, `light/full` проверяются через `detmir-dlp` | `detmir-check` | да | да |
|
||||
| Grafana evidence | свежий JSON артефакт Grafana check | `detmir-check` | да | да |
|
||||
| Security events backend | ClickHouse events, если включено | `detmir-check` | да | да |
|
||||
| Portal contract | role/API smoke | `scripts/awatch-production-hardening-smoke.mjs` | нет | да |
|
||||
| Pilot contract | demo/API smoke | `scripts/detmir-pilot-demo-smoke.mjs` | нет | да |
|
||||
| Registry/readiness docs | registry readiness check | `scripts/registry_readiness_check.sh` | опционально | да |
|
||||
|
||||
## Timeout/fail-closed параметры live contour
|
||||
|
||||
После ручного live-прогона 2026-06-24 production
|
||||
`/etc/detmir/detmir-check.env` должен содержать bounded timeouts, соответствующие
|
||||
фактической latency AW datastore:
|
||||
|
||||
```env
|
||||
DETMIR_SERVICE_TIMEOUT_SECONDS=35
|
||||
DETMIR_BUCKET_TIMEOUT_SECONDS=35
|
||||
DETMIR_DLP_TIMEOUT_SECONDS=120
|
||||
DETMIR_CHECK_OVERALL_TIMEOUT_SECONDS=300
|
||||
```
|
||||
|
||||
Назначение:
|
||||
|
||||
- не считать bucket `DEAD` только из-за штатной 15-30 секундной latency
|
||||
большого SQLite datastore;
|
||||
- не оставлять `detmir-check`, `detmir-dlp`, `ssh` и remote
|
||||
`dlp-health-check` хвосты при timeout;
|
||||
- сохранять красный non-zero результат при реальной недоступности, но
|
||||
завершать проверку bounded.
|
||||
|
||||
## Fail-closed политика
|
||||
|
||||
Ежедневный check должен завершаться non-zero, если падает обязательная область:
|
||||
@@ -108,7 +145,9 @@ file outside the public repository.
|
||||
- Gateway/Portal health;
|
||||
- RDP/Windows reachability;
|
||||
- свежесть обязательных bucket streams;
|
||||
- AWatch DLP health;
|
||||
- AWatch DLP health только если DLP runtime включен; при штатном
|
||||
`AW_DLP_ENABLED=false`/`core_only` disabled-state не является отказом
|
||||
Workforce/Worktime core;
|
||||
- Grafana evidence freshness.
|
||||
|
||||
Event-driven buckets не должны считаться stale только из-за отсутствия новых
|
||||
|
||||
@@ -7,6 +7,20 @@ ClickHouse не является обязательной зависимость
|
||||
перезапускайте ClickHouse для восстановления отчетов рабочего времени, если нет
|
||||
отдельного подтвержденного отказа ClickHouse.
|
||||
|
||||
## Stable host id
|
||||
|
||||
Worktime reports используют stable logical host id, а не обязательно текущее
|
||||
Windows `COMPUTERNAME`. Для DetMir production текущий logical id:
|
||||
|
||||
```text
|
||||
SHARKON2025
|
||||
```
|
||||
|
||||
При переименовании RDP-сервера не меняйте `awHostname` автоматически. Сначала
|
||||
обновите Windows account domain для задач, затем проверьте, что collectors
|
||||
продолжают писать в bucket-и `*_SHARKON2025`. Подробный порядок:
|
||||
`docs/WINDOWS_LOGICAL_HOST_ID_RU.md`.
|
||||
|
||||
## Симптомы перегруза
|
||||
|
||||
- `/portal/api/reports?role=executive` открывается медленно или отвечает
|
||||
@@ -226,6 +240,153 @@ curl -sS --max-time 8 http://<PORTAL_HOST>/portal/api/health | jq
|
||||
curl -sS --max-time 12 "http://<PORTAL_HOST>/portal/api/reports?role=executive" | jq '.status'
|
||||
```
|
||||
|
||||
## Production repair: AW SQLite hot path, 2026-06-30
|
||||
|
||||
Симптомы:
|
||||
|
||||
- `activitywatch-server` отвечает `503` на bucket API;
|
||||
- журнал содержит `poisoned lock` / `database is locked`;
|
||||
- `aw-worktime-api` уходит в bounded `DEGRADED`;
|
||||
- `/buckets/aw-worktime-sessions_<HOST>/events?limit=...` тайм-аутится даже
|
||||
при малом лимите;
|
||||
- RDP browser/category collector пишет `bucket create failed` или timeout.
|
||||
|
||||
Порядок безопасного восстановления:
|
||||
|
||||
1. Остановить RDP guard и процессы `aw-windows-telemetry`, чтобы не продолжать
|
||||
штурмовать AW API.
|
||||
2. Остановить `aw-worktime-*` timers/services и другие локальные потребители AW
|
||||
API.
|
||||
3. Перезапустить `activitywatch-server` отдельно и проверить `/api/0/info`.
|
||||
4. Если bucket metadata отвечает, но `/events` медленный, проверить SQLite plan:
|
||||
|
||||
```sql
|
||||
EXPLAIN QUERY PLAN
|
||||
SELECT id,starttime,endtime,data
|
||||
FROM events
|
||||
WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_<HOST>')
|
||||
ORDER BY starttime DESC
|
||||
LIMIT 100;
|
||||
```
|
||||
|
||||
Если план строит `TEMP B-TREE FOR ORDER BY`, нужен составной индекс:
|
||||
|
||||
```sql
|
||||
CREATE INDEX IF NOT EXISTS events_bucketrow_starttime_desc_index
|
||||
ON events(bucketrow, starttime DESC);
|
||||
ANALYZE;
|
||||
PRAGMA optimize;
|
||||
PRAGMA integrity_check;
|
||||
```
|
||||
|
||||
Индекс добавлять только в controlled window:
|
||||
|
||||
- остановить `activitywatch-server`;
|
||||
- сделать rollback backup SQLite DB;
|
||||
- создать индекс;
|
||||
- проверить `PRAGMA integrity_check = ok`;
|
||||
- запустить `activitywatch-server`;
|
||||
- проверить, что `/events?limit=100` больше не тайм-аутится.
|
||||
|
||||
Production DetMir repair 2026-06-30:
|
||||
|
||||
- оставлены две свежие ежедневные SQLite VACUUM backup-копии, старые backup-и
|
||||
ротированы для освобождения места;
|
||||
- создан rollback backup:
|
||||
`/var/lib/activitywatch/backups/db/aw-sqlite-before-hotpath-index-20260630T035032Z.db`;
|
||||
- добавлен индекс `events_bucketrow_starttime_desc_index`;
|
||||
- `ROCKET_WORKERS=8` добавлен в `/etc/activitywatch/aw-server.env`;
|
||||
- для `aw-worktime-api.service` добавлен stabilization drop-in:
|
||||
`AW_WORKTIME_EVENTS_LIMIT=100`,
|
||||
`AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS=25`,
|
||||
`AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS=600`,
|
||||
`AW_WORKTIME_REPORT_STALE_TTL_SECONDS=7200`.
|
||||
|
||||
После ремонта проверить:
|
||||
|
||||
```bash
|
||||
curl -sS --max-time 10 http://127.0.0.1:5600/api/0/info
|
||||
curl -sS --max-time 15 \
|
||||
'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100'
|
||||
curl -sS --max-time 15 \
|
||||
'http://127.0.0.1:5610/reports/worktime/today?format=json' | jq '{rows:(.rows|length),degraded,runtime}'
|
||||
```
|
||||
|
||||
Также проверить отсутствие новых `poisoned lock` после финального старта:
|
||||
|
||||
```bash
|
||||
journalctl -u activitywatch-server --since '<FINAL_START_TIME>' --no-pager |
|
||||
grep -E 'poisoned lock|Taking datastore lock failed|database is locked'
|
||||
```
|
||||
|
||||
## Crash/readiness test after AW repair
|
||||
|
||||
Цель: проверить, что контур выдерживает restart и короткую параллельную
|
||||
нагрузку, а проверки не путают `systemctl active` с готовым API.
|
||||
|
||||
Порядок:
|
||||
|
||||
1. Зафиксировать baseline:
|
||||
|
||||
```bash
|
||||
./check-aw-full.sh
|
||||
```
|
||||
|
||||
2. На AW server проверить readiness, hot-path и Worktime API:
|
||||
|
||||
```bash
|
||||
AW_API=http://127.0.0.1:5600 \
|
||||
AW_WORKTIME_API=http://127.0.0.1:5610 \
|
||||
AW_LOGICAL_HOST_ID=SHARKON2025 \
|
||||
scripts/detmir_resilience_check.sh --live
|
||||
```
|
||||
|
||||
Если скрипт запускается с ноутбука, live-mode нужно выполнять на самом
|
||||
AW-сервере через SSH/Ansible, потому что он проверяет local systemd и SQLite.
|
||||
|
||||
3. Controlled restart:
|
||||
|
||||
```bash
|
||||
systemctl restart aw-worktime-api
|
||||
curl -sS --max-time 12 \
|
||||
'http://127.0.0.1:5610/reports/worktime/today?format=json&host=SHARKON2025&allow_stale=1' |
|
||||
jq '{rows:(.rows|length),degraded}'
|
||||
|
||||
systemctl restart activitywatch-server
|
||||
# Не считать "active" готовностью: дождаться HTTP readiness.
|
||||
timeout 90 bash -c 'until curl -fsS --max-time 8 http://127.0.0.1:5600/api/0/info >/dev/null; do sleep 2; done'
|
||||
curl -sS --max-time 15 \
|
||||
'http://127.0.0.1:5600/api/0/buckets/aw-worktime-sessions_SHARKON2025/events?limit=100' >/dev/null
|
||||
```
|
||||
|
||||
4. Проверить, что после рестарта нет новых lock/503:
|
||||
|
||||
```bash
|
||||
journalctl -u activitywatch-server --since '<RESTART_TIME>' --no-pager |
|
||||
grep -E 'poisoned lock|Taking datastore lock failed|database is locked|503'
|
||||
```
|
||||
|
||||
5. Проверить RDP guard restart отдельно:
|
||||
|
||||
```powershell
|
||||
Restart-Service AWatchRusCollectorGuard -Force
|
||||
Start-Sleep -Seconds 75
|
||||
Get-Service AWatchRusCollectorGuard
|
||||
Get-Process aw-windows-telemetry -ErrorAction SilentlyContinue | Measure-Object
|
||||
```
|
||||
|
||||
Ожидаемый результат для DetMir после ремонта 2026-06-30:
|
||||
|
||||
- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`;
|
||||
- `/events?limit=100` отвечает за bounded time и использует
|
||||
`events_bucketrow_starttime_desc_index`;
|
||||
- `aw-rus-healthd.service` завершается `status=0/SUCCESS`;
|
||||
- server-side TCP до RDP может быть `warn`, если
|
||||
`AW_RUS_HEALTH_RDP_TCP_REQUIRED=false`, но bucket freshness и WinRM/SSH
|
||||
через admin path должны оставаться зелёными;
|
||||
- optional DLP/Loki heavy runtime units должны быть inactive в экономном
|
||||
production profile.
|
||||
|
||||
## Rollback
|
||||
|
||||
Rollback нужен, если после обновления бинарника или env-настроек:
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
# Workforce Operations Model
|
||||
|
||||
Статус: implemented in Worktime API and DetMir portal.
|
||||
|
||||
Модель отвечает на главный управленческий вопрос AWatch-rus Workforce:
|
||||
рабочая активность сотрудников, загрузка, простои, перегруз и дисциплина
|
||||
рабочего процесса. Это rule-based слой операционного контроля. Он не является
|
||||
HR-оценкой, не использует ML/LLM и не выполняет автоматических санкций.
|
||||
|
||||
## Где смотреть
|
||||
|
||||
Основные точки:
|
||||
|
||||
- Worktime API:
|
||||
`GET /reports/worktime/management?format=json`;
|
||||
- Worktime HTML:
|
||||
`GET /reports/worktime/management?format=html`;
|
||||
- DetMir portal:
|
||||
`/api/reports`, блок `workforce_operations`;
|
||||
- UI портала:
|
||||
роли `Руководитель` и вкладка `Отчеты`, блок `Операционная загрузка`.
|
||||
|
||||
## Источники
|
||||
|
||||
Модель использует только подтвержденные рабочие источники:
|
||||
|
||||
- ActivityWatch worktime rows;
|
||||
- bucket рабочих сессий RDP;
|
||||
- интервалы активности в рабочем окне;
|
||||
- configured owner/department aliases;
|
||||
- freshness/coverage metadata, которые уже возвращает Worktime API.
|
||||
|
||||
Отсутствие данных не считается простоем. При пропусках источников строка
|
||||
получает `data_confidence=low` и guardrail
|
||||
`low_confidence_not_for_discipline`.
|
||||
|
||||
## Runtime-настройки
|
||||
|
||||
Основной файл политики:
|
||||
|
||||
- пример: `configs/worktime-interpretation-policy.example.json`;
|
||||
- runtime: `/etc/activitywatch/worktime-interpretation-policy.json`;
|
||||
- env path: `AW_WORKTIME_MANAGER_INTERPRETATION_POLICY`.
|
||||
|
||||
Поля policy:
|
||||
|
||||
| Поле | Смысл | Рекомендуемое значение |
|
||||
| --- | --- | --- |
|
||||
| `underload_threshold` | порог недогруза от рабочего окна | `0.35..0.45` |
|
||||
| `overload_threshold` | порог перегруза от рабочего окна | `1.10..1.25` |
|
||||
| `drop_threshold_pct` | порог просадки тренда | `10..25` |
|
||||
| `night_work_after` | начало вечернего/ночного отклонения | `20:00` |
|
||||
| `weekend_work` | учитывать выходные отклонения | `true` |
|
||||
| `min_trend_points` | минимум daily points для тренда | `3..7` |
|
||||
| `off_hours_threshold_seconds` | минимум внерабочей активности для флага | `1800` |
|
||||
|
||||
`underload_threshold` и `overload_threshold` можно задавать дробью или
|
||||
процентом: `0.45` равно `45`, `1.15` равно `115`.
|
||||
Для перегруза effective threshold fail-closed зажат в диапазон `100..300`, чтобы
|
||||
значение ниже 100% не создавало ложный статус перегруза.
|
||||
|
||||
Env fallback:
|
||||
|
||||
- `AW_WORKTIME_MANAGER_TARGET_COVERAGE_PCT`;
|
||||
- `AW_WORKTIME_MANAGER_LOW_COVERAGE_PCT`;
|
||||
- `AW_WORKTIME_MANAGER_OVERLOAD_COVERAGE_PCT`;
|
||||
- `AW_WORKTIME_MANAGER_TREND_MIN_POINTS`;
|
||||
- `AW_WORKTIME_MANAGER_TREND_DELTA_PCT`;
|
||||
- `AW_WORKTIME_MANAGER_OFF_HOURS_THRESHOLD_SECONDS`;
|
||||
- `AW_WORKTIME_MANAGER_NIGHT_WORK_AFTER`;
|
||||
- `AW_WORKTIME_MANAGER_WEEKEND_WORK_ENABLED`.
|
||||
|
||||
Веса приложений остаются отдельной политикой:
|
||||
|
||||
- пример: `configs/detmir-workforce-policy.example.json`;
|
||||
- runtime: `/etc/detmir-portal-workforce-policy.json`.
|
||||
|
||||
Она влияет на explainable KPI и weighted activity, но не подменяет
|
||||
операционные статусы загрузки/простоя.
|
||||
|
||||
## API contract
|
||||
|
||||
`/reports/worktime/management?format=json` содержит:
|
||||
|
||||
```json
|
||||
{
|
||||
"workday": {
|
||||
"target_coverage_pct": 75,
|
||||
"low_coverage_pct": 35,
|
||||
"overload_coverage_pct": 115
|
||||
},
|
||||
"workforce_operations": {
|
||||
"status": "ATTENTION",
|
||||
"summary": {},
|
||||
"rows": [],
|
||||
"model": {
|
||||
"type": "rule_based",
|
||||
"ml": false,
|
||||
"llm": false,
|
||||
"version": "workforce-operations-v1"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Каждая строка сотрудника содержит:
|
||||
|
||||
- `workday_active_seconds`, `workday_active_hhmm`;
|
||||
- `workday_idle_seconds`, `workday_idle_hhmm`;
|
||||
- `coverage_pct`;
|
||||
- `load_status`;
|
||||
- `idle_status`;
|
||||
- `discipline_status`;
|
||||
- `data_confidence`;
|
||||
- `recommended_action`.
|
||||
|
||||
Полный roster в `rows[]` дополнительно содержит `operations`,
|
||||
`operations.evidence`, `operations.guardrail` и
|
||||
`operations_recommended_action`.
|
||||
|
||||
## Статусы загрузки
|
||||
|
||||
| Status | Значение | Действие |
|
||||
| --- | --- | --- |
|
||||
| `insufficient_data` | рабочее окно еще не началось или равно нулю | не делать вывод |
|
||||
| `no_data` | нет сессий или worktime samples | проверить источники |
|
||||
| `no_activity` | сессия/данные есть, активности в окне нет | проверить присутствие и задачи |
|
||||
| `underloaded` | ниже low threshold | проверить загрузку и доступ к процессам |
|
||||
| `below_target` | ниже target threshold | уточнить причину отклонения |
|
||||
| `normal` | в рабочем диапазоне | наблюдать |
|
||||
| `overloaded` | выше overload threshold | проверить переработку и риск аврала |
|
||||
|
||||
## Статусы простоя
|
||||
|
||||
| Status | Значение |
|
||||
| --- | --- |
|
||||
| `not_applicable` | нет рабочего окна |
|
||||
| `unknown` | нет достаточных источников |
|
||||
| `full_workday_idle_or_absent` | активность в рабочем окне отсутствует |
|
||||
| `idle_detected` | простой выше порога |
|
||||
| `no_significant_idle` | существенный простой не найден |
|
||||
|
||||
## Дисциплина процесса
|
||||
|
||||
`discipline_status` показывает отклонение от рабочего процесса, а не
|
||||
автоматическое нарушение:
|
||||
|
||||
- `ok`;
|
||||
- `off_hours`;
|
||||
- `late_start`;
|
||||
- `early_finish`;
|
||||
- `multiple_flags`.
|
||||
|
||||
Для текущего дня `early_finish` не выставляется до завершения рабочего окна.
|
||||
|
||||
## Достоверность
|
||||
|
||||
`data_confidence`:
|
||||
|
||||
- `high`: есть session samples, worktime samples и active samples;
|
||||
- `medium`: данных мало или нет active samples;
|
||||
- `low`: нет сессий/worktime samples или рабочее окно невалидно.
|
||||
|
||||
Правило: low confidence строки сначала проверяются как проблема источников.
|
||||
Их нельзя использовать как персональный дисциплинарный вывод.
|
||||
|
||||
## Summary
|
||||
|
||||
`workforce_operations.summary` содержит:
|
||||
|
||||
- `users_count`;
|
||||
- `action_required_users`;
|
||||
- `load.unknown_or_no_data_users`;
|
||||
- `load.underloaded_users`;
|
||||
- `load.normal_users`;
|
||||
- `load.overloaded_users`;
|
||||
- `idle.idle_users`;
|
||||
- `discipline.review_users`;
|
||||
- `confidence.low_users`;
|
||||
- `confidence.medium_users`;
|
||||
- `confidence.high_users`;
|
||||
- `guardrail`.
|
||||
|
||||
Summary status:
|
||||
|
||||
- `LOW_CONFIDENCE`: нет строк или все строки low confidence;
|
||||
- `ATTENTION`: есть перегруз, простой или дисциплинарные флаги;
|
||||
- `WATCH`: есть недогруз, нет данных или low confidence;
|
||||
- `OK`: отклонений нет.
|
||||
|
||||
## UI contract
|
||||
|
||||
Портал показывает отдельный блок `Операционная загрузка`:
|
||||
|
||||
- сводка: требуют разбора, недогруз, перегруз, простой, дисциплина, low
|
||||
confidence;
|
||||
- таблица сотрудников: active/idle/coverage/load/idle/discipline/confidence;
|
||||
- рекомендуемое действие;
|
||||
- guardrail и версию rule-based модели.
|
||||
|
||||
Это отдельный блок от `Почему такой индекс активности?`: explainable KPI
|
||||
отвечает на вопрос "почему такой процент", а Workforce Operations отвечает
|
||||
"кого и почему нужно разобрать".
|
||||
|
||||
## Ограничения
|
||||
|
||||
- Не утверждать автоматическую оценку эффективности сотрудника.
|
||||
- Не считать missing data простоем.
|
||||
- Не смешивать Security/Forensics claims с Workforce Operations.
|
||||
- Не заявлять ML/LLM detection.
|
||||
- Не выполнять автоматическое remediation/action.
|
||||
- Не использовать GitHub Actions или демо-данные как registry release evidence.
|
||||
|
||||
## Проверка после изменения
|
||||
|
||||
Минимальный локальный контур:
|
||||
|
||||
```bash
|
||||
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust
|
||||
export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
|
||||
cargo fmt --all --check
|
||||
cargo test -p worktime-api -p detmir-portal --locked
|
||||
cargo clippy -p worktime-api -p detmir-portal --all-targets --locked -- -D warnings
|
||||
```
|
||||
|
||||
Минимальный live smoke:
|
||||
|
||||
```bash
|
||||
curl -fsS 'http://10.10.10.13:5610/reports/worktime/management?format=json' \
|
||||
| jq '.workforce_operations.summary'
|
||||
|
||||
curl -fsS 'http://10.10.10.2:8720/api/reports?role=manager' \
|
||||
| jq '{status: .workforce_operations.summary.status, rows: (.workforce_operations.rows | length)}'
|
||||
```
|
||||
|
||||
Браузерный smoke: открыть `http://10.10.10.2:8720/`, выбрать представление
|
||||
менеджера и проверить блок `Операционная загрузка`. В рабочем состоянии должны
|
||||
быть видны summary-карточки, таблица сотрудников, `workforce-operations-v1` и
|
||||
guardrail про `low confidence`.
|
||||
@@ -12,6 +12,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp
|
||||
Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs
|
||||
Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env
|
||||
Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720
|
||||
Environment=DETMIR_DLP_ENABLED=false
|
||||
EnvironmentFile=-/etc/detmir/detmir-check.env
|
||||
EnvironmentFile=-/etc/awatch-rus/contour-check.env
|
||||
ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check
|
||||
|
||||
@@ -13,6 +13,7 @@ Environment=DETMIR_DLP_COMMAND=detmir-dlp
|
||||
Environment=CONTOUR_CHECK_OUTPUT_ROOT=/var/lib/detmir-ai/contour-check-runs
|
||||
Environment=CONTOUR_CHECK_ENV_FILE=/etc/detmir/detmir-check.env
|
||||
Environment=DETMIR_PORTAL_URL=http://127.0.0.1:8720
|
||||
Environment=DETMIR_DLP_ENABLED=false
|
||||
EnvironmentFile=-/etc/detmir/detmir-check.env
|
||||
EnvironmentFile=-/etc/awatch-rus/contour-check.env
|
||||
ExecStart=/usr/bin/env bash /usr/local/sbin/awatch-contour-check
|
||||
|
||||
@@ -0,0 +1,471 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
MODE="repo"
|
||||
AW_API="${AW_API:-http://127.0.0.1:5600}"
|
||||
AW_WORKTIME_API="${AW_WORKTIME_API:-http://127.0.0.1:5610}"
|
||||
AW_LOGICAL_HOST_ID="${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-SHARKON2025}}"
|
||||
AW_READINESS_TIMEOUT_SECONDS="${AW_READINESS_TIMEOUT_SECONDS:-60}"
|
||||
AW_READINESS_INTERVAL_SECONDS="${AW_READINESS_INTERVAL_SECONDS:-2}"
|
||||
AW_EVENTS_LIMIT="${AW_EVENTS_LIMIT:-100}"
|
||||
AW_EVENTS_MAX_SECONDS="${AW_EVENTS_MAX_SECONDS:-15}"
|
||||
HAYA_ROOT="${AW_HAYABUSA_ROOT:-/opt/hayabusa}"
|
||||
HAYA_DROP_DIR="${AW_HAYABUSA_DROP_DIR:-/opt/activitywatch/aw-rus-ops/drop}"
|
||||
SQLITE_DB="${AW_SQLITE_DB:-/var/lib/activitywatch/aw-server-rust/sqlite.db}"
|
||||
MAX_INCOMING_AGE_SECONDS="${MAX_HAYABUSA_INCOMING_AGE_SECONDS:-900}"
|
||||
STRICT_SECRETS="${DETMIR_RESILIENCE_STRICT_SECRETS:-0}"
|
||||
EXPECT_DLP_PROFILE="${DETMIR_RESILIENCE_EXPECT_DLP_PROFILE:-light}"
|
||||
EXPECT_OPTIONAL_DLP_OFF="${DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF:-0}"
|
||||
EXPECT_LOKI_OFF="${DETMIR_RESILIENCE_EXPECT_LOKI_OFF:-1}"
|
||||
|
||||
DLP_RUNTIME_UNITS=(
|
||||
aw-dlp-influx-exporter.timer
|
||||
aw-dlp-influx-exporter.service
|
||||
activitywatch-dlp-aggregator.timer
|
||||
activitywatch-dlp-aggregator.service
|
||||
aw-dlp-report-scheduler.timer
|
||||
aw-dlp-report-scheduler.service
|
||||
aw-dlp-syslog-forwarder.timer
|
||||
aw-dlp-syslog-forwarder.service
|
||||
aw-dlp-webhook-sender.timer
|
||||
aw-dlp-webhook-sender.service
|
||||
aw-dlp-cef-exporter.timer
|
||||
aw-dlp-cef-exporter.service
|
||||
aw-dlp-ioc-refresh.timer
|
||||
aw-dlp-ioc-refresh.service
|
||||
aw-dlp-policy-engine.service
|
||||
aw-dlp-case-management.service
|
||||
detmir-portal-evidence.service
|
||||
)
|
||||
|
||||
DLP_LIGHT_ALLOWED_UNITS=(
|
||||
activitywatch-dlp-aggregator.timer
|
||||
activitywatch-dlp-aggregator.service
|
||||
aw-dlp-ioc-refresh.timer
|
||||
aw-dlp-ioc-refresh.service
|
||||
detmir-dlp-load-guard.timer
|
||||
detmir-dlp-load-guard.service
|
||||
)
|
||||
|
||||
DLP_HEAVY_RUNTIME_UNITS=(
|
||||
aw-dlp-influx-exporter.timer
|
||||
aw-dlp-influx-exporter.service
|
||||
aw-dlp-report-scheduler.timer
|
||||
aw-dlp-report-scheduler.service
|
||||
aw-dlp-syslog-forwarder.timer
|
||||
aw-dlp-syslog-forwarder.service
|
||||
aw-dlp-webhook-sender.timer
|
||||
aw-dlp-webhook-sender.service
|
||||
aw-dlp-cef-exporter.timer
|
||||
aw-dlp-cef-exporter.service
|
||||
aw-dlp-policy-engine.service
|
||||
aw-dlp-case-management.service
|
||||
detmir-portal-evidence.service
|
||||
)
|
||||
|
||||
LOKI_RUNTIME_UNITS=(
|
||||
loki.service
|
||||
promtail.service
|
||||
)
|
||||
|
||||
OK_COUNT=0
|
||||
WARN_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/detmir_resilience_check.sh [--repo|--live|--all]
|
||||
|
||||
Modes:
|
||||
--repo check repository hardening and docs only (default, CI-safe)
|
||||
--live read-only checks for the local AW server/Hayabusa host
|
||||
--all repo + live
|
||||
|
||||
Environment:
|
||||
AW_API=http://127.0.0.1:5600
|
||||
AW_WORKTIME_API=http://127.0.0.1:5610
|
||||
AW_LOGICAL_HOST_ID=SHARKON2025
|
||||
AW_READINESS_TIMEOUT_SECONDS=60
|
||||
AW_EVENTS_MAX_SECONDS=15
|
||||
AW_HAYABUSA_ROOT=/opt/hayabusa
|
||||
AW_HAYABUSA_DROP_DIR=/opt/activitywatch/aw-rus-ops/drop
|
||||
AW_SQLITE_DB=/var/lib/activitywatch/aw-server-rust/sqlite.db
|
||||
DETMIR_RESILIENCE_EXPECT_DLP_PROFILE=light
|
||||
DETMIR_RESILIENCE_EXPECT_OPTIONAL_DLP_OFF=0
|
||||
DETMIR_RESILIENCE_EXPECT_LOKI_OFF=1
|
||||
DETMIR_RESILIENCE_STRICT_SECRETS=1
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--repo) MODE="repo"; shift ;;
|
||||
--live) MODE="live"; shift ;;
|
||||
--all) MODE="all"; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
ok() {
|
||||
OK_COUNT=$((OK_COUNT + 1))
|
||||
printf '[OK] %s\n' "$*"
|
||||
}
|
||||
|
||||
warn() {
|
||||
WARN_COUNT=$((WARN_COUNT + 1))
|
||||
printf '[WARN] %s\n' "$*"
|
||||
}
|
||||
|
||||
fail() {
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
printf '[FAIL] %s\n' "$*"
|
||||
}
|
||||
|
||||
have() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
require_file() {
|
||||
local path="$1"
|
||||
if [[ -f "$ROOT_DIR/$path" ]]; then
|
||||
ok "file exists: $path"
|
||||
else
|
||||
fail "missing file: $path"
|
||||
fi
|
||||
}
|
||||
|
||||
require_pattern() {
|
||||
local path="$1"
|
||||
local pattern="$2"
|
||||
local label="$3"
|
||||
if grep -Eq "$pattern" "$ROOT_DIR/$path"; then
|
||||
ok "$label"
|
||||
else
|
||||
fail "$label"
|
||||
fi
|
||||
}
|
||||
|
||||
check_repo() {
|
||||
printf '== repo resilience checks ==\n'
|
||||
|
||||
require_file "scripts/detmir_resilience_check.sh"
|
||||
require_file "docs/DETMIR_RESILIENCE_HARDENING_RU.md"
|
||||
require_file "docs/DLP_RESOURCE_PROFILES_RU.md"
|
||||
require_file "docs/DLP_OPTIONAL_RUNTIME_RU.md"
|
||||
require_file "scripts/detmir_dlp_load_guard.sh"
|
||||
require_file "scripts/detmir_dlp_warehouse_sync.sh"
|
||||
require_file "aw-server/hayabusa/aw-hayabusa.sh"
|
||||
require_file "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs"
|
||||
require_file "windows/AWatchRusCollectorGuardService.cs"
|
||||
require_file "windows/install-collector-guard-service.ps1"
|
||||
|
||||
require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "HAYA_QUARANTINE_DIR" "Hayabusa wrapper has quarantine root"
|
||||
require_pattern "aw-server/hayabusa/aw-hayabusa.sh" "quarantine_incoming_package" "Hayabusa wrapper isolates incoming poison packages"
|
||||
require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "validate_drop_inputs" "Hayabusa autoprocess validates drop package before accept"
|
||||
require_pattern "adk-rust/crates/hayabusa-tools/src/bin/autoprocess.rs" "quarantine_drop_package" "Hayabusa autoprocess quarantines bad drop package"
|
||||
require_pattern "windows/AWatchRusCollectorGuardService.cs" "Process\\.Exited|ChildExited" "Collector guard service watches child exit"
|
||||
require_pattern "windows/AWatchRusCollectorGuardService.cs" "MaxChildRestartsInWindow" "Collector guard service has bounded child restart budget"
|
||||
require_pattern "windows/install-collector-guard-service.ps1" "failureflag" "Collector guard installer enables SCM failureflag"
|
||||
require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Hayabusa poison-package isolation" "Resilience doc records Hayabusa hardening"
|
||||
require_pattern "docs/DETMIR_RESILIENCE_HARDENING_RU.md" "Windows collector guard service child watchdog" "Resilience doc records guard child watchdog"
|
||||
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "core_only" "DLP resource profiles document core_only"
|
||||
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "auto.?disable|автоотключ" "DLP resource profiles document auto-disable guard"
|
||||
require_pattern "docs/DLP_RESOURCE_PROFILES_RU.md" "rollback" "DLP resource profiles document rollback"
|
||||
require_pattern "docs/DETMIR_CURRENT_STATE_RU.md" "AW_DLP_PROFILE=light" "Current state records DLP light profile"
|
||||
require_pattern "scripts/detmir_dlp_runtime_control.sh" "set-profile" "DLP runtime control supports profile switching"
|
||||
require_pattern "scripts/detmir_dlp_runtime_control.sh" "rollback_dlp" "DLP runtime control supports rollback"
|
||||
require_pattern "scripts/detmir_dlp_load_guard.sh" "set-profile core_only" "DLP load guard can auto-disable DLP to core_only"
|
||||
require_pattern "scripts/detmir_dlp_load_guard.sh" "STRIKES_REQUIRED" "DLP load guard requires consecutive overload checks"
|
||||
require_pattern "scripts/detmir_dlp_warehouse_sync.sh" "sqlite3 .*\\.backup" "DLP warehouse sync uses SQLite backup"
|
||||
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_profile: "light"' "Production defaults keep DLP profile light"
|
||||
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_enabled: true' "Production defaults enable lightweight DLP"
|
||||
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_influx_enabled: false' "Production defaults keep DLP Influx disabled"
|
||||
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_collector_enabled: true' "Production defaults enable lightweight DLP collector"
|
||||
require_pattern "ansible/group_vars/all.yml" 'aw_dlp_light_guard_enabled: true' "Production defaults enable DLP load guard"
|
||||
require_pattern "ansible/group_vars/all.yml" 'detmir_portal_dlp_module_enabled_override: true' "Production defaults expose DLP light status to portal"
|
||||
require_pattern "ansible/deploy_aw_server.yml" "detmir-dlp-load-guard.service" "AW server deploy installs DLP load guard service"
|
||||
require_pattern "ansible/deploy_aw_server.yml" "CPUQuota=.*aw_dlp_aggregator_cpu_quota" "DLP aggregator has systemd CPU quota"
|
||||
require_pattern "ansible/deploy_detmir_portal.yml" "detmir-dlp-warehouse-sync.service" "Portal deploy installs DLP warehouse sync service"
|
||||
require_pattern "ansible/deploy_detmir_portal.yml" "detmir_portal_dlp_module_enabled_override \\| default\\(false\\)" "Portal deploy defaults DLP module to disabled"
|
||||
|
||||
if [[ -f "$ROOT_DIR/ansible/inventory.ini" ]] && grep -Eq '(^|[[:space:]])ansible_(become_)?password[[:space:]]*=[[:space:]]*[^<{]' "$ROOT_DIR/ansible/inventory.ini"; then
|
||||
if [[ "$STRICT_SECRETS" == "1" ]]; then
|
||||
fail "ansible/inventory.ini appears to contain literal password assignments; move them to vault/env"
|
||||
else
|
||||
warn "ansible/inventory.ini appears to contain literal password assignments; strict mode would fail"
|
||||
fi
|
||||
else
|
||||
ok "no literal ansible password assignments detected in ansible/inventory.ini"
|
||||
fi
|
||||
}
|
||||
|
||||
check_systemd_unit() {
|
||||
local unit="$1"
|
||||
if ! have systemctl; then
|
||||
warn "systemctl unavailable; skipping $unit"
|
||||
return
|
||||
fi
|
||||
if systemctl is-active --quiet "$unit"; then
|
||||
ok "systemd active: $unit"
|
||||
else
|
||||
fail "systemd not active: $unit"
|
||||
fi
|
||||
}
|
||||
|
||||
check_aw_api() {
|
||||
if ! have curl; then
|
||||
warn "curl unavailable; skipping AW API check"
|
||||
return
|
||||
fi
|
||||
local deadline last_code elapsed
|
||||
deadline=$((SECONDS + AW_READINESS_TIMEOUT_SECONDS))
|
||||
last_code=""
|
||||
while (( SECONDS <= deadline )); do
|
||||
elapsed="$(
|
||||
curl -sS --connect-timeout 3 --max-time 8 -o /dev/null -w '%{http_code} %{time_total}' \
|
||||
"$AW_API/api/0/info" 2>/dev/null || true
|
||||
)"
|
||||
last_code="${elapsed%% *}"
|
||||
if [[ "$last_code" == "200" ]]; then
|
||||
ok "ActivityWatch API readiness /api/0/info returns 200 (${elapsed#* }s)"
|
||||
return
|
||||
fi
|
||||
sleep "$AW_READINESS_INTERVAL_SECONDS"
|
||||
done
|
||||
case "$last_code" in
|
||||
503) fail "ActivityWatch API readiness ended on 503; possible datastore lock poisoning" ;;
|
||||
""|000) fail "ActivityWatch API did not become ready within ${AW_READINESS_TIMEOUT_SECONDS}s" ;;
|
||||
*) fail "ActivityWatch API readiness unexpected final HTTP status: $last_code" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_aw_hot_path() {
|
||||
if ! have curl; then
|
||||
warn "curl unavailable; skipping AW hot-path event check"
|
||||
return
|
||||
fi
|
||||
local url result code elapsed
|
||||
url="${AW_API%/}/api/0/buckets/aw-worktime-sessions_${AW_LOGICAL_HOST_ID}/events?limit=${AW_EVENTS_LIMIT}"
|
||||
result="$(curl -sS --connect-timeout 3 --max-time "$AW_EVENTS_MAX_SECONDS" -o /dev/null -w '%{http_code} %{time_total}' "$url" 2>/dev/null || true)"
|
||||
code="${result%% *}"
|
||||
elapsed="${result#* }"
|
||||
if [[ "$code" != "200" ]]; then
|
||||
fail "ActivityWatch worktime events hot path returned HTTP ${code:-none}"
|
||||
return
|
||||
fi
|
||||
if awk -v t="$elapsed" -v max="$AW_EVENTS_MAX_SECONDS" 'BEGIN { exit !(t <= max) }'; then
|
||||
ok "ActivityWatch worktime events hot path returns 200 (${elapsed}s, limit=${AW_EVENTS_LIMIT})"
|
||||
else
|
||||
fail "ActivityWatch worktime events hot path exceeded ${AW_EVENTS_MAX_SECONDS}s (${elapsed}s)"
|
||||
fi
|
||||
}
|
||||
|
||||
check_worktime_api() {
|
||||
if ! have curl; then
|
||||
warn "curl unavailable; skipping Worktime API check"
|
||||
return
|
||||
fi
|
||||
local tmp code
|
||||
tmp="$(mktemp)"
|
||||
code="$(curl -sS --connect-timeout 3 --max-time 12 -o "$tmp" -w '%{http_code}' \
|
||||
"${AW_WORKTIME_API%/}/reports/worktime/today?format=json&host=${AW_LOGICAL_HOST_ID}&allow_stale=1" 2>/dev/null || true)"
|
||||
if [[ "$code" != "200" ]]; then
|
||||
fail "Worktime API today report returned HTTP ${code:-none}"
|
||||
rm -f "$tmp"
|
||||
return
|
||||
fi
|
||||
if have jq; then
|
||||
if jq -e '(.degraded // false) == false' "$tmp" >/dev/null 2>&1; then
|
||||
ok "Worktime API today report is not degraded"
|
||||
else
|
||||
fail "Worktime API today report is degraded"
|
||||
fi
|
||||
if jq -e '((.rows // .users // []) | length) > 0' "$tmp" >/dev/null 2>&1; then
|
||||
ok "Worktime API today report has employee rows"
|
||||
else
|
||||
fail "Worktime API today report has no employee rows"
|
||||
fi
|
||||
else
|
||||
ok "Worktime API today report returns 200"
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
check_failed_units() {
|
||||
if ! have systemctl; then
|
||||
return
|
||||
fi
|
||||
local failed
|
||||
failed="$(systemctl --failed --no-legend --plain 2>/dev/null | wc -l | tr -d ' ')"
|
||||
if [[ "$failed" == "0" ]]; then
|
||||
ok "systemd failed units count is 0"
|
||||
else
|
||||
fail "systemd failed units count is $failed"
|
||||
fi
|
||||
}
|
||||
|
||||
count_files() {
|
||||
local dir="$1"
|
||||
local pattern="$2"
|
||||
if [[ ! -d "$dir" ]]; then
|
||||
printf '0\n'
|
||||
return
|
||||
fi
|
||||
find "$dir" -maxdepth 1 -type f -name "$pattern" | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
check_hayabusa_queues() {
|
||||
local incoming_dir="$HAYA_ROOT/inbox/incoming"
|
||||
local quarantine_dir="$HAYA_ROOT/quarantine"
|
||||
local incoming_count drop_count old_count quarantine_count
|
||||
incoming_count="$(count_files "$incoming_dir" '*.zip')"
|
||||
drop_count="$(count_files "$HAYA_DROP_DIR" '*.zip')"
|
||||
old_count="0"
|
||||
if [[ -d "$incoming_dir" ]]; then
|
||||
old_count="$(find "$incoming_dir" -maxdepth 1 -type f -name '*.zip' -mmin "+$((MAX_INCOMING_AGE_SECONDS / 60))" | wc -l | tr -d ' ')"
|
||||
fi
|
||||
if [[ "$incoming_count" == "0" ]]; then
|
||||
ok "Hayabusa incoming zip count is 0"
|
||||
else
|
||||
warn "Hayabusa incoming zip count is $incoming_count"
|
||||
fi
|
||||
if [[ "$drop_count" == "0" ]]; then
|
||||
ok "Hayabusa drop zip count is 0"
|
||||
else
|
||||
warn "Hayabusa drop zip count is $drop_count"
|
||||
fi
|
||||
if [[ "$old_count" == "0" ]]; then
|
||||
ok "Hayabusa incoming has no stale zip older than ${MAX_INCOMING_AGE_SECONDS}s"
|
||||
else
|
||||
fail "Hayabusa incoming has $old_count stale zip package(s)"
|
||||
fi
|
||||
if [[ -d "$quarantine_dir" ]]; then
|
||||
quarantine_count="$(find "$quarantine_dir" -type f -name reason.json | wc -l | tr -d ' ')"
|
||||
if [[ "$quarantine_count" == "0" ]]; then
|
||||
ok "Hayabusa quarantine reason count is 0"
|
||||
else
|
||||
warn "Hayabusa quarantine reason count is $quarantine_count; review/replay policy required"
|
||||
fi
|
||||
else
|
||||
warn "Hayabusa quarantine root not found yet: $quarantine_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
check_sqlite_files() {
|
||||
if [[ ! -f "$SQLITE_DB" ]]; then
|
||||
warn "AW SQLite DB not found at $SQLITE_DB; skipping local DB size check"
|
||||
return
|
||||
fi
|
||||
local db_size wal_size
|
||||
db_size="$(stat -c '%s' "$SQLITE_DB")"
|
||||
wal_size="0"
|
||||
[[ -f "$SQLITE_DB-wal" ]] && wal_size="$(stat -c '%s' "$SQLITE_DB-wal")"
|
||||
if (( db_size > 5 * 1024 * 1024 * 1024 )); then
|
||||
fail "AW SQLite DB exceeds 5GiB"
|
||||
elif (( db_size > 2 * 1024 * 1024 * 1024 )); then
|
||||
warn "AW SQLite DB exceeds 2GiB"
|
||||
else
|
||||
ok "AW SQLite DB size below 2GiB"
|
||||
fi
|
||||
if (( wal_size > 1024 * 1024 * 1024 )); then
|
||||
fail "AW SQLite WAL exceeds 1GiB"
|
||||
elif (( wal_size > 256 * 1024 * 1024 )); then
|
||||
warn "AW SQLite WAL exceeds 256MiB"
|
||||
else
|
||||
ok "AW SQLite WAL size below 256MiB"
|
||||
fi
|
||||
}
|
||||
|
||||
check_sqlite_hot_path_index() {
|
||||
if [[ ! -f "$SQLITE_DB" ]]; then
|
||||
warn "AW SQLite DB not found at $SQLITE_DB; skipping hot-path index check"
|
||||
return
|
||||
fi
|
||||
if ! have sqlite3; then
|
||||
warn "sqlite3 unavailable; skipping hot-path index check"
|
||||
return
|
||||
fi
|
||||
local index_exists plan
|
||||
index_exists="$(sqlite3 "$SQLITE_DB" "SELECT name FROM sqlite_master WHERE type='index' AND name='events_bucketrow_starttime_desc_index';" 2>/dev/null || true)"
|
||||
if [[ "$index_exists" == "events_bucketrow_starttime_desc_index" ]]; then
|
||||
ok "AW SQLite hot-path index exists"
|
||||
else
|
||||
fail "AW SQLite hot-path index events_bucketrow_starttime_desc_index is missing"
|
||||
return
|
||||
fi
|
||||
plan="$(
|
||||
sqlite3 "$SQLITE_DB" "EXPLAIN QUERY PLAN SELECT id,starttime,endtime,data FROM events WHERE bucketrow=(SELECT id FROM buckets WHERE name='aw-worktime-sessions_${AW_LOGICAL_HOST_ID}') ORDER BY starttime DESC LIMIT ${AW_EVENTS_LIMIT};" 2>/dev/null || true
|
||||
)"
|
||||
if printf '%s' "$plan" | grep -q 'events_bucketrow_starttime_desc_index'; then
|
||||
ok "AW SQLite worktime event query uses hot-path index"
|
||||
else
|
||||
fail "AW SQLite worktime event query does not use hot-path index"
|
||||
fi
|
||||
if printf '%s' "$plan" | grep -q 'TEMP B-TREE'; then
|
||||
fail "AW SQLite worktime event query still builds TEMP B-TREE"
|
||||
else
|
||||
ok "AW SQLite worktime event query avoids TEMP B-TREE"
|
||||
fi
|
||||
}
|
||||
|
||||
check_units_inactive() {
|
||||
local label="$1"
|
||||
shift
|
||||
if ! have systemctl; then
|
||||
warn "systemctl unavailable; skipping $label runtime check"
|
||||
return
|
||||
fi
|
||||
local active_units=()
|
||||
local unit active
|
||||
for unit in "$@"; do
|
||||
active="$(systemctl is-active "$unit" 2>/dev/null || true)"
|
||||
if [[ "$active" == "active" || "$active" == "activating" ]]; then
|
||||
active_units+=("$unit:$active")
|
||||
fi
|
||||
done
|
||||
if [[ "${#active_units[@]}" -eq 0 ]]; then
|
||||
ok "$label runtime units are inactive"
|
||||
else
|
||||
fail "$label runtime units active: ${active_units[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
check_live() {
|
||||
printf '== live resilience checks ==\n'
|
||||
check_systemd_unit "activitywatch-server"
|
||||
check_systemd_unit "aw-worktime-api"
|
||||
check_aw_api
|
||||
check_aw_hot_path
|
||||
check_worktime_api
|
||||
check_failed_units
|
||||
check_hayabusa_queues
|
||||
check_sqlite_files
|
||||
check_sqlite_hot_path_index
|
||||
if [[ "$EXPECT_OPTIONAL_DLP_OFF" == "1" || "$EXPECT_DLP_PROFILE" == "core_only" ]]; then
|
||||
check_units_inactive "optional DLP" "${DLP_RUNTIME_UNITS[@]}"
|
||||
elif [[ "$EXPECT_DLP_PROFILE" == "light" ]]; then
|
||||
check_units_inactive "heavy DLP" "${DLP_HEAVY_RUNTIME_UNITS[@]}"
|
||||
else
|
||||
warn "optional DLP runtime inactive check skipped for profile=$EXPECT_DLP_PROFILE"
|
||||
fi
|
||||
if [[ "$EXPECT_LOKI_OFF" == "1" ]]; then
|
||||
check_units_inactive "Loki" "${LOKI_RUNTIME_UNITS[@]}"
|
||||
else
|
||||
warn "Loki inactive check skipped by env"
|
||||
fi
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
repo) check_repo ;;
|
||||
live) check_live ;;
|
||||
all) check_repo; check_live ;;
|
||||
*) fail "invalid mode: $MODE" ;;
|
||||
esac
|
||||
|
||||
printf 'summary: ok=%s warn=%s fail=%s\n' "$OK_COUNT" "$WARN_COUNT" "$FAIL_COUNT"
|
||||
if (( FAIL_COUNT > 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user