awatch-rus/AWatch-rus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(portal): restore production runtime health

Browse Source
This commit is contained in:
igor04091968
2026-06-03 23:22:02 +03:00
parent a297220353
commit 1e869e1a14
7 changed files with 314 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adk-rust/crates/detmir-check/src/main.rs
+116 -37
View File
@@ -15,6 +15,7 @@ use serde_json::Value;
const DEFAULT_AW_API: &str = "http://192.0.2.13:5600/api/0";
const DEFAULT_WORKTIME_URL: &str = "http://192.0.2.13:5610";
const DEFAULT_ONE_C_URL: &str = "http://192.0.2.2:8710";
const DEFAULT_RDP_HOST: &str = "198.51.100.18";
const DEFAULT_HOSTNAME: &str = "HOST-EXAMPLE";
const DEFAULT_GATEWAY_HOST: &str = "detmir.example.local";
@@ -33,6 +34,9 @@ struct Cli {
#[arg(long, default_value = DEFAULT_ONE_C_URL)]
one_c_url: String,
#[arg(long, default_value = DEFAULT_RDP_HOST)]
rdp_host: String,
#[arg(long, default_value = DEFAULT_HOSTNAME)]
hostname: String,
@@ -61,6 +65,7 @@ struct Cli {
#[derive(Debug, Clone, Copy)]
enum BucketMode {
Fresh,
InteractiveFresh,
InactiveOk,
EventDriven,
}
@@ -69,6 +74,7 @@ impl BucketMode {
fn as_str(self) -> &'static str {
match self {
Self::Fresh => "fresh",
Self::InteractiveFresh => "interactive_fresh",
Self::InactiveOk => "inactive_ok",
Self::EventDriven => "event_driven",
}
@@ -145,7 +151,7 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
label: "AFK watcher",
bucket: format!("aw-watcher-afk_{hostname}"),
max_age_seconds: Some(15 * 60),
mode: BucketMode::Fresh,
mode: BucketMode::InteractiveFresh,
},
BucketSpec {
label: "Window watcher",
@@ -169,7 +175,7 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
label: "DLP signals",
bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
max_age_seconds: Some(10 * 60),
mode: BucketMode::Fresh,
mode: BucketMode::InteractiveFresh,
},
BucketSpec {
label: "DLP incidents",
@@ -298,13 +304,13 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
}
checks.push(tcp_check(
"198.51.100.18",
&args.rdp_host,
5985,
args.tcp_timeout_seconds,
true,
));
checks.push(tcp_check(
"198.51.100.18",
&args.rdp_host,
22,
args.tcp_timeout_seconds,
true,
@@ -463,6 +469,7 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
Duration::from_secs(args.bucket_timeout_seconds),
)?;
let now = Utc::now();
let interactive_required = interactive_required(&client, &args.hostname, now);
let mut out = Vec::new();
for spec in bucket_specs(&args.hostname) {
@@ -485,28 +492,8 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
Ok(Some(event)) => {
let ts = event.timestamp_utc()?;
let age = (now - ts).num_seconds();
let status = match spec.mode {
BucketMode::InactiveOk => {
if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) {
"FRESH"
} else {
"INACTIVE"
}
}
BucketMode::Fresh => {
if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) {
"FRESH"
} else {
"STALE"
}
}
BucketMode::EventDriven => "EVENT-DRIVEN",
};
let ok = match spec.mode {
BucketMode::InactiveOk => true,
BucketMode::Fresh => status == "FRESH",
BucketMode::EventDriven => true,
};
let (status, ok) =
classify_bucket(spec.mode, spec.max_age_seconds, age, interactive_required);
out.push(BucketCheck {
label: spec.label.to_string(),
bucket: spec.bucket,
@@ -519,17 +506,20 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
error: None,
});
}
Ok(None) => out.push(BucketCheck {
label: spec.label.to_string(),
bucket: spec.bucket,
mode: spec.mode.as_str().to_string(),
status: "DEAD".to_string(),
ok: false,
event_count_sample: Some(0),
latest: None,
age_seconds: None,
error: None,
}),
Ok(None) => {
let (status, ok) = classify_missing_bucket(spec.mode, interactive_required);
out.push(BucketCheck {
label: spec.label.to_string(),
bucket: spec.bucket,
mode: spec.mode.as_str().to_string(),
status: status.to_string(),
ok,
event_count_sample: Some(0),
latest: None,
age_seconds: None,
error: None,
});
}
Err(err) => out.push(BucketCheck {
label: spec.label.to_string(),
bucket: spec.bucket,
@@ -546,6 +536,58 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
Ok(out)
}
fn interactive_required(client: &ActivityWatchClient, hostname: &str, now: DateTime<Utc>) -> bool {
let bucket = format!("aw-worktime-sessions_{hostname}");
let Ok(Some(event)) = client.latest_event(&bucket) else {
return false;
};
let Ok(ts) = event.timestamp_utc() else {
return false;
};
let age = (now - ts).num_seconds();
let fresh = (0..=5 * 60).contains(&age);
let active = event
.data
.get("active")
.and_then(Value::as_bool)
.unwrap_or(false);
fresh && active
}
fn classify_bucket(
mode: BucketMode,
max_age_seconds: Option<i64>,
age_seconds: i64,
interactive_required: bool,
) -> (&'static str, bool) {
match mode {
BucketMode::InactiveOk => {
if max_age_seconds.is_some_and(|max_age| age_seconds <= max_age) {
("FRESH", true)
} else {
("INACTIVE", true)
}
}
BucketMode::InteractiveFresh if !interactive_required => ("INACTIVE", true),
BucketMode::InteractiveFresh | BucketMode::Fresh => {
if max_age_seconds.is_some_and(|max_age| age_seconds <= max_age) {
("FRESH", true)
} else {
("STALE", false)
}
}
BucketMode::EventDriven => ("EVENT-DRIVEN", true),
}
}
fn classify_missing_bucket(mode: BucketMode, interactive_required: bool) -> (&'static str, bool) {
match mode {
BucketMode::InteractiveFresh if !interactive_required => ("INACTIVE", true),
BucketMode::EventDriven => ("EVENT-DRIVEN", true),
_ => ("DEAD", false),
}
}
fn build_report(args: &Cli) -> Result<CheckReport> {
let services = service_checks(args);
let buckets = bucket_health(args)?;
@@ -630,6 +672,7 @@ fn main() -> Result<()> {
args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api);
args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
args.rdp_host = env_or_default("DETMIR_RDP_HOST", &args.rdp_host);
args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname);
args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json);
@@ -645,3 +688,39 @@ fn main() -> Result<()> {
exit_codes::CHECK_FAILED
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn interactive_bucket_is_inactive_when_no_interactive_session() {
let (status, ok) = classify_bucket(
BucketMode::InteractiveFresh,
Some(15 * 60),
3 * 60 * 60,
false,
);
assert_eq!(status, "INACTIVE");
assert!(ok);
}
#[test]
fn interactive_bucket_is_stale_when_session_is_active() {
let (status, ok) = classify_bucket(
BucketMode::InteractiveFresh,
Some(15 * 60),
3 * 60 * 60,
true,
);
assert_eq!(status, "STALE");
assert!(!ok);
}
#[test]
fn missing_interactive_bucket_is_inactive_when_no_interactive_session() {
let (status, ok) = classify_missing_bucket(BucketMode::InteractiveFresh, false);
assert_eq!(status, "INACTIVE");
assert!(ok);
}
}
adk-rust/crates/detmir-portal/src/main.rs
+56 -20
View File
@@ -3,6 +3,7 @@ use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
@@ -23,8 +24,17 @@ const INDEX_HTML: &str = include_str!("static/index.html");
const APP_CSS: &str = include_str!("static/app.css");
const APP_JS: &str = include_str!("static/app.js");
const UEBA_BASELINE_MIN_SAMPLES: usize = 3;
const SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(5);
#[derive(Debug, Parser)]
type SnapshotCache = Arc<Mutex<Option<CachedSnapshot>>>;
#[derive(Clone, Debug)]
struct CachedSnapshot {
created: Instant,
snapshot: Snapshot,
}
#[derive(Clone, Debug, Parser)]
#[command(about = "Read-only DetMir operator/manager/owner web portal")]
struct Cli {
#[arg(long, default_value = "127.0.0.1:8720", env = "DETMIR_PORTAL_BIND")]
@@ -130,7 +140,7 @@ struct Cli {
evidence_upload_token: Option<String>,
}
#[derive(Debug, Serialize)]
#[derive(Clone, Debug, Serialize)]
struct SourceStatus {
ok: bool,
status: String,
@@ -369,7 +379,7 @@ struct PortalLinks {
file1c_actions: String,
}
#[derive(Debug)]
#[derive(Clone, Debug)]
struct Snapshot {
generated_at_utc: String,
detmir_status: SourceStatus,
@@ -570,21 +580,26 @@ fn run() -> Result<i32> {
}
let server = Server::http(&args.bind).map_err(|err| anyhow!("bind {}: {err}", args.bind))?;
let snapshot_cache: SnapshotCache = Arc::new(Mutex::new(None));
eprintln!("detmir-portal listening on http://{}", args.bind);
for request in server.incoming_requests() {
let result = if args.evidence_only {
handle_evidence_only_request(request, &args)
} else {
handle_request(request, &args)
};
if let Err(err) = result {
eprintln!("detmir-portal request failed: {err:#}");
}
let args = args.clone();
let snapshot_cache = Arc::clone(&snapshot_cache);
thread::spawn(move || {
let result = if args.evidence_only {
handle_evidence_only_request(request, &args)
} else {
handle_request(request, &args, &snapshot_cache)
};
if let Err(err) = result {
eprintln!("detmir-portal request failed: {err:#}");
}
});
}
Ok(0)
}
fn handle_request(request: Request, args: &Cli) -> Result<()> {
fn handle_request(request: Request, args: &Cli, snapshot_cache: &SnapshotCache) -> Result<()> {
let method = request.method().clone();
let url = request.url().to_string();
let path = normalize_path(&url);
@@ -616,33 +631,39 @@ fn handle_request(request: Request, args: &Cli) -> Result<()> {
"application/javascript; charset=utf-8",
),
"/favicon.ico" => respond_text(request, StatusCode(204), "", "image/x-icon"),
"/api/health" => respond_json(request, &build_health(&build_snapshot(args))),
"/api/health" => respond_json(
request,
&build_health(&cached_snapshot(args, snapshot_cache)),
),
"/api/readiness/latest" => respond_json(request, &readiness_latest(args)),
"/api/readiness/bundle" => respond_json(request, &readiness_bundle(args)),
"/api/readiness/verify" => respond_json(request, &readiness_verify(args)),
"/api/summary" => respond_json(request, &build_summary(&build_snapshot(args))),
"/api/summary" => respond_json(
request,
&build_summary(&cached_snapshot(args, snapshot_cache)),
),
"/api/operator" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
let incident_state = load_incident_state_best_effort(args);
respond_json(request, &build_operator(&snapshot, &incident_state))
}
"/api/manager" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
respond_json(request, &build_manager(&snapshot))
}
"/api/workforce/policy/explain" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
respond_json(
request,
&build_workforce_policy_explain(&snapshot, &args.workforce_policy_path, anonymize),
)
}
"/api/owner" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
respond_json(request, &build_owner(&snapshot))
}
"/api/reports" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
let incident_state = load_incident_state_best_effort(args);
let evidence = build_dlp_evidence_response(args);
let ueba_baseline_path = ueba_baseline_state_path(args);
@@ -660,7 +681,7 @@ fn handle_request(request: Request, args: &Cli) -> Result<()> {
)
}
"/api/incidents" => {
let snapshot = build_snapshot(args);
let snapshot = cached_snapshot(args, snapshot_cache);
let incident_state = load_incident_state_best_effort(args);
respond_json(request, &build_incidents(&snapshot, &incident_state))
}
@@ -854,6 +875,21 @@ fn query_flag(url: &str, key: &str) -> bool {
})
}
fn cached_snapshot(args: &Cli, cache: &SnapshotCache) -> Snapshot {
let mut guard = cache.lock().expect("snapshot cache mutex poisoned");
if let Some(cached) = guard.as_ref() {
if cached.created.elapsed() <= SNAPSHOT_CACHE_TTL {
return cached.snapshot.clone();
}
}
let snapshot = build_snapshot(args);
*guard = Some(CachedSnapshot {
created: Instant::now(),
snapshot: snapshot.clone(),
});
snapshot
}
fn build_snapshot(args: &Cli) -> Snapshot {
let timeout = Duration::from_secs(args.timeout_seconds);
Snapshot {