fix(portal): restore production runtime health
This commit is contained in:
@@ -15,6 +15,7 @@ use serde_json::Value;
|
|||||||
const DEFAULT_AW_API: &str = "http://192.0.2.13:5600/api/0";
|
const DEFAULT_AW_API: &str = "http://192.0.2.13:5600/api/0";
|
||||||
const DEFAULT_WORKTIME_URL: &str = "http://192.0.2.13:5610";
|
const DEFAULT_WORKTIME_URL: &str = "http://192.0.2.13:5610";
|
||||||
const DEFAULT_ONE_C_URL: &str = "http://192.0.2.2:8710";
|
const DEFAULT_ONE_C_URL: &str = "http://192.0.2.2:8710";
|
||||||
|
const DEFAULT_RDP_HOST: &str = "198.51.100.18";
|
||||||
const DEFAULT_HOSTNAME: &str = "HOST-EXAMPLE";
|
const DEFAULT_HOSTNAME: &str = "HOST-EXAMPLE";
|
||||||
const DEFAULT_GATEWAY_HOST: &str = "detmir.example.local";
|
const DEFAULT_GATEWAY_HOST: &str = "detmir.example.local";
|
||||||
|
|
||||||
@@ -33,6 +34,9 @@ struct Cli {
|
|||||||
#[arg(long, default_value = DEFAULT_ONE_C_URL)]
|
#[arg(long, default_value = DEFAULT_ONE_C_URL)]
|
||||||
one_c_url: String,
|
one_c_url: String,
|
||||||
|
|
||||||
|
#[arg(long, default_value = DEFAULT_RDP_HOST)]
|
||||||
|
rdp_host: String,
|
||||||
|
|
||||||
#[arg(long, default_value = DEFAULT_HOSTNAME)]
|
#[arg(long, default_value = DEFAULT_HOSTNAME)]
|
||||||
hostname: String,
|
hostname: String,
|
||||||
|
|
||||||
@@ -61,6 +65,7 @@ struct Cli {
|
|||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
enum BucketMode {
|
enum BucketMode {
|
||||||
Fresh,
|
Fresh,
|
||||||
|
InteractiveFresh,
|
||||||
InactiveOk,
|
InactiveOk,
|
||||||
EventDriven,
|
EventDriven,
|
||||||
}
|
}
|
||||||
@@ -69,6 +74,7 @@ impl BucketMode {
|
|||||||
fn as_str(self) -> &'static str {
|
fn as_str(self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Fresh => "fresh",
|
Self::Fresh => "fresh",
|
||||||
|
Self::InteractiveFresh => "interactive_fresh",
|
||||||
Self::InactiveOk => "inactive_ok",
|
Self::InactiveOk => "inactive_ok",
|
||||||
Self::EventDriven => "event_driven",
|
Self::EventDriven => "event_driven",
|
||||||
}
|
}
|
||||||
@@ -145,7 +151,7 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
|
|||||||
label: "AFK watcher",
|
label: "AFK watcher",
|
||||||
bucket: format!("aw-watcher-afk_{hostname}"),
|
bucket: format!("aw-watcher-afk_{hostname}"),
|
||||||
max_age_seconds: Some(15 * 60),
|
max_age_seconds: Some(15 * 60),
|
||||||
mode: BucketMode::Fresh,
|
mode: BucketMode::InteractiveFresh,
|
||||||
},
|
},
|
||||||
BucketSpec {
|
BucketSpec {
|
||||||
label: "Window watcher",
|
label: "Window watcher",
|
||||||
@@ -169,7 +175,7 @@ fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
|
|||||||
label: "DLP signals",
|
label: "DLP signals",
|
||||||
bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
|
bucket: format!("aw-dlp-endpoint-signals_{hostname}"),
|
||||||
max_age_seconds: Some(10 * 60),
|
max_age_seconds: Some(10 * 60),
|
||||||
mode: BucketMode::Fresh,
|
mode: BucketMode::InteractiveFresh,
|
||||||
},
|
},
|
||||||
BucketSpec {
|
BucketSpec {
|
||||||
label: "DLP incidents",
|
label: "DLP incidents",
|
||||||
@@ -298,13 +304,13 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
checks.push(tcp_check(
|
checks.push(tcp_check(
|
||||||
"198.51.100.18",
|
&args.rdp_host,
|
||||||
5985,
|
5985,
|
||||||
args.tcp_timeout_seconds,
|
args.tcp_timeout_seconds,
|
||||||
true,
|
true,
|
||||||
));
|
));
|
||||||
checks.push(tcp_check(
|
checks.push(tcp_check(
|
||||||
"198.51.100.18",
|
&args.rdp_host,
|
||||||
22,
|
22,
|
||||||
args.tcp_timeout_seconds,
|
args.tcp_timeout_seconds,
|
||||||
true,
|
true,
|
||||||
@@ -463,6 +469,7 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
|
|||||||
Duration::from_secs(args.bucket_timeout_seconds),
|
Duration::from_secs(args.bucket_timeout_seconds),
|
||||||
)?;
|
)?;
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
|
let interactive_required = interactive_required(&client, &args.hostname, now);
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
|
|
||||||
for spec in bucket_specs(&args.hostname) {
|
for spec in bucket_specs(&args.hostname) {
|
||||||
@@ -485,28 +492,8 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
|
|||||||
Ok(Some(event)) => {
|
Ok(Some(event)) => {
|
||||||
let ts = event.timestamp_utc()?;
|
let ts = event.timestamp_utc()?;
|
||||||
let age = (now - ts).num_seconds();
|
let age = (now - ts).num_seconds();
|
||||||
let status = match spec.mode {
|
let (status, ok) =
|
||||||
BucketMode::InactiveOk => {
|
classify_bucket(spec.mode, spec.max_age_seconds, age, interactive_required);
|
||||||
if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) {
|
|
||||||
"FRESH"
|
|
||||||
} else {
|
|
||||||
"INACTIVE"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BucketMode::Fresh => {
|
|
||||||
if spec.max_age_seconds.is_some_and(|max_age| age <= max_age) {
|
|
||||||
"FRESH"
|
|
||||||
} else {
|
|
||||||
"STALE"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BucketMode::EventDriven => "EVENT-DRIVEN",
|
|
||||||
};
|
|
||||||
let ok = match spec.mode {
|
|
||||||
BucketMode::InactiveOk => true,
|
|
||||||
BucketMode::Fresh => status == "FRESH",
|
|
||||||
BucketMode::EventDriven => true,
|
|
||||||
};
|
|
||||||
out.push(BucketCheck {
|
out.push(BucketCheck {
|
||||||
label: spec.label.to_string(),
|
label: spec.label.to_string(),
|
||||||
bucket: spec.bucket,
|
bucket: spec.bucket,
|
||||||
@@ -519,17 +506,20 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
|
|||||||
error: None,
|
error: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Ok(None) => out.push(BucketCheck {
|
Ok(None) => {
|
||||||
label: spec.label.to_string(),
|
let (status, ok) = classify_missing_bucket(spec.mode, interactive_required);
|
||||||
bucket: spec.bucket,
|
out.push(BucketCheck {
|
||||||
mode: spec.mode.as_str().to_string(),
|
label: spec.label.to_string(),
|
||||||
status: "DEAD".to_string(),
|
bucket: spec.bucket,
|
||||||
ok: false,
|
mode: spec.mode.as_str().to_string(),
|
||||||
event_count_sample: Some(0),
|
status: status.to_string(),
|
||||||
latest: None,
|
ok,
|
||||||
age_seconds: None,
|
event_count_sample: Some(0),
|
||||||
error: None,
|
latest: None,
|
||||||
}),
|
age_seconds: None,
|
||||||
|
error: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
Err(err) => out.push(BucketCheck {
|
Err(err) => out.push(BucketCheck {
|
||||||
label: spec.label.to_string(),
|
label: spec.label.to_string(),
|
||||||
bucket: spec.bucket,
|
bucket: spec.bucket,
|
||||||
@@ -546,6 +536,58 @@ fn bucket_health(args: &Cli) -> Result<Vec<BucketCheck>> {
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn interactive_required(client: &ActivityWatchClient, hostname: &str, now: DateTime<Utc>) -> bool {
|
||||||
|
let bucket = format!("aw-worktime-sessions_{hostname}");
|
||||||
|
let Ok(Some(event)) = client.latest_event(&bucket) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Ok(ts) = event.timestamp_utc() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let age = (now - ts).num_seconds();
|
||||||
|
let fresh = (0..=5 * 60).contains(&age);
|
||||||
|
let active = event
|
||||||
|
.data
|
||||||
|
.get("active")
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(false);
|
||||||
|
fresh && active
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_bucket(
|
||||||
|
mode: BucketMode,
|
||||||
|
max_age_seconds: Option<i64>,
|
||||||
|
age_seconds: i64,
|
||||||
|
interactive_required: bool,
|
||||||
|
) -> (&'static str, bool) {
|
||||||
|
match mode {
|
||||||
|
BucketMode::InactiveOk => {
|
||||||
|
if max_age_seconds.is_some_and(|max_age| age_seconds <= max_age) {
|
||||||
|
("FRESH", true)
|
||||||
|
} else {
|
||||||
|
("INACTIVE", true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BucketMode::InteractiveFresh if !interactive_required => ("INACTIVE", true),
|
||||||
|
BucketMode::InteractiveFresh | BucketMode::Fresh => {
|
||||||
|
if max_age_seconds.is_some_and(|max_age| age_seconds <= max_age) {
|
||||||
|
("FRESH", true)
|
||||||
|
} else {
|
||||||
|
("STALE", false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BucketMode::EventDriven => ("EVENT-DRIVEN", true),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_missing_bucket(mode: BucketMode, interactive_required: bool) -> (&'static str, bool) {
|
||||||
|
match mode {
|
||||||
|
BucketMode::InteractiveFresh if !interactive_required => ("INACTIVE", true),
|
||||||
|
BucketMode::EventDriven => ("EVENT-DRIVEN", true),
|
||||||
|
_ => ("DEAD", false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn build_report(args: &Cli) -> Result<CheckReport> {
|
fn build_report(args: &Cli) -> Result<CheckReport> {
|
||||||
let services = service_checks(args);
|
let services = service_checks(args);
|
||||||
let buckets = bucket_health(args)?;
|
let buckets = bucket_health(args)?;
|
||||||
@@ -630,6 +672,7 @@ fn main() -> Result<()> {
|
|||||||
args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api);
|
args.aw_api = env_or_default("DETMIR_AW_API", &args.aw_api);
|
||||||
args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
|
args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
|
||||||
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
|
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
|
||||||
|
args.rdp_host = env_or_default("DETMIR_RDP_HOST", &args.rdp_host);
|
||||||
args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname);
|
args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname);
|
||||||
args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json);
|
args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json);
|
||||||
|
|
||||||
@@ -645,3 +688,39 @@ fn main() -> Result<()> {
|
|||||||
exit_codes::CHECK_FAILED
|
exit_codes::CHECK_FAILED
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn interactive_bucket_is_inactive_when_no_interactive_session() {
|
||||||
|
let (status, ok) = classify_bucket(
|
||||||
|
BucketMode::InteractiveFresh,
|
||||||
|
Some(15 * 60),
|
||||||
|
3 * 60 * 60,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert_eq!(status, "INACTIVE");
|
||||||
|
assert!(ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn interactive_bucket_is_stale_when_session_is_active() {
|
||||||
|
let (status, ok) = classify_bucket(
|
||||||
|
BucketMode::InteractiveFresh,
|
||||||
|
Some(15 * 60),
|
||||||
|
3 * 60 * 60,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
assert_eq!(status, "STALE");
|
||||||
|
assert!(!ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_interactive_bucket_is_inactive_when_no_interactive_session() {
|
||||||
|
let (status, ok) = classify_missing_bucket(BucketMode::InteractiveFresh, false);
|
||||||
|
assert_eq!(status, "INACTIVE");
|
||||||
|
assert!(ok);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use std::fs::{self, File, OpenOptions};
|
|||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Command, Stdio};
|
use std::process::{Command, Stdio};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -23,8 +24,17 @@ const INDEX_HTML: &str = include_str!("static/index.html");
|
|||||||
const APP_CSS: &str = include_str!("static/app.css");
|
const APP_CSS: &str = include_str!("static/app.css");
|
||||||
const APP_JS: &str = include_str!("static/app.js");
|
const APP_JS: &str = include_str!("static/app.js");
|
||||||
const UEBA_BASELINE_MIN_SAMPLES: usize = 3;
|
const UEBA_BASELINE_MIN_SAMPLES: usize = 3;
|
||||||
|
const SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
#[derive(Debug, Parser)]
|
type SnapshotCache = Arc<Mutex<Option<CachedSnapshot>>>;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct CachedSnapshot {
|
||||||
|
created: Instant,
|
||||||
|
snapshot: Snapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Parser)]
|
||||||
#[command(about = "Read-only DetMir operator/manager/owner web portal")]
|
#[command(about = "Read-only DetMir operator/manager/owner web portal")]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
#[arg(long, default_value = "127.0.0.1:8720", env = "DETMIR_PORTAL_BIND")]
|
#[arg(long, default_value = "127.0.0.1:8720", env = "DETMIR_PORTAL_BIND")]
|
||||||
@@ -130,7 +140,7 @@ struct Cli {
|
|||||||
evidence_upload_token: Option<String>,
|
evidence_upload_token: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Clone, Debug, Serialize)]
|
||||||
struct SourceStatus {
|
struct SourceStatus {
|
||||||
ok: bool,
|
ok: bool,
|
||||||
status: String,
|
status: String,
|
||||||
@@ -369,7 +379,7 @@ struct PortalLinks {
|
|||||||
file1c_actions: String,
|
file1c_actions: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Clone, Debug)]
|
||||||
struct Snapshot {
|
struct Snapshot {
|
||||||
generated_at_utc: String,
|
generated_at_utc: String,
|
||||||
detmir_status: SourceStatus,
|
detmir_status: SourceStatus,
|
||||||
@@ -570,21 +580,26 @@ fn run() -> Result<i32> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let server = Server::http(&args.bind).map_err(|err| anyhow!("bind {}: {err}", args.bind))?;
|
let server = Server::http(&args.bind).map_err(|err| anyhow!("bind {}: {err}", args.bind))?;
|
||||||
|
let snapshot_cache: SnapshotCache = Arc::new(Mutex::new(None));
|
||||||
eprintln!("detmir-portal listening on http://{}", args.bind);
|
eprintln!("detmir-portal listening on http://{}", args.bind);
|
||||||
for request in server.incoming_requests() {
|
for request in server.incoming_requests() {
|
||||||
let result = if args.evidence_only {
|
let args = args.clone();
|
||||||
handle_evidence_only_request(request, &args)
|
let snapshot_cache = Arc::clone(&snapshot_cache);
|
||||||
} else {
|
thread::spawn(move || {
|
||||||
handle_request(request, &args)
|
let result = if args.evidence_only {
|
||||||
};
|
handle_evidence_only_request(request, &args)
|
||||||
if let Err(err) = result {
|
} else {
|
||||||
eprintln!("detmir-portal request failed: {err:#}");
|
handle_request(request, &args, &snapshot_cache)
|
||||||
}
|
};
|
||||||
|
if let Err(err) = result {
|
||||||
|
eprintln!("detmir-portal request failed: {err:#}");
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
Ok(0)
|
Ok(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_request(request: Request, args: &Cli) -> Result<()> {
|
fn handle_request(request: Request, args: &Cli, snapshot_cache: &SnapshotCache) -> Result<()> {
|
||||||
let method = request.method().clone();
|
let method = request.method().clone();
|
||||||
let url = request.url().to_string();
|
let url = request.url().to_string();
|
||||||
let path = normalize_path(&url);
|
let path = normalize_path(&url);
|
||||||
@@ -616,33 +631,39 @@ fn handle_request(request: Request, args: &Cli) -> Result<()> {
|
|||||||
"application/javascript; charset=utf-8",
|
"application/javascript; charset=utf-8",
|
||||||
),
|
),
|
||||||
"/favicon.ico" => respond_text(request, StatusCode(204), "", "image/x-icon"),
|
"/favicon.ico" => respond_text(request, StatusCode(204), "", "image/x-icon"),
|
||||||
"/api/health" => respond_json(request, &build_health(&build_snapshot(args))),
|
"/api/health" => respond_json(
|
||||||
|
request,
|
||||||
|
&build_health(&cached_snapshot(args, snapshot_cache)),
|
||||||
|
),
|
||||||
"/api/readiness/latest" => respond_json(request, &readiness_latest(args)),
|
"/api/readiness/latest" => respond_json(request, &readiness_latest(args)),
|
||||||
"/api/readiness/bundle" => respond_json(request, &readiness_bundle(args)),
|
"/api/readiness/bundle" => respond_json(request, &readiness_bundle(args)),
|
||||||
"/api/readiness/verify" => respond_json(request, &readiness_verify(args)),
|
"/api/readiness/verify" => respond_json(request, &readiness_verify(args)),
|
||||||
"/api/summary" => respond_json(request, &build_summary(&build_snapshot(args))),
|
"/api/summary" => respond_json(
|
||||||
|
request,
|
||||||
|
&build_summary(&cached_snapshot(args, snapshot_cache)),
|
||||||
|
),
|
||||||
"/api/operator" => {
|
"/api/operator" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
let incident_state = load_incident_state_best_effort(args);
|
let incident_state = load_incident_state_best_effort(args);
|
||||||
respond_json(request, &build_operator(&snapshot, &incident_state))
|
respond_json(request, &build_operator(&snapshot, &incident_state))
|
||||||
}
|
}
|
||||||
"/api/manager" => {
|
"/api/manager" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
respond_json(request, &build_manager(&snapshot))
|
respond_json(request, &build_manager(&snapshot))
|
||||||
}
|
}
|
||||||
"/api/workforce/policy/explain" => {
|
"/api/workforce/policy/explain" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
respond_json(
|
respond_json(
|
||||||
request,
|
request,
|
||||||
&build_workforce_policy_explain(&snapshot, &args.workforce_policy_path, anonymize),
|
&build_workforce_policy_explain(&snapshot, &args.workforce_policy_path, anonymize),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
"/api/owner" => {
|
"/api/owner" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
respond_json(request, &build_owner(&snapshot))
|
respond_json(request, &build_owner(&snapshot))
|
||||||
}
|
}
|
||||||
"/api/reports" => {
|
"/api/reports" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
let incident_state = load_incident_state_best_effort(args);
|
let incident_state = load_incident_state_best_effort(args);
|
||||||
let evidence = build_dlp_evidence_response(args);
|
let evidence = build_dlp_evidence_response(args);
|
||||||
let ueba_baseline_path = ueba_baseline_state_path(args);
|
let ueba_baseline_path = ueba_baseline_state_path(args);
|
||||||
@@ -660,7 +681,7 @@ fn handle_request(request: Request, args: &Cli) -> Result<()> {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
"/api/incidents" => {
|
"/api/incidents" => {
|
||||||
let snapshot = build_snapshot(args);
|
let snapshot = cached_snapshot(args, snapshot_cache);
|
||||||
let incident_state = load_incident_state_best_effort(args);
|
let incident_state = load_incident_state_best_effort(args);
|
||||||
respond_json(request, &build_incidents(&snapshot, &incident_state))
|
respond_json(request, &build_incidents(&snapshot, &incident_state))
|
||||||
}
|
}
|
||||||
@@ -854,6 +875,21 @@ fn query_flag(url: &str, key: &str) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn cached_snapshot(args: &Cli, cache: &SnapshotCache) -> Snapshot {
|
||||||
|
let mut guard = cache.lock().expect("snapshot cache mutex poisoned");
|
||||||
|
if let Some(cached) = guard.as_ref() {
|
||||||
|
if cached.created.elapsed() <= SNAPSHOT_CACHE_TTL {
|
||||||
|
return cached.snapshot.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let snapshot = build_snapshot(args);
|
||||||
|
*guard = Some(CachedSnapshot {
|
||||||
|
created: Instant::now(),
|
||||||
|
snapshot: snapshot.clone(),
|
||||||
|
});
|
||||||
|
snapshot
|
||||||
|
}
|
||||||
|
|
||||||
fn build_snapshot(args: &Cli) -> Snapshot {
|
fn build_snapshot(args: &Cli) -> Snapshot {
|
||||||
let timeout = Duration::from_secs(args.timeout_seconds);
|
let timeout = Duration::from_secs(args.timeout_seconds);
|
||||||
Snapshot {
|
Snapshot {
|
||||||
|
|||||||
@@ -593,11 +593,13 @@
|
|||||||
failed_when: false
|
failed_when: false
|
||||||
no_log: true
|
no_log: true
|
||||||
|
|
||||||
- name: Подготовить effective Influx tokens без вывода секретов
|
- name: Подготовить effective runtime values без вывода секретов
|
||||||
ansible.builtin.set_fact:
|
ansible.builtin.set_fact:
|
||||||
aw_existing_server_env_text: "{{ aw_existing_server_env_raw.content | default('') | b64decode }}"
|
aw_existing_server_env_text: "{{ aw_existing_server_env_raw.content | default('') | b64decode }}"
|
||||||
aw_existing_worktime_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_WORKTIME_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_WORKTIME_INFLUX_TOKEN=', '')) }}"
|
aw_existing_worktime_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_WORKTIME_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_WORKTIME_INFLUX_TOKEN=', '')) }}"
|
||||||
aw_existing_dlp_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_DLP_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_DLP_INFLUX_TOKEN=', '')) }}"
|
aw_existing_dlp_influx_token: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_DLP_INFLUX_TOKEN=.*$') | default('', true) | regex_replace('^AW_DLP_INFLUX_TOKEN=', '')) }}"
|
||||||
|
aw_existing_monitored_windows_host: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_MONITORED_WINDOWS_HOST=.*$') | default('', true) | regex_replace('^AW_MONITORED_WINDOWS_HOST=', '')) }}"
|
||||||
|
aw_existing_monitored_windows_hostname: "{{ (aw_existing_server_env_raw.content | default('') | b64decode | regex_search('(?m)^AW_MONITORED_WINDOWS_HOSTNAME=.*$') | default('', true) | regex_replace('^AW_MONITORED_WINDOWS_HOSTNAME=', '')) }}"
|
||||||
no_log: true
|
no_log: true
|
||||||
|
|
||||||
- name: Выбрать effective Influx tokens
|
- name: Выбрать effective Influx tokens
|
||||||
@@ -606,6 +608,53 @@
|
|||||||
aw_effective_dlp_influx_token: "{{ (aw_dlp_influx_token | default('') | string) if ((aw_dlp_influx_token | default('') | string | length) > 0) else aw_existing_dlp_influx_token }}"
|
aw_effective_dlp_influx_token: "{{ (aw_dlp_influx_token | default('') | string) if ((aw_dlp_influx_token | default('') | string | length) > 0) else aw_existing_dlp_influx_token }}"
|
||||||
no_log: true
|
no_log: true
|
||||||
|
|
||||||
|
- name: Выбрать effective monitored Windows target
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
aw_effective_monitored_windows_host: >-
|
||||||
|
{{
|
||||||
|
(aw_monitored_windows_host | default('') | string)
|
||||||
|
if (
|
||||||
|
(aw_monitored_windows_host | default('') | string | length) > 0
|
||||||
|
and (aw_monitored_windows_host | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
)
|
||||||
|
else aw_existing_monitored_windows_host
|
||||||
|
}}
|
||||||
|
aw_effective_monitored_windows_hostname: >-
|
||||||
|
{{
|
||||||
|
(aw_monitored_windows_hostname | default('') | string)
|
||||||
|
if (
|
||||||
|
(aw_monitored_windows_hostname | default('') | string | length) > 0
|
||||||
|
and (aw_monitored_windows_hostname | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
)
|
||||||
|
else aw_existing_monitored_windows_hostname
|
||||||
|
}}
|
||||||
|
no_log: true
|
||||||
|
|
||||||
|
- name: Выбрать effective worktime host
|
||||||
|
ansible.builtin.set_fact:
|
||||||
|
aw_effective_worktime_host: >-
|
||||||
|
{{
|
||||||
|
(aw_worktime_host | default('') | string)
|
||||||
|
if (
|
||||||
|
(aw_worktime_host | default('') | string | length) > 0
|
||||||
|
and (aw_worktime_host | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
)
|
||||||
|
else aw_effective_monitored_windows_hostname
|
||||||
|
}}
|
||||||
|
no_log: true
|
||||||
|
|
||||||
|
- name: Проверить monitored Windows target для production healthd
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- (aw_effective_monitored_windows_host | default('') | string | length) > 0
|
||||||
|
- (aw_effective_monitored_windows_host | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
- (aw_effective_monitored_windows_hostname | default('') | string | length) > 0
|
||||||
|
- (aw_effective_monitored_windows_hostname | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
- (aw_effective_worktime_host | default('') | string | length) > 0
|
||||||
|
- (aw_effective_worktime_host | default('') | string | regex_search('192\\.0\\.2\\.|198\\.51\\.100\\.|203\\.0\\.113\\.|<|>|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE')) is none
|
||||||
|
fail_msg: "aw-rus-healthd получил public example/TEST-NET Windows target. Задайте live значения в private inventory/env или сохраните их в runtime /etc/activitywatch/aw-server.env."
|
||||||
|
when: aw_server_post_deploy_health_check_enabled | default(true) | bool
|
||||||
|
|
||||||
- name: Проверить Influx token для AW worktime exporter
|
- name: Проверить Influx token для AW worktime exporter
|
||||||
ansible.builtin.assert:
|
ansible.builtin.assert:
|
||||||
that:
|
that:
|
||||||
@@ -675,7 +724,7 @@
|
|||||||
AW_SERVER_GROUP={{ aw_server_group }}
|
AW_SERVER_GROUP={{ aw_server_group }}
|
||||||
AW_WORKTIME_REPORT_BASE={{ aw_worktime_report_base }}
|
AW_WORKTIME_REPORT_BASE={{ aw_worktime_report_base }}
|
||||||
AW_WORKTIME_TZ={{ aw_worktime_timezone }}
|
AW_WORKTIME_TZ={{ aw_worktime_timezone }}
|
||||||
AW_WORKTIME_HOST={{ aw_worktime_host | default(aw_monitored_windows_hostname | default('HOST-EXAMPLE')) }}
|
AW_WORKTIME_HOST={{ aw_effective_worktime_host | default(aw_effective_monitored_windows_hostname | default('HOST-EXAMPLE')) }}
|
||||||
AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output
|
AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output
|
||||||
AW_DLP_POLICY_ENGINE_BIND_HOST={{ aw_dlp_policy_engine_bind_host }}
|
AW_DLP_POLICY_ENGINE_BIND_HOST={{ aw_dlp_policy_engine_bind_host }}
|
||||||
AW_DLP_POLICY_ENGINE_PORT={{ aw_dlp_policy_engine_port }}
|
AW_DLP_POLICY_ENGINE_PORT={{ aw_dlp_policy_engine_port }}
|
||||||
@@ -717,8 +766,8 @@
|
|||||||
AW_EXPECT_ALWAYS_ACTIVE_PATTERN={{ aw_server_always_active_pattern | default('') }}
|
AW_EXPECT_ALWAYS_ACTIVE_PATTERN={{ aw_server_always_active_pattern | default('') }}
|
||||||
AW_EXPECT_LANDINGPAGE={{ aw_server_landingpage | default('') }}
|
AW_EXPECT_LANDINGPAGE={{ aw_server_landingpage | default('') }}
|
||||||
AW_HEALTH_STRICT_FILEOPS={{ aw_health_strict_fileops | default(0) }}
|
AW_HEALTH_STRICT_FILEOPS={{ aw_health_strict_fileops | default(0) }}
|
||||||
AW_MONITORED_WINDOWS_HOST={{ aw_monitored_windows_host }}
|
AW_MONITORED_WINDOWS_HOST={{ aw_effective_monitored_windows_host }}
|
||||||
AW_MONITORED_WINDOWS_HOSTNAME={{ aw_monitored_windows_hostname }}
|
AW_MONITORED_WINDOWS_HOSTNAME={{ aw_effective_monitored_windows_hostname }}
|
||||||
AW_RUS_HEALTH_WORKTIME_API={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }}
|
AW_RUS_HEALTH_WORKTIME_API={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }}
|
||||||
AW_RUS_HEALTH_STATE_DIR={{ aw_rus_health_state_dir }}
|
AW_RUS_HEALTH_STATE_DIR={{ aw_rus_health_state_dir }}
|
||||||
AW_RUS_HEALTH_VALIDATION_DIR={{ aw_rus_health_validation_dir }}
|
AW_RUS_HEALTH_VALIDATION_DIR={{ aw_rus_health_validation_dir }}
|
||||||
@@ -731,7 +780,7 @@
|
|||||||
AW_RUS_SLO_TARGET_PERCENT={{ aw_rus_slo_target_percent | default('99.97') }}
|
AW_RUS_SLO_TARGET_PERCENT={{ aw_rus_slo_target_percent | default('99.97') }}
|
||||||
AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600
|
AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600
|
||||||
AW_BROWSER_SMOKE_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }}
|
AW_BROWSER_SMOKE_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }}
|
||||||
AW_BROWSER_SMOKE_HOST={{ aw_monitored_windows_hostname }}
|
AW_BROWSER_SMOKE_HOST={{ aw_effective_monitored_windows_hostname }}
|
||||||
AW_BROWSER_SMOKE_OUTPUT_DIR={{ aw_server_data_dir }}/browser-smoke
|
AW_BROWSER_SMOKE_OUTPUT_DIR={{ aw_server_data_dir }}/browser-smoke
|
||||||
AW_BROWSER_SMOKE_ENGINE={{ aw_browser_smoke_engine | default('chromium-cli') }}
|
AW_BROWSER_SMOKE_ENGINE={{ aw_browser_smoke_engine | default('chromium-cli') }}
|
||||||
AW_BROWSER_SMOKE_TIMEOUT_MS={{ aw_browser_smoke_timeout_ms | default(20000) }}
|
AW_BROWSER_SMOKE_TIMEOUT_MS={{ aw_browser_smoke_timeout_ms | default(20000) }}
|
||||||
|
|||||||
@@ -105,6 +105,7 @@
|
|||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
EnvironmentFile=-{{ detmir_portal_env_path }}
|
EnvironmentFile=-{{ detmir_portal_env_path }}
|
||||||
|
EnvironmentFile=-/etc/detmir/detmir-check.env
|
||||||
ExecStart=/usr/local/bin/detmir-portal
|
ExecStart=/usr/local/bin/detmir-portal
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=5s
|
RestartSec=5s
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Боевая проверка DetMir Portal, 2026-06-03
|
||||||
|
|
||||||
|
Документ фиксирует end-to-end проверку портала DetMir в production-контуре без
|
||||||
|
публикации live IP, доменов, имен хостов и путей с персональными данными.
|
||||||
|
|
||||||
|
## Область проверки
|
||||||
|
|
||||||
|
- gateway entrypoint: `https://<PUBLIC_GATEWAY_FQDN>/portal/`;
|
||||||
|
- локальный backend портала на `<GATEWAY_HOST>`;
|
||||||
|
- evidence/readiness API на `<AW_SERVER_HOST>`;
|
||||||
|
- `detmir-check`, `detmir-auto`, `aw-rus-healthd`;
|
||||||
|
- вкладки портала: оператор, руководитель, владелец, инциденты ИБ, отчеты;
|
||||||
|
- API: health, summary, operator, manager, owner, incidents, reports,
|
||||||
|
workforce policy explain, readiness bundle/verify, DLP evidence.
|
||||||
|
|
||||||
|
## Найденные проблемы
|
||||||
|
|
||||||
|
1. `aw-rus-healthd` запускался с public-safe placeholder значениями Windows
|
||||||
|
target из публичных group vars. В результате healthd проверял не production
|
||||||
|
host/buckets и давал ложный FAIL.
|
||||||
|
2. `detmir-check` считал AFK bucket строго fresh даже при отсутствии активной
|
||||||
|
интерактивной сессии. Это создавало ложный FAIL для `detmir-auto` и портала.
|
||||||
|
3. `detmir-auto` и `detmir-portal` не имели единого приватного runtime env для
|
||||||
|
`detmir-check`.
|
||||||
|
4. `detmir-portal` обрабатывал HTTP-запросы последовательно. Параллельные fetch
|
||||||
|
из браузера могли блокироваться тяжелыми snapshot-route.
|
||||||
|
|
||||||
|
## Исправления
|
||||||
|
|
||||||
|
- Production runtime на AW-сервере восстановлен из приватного inventory/runtime
|
||||||
|
values; публичные placeholders не записывались в Git.
|
||||||
|
- В `deploy_aw_server.yml` добавлена защита: playbook сохраняет уже рабочие
|
||||||
|
runtime значения monitored Windows target и не принимает TEST-NET/example
|
||||||
|
значения для production healthd.
|
||||||
|
- В `detmir-check` добавлен режим `interactive_fresh` для AFK/DLP heartbeat:
|
||||||
|
при активной свежей worktime-сессии bucket обязан быть fresh; при отсутствии
|
||||||
|
активной интерактивной сессии stale/missing bucket классифицируется как
|
||||||
|
`INACTIVE` и не валит контур.
|
||||||
|
- В `detmir-check` добавлен `DETMIR_RDP_HOST` / `--rdp-host`, чтобы TCP checks
|
||||||
|
не зависели от public placeholder defaults.
|
||||||
|
- На Proxmox создан приватный `/etc/detmir/detmir-check.env`; systemd units
|
||||||
|
читают его через `EnvironmentFile`.
|
||||||
|
- `detmir-portal` переведен на concurrent request handling и короткий TTL-cache
|
||||||
|
snapshot, чтобы несколько одновременных UI/API-запросов использовали один
|
||||||
|
общий снимок состояния.
|
||||||
|
|
||||||
|
## Результаты проверки
|
||||||
|
|
||||||
|
- `aw-rus-healthd`: `ok=14`, `warn=0`, `fail=0`.
|
||||||
|
- `detmir-check`: `rc=0`, `bucket_stale=0`, `bucket_dead=0`,
|
||||||
|
`service_failures=0`.
|
||||||
|
- `detmir-auto-rust --no-heal`: `severity=OK`, `needs_heal=false`,
|
||||||
|
DLP `ok=22`, `warn=0`, `fail=0`.
|
||||||
|
- `detmir-auto.service`: `Result=success`, `ExecMainStatus=0`.
|
||||||
|
- Proxmox failed units: `0`.
|
||||||
|
- AW server failed units: `0`.
|
||||||
|
- Authenticated gateway smoke:
|
||||||
|
- `/portal/` -> `200`;
|
||||||
|
- `/portal/app.js` -> `200`;
|
||||||
|
- `/portal/api/health` -> `200`;
|
||||||
|
- `/portal/api/summary` -> `200`;
|
||||||
|
- `/portal/api/operator` -> `200`;
|
||||||
|
- `/portal/api/reports` -> `200`;
|
||||||
|
- `/portal/api/readiness/bundle` -> `200`;
|
||||||
|
- `/portal/api/readiness/verify` -> `200`;
|
||||||
|
- `/portal/api/dlp/evidence` -> `200`.
|
||||||
|
- Parallel backend API smoke: all checked endpoints returned `200`; heavy
|
||||||
|
snapshot endpoints completed from shared cache in about one snapshot window.
|
||||||
|
- Playwright browser smoke through `<PUBLIC_GATEWAY_FQDN>`:
|
||||||
|
- title: `DetMir Portal`;
|
||||||
|
- JavaScript/page errors: none;
|
||||||
|
- all checked `/portal/api/*` endpoints returned `200`;
|
||||||
|
- tabs opened: руководитель, инциденты ИБ, отчеты;
|
||||||
|
- report UI rendered `Markdown для отчета` and `Печать / PDF`;
|
||||||
|
- screenshot saved as runtime artifact outside the repository.
|
||||||
|
|
||||||
|
## Ожидаемое поведение
|
||||||
|
|
||||||
|
- Unauthenticated `/portal/*` requests return `401`; this is normal gateway
|
||||||
|
protection.
|
||||||
|
- Direct local portal API may expose only local evidence/readiness placeholders.
|
||||||
|
Production UI must access readiness/evidence through gateway routes, where
|
||||||
|
nginx proxies these paths to the AW evidence API.
|
||||||
|
- Public repository files must keep placeholders. Live endpoint values belong
|
||||||
|
only to private inventory, systemd env files, and runtime state.
|
||||||
@@ -10,6 +10,7 @@ Group=igor
|
|||||||
ExecCondition=/bin/sh -c '! systemctl -q is-active detmir-auto.service'
|
ExecCondition=/bin/sh -c '! systemctl -q is-active detmir-auto.service'
|
||||||
Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust
|
Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust
|
||||||
Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust/locks
|
Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/shadow/detmir-auto-rust/locks
|
||||||
|
EnvironmentFile=-/etc/detmir/detmir-check.env
|
||||||
Environment=no_proxy=localhost,127.0.0.1,198.51.100.18,192.0.2.13,192.0.2.2,192.0.2.0/24,198.51.100.0/24
|
Environment=no_proxy=localhost,127.0.0.1,198.51.100.18,192.0.2.13,192.0.2.2,192.0.2.0/24,198.51.100.0/24
|
||||||
Environment=NO_PROXY=localhost,127.0.0.1,198.51.100.18,192.0.2.13,192.0.2.2,192.0.2.0/24,198.51.100.0/24
|
Environment=NO_PROXY=localhost,127.0.0.1,198.51.100.18,192.0.2.13,192.0.2.2,192.0.2.0/24,198.51.100.0/24
|
||||||
ExecStart=/usr/local/bin/detmir-auto-rust --no-heal --no-report --command-timeout-seconds 180
|
ExecStart=/usr/local/bin/detmir-auto-rust --no-heal --no-report --command-timeout-seconds 180
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ Group=igor
|
|||||||
Environment=DETMIR_AUTO_HEAL=1
|
Environment=DETMIR_AUTO_HEAL=1
|
||||||
Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai
|
Environment=DETMIR_AI_STATE_DIR=/var/lib/detmir-ai
|
||||||
Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/locks
|
Environment=DETMIR_AI_RUN_DIR=/var/lib/detmir-ai/locks
|
||||||
|
EnvironmentFile=-/etc/detmir/detmir-check.env
|
||||||
ExecStart=/usr/local/bin/detmir-auto
|
ExecStart=/usr/local/bin/detmir-auto
|
||||||
TimeoutStartSec=300
|
TimeoutStartSec=300
|
||||||
Nice=5
|
Nice=5
|
||||||
|
|||||||
Reference in New Issue
Block a user