diff --git a/README.md b/README.md index 912183b..1256611 100755 --- a/README.md +++ b/README.md @@ -279,6 +279,8 @@ collectors. [docs/REVIEW_CHECKLIST_RU.md](docs/REVIEW_CHECKLIST_RU.md). - Operational validation runbook: [docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md). +- Daily maintenance runbook with optional Pollinations AI evidence assistant: + [docs/DAILY_MAINTENANCE_RU.md](docs/DAILY_MAINTENANCE_RU.md). - Operational maturity harness: [docs/OPERATIONAL_MATURITY_RU.md](docs/OPERATIONAL_MATURITY_RU.md). - Residual risks register: diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index a9c739d..730dd19 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -169,12 +169,12 @@ name = "aw-1c-ingest" version = "0.1.0" dependencies = [ "anyhow", - "calamine", "chrono", "clap", "csv", "filetime", "fs2", + "quick-xml", "regex", "reqwest", "serde", @@ -182,6 +182,7 @@ dependencies = [ "serde_yaml", "sha1", "tempfile", + "zip", ] [[package]] @@ -397,33 +398,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" -[[package]] -name = "calamine" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a3a315226fdc5b1c3e33521073e1712a05944bc0664d665ff1f6ff0396334da" -dependencies = [ - "byteorder", - "codepage", - "encoding_rs", - "log", - "quick-xml", - "serde", - "zip 0.6.6", -] - [[package]] name = "cc" version = "1.2.63" @@ -540,15 +520,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "codepage" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" -dependencies = [ - "encoding_rs", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -949,15 +920,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -1801,11 +1763,10 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ - "encoding_rs", "memchr", ] @@ -1936,7 +1897,7 @@ dependencies = [ "sha2", "tar", "tempfile", - "zip 2.4.2", + "zip", ] [[package]] @@ -2609,7 +2570,7 @@ dependencies = [ "sha2", "tar", "tempfile", - "zip 2.4.2", + "zip", ] [[package]] @@ -3455,18 +3416,6 @@ dependencies = [ "syn", ] -[[package]] -name = "zip" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" -dependencies = [ - "byteorder", - "crc32fast", - "crossbeam-utils", - "flate2", -] - [[package]] name = "zip" version = "2.4.2" diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml index 31a587e..b9e59e1 100644 --- a/adk-rust/Cargo.toml +++ b/adk-rust/Cargo.toml @@ -71,7 +71,6 @@ publish = false adk-rust = { version = "0.9.1", default-features = false } anyhow = "1" base64 = "0.22" -calamine = "=0.24.0" chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } clap = { version = "4", features = ["derive", "env"] } csv = "1" @@ -81,6 +80,7 @@ detmir-state = { path = "crates/detmir-state" } filetime = "0.2" fs2 = "0.4" notify = "6" +quick-xml = "0.41" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } regex = "1" rusqlite = "0.32" @@ -93,4 +93,5 @@ tempfile = "3" tiny_http = "0.12" url = "2" urlencoding = "2" +zip = { version = "2", default-features = false, features = ["deflate"] } windows-sys = "0.59" diff --git a/adk-rust/crates/aw-1c-ingest/Cargo.toml b/adk-rust/crates/aw-1c-ingest/Cargo.toml index 68b0849..8ba8d1c 100644 --- a/adk-rust/crates/aw-1c-ingest/Cargo.toml +++ b/adk-rust/crates/aw-1c-ingest/Cargo.toml @@ -8,12 +8,12 @@ publish = false [dependencies] anyhow.workspace = true -calamine.workspace = true chrono.workspace = true clap.workspace = true csv.workspace = true filetime.workspace = true fs2.workspace = true +quick-xml.workspace = true regex.workspace = true reqwest.workspace = true serde.workspace = true @@ -21,3 +21,4 @@ serde_json.workspace = true serde_yaml.workspace = true sha1.workspace = true tempfile.workspace = true +zip.workspace = true diff --git a/adk-rust/crates/aw-1c-ingest/src/main.rs b/adk-rust/crates/aw-1c-ingest/src/main.rs index 03cba5e..3e63c52 100644 --- a/adk-rust/crates/aw-1c-ingest/src/main.rs +++ b/adk-rust/crates/aw-1c-ingest/src/main.rs @@ -6,16 +6,19 @@ use std::process::Command; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; -use calamine::{Data, Reader, open_workbook_auto}; use chrono::{DateTime, NaiveDateTime, Utc}; use clap::Parser; use filetime::{FileTime, set_file_times}; use fs2::FileExt; +use quick_xml::Reader as XmlReader; +use quick_xml::XmlVersion; +use quick_xml::events::{BytesStart, Event}; use regex::Regex; use reqwest::blocking::Client; use serde::Deserialize; use serde_json::{Map, Value, json}; use sha1::{Digest, Sha1}; +use zip::ZipArchive; const DATASETS: &[&str] = &[ "documents", @@ -1103,15 +1106,14 @@ fn load_registry( } fn parse_registry_xlsx(path: &Path) -> Result> { - let mut workbook = - open_workbook_auto(path).with_context(|| format!("open xlsx {}", path.display()))?; + let workbook = read_xlsx_workbook(path)?; let now = Utc::now() .naive_utc() .format("%Y-%m-%d %H:%M:%S") .to_string(); let mut tax_map: HashMap = HashMap::new(); - if let Ok(range) = workbook.worksheet_range("Лист2") { - for row in range.rows().skip(2) { + if let Some(rows) = workbook.get("Лист2") { + for row in rows.iter().skip(2) { let company_name = cell_text(row.get(1)); if company_name.is_empty() { continue; @@ -1123,10 +1125,10 @@ fn parse_registry_xlsx(path: &Path) -> Result> { } } let mut out = Vec::new(); - let Ok(range) = workbook.worksheet_range("ОСНОВНОЙ") else { + let Some(sheet_rows) = workbook.get("ОСНОВНОЙ") else { return Ok(out); }; - let rows: Vec<_> = range.rows().collect(); + let rows: Vec<_> = sheet_rows.iter().collect(); if rows.len() < 3 { return Ok(out); } @@ -1134,7 +1136,8 @@ fn parse_registry_xlsx(path: &Path) -> Result> { let manager_headers: Vec = rows[1].iter().map(|v| cell_text(Some(v))).collect(); let mut specs = Vec::<(usize, String, Option, String, String)>::new(); let mut current: Option<(usize, String, String)> = None; - for idx in 1..manager_headers.len() { + let header_len = top_headers.len().max(manager_headers.len()); + for idx in 1..header_len { let manager = manager_headers.get(idx).cloned().unwrap_or_default(); let top = top_headers.get(idx).cloned().unwrap_or_default(); if !manager.is_empty() { @@ -1194,26 +1197,291 @@ fn parse_registry_xlsx(path: &Path) -> Result> { Ok(out) } -fn cell_text(value: Option<&Data>) -> String { - match value { - Some(Data::String(v)) => v.trim().to_string(), - Some(Data::Float(v)) => { - if v.fract() == 0.0 { - format!("{}", *v as i64) +type XlsxWorkbook = HashMap>>; + +fn read_xlsx_workbook(path: &Path) -> Result { + let file = File::open(path).with_context(|| format!("open xlsx {}", path.display()))?; + let mut archive = + ZipArchive::new(file).with_context(|| format!("read xlsx {}", path.display()))?; + let shared_strings = read_xlsx_shared_strings(&mut archive)?; + let sheets = read_xlsx_sheet_paths(&mut archive)?; + let mut workbook = XlsxWorkbook::new(); + for (name, sheet_path) in sheets { + let xml = read_zip_text(&mut archive, &sheet_path) + .with_context(|| format!("read sheet {name} from {}", path.display()))?; + let rows = parse_xlsx_sheet(&xml, &shared_strings) + .with_context(|| format!("parse sheet {name} from {}", path.display()))?; + workbook.insert(name, rows); + } + Ok(workbook) +} + +fn read_xlsx_shared_strings( + archive: &mut ZipArchive, +) -> Result> { + let Some(xml) = read_zip_text_optional(archive, "xl/sharedStrings.xml")? else { + return Ok(Vec::new()); + }; + let mut reader = XmlReader::from_str(&xml); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut strings = Vec::new(); + let mut in_si = false; + let mut in_t = false; + let mut current = String::new(); + loop { + match reader.read_event_into(&mut buf)? { + Event::Start(e) => match local_name(e.name().as_ref()) { + b"si" => { + in_si = true; + current.clear(); + } + b"t" if in_si => in_t = true, + _ => {} + }, + Event::Text(e) if in_si && in_t => current.push_str(e.decode()?.as_ref()), + Event::End(e) => match local_name(e.name().as_ref()) { + b"t" => in_t = false, + b"si" => { + strings.push(current.clone()); + in_si = false; + } + _ => {} + }, + Event::Eof => break, + _ => {} + } + buf.clear(); + } + Ok(strings) +} + +fn read_xlsx_sheet_paths( + archive: &mut ZipArchive, +) -> Result> { + let workbook_xml = read_zip_text(archive, "xl/workbook.xml")?; + let rels_xml = read_zip_text(archive, "xl/_rels/workbook.xml.rels")?; + let rels = parse_xlsx_relationships(&rels_xml)?; + let mut reader = XmlReader::from_str(&workbook_xml); + reader.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut sheets = Vec::new(); + loop { + match reader.read_event_into(&mut buf)? { + Event::Empty(e) | Event::Start(e) if local_name(e.name().as_ref()) == b"sheet" => { + let name = xml_attr(&reader, &e, b"name")?.unwrap_or_default(); + let mut rel_id = xml_attr(&reader, &e, b"r:id")?.unwrap_or_default(); + if rel_id.is_empty() { + rel_id = xml_attr(&reader, &e, b"id")?.unwrap_or_default(); + } + if !name.is_empty() && !rel_id.is_empty() { + let target = rels + .get(&rel_id) + .with_context(|| format!("missing workbook relationship {rel_id}"))?; + sheets.push((name, normalize_xlsx_target(target)?)); + } + } + Event::Eof => break, + _ => {} + } + buf.clear(); + } + Ok(sheets) +} + +fn parse_xlsx_relationships(xml: &str) -> Result> { + let mut reader = XmlReader::from_str(xml); + reader.config_mut().trim_text(true); + let mut buf = Vec::new(); + let mut rels = HashMap::new(); + loop { + match reader.read_event_into(&mut buf)? { + Event::Empty(e) | Event::Start(e) + if local_name(e.name().as_ref()) == b"Relationship" => + { + let id = xml_attr(&reader, &e, b"Id")?.unwrap_or_default(); + let target = xml_attr(&reader, &e, b"Target")?.unwrap_or_default(); + if !id.is_empty() && !target.is_empty() { + rels.insert(id, target); + } + } + Event::Eof => break, + _ => {} + } + buf.clear(); + } + Ok(rels) +} + +fn parse_xlsx_sheet(xml: &str, shared_strings: &[String]) -> Result>> { + let mut reader = XmlReader::from_str(xml); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut rows = Vec::>::new(); + let mut current_row = Vec::::new(); + let mut current_cell: Option<(usize, String, String)> = None; + let mut in_value = false; + let mut in_inline_text = false; + loop { + match reader.read_event_into(&mut buf)? { + Event::Empty(e) => match local_name(e.name().as_ref()) { + b"row" => rows.push(Vec::new()), + b"c" => { + let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default(); + let column = cell_ref + .chars() + .take_while(|ch| ch.is_ascii_alphabetic()) + .collect::(); + let col = xlsx_column_index(&column).unwrap_or(current_row.len()); + if current_row.len() <= col { + current_row.resize(col + 1, String::new()); + } + } + _ => {} + }, + Event::Start(e) => match local_name(e.name().as_ref()) { + b"row" => current_row.clear(), + b"c" => { + let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default(); + let cell_type = xml_attr(&reader, &e, b"t")?.unwrap_or_default(); + let column = cell_ref + .chars() + .take_while(|ch| ch.is_ascii_alphabetic()) + .collect::(); + let col = xlsx_column_index(&column).unwrap_or(current_row.len()); + current_cell = Some((col, cell_type, String::new())); + } + b"v" => in_value = true, + b"t" if current_cell + .as_ref() + .is_some_and(|(_, kind, _)| kind == "inlineStr") => + { + in_inline_text = true; + } + _ => {} + }, + Event::Text(e) if in_value || in_inline_text => { + if let Some((_, _, value)) = current_cell.as_mut() { + value.push_str(e.decode()?.as_ref()); + } + } + Event::End(e) => match local_name(e.name().as_ref()) { + b"v" => in_value = false, + b"t" => in_inline_text = false, + b"c" => { + if let Some((col, cell_type, raw)) = current_cell.take() { + let value = xlsx_cell_value(&cell_type, &raw, shared_strings); + if current_row.len() <= col { + current_row.resize(col + 1, String::new()); + } + current_row[col] = value; + } + } + b"row" => rows.push(current_row.clone()), + _ => {} + }, + Event::Eof => break, + _ => {} + } + buf.clear(); + } + Ok(rows) +} + +fn read_zip_text( + archive: &mut ZipArchive, + path: &str, +) -> Result { + read_zip_text_optional(archive, path)?.with_context(|| format!("missing xlsx entry {path}")) +} + +fn read_zip_text_optional( + archive: &mut ZipArchive, + path: &str, +) -> Result> { + let Ok(mut file) = archive.by_name(path) else { + return Ok(None); + }; + let mut text = String::new(); + file.read_to_string(&mut text) + .with_context(|| format!("read xlsx entry {path}"))?; + Ok(Some(text)) +} + +fn normalize_xlsx_target(target: &str) -> Result { + let normalized = target.replace('\\', "/"); + if normalized.split('/').any(|part| part == "..") { + bail!("unsafe xlsx relationship target: {target}"); + } + let path = normalized.trim_start_matches('/'); + if path.starts_with("xl/") { + Ok(path.to_string()) + } else { + Ok(format!("xl/{path}")) + } +} + +fn xml_attr(reader: &XmlReader<&[u8]>, e: &BytesStart<'_>, key: &[u8]) -> Result> { + for attr in e.attributes() { + let attr = attr?; + if attr.key.as_ref() == key { + return Ok(Some( + attr.decoded_and_normalized_value(XmlVersion::Implicit1_0, reader.decoder())? + .into_owned(), + )); + } + } + Ok(None) +} + +fn local_name(name: &[u8]) -> &[u8] { + name.rsplit(|b| *b == b':').next().unwrap_or(name) +} + +fn xlsx_column_index(column: &str) -> Option { + let mut value = 0usize; + for byte in column.bytes() { + if !byte.is_ascii_alphabetic() { + return None; + } + value = value * 26 + usize::from(byte.to_ascii_uppercase() - b'A' + 1); + } + value.checked_sub(1) +} + +fn xlsx_cell_value(cell_type: &str, raw: &str, shared_strings: &[String]) -> String { + match cell_type { + "s" => raw + .parse::() + .ok() + .and_then(|idx| shared_strings.get(idx)) + .cloned() + .unwrap_or_default(), + "b" => { + if raw.trim() == "1" { + "true".to_string() } else { - v.to_string() + "false".to_string() } } - Some(Data::Int(v)) => v.to_string(), - Some(Data::Bool(v)) => v.to_string(), - Some(Data::DateTime(v)) => v.to_string(), - Some(Data::DateTimeIso(v)) => v.trim().to_string(), - Some(Data::DurationIso(v)) => v.trim().to_string(), - Some(Data::Empty) | None => String::new(), - Some(other) => other.to_string(), + _ => raw.to_string(), } } +fn cell_text(value: Option<&String>) -> String { + value + .map(|v| { + let trimmed = v.trim(); + if let Ok(number) = trimmed.parse::() { + if number.fract() == 0.0 { + return format!("{}", number as i64); + } + } + trimmed.to_string() + }) + .unwrap_or_default() +} + fn archive_or_delete( config: &RawConfig, dataset: &str, @@ -1507,4 +1775,81 @@ mod tests { fn split_sql_ignores_empty_chunks() { assert_eq!(split_sql_statements("SELECT 1; ; SELECT 2;").len(), 2); } + + #[test] + fn parses_registry_xlsx_with_inline_strings() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("registry.xlsx"); + let file = File::create(&path)?; + let mut zip = zip::ZipWriter::new(file); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + for (name, xml) in [ + ( + "xl/workbook.xml", + r#" + + + + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + + +"#, + ), + ( + "xl/worksheets/sheet1.xml", + r#" + + + + + + ООО Ромашка + 1101000000 + 110101001 + + +"#, + ), + ( + "xl/worksheets/sheet2.xml", + r#" + + + + Ключевой контур + + + Иванов И.И. + + + ООО Ромашка + ЕСТЬ + + +"#, + ), + ] { + zip.start_file(name, options)?; + zip.write_all(xml.as_bytes())?; + } + zip.finish()?; + + let rows = parse_registry_xlsx(&path)?; + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["company_name"], "ООО Ромашка"); + assert_eq!(rows[0]["company_key"], "ООО РОМАШКА"); + assert_eq!(rows[0]["assignee_name"], "Иванов И.И."); + assert_eq!(rows[0]["key_contour"], 1); + assert_eq!(rows[0]["inn"], "1101000000"); + assert_eq!(rows[0]["kpp"], "110101001"); + Ok(()) + } } diff --git a/adk-rust/crates/aw-windows-telemetry/src/main.rs b/adk-rust/crates/aw-windows-telemetry/src/main.rs index 8bc6692..94a9e0e 100644 --- a/adk-rust/crates/aw-windows-telemetry/src/main.rs +++ b/adk-rust/crates/aw-windows-telemetry/src/main.rs @@ -4037,6 +4037,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime) .unwrap_or_else(|| "powershell_primary".to_string()); let worktime_legacy_fallback_enabled = json_bool(&config, &["collectors", "worktimeLegacyFallbackEnabled"]).unwrap_or(true); + let window_enabled = json_bool(&config, &["collectors", "windowEnabled"]).unwrap_or(true); let file_ops_enabled = json_bool(&config, &["collectors", "fileOpsEnabled"]).unwrap_or(true); let file_ops_mode = json_string(&config, &["collectors", "fileOpsMode"]) .unwrap_or_else(|| "powershell_primary".to_string()); @@ -4047,7 +4048,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime) args.interactive_max_age_seconds, 15, ); - let bucket_checks = vec![ + let mut bucket_checks = vec![ worktime_bucket.clone(), get_bucket_health( &api_base, @@ -4055,19 +4056,21 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime) args.interactive_max_age_seconds, 15, ), - get_bucket_health( + ]; + if window_enabled { + bucket_checks.push(get_bucket_health( &api_base, &format!("aw-watcher-window_{aw_hostname}"), args.interactive_max_age_seconds, 15, - ), - get_bucket_health( - &api_base, - &format!("aw-dlp-endpoint-signals_{aw_hostname}"), - args.interactive_max_age_seconds, - 15, - ), - ]; + )); + } + bucket_checks.push(get_bucket_health( + &api_base, + &format!("aw-dlp-endpoint-signals_{aw_hostname}"), + args.interactive_max_age_seconds, + 15, + )); let interactive_stale = bucket_checks .iter() .skip(1) diff --git a/docs/AUTONOMOUS_VALIDATION_RU.md b/docs/AUTONOMOUS_VALIDATION_RU.md new file mode 100644 index 0000000..2fa85f1 --- /dev/null +++ b/docs/AUTONOMOUS_VALIDATION_RU.md @@ -0,0 +1,192 @@ +# Автономная проверка AWatch-rus + +Документ описывает единый validation entry point для длительной +эксплуатационной проверки AWatch-rus/DetMir без ручного надзора. + +Цель framework - не менять production runtime, а регулярно доказывать состояние +репозитория, release artifacts, operational maturity, runtime health и evidence. + +## Запуск + +Из корня репозитория: + +```bash +scripts/run_full_validation.sh --profile full +``` + +Профили: + +- `quick` - repository, JSON/YAML/Markdown, retention/recovery docs, secret and + contract checks. +- `standard` - `quick` плюс operational maturity, deployment readiness, pilot + validation, release artifact checks, binary parity self-test. +- `full` - `standard` плюс Rust, Cargo и dependency hygiene gates. +- `runtime` - `standard` с акцентом на runtime diagnostics и health endpoints. + +Для unattended запуска можно задать каталог reports: + +```bash +AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation \ +AW_VALIDATION_RETENTION_DAYS=45 \ +scripts/run_full_validation.sh --profile standard +``` + +## Production evidence inputs + +Framework не собирает production binary parity evidence самостоятельно. Это +намеренное ограничение: оператор сначала формирует approved evidence JSON, затем +gate проверяет его в репозитории. + +```bash +PRODUCTION_BINARY_PARITY_EVIDENCE=/var/lib/activitywatch/evidence/production-binary-parity.json \ +scripts/run_full_validation.sh --profile standard +``` + +Release evidence проверяется только когда явно указан каталог evidence package: + +```bash +AW_RELEASE_EVIDENCE_DIR=/var/lib/activitywatch/release-evidence/rc-1 \ +scripts/run_full_validation.sh --profile standard +``` + +## Runtime diagnostics + +Каждый запуск собирает machine-readable диагностику: + +- CPU/load average; +- RAM/swap; +- disk usage; +- размеры известных runtime paths; +- queue directories; +- systemd unit state, restart count, exit status; +- ActivityWatch, Worktime, Grafana, Prometheus, ClickHouse endpoints. +- TLS certificate expiry for hosts listed in `AW_VALIDATION_TLS_HOSTS`. + +Endpoint defaults безопасны для локального запуска: + +- `AW_VALIDATION_ACTIVITYWATCH_URL`, default + `http://127.0.0.1:5600/api/0/info`; +- `AW_VALIDATION_WORKTIME_URL`, default `http://127.0.0.1:5610/healthz`; +- `AW_VALIDATION_GRAFANA_URL`; +- `AW_VALIDATION_PROMETHEUS_URL`; +- `AW_VALIDATION_CLICKHOUSE_URL`. +- `AW_VALIDATION_TLS_HOSTS`, comma-separated `host:port` list. + +ActivityWatch и Worktime по умолчанию проверяются на localhost. Grafana, +Prometheus, ClickHouse и TLS hosts проверяются только если заданы. Если заданный +endpoint недоступен, создается machine-readable alert. + +## Reports + +Каждый запуск пишет reports в: + +```text +output/validation/history// +output/validation/latest/ +``` + +Файлы: + +- `validation-report.json`; +- `validation-report.md`; +- `production-health.json`; +- `deployment-health.json`; +- `runtime-health.json`; +- `release-evidence.json`; +- `operational-evidence.json`; +- `recovery-evidence.json`; +- `build-evidence.json`; +- `operational-dashboard.json`; +- `operational-dashboard.md`. + +Все JSON reports содержат timestamp, Git SHA, host identifier, profile, +validation outcome и evidence summary. + +## Regression detection + +Framework сравнивает текущий запуск с предыдущим report из history и фиксирует: + +- новые failures; +- восстановленные failures; +- ухудшение длительности checks; +- изменение `Cargo.lock`; +- configuration drift; +- unexpected release artifact changes; +- disk/database/log path growth above threshold; +- новые alerts. + +Первый запуск создает baseline. Дальнейшие запуски пишут regression summary в +`validation-report.json` и `validation-report.md`. + +## Alert rules + +Alerts создаются при: + +- failed validation; +- недоступном настроенном health endpoint; +- превышении disk threshold; +- превышении memory threshold; +- failed systemd unit; +- restart count больше нуля; +- queue backlog выше threshold; +- отсутствии production binary parity evidence; +- отсутствии release evidence directory; +- expired или missing backup evidence path, если он задан; +- secret pattern check failure; +- configuration drift; +- unexpected artifact changes; +- certificate expiry within `AW_VALIDATION_CERT_EXPIRY_WARN_DAYS`. + +Thresholds: + +```bash +AW_VALIDATION_DISK_USED_WARN_PCT=85 +AW_VALIDATION_DISK_USED_CRIT_PCT=95 +AW_VALIDATION_MEMORY_USED_WARN_PCT=90 +AW_VALIDATION_MEMORY_USED_CRIT_PCT=97 +AW_VALIDATION_CERT_EXPIRY_WARN_DAYS=14 +AW_VALIDATION_PATH_GROWTH_WARN_BYTES=1073741824 +AW_VALIDATION_PATH_GROWTH_WARN_RATIO=1.25 +AW_VALIDATION_QUEUE_FILES_WARN=1000 +AW_VALIDATION_BACKUP_EVIDENCE_PATH=/var/lib/activitywatch/evidence/latest-backup.json +AW_VALIDATION_BACKUP_EVIDENCE_MAX_AGE_DAYS=7 +``` + +## Scheduling + +Пример systemd timer должен выполняться вне hot path и не должен включать DLP, +Loki или always-on Velociraptor: + +```ini +[Service] +Type=oneshot +WorkingDirectory=/mnt/usb_hdd2/Projects/ActivityWatch-Russian +Environment=AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation +Environment=AW_VALIDATION_RETENTION_DAYS=45 +ExecStart=/mnt/usb_hdd2/Projects/ActivityWatch-Russian/scripts/run_full_validation.sh --profile standard +``` + +Для weekly deep validation использовать `--profile full`, потому что Rust и +dependency gates могут быть дорогими. + +## Failure investigation + +1. Открыть `output/validation/latest/validation-report.md`. +2. Проверить `Alerts`. +3. Для failed command смотреть `validation-report.json`: `stdout`, `stderr`, + `exit_code`, `timed_out`. +4. Если alert относится к runtime endpoint, проверить соответствующий service + через runbook [эксплуатационной проверки](OPERATIONS_VALIDATION_RUNBOOK_RU.md). +5. Если alert относится к binary parity, обновить production evidence JSON и + повторить запуск. +6. Если alert относится к release evidence, проверить controlled release package + через registry runbook. + +## History retention + +История хранится в `history//`. Старые запуски удаляются по +`AW_VALIDATION_RETENTION_DAYS` или `--retention-days`. + +Минимальное значение retention: 1 день. Для production рекомендуется 30-45 +дней, чтобы видеть недельные тренды по runtime failures, restart frequency, +disk growth и validation score. diff --git a/docs/DAILY_MAINTENANCE_RU.md b/docs/DAILY_MAINTENANCE_RU.md new file mode 100644 index 0000000..08f28b7 --- /dev/null +++ b/docs/DAILY_MAINTENANCE_RU.md @@ -0,0 +1,310 @@ +# Ежедневное обслуживание AWatch-rus + +Дата актуализации: 2026-07-02 + +Документ описывает ежедневный операторский цикл обслуживания AWatch-rus / +DetMir и безопасное использование Pollinations AI как вспомогательного +аналитика evidence-артефактов. Pollinations не является production runtime +dependency, не принимает решения автоматически и не имеет прямого доступа к +customer contour. + +## Цели + +- Подтвердить, что сервисы отвечают и не находятся в скрытой деградации. +- Проверить свежесть данных, состояние очередей, timers, retention и readiness + evidence. +- Сформировать короткий дневной maintenance summary. +- Использовать Pollinations только для сжатия и первичного анализа локальных + JSON/Markdown evidence без секретов. + +## Guardrails + +- Не включать heavy DLP, Loki или always-on Velociraptor в рамках ежедневного + обслуживания. +- DLP runtime для DetMir production считать нормальным в light/disabled profile, + если оператор отдельно не принял решение о включении. +- Не удалять ClickHouse, Grafana, DLP evidence, Hayabusa archives, Windows + queues, release evidence и diagnostic bundles вручную без отдельного change. +- Pollinations failure не является отказом AWatch-rus. Если AI недоступен, + ежедневное обслуживание продолжается по обычному runbook. +- Не отправлять в Pollinations secrets, private env files, bearer tokens, + пароли, raw customer evidence, user identifiers, скриншоты с ПДн или полные + журналы production. + +## Ежедневный цикл + +### 1. Быстрый статус сервисов + +Проверить локальный health/readiness и failed units: + +```bash +curl -fsS http://127.0.0.1:5600/api/0/info >/dev/null +curl -fsS http://127.0.0.1:5610/healthz >/dev/null +systemctl --failed --no-pager +systemctl list-timers \ + aw-prune-local-state.timer \ + aw-db-maintenance.timer \ + aw-db-vacuum.timer \ + detmir-readiness.timer +``` + +Критерий: ActivityWatch и Worktime API отвечают, нет неожиданных failed units, +ежедневные timers присутствуют в расписании. + +### 2. Readiness bundle + +Проверить, что ежедневный bundle сформирован и проверяется: + +```bash +cd /var/lib/activitywatch/health/readiness-bundle +sha256sum -c sha256sums.txt +openssl dgst -sha256 -verify public-key.pem \ + -signature sha256sums.txt.sig sha256sums.txt +jq -r '.status // .overall_status // empty' detmir-readiness-status.json +``` + +Критерий: checksum/signature проходят, статус не `FAIL`, дата bundle относится +к текущему maintenance window. + +### 3. Свежесть telemetry и reports + +Проверить штатные smoke/validation команды из репозитория: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +node scripts/operational-maturity-check.mjs --json +node scripts/pilot-validation-smoke.mjs +node scripts/deployment-readiness-smoke.mjs +``` + +Если включён автономный validation orchestrator, использовать его как единый +источник daily evidence: + +```bash +./scripts/run_full_validation.sh --profile standard +``` + +Критерий: нет новых hard-fail проверок. Warning допустим только при известном +и задокументированном operational constraint. + +### 4. Очереди, backlog и storage growth + +Проверить отсутствие неконтролируемого роста: + +```bash +du -sh /var/lib/activitywatch /var/log/activitywatch 2>/dev/null || true +docker system df 2>/dev/null || true +docker exec aw-rus-1c-clickhouse clickhouse-client --query \ + "SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" \ + 2>/dev/null || true +``` + +Для Windows queues не выполнять ручную очистку. Если backlog растёт, сначала +проверить доступность AW server, scheduled tasks и upload logs. + +### 5. Retention safety + +Проверить dry-run cleanup, если есть подозрение на рост диска или после +изменения retention-конфигурации: + +```bash +sudo AW_DATA_DIR=/var/lib/activitywatch \ + AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache \ + AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400 \ + /usr/local/bin/aw-prune-local-state-rust --json +``` + +Apply не является ежедневной обязательной операцией. Apply выполнять только +после просмотра planned items и подтверждения, что cleanup не затрагивает +configuration, dashboards, forensic/security evidence, release evidence или +Windows queues. + +### 6. Короткая запись результата + +Сохранить дневной summary в operator journal или internal change log: + +```text +date: +operator: +health: +readiness_bundle: +telemetry_freshness: +storage: +retention: +new_alerts: +known_constraints: +actions_taken: +``` + +## Pollinations AI assistant + +### Роль + +Pollinations используется только как локально запускаемый помощник для: + +- сжатия daily evidence в операторский summary; +- выделения новых failures/warnings из JSON/Markdown reports; +- подготовки списка вопросов для ручной проверки; +- сравнения текущего maintenance summary с предыдущим summary. + +Pollinations не должен: + +- запускать production commands самостоятельно; +- получать secrets/private env/raw logs; +- менять конфигурацию; +- принимать решение о cleanup apply, DLP enablement, restart или rollback. + +### Проверка доступности CLI + +```bash +/home/igor/.local/bin/polli-chat --help +/home/igor/.local/bin/polli-agent --help +/home/igor/.local/bin/polli-chat --max-tokens 16 'ping' +``` + +Для `polli-agent` использовать `--collection`, не `--db`: + +```bash +/home/igor/.local/bin/polli-agent index \ + --collection /tmp/awatch-maintenance-index \ + /path/to/sanitized-evidence.md \ + /path/to/sanitized-validation-report.json + +/home/igor/.local/bin/polli-agent ask \ + --collection /tmp/awatch-maintenance-index \ + --context-only \ + 'Какие новые failures или warnings есть в сегодняшнем evidence?' +``` + +### Sanitization перед AI-анализом + +Перед передачей в Pollinations оставить только технические поля: + +- status/outcome; +- timestamps; +- Git SHA/version; +- service names; +- check names; +- counters; +- sizes; +- non-sensitive error class; +- documented operational constraints. + +Удалить или заменить placeholders: + +- usernames and user identifiers; +- tokens/passwords/API keys; +- Basic Auth credentials; +- private IP details, если summary планируется публиковать; +- screenshots and raw event payloads; +- full journal lines with command arguments. + +### Prompt для daily summary + +```text +Ты анализируешь sanitized AWatch-rus daily maintenance evidence. +Не придумывай факты. Не предлагай включать DLP/Loki/always-on Velociraptor. +Раздели вывод на: +1. OK +2. New warnings +3. New failures +4. Evidence gaps +5. Required operator actions +Если evidence недостаточно, напиши "не доказано". +``` + +### Fail-closed правило + +Если Pollinations summary расходится с raw evidence, считать источником истины +raw evidence и runbook. AI-вывод можно использовать только как черновик для +операторского анализа. + +## Журнал выполненного обслуживания + +### 2026-07-03: production cleanup, collector guard и dependency hardening + +Выполнено: + +- очищены старые ActivityWatch SQLite backup files на AW server с сохранением + последних двух rollback-точек; +- `/` на AW server снижен с `85%` до `40%`, свободно `19G`; +- заменен Windows production binary + `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe`; +- заменен gateway/ClickHouse production binary + `/usr/local/bin/aw-1c-ingest-rust`; +- закрыт RustSec риск `quick-xml` через удаление `calamine` из + `aw-1c-ingest` и переход на прямое чтение XLSX через `zip` + + `quick-xml 0.41.0`; +- добавлен regression test на разбор registry XLSX листов `Лист2` и + `ОСНОВНОЙ`. + +Production checksums: + +| Binary | SHA256 | +|---|---| +| `aw-windows-telemetry.exe` | `A8F517AB636C8537413201B84EA6540CB198045F388FECE6C8D6810537FBDDF1` | +| `aw-1c-ingest-rust` | `e0c4e665ea8946ba9c243127a65fd69554cacdc4e3191ec798b8ca19acf89623` | + +Rollback files: + +- `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe.bak.20260702T202626Z`; +- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T211637Z`; +- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T212258Z`. + +Post-change production evidence: + +- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`; +- `aw-rus-healthd`: `ok=True`; +- AW server `systemctl --failed`: `0 loaded units listed`; +- `pve-detmir systemctl --failed`: `0 loaded units listed`; +- `aw-1c-ingest.service`: `Result=success`, `ExecMainStatus=0`, + `files_loaded=8`, `rows_loaded=561`; +- ClickHouse container `aw-rus-1c-clickhouse`: `healthy`; +- gateway `/healthz`: `ok`. + +Validation executed before documenting: + +```bash +cargo fmt --all --check +cargo test --workspace --locked +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo audit --no-fetch +cargo deny --manifest-path adk-rust/Cargo.toml check --config deny.toml --hide-inclusion-graph --show-stats +cargo metadata --locked --format-version 1 +cargo tree --duplicates --locked +cargo machete --with-metadata +python3 scripts/public_secret_pattern_check.py +node scripts/operational-maturity-check.mjs --json +git diff --check +``` + +Cargo/RustSec network note: if direct `https://index.crates.io` is unavailable +from the laptop, use the local HTTP CONNECT proxy: + +```bash +export HTTP_PROXY=http://127.0.0.1:10808 +export HTTPS_PROXY=http://127.0.0.1:10808 +export http_proxy=http://127.0.0.1:10808 +export https_proxy=http://127.0.0.1:10808 +unset ALL_PROXY all_proxy +``` + +## Эскалация + +Эскалировать как production issue, если: + +- `/healthz`, `/readyz` или Worktime reports не отвечают; +- readiness bundle не сформирован или не проходит checksum/signature; +- есть неожиданные failed systemd units; +- storage growth продолжается после штатного retention; +- Windows queues растут и upload не восстанавливается; +- DLP/security evidence удалено, повреждено или изменилось без change; +- Pollinations обнаружил warning, который подтверждается raw evidence. + +## Связанные документы + +- [Operations Runbook](OPERATIONS_RUNBOOK_RU.md) +- [Эксплуатационная проверка контура](OPERATIONS_VALIDATION_RUNBOOK_RU.md) +- [Retention and Cleanup Policy](RETENTION_POLICY_RU.md) +- [Production readiness](PRODUCTION_READINESS_RU.md) +- [Autonomous validation](AUTONOMOUS_VALIDATION_RU.md) diff --git a/docs/OPERATIONS_RUNBOOK_RU.md b/docs/OPERATIONS_RUNBOOK_RU.md index ffb55c0..8e53755 100644 --- a/docs/OPERATIONS_RUNBOOK_RU.md +++ b/docs/OPERATIONS_RUNBOOK_RU.md @@ -2,6 +2,10 @@ Документ описывает базовые эксплуатационные проверки AWatch-rus. +Ежедневный операторский цикл обслуживания, retention safety и безопасное +использование Pollinations AI для анализа sanitized evidence описаны отдельно: +[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md). + ## Быстрая проверка Проверить доступность: diff --git a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md index 5503de1..ccb98cd 100644 --- a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md +++ b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @@ -61,6 +61,14 @@ cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian Подробности эксплуатационного maturity harness: [эксплуатационная зрелость DetMir/AWatch-rus](OPERATIONAL_MATURITY_RU.md). +Единый автономный entry point для unattended validation, runtime diagnostics, +history, regression detection, alerts и reports описан отдельно: +[автономная проверка AWatch-rus](AUTONOMOUS_VALIDATION_RU.md). + +Ежедневный maintenance-порядок с optional Pollinations AI assistant для +sanitized evidence summaries описан в отдельном runbook: +[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md). + ## Production binary parity gate Production binary parity gate доказывает, что production unit/timer/task diff --git a/scripts/full-validation-orchestrator.mjs b/scripts/full-validation-orchestrator.mjs new file mode 100644 index 0000000..ecdb902 --- /dev/null +++ b/scripts/full-validation-orchestrator.mjs @@ -0,0 +1,1530 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import https from "node:https"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import tls from "node:tls"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const defaultOutputDir = path.join(root, "output", "validation"); +const maxCapturedBytes = 256 * 1024; +const criticalAlertSeverities = new Set(["critical"]); +const failureStatuses = new Set(["fail"]); + +function utcNow() { + return new Date().toISOString().replace(/\.\d{3}Z$/, "Z"); +} + +function stampForPath(value) { + return value.replaceAll(":", "").replaceAll("-", "").replace("T", "T").replace("Z", "Z"); +} + +function parseArgs(argv) { + const args = { + profile: process.env.AW_VALIDATION_PROFILE || "full", + outputDir: process.env.AW_VALIDATION_OUTPUT_DIR || defaultOutputDir, + retentionDays: Number.parseInt(process.env.AW_VALIDATION_RETENTION_DAYS || "30", 10), + commandTimeoutSeconds: Number.parseInt(process.env.AW_VALIDATION_COMMAND_TIMEOUT_SECONDS || "900", 10), + endpointTimeoutSeconds: Number.parseInt(process.env.AW_VALIDATION_ENDPOINT_TIMEOUT_SECONDS || "5", 10), + productionEvidence: process.env.PRODUCTION_BINARY_PARITY_EVIDENCE || "", + releaseEvidenceDir: process.env.AW_RELEASE_EVIDENCE_DIR || "", + releaseAssetDir: process.env.AW_RELEASE_ASSET_DIR || "", + json: false, + selfTest: false, + allowFailures: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => { + index += 1; + if (index >= argv.length) throw new Error(`${arg} requires a value`); + return argv[index]; + }; + if (arg === "--profile") args.profile = next(); + else if (arg === "--output-dir") args.outputDir = path.resolve(next()); + else if (arg === "--retention-days") args.retentionDays = Number.parseInt(next(), 10); + else if (arg === "--command-timeout-seconds") args.commandTimeoutSeconds = Number.parseInt(next(), 10); + else if (arg === "--endpoint-timeout-seconds") args.endpointTimeoutSeconds = Number.parseInt(next(), 10); + else if (arg === "--production-evidence") args.productionEvidence = path.resolve(next()); + else if (arg === "--release-evidence-dir") args.releaseEvidenceDir = path.resolve(next()); + else if (arg === "--release-asset-dir") args.releaseAssetDir = path.resolve(next()); + else if (arg === "--json") args.json = true; + else if (arg === "--self-test") args.selfTest = true; + else if (arg === "--allow-failures") args.allowFailures = true; + else if (arg === "--help" || arg === "-h") args.help = true; + else throw new Error(`unsupported argument: ${arg}`); + } + + if (!["quick", "standard", "full", "runtime"].includes(args.profile)) { + throw new Error(`unsupported --profile=${args.profile}; expected quick, standard, full or runtime`); + } + if (!Number.isFinite(args.retentionDays) || args.retentionDays < 1) { + throw new Error("--retention-days must be a positive integer"); + } + if (!Number.isFinite(args.commandTimeoutSeconds) || args.commandTimeoutSeconds < 5) { + throw new Error("--command-timeout-seconds must be at least 5"); + } + if (!Number.isFinite(args.endpointTimeoutSeconds) || args.endpointTimeoutSeconds < 1) { + throw new Error("--endpoint-timeout-seconds must be at least 1"); + } + return args; +} + +function usage() { + return `Usage: + scripts/run_full_validation.sh [options] + node scripts/full-validation-orchestrator.mjs [options] + +Options: + --profile quick|standard|full|runtime + quick: repository, docs, schema and self-test checks + standard: quick + operational/release validation wrappers + full: standard + Rust/dependency gates + runtime: standard + runtime diagnostics focus + --output-dir default: output/validation + --retention-days default: 30 + --production-evidence production binary parity evidence + --release-evidence-dir release evidence package to verify + --release-asset-dir signed release assets directory to verify + --command-timeout-seconds default: 900 + --endpoint-timeout-seconds default: 5 + --json print compact JSON summary + --allow-failures exit 0 after writing reports + --self-test validate the orchestrator itself + +Environment: + AW_VALIDATION_PROFILE, AW_VALIDATION_OUTPUT_DIR, + AW_VALIDATION_RETENTION_DAYS, PRODUCTION_BINARY_PARITY_EVIDENCE, + AW_RELEASE_EVIDENCE_DIR, AW_RELEASE_ASSET_DIR, + AW_VALIDATION_ACTIVITYWATCH_URL, AW_VALIDATION_WORKTIME_URL, + AW_VALIDATION_GRAFANA_URL, AW_VALIDATION_PROMETHEUS_URL, + AW_VALIDATION_CLICKHOUSE_URL, AW_VALIDATION_SYSTEMD_UNITS, + AW_VALIDATION_QUEUE_DIRS, AW_VALIDATION_SIZE_PATHS, + AW_VALIDATION_TLS_HOSTS +`; +} + +function commandExists(name) { + if (name.includes(path.sep)) { + try { + fs.accessSync(name, fs.constants.X_OK); + return true; + } catch { + return false; + } + } + const dirs = String(process.env.PATH || "").split(path.delimiter); + const extensions = process.platform === "win32" ? ["", ".exe", ".cmd", ".bat"] : [""]; + for (const dir of dirs) { + for (const ext of extensions) { + const candidate = path.join(dir, `${name}${ext}`); + try { + fs.accessSync(candidate, fs.constants.X_OK); + return true; + } catch { + // continue + } + } + } + return false; +} + +function readTextIfExists(file) { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return ""; + } +} + +function truncate(value) { + const text = String(value || ""); + if (Buffer.byteLength(text, "utf8") <= maxCapturedBytes) return text; + const buffer = Buffer.from(text, "utf8"); + return `\n${buffer.subarray(buffer.length - maxCapturedBytes).toString("utf8")}`; +} + +function normalizeRelative(file) { + return path.relative(root, file).replaceAll(path.sep, "/"); +} + +function splitEnvList(value) { + return String(value || "") + .split(/[,\n]/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function sha256Bytes(data) { + return crypto.createHash("sha256").update(data).digest("hex"); +} + +function sha256File(file) { + return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +function mkdirp(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function writeJson(file, data) { + mkdirp(path.dirname(file)); + fs.writeFileSync(file, `${JSON.stringify(data, null, 2)}\n`, "utf8"); +} + +function writeText(file, data) { + mkdirp(path.dirname(file)); + fs.writeFileSync(file, data, "utf8"); +} + +function runCommand({ id, category, command, args = [], cwd = root, timeoutSeconds, env = {}, optional = false }) { + const started = Date.now(); + const cmdline = [command, ...args].join(" "); + if (!commandExists(command)) { + return Promise.resolve({ + id, + category, + status: optional ? "skipped" : "fail", + ok: false, + command: cmdline, + cwd: normalizeRelative(cwd), + duration_ms: 0, + skipped_reason: `tool_not_found:${command}`, + stdout: "", + stderr: "", + }); + } + + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd, + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + setTimeout(() => { + if (!child.killed) child.kill("SIGKILL"); + }, 2000).unref(); + }, timeoutSeconds * 1000); + + child.stdout.on("data", (chunk) => { + stdout = truncate(stdout + chunk.toString("utf8")); + }); + child.stderr.on("data", (chunk) => { + stderr = truncate(stderr + chunk.toString("utf8")); + }); + child.on("error", (error) => { + clearTimeout(timeout); + resolve({ + id, + category, + status: optional ? "skipped" : "fail", + ok: false, + command: cmdline, + cwd: normalizeRelative(cwd), + duration_ms: Date.now() - started, + error: error.message, + stdout, + stderr, + }); + }); + child.on("close", (code, signal) => { + clearTimeout(timeout); + const ok = code === 0 && !timedOut; + resolve({ + id, + category, + status: ok ? "pass" : "fail", + ok, + command: cmdline, + cwd: normalizeRelative(cwd), + exit_code: code, + signal, + timed_out: timedOut, + duration_ms: Date.now() - started, + stdout: truncate(stdout), + stderr: truncate(stderr), + }); + }); + }); +} + +async function runInternalCheck(id, category, fn) { + const started = Date.now(); + try { + const details = (await fn()) || {}; + return { + id, + category, + status: details.status || "pass", + ok: details.status ? details.status === "pass" : true, + duration_ms: Date.now() - started, + ...details, + }; + } catch (error) { + return { + id, + category, + status: "fail", + ok: false, + duration_ms: Date.now() - started, + error: error.message, + }; + } +} + +function gitOutput(args, fallback = "") { + if (!commandExists("git")) return fallback; + const result = spawnSyncText("git", args, root, 30); + return result.status === 0 ? result.stdout.trim() : fallback; +} + +function spawnSyncText(command, args, cwd, timeoutSeconds) { + const started = Date.now(); + const result = spawn(command, args, { + cwd, + stdio: ["ignore", "pipe", "pipe"], + env: process.env, + }); + return new Promise((resolve) => { + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + result.kill("SIGTERM"); + }, timeoutSeconds * 1000); + result.stdout.on("data", (chunk) => { + stdout = truncate(stdout + chunk.toString("utf8")); + }); + result.stderr.on("data", (chunk) => { + stderr = truncate(stderr + chunk.toString("utf8")); + }); + result.on("close", (status) => { + clearTimeout(timeout); + resolve({ status, stdout, stderr, duration_ms: Date.now() - started }); + }); + }); +} + +async function collectGitContext() { + const rev = commandExists("git") ? await spawnSyncText("git", ["rev-parse", "HEAD"], root, 30) : null; + const describe = commandExists("git") ? await spawnSyncText("git", ["describe", "--tags", "--always", "--dirty"], root, 30) : null; + const branch = commandExists("git") ? await spawnSyncText("git", ["branch", "--show-current"], root, 30) : null; + return { + git_sha: rev?.status === 0 ? rev.stdout.trim() : "unknown", + git_describe: describe?.status === 0 ? describe.stdout.trim() : "unknown", + git_branch: branch?.status === 0 ? branch.stdout.trim() : "unknown", + }; +} + +async function trackedFiles() { + if (!commandExists("git")) { + return walk(root) + .map((file) => normalizeRelative(file)) + .filter((file) => !file.startsWith("output/") && !file.startsWith("dist/")); + } + const result = await spawnSyncText("git", ["ls-files", "-z"], root, 60); + if (result.status !== 0) return []; + return result.stdout.split("\0").filter(Boolean); +} + +function walk(dir) { + const found = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if ([".git", "output", "dist", "target", "node_modules"].includes(entry.name)) continue; + const absolute = path.join(dir, entry.name); + if (entry.isDirectory()) found.push(...walk(absolute)); + else found.push(absolute); + } + return found; +} + +async function validateJsonFiles(files) { + const findings = []; + let checked = 0; + for (const file of files.filter((item) => item.endsWith(".json"))) { + try { + JSON.parse(fs.readFileSync(path.join(root, file), "utf8")); + checked += 1; + } catch (error) { + findings.push({ file, error: error.message }); + } + } + return { + checked, + findings, + status: findings.length === 0 ? "pass" : "fail", + ok: findings.length === 0, + }; +} + +function parseMarkdownLinks(text) { + const links = []; + const pattern = /!?\[[^\]]+\]\(([^)]+)\)/g; + let match = null; + while ((match = pattern.exec(text)) !== null) { + links.push(String(match[1] || "").trim()); + } + return links; +} + +async function validateMarkdownLinks(files) { + const findings = []; + let checked = 0; + for (const file of files.filter((item) => item.endsWith(".md"))) { + const text = fs.readFileSync(path.join(root, file), "utf8"); + for (const raw of parseMarkdownLinks(text)) { + if (raw.includes("")) continue; + let target = raw.split(/\s+/)[0].replace(/^<|>$/g, ""); + if ( + !target + || target.startsWith("#") + || target.startsWith("http://") + || target.startsWith("https://") + || target.startsWith("mailto:") + || target.includes("") + || target.startsWith("file:") + ) { + continue; + } + target = target.split("#", 1)[0]; + if (!target) continue; + const resolved = path.resolve(root, path.dirname(file), target); + const wikiResolved = path.resolve(root, path.dirname(file), `${target}.md`); + checked += 1; + if (!fs.existsSync(resolved) && !fs.existsSync(wikiResolved)) findings.push({ file, target }); + } + } + return { + checked, + findings, + status: findings.length === 0 ? "pass" : "fail", + ok: findings.length === 0, + }; +} + +async function validateYamlFiles(files, timeoutSeconds) { + const yamlFiles = files.filter((item) => item.endsWith(".yml") || item.endsWith(".yaml")); + if (yamlFiles.length === 0) return { checked: 0, status: "pass", ok: true }; + if (commandExists("ruby")) { + const script = "require 'psych'; ARGV.each { |f| Psych.parse_file(f) }"; + return runCommand({ + id: "config.yaml_parse", + category: "configuration", + command: "ruby", + args: ["-e", script, ...yamlFiles], + timeoutSeconds, + optional: false, + }); + } + return { + status: "skipped", + ok: false, + checked: 0, + skipped_reason: "ruby_not_available_for_yaml_parse", + }; +} + +function commandDefinitions(args) { + const timeout = args.commandTimeoutSeconds; + const adkRust = path.join(root, "adk-rust"); + const defs = [ + { id: "repository.git_diff_check", category: "repository", command: "git", args: ["diff", "--check"], timeoutSeconds: timeout }, + { id: "security.secret_pattern_check", category: "security", command: "python3", args: ["scripts/public_secret_pattern_check.py"], timeoutSeconds: timeout }, + { id: "repository.private_config_guard", category: "security", command: "bash", args: ["scripts/check_private_config_guard.sh"], timeoutSeconds: timeout }, + { id: "contracts.portal_contract_sync", category: "configuration", command: "node", args: ["scripts/check_portal_contract_sync.mjs"], timeoutSeconds: timeout }, + ]; + + if (["standard", "full", "runtime"].includes(args.profile)) { + defs.push( + { id: "operational.operational_maturity", category: "operational", command: "node", args: ["scripts/operational-maturity-check.mjs", "--json"], timeoutSeconds: timeout }, + { id: "operational.deployment_readiness_smoke", category: "deployment", command: "node", args: ["scripts/deployment-readiness-smoke.mjs"], timeoutSeconds: timeout }, + { id: "operational.pilot_validation_smoke", category: "operational", command: "node", args: ["scripts/pilot-validation-smoke.mjs"], timeoutSeconds: timeout }, + { id: "repository.production_inventory_placeholder_self_test", category: "configuration", command: "bash", args: ["scripts/check_production_inventory_placeholders.sh", "--self-test"], timeoutSeconds: timeout, optional: false }, + { id: "release.release_assets_self_test", category: "release", command: "bash", args: ["scripts/verify_release_assets.sh", "--self-test"], timeoutSeconds: timeout, optional: true }, + { id: "release.rust_release_artifacts", category: "release", command: "bash", args: ["scripts/check_detmir_rust_release_artifacts.sh"], timeoutSeconds: timeout, optional: false }, + { id: "release.binary_parity_self_test", category: "release", command: "python3", args: ["scripts/check_production_binary_parity.py", "--self-test"], timeoutSeconds: timeout, optional: false }, + ); + if (process.env.DETMIR_PRODUCTION_CONFIG_PATHS) { + defs.push({ + id: "repository.production_inventory_placeholders", + category: "configuration", + command: "bash", + args: ["scripts/check_production_inventory_placeholders.sh", "--strict"], + timeoutSeconds: timeout, + optional: false, + }); + } + if (args.productionEvidence) { + defs.push({ + id: "release.production_binary_parity", + category: "release", + command: "python3", + args: ["scripts/check_production_binary_parity.py", "--evidence", args.productionEvidence], + timeoutSeconds: timeout, + optional: false, + }); + } + if (args.releaseEvidenceDir) { + defs.push({ + id: "release.release_evidence_check", + category: "release", + command: "bash", + args: ["scripts/check_release_evidence.sh", args.releaseEvidenceDir], + timeoutSeconds: timeout, + optional: false, + }); + } + if (args.releaseAssetDir) { + defs.push({ + id: "release.release_asset_check", + category: "release", + command: "bash", + args: ["scripts/verify_release_assets.sh", args.releaseAssetDir], + timeoutSeconds: timeout, + optional: false, + }); + } + } + + if (args.profile === "full") { + defs.push( + { id: "rust.cargo_metadata", category: "rust", command: "cargo", args: ["metadata", "--locked", "--format-version", "1"], cwd: adkRust, timeoutSeconds: timeout }, + { id: "rust.cargo_fmt", category: "rust", command: "cargo", args: ["fmt", "--all", "--check"], cwd: adkRust, timeoutSeconds: timeout }, + { id: "rust.cargo_clippy", category: "rust", command: "cargo", args: ["clippy", "--workspace", "--all-targets", "--all-features", "--", "-D", "warnings"], cwd: adkRust, timeoutSeconds: timeout }, + { id: "rust.cargo_test", category: "rust", command: "cargo", args: ["test", "--workspace", "--locked"], cwd: adkRust, timeoutSeconds: timeout }, + { id: "dependency.cargo_tree_duplicates", category: "dependency", command: "cargo", args: ["tree", "--duplicates", "--locked"], cwd: adkRust, timeoutSeconds: timeout, optional: true }, + { id: "dependency.cargo_audit", category: "dependency", command: "cargo", args: ["audit"], cwd: adkRust, timeoutSeconds: timeout, optional: true }, + { id: "dependency.cargo_deny", category: "dependency", command: "cargo", args: ["deny", "check", "--config", "../deny.toml"], cwd: adkRust, timeoutSeconds: timeout, optional: true }, + { id: "dependency.cargo_machete", category: "dependency", command: "cargo", args: ["machete", "--with-metadata"], cwd: adkRust, timeoutSeconds: timeout, optional: true }, + ); + } + + return defs; +} + +function summarizeCheckForEvidence(check) { + return { + id: check.id, + category: check.category, + status: check.status, + exit_code: check.exit_code, + duration_ms: check.duration_ms, + skipped_reason: check.skipped_reason, + error: check.error, + }; +} + +function collectDiskUsage() { + const disks = []; + const lines = readTextIfExists("/proc/mounts").split("\n").filter(Boolean); + const seen = new Set(); + for (const line of lines) { + const [, mountPoint, fsType] = line.split(/\s+/); + if (!mountPoint || seen.has(mountPoint)) continue; + if (["proc", "sysfs", "tmpfs", "devtmpfs", "devpts", "cgroup", "cgroup2", "overlay", "squashfs"].includes(fsType)) continue; + if (fsType.startsWith("fuse.")) continue; + seen.add(mountPoint); + try { + const stat = fs.statfsSync(mountPoint); + const total = Number(stat.blocks) * Number(stat.bsize); + const free = Number(stat.bavail) * Number(stat.bsize); + const used = Math.max(total - free, 0); + disks.push({ + mount: mountPoint, + fs_type: fsType, + total_bytes: total, + used_bytes: used, + free_bytes: free, + used_pct: total > 0 ? Number(((used / total) * 100).toFixed(2)) : 0, + }); + } catch { + // ignore inaccessible mounts + } + } + return disks.sort((a, b) => b.used_pct - a.used_pct); +} + +function collectMemory() { + const meminfo = {}; + for (const line of readTextIfExists("/proc/meminfo").split("\n")) { + const match = line.match(/^([^:]+):\s+(\d+)\s+kB/); + if (match) meminfo[match[1]] = Number(match[2]) * 1024; + } + const total = meminfo.MemTotal || os.totalmem(); + const available = meminfo.MemAvailable || os.freemem(); + const used = Math.max(total - available, 0); + return { + total_bytes: total, + available_bytes: available, + used_bytes: used, + used_pct: total > 0 ? Number(((used / total) * 100).toFixed(2)) : 0, + swap_total_bytes: meminfo.SwapTotal || 0, + swap_free_bytes: meminfo.SwapFree || 0, + }; +} + +function collectCpu() { + const load = os.loadavg(); + return { + cpu_count: os.cpus().length, + load_1m: load[0], + load_5m: load[1], + load_15m: load[2], + }; +} + +function sizePath(target) { + try { + const stat = fs.statSync(target); + if (stat.isFile()) return stat.size; + if (!stat.isDirectory()) return 0; + let total = 0; + for (const entry of fs.readdirSync(target, { withFileTypes: true })) { + const child = path.join(target, entry.name); + if (entry.isSymbolicLink()) continue; + total += sizePath(child); + } + return total; + } catch { + return null; + } +} + +function collectPathSizes() { + const defaults = [ + "/var/lib/activitywatch", + "/var/log/activitywatch", + "/opt/activitywatch/clickhouse-1c", + "/opt/activitywatch/clickhouse-workforce", + "/opt/hayabusa", + ]; + const configured = splitEnvList(process.env.AW_VALIDATION_SIZE_PATHS); + const paths = configured.length > 0 ? configured : defaults; + return paths.map((item) => ({ + path: item, + exists: fs.existsSync(item), + size_bytes: sizePath(item), + })); +} + +function countQueueFiles() { + const defaults = [ + "/var/lib/activitywatch/queues", + "/var/lib/activitywatch/worktime-report-cache", + "/opt/activitywatch/aw-rus-ops/drop", + ]; + const dirs = splitEnvList(process.env.AW_VALIDATION_QUEUE_DIRS); + const targets = dirs.length > 0 ? dirs : defaults; + return targets.map((dir) => { + try { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + return { + path: dir, + exists: true, + files: entries.filter((entry) => entry.isFile()).length, + directories: entries.filter((entry) => entry.isDirectory()).length, + }; + } catch { + return { path: dir, exists: false, files: null, directories: null }; + } + }); +} + +async function probeEndpoint(name, url, timeoutSeconds) { + if (!url) return { name, status: "skipped", skipped_reason: "not_configured" }; + return new Promise((resolve) => { + const started = Date.now(); + const parsed = new URL(url); + const transport = parsed.protocol === "https:" ? https : http; + const req = transport.request(parsed, { method: "GET", timeout: timeoutSeconds * 1000 }, (res) => { + res.resume(); + res.on("end", () => { + resolve({ + name, + url, + status: res.statusCode >= 200 && res.statusCode < 500 ? "pass" : "fail", + http_status: res.statusCode, + duration_ms: Date.now() - started, + }); + }); + }); + req.on("timeout", () => { + req.destroy(new Error("timeout")); + }); + req.on("error", (error) => { + resolve({ + name, + url, + status: "fail", + error: error.message, + duration_ms: Date.now() - started, + }); + }); + req.end(); + }); +} + +async function collectSystemd(units, timeoutSeconds) { + if (!commandExists("systemctl")) return { available: false, units: [] }; + const results = []; + for (const unit of units) { + const result = await spawnSyncText( + "systemctl", + ["show", unit, "--property=Id,LoadState,ActiveState,SubState,Result,NRestarts,ExecMainStatus"], + root, + timeoutSeconds, + ); + const values = {}; + for (const line of result.stdout.split("\n")) { + const [key, ...rest] = line.split("="); + if (key) values[key] = rest.join("="); + } + results.push({ + unit, + command_status: result.status, + load_state: values.LoadState || "unknown", + active_state: values.ActiveState || "unknown", + sub_state: values.SubState || "unknown", + result: values.Result || "", + restarts: Number.parseInt(values.NRestarts || "0", 10), + exec_main_status: values.ExecMainStatus || "", + }); + } + return { available: true, units: results }; +} + +async function probeTlsCertificate(target, timeoutSeconds) { + const [host, portText] = target.split(":"); + const port = Number.parseInt(portText || "443", 10); + if (!host || !Number.isFinite(port)) { + return { target, status: "fail", error: "expected host:port" }; + } + return new Promise((resolve) => { + const started = Date.now(); + const socket = tls.connect({ + host, + port, + servername: host, + rejectUnauthorized: false, + timeout: timeoutSeconds * 1000, + }, () => { + const cert = socket.getPeerCertificate(); + socket.end(); + const expiresAt = cert?.valid_to ? new Date(cert.valid_to) : null; + resolve({ + target, + status: cert && cert.valid_to ? "pass" : "fail", + subject: cert?.subject || {}, + issuer: cert?.issuer || {}, + valid_from: cert?.valid_from || null, + valid_to: cert?.valid_to || null, + expires_at_utc: expiresAt && !Number.isNaN(expiresAt.valueOf()) ? expiresAt.toISOString() : null, + days_until_expiry: expiresAt && !Number.isNaN(expiresAt.valueOf()) + ? Math.floor((expiresAt.valueOf() - Date.now()) / (24 * 60 * 60 * 1000)) + : null, + authorized: socket.authorized, + authorization_error: socket.authorizationError || null, + duration_ms: Date.now() - started, + }); + }); + socket.on("timeout", () => socket.destroy(new Error("timeout"))); + socket.on("error", (error) => { + resolve({ target, status: "fail", error: error.message, duration_ms: Date.now() - started }); + }); + }); +} + +async function collectTlsCertificates(timeoutSeconds) { + const targets = splitEnvList(process.env.AW_VALIDATION_TLS_HOSTS); + if (targets.length === 0) return []; + return Promise.all(targets.map((target) => probeTlsCertificate(target, timeoutSeconds))); +} + +async function collectRuntimeDiagnostics(args) { + const endpoints = [ + ["activitywatch", process.env.AW_VALIDATION_ACTIVITYWATCH_URL || "http://127.0.0.1:5600/api/0/info"], + ["worktime", process.env.AW_VALIDATION_WORKTIME_URL || "http://127.0.0.1:5610/healthz"], + ["grafana", process.env.AW_VALIDATION_GRAFANA_URL || ""], + ["prometheus", process.env.AW_VALIDATION_PROMETHEUS_URL || ""], + ["clickhouse", process.env.AW_VALIDATION_CLICKHOUSE_URL || ""], + ]; + const systemdUnits = splitEnvList(process.env.AW_VALIDATION_SYSTEMD_UNITS); + const defaultUnits = [ + "activitywatch-server.service", + "aw-worktime-api.service", + "detmir-readiness.service", + "aw-prune-local-state.service", + "aw-db-maintenance.service", + "aw-db-vacuum.service", + ]; + + return { + host: { + hostname: os.hostname(), + platform: os.platform(), + release: os.release(), + uptime_seconds: Math.floor(os.uptime()), + }, + cpu: collectCpu(), + memory: collectMemory(), + disks: collectDiskUsage(), + path_sizes: collectPathSizes(), + queues: countQueueFiles(), + endpoints: await Promise.all( + endpoints.map(([name, url]) => probeEndpoint(name, url, args.endpointTimeoutSeconds)), + ), + systemd: await collectSystemd(systemdUnits.length > 0 ? systemdUnits : defaultUnits, args.endpointTimeoutSeconds), + tls_certificates: await collectTlsCertificates(args.endpointTimeoutSeconds), + }; +} + +function buildEvidence(report, checks) { + const byCategory = (category) => checks.filter((check) => check.category === category).map(summarizeCheckForEvidence); + return { + deployment: { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + checks: [...byCategory("deployment"), ...byCategory("configuration")], + }, + release: { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + checks: byCategory("release"), + production_binary_parity_evidence_configured: Boolean(report.inputs.productionEvidence), + release_evidence_dir_configured: Boolean(report.inputs.releaseEvidenceDir), + }, + operational: { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + checks: byCategory("operational"), + }, + recovery: { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + checks: checks + .filter((check) => ["retention", "recovery"].includes(check.category)) + .map(summarizeCheckForEvidence), + }, + build: { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + git_describe: report.git_describe, + checks: [...byCategory("rust"), ...byCategory("dependency"), ...byCategory("repository")], + tools: report.tools, + }, + }; +} + +async function retentionRecoveryChecks() { + const checks = []; + checks.push(await runInternalCheck("retention.policy_documented", "retention", () => { + const file = path.join(root, "docs", "RETENTION_POLICY_RU.md"); + const text = readTextIfExists(file); + const required = ["Component", "Data type", "Default retention", "Cleanup method", "Recovery impact"]; + const missing = required.filter((marker) => !text.includes(marker)); + return { + status: fs.existsSync(file) && missing.length === 0 ? "pass" : "fail", + file: "docs/RETENTION_POLICY_RU.md", + missing, + }; + })); + checks.push(await runInternalCheck("recovery.audit_documented", "recovery", () => { + const required = ["RECOVERY_AUDIT.md", "DISASTER_RECOVERY_PROOF.md", "docs/RETENTION_POLICY_RU.md"]; + const missing = required.filter((file) => !fs.existsSync(path.join(root, file))); + return { status: missing.length === 0 ? "pass" : "fail", missing }; + })); + return checks; +} + +function fileFingerprint(files) { + const hashes = {}; + for (const file of files) { + const full = path.join(root, file); + if (fs.existsSync(full) && fs.statSync(full).isFile()) { + hashes[file] = sha256File(full); + } + } + return hashes; +} + +async function buildFingerprints(files) { + const configFiles = files.filter((file) => + file.endsWith(".json") + || file.endsWith(".yml") + || file.endsWith(".yaml") + || file.endsWith(".toml") + || file.endsWith(".service") + || file.endsWith(".timer") + || file.includes("grafana/") + ); + const artifactFiles = [ + "adk-rust/target/release/detmir-readiness", + "adk-rust/target/release/detmir-portal", + "adk-rust/target/release/worktime-api", + "adk-rust/target/release/aw-rus-healthd", + "adk-rust/target/x86_64-pc-windows-gnu/release/aw-windows-telemetry.exe", + ]; + return { + config_hash: sha256Bytes(JSON.stringify(fileFingerprint(configFiles), Object.keys(fileFingerprint(configFiles)).sort())), + artifact_hashes: fileFingerprint(artifactFiles), + cargo_lock_sha256: fs.existsSync(path.join(root, "adk-rust", "Cargo.lock")) + ? sha256File(path.join(root, "adk-rust", "Cargo.lock")) + : null, + }; +} + +function latestHistoryReports(historyDir, currentRunDir) { + if (!fs.existsSync(historyDir)) return []; + return fs + .readdirSync(historyDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(historyDir, entry.name, "validation-report.json")) + .filter((file) => path.dirname(file) !== currentRunDir && fs.existsSync(file)) + .sort(); +} + +function loadPreviousReport(historyDir, currentRunDir) { + const reports = latestHistoryReports(historyDir, currentRunDir); + if (reports.length === 0) return null; + try { + return JSON.parse(fs.readFileSync(reports[reports.length - 1], "utf8")); + } catch { + return null; + } +} + +function checkMap(report) { + const map = new Map(); + for (const check of report?.checks || []) map.set(check.id, check); + return map; +} + +function detectRegressions(current, previous) { + if (!previous) { + return { + baseline: "created", + new_failures: [], + recovered_failures: [], + performance_degradations: [], + new_alerts: [], + dependency_changes: [], + configuration_drift: [], + artifact_changes: [], + resource_growth: [], + }; + } + + const prevChecks = checkMap(previous); + const newFailures = []; + const recovered = []; + const performance = []; + for (const check of current.checks) { + const prev = prevChecks.get(check.id); + if (!prev) continue; + if (check.status === "fail" && prev.status !== "fail") newFailures.push({ id: check.id, previous: prev.status, current: check.status }); + if (check.status !== "fail" && prev.status === "fail") recovered.push({ id: check.id, previous: prev.status, current: check.status }); + if ( + Number.isFinite(check.duration_ms) + && Number.isFinite(prev.duration_ms) + && check.duration_ms > Math.max(prev.duration_ms * 1.5, prev.duration_ms + 5000) + ) { + performance.push({ id: check.id, previous_ms: prev.duration_ms, current_ms: check.duration_ms }); + } + } + + const previousAlertIds = new Set((previous.alerts || []).map((alert) => alert.id)); + const newAlerts = (current.alerts || []).filter((alert) => !previousAlertIds.has(alert.id)); + const dependencyChanges = []; + if (previous.fingerprints?.cargo_lock_sha256 && current.fingerprints?.cargo_lock_sha256 + && previous.fingerprints.cargo_lock_sha256 !== current.fingerprints.cargo_lock_sha256) { + dependencyChanges.push({ + file: "adk-rust/Cargo.lock", + previous_sha256: previous.fingerprints.cargo_lock_sha256, + current_sha256: current.fingerprints.cargo_lock_sha256, + }); + } + const configurationDrift = []; + if (previous.fingerprints?.config_hash && current.fingerprints?.config_hash + && previous.fingerprints.config_hash !== current.fingerprints.config_hash) { + configurationDrift.push({ previous_hash: previous.fingerprints.config_hash, current_hash: current.fingerprints.config_hash }); + } + const artifactChanges = []; + for (const [file, hash] of Object.entries(current.fingerprints?.artifact_hashes || {})) { + const previousHash = previous.fingerprints?.artifact_hashes?.[file]; + if (previousHash && previousHash !== hash) { + artifactChanges.push({ file, previous_sha256: previousHash, current_sha256: hash }); + } + } + + const resourceGrowth = []; + const growthBytes = Number.parseInt(process.env.AW_VALIDATION_PATH_GROWTH_WARN_BYTES || String(1024 * 1024 * 1024), 10); + const growthRatio = Number.parseFloat(process.env.AW_VALIDATION_PATH_GROWTH_WARN_RATIO || "1.25"); + const previousSizes = new Map((previous.runtime_health?.path_sizes || []).map((item) => [item.path, item.size_bytes])); + for (const item of current.runtime_health?.path_sizes || []) { + const previousSize = previousSizes.get(item.path); + if (!Number.isFinite(previousSize) || !Number.isFinite(item.size_bytes)) continue; + const delta = item.size_bytes - previousSize; + if (delta > growthBytes && item.size_bytes > previousSize * growthRatio) { + resourceGrowth.push({ + path: item.path, + previous_size_bytes: previousSize, + current_size_bytes: item.size_bytes, + growth_bytes: delta, + }); + } + } + + return { + baseline: "compared", + previous_generated_at_utc: previous.generated_at_utc, + new_failures: newFailures, + recovered_failures: recovered, + performance_degradations: performance, + new_alerts: newAlerts.map((alert) => ({ id: alert.id, severity: alert.severity, message: alert.message })), + dependency_changes: dependencyChanges, + configuration_drift: configurationDrift, + artifact_changes: artifactChanges, + resource_growth: resourceGrowth, + }; +} + +function buildAlerts(report) { + const alerts = []; + const push = (id, severity, category, message, evidence = {}) => { + alerts.push({ id, severity, category, message, evidence }); + }; + + for (const check of report.checks) { + if (check.status === "fail") { + push(`validation_failed:${check.id}`, "critical", check.category, `Validation failed: ${check.id}`, { + exit_code: check.exit_code, + error: check.error, + timed_out: check.timed_out, + }); + } + } + + const diskWarn = Number.parseFloat(process.env.AW_VALIDATION_DISK_USED_WARN_PCT || "85"); + const diskCrit = Number.parseFloat(process.env.AW_VALIDATION_DISK_USED_CRIT_PCT || "95"); + for (const disk of report.runtime_health.disks || []) { + if (disk.used_pct >= diskCrit) push(`disk_critical:${disk.mount}`, "critical", "runtime", `Disk ${disk.mount} usage ${disk.used_pct}%`, disk); + else if (disk.used_pct >= diskWarn) push(`disk_warning:${disk.mount}`, "warning", "runtime", `Disk ${disk.mount} usage ${disk.used_pct}%`, disk); + } + + const memWarn = Number.parseFloat(process.env.AW_VALIDATION_MEMORY_USED_WARN_PCT || "90"); + const memCrit = Number.parseFloat(process.env.AW_VALIDATION_MEMORY_USED_CRIT_PCT || "97"); + const memory = report.runtime_health.memory || {}; + if (memory.used_pct >= memCrit) push("memory_critical", "critical", "runtime", `Memory usage ${memory.used_pct}%`, memory); + else if (memory.used_pct >= memWarn) push("memory_warning", "warning", "runtime", `Memory usage ${memory.used_pct}%`, memory); + + for (const endpoint of report.runtime_health.endpoints || []) { + if (endpoint.status === "fail") { + const severity = endpoint.url ? "critical" : "warning"; + push(`endpoint_unavailable:${endpoint.name}`, severity, "runtime", `${endpoint.name} endpoint unavailable`, endpoint); + } + } + + for (const unit of report.runtime_health.systemd?.units || []) { + if (unit.load_state === "loaded" && unit.active_state === "failed") { + push(`systemd_failed:${unit.unit}`, "critical", "runtime", `${unit.unit} is failed`, unit); + } + if (unit.restarts > 0) { + push(`systemd_restarts:${unit.unit}`, "warning", "runtime", `${unit.unit} restart count is ${unit.restarts}`, unit); + } + } + + const queueWarn = Number.parseInt(process.env.AW_VALIDATION_QUEUE_FILES_WARN || "1000", 10); + for (const queue of report.runtime_health.queues || []) { + if (queue.exists && Number.isFinite(queue.files) && queue.files > queueWarn) { + push(`queue_backlog:${queue.path}`, "warning", "runtime", `Queue ${queue.path} has ${queue.files} files`, queue); + } + } + + const certWarnDays = Number.parseInt(process.env.AW_VALIDATION_CERT_EXPIRY_WARN_DAYS || "14", 10); + for (const cert of report.runtime_health.tls_certificates || []) { + if (cert.status === "fail") { + push(`certificate_probe_failed:${cert.target}`, "warning", "runtime", `TLS certificate probe failed for ${cert.target}`, cert); + } else if (Number.isFinite(cert.days_until_expiry) && cert.days_until_expiry <= certWarnDays) { + push(`certificate_expiring:${cert.target}`, "warning", "runtime", `TLS certificate expires in ${cert.days_until_expiry} days for ${cert.target}`, cert); + } + } + + const backupEvidencePath = process.env.AW_VALIDATION_BACKUP_EVIDENCE_PATH || ""; + if (backupEvidencePath) { + const maxAgeDays = Number.parseInt(process.env.AW_VALIDATION_BACKUP_EVIDENCE_MAX_AGE_DAYS || "7", 10); + try { + const stat = fs.statSync(backupEvidencePath); + const ageDays = Math.floor((Date.now() - stat.mtimeMs) / (24 * 60 * 60 * 1000)); + if (ageDays > maxAgeDays) { + push("backup_evidence_expired", "warning", "recovery", `Backup evidence is ${ageDays} days old`, { + path: backupEvidencePath, + age_days: ageDays, + max_age_days: maxAgeDays, + }); + } + } catch (error) { + push("backup_evidence_missing", "warning", "recovery", `Backup evidence path is unavailable: ${backupEvidencePath}`, { + path: backupEvidencePath, + error: error.message, + }); + } + } + + if (!report.inputs.productionEvidence) { + push("binary_parity_evidence_missing", report.profile === "full" ? "warning" : "info", "release", "Production binary parity evidence was not provided"); + } + if (!report.inputs.releaseEvidenceDir) { + push("release_evidence_missing", report.profile === "full" ? "warning" : "info", "release", "Release evidence directory was not provided"); + } + const secretCheck = report.checks.find((check) => check.id === "security.secret_pattern_check"); + if (secretCheck?.status === "fail") { + push("secret_leak_detected", "critical", "security", "Secret pattern check failed", summarizeCheckForEvidence(secretCheck)); + } + + if (report.regressions) { + for (const item of report.regressions.new_failures || []) { + push(`new_failure:${item.id}`, "critical", "regression", `New validation failure: ${item.id}`, item); + } + for (const item of report.regressions.configuration_drift || []) { + push("configuration_drift_detected", "warning", "regression", "Configuration fingerprint changed since previous run", item); + } + for (const item of report.regressions.artifact_changes || []) { + push(`artifact_changed:${item.file}`, "warning", "regression", `Release artifact changed: ${item.file}`, item); + } + for (const item of report.regressions.resource_growth || []) { + push(`resource_growth:${item.path}`, "warning", "regression", `Runtime path grew by ${item.growth_bytes} bytes: ${item.path}`, item); + } + } + + return alerts; +} + +function calculateScore(report) { + const checks = report.checks || []; + if (checks.length === 0) return 0; + const passed = checks.filter((check) => check.status === "pass").length; + const skipped = checks.filter((check) => check.status === "skipped").length; + const base = (passed + skipped * 0.5) / checks.length; + const criticalPenalty = (report.alerts || []).filter((alert) => alert.severity === "critical").length * 0.08; + const warningPenalty = (report.alerts || []).filter((alert) => alert.severity === "warning").length * 0.02; + return Math.max(0, Math.min(100, Math.round((base - criticalPenalty - warningPenalty) * 100))); +} + +function renderMarkdown(report) { + const lines = []; + lines.push("# AWatch-rus Full Validation Report"); + lines.push(""); + lines.push(`Generated: ${report.generated_at_utc}`); + lines.push(`Host: ${report.host_identifier}`); + lines.push(`Git SHA: ${report.git_sha}`); + lines.push(`Version: ${report.version}`); + lines.push(`Profile: ${report.profile}`); + lines.push(`Status: ${report.status}`); + lines.push(`Validation score: ${report.validation_score}`); + lines.push(`Release readiness score: ${report.release_readiness_score}`); + lines.push(""); + lines.push("## Summary"); + lines.push(""); + lines.push(`- Checks: ${report.summary.total}`); + lines.push(`- Passed: ${report.summary.pass}`); + lines.push(`- Failed: ${report.summary.fail}`); + lines.push(`- Skipped: ${report.summary.skipped}`); + lines.push(`- Alerts: ${report.alerts.length}`); + lines.push(""); + lines.push("## Alerts"); + lines.push(""); + if (report.alerts.length === 0) { + lines.push("No active alerts."); + } else { + for (const alert of report.alerts) { + lines.push(`- ${alert.severity}: ${alert.id} - ${alert.message}`); + } + } + lines.push(""); + lines.push("## Regressions"); + lines.push(""); + lines.push(`Baseline: ${report.regressions.baseline}`); + for (const key of ["new_failures", "performance_degradations", "dependency_changes", "configuration_drift", "artifact_changes"]) { + lines.push(`- ${key}: ${(report.regressions[key] || []).length}`); + } + lines.push(`- resource_growth: ${(report.regressions.resource_growth || []).length}`); + lines.push(""); + lines.push("## Checks"); + lines.push(""); + lines.push("| Check | Category | Status | Duration ms |"); + lines.push("| --- | --- | --- | ---: |"); + for (const check of report.checks) { + lines.push(`| ${check.id} | ${check.category} | ${check.status} | ${check.duration_ms ?? 0} |`); + } + lines.push(""); + lines.push("## Runtime"); + lines.push(""); + lines.push(`- CPU load 1m: ${report.runtime_health.cpu.load_1m}`); + lines.push(`- Memory used: ${report.runtime_health.memory.used_pct}%`); + const topDisk = (report.runtime_health.disks || [])[0]; + if (topDisk) lines.push(`- Highest disk usage: ${topDisk.mount} ${topDisk.used_pct}%`); + lines.push(""); + lines.push("## Output Files"); + lines.push(""); + for (const [name, file] of Object.entries(report.outputs || {})) { + lines.push(`- ${name}: ${file}`); + } + lines.push(""); + return `${lines.join("\n")}\n`; +} + +function pruneHistory(historyDir, retentionDays, nowMs) { + if (!fs.existsSync(historyDir)) return []; + const cutoff = nowMs - retentionDays * 24 * 60 * 60 * 1000; + const removed = []; + for (const entry of fs.readdirSync(historyDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const full = path.join(historyDir, entry.name); + const stat = fs.statSync(full); + if (stat.mtimeMs < cutoff) { + fs.rmSync(full, { recursive: true, force: true }); + removed.push(entry.name); + } + } + return removed; +} + +function buildDashboard(report) { + return { + generated_at_utc: report.generated_at_utc, + host_identifier: report.host_identifier, + git_sha: report.git_sha, + version: report.version, + overall_health: report.status, + overall_readiness_score: report.validation_score, + release_readiness_score: report.release_readiness_score, + system_uptime_seconds: report.runtime_health.host.uptime_seconds, + active_alerts: report.alerts, + recent_regressions: report.regressions, + resource_usage: { + cpu: report.runtime_health.cpu, + memory: report.runtime_health.memory, + disks: report.runtime_health.disks, + }, + service_state: report.runtime_health.systemd, + backup_state: { + release_evidence_configured: Boolean(report.inputs.releaseEvidenceDir), + recovery_documents_present: report.checks.find((check) => check.id === "recovery.audit_documented")?.status === "pass", + }, + recovery_readiness: report.evidence.recovery, + }; +} + +async function executeValidation(args) { + const generatedAt = utcNow(); + const outputDir = path.resolve(args.outputDir); + const historyDir = path.join(outputDir, "history"); + const runDir = path.join(historyDir, stampForPath(generatedAt)); + const latestDir = path.join(outputDir, "latest"); + mkdirp(runDir); + mkdirp(latestDir); + + const git = await collectGitContext(); + const files = await trackedFiles(); + const tools = {}; + for (const tool of ["git", "node", "python3", "bash", "cargo", "ruby", "shellcheck", "ansible-playbook", "openssl", "systemctl"]) { + tools[tool] = commandExists(tool); + } + + const checks = []; + checks.push(await runInternalCheck("repository.tracked_files_inventory", "repository", () => ({ + status: files.length > 0 ? "pass" : "fail", + tracked_files: files.length, + }))); + checks.push(await runInternalCheck("configuration.json_parse", "configuration", () => validateJsonFiles(files))); + checks.push(await runInternalCheck("documentation.markdown_links", "documentation", () => validateMarkdownLinks(files))); + checks.push(await runInternalCheck("configuration.yaml_parse", "configuration", () => validateYamlFiles(files, args.commandTimeoutSeconds))); + checks.push(...await retentionRecoveryChecks()); + + for (const def of commandDefinitions(args)) { + checks.push(await runCommand(def)); + } + + const runtimeHealth = await collectRuntimeDiagnostics(args); + const fingerprints = await buildFingerprints(files); + + const summary = { + total: checks.length, + pass: checks.filter((check) => check.status === "pass").length, + fail: checks.filter((check) => check.status === "fail").length, + skipped: checks.filter((check) => check.status === "skipped").length, + }; + + const report = { + schema_version: 1, + generated_at_utc: generatedAt, + product: "AWatch-rus", + profile: args.profile, + host_identifier: os.hostname(), + version: git.git_describe, + ...git, + inputs: { + outputDir, + retentionDays: args.retentionDays, + productionEvidence: args.productionEvidence || null, + releaseEvidenceDir: args.releaseEvidenceDir || null, + releaseAssetDir: args.releaseAssetDir || null, + }, + tools, + summary, + status: "pending", + checks, + runtime_health: runtimeHealth, + fingerprints, + regressions: null, + alerts: [], + validation_score: 0, + release_readiness_score: 0, + outputs: {}, + }; + + report.evidence = buildEvidence(report, checks); + const previous = loadPreviousReport(historyDir, runDir); + report.regressions = detectRegressions(report, previous); + report.alerts = buildAlerts(report); + report.validation_score = calculateScore(report); + report.release_readiness_score = Math.max( + 0, + report.validation_score + - (report.inputs.productionEvidence ? 0 : 5) + - (report.inputs.releaseEvidenceDir ? 0 : 5), + ); + const criticalAlerts = report.alerts.filter((alert) => criticalAlertSeverities.has(alert.severity)).length; + report.status = summary.fail === 0 && criticalAlerts === 0 ? "pass" : "fail"; + + const dashboard = buildDashboard(report); + const outputs = { + validation_report_json: path.join(runDir, "validation-report.json"), + validation_report_md: path.join(runDir, "validation-report.md"), + production_health_json: path.join(runDir, "production-health.json"), + deployment_health_json: path.join(runDir, "deployment-health.json"), + runtime_health_json: path.join(runDir, "runtime-health.json"), + release_evidence_json: path.join(runDir, "release-evidence.json"), + operational_evidence_json: path.join(runDir, "operational-evidence.json"), + recovery_evidence_json: path.join(runDir, "recovery-evidence.json"), + build_evidence_json: path.join(runDir, "build-evidence.json"), + operational_dashboard_json: path.join(runDir, "operational-dashboard.json"), + operational_dashboard_md: path.join(runDir, "operational-dashboard.md"), + }; + report.outputs = Object.fromEntries(Object.entries(outputs).map(([key, value]) => [key, normalizeRelative(value)])); + + writeJson(outputs.validation_report_json, report); + writeText(outputs.validation_report_md, renderMarkdown(report)); + writeJson(outputs.production_health_json, { + generated_at_utc: report.generated_at_utc, + git_sha: report.git_sha, + host_identifier: report.host_identifier, + status: report.status, + alerts: report.alerts, + runtime_health: report.runtime_health, + }); + writeJson(outputs.deployment_health_json, report.evidence.deployment); + writeJson(outputs.runtime_health_json, report.runtime_health); + writeJson(outputs.release_evidence_json, report.evidence.release); + writeJson(outputs.operational_evidence_json, report.evidence.operational); + writeJson(outputs.recovery_evidence_json, report.evidence.recovery); + writeJson(outputs.build_evidence_json, report.evidence.build); + writeJson(outputs.operational_dashboard_json, dashboard); + writeText(outputs.operational_dashboard_md, renderDashboardMarkdown(dashboard)); + + fs.rmSync(latestDir, { recursive: true, force: true }); + mkdirp(latestDir); + for (const [name, file] of Object.entries(outputs)) { + const latestName = path.basename(file); + fs.copyFileSync(file, path.join(latestDir, latestName)); + report.outputs[`latest_${name}`] = normalizeRelative(path.join(latestDir, latestName)); + } + writeJson(outputs.validation_report_json, report); + writeText(outputs.validation_report_md, renderMarkdown(report)); + writeJson(path.join(latestDir, "validation-report.json"), report); + writeText(path.join(latestDir, "validation-report.md"), renderMarkdown(report)); + + report.history_retention = { + history_dir: normalizeRelative(historyDir), + retention_days: args.retentionDays, + pruned_runs: pruneHistory(historyDir, args.retentionDays, Date.now()), + }; + writeJson(outputs.validation_report_json, report); + writeJson(path.join(latestDir, "validation-report.json"), report); + + return report; +} + +function renderDashboardMarkdown(dashboard) { + const lines = []; + lines.push("# AWatch-rus Operational Dashboard"); + lines.push(""); + lines.push(`Generated: ${dashboard.generated_at_utc}`); + lines.push(`Host: ${dashboard.host_identifier}`); + lines.push(`Git SHA: ${dashboard.git_sha}`); + lines.push(`Version: ${dashboard.version}`); + lines.push(`Overall health: ${dashboard.overall_health}`); + lines.push(`Overall readiness score: ${dashboard.overall_readiness_score}`); + lines.push(`Release readiness score: ${dashboard.release_readiness_score}`); + lines.push(`System uptime seconds: ${dashboard.system_uptime_seconds}`); + lines.push(""); + lines.push("## Active Alerts"); + lines.push(""); + if (dashboard.active_alerts.length === 0) lines.push("No active alerts."); + for (const alert of dashboard.active_alerts) lines.push(`- ${alert.severity}: ${alert.message}`); + lines.push(""); + lines.push("## Resources"); + lines.push(""); + lines.push(`- CPU load 1m: ${dashboard.resource_usage.cpu.load_1m}`); + lines.push(`- Memory used: ${dashboard.resource_usage.memory.used_pct}%`); + for (const disk of dashboard.resource_usage.disks.slice(0, 5)) { + lines.push(`- Disk ${disk.mount}: ${disk.used_pct}%`); + } + lines.push(""); + return `${lines.join("\n")}\n`; +} + +async function selfTest() { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "aw-full-validation-selftest-")); + try { + const commandOk = await runCommand({ + id: "self.command_ok", + category: "self", + command: process.execPath, + args: ["-e", "process.stdout.write('ok')"], + cwd: root, + timeoutSeconds: 10, + }); + if (commandOk.status !== "pass" || commandOk.stdout !== "ok") throw new Error("command runner pass case failed"); + + const commandFail = await runCommand({ + id: "self.command_fail", + category: "self", + command: process.execPath, + args: ["-e", "process.exit(3)"], + cwd: root, + timeoutSeconds: 10, + }); + if (commandFail.status !== "fail" || commandFail.exit_code !== 3) throw new Error("command runner fail case failed"); + + const previous = { + generated_at_utc: "2026-07-01T00:00:00Z", + checks: [{ id: "x", status: "pass", duration_ms: 100 }], + alerts: [], + fingerprints: { + config_hash: "a", + artifact_hashes: { bin: "1" }, + cargo_lock_sha256: "lock-a", + }, + }; + const current = { + checks: [{ id: "x", status: "fail", duration_ms: 1000 }], + alerts: [{ id: "alert-x", severity: "critical", message: "x" }], + fingerprints: { + config_hash: "b", + artifact_hashes: { bin: "2" }, + cargo_lock_sha256: "lock-b", + }, + }; + const regressions = detectRegressions(current, previous); + if (regressions.new_failures.length !== 1) throw new Error("new failure regression not detected"); + if (regressions.configuration_drift.length !== 1) throw new Error("configuration drift not detected"); + if (regressions.artifact_changes.length !== 1) throw new Error("artifact change not detected"); + if (regressions.dependency_changes.length !== 1) throw new Error("dependency change not detected"); + + const fakeReport = { + checks: [{ id: "x", category: "self", status: "fail" }], + runtime_health: { + disks: [{ mount: "/", used_pct: 99 }], + memory: { used_pct: 98 }, + endpoints: [{ name: "activitywatch", url: "http://127.0.0.1", status: "fail" }], + systemd: { units: [] }, + }, + inputs: {}, + profile: "full", + regressions, + }; + const alerts = buildAlerts(fakeReport); + if (!alerts.some((alert) => alert.id === "validation_failed:x")) throw new Error("validation failure alert missing"); + if (!alerts.some((alert) => alert.id.startsWith("disk_critical:"))) throw new Error("disk alert missing"); + + const markdown = renderMarkdown({ + generated_at_utc: utcNow(), + host_identifier: "self", + git_sha: "0".repeat(40), + version: "self", + profile: "quick", + status: "pass", + validation_score: 100, + release_readiness_score: 100, + summary: { total: 1, pass: 1, fail: 0, skipped: 0 }, + alerts: [], + regressions: { baseline: "created", new_failures: [], performance_degradations: [], dependency_changes: [], configuration_drift: [], artifact_changes: [] }, + checks: [{ id: "self", category: "self", status: "pass", duration_ms: 1 }], + runtime_health: { cpu: { load_1m: 0 }, memory: { used_pct: 0 }, disks: [] }, + outputs: {}, + }); + if (!markdown.includes("AWatch-rus Full Validation Report")) throw new Error("markdown renderer failed"); + + writeJson(path.join(tmp, "report.json"), { ok: true }); + JSON.parse(fs.readFileSync(path.join(tmp, "report.json"), "utf8")); + console.log("full_validation_orchestrator_self_test=ok"); + return 0; + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +async function main() { + let args; + try { + args = parseArgs(process.argv.slice(2)); + } catch (error) { + console.error(error.message); + console.error(usage()); + return 2; + } + if (args.help) { + process.stdout.write(usage()); + return 0; + } + if (args.selfTest) return selfTest(); + + const report = await executeValidation(args); + const summary = { + status: report.status, + validation_score: report.validation_score, + release_readiness_score: report.release_readiness_score, + checks: report.summary, + alerts: report.alerts.length, + output_dir: normalizeRelative(path.join(path.resolve(args.outputDir), "latest")), + }; + if (args.json) { + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); + } else { + process.stdout.write( + `full_validation=${summary.status} score=${summary.validation_score} release_score=${summary.release_readiness_score} ` + + `checks=${summary.checks.total} failed=${summary.checks.fail} alerts=${summary.alerts} output=${summary.output_dir}\n`, + ); + } + if (args.allowFailures) return 0; + return report.status === "pass" ? 0 : 1; +} + +main().then((code) => { + process.exitCode = code; +}).catch((error) => { + console.error(`full_validation_orchestrator=fail error=${error.stack || error.message}`); + process.exitCode = 1; +}); diff --git a/scripts/run_full_validation.sh b/scripts/run_full_validation.sh new file mode 100644 index 0000000..9e13ece --- /dev/null +++ b/scripts/run_full_validation.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +if ! command -v node >/dev/null 2>&1; then + echo "run_full_validation=fail node_not_found" >&2 + exit 127 +fi + +run_preflight_self_test=1 +for arg in "$@"; do + if [[ "$arg" == "--self-test" || "$arg" == "--help" || "$arg" == "-h" ]]; then + run_preflight_self_test=0 + fi +done + +if [[ "${AW_VALIDATION_SKIP_SELF_TEST:-0}" != "1" && "$run_preflight_self_test" == "1" ]]; then + node scripts/full-validation-orchestrator.mjs --self-test +fi + +exec node scripts/full-validation-orchestrator.mjs "$@" diff --git a/windows/validate-deployment.ps1 b/windows/validate-deployment.ps1 index 9f5f96b..40f0f6b 100644 --- a/windows/validate-deployment.ps1 +++ b/windows/validate-deployment.ps1 @@ -594,7 +594,15 @@ $watcherCountsOk = $true $endpointProcessOk = $true $fileProcessOk = $true $browserProcessOk = $true -$sessionCollectorOk = (@($sessionCollectorProcesses).Count -eq 1) +$sessionCollectorOk = if (@($sessionCollectorProcesses).Count -eq 1) { + $true +} +elseif (-not $liveSessionScopedCollectorsRequired) { + $true +} +else { + $false +} $result = [ordered]@{ generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o')