diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index 4fca3cf..6e129da 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -212,6 +212,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "aw-ensure-reliability" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "aw-health-check" version = "0.1.0" diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml index 507acb1..d933a4a 100644 --- a/adk-rust/Cargo.toml +++ b/adk-rust/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/detmir-aw-client", "crates/aw-db-health", "crates/aw-db-maintenance", + "crates/aw-ensure-reliability", "crates/aw-health-check", "crates/aw-prune-local-state", "crates/check-aw-data", diff --git a/adk-rust/README.md b/adk-rust/README.md index 9075563..dfa902a 100644 --- a/adk-rust/README.md +++ b/adk-rust/README.md @@ -23,6 +23,8 @@ scripts with durable standalone Rust modules. - `dlp-health-check` - AW server DLP health check replacement. - `aw-db-maintenance` - guarded weekly SQLite maintenance for old allowlisted process-level session events, with backup-before-delete. +- `aw-ensure-reliability` - safe dry-run/apply planner for AW service + reliability repair actions that were previously immediate Bash mutations. - `check-aw-full` - read-only local AW/RDP full check replacement for the legacy shell helper. - `dlp-aggregator` - AW server DLP warehouse aggregator replacement. diff --git a/adk-rust/RUNBOOK.md b/adk-rust/RUNBOOK.md index 362af88..40130ec 100644 --- a/adk-rust/RUNBOOK.md +++ b/adk-rust/RUNBOOK.md @@ -1470,6 +1470,37 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg (`3 passed`), `cargo clippy -p prod-rollout --all-targets -- -D warnings`, release build OK, `bash -n scripts/prod_rollout.sh`, artifact check OK. +53. `[done]` Перенести `aw-server/ensure-reliability.sh` в Rust-first + dry-run/apply helper: + - добавлен crate `aw-ensure-reliability`; + - `aw-server/ensure-reliability.sh` теперь Rust-first wrapper: + ищет `AW_ENSURE_RELIABILITY_RUST`, + `$CARGO_TARGET_DIR/release/aw-ensure-reliability`, + `adk-rust/target/release/aw-ensure-reliability`, + `/usr/local/bin/aw-ensure-reliability`; + - обычный запуск больше не делает `chown`, `systemctl stop/start`, + logrotate write или health timer write; он показывает dry-run plan; + - реальный Rust repair требует explicit `--apply`; + - старый Bash repair доступен только explicit `--apply-legacy`; + - Rust helper сохраняет legacy action set: env check, + ownership/mode repair for `/var/lib/activitywatch`, + `/var/log/activitywatch`, `/opt/activitywatch`, logrotate install, + health-check script/timer/service install, ordered AW service restart and + enable; + - `deploy_aw_server.yml` optional устанавливает + `/usr/local/bin/aw-ensure-reliability` до Influx token checks, если + release artifact доступен; + - production binary доставлен на AW server, но `--apply` не запускался; + - production dry-run: `apply=false`, `ok=true`, `missing=0`, + `executed=0`, `steps=25`; + - final production gates: AW failed units `0`, DetMir status OK with + `service_warnings=0`, `dlp_counts={ok:22,warn:0,fail:0}`, + `ok_for_operator=true`; + - gates: `cargo fmt --all -- --check`, `cargo test -p + aw-ensure-reliability` (`2 passed`), `cargo clippy -p + aw-ensure-reliability --all-targets -- -D warnings`, release build OK, + `bash -n aw-server/ensure-reliability.sh`, artifact check OK, + `ansible-playbook deploy_aw_server.yml --syntax-check` OK. Отложить: diff --git a/adk-rust/crates/aw-ensure-reliability/Cargo.toml b/adk-rust/crates/aw-ensure-reliability/Cargo.toml new file mode 100644 index 0000000..1216dc9 --- /dev/null +++ b/adk-rust/crates/aw-ensure-reliability/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "aw-ensure-reliability" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/aw-ensure-reliability/src/main.rs b/adk-rust/crates/aw-ensure-reliability/src/main.rs new file mode 100644 index 0000000..6af41c4 --- /dev/null +++ b/adk-rust/crates/aw-ensure-reliability/src/main.rs @@ -0,0 +1,547 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use serde::Serialize; + +const LOGROTATE_CONTENT: &str = include_str!("../../../../aw-server/logrotate.conf"); +const HEALTH_TIMER_CONTENT: &str = r#"[Unit] +Description=AW Health Check Timer +Requires=aw-health-check.service + +[Timer] +OnCalendar=*:0/5:00 +Persistent=true + +[Install] +WantedBy=timers.target +"#; +const HEALTH_SERVICE_CONTENT: &str = r#"[Unit] +Description=AW Health Check +After=network.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/aw-health-check +User=root +Group=root +"#; + +#[derive(Debug, Parser)] +#[command(about = "Plan or apply AW service reliability hardening")] +struct Cli { + #[arg(long, default_value = "/etc/activitywatch/aw-server.env")] + env_file: PathBuf, + + #[arg(long, default_value = "/var/lib/activitywatch")] + data_dir: PathBuf, + + #[arg(long, default_value = "/var/log/activitywatch")] + log_dir: PathBuf, + + #[arg(long, default_value = "/opt/activitywatch")] + opt_dir: PathBuf, + + #[arg(long, default_value = "/etc/logrotate.d/activitywatch")] + logrotate_target: PathBuf, + + #[arg(long, default_value = "/usr/local/bin/aw-health-check")] + health_script_target: PathBuf, + + #[arg(long, default_value = "/etc/systemd/system/aw-health-check.timer")] + health_timer_target: PathBuf, + + #[arg(long, default_value = "/etc/systemd/system/aw-health-check.service")] + health_service_target: PathBuf, + + #[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/health-check.sh")] + health_script_source: PathBuf, + + #[arg(long, default_value_t = false)] + apply: bool, + + #[arg(long, default_value_t = false)] + json: bool, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum StepKind { + Check, + Chown, + Chmod, + Copy, + Write, + Systemd, + Sleep, +} + +#[derive(Debug, Clone, Serialize)] +struct Step { + order: usize, + name: String, + kind: StepKind, + command: String, + mutation: bool, + needed: bool, + reason: String, +} + +#[derive(Debug, Serialize)] +struct Report { + apply: bool, + ok: bool, + env_file: PathBuf, + missing_required: Vec, + steps: Vec, + executed: Vec, +} + +#[derive(Debug, Serialize)] +struct ExecResult { + order: usize, + name: String, + ok: bool, + exit_code: Option, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let mut report = build_report(&cli); + if cli.apply { + if !report.missing_required.is_empty() { + report.ok = false; + print_report(&report, cli.json)?; + bail!("refusing --apply because required inputs are missing"); + } + apply_steps(&mut report)?; + } + print_report(&report, cli.json)?; + Ok(if report.ok { 0 } else { 1 }) +} + +fn build_report(cli: &Cli) -> Report { + let mut missing_required = Vec::new(); + if !cli.env_file.is_file() { + missing_required.push(format!("env file missing: {}", cli.env_file.display())); + } + if !cli.health_script_target.is_file() && !cli.health_script_source.is_file() { + missing_required.push(format!( + "health script target and source missing: {}, {}", + cli.health_script_target.display(), + cli.health_script_source.display() + )); + } + + let mut steps = Vec::new(); + push_step( + &mut steps, + "check-env-file", + StepKind::Check, + format!("test -f {}", shell_quote(&cli.env_file)), + false, + !cli.env_file.is_file(), + "required before reliability actions".to_string(), + ); + for dir in [&cli.data_dir, &cli.log_dir, &cli.opt_dir] { + push_step( + &mut steps, + format!("chown-{}", dir.display()), + StepKind::Chown, + format!("chown -R activitywatch:activitywatch {}", shell_quote(dir)), + true, + true, + "preserve legacy ownership repair".to_string(), + ); + } + for dir in [&cli.data_dir, &cli.log_dir, &cli.opt_dir] { + push_step( + &mut steps, + format!("chmod-{}", dir.display()), + StepKind::Chmod, + format!("chmod 755 {}", shell_quote(dir)), + true, + true, + "preserve legacy directory mode repair".to_string(), + ); + } + push_step( + &mut steps, + "install-logrotate", + StepKind::Write, + format!("write {}", shell_quote(&cli.logrotate_target)), + true, + !cli.logrotate_target.is_file(), + if cli.logrotate_target.is_file() { + "logrotate already configured".to_string() + } else { + "logrotate target missing".to_string() + }, + ); + push_step( + &mut steps, + "install-health-script", + StepKind::Copy, + format!( + "copy {} {}", + shell_quote(&cli.health_script_source), + shell_quote(&cli.health_script_target) + ), + true, + !cli.health_script_target.is_file(), + if cli.health_script_target.is_file() { + "health script already installed".to_string() + } else { + "health script target missing".to_string() + }, + ); + for (name, path, content_name) in [ + ( + "install-health-timer", + &cli.health_timer_target, + "aw-health-check.timer", + ), + ( + "install-health-service", + &cli.health_service_target, + "aw-health-check.service", + ), + ] { + push_step( + &mut steps, + name, + StepKind::Write, + format!("write {} ({content_name})", shell_quote(path)), + true, + !path.is_file(), + if path.is_file() { + format!("{content_name} already installed") + } else { + format!("{content_name} target missing") + }, + ); + } + for (name, command) in [ + ("daemon-reload-before-restart", "systemctl daemon-reload"), + ( + "stop-services", + "systemctl stop aw-worktime-api aw-worktime-ui-bridge activitywatch-server || true", + ), + ("sleep-after-stop", "sleep 2"), + ( + "start-activitywatch-server", + "systemctl start activitywatch-server", + ), + ("sleep-after-server-start", "sleep 3"), + ("start-worktime-api", "systemctl start aw-worktime-api"), + ("sleep-after-api-start", "sleep 2"), + ( + "start-worktime-ui-bridge", + "systemctl start aw-worktime-ui-bridge", + ), + ( + "enable-activitywatch-server", + "systemctl enable activitywatch-server", + ), + ("enable-worktime-api", "systemctl enable aw-worktime-api"), + ( + "enable-worktime-ui-bridge", + "systemctl enable aw-worktime-ui-bridge", + ), + ("daemon-reload-health", "systemctl daemon-reload"), + ( + "enable-health-timer", + "systemctl enable aw-health-check.timer", + ), + ( + "start-health-timer", + "systemctl start aw-health-check.timer", + ), + ] { + let kind = if command.starts_with("sleep") { + StepKind::Sleep + } else { + StepKind::Systemd + }; + push_step( + &mut steps, + name, + kind, + command.to_string(), + true, + true, + "preserve legacy reliability action".to_string(), + ); + } + + Report { + apply: cli.apply, + ok: missing_required.is_empty(), + env_file: cli.env_file.clone(), + missing_required, + steps, + executed: Vec::new(), + } +} + +fn push_step( + steps: &mut Vec, + name: impl Into, + kind: StepKind, + command: String, + mutation: bool, + needed: bool, + reason: String, +) { + steps.push(Step { + order: steps.len() + 1, + name: name.into(), + kind, + command, + mutation, + needed, + reason, + }); +} + +fn apply_steps(report: &mut Report) -> Result<()> { + let steps = report.steps.clone(); + for step in steps.iter().filter(|step| step.needed) { + let result = match step.name.as_str() { + "check-env-file" => ExecResult { + order: step.order, + name: step.name.clone(), + ok: Path::new(&report.env_file).is_file(), + exit_code: Some(if Path::new(&report.env_file).is_file() { + 0 + } else { + 1 + }), + }, + "install-logrotate" => write_file_result(step, report, LOGROTATE_CONTENT, 0o644)?, + "install-health-script" => copy_health_script(step, report)?, + "install-health-timer" => write_file_result(step, report, HEALTH_TIMER_CONTENT, 0o644)?, + "install-health-service" => { + write_file_result(step, report, HEALTH_SERVICE_CONTENT, 0o644)? + } + _ => run_shell_step(step)?, + }; + let ok = result.ok; + report.executed.push(result); + if !ok { + report.ok = false; + return Ok(()); + } + } + report.ok = true; + Ok(()) +} + +fn write_file_result(step: &Step, report: &Report, content: &str, mode: u32) -> Result { + let target = match step.name.as_str() { + "install-logrotate" => target_from_command(&step.command)?, + "install-health-timer" => target_from_command(&step.command)?, + "install-health-service" => target_from_command(&step.command)?, + _ => bail!("unsupported write step {}", step.name), + }; + let _ = report; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::write(&target, content).with_context(|| format!("write {}", target.display()))?; + set_mode(&target, mode).with_context(|| format!("chmod {:o} {}", mode, target.display()))?; + Ok(ExecResult { + order: step.order, + name: step.name.clone(), + ok: true, + exit_code: Some(0), + }) +} + +fn copy_health_script(step: &Step, report: &Report) -> Result { + let source = source_from_copy_command(&step.command)?; + let target = copy_target_from_command(&step.command)?; + let _ = report; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::copy(&source, &target) + .with_context(|| format!("copy {} -> {}", source.display(), target.display()))?; + set_mode(&target, 0o755).with_context(|| format!("chmod 0755 {}", target.display()))?; + Ok(ExecResult { + order: step.order, + name: step.name.clone(), + ok: true, + exit_code: Some(0), + }) +} + +fn run_shell_step(step: &Step) -> Result { + let status = Command::new("sh") + .arg("-c") + .arg(&step.command) + .status() + .with_context(|| format!("run {}", step.command))?; + Ok(ExecResult { + order: step.order, + name: step.name.clone(), + ok: status.success(), + exit_code: status.code(), + }) +} + +fn target_from_command(command: &str) -> Result { + let raw = command + .split_whitespace() + .nth(1) + .or_else(|| command.split_whitespace().nth(2)) + .context("parse target from command")?; + Ok(PathBuf::from(raw.trim_matches('\''))) +} + +fn source_from_copy_command(command: &str) -> Result { + let raw = command + .split_whitespace() + .nth(1) + .context("parse source from copy command")?; + Ok(PathBuf::from(raw.trim_matches('\''))) +} + +fn copy_target_from_command(command: &str) -> Result { + let raw = command + .split_whitespace() + .nth(2) + .context("parse target from copy command")?; + Ok(PathBuf::from(raw.trim_matches('\''))) +} + +#[cfg(unix)] +fn set_mode(path: &Path, mode: u32) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(path)?.permissions(); + perms.set_mode(mode); + fs::set_permissions(path, perms)?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_mode(_path: &Path, _mode: u32) -> Result<()> { + Ok(()) +} + +fn print_report(report: &Report, json: bool) -> Result<()> { + if json { + println!("{}", serde_json::to_string_pretty(report)?); + return Ok(()); + } + println!( + "aw-ensure-reliability: {}", + if report.apply { "apply" } else { "dry-run" } + ); + println!("env_file: {}", report.env_file.display()); + println!("ok: {}", report.ok); + if !report.missing_required.is_empty() { + println!("missing_required:"); + for item in &report.missing_required { + println!(" - {item}"); + } + } + println!("planned steps:"); + for step in &report.steps { + let risk = if step.mutation { "MUTATION" } else { "check" }; + let needed = if step.needed { "needed" } else { "skip" }; + println!( + " {:02}. {:<28} {:<8} {:<6} {}", + step.order, step.name, risk, needed, step.command + ); + } + if report.executed.is_empty() { + println!("No mutation executed. Use --apply for explicit reliability fix."); + } else { + println!("executed:"); + for item in &report.executed { + println!( + " {:02}. {:<28} ok={} exit={:?}", + item.order, item.name, item.ok, item.exit_code + ); + } + } + Ok(()) +} + +fn shell_quote(path: &Path) -> String { + format!("'{}'", path.display().to_string().replace('\'', "'\\''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dry_run_marks_legacy_mutations() { + let dir = tempfile::tempdir().unwrap(); + let env_file = dir.path().join("aw-server.env"); + let health = dir.path().join("health-check.sh"); + fs::write(&env_file, "AW_BASE_URL=http://127.0.0.1:5600\n").unwrap(); + fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap(); + let cli = Cli { + env_file, + data_dir: dir.path().join("data"), + log_dir: dir.path().join("log"), + opt_dir: dir.path().join("opt"), + logrotate_target: dir.path().join("logrotate/activitywatch"), + health_script_target: dir.path().join("bin/aw-health-check"), + health_timer_target: dir.path().join("systemd/aw-health-check.timer"), + health_service_target: dir.path().join("systemd/aw-health-check.service"), + health_script_source: health, + apply: false, + json: true, + }; + let report = build_report(&cli); + assert!(report.ok); + assert!(report.steps.iter().any(|step| step.name == "stop-services")); + assert!( + report + .steps + .iter() + .any(|step| step.name == "install-logrotate") + ); + assert!(report.steps.iter().any(|step| step.mutation)); + } + + #[test] + fn missing_env_blocks_apply() { + let dir = tempfile::tempdir().unwrap(); + let health = dir.path().join("health-check.sh"); + fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap(); + let cli = Cli { + env_file: dir.path().join("missing.env"), + data_dir: dir.path().join("data"), + log_dir: dir.path().join("log"), + opt_dir: dir.path().join("opt"), + logrotate_target: dir.path().join("logrotate/activitywatch"), + health_script_target: dir.path().join("bin/aw-health-check"), + health_timer_target: dir.path().join("systemd/aw-health-check.timer"), + health_service_target: dir.path().join("systemd/aw-health-check.service"), + health_script_source: health, + apply: true, + json: true, + }; + let report = build_report(&cli); + assert!(!report.ok); + assert_eq!(report.missing_required.len(), 1); + } +} diff --git a/ansible/deploy_aw_server.yml b/ansible/deploy_aw_server.yml index 4cab6f9..66defd1 100644 --- a/ansible/deploy_aw_server.yml +++ b/ansible/deploy_aw_server.yml @@ -471,6 +471,22 @@ mode: "0755" when: aw_db_health_rust_binary_early.stat.exists | default(false) + - name: Проверить локальный Rust aw-ensure-reliability до Influx проверок + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/aw-ensure-reliability" + delegate_to: localhost + register: aw_ensure_reliability_rust_binary_early + become: false + + - name: Установить Rust aw-ensure-reliability до Influx проверок + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/aw-ensure-reliability" + dest: /usr/local/bin/aw-ensure-reliability + owner: root + group: root + mode: "0755" + when: aw_ensure_reliability_rust_binary_early.stat.exists | default(false) + - name: Проверить локальный Rust aw-db-maintenance до Influx проверок ansible.builtin.stat: path: "{{ aw_rust_release_dir }}/aw-db-maintenance" diff --git a/aw-server/ensure-reliability.sh b/aw-server/ensure-reliability.sh index 093fcd1..8b2c6ce 100644 --- a/aw-server/ensure-reliability.sh +++ b/aw-server/ensure-reliability.sh @@ -5,8 +5,39 @@ set -euo pipefail # Fixes common issues and ensures proper configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" ENV_FILE="/etc/activitywatch/aw-server.env" +if [[ "${1:-}" == "--apply-legacy" ]]; then + shift +else + TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}" + for candidate in \ + "${AW_ENSURE_RELIABILITY_RUST:-}" \ + "$TARGET_ROOT/release/aw-ensure-reliability" \ + "$ROOT_DIR/adk-rust/target/release/aw-ensure-reliability" \ + "/usr/local/bin/aw-ensure-reliability"; do + if [[ -n "$candidate" && -x "$candidate" ]]; then + exec "$candidate" --health-script-source "$SCRIPT_DIR/health-check.sh" "$@" + fi + done + cat >&2 <<'EOF' +ensure-reliability.sh now requires the Rust planner for safe default runs. +Build it first: + cd adk-rust && cargo build --release -p aw-ensure-reliability + +Safe dry-run: + aw-server/ensure-reliability.sh --json + +Explicit Rust apply: + aw-server/ensure-reliability.sh --apply + +Old Bash apply: + aw-server/ensure-reliability.sh --apply-legacy +EOF + exit 2 +fi + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" } @@ -143,4 +174,4 @@ main() { if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@" -fi \ No newline at end of file +fi diff --git a/scripts/check_detmir_rust_release_artifacts.sh b/scripts/check_detmir_rust_release_artifacts.sh index 1d7880c..d3f506a 100644 --- a/scripts/check_detmir_rust_release_artifacts.sh +++ b/scripts/check_detmir_rust_release_artifacts.sh @@ -17,6 +17,7 @@ required_bins=( aw-rus-healthd aw-db-health aw-db-maintenance + aw-ensure-reliability aw-health-check aw-prune-local-state check-aw-data