refactor(detmir): retire python runtime paths

This commit is contained in:
igor04091968
2026-06-03 03:27:52 +03:00
parent 109c31f291
commit dbef90a09e
103 changed files with 914 additions and 19800 deletions
+18 -25
View File
@@ -52,8 +52,8 @@ Ansible, PowerShell или Playwright, он остается там до поя
| `detmir-heal-safe` | Rust binary deployed + legacy script retained | switched for Rust auto |
| `tsj_guardian_watchdog.sh` | Rust service via systemd drop-in + legacy shell retained | switched |
| Telegram `/status`/decision backend | Rust helper + permanent Python bot runtime | backend only |
| `aw-rus-healthd.py` | Rust production via systemd drop-in + legacy Python retained | switched |
| `scripts/dlp-health-check.py` | Rust production binary + legacy Python backup retained | switched |
| `aw-rus-healthd` | Rust production unit, Python entrypoint removed from repo | done |
| `dlp-health-check` | Rust production binary, Python entrypoint removed from repo | done |
| DLP aggregator | Rust production via systemd drop-in + legacy Python retained | switched |
| AW DLP Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched |
| AW worktime Influx exporter | Rust production via systemd drop-in + legacy Python retained | switched |
@@ -763,7 +763,8 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
`cases list/create`, `health check`;
- HTTP client использует no-proxy для локальных AW/DLP сервисов;
- `/usr/local/bin/dlp-admin-cli` установлен как Rust production CLI,
legacy `/usr/local/bin/dlp-admin-cli.py` оставлен для rollback/reference;
legacy `/usr/local/bin/dlp-admin-cli.py` больше не является repo/runtime
path; рабочий CLI: `/usr/local/bin/dlp-admin-cli`;
- parity на AW server совпал с Python для `health`, `policies active`,
`incidents list --since-hours 24 --limit 5`, `cases list --limit 5`;
- production verification зеленый: AW failed units 0,
@@ -885,9 +886,9 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
freshness через `dlp-health-check --json`, и drift checks для
`startOfDay`, `always_active_pattern`, `landingpage`;
- env contract читается из процесса и `/etc/activitywatch/aw-server.env`;
- production `/usr/local/bin/aw-health-check` теперь wrapper, который
предпочитает `/usr/local/bin/aw-health-check-rust` и fallback на
`/opt/activitywatch/aw-rus-ops/health-check.sh`;
- production `/usr/local/bin/aw-health-check` теперь Rust-required wrapper:
вызывает `/usr/local/bin/aw-health-check-rust` и падает с ошибкой, если
Rust binary отсутствует;
- shadow parity на AW server совпал с shell по exit code и ключевым
health-строкам;
- production verification зеленый: `aw-health-check` OK,
@@ -1380,32 +1381,24 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
warnings`, release build OK, `bash -n` OK, artifact check OK,
`quality-gate.sh` OK, DetMir read-only OK with
`dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`.
50. `[done]` Перенести `scripts/merge_aw_server_dbs.py` merge engine на Rust:
50. `[done]` Перенести merge engine на Rust:
- добавлен crate `merge-aw-server-dbs`;
- `scripts/merge_aw_server_dbs.py` остается совместимым с текущим
`python3 merge_aw_server_dbs.py` вызовом, но в начале exec'ает Rust
binary при наличии `MERGE_AW_SERVER_DBS_RUST`,
`$CARGO_TARGET_DIR/release/merge-aw-server-dbs`,
`adk-rust/target/release/merge-aw-server-dbs` или
- рабочий runtime path: `adk-rust/target/release/merge-aw-server-dbs` и
`/usr/local/bin/merge-aw-server-dbs`;
- `MERGE_AW_SERVER_DBS_FORCE_LEGACY=1` форсирует Python fallback;
- `ansible/deploy_aw_server.yml` теперь optional устанавливает
`/usr/local/bin/merge-aw-server-dbs`, если release artifact существует,
сохраняя прежний `python3 /usr/local/bin/merge_aw_server_dbs.py` command;
- legacy `scripts/merge_aw_server_dbs.py` удален из repo/runtime path;
- `ansible/deploy_aw_server.yml` устанавливает
`/usr/local/bin/merge-aw-server-dbs` и использует его без Python;
- Rust сохраняет контракт `--base`, `--overlay`, `--output`, SQLite
backup-copy base DB, bucket matching by id/key/name, duplicate event
suppression by `(starttime,endtime,data)`, JSON summary;
- fallback bugfix: Python legacy больше не падает на `UNIQUE(name)`
name-conflict/update path и считает только реально вставленные buckets;
- offline parity fixture: Rust и Python fallback дали одинаковый JSON
- offline parity fixture: Rust replacement дал ожидаемый JSON
`inserted_buckets=1`, `inserted_events=3` и одинаковые SQLite table rows
(`buckets=3`, `events=5`) на кейсе duplicate event + name-conflict
bucket;
- gates: `cargo fmt --all -- --check`, `cargo test -p
merge-aw-server-dbs` (`2 passed`), `cargo clippy -p
merge-aw-server-dbs --all-targets -- -D warnings`, release build OK,
`python3 -m py_compile scripts/merge_aw_server_dbs.py`, artifact check
OK, `quality-gate.sh` OK, DetMir read-only OK after DLP baseline retry
artifact check OK, `quality-gate.sh` OK, DetMir read-only OK after DLP baseline retry
with `dlp_counts={ok:22,warn:0,fail:0}` and `ok_for_operator=true`.
51. `[done]` Перенести `scripts/prod_backup_restore.sh` в safe-by-default
Rust planner/checker:
@@ -1419,13 +1412,13 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
он печатает plan-only flow и помечает destructive steps;
- если Rust artifact отсутствует, wrapper отказывается запускать
destructive legacy случайно и просит собрать Rust planner;
- старый destructive flow доступен только явно:
`scripts/prod_backup_restore.sh --apply-legacy`;
- старый Python destructive flow удален; `--apply-legacy` больше не
поддерживается;
- Rust `--apply` намеренно запрещен на этом этапе;
- `--check-inputs --json` валидирует env/files/commands без вывода
секретов, проверяет `AW_SSH_PASSWORD`, `AW_WINRM_PASSWORD`, `sshpass`,
`ansible-playbook`, inventory и `merge_aw_server_dbs.py`;
- план сохраняет порядок legacy flow: scp merge script, remote backup dir,
`ansible-playbook`, inventory и `merge-aw-server-dbs`;
- план сохраняет порядок restore flow: scp Rust merge binary, remote backup dir,
DB existence checks, DB backups, stop `activitywatch-server`, merge DB,
install merged DB, 3 ansible playbooks, final AW validation;
- safe smoke: plan text OK, JSON valid (`steps=12`, `missing_count=0`),
@@ -56,9 +56,6 @@ struct Cli {
#[arg(long, default_value = "/etc/systemd/system/aw-health-check.service")]
health_service_target: PathBuf,
#[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/health-check.sh")]
health_script_source: PathBuf,
#[arg(long, default_value_t = false)]
apply: bool,
@@ -72,7 +69,6 @@ enum StepKind {
Check,
Chown,
Chmod,
Copy,
Write,
Systemd,
Sleep,
@@ -138,11 +134,10 @@ fn build_report(cli: &Cli) -> Report {
if !cli.env_file.is_file() {
missing_required.push(format!("env file missing: {}", cli.env_file.display()));
}
if !cli.health_script_target.is_file() && !cli.health_script_source.is_file() {
if !cli.health_script_target.is_file() {
missing_required.push(format!(
"health script target and source missing: {}, {}",
cli.health_script_target.display(),
cli.health_script_source.display()
"health script target missing: {}",
cli.health_script_target.display()
));
}
@@ -193,17 +188,13 @@ fn build_report(cli: &Cli) -> Report {
);
push_step(
&mut steps,
"install-health-script",
StepKind::Copy,
format!(
"copy {} {}",
shell_quote(&cli.health_script_source),
shell_quote(&cli.health_script_target)
),
true,
"check-health-script",
StepKind::Check,
format!("test -x {}", shell_quote(&cli.health_script_target)),
false,
!cli.health_script_target.is_file(),
if cli.health_script_target.is_file() {
"health script already installed".to_string()
"health script already installed by Ansible".to_string()
} else {
"health script target missing".to_string()
},
@@ -331,8 +322,16 @@ fn apply_steps(report: &mut Report) -> Result<()> {
1
}),
},
"check-health-script" => {
let target = Path::new("/usr/local/bin/aw-health-check");
ExecResult {
order: step.order,
name: step.name.clone(),
ok: target.is_file(),
exit_code: Some(if target.is_file() { 0 } else { 1 }),
}
}
"install-logrotate" => write_file_result(step, report, LOGROTATE_CONTENT, 0o644)?,
"install-health-script" => copy_health_script(step, report)?,
"install-health-timer" => write_file_result(step, report, HEALTH_TIMER_CONTENT, 0o644)?,
"install-health-service" => {
write_file_result(step, report, HEALTH_SERVICE_CONTENT, 0o644)?
@@ -371,24 +370,6 @@ fn write_file_result(step: &Step, report: &Report, content: &str, mode: u32) ->
})
}
fn copy_health_script(step: &Step, report: &Report) -> Result<ExecResult> {
let source = source_from_copy_command(&step.command)?;
let target = copy_target_from_command(&step.command)?;
let _ = report;
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
fs::copy(&source, &target)
.with_context(|| format!("copy {} -> {}", source.display(), target.display()))?;
set_mode(&target, 0o755).with_context(|| format!("chmod 0755 {}", target.display()))?;
Ok(ExecResult {
order: step.order,
name: step.name.clone(),
ok: true,
exit_code: Some(0),
})
}
fn run_shell_step(step: &Step) -> Result<ExecResult> {
let status = Command::new("sh")
.arg("-c")
@@ -412,22 +393,6 @@ fn target_from_command(command: &str) -> Result<PathBuf> {
Ok(PathBuf::from(raw.trim_matches('\'')))
}
fn source_from_copy_command(command: &str) -> Result<PathBuf> {
let raw = command
.split_whitespace()
.nth(1)
.context("parse source from copy command")?;
Ok(PathBuf::from(raw.trim_matches('\'')))
}
fn copy_target_from_command(command: &str) -> Result<PathBuf> {
let raw = command
.split_whitespace()
.nth(2)
.context("parse target from copy command")?;
Ok(PathBuf::from(raw.trim_matches('\'')))
}
#[cfg(unix)]
fn set_mode(path: &Path, mode: u32) -> Result<()> {
use std::os::unix::fs::PermissionsExt;
@@ -494,19 +459,19 @@ mod tests {
fn dry_run_marks_legacy_mutations() {
let dir = tempfile::tempdir().unwrap();
let env_file = dir.path().join("aw-server.env");
let health = dir.path().join("health-check.sh");
let health_target = dir.path().join("bin/aw-health-check");
fs::write(&env_file, "AW_BASE_URL=http://127.0.0.1:5600\n").unwrap();
fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap();
fs::create_dir_all(health_target.parent().unwrap()).unwrap();
fs::write(&health_target, "#!/bin/sh\nexit 0\n").unwrap();
let cli = Cli {
env_file,
data_dir: dir.path().join("data"),
log_dir: dir.path().join("log"),
opt_dir: dir.path().join("opt"),
logrotate_target: dir.path().join("logrotate/activitywatch"),
health_script_target: dir.path().join("bin/aw-health-check"),
health_script_target: health_target,
health_timer_target: dir.path().join("systemd/aw-health-check.timer"),
health_service_target: dir.path().join("systemd/aw-health-check.service"),
health_script_source: health,
apply: false,
json: true,
};
@@ -525,18 +490,18 @@ mod tests {
#[test]
fn missing_env_blocks_apply() {
let dir = tempfile::tempdir().unwrap();
let health = dir.path().join("health-check.sh");
fs::write(&health, "#!/bin/sh\nexit 0\n").unwrap();
let health_target = dir.path().join("bin/aw-health-check");
fs::create_dir_all(health_target.parent().unwrap()).unwrap();
fs::write(&health_target, "#!/bin/sh\nexit 0\n").unwrap();
let cli = Cli {
env_file: dir.path().join("missing.env"),
data_dir: dir.path().join("data"),
log_dir: dir.path().join("log"),
opt_dir: dir.path().join("opt"),
logrotate_target: dir.path().join("logrotate/activitywatch"),
health_script_target: dir.path().join("bin/aw-health-check"),
health_script_target: health_target,
health_timer_target: dir.path().join("systemd/aw-health-check.timer"),
health_service_target: dir.path().join("systemd/aw-health-check.service"),
health_script_source: health,
apply: true,
json: true,
};
+1 -1
View File
@@ -20,7 +20,7 @@ use serde_json::{Value, json};
const ENV_FILE: &str = "/etc/activitywatch/aw-server.env";
const DEFAULT_STATE_DIR: &str = "/var/lib/activitywatch/slo";
const DEFAULT_HEALTHD_CMD: &str = "/usr/local/bin/aw-rus-healthd.py --json";
const DEFAULT_HEALTHD_CMD: &str = "/usr/local/bin/aw-rus-healthd-rust --json";
const DEFAULT_HEALTHD_STATE_FILE: &str = "/var/lib/activitywatch/health/aw-rus-health.json";
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
+25 -26
View File
@@ -13,7 +13,7 @@ const DEFAULT_SERVER_USER: &str = "igor";
const DEFAULT_LEGACY_DB: &str = "/root/.local/share/activitywatch/aw-server-rust/sqlite.db";
const DEFAULT_TARGET_DB: &str =
"/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db";
const DEFAULT_REMOTE_MERGE_SCRIPT: &str = "/tmp/merge_aw_server_dbs.py";
const DEFAULT_REMOTE_MERGE_BIN: &str = "/tmp/merge-aw-server-dbs";
#[derive(Debug, Parser)]
#[command(about = "Safe planner/checker for the destructive prod backup restore flow")]
@@ -45,8 +45,8 @@ struct Cli {
#[arg(long, default_value = DEFAULT_TARGET_DB)]
target_db: String,
#[arg(long, default_value = DEFAULT_REMOTE_MERGE_SCRIPT)]
remote_merge_script: String,
#[arg(long, default_value = DEFAULT_REMOTE_MERGE_BIN)]
remote_merge_bin: String,
#[arg(long)]
apply: bool,
@@ -63,7 +63,7 @@ struct Plan {
remote_backup_dir: String,
legacy_db: String,
target_db: String,
remote_merge_script: String,
remote_merge_bin: String,
required_env: Vec<Requirement>,
required_commands: Vec<Requirement>,
required_files: Vec<Requirement>,
@@ -101,7 +101,7 @@ fn run() -> Result<i32> {
let cli = Cli::parse();
if cli.apply {
bail!(
"Rust apply is intentionally disabled for this stage; use scripts/prod_backup_restore.sh --apply-legacy only for explicit legacy restore"
"Rust apply is intentionally disabled for this stage; run without --apply to review the safe restore plan"
);
}
let root = cli
@@ -128,7 +128,7 @@ fn run() -> Result<i32> {
} else {
root.join(&cli.inventory)
};
let merge_script = root.join("scripts/merge_aw_server_dbs.py");
let merge_bin = root.join("adk-rust/target/release/merge-aw-server-dbs");
let plan = build_plan(
&root,
&env_file,
@@ -139,9 +139,9 @@ fn run() -> Result<i32> {
&remote_backup_dir,
&cli.legacy_db,
&cli.target_db,
&cli.remote_merge_script,
&cli.remote_merge_bin,
&inventory,
&merge_script,
&merge_bin,
);
if cli.json {
@@ -168,9 +168,9 @@ fn build_plan(
remote_backup_dir: &str,
legacy_db: &str,
target_db: &str,
remote_merge_script: &str,
remote_merge_bin: &str,
inventory: &Path,
merge_script: &Path,
merge_bin: &Path,
) -> Plan {
let required_env = ["AW_SSH_PASSWORD", "AW_WINRM_PASSWORD"]
.into_iter()
@@ -199,24 +199,21 @@ fn build_plan(
},
})
.collect::<Vec<_>>();
let required_files = [
("inventory", inventory),
("merge_aw_server_dbs.py", merge_script),
]
.into_iter()
.map(|(name, path)| Requirement {
name: name.to_string(),
ok: path.is_file(),
detail: path.display().to_string(),
})
.collect::<Vec<_>>();
let required_files = [("inventory", inventory), ("merge-aw-server-dbs", merge_bin)]
.into_iter()
.map(|(name, path)| Requirement {
name: name.to_string(),
ok: path.is_file(),
detail: path.display().to_string(),
})
.collect::<Vec<_>>();
let mut steps = Vec::new();
push_step(
&mut steps,
"scp",
format!(
"sshpass scp scripts/merge_aw_server_dbs.py {server_user}@{server_host}:{remote_merge_script}"
"sshpass scp adk-rust/target/release/merge-aw-server-dbs {server_user}@{server_host}:{remote_merge_bin}"
),
false,
);
@@ -258,7 +255,7 @@ fn build_plan(
&mut steps,
"ssh",
format!(
"sudo python3 '{remote_merge_script}' --base '{legacy_db}' --overlay '{target_db}' --output '{remote_backup_dir}/sqlite.merged.db'"
"sudo chmod 0755 '{remote_merge_bin}' && sudo '{remote_merge_bin}' --base '{legacy_db}' --overlay '{target_db}' --output '{remote_backup_dir}/sqlite.merged.db'"
),
true,
);
@@ -307,7 +304,7 @@ fn build_plan(
remote_backup_dir: remote_backup_dir.to_string(),
legacy_db: legacy_db.to_string(),
target_db: target_db.to_string(),
remote_merge_script: remote_merge_script.to_string(),
remote_merge_bin: remote_merge_bin.to_string(),
required_env,
required_commands,
required_files,
@@ -443,7 +440,9 @@ mod tests {
fn plan_marks_destructive_steps() {
let dir = tempfile::tempdir().unwrap();
let inventory = dir.path().join("ansible/inventory.ini");
let merge = dir.path().join("scripts/merge_aw_server_dbs.py");
let merge = dir
.path()
.join("adk-rust/target/release/merge-aw-server-dbs");
fs::create_dir_all(inventory.parent().unwrap()).unwrap();
fs::create_dir_all(merge.parent().unwrap()).unwrap();
fs::write(&inventory, "").unwrap();
@@ -461,7 +460,7 @@ mod tests {
"/var/lib/activitywatch/backups/prod-restore-20260602-000000",
DEFAULT_LEGACY_DB,
DEFAULT_TARGET_DB,
DEFAULT_REMOTE_MERGE_SCRIPT,
DEFAULT_REMOTE_MERGE_BIN,
&inventory,
&merge,
);
@@ -47,23 +47,18 @@ const AW_SERVER_FILES: &[&str] = &[
"aw-server/apply_webui_ru_patch.sh",
"aw-server/aw-host-groups.json",
"aw-server/aw-ru-patch.js",
"aw-server/aw-rus-healthd.py",
"aw-server/aw-rus-healthd.service",
"aw-server/aw-rus-healthd.timer",
"aw-server/aw-browser-smoke.service",
"aw-server/aw-browser-smoke.timer",
"aw-server/aw-slo-monitor.py",
"aw-server/aw-slo-monitor.service",
"aw-server/aw-slo-monitor.timer",
"aw-server/aw-server.env.example",
"aw-server/aw-sw-cleanup.js",
"aw-server/aw-worktime-api.py",
"aw-server/aw-worktime-api.service",
"aw-server/aw-worktime-prewarm.sh",
"aw-server/aw-worktime-prewarm.service",
"aw-server/aw-worktime-prewarm.timer",
"aw-server/aw-worktime-panel.js",
"aw-server/health-check.sh",
"aw-server/install_aw_server.sh",
"aw-server/settings/classes-worktime.json",
"aw-server/settings/views-default.json",