feat(ops): add DetMir production readiness check
This commit is contained in:
@@ -23,6 +23,18 @@ jobs:
|
|||||||
- name: Run production inventory placeholder guard self-test
|
- name: Run production inventory placeholder guard self-test
|
||||||
run: bash scripts/check_production_inventory_placeholders.sh --self-test
|
run: bash scripts/check_production_inventory_placeholders.sh --self-test
|
||||||
|
|
||||||
|
rust-runtime-guard:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Test detmir-core runtime guard
|
||||||
|
run: cargo test --manifest-path adk-rust/Cargo.toml -p detmir-core runtime_guard
|
||||||
|
|
||||||
|
- name: Test detmir readiness crate
|
||||||
|
run: cargo test --manifest-path adk-rust/Cargo.toml -p detmir-readiness
|
||||||
|
|
||||||
powershell-analyzer:
|
powershell-analyzer:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
Generated
+15
@@ -590,6 +590,21 @@ dependencies = [
|
|||||||
"tiny_http",
|
"tiny_http",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "detmir-readiness"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"chrono",
|
||||||
|
"clap",
|
||||||
|
"detmir-core",
|
||||||
|
"reqwest",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"tempfile",
|
||||||
|
"urlencoding",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "detmir-state"
|
name = "detmir-state"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ members = [
|
|||||||
"crates/aw-browser-smoke",
|
"crates/aw-browser-smoke",
|
||||||
"crates/diag-and-manual-restart",
|
"crates/diag-and-manual-restart",
|
||||||
"crates/detmir-grafana-check",
|
"crates/detmir-grafana-check",
|
||||||
|
"crates/detmir-readiness",
|
||||||
"crates/detmir-portal",
|
"crates/detmir-portal",
|
||||||
"crates/aw-slo-monitor",
|
"crates/aw-slo-monitor",
|
||||||
"crates/aw-rus-healthd",
|
"crates/aw-rus-healthd",
|
||||||
|
|||||||
@@ -638,12 +638,16 @@ Production readiness checklist:
|
|||||||
- [ ] `ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml --syntax-check` passes;
|
- [ ] `ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml --syntax-check` passes;
|
||||||
- [ ] `aw-worktime-influx-exporter.service` and `aw-dlp-influx-exporter.service`
|
- [ ] `aw-worktime-influx-exporter.service` and `aw-dlp-influx-exporter.service`
|
||||||
complete once and write points;
|
complete once and write points;
|
||||||
|
- [ ] `detmir-readiness --json` returns `status=OK`;
|
||||||
- [ ] `detmir-check --json`, `detmir-status --json` and Grafana check are green;
|
- [ ] `detmir-check --json`, `detmir-status --json` and Grafana check are green;
|
||||||
- [ ] rollback path for changed binaries/env files is known.
|
- [ ] rollback path for changed binaries/env files is known.
|
||||||
|
|
||||||
|
Full readiness procedure is documented in `docs/PRODUCTION_READINESS_RU.md`.
|
||||||
|
|
||||||
На Proxmox:
|
На Proxmox:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
detmir-readiness --json
|
||||||
detmir-status --json
|
detmir-status --json
|
||||||
detmir-check --json
|
detmir-check --json
|
||||||
detmir-dlp
|
detmir-dlp
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "detmir-readiness"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
publish.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
anyhow.workspace = true
|
||||||
|
chrono.workspace = true
|
||||||
|
clap.workspace = true
|
||||||
|
detmir-core.workspace = true
|
||||||
|
reqwest.workspace = true
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
tempfile.workspace = true
|
||||||
|
urlencoding.workspace = true
|
||||||
@@ -0,0 +1,581 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::Command;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use chrono::Utc;
|
||||||
|
use clap::Parser;
|
||||||
|
use detmir_core::runtime_guard::ensure_influx_runtime_config;
|
||||||
|
use detmir_core::{StatusLevel, exit_codes, now_utc_rfc3339};
|
||||||
|
use reqwest::blocking::Client;
|
||||||
|
use serde::Serialize;
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
const DEFAULT_AW_ENV_FILE: &str = "/etc/activitywatch/aw-server.env";
|
||||||
|
const DEFAULT_GRAFANA_ENV_FILE: &str = "/etc/detmir-grafana-check.env";
|
||||||
|
const DEFAULT_GRAFANA_URL: &str = "http://127.0.0.1:3000";
|
||||||
|
const DEFAULT_GRAFANA_DATASOURCE_UID: &str = "influxdb_aw";
|
||||||
|
const DEFAULT_SYSTEMD_SERVICES: &str = "activitywatch-server,aw-worktime-api,aw-worktime-influx-exporter.timer,aw-dlp-influx-exporter.timer";
|
||||||
|
|
||||||
|
#[derive(Debug, Parser)]
|
||||||
|
#[command(
|
||||||
|
about = "DetMir production readiness check: env, systemd, Influx write and Grafana datasource"
|
||||||
|
)]
|
||||||
|
struct Cli {
|
||||||
|
#[arg(long)]
|
||||||
|
json: bool,
|
||||||
|
|
||||||
|
#[arg(long, default_value = DEFAULT_AW_ENV_FILE)]
|
||||||
|
aw_env_file: PathBuf,
|
||||||
|
|
||||||
|
#[arg(long, default_value = DEFAULT_GRAFANA_ENV_FILE)]
|
||||||
|
grafana_env_file: PathBuf,
|
||||||
|
|
||||||
|
#[arg(long, default_value = DEFAULT_SYSTEMD_SERVICES)]
|
||||||
|
systemd_services: String,
|
||||||
|
|
||||||
|
#[arg(long, default_value_t = 15)]
|
||||||
|
timeout_seconds: u64,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
skip_systemd: bool,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
skip_influx_write: bool,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
allow_disabled_influx: bool,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
skip_grafana: bool,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
grafana_url: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
grafana_user: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
grafana_password: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long, default_value = DEFAULT_GRAFANA_DATASOURCE_UID)]
|
||||||
|
grafana_datasource_uid: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct Report {
|
||||||
|
ok: bool,
|
||||||
|
status: StatusLevel,
|
||||||
|
generated_at_utc: String,
|
||||||
|
counts: Counts,
|
||||||
|
checks: Vec<Check>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Serialize)]
|
||||||
|
struct Counts {
|
||||||
|
ok: usize,
|
||||||
|
warn: usize,
|
||||||
|
fail: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct Check {
|
||||||
|
name: String,
|
||||||
|
status: StatusLevel,
|
||||||
|
summary: String,
|
||||||
|
details: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct InfluxConfig {
|
||||||
|
prefix: &'static str,
|
||||||
|
enabled_key: &'static str,
|
||||||
|
enabled: bool,
|
||||||
|
url: String,
|
||||||
|
org: String,
|
||||||
|
bucket: String,
|
||||||
|
token: String,
|
||||||
|
hosts: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let cli = Cli::parse();
|
||||||
|
let code = match run(&cli) {
|
||||||
|
Ok(report) => {
|
||||||
|
if cli.json {
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&report).expect("serialize report")
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
print_text(&report);
|
||||||
|
}
|
||||||
|
report.status.exit_code()
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
eprintln!("{err:#}");
|
||||||
|
exit_codes::ERROR
|
||||||
|
}
|
||||||
|
};
|
||||||
|
std::process::exit(code);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(cli: &Cli) -> Result<Report> {
|
||||||
|
let mut aw_env = load_env_file(&cli.aw_env_file)?;
|
||||||
|
overlay_process_env(&mut aw_env);
|
||||||
|
let mut grafana_env = load_env_file(&cli.grafana_env_file)?;
|
||||||
|
overlay_process_env(&mut grafana_env);
|
||||||
|
|
||||||
|
let client = Client::builder()
|
||||||
|
.timeout(Duration::from_secs(cli.timeout_seconds))
|
||||||
|
.no_proxy()
|
||||||
|
.build()
|
||||||
|
.context("build HTTP client")?;
|
||||||
|
|
||||||
|
let mut checks = Vec::new();
|
||||||
|
let worktime = influx_config(&aw_env, "AW_WORKTIME_INFLUX");
|
||||||
|
let dlp = influx_config(&aw_env, "AW_DLP_INFLUX");
|
||||||
|
|
||||||
|
checks.push(check_influx_env(&worktime, cli.allow_disabled_influx));
|
||||||
|
checks.push(check_influx_env(&dlp, cli.allow_disabled_influx));
|
||||||
|
|
||||||
|
if cli.skip_systemd {
|
||||||
|
checks.push(warn("systemd", "systemd checks skipped", json!({})));
|
||||||
|
} else {
|
||||||
|
checks.extend(check_systemd_services(&cli.systemd_services));
|
||||||
|
}
|
||||||
|
|
||||||
|
if cli.skip_influx_write {
|
||||||
|
checks.push(warn(
|
||||||
|
"influx:write",
|
||||||
|
"Influx write probes skipped",
|
||||||
|
json!({}),
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
checks.push(check_influx_write(&client, "worktime", &worktime));
|
||||||
|
checks.push(check_influx_write(&client, "dlp", &dlp));
|
||||||
|
}
|
||||||
|
|
||||||
|
if cli.skip_grafana {
|
||||||
|
checks.push(warn(
|
||||||
|
"grafana:datasource",
|
||||||
|
"Grafana datasource check skipped",
|
||||||
|
json!({}),
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
checks.push(check_grafana_datasource(
|
||||||
|
&client,
|
||||||
|
cli,
|
||||||
|
&grafana_env,
|
||||||
|
&cli.grafana_datasource_uid,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let (status, counts) = summarize(&checks);
|
||||||
|
Ok(Report {
|
||||||
|
ok: status == StatusLevel::Ok,
|
||||||
|
status,
|
||||||
|
generated_at_utc: now_utc_rfc3339(),
|
||||||
|
counts,
|
||||||
|
checks,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_env_file(path: &Path) -> Result<BTreeMap<String, String>> {
|
||||||
|
let mut values = BTreeMap::new();
|
||||||
|
let Ok(text) = fs::read_to_string(path) else {
|
||||||
|
return Ok(values);
|
||||||
|
};
|
||||||
|
for line in text.lines() {
|
||||||
|
let line = line.trim();
|
||||||
|
if line.is_empty() || line.starts_with('#') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let line = line.strip_prefix("export ").unwrap_or(line);
|
||||||
|
let Some((key, value)) = line.split_once('=') else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
values.insert(key.trim().to_string(), unquote(value.trim()));
|
||||||
|
}
|
||||||
|
Ok(values)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn overlay_process_env(values: &mut BTreeMap<String, String>) {
|
||||||
|
for (key, value) in std::env::vars() {
|
||||||
|
if !value.trim().is_empty() {
|
||||||
|
values.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unquote(value: &str) -> String {
|
||||||
|
let bytes = value.as_bytes();
|
||||||
|
if bytes.len() >= 2
|
||||||
|
&& ((bytes[0] == b'"' && bytes[bytes.len() - 1] == b'"')
|
||||||
|
|| (bytes[0] == b'\'' && bytes[bytes.len() - 1] == b'\''))
|
||||||
|
{
|
||||||
|
value[1..value.len() - 1].to_string()
|
||||||
|
} else {
|
||||||
|
value.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_value(env: &BTreeMap<String, String>, key: &str, default: &str) -> String {
|
||||||
|
env.get(key)
|
||||||
|
.map(|value| value.trim())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.unwrap_or(default)
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_bool(env: &BTreeMap<String, String>, key: &str, default: bool) -> bool {
|
||||||
|
env.get(key)
|
||||||
|
.map(|value| value.trim().to_ascii_lowercase())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(|value| matches!(value.as_str(), "1" | "true" | "yes" | "on"))
|
||||||
|
.unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_csv(value: &str) -> Vec<String> {
|
||||||
|
value
|
||||||
|
.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|item| !item.is_empty())
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn influx_config(env: &BTreeMap<String, String>, prefix: &'static str) -> InfluxConfig {
|
||||||
|
let enabled_key = match prefix {
|
||||||
|
"AW_WORKTIME_INFLUX" => "AW_WORKTIME_INFLUX_ENABLED",
|
||||||
|
"AW_DLP_INFLUX" => "AW_DLP_INFLUX_ENABLED",
|
||||||
|
_ => unreachable!("unknown Influx prefix"),
|
||||||
|
};
|
||||||
|
InfluxConfig {
|
||||||
|
prefix,
|
||||||
|
enabled_key,
|
||||||
|
enabled: env_bool(env, enabled_key, false),
|
||||||
|
url: env_value(env, &format!("{prefix}_URL"), ""),
|
||||||
|
org: env_value(env, &format!("{prefix}_ORG"), ""),
|
||||||
|
bucket: env_value(env, &format!("{prefix}_BUCKET"), ""),
|
||||||
|
token: env_value(env, &format!("{prefix}_TOKEN"), ""),
|
||||||
|
hosts: split_csv(&env_value(env, &format!("{prefix}_HOSTS"), "")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_influx_env(config: &InfluxConfig, allow_disabled: bool) -> Check {
|
||||||
|
if !config.enabled {
|
||||||
|
let status = if allow_disabled {
|
||||||
|
StatusLevel::Warn
|
||||||
|
} else {
|
||||||
|
StatusLevel::Fail
|
||||||
|
};
|
||||||
|
return Check {
|
||||||
|
name: format!("env:{}", config.prefix),
|
||||||
|
status,
|
||||||
|
summary: format!("{} is disabled", config.enabled_key),
|
||||||
|
details: json!({ "enabled": false, "enabled_key": config.enabled_key }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
match ensure_influx_runtime_config(
|
||||||
|
config.prefix,
|
||||||
|
&config.url,
|
||||||
|
&config.org,
|
||||||
|
&config.bucket,
|
||||||
|
&config.token,
|
||||||
|
&config.hosts,
|
||||||
|
) {
|
||||||
|
Ok(()) => ok(
|
||||||
|
format!("env:{}", config.prefix),
|
||||||
|
"Influx runtime env is production-ready",
|
||||||
|
json!({
|
||||||
|
"enabled": true,
|
||||||
|
"url_present": !config.url.is_empty(),
|
||||||
|
"org": config.org,
|
||||||
|
"bucket": config.bucket,
|
||||||
|
"token_present": !config.token.is_empty(),
|
||||||
|
"host_count": config.hosts.len(),
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Err(err) => fail(
|
||||||
|
format!("env:{}", config.prefix),
|
||||||
|
format!("Influx runtime env failed validation: {err}"),
|
||||||
|
json!({ "enabled": true, "token_redacted": true }),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_systemd_services(csv: &str) -> Vec<Check> {
|
||||||
|
split_csv(csv)
|
||||||
|
.into_iter()
|
||||||
|
.map(|service| {
|
||||||
|
let output = Command::new("systemctl")
|
||||||
|
.arg("is-active")
|
||||||
|
.arg(&service)
|
||||||
|
.output();
|
||||||
|
match output {
|
||||||
|
Ok(output) if output.status.success() => ok(
|
||||||
|
format!("systemd:{service}"),
|
||||||
|
"service is active",
|
||||||
|
json!({ "service": service }),
|
||||||
|
),
|
||||||
|
Ok(output) => fail(
|
||||||
|
format!("systemd:{service}"),
|
||||||
|
format!(
|
||||||
|
"service is not active: {}",
|
||||||
|
String::from_utf8_lossy(&output.stdout).trim()
|
||||||
|
),
|
||||||
|
json!({ "service": service, "exit_code": output.status.code() }),
|
||||||
|
),
|
||||||
|
Err(err) => fail(
|
||||||
|
format!("systemd:{service}"),
|
||||||
|
format!("systemctl failed: {err}"),
|
||||||
|
json!({ "service": service }),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_influx_write(client: &Client, label: &str, config: &InfluxConfig) -> Check {
|
||||||
|
if !config.enabled {
|
||||||
|
return fail(
|
||||||
|
format!("influx:write:{label}"),
|
||||||
|
format!("{} is disabled", config.enabled_key),
|
||||||
|
json!({ "enabled": false }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Err(err) = ensure_influx_runtime_config(
|
||||||
|
config.prefix,
|
||||||
|
&config.url,
|
||||||
|
&config.org,
|
||||||
|
&config.bucket,
|
||||||
|
&config.token,
|
||||||
|
&config.hosts,
|
||||||
|
) {
|
||||||
|
return fail(
|
||||||
|
format!("influx:write:{label}"),
|
||||||
|
format!("Influx write probe skipped because env is invalid: {err}"),
|
||||||
|
json!({ "token_redacted": true }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let url = format!(
|
||||||
|
"{}/api/v2/write?org={}&bucket={}&precision=ns",
|
||||||
|
config.url.trim_end_matches('/'),
|
||||||
|
urlencoding::encode(&config.org),
|
||||||
|
urlencoding::encode(&config.bucket)
|
||||||
|
);
|
||||||
|
let host = config
|
||||||
|
.hosts
|
||||||
|
.first()
|
||||||
|
.map(String::as_str)
|
||||||
|
.unwrap_or("unknown-host");
|
||||||
|
let ts = Utc::now().timestamp_nanos_opt().unwrap_or(0);
|
||||||
|
let payload = format!(
|
||||||
|
"detmir_readiness_heartbeat,channel={},host={} value=1i {}\n",
|
||||||
|
escape_tag(label),
|
||||||
|
escape_tag(host),
|
||||||
|
ts
|
||||||
|
);
|
||||||
|
match client
|
||||||
|
.post(url)
|
||||||
|
.header("Authorization", format!("Token {}", config.token))
|
||||||
|
.header("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
.body(payload)
|
||||||
|
.send()
|
||||||
|
.and_then(|resp| resp.error_for_status())
|
||||||
|
{
|
||||||
|
Ok(_) => ok(
|
||||||
|
format!("influx:write:{label}"),
|
||||||
|
"Influx write probe succeeded",
|
||||||
|
json!({ "bucket": config.bucket, "host": host, "token_redacted": true }),
|
||||||
|
),
|
||||||
|
Err(err) => fail(
|
||||||
|
format!("influx:write:{label}"),
|
||||||
|
format!("Influx write probe failed: {err}"),
|
||||||
|
json!({ "bucket": config.bucket, "token_redacted": true }),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_grafana_datasource(
|
||||||
|
client: &Client,
|
||||||
|
cli: &Cli,
|
||||||
|
env: &BTreeMap<String, String>,
|
||||||
|
uid: &str,
|
||||||
|
) -> Check {
|
||||||
|
let grafana_url = cli
|
||||||
|
.grafana_url
|
||||||
|
.clone()
|
||||||
|
.or_else(|| env.get("DETMIR_GRAFANA_URL").cloned())
|
||||||
|
.or_else(|| env.get("GRAFANA_URL").cloned())
|
||||||
|
.unwrap_or_else(|| DEFAULT_GRAFANA_URL.to_string())
|
||||||
|
.trim_end_matches('/')
|
||||||
|
.to_string();
|
||||||
|
let user = cli
|
||||||
|
.grafana_user
|
||||||
|
.clone()
|
||||||
|
.or_else(|| env.get("DETMIR_GRAFANA_USER").cloned())
|
||||||
|
.or_else(|| env.get("GRAFANA_USER").cloned());
|
||||||
|
let password = cli
|
||||||
|
.grafana_password
|
||||||
|
.clone()
|
||||||
|
.or_else(|| env.get("DETMIR_GRAFANA_PASSWORD").cloned())
|
||||||
|
.or_else(|| env.get("GRAFANA_PASSWORD").cloned());
|
||||||
|
let url = format!("{grafana_url}/api/datasources/uid/{uid}/health");
|
||||||
|
let mut request = client.get(url);
|
||||||
|
if let Some(user) = user.as_deref() {
|
||||||
|
request = request.basic_auth(user, password.as_deref());
|
||||||
|
}
|
||||||
|
match request.send().and_then(|resp| resp.error_for_status()) {
|
||||||
|
Ok(resp) => {
|
||||||
|
let value = resp.json::<Value>().unwrap_or_else(|_| json!({}));
|
||||||
|
let status = value
|
||||||
|
.get("status")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("unknown");
|
||||||
|
if status.eq_ignore_ascii_case("ok") {
|
||||||
|
ok(
|
||||||
|
"grafana:datasource",
|
||||||
|
"Grafana datasource health is OK",
|
||||||
|
json!({ "uid": uid, "grafana_url": grafana_url, "auth_present": user.is_some() }),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
fail(
|
||||||
|
"grafana:datasource",
|
||||||
|
format!("Grafana datasource health is not OK: {status}"),
|
||||||
|
json!({ "uid": uid, "grafana_url": grafana_url }),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => fail(
|
||||||
|
"grafana:datasource",
|
||||||
|
format!("Grafana datasource health request failed: {err}"),
|
||||||
|
json!({ "uid": uid, "grafana_url": grafana_url, "auth_present": user.is_some() }),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn escape_tag(value: &str) -> String {
|
||||||
|
value
|
||||||
|
.replace('\\', "\\\\")
|
||||||
|
.replace(' ', "\\ ")
|
||||||
|
.replace(',', "\\,")
|
||||||
|
.replace('=', "\\=")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ok(name: impl Into<String>, summary: impl Into<String>, details: Value) -> Check {
|
||||||
|
Check {
|
||||||
|
name: name.into(),
|
||||||
|
status: StatusLevel::Ok,
|
||||||
|
summary: summary.into(),
|
||||||
|
details,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn warn(name: impl Into<String>, summary: impl Into<String>, details: Value) -> Check {
|
||||||
|
Check {
|
||||||
|
name: name.into(),
|
||||||
|
status: StatusLevel::Warn,
|
||||||
|
summary: summary.into(),
|
||||||
|
details,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fail(name: impl Into<String>, summary: impl Into<String>, details: Value) -> Check {
|
||||||
|
Check {
|
||||||
|
name: name.into(),
|
||||||
|
status: StatusLevel::Fail,
|
||||||
|
summary: summary.into(),
|
||||||
|
details,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summarize(checks: &[Check]) -> (StatusLevel, Counts) {
|
||||||
|
let mut counts = Counts::default();
|
||||||
|
for check in checks {
|
||||||
|
match check.status {
|
||||||
|
StatusLevel::Ok => counts.ok += 1,
|
||||||
|
StatusLevel::Warn => counts.warn += 1,
|
||||||
|
StatusLevel::Fail | StatusLevel::Unknown => counts.fail += 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let status = if counts.fail > 0 {
|
||||||
|
StatusLevel::Fail
|
||||||
|
} else if counts.warn > 0 {
|
||||||
|
StatusLevel::Warn
|
||||||
|
} else {
|
||||||
|
StatusLevel::Ok
|
||||||
|
};
|
||||||
|
(status, counts)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print_text(report: &Report) {
|
||||||
|
println!("DetMir readiness: {}", report.status);
|
||||||
|
for check in &report.checks {
|
||||||
|
println!("- {}: {} - {}", check.status, check.name, check.summary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_env_file_without_quotes() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("runtime.env");
|
||||||
|
fs::write(
|
||||||
|
&path,
|
||||||
|
"export A=\"one\"\nB='two'\n# skip\nBROKEN\nC=three\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let env = load_env_file(&path).unwrap();
|
||||||
|
assert_eq!(env.get("A").unwrap(), "one");
|
||||||
|
assert_eq!(env.get("B").unwrap(), "two");
|
||||||
|
assert_eq!(env.get("C").unwrap(), "three");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validates_influx_env_placeholders() {
|
||||||
|
let mut env = BTreeMap::new();
|
||||||
|
env.insert("AW_WORKTIME_INFLUX_ENABLED".to_string(), "true".to_string());
|
||||||
|
env.insert(
|
||||||
|
"AW_WORKTIME_INFLUX_URL".to_string(),
|
||||||
|
"http://192.0.2.10:8086".to_string(),
|
||||||
|
);
|
||||||
|
env.insert("AW_WORKTIME_INFLUX_ORG".to_string(), "proxmox".to_string());
|
||||||
|
env.insert(
|
||||||
|
"AW_WORKTIME_INFLUX_BUCKET".to_string(),
|
||||||
|
"aw_metrics".to_string(),
|
||||||
|
);
|
||||||
|
env.insert(
|
||||||
|
"AW_WORKTIME_INFLUX_TOKEN".to_string(),
|
||||||
|
"CHANGE_ME".to_string(),
|
||||||
|
);
|
||||||
|
env.insert(
|
||||||
|
"AW_WORKTIME_INFLUX_HOSTS".to_string(),
|
||||||
|
"HOST-EXAMPLE".to_string(),
|
||||||
|
);
|
||||||
|
let config = influx_config(&env, "AW_WORKTIME_INFLUX");
|
||||||
|
let check = check_influx_env(&config, false);
|
||||||
|
assert_eq!(check.status, StatusLevel::Fail);
|
||||||
|
assert!(check.summary.contains("AW_WORKTIME_INFLUX_URL"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn summarizes_warn_and_fail() {
|
||||||
|
let checks = vec![
|
||||||
|
ok("a", "a", json!({})),
|
||||||
|
warn("b", "b", json!({})),
|
||||||
|
fail("c", "c", json!({})),
|
||||||
|
];
|
||||||
|
let (status, counts) = summarize(&checks);
|
||||||
|
assert_eq!(status, StatusLevel::Fail);
|
||||||
|
assert_eq!(counts.ok, 1);
|
||||||
|
assert_eq!(counts.warn, 1);
|
||||||
|
assert_eq!(counts.fail, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1586,6 +1586,28 @@
|
|||||||
mode: "0644"
|
mode: "0644"
|
||||||
when: aw_dlp_influx_enabled | default(false) | bool
|
when: aw_dlp_influx_enabled | default(false) | bool
|
||||||
|
|
||||||
|
- name: Проверить локальный Rust DetMir readiness checker
|
||||||
|
ansible.builtin.stat:
|
||||||
|
path: "{{ aw_rust_release_dir }}/detmir-readiness"
|
||||||
|
delegate_to: localhost
|
||||||
|
register: detmir_readiness_rust_binary
|
||||||
|
become: false
|
||||||
|
|
||||||
|
- name: Требовать Rust DetMir readiness checker artifact
|
||||||
|
ansible.builtin.assert:
|
||||||
|
that:
|
||||||
|
- detmir_readiness_rust_binary.stat.exists | default(false)
|
||||||
|
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/detmir-readiness"
|
||||||
|
|
||||||
|
- name: Установить Rust DetMir readiness checker
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: "{{ aw_rust_release_dir }}/detmir-readiness"
|
||||||
|
dest: /usr/local/bin/detmir-readiness
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
mode: "0755"
|
||||||
|
when: detmir_readiness_rust_binary.stat.exists | default(false)
|
||||||
|
|
||||||
- name: Перезагрузить systemd после установки AW worktime API
|
- name: Перезагрузить systemd после установки AW worktime API
|
||||||
ansible.builtin.systemd:
|
ansible.builtin.systemd:
|
||||||
daemon_reload: true
|
daemon_reload: true
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# Контроль готовности промышленного внедрения
|
||||||
|
|
||||||
|
`detmir-readiness` - единая команда preflight-контроля перед внедрением,
|
||||||
|
релизом или изменением production runtime.
|
||||||
|
|
||||||
|
Команда проверяет:
|
||||||
|
|
||||||
|
- runtime env без public placeholders;
|
||||||
|
- активность обязательных systemd units;
|
||||||
|
- реальную запись в InfluxDB;
|
||||||
|
- health Grafana datasource.
|
||||||
|
|
||||||
|
## Базовый запуск
|
||||||
|
|
||||||
|
На AW server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
detmir-readiness --json
|
||||||
|
```
|
||||||
|
|
||||||
|
Ожидаемый результат:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"status": "OK"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Коды возврата:
|
||||||
|
|
||||||
|
- `0` - готово к промышленной эксплуатации;
|
||||||
|
- `2` - readiness check нашел `WARN` или `FAIL`;
|
||||||
|
- `1` - сама команда не смогла выполниться.
|
||||||
|
|
||||||
|
## Private production inventory
|
||||||
|
|
||||||
|
Перед rollout private override-файлы проверяются отдельно:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/check_production_inventory_placeholders.sh --strict \
|
||||||
|
private-config/runtime.env \
|
||||||
|
private-config/ansible-vars.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
`--strict` предназначен только для private production-файлов. Публичные
|
||||||
|
tracked defaults и `.example` файлы могут содержать `HOST-EXAMPLE` и TEST-NET
|
||||||
|
адреса, потому что они не являются production source of truth.
|
||||||
|
|
||||||
|
## Что считается отказом
|
||||||
|
|
||||||
|
`detmir-readiness` возвращает `FAIL`, если:
|
||||||
|
|
||||||
|
- включенный Influx exporter получил пустой или example URL/org/bucket/token/host;
|
||||||
|
- systemd unit из обязательного списка не active;
|
||||||
|
- Influx write-probe не смог записать heartbeat;
|
||||||
|
- Grafana datasource health не `OK`.
|
||||||
|
|
||||||
|
## Полезные параметры
|
||||||
|
|
||||||
|
```bash
|
||||||
|
detmir-readiness --json \
|
||||||
|
--aw-env-file /etc/activitywatch/aw-server.env \
|
||||||
|
--grafana-env-file /etc/detmir-grafana-check.env \
|
||||||
|
--grafana-datasource-uid influxdb_aw
|
||||||
|
```
|
||||||
|
|
||||||
|
Для диагностики без write-probe:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
detmir-readiness --json --skip-influx-write
|
||||||
|
```
|
||||||
|
|
||||||
|
Для контура, где Influx временно не входит в профиль внедрения:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
detmir-readiness --json --allow-disabled-influx
|
||||||
|
```
|
||||||
|
|
||||||
|
Такой запуск допустим только как временный исключительный режим; для полного
|
||||||
|
commercial DetMir contour Influx/Grafana должны быть зелеными.
|
||||||
@@ -37,6 +37,7 @@ required_bins=(
|
|||||||
aw-browser-smoke
|
aw-browser-smoke
|
||||||
diag-and-manual-restart
|
diag-and-manual-restart
|
||||||
detmir-grafana-check
|
detmir-grafana-check
|
||||||
|
detmir-readiness
|
||||||
detmir-portal
|
detmir-portal
|
||||||
dlp-health-check
|
dlp-health-check
|
||||||
dlp-content-analyzer
|
dlp-content-analyzer
|
||||||
|
|||||||
@@ -4,18 +4,22 @@ set -euo pipefail
|
|||||||
usage() {
|
usage() {
|
||||||
cat <<'EOF'
|
cat <<'EOF'
|
||||||
Usage:
|
Usage:
|
||||||
scripts/check_production_inventory_placeholders.sh [--allow-missing] FILE...
|
scripts/check_production_inventory_placeholders.sh [--allow-missing] [--strict] FILE...
|
||||||
DETMIR_PRODUCTION_CONFIG_PATHS="file1:file2" scripts/check_production_inventory_placeholders.sh
|
DETMIR_PRODUCTION_CONFIG_PATHS="file1:file2" scripts/check_production_inventory_placeholders.sh
|
||||||
scripts/check_production_inventory_placeholders.sh --self-test
|
scripts/check_production_inventory_placeholders.sh --self-test
|
||||||
|
|
||||||
Fails when production inventory/env files contain public placeholder values:
|
Fails when production inventory/env files contain public placeholder values:
|
||||||
TEST-NET addresses, HOST-EXAMPLE, WINDOWS_USER_EXAMPLE, CHANGE_ME, YOUR_*,
|
TEST-NET addresses, HOST-EXAMPLE, WINDOWS_USER_EXAMPLE, CHANGE_ME, YOUR_*,
|
||||||
replace-me, or angle-bracket placeholders.
|
replace-me, or angle-bracket placeholders.
|
||||||
|
|
||||||
|
Use --strict for private production override files. Strict mode requires at
|
||||||
|
least one existing path and rejects public/example/default files.
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
pattern='(192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE|CHANGE_ME|CHANGEME|REPLACE_ME|replace-me|YOUR_[A-Z0-9_]*|<[A-Z0-9_ -]+>)'
|
pattern='(192\.0\.2\.|198\.51\.100\.|203\.0\.113\.|HOST-EXAMPLE|WINDOWS_USER_EXAMPLE|CHANGE_ME|CHANGEME|REPLACE_ME|replace-me|YOUR_[A-Z0-9_]*|<[A-Z0-9_ -]+>)'
|
||||||
allow_missing=0
|
allow_missing=0
|
||||||
|
strict=0
|
||||||
self_test=0
|
self_test=0
|
||||||
paths=()
|
paths=()
|
||||||
|
|
||||||
@@ -24,6 +28,9 @@ while (($#)); do
|
|||||||
--allow-missing)
|
--allow-missing)
|
||||||
allow_missing=1
|
allow_missing=1
|
||||||
;;
|
;;
|
||||||
|
--strict)
|
||||||
|
strict=1
|
||||||
|
;;
|
||||||
--self-test)
|
--self-test)
|
||||||
self_test=1
|
self_test=1
|
||||||
;;
|
;;
|
||||||
@@ -42,6 +49,11 @@ run_scan() {
|
|||||||
local file
|
local file
|
||||||
local found=0
|
local found=0
|
||||||
for file in "$@"; do
|
for file in "$@"; do
|
||||||
|
if (( strict == 1 )) && [[ "$file" == ansible/group_vars* || "$file" == ansible/inventory.ini || "$file" == *".example."* || "$file" == *"example."* ]]; then
|
||||||
|
printf 'strict mode refuses public/default config path: %s\n' "$file" >&2
|
||||||
|
found=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
if [[ ! -e "$file" ]]; then
|
if [[ ! -e "$file" ]]; then
|
||||||
if (( allow_missing == 0 )); then
|
if (( allow_missing == 0 )); then
|
||||||
printf 'missing production config path: %s\n' "$file" >&2
|
printf 'missing production config path: %s\n' "$file" >&2
|
||||||
@@ -86,6 +98,13 @@ EOF
|
|||||||
echo "self-test failed: bad fixture was accepted" >&2
|
echo "self-test failed: bad fixture was accepted" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
saved_strict="$strict"
|
||||||
|
strict=1
|
||||||
|
if run_scan "ansible/group_vars/all.yml" >/dev/null 2>&1; then
|
||||||
|
echo "self-test failed: strict mode accepted public/default file name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
strict="$saved_strict"
|
||||||
echo "production inventory placeholder guard self-test: OK"
|
echo "production inventory placeholder guard self-test: OK"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
@@ -95,7 +114,7 @@ if ((${#paths[@]} == 0)) && [[ -n "${DETMIR_PRODUCTION_CONFIG_PATHS:-}" ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
if ((${#paths[@]} == 0)); then
|
if ((${#paths[@]} == 0)); then
|
||||||
if (( allow_missing == 1 )); then
|
if (( allow_missing == 1 && strict == 0 )); then
|
||||||
echo "production inventory placeholder guard: skipped (no production paths)"
|
echo "production inventory placeholder guard: skipped (no production paths)"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user