From f513836396923f7067285eed520eb7708df37658 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Tue, 2 Jun 2026 20:06:06 +0300 Subject: [PATCH] feat(detmir): add grafana data correctness guard --- adk-rust/Cargo.lock | 12 + adk-rust/Cargo.toml | 1 + adk-rust/RUNBOOK.md | 23 + adk-rust/crates/detmir-check/src/main.rs | 127 +++- .../crates/detmir-grafana-check/Cargo.toml | 16 + .../crates/detmir-grafana-check/src/main.rs | 711 ++++++++++++++++++ ansible/deploy_grafana_check.yml | 163 ++++ .../check_detmir_rust_release_artifacts.sh | 1 + 8 files changed, 1053 insertions(+), 1 deletion(-) create mode 100644 adk-rust/crates/detmir-grafana-check/Cargo.toml create mode 100644 adk-rust/crates/detmir-grafana-check/src/main.rs create mode 100644 ansible/deploy_grafana_check.yml diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index b91ab22..b7012fe 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -549,6 +549,18 @@ dependencies = [ "clap", ] +[[package]] +name = "detmir-grafana-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "reqwest", + "serde", + "serde_json", +] + [[package]] name = "detmir-heal-safe" version = "0.1.0" diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml index f861ee9..4577da0 100644 --- a/adk-rust/Cargo.toml +++ b/adk-rust/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/aw-contour-smoke", "crates/aw-browser-smoke", "crates/diag-and-manual-restart", + "crates/detmir-grafana-check", "crates/aw-slo-monitor", "crates/aw-rus-healthd", "crates/detmir-check", diff --git a/adk-rust/RUNBOOK.md b/adk-rust/RUNBOOK.md index e783205..464fe68 100644 --- a/adk-rust/RUNBOOK.md +++ b/adk-rust/RUNBOOK.md @@ -1523,6 +1523,29 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg (`2 passed`), `cargo clippy -p aw-linux-install --all-targets -- -D warnings`, release build OK, `sh -n` для всех пяти wrappers OK, dry-run JSON для всех пяти wrappers OK, artifact check OK. +55. `[done]` Закрепить проверку актуализации и правильности Grafana данных: + - добавлен crate `detmir-grafana-check`; + - check читает Grafana dashboard API, валидирует форму dashboard, + отсутствие старых `aw_window_event`/`aw_afk_event`, наличие текущих + worktime measurements и выполняет все panel queries через Grafana + datasource API; + - freshness panel считается обязательным: значение свежести должно быть не + старше `DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES` (production default 360); + - результат пишется в Grafana CT 201: + `/var/lib/detmir-grafana-check/latest.json` и `latest.txt`; + - установлен `detmir-grafana-check.service` и timer каждые 15 минут; + - `detmir-check` теперь читает последний Grafana-check artifact через + `sudo -n /usr/sbin/pct exec 201` и считает `grafana-data` required + service check, так что общий `detmir-status` краснеет при сломанной или + устаревшей Grafana; + - добавлен playbook `ansible/deploy_grafana_check.yml`; + - `scripts/check_detmir_rust_release_artifacts.sh` теперь требует + `detmir-grafana-check`; + - production verification: `detmir-grafana-check` OK + (`ok=13,warn=0,fail=0`), 7/7 panels have rows, freshness около 120 min, + timer active, Grafana CT failed units 0, Proxmox failed units 0, + `detmir-check` `service_failures=0`, `detmir-auto --no-heal` rc 0, + `detmir-status` OK / `ok_for_operator=true`. Отложить: diff --git a/adk-rust/crates/detmir-check/src/main.rs b/adk-rust/crates/detmir-check/src/main.rs index da5c51f..4b68ba4 100644 --- a/adk-rust/crates/detmir-check/src/main.rs +++ b/adk-rust/crates/detmir-check/src/main.rs @@ -1,8 +1,9 @@ use std::net::{SocketAddr, TcpStream}; +use std::process::Command; use std::time::Duration; use anyhow::{Context, Result}; -use chrono::{SecondsFormat, Utc}; +use chrono::{DateTime, SecondsFormat, Utc}; use clap::Parser; use detmir_aw_client::ActivityWatchClient; use detmir_core::{exit_codes, now_utc_rfc3339}; @@ -43,6 +44,18 @@ struct Cli { #[arg(long, default_value_t = 3.0)] tcp_timeout_seconds: f64, + + #[arg(long, default_value_t = 201)] + grafana_ct_id: u32, + + #[arg(long, default_value = "/var/lib/detmir-grafana-check/latest.json")] + grafana_check_json: String, + + #[arg(long, default_value_t = 30 * 60)] + grafana_check_max_age_seconds: i64, + + #[arg(long, default_value_t = false)] + disable_grafana_check: bool, } #[derive(Debug, Clone, Copy)] @@ -296,9 +309,120 @@ fn service_checks(args: &Cli) -> Vec { args.tcp_timeout_seconds, true, )); + if !args.disable_grafana_check { + checks.push(grafana_data_check(args)); + } checks } +fn grafana_data_check(args: &Cli) -> ServiceCheck { + let name = "grafana-data".to_string(); + let output = read_grafana_check_json_from_ct(args); + let output = match output { + Ok(output) => output, + Err(err) => { + return ServiceCheck { + name, + required: true, + ok: false, + url: None, + payload: None, + error: Some(format!("cannot execute pct for Grafana check: {err}")), + }; + } + }; + if !output.status.success() { + return ServiceCheck { + name, + required: true, + ok: false, + url: None, + payload: None, + error: Some(format!( + "pct grafana check read failed with status {:?}: {}", + output.status.code(), + String::from_utf8_lossy(&output.stderr).trim() + )), + }; + } + let raw = String::from_utf8_lossy(&output.stdout); + let payload = match serde_json::from_str::(&raw) { + Ok(payload) => payload, + Err(err) => { + return ServiceCheck { + name, + required: true, + ok: false, + url: None, + payload: None, + error: Some(format!("cannot parse Grafana check JSON: {err}")), + }; + } + }; + let check_ok = payload.get("ok").and_then(Value::as_bool).unwrap_or(false); + let fail_count = payload + .pointer("/counts/fail") + .and_then(Value::as_u64) + .unwrap_or(1); + let generated_at = payload + .get("generated_at_utc") + .and_then(Value::as_str) + .unwrap_or(""); + let age_seconds = DateTime::parse_from_rfc3339(generated_at) + .map(|ts| (Utc::now() - ts.with_timezone(&Utc)).num_seconds()) + .unwrap_or(i64::MAX); + let fresh = (0..=args.grafana_check_max_age_seconds).contains(&age_seconds); + let ok = check_ok && fail_count == 0 && fresh; + ServiceCheck { + name, + required: true, + ok, + url: None, + payload: Some(serde_json::json!({ + "generated_at_utc": generated_at, + "age_seconds": age_seconds, + "max_age_seconds": args.grafana_check_max_age_seconds, + "check_ok": check_ok, + "fail_count": fail_count, + "dashboard_uid": payload.get("dashboard_uid"), + "counts": payload.get("counts"), + })), + error: if ok { + None + } else { + Some(format!( + "Grafana check unhealthy: ok={check_ok} fail_count={fail_count} age_seconds={age_seconds}" + )) + }, + } +} + +fn read_grafana_check_json_from_ct(args: &Cli) -> std::io::Result { + let ct_id = args.grafana_ct_id.to_string(); + let pct_args = [ + "exec", + ct_id.as_str(), + "--", + "cat", + args.grafana_check_json.as_str(), + ]; + if let Some(custom_pct) = std::env::var("DETMIR_PCT_BIN") + .ok() + .filter(|value| !value.is_empty()) + { + return Command::new(custom_pct).args(pct_args).output(); + } + let sudo_output = Command::new("/usr/bin/sudo") + .args(["-n", "/usr/sbin/pct"]) + .args(pct_args) + .output(); + match sudo_output { + Ok(output) if output.status.success() => Ok(output), + Ok(output) if output.status.code() != Some(127) => Ok(output), + _ => Command::new("/usr/sbin/pct").args(pct_args).output(), + } +} + fn tcp_check(host: &str, port: u16, timeout_seconds: f64, required: bool) -> ServiceCheck { let name = format!("tcp:{host}:{port}"); let timeout = Duration::from_secs_f64(timeout_seconds); @@ -507,6 +631,7 @@ fn main() -> Result<()> { args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url); args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url); args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname); + args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json); let report = build_report(&args)?; if args.json { diff --git a/adk-rust/crates/detmir-grafana-check/Cargo.toml b/adk-rust/crates/detmir-grafana-check/Cargo.toml new file mode 100644 index 0000000..937b4cd --- /dev/null +++ b/adk-rust/crates/detmir-grafana-check/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "detmir-grafana-check" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true + diff --git a/adk-rust/crates/detmir-grafana-check/src/main.rs b/adk-rust/crates/detmir-grafana-check/src/main.rs new file mode 100644 index 0000000..9ec9ca1 --- /dev/null +++ b/adk-rust/crates/detmir-grafana-check/src/main.rs @@ -0,0 +1,711 @@ +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use chrono::{SecondsFormat, Utc}; +use clap::Parser; +use reqwest::blocking::Client; +use serde::Serialize; +use serde_json::{Value, json}; + +const DEFAULT_GRAFANA_URL: &str = "http://127.0.0.1:3000"; +const DEFAULT_DASHBOARD_UID: &str = "detmir-aw-main"; +const DEFAULT_DASHBOARD_FILE: &str = "/etc/grafana/provisioning/dashboards/aw/detmir-aw-main.json"; +const DEFAULT_HOST: &str = "SHARKON2025"; +const OLD_MEASUREMENTS: &[&str] = &["aw_window_event", "aw_afk_event"]; +const REQUIRED_MEASUREMENTS: &[&str] = &[ + "aw_rdp_worktime_hourly", + "aw_rdp_worktime_daily", + "aw_rdp_worktime_summary_daily", + "aw_true_active_app_daily", +]; + +#[derive(Debug, Parser)] +#[command(about = "Read-only DetMir Grafana dashboard freshness and correctness check")] +struct Cli { + #[arg(long, env = "DETMIR_GRAFANA_URL")] + grafana_url: Option, + + #[arg(long, env = "DETMIR_GRAFANA_USER")] + user: Option, + + #[arg(long, env = "DETMIR_GRAFANA_PASSWORD")] + password: Option, + + #[arg(long, default_value = DEFAULT_DASHBOARD_UID, env = "DETMIR_GRAFANA_DASHBOARD_UID")] + dashboard_uid: String, + + #[arg(long, default_value = DEFAULT_DASHBOARD_FILE, env = "DETMIR_GRAFANA_DASHBOARD_FILE")] + dashboard_file: PathBuf, + + #[arg(long, default_value = DEFAULT_HOST, env = "DETMIR_GRAFANA_HOST")] + host: String, + + #[arg(long, default_value_t = 15, env = "DETMIR_GRAFANA_TIMEOUT_SECONDS")] + timeout_seconds: u64, + + #[arg( + long, + default_value_t = 360.0, + env = "DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES" + )] + max_freshness_minutes: f64, + + #[arg(long, default_value_t = 1, env = "DETMIR_GRAFANA_MIN_PANEL_ROWS")] + min_panel_rows: usize, + + #[arg(long, default_value_t = 4, env = "DETMIR_GRAFANA_MIN_PANELS")] + min_panels: usize, + + #[arg(long)] + json: bool, + + #[arg(long, env = "DETMIR_GRAFANA_OUTPUT_JSON")] + output_json: Option, + + #[arg(long, env = "DETMIR_GRAFANA_OUTPUT_TEXT")] + output_text: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +enum Status { + Ok, + Warn, + Fail, +} + +#[derive(Debug, Serialize)] +struct CheckResult { + name: String, + status: Status, + summary: String, + details: Value, +} + +#[derive(Debug, Default, Serialize)] +struct Counts { + ok: usize, + warn: usize, + fail: usize, +} + +#[derive(Debug, Serialize)] +struct Report { + ok: bool, + generated_at_utc: String, + dashboard_uid: String, + grafana_url: String, + counts: Counts, + results: Vec, +} + +#[derive(Debug)] +struct PanelTarget { + panel_id: i64, + panel_title: String, + ref_id: String, + datasource_type: String, + datasource_uid: String, + query: String, +} + +fn main() { + let code = match run() { + Ok(code) => code, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} + +fn run() -> Result { + let cli = Cli::parse(); + let report = build_report(&cli)?; + let text = render_text(&report); + if let Some(path) = &cli.output_json { + write_report_file(path, &serde_json::to_string_pretty(&report)?)?; + } + if let Some(path) = &cli.output_text { + write_report_file(path, &text)?; + } + if cli.json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + print!("{text}"); + } + Ok(if report.ok { 0 } else { 2 }) +} + +fn build_report(cli: &Cli) -> Result { + let grafana_url = cli + .grafana_url + .clone() + .or_else(|| env_nonempty("GRAFANA_URL")) + .unwrap_or_else(|| DEFAULT_GRAFANA_URL.to_string()) + .trim_end_matches('/') + .to_string(); + let user = cli.user.clone().or_else(|| env_nonempty("GRAFANA_USER")); + let password = cli + .password + .clone() + .or_else(|| env_nonempty("GRAFANA_PASSWORD")); + let client = Client::builder() + .timeout(Duration::from_secs(cli.timeout_seconds)) + .no_proxy() + .build() + .context("build HTTP client")?; + let mut results = Vec::new(); + + results.push(check_health(&client, &grafana_url)); + let dashboard_api = match get_json_auth( + &client, + &format!("{grafana_url}/api/dashboards/uid/{}", cli.dashboard_uid), + user.as_deref(), + password.as_deref(), + ) { + Ok(value) => { + results.push(CheckResult { + name: "grafana:dashboard-api".to_string(), + status: Status::Ok, + summary: format!("dashboard {} loaded through Grafana API", cli.dashboard_uid), + details: json!({ + "uid": cli.dashboard_uid, + "title": value.pointer("/dashboard/title").and_then(Value::as_str), + "version": value.pointer("/dashboard/version").and_then(Value::as_i64), + }), + }); + Some(value) + } + Err(err) => { + results.push(CheckResult { + name: "grafana:dashboard-api".to_string(), + status: Status::Fail, + summary: format!("dashboard API request failed: {err:#}"), + details: json!({ "uid": cli.dashboard_uid }), + }); + None + } + }; + + if let Some(api_value) = &dashboard_api { + let dashboard = api_value.get("dashboard").unwrap_or(api_value); + results.push(check_dashboard_shape(dashboard, cli.min_panels)); + results.push(check_measurements( + "grafana:dashboard-api-measurements", + dashboard, + )); + let targets = collect_panel_targets(dashboard); + results.push(CheckResult { + name: "grafana:panel-targets".to_string(), + status: if targets.is_empty() { + Status::Fail + } else { + Status::Ok + }, + summary: format!("{} query targets found", targets.len()), + details: json!({ + "targets": targets.iter().map(|target| json!({ + "panel_id": target.panel_id, + "panel_title": target.panel_title, + "ref_id": target.ref_id, + "datasource_uid": target.datasource_uid, + })).collect::>() + }), + }); + for target in &targets { + let rendered_query = render_query_vars(&target.query, &cli.host); + let query_result = query_panel_target( + &client, + &grafana_url, + user.as_deref(), + password.as_deref(), + target, + &rendered_query, + ); + match query_result { + Ok((rows, first_number)) => { + let freshness_failed = is_freshness_panel(&target.panel_title) + && first_number + .map(|value| value > cli.max_freshness_minutes) + .unwrap_or(true); + let status = if rows < cli.min_panel_rows || freshness_failed { + Status::Fail + } else { + Status::Ok + }; + let summary = if is_freshness_panel(&target.panel_title) { + match first_number { + Some(value) => format!( + "panel '{}' returned {rows} rows, freshness {:.1} min", + target.panel_title, value + ), + None => format!( + "panel '{}' returned {rows} rows, but freshness value was absent", + target.panel_title + ), + } + } else { + format!("panel '{}' returned {rows} rows", target.panel_title) + }; + results.push(CheckResult { + name: format!("grafana:panel-query:{}:{}", target.panel_id, target.ref_id), + status, + summary, + details: json!({ + "panel_id": target.panel_id, + "panel_title": target.panel_title, + "ref_id": target.ref_id, + "rows": rows, + "first_number": first_number, + "min_panel_rows": cli.min_panel_rows, + "max_freshness_minutes": cli.max_freshness_minutes, + }), + }); + } + Err(err) => results.push(CheckResult { + name: format!("grafana:panel-query:{}:{}", target.panel_id, target.ref_id), + status: Status::Fail, + summary: format!("panel '{}' query failed: {err:#}", target.panel_title), + details: json!({ + "panel_id": target.panel_id, + "panel_title": target.panel_title, + "ref_id": target.ref_id, + "datasource_uid": target.datasource_uid, + }), + }), + } + } + } + + if cli.dashboard_file.exists() { + match read_json_file(&cli.dashboard_file) { + Ok(file_dashboard) => { + results.push(check_measurements( + "grafana:provisioned-file-measurements", + &file_dashboard, + )); + } + Err(err) => results.push(CheckResult { + name: "grafana:provisioned-file".to_string(), + status: Status::Warn, + summary: format!( + "cannot read provisioned dashboard file {}: {err:#}", + cli.dashboard_file.display() + ), + details: json!({ "path": cli.dashboard_file }), + }), + } + } else { + results.push(CheckResult { + name: "grafana:provisioned-file".to_string(), + status: Status::Warn, + summary: format!("dashboard file not found: {}", cli.dashboard_file.display()), + details: json!({ "path": cli.dashboard_file }), + }); + } + + let counts = count_statuses(&results); + Ok(Report { + ok: counts.fail == 0, + generated_at_utc: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + dashboard_uid: cli.dashboard_uid.clone(), + grafana_url, + counts, + results, + }) +} + +fn check_health(client: &Client, grafana_url: &str) -> CheckResult { + match get_json_auth(client, &format!("{grafana_url}/api/health"), None, None) { + Ok(value) => { + let database = value + .get("database") + .and_then(Value::as_str) + .unwrap_or("unknown"); + CheckResult { + name: "grafana:health".to_string(), + status: if database.eq_ignore_ascii_case("ok") { + Status::Ok + } else { + Status::Warn + }, + summary: format!("Grafana health database={database}"), + details: value, + } + } + Err(err) => CheckResult { + name: "grafana:health".to_string(), + status: Status::Fail, + summary: format!("Grafana health request failed: {err:#}"), + details: json!({ "url": grafana_url }), + }, + } +} + +fn check_dashboard_shape(dashboard: &Value, min_panels: usize) -> CheckResult { + let title = dashboard + .get("title") + .and_then(Value::as_str) + .unwrap_or("unknown"); + let panels = collect_panels(dashboard); + CheckResult { + name: "grafana:dashboard-shape".to_string(), + status: if panels.len() >= min_panels { + Status::Ok + } else { + Status::Fail + }, + summary: format!("dashboard '{title}' has {} panels", panels.len()), + details: json!({ + "title": title, + "panel_count": panels.len(), + "min_panels": min_panels, + }), + } +} + +fn check_measurements(name: &str, dashboard: &Value) -> CheckResult { + let text = dashboard.to_string(); + let old = contains_any(&text, OLD_MEASUREMENTS); + let missing_required = REQUIRED_MEASUREMENTS + .iter() + .filter(|measurement| !text.contains(**measurement)) + .copied() + .collect::>(); + let status = if old || !missing_required.is_empty() { + Status::Fail + } else { + Status::Ok + }; + CheckResult { + name: name.to_string(), + status, + summary: if status == Status::Ok { + "dashboard uses current DetMir worktime measurements".to_string() + } else { + "dashboard measurement set is stale or incomplete".to_string() + }, + details: json!({ + "old_measurements_present": OLD_MEASUREMENTS + .iter() + .filter(|measurement| text.contains(**measurement)) + .copied() + .collect::>(), + "required_measurements_missing": missing_required, + "required_measurements": REQUIRED_MEASUREMENTS, + }), + } +} + +fn query_panel_target( + client: &Client, + grafana_url: &str, + user: Option<&str>, + password: Option<&str>, + target: &PanelTarget, + query: &str, +) -> Result<(usize, Option)> { + let body = json!({ + "from": "now-48h", + "to": "now", + "queries": [{ + "refId": target.ref_id, + "datasource": { + "type": target.datasource_type, + "uid": target.datasource_uid, + }, + "query": query, + "rawQuery": true, + "format": "table", + "intervalMs": 60000, + "maxDataPoints": 1000, + }] + }); + let response = post_json_auth( + client, + &format!("{grafana_url}/api/ds/query"), + user, + password, + &body, + )?; + let result = response + .pointer(&format!("/results/{}", target.ref_id)) + .ok_or_else(|| anyhow!("missing result for refId {}", target.ref_id))?; + if let Some(error) = result.get("error").and_then(Value::as_str) { + return Err(anyhow!("{error}")); + } + if let Some(status) = result.get("status").and_then(Value::as_i64) { + if status >= 400 { + return Err(anyhow!("Grafana datasource status {status}")); + } + } + Ok((frame_row_count(result), first_numeric_value(result))) +} + +fn get_json_auth( + client: &Client, + url: &str, + user: Option<&str>, + password: Option<&str>, +) -> Result { + let mut request = client.get(url); + if let Some(user) = user { + request = request.basic_auth(user, password); + } + let response = request + .send() + .with_context(|| format!("GET {url}"))? + .error_for_status() + .with_context(|| format!("GET {url} returned non-success status"))?; + response + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn post_json_auth( + client: &Client, + url: &str, + user: Option<&str>, + password: Option<&str>, + body: &Value, +) -> Result { + let mut request = client.post(url).json(body); + if let Some(user) = user { + request = request.basic_auth(user, password); + } + let response = request + .send() + .with_context(|| format!("POST {url}"))? + .error_for_status() + .with_context(|| format!("POST {url} returned non-success status"))?; + response + .json::() + .with_context(|| format!("decode JSON from {url}")) +} + +fn read_json_file(path: &PathBuf) -> Result { + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parse {}", path.display())) +} + +fn collect_panel_targets(dashboard: &Value) -> Vec { + let mut targets = Vec::new(); + for panel in collect_panels(dashboard) { + let panel_id = panel.get("id").and_then(Value::as_i64).unwrap_or(0); + let panel_title = panel + .get("title") + .and_then(Value::as_str) + .unwrap_or("untitled") + .to_string(); + let datasource_type = panel + .pointer("/datasource/type") + .and_then(Value::as_str) + .unwrap_or("influxdb") + .to_string(); + let datasource_uid = panel + .pointer("/datasource/uid") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let Some(items) = panel.get("targets").and_then(Value::as_array) else { + continue; + }; + for item in items { + let Some(query) = item.get("query").and_then(Value::as_str) else { + continue; + }; + targets.push(PanelTarget { + panel_id, + panel_title: panel_title.clone(), + ref_id: item + .get("refId") + .and_then(Value::as_str) + .unwrap_or("A") + .to_string(), + datasource_type: item + .pointer("/datasource/type") + .and_then(Value::as_str) + .unwrap_or(&datasource_type) + .to_string(), + datasource_uid: item + .pointer("/datasource/uid") + .and_then(Value::as_str) + .unwrap_or(&datasource_uid) + .to_string(), + query: query.to_string(), + }); + } + } + targets +} + +fn collect_panels(dashboard: &Value) -> Vec<&Value> { + let mut panels = Vec::new(); + collect_panels_inner(dashboard, &mut panels); + panels +} + +fn collect_panels_inner<'a>(value: &'a Value, panels: &mut Vec<&'a Value>) { + if value.get("targets").is_some() && value.get("type").is_some() { + panels.push(value); + } + if let Some(children) = value.get("panels").and_then(Value::as_array) { + for child in children { + collect_panels_inner(child, panels); + } + } +} + +fn render_query_vars(query: &str, host: &str) -> String { + query.replace("${host}", host).replace("$host", host) +} + +fn contains_any(text: &str, needles: &[&str]) -> bool { + needles.iter().any(|needle| text.contains(needle)) +} + +fn is_freshness_panel(title: &str) -> bool { + title.to_lowercase().contains("свеж") +} + +fn frame_row_count(value: &Value) -> usize { + value + .get("frames") + .and_then(Value::as_array) + .map(|frames| { + frames + .iter() + .filter_map(|frame| frame.pointer("/data/values").and_then(Value::as_array)) + .map(|columns| { + columns + .iter() + .filter_map(Value::as_array) + .map(Vec::len) + .max() + .unwrap_or(0) + }) + .sum() + }) + .unwrap_or(0) +} + +fn first_numeric_value(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_f64(), + Value::Array(items) => items.iter().find_map(first_numeric_value), + Value::Object(map) => map.values().find_map(first_numeric_value), + _ => None, + } +} + +fn count_statuses(results: &[CheckResult]) -> Counts { + let mut counts = Counts::default(); + for result in results { + match result.status { + Status::Ok => counts.ok += 1, + Status::Warn => counts.warn += 1, + Status::Fail => counts.fail += 1, + } + } + counts +} + +fn env_nonempty(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn write_report_file(path: &PathBuf, content: &str) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + } + fs::write(path, content).with_context(|| format!("write {}", path.display())) +} + +fn render_text(report: &Report) -> String { + let mut out = String::new(); + out.push_str("=== DetMir Grafana Check ===\n"); + out.push_str(&format!("dashboard_uid={}\n", report.dashboard_uid)); + out.push_str(&format!("grafana_url={}\n", report.grafana_url)); + out.push_str(&format!("generated_at_utc={}\n\n", report.generated_at_utc)); + for result in &report.results { + out.push_str(&format!( + "{:<5} {:<42} {}\n", + format!("{:?}", result.status).to_uppercase(), + result.name, + result.summary + )); + } + out.push('\n'); + out.push_str(&format!( + "counts: ok={} warn={} fail={}", + report.counts.ok, report.counts.warn, report.counts.fail + )); + out.push('\n'); + out.push_str(&format!("ok={}\n", report.ok)); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_vars_render_host_forms() { + assert_eq!( + render_query_vars( + "r.host == \"${host}\" or r.host == \"$host\"", + "SHARKON2025" + ), + "r.host == \"SHARKON2025\" or r.host == \"SHARKON2025\"" + ); + } + + #[test] + fn frame_row_count_reads_grafana_frames() { + let value = json!({ + "frames": [{ + "data": { + "values": [ + ["2026-06-02T10:00:00Z", "2026-06-02T11:00:00Z"], + [1.0, 2.0] + ] + } + }] + }); + assert_eq!(frame_row_count(&value), 2); + assert_eq!(first_numeric_value(&value), Some(1.0)); + } + + #[test] + fn measurement_check_catches_old_and_missing() { + let dashboard = json!({ + "panels": [{ + "type": "stat", + "targets": [{"query": "from(bucket:\"aw_metrics\") |> filter(fn:(r)=>r._measurement == \"aw_window_event\")"}] + }] + }); + let result = check_measurements("test", &dashboard); + assert_eq!(result.status, Status::Fail); + } + + #[test] + fn collect_targets_from_dashboard_panels() { + let dashboard = json!({ + "panels": [{ + "id": 5, + "type": "stat", + "title": "Свежесть worktime данных", + "datasource": {"type": "influxdb", "uid": "influxdb_aw"}, + "targets": [{"refId": "A", "query": "from(bucket:\"aw_metrics\")"}] + }] + }); + let targets = collect_panel_targets(&dashboard); + assert_eq!(targets.len(), 1); + assert_eq!(targets[0].panel_id, 5); + assert_eq!(targets[0].datasource_uid, "influxdb_aw"); + } +} diff --git a/ansible/deploy_grafana_check.yml b/ansible/deploy_grafana_check.yml new file mode 100644 index 0000000..1d6d286 --- /dev/null +++ b/ansible/deploy_grafana_check.yml @@ -0,0 +1,163 @@ +--- +- name: Deploy DetMir Grafana correctness check into Grafana CT + hosts: proxmox + become: true + gather_facts: false + + vars: + aw_repo_root: "{{ playbook_dir | dirname }}" + aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}" + grafana_check_ct_id: "{{ detmir_grafana_ct_id | default(201) }}" + grafana_check_url: "{{ detmir_grafana_check_url | default('http://127.0.0.1:3000') }}" + grafana_check_user: "{{ detmir_grafana_check_user | default(lookup('env', 'DETMIR_GRAFANA_USER') | default(lookup('env', 'GRAFANA_USER'), true), true) }}" + grafana_check_password: "{{ detmir_grafana_check_password | default(lookup('env', 'DETMIR_GRAFANA_PASSWORD') | default(lookup('env', 'GRAFANA_PASSWORD'), true), true) }}" + grafana_check_dashboard_uid: "{{ detmir_grafana_check_dashboard_uid | default('detmir-aw-main') }}" + grafana_check_host: "{{ detmir_grafana_check_host | default('SHARKON2025') }}" + grafana_check_max_freshness_minutes: "{{ detmir_grafana_check_max_freshness_minutes | default(360) }}" + grafana_check_on_calendar: "{{ detmir_grafana_check_on_calendar | default('*:0/15') }}" + + tasks: + - name: Validate Grafana check credentials + ansible.builtin.assert: + that: + - grafana_check_user | length > 0 + - grafana_check_password | length > 0 + fail_msg: "Set DETMIR_GRAFANA_USER/DETMIR_GRAFANA_PASSWORD or detmir_grafana_check_user/password." + no_log: true + + - name: Check local detmir-grafana-check binary + ansible.builtin.stat: + path: "{{ aw_rust_release_dir }}/detmir-grafana-check" + delegate_to: localhost + become: false + register: grafana_check_binary + + - name: Fail when Rust binary is absent + ansible.builtin.fail: + msg: "Missing {{ aw_rust_release_dir }}/detmir-grafana-check. Build with cargo build --release -p detmir-grafana-check." + when: not (grafana_check_binary.stat.exists | default(false)) + + - name: Copy detmir-grafana-check binary to Proxmox staging + ansible.builtin.copy: + src: "{{ aw_rust_release_dir }}/detmir-grafana-check" + dest: /tmp/detmir-grafana-check + owner: root + group: root + mode: "0755" + + - name: Install detmir-grafana-check into Grafana CT + ansible.builtin.command: + argv: + - pct + - push + - "{{ grafana_check_ct_id }}" + - /tmp/detmir-grafana-check + - /usr/local/bin/detmir-grafana-check + changed_when: true + + - name: Set Grafana check binary permissions in CT + ansible.builtin.command: + argv: + - pct + - exec + - "{{ grafana_check_ct_id }}" + - -- + - chmod + - "0755" + - /usr/local/bin/detmir-grafana-check + changed_when: true + + - name: Create Grafana check runtime in CT + ansible.builtin.command: + argv: + - pct + - exec + - "{{ grafana_check_ct_id }}" + - -- + - install + - "-d" + - "-m" + - "0755" + - /var/lib/detmir-grafana-check + changed_when: true + + - name: Install Grafana check env in CT + ansible.builtin.shell: | + set -euo pipefail + pct exec {{ grafana_check_ct_id }} -- bash -lc 'umask 077; cat > /etc/detmir-grafana-check.env' <<'EOF' + DETMIR_GRAFANA_URL={{ grafana_check_url }} + DETMIR_GRAFANA_USER={{ grafana_check_user }} + DETMIR_GRAFANA_PASSWORD={{ grafana_check_password }} + DETMIR_GRAFANA_DASHBOARD_UID={{ grafana_check_dashboard_uid }} + DETMIR_GRAFANA_DASHBOARD_FILE=/etc/grafana/provisioning/dashboards/aw/detmir-aw-main.json + DETMIR_GRAFANA_HOST={{ grafana_check_host }} + DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES={{ grafana_check_max_freshness_minutes }} + DETMIR_GRAFANA_OUTPUT_JSON=/var/lib/detmir-grafana-check/latest.json + DETMIR_GRAFANA_OUTPUT_TEXT=/var/lib/detmir-grafana-check/latest.txt + EOF + args: + executable: /bin/bash + changed_when: true + no_log: true + + - name: Install Grafana check service in CT + ansible.builtin.shell: | + set -euo pipefail + pct exec {{ grafana_check_ct_id }} -- bash -lc 'cat > /etc/systemd/system/detmir-grafana-check.service' <<'EOF' + [Unit] + Description=DetMir Grafana dashboard data correctness check + After=grafana-server.service network-online.target + Wants=network-online.target + + [Service] + Type=oneshot + EnvironmentFile=/etc/detmir-grafana-check.env + ExecStart=/usr/local/bin/detmir-grafana-check + EOF + args: + executable: /bin/bash + changed_when: true + + - name: Install Grafana check timer in CT + ansible.builtin.shell: | + set -euo pipefail + pct exec {{ grafana_check_ct_id }} -- bash -lc 'cat > /etc/systemd/system/detmir-grafana-check.timer' <<'EOF' + [Unit] + Description=Run DetMir Grafana dashboard data correctness check + + [Timer] + OnBootSec=2min + OnCalendar={{ grafana_check_on_calendar }} + Persistent=true + RandomizedDelaySec=2min + + [Install] + WantedBy=timers.target + EOF + args: + executable: /bin/bash + changed_when: true + + - name: Enable and run Grafana check timer in CT + ansible.builtin.command: + argv: + - pct + - exec + - "{{ grafana_check_ct_id }}" + - -- + - bash + - "-lc" + - systemctl daemon-reload && systemctl enable --now detmir-grafana-check.timer && systemctl start detmir-grafana-check.service + changed_when: true + + - name: Verify Grafana check result in CT + ansible.builtin.command: + argv: + - pct + - exec + - "{{ grafana_check_ct_id }}" + - -- + - bash + - "-lc" + - test -s /var/lib/detmir-grafana-check/latest.json && jq -e '.ok == true and .counts.fail == 0' /var/lib/detmir-grafana-check/latest.json + changed_when: false diff --git a/scripts/check_detmir_rust_release_artifacts.sh b/scripts/check_detmir_rust_release_artifacts.sh index cb414df..2b24e94 100644 --- a/scripts/check_detmir_rust_release_artifacts.sh +++ b/scripts/check_detmir_rust_release_artifacts.sh @@ -36,6 +36,7 @@ required_bins=( aw-contour-smoke aw-browser-smoke diag-and-manual-restart + detmir-grafana-check dlp-health-check dlp-content-analyzer dlp-admin-cli