feat(detmir): add grafana data correctness guard

adk-rust/Cargo.lock

@@ -549,6 +549,18 @@ dependencies = [
  "clap",
 ]
 
+[[package]]
+name = "detmir-grafana-check"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "clap",
+ "reqwest",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "detmir-heal-safe"
 version = "0.1.0"

adk-rust/Cargo.toml

@@ -24,6 +24,7 @@ members = [
     "crates/aw-contour-smoke",
     "crates/aw-browser-smoke",
     "crates/diag-and-manual-restart",
+    "crates/detmir-grafana-check",
     "crates/aw-slo-monitor",
     "crates/aw-rus-healthd",
     "crates/detmir-check",

adk-rust/RUNBOOK.md

@@ -1523,6 +1523,29 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
       (`2 passed`), `cargo clippy -p aw-linux-install --all-targets -- -D
       warnings`, release build OK, `sh -n` для всех пяти wrappers OK,
       dry-run JSON для всех пяти wrappers OK, artifact check OK.
+55. `[done]` Закрепить проверку актуализации и правильности Grafana данных:
+    - добавлен crate `detmir-grafana-check`;
+    - check читает Grafana dashboard API, валидирует форму dashboard,
+      отсутствие старых `aw_window_event`/`aw_afk_event`, наличие текущих
+      worktime measurements и выполняет все panel queries через Grafana
+      datasource API;
+    - freshness panel считается обязательным: значение свежести должно быть не
+      старше `DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES` (production default 360);
+    - результат пишется в Grafana CT 201:
+      `/var/lib/detmir-grafana-check/latest.json` и `latest.txt`;
+    - установлен `detmir-grafana-check.service` и timer каждые 15 минут;
+    - `detmir-check` теперь читает последний Grafana-check artifact через
+      `sudo -n /usr/sbin/pct exec 201` и считает `grafana-data` required
+      service check, так что общий `detmir-status` краснеет при сломанной или
+      устаревшей Grafana;
+    - добавлен playbook `ansible/deploy_grafana_check.yml`;
+    - `scripts/check_detmir_rust_release_artifacts.sh` теперь требует
+      `detmir-grafana-check`;
+    - production verification: `detmir-grafana-check` OK
+      (`ok=13,warn=0,fail=0`), 7/7 panels have rows, freshness около 120 min,
+      timer active, Grafana CT failed units 0, Proxmox failed units 0,
+      `detmir-check` `service_failures=0`, `detmir-auto --no-heal` rc 0,
+      `detmir-status` OK / `ok_for_operator=true`.
 
 Отложить:
 

adk-rust/crates/detmir-check/src/main.rs

@@ -1,8 +1,9 @@
 use std::net::{SocketAddr, TcpStream};
+use std::process::Command;
 use std::time::Duration;
 
 use anyhow::{Context, Result};
-use chrono::{SecondsFormat, Utc};
+use chrono::{DateTime, SecondsFormat, Utc};
 use clap::Parser;
 use detmir_aw_client::ActivityWatchClient;
 use detmir_core::{exit_codes, now_utc_rfc3339};
@@ -43,6 +44,18 @@ struct Cli {
 
     #[arg(long, default_value_t = 3.0)]
     tcp_timeout_seconds: f64,
+
+    #[arg(long, default_value_t = 201)]
+    grafana_ct_id: u32,
+
+    #[arg(long, default_value = "/var/lib/detmir-grafana-check/latest.json")]
+    grafana_check_json: String,
+
+    #[arg(long, default_value_t = 30 * 60)]
+    grafana_check_max_age_seconds: i64,
+
+    #[arg(long, default_value_t = false)]
+    disable_grafana_check: bool,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -296,9 +309,120 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
         args.tcp_timeout_seconds,
         true,
     ));
+    if !args.disable_grafana_check {
+        checks.push(grafana_data_check(args));
+    }
     checks
 }
 
+fn grafana_data_check(args: &Cli) -> ServiceCheck {
+    let name = "grafana-data".to_string();
+    let output = read_grafana_check_json_from_ct(args);
+    let output = match output {
+        Ok(output) => output,
+        Err(err) => {
+            return ServiceCheck {
+                name,
+                required: true,
+                ok: false,
+                url: None,
+                payload: None,
+                error: Some(format!("cannot execute pct for Grafana check: {err}")),
+            };
+        }
+    };
+    if !output.status.success() {
+        return ServiceCheck {
+            name,
+            required: true,
+            ok: false,
+            url: None,
+            payload: None,
+            error: Some(format!(
+                "pct grafana check read failed with status {:?}: {}",
+                output.status.code(),
+                String::from_utf8_lossy(&output.stderr).trim()
+            )),
+        };
+    }
+    let raw = String::from_utf8_lossy(&output.stdout);
+    let payload = match serde_json::from_str::<Value>(&raw) {
+        Ok(payload) => payload,
+        Err(err) => {
+            return ServiceCheck {
+                name,
+                required: true,
+                ok: false,
+                url: None,
+                payload: None,
+                error: Some(format!("cannot parse Grafana check JSON: {err}")),
+            };
+        }
+    };
+    let check_ok = payload.get("ok").and_then(Value::as_bool).unwrap_or(false);
+    let fail_count = payload
+        .pointer("/counts/fail")
+        .and_then(Value::as_u64)
+        .unwrap_or(1);
+    let generated_at = payload
+        .get("generated_at_utc")
+        .and_then(Value::as_str)
+        .unwrap_or("");
+    let age_seconds = DateTime::parse_from_rfc3339(generated_at)
+        .map(|ts| (Utc::now() - ts.with_timezone(&Utc)).num_seconds())
+        .unwrap_or(i64::MAX);
+    let fresh = (0..=args.grafana_check_max_age_seconds).contains(&age_seconds);
+    let ok = check_ok && fail_count == 0 && fresh;
+    ServiceCheck {
+        name,
+        required: true,
+        ok,
+        url: None,
+        payload: Some(serde_json::json!({
+            "generated_at_utc": generated_at,
+            "age_seconds": age_seconds,
+            "max_age_seconds": args.grafana_check_max_age_seconds,
+            "check_ok": check_ok,
+            "fail_count": fail_count,
+            "dashboard_uid": payload.get("dashboard_uid"),
+            "counts": payload.get("counts"),
+        })),
+        error: if ok {
+            None
+        } else {
+            Some(format!(
+                "Grafana check unhealthy: ok={check_ok} fail_count={fail_count} age_seconds={age_seconds}"
+            ))
+        },
+    }
+}
+
+fn read_grafana_check_json_from_ct(args: &Cli) -> std::io::Result<std::process::Output> {
+    let ct_id = args.grafana_ct_id.to_string();
+    let pct_args = [
+        "exec",
+        ct_id.as_str(),
+        "--",
+        "cat",
+        args.grafana_check_json.as_str(),
+    ];
+    if let Some(custom_pct) = std::env::var("DETMIR_PCT_BIN")
+        .ok()
+        .filter(|value| !value.is_empty())
+    {
+        return Command::new(custom_pct).args(pct_args).output();
+    }
+    let sudo_output = Command::new("/usr/bin/sudo")
+        .args(["-n", "/usr/sbin/pct"])
+        .args(pct_args)
+        .output();
+    match sudo_output {
+        Ok(output) if output.status.success() => Ok(output),
+        Ok(output) if output.status.code() != Some(127) => Ok(output),
+        _ => Command::new("/usr/sbin/pct").args(pct_args).output(),
+    }
+}
+
 fn tcp_check(host: &str, port: u16, timeout_seconds: f64, required: bool) -> ServiceCheck {
     let name = format!("tcp:{host}:{port}");
     let timeout = Duration::from_secs_f64(timeout_seconds);
@@ -507,6 +631,7 @@ fn main() -> Result<()> {
     args.worktime_url = env_or_default("DETMIR_WORKTIME_URL", &args.worktime_url);
     args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
     args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname);
+    args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json);
 
     let report = build_report(&args)?;
     if args.json {

adk-rust/crates/detmir-grafana-check/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "detmir-grafana-check"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+chrono.workspace = true
+clap.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+

adk-rust/crates/detmir-grafana-check/src/main.rs

@@ -0,0 +1,711 @@
+use std::fs;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use anyhow::{Context, Result, anyhow};
+use chrono::{SecondsFormat, Utc};
+use clap::Parser;
+use reqwest::blocking::Client;
+use serde::Serialize;
+use serde_json::{Value, json};
+
+const DEFAULT_GRAFANA_URL: &str = "http://127.0.0.1:3000";
+const DEFAULT_DASHBOARD_UID: &str = "detmir-aw-main";
+const DEFAULT_DASHBOARD_FILE: &str = "/etc/grafana/provisioning/dashboards/aw/detmir-aw-main.json";
+const DEFAULT_HOST: &str = "SHARKON2025";
+const OLD_MEASUREMENTS: &[&str] = &["aw_window_event", "aw_afk_event"];
+const REQUIRED_MEASUREMENTS: &[&str] = &[
+    "aw_rdp_worktime_hourly",
+    "aw_rdp_worktime_daily",
+    "aw_rdp_worktime_summary_daily",
+    "aw_true_active_app_daily",
+];
+
+#[derive(Debug, Parser)]
+#[command(about = "Read-only DetMir Grafana dashboard freshness and correctness check")]
+struct Cli {
+    #[arg(long, env = "DETMIR_GRAFANA_URL")]
+    grafana_url: Option<String>,
+
+    #[arg(long, env = "DETMIR_GRAFANA_USER")]
+    user: Option<String>,
+
+    #[arg(long, env = "DETMIR_GRAFANA_PASSWORD")]
+    password: Option<String>,
+
+    #[arg(long, default_value = DEFAULT_DASHBOARD_UID, env = "DETMIR_GRAFANA_DASHBOARD_UID")]
+    dashboard_uid: String,
+
+    #[arg(long, default_value = DEFAULT_DASHBOARD_FILE, env = "DETMIR_GRAFANA_DASHBOARD_FILE")]
+    dashboard_file: PathBuf,
+
+    #[arg(long, default_value = DEFAULT_HOST, env = "DETMIR_GRAFANA_HOST")]
+    host: String,
+
+    #[arg(long, default_value_t = 15, env = "DETMIR_GRAFANA_TIMEOUT_SECONDS")]
+    timeout_seconds: u64,
+
+    #[arg(
+        long,
+        default_value_t = 360.0,
+        env = "DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES"
+    )]
+    max_freshness_minutes: f64,
+
+    #[arg(long, default_value_t = 1, env = "DETMIR_GRAFANA_MIN_PANEL_ROWS")]
+    min_panel_rows: usize,
+
+    #[arg(long, default_value_t = 4, env = "DETMIR_GRAFANA_MIN_PANELS")]
+    min_panels: usize,
+
+    #[arg(long)]
+    json: bool,
+
+    #[arg(long, env = "DETMIR_GRAFANA_OUTPUT_JSON")]
+    output_json: Option<PathBuf>,
+
+    #[arg(long, env = "DETMIR_GRAFANA_OUTPUT_TEXT")]
+    output_text: Option<PathBuf>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "lowercase")]
+enum Status {
+    Ok,
+    Warn,
+    Fail,
+}
+
+#[derive(Debug, Serialize)]
+struct CheckResult {
+    name: String,
+    status: Status,
+    summary: String,
+    details: Value,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct Counts {
+    ok: usize,
+    warn: usize,
+    fail: usize,
+}
+
+#[derive(Debug, Serialize)]
+struct Report {
+    ok: bool,
+    generated_at_utc: String,
+    dashboard_uid: String,
+    grafana_url: String,
+    counts: Counts,
+    results: Vec<CheckResult>,
+}
+
+#[derive(Debug)]
+struct PanelTarget {
+    panel_id: i64,
+    panel_title: String,
+    ref_id: String,
+    datasource_type: String,
+    datasource_uid: String,
+    query: String,
+}
+
+fn main() {
+    let code = match run() {
+        Ok(code) => code,
+        Err(err) => {
+            eprintln!("{err:#}");
+            1
+        }
+    };
+    std::process::exit(code);
+}
+
+fn run() -> Result<i32> {
+    let cli = Cli::parse();
+    let report = build_report(&cli)?;
+    let text = render_text(&report);
+    if let Some(path) = &cli.output_json {
+        write_report_file(path, &serde_json::to_string_pretty(&report)?)?;
+    }
+    if let Some(path) = &cli.output_text {
+        write_report_file(path, &text)?;
+    }
+    if cli.json {
+        println!("{}", serde_json::to_string_pretty(&report)?);
+    } else {
+        print!("{text}");
+    }
+    Ok(if report.ok { 0 } else { 2 })
+}
+
+fn build_report(cli: &Cli) -> Result<Report> {
+    let grafana_url = cli
+        .grafana_url
+        .clone()
+        .or_else(|| env_nonempty("GRAFANA_URL"))
+        .unwrap_or_else(|| DEFAULT_GRAFANA_URL.to_string())
+        .trim_end_matches('/')
+        .to_string();
+    let user = cli.user.clone().or_else(|| env_nonempty("GRAFANA_USER"));
+    let password = cli
+        .password
+        .clone()
+        .or_else(|| env_nonempty("GRAFANA_PASSWORD"));
+    let client = Client::builder()
+        .timeout(Duration::from_secs(cli.timeout_seconds))
+        .no_proxy()
+        .build()
+        .context("build HTTP client")?;
+    let mut results = Vec::new();
+
+    results.push(check_health(&client, &grafana_url));
+    let dashboard_api = match get_json_auth(
+        &client,
+        &format!("{grafana_url}/api/dashboards/uid/{}", cli.dashboard_uid),
+        user.as_deref(),
+        password.as_deref(),
+    ) {
+        Ok(value) => {
+            results.push(CheckResult {
+                name: "grafana:dashboard-api".to_string(),
+                status: Status::Ok,
+                summary: format!("dashboard {} loaded through Grafana API", cli.dashboard_uid),
+                details: json!({
+                    "uid": cli.dashboard_uid,
+                    "title": value.pointer("/dashboard/title").and_then(Value::as_str),
+                    "version": value.pointer("/dashboard/version").and_then(Value::as_i64),
+                }),
+            });
+            Some(value)
+        }
+        Err(err) => {
+            results.push(CheckResult {
+                name: "grafana:dashboard-api".to_string(),
+                status: Status::Fail,
+                summary: format!("dashboard API request failed: {err:#}"),
+                details: json!({ "uid": cli.dashboard_uid }),
+            });
+            None
+        }
+    };
+
+    if let Some(api_value) = &dashboard_api {
+        let dashboard = api_value.get("dashboard").unwrap_or(api_value);
+        results.push(check_dashboard_shape(dashboard, cli.min_panels));
+        results.push(check_measurements(
+            "grafana:dashboard-api-measurements",
+            dashboard,
+        ));
+        let targets = collect_panel_targets(dashboard);
+        results.push(CheckResult {
+            name: "grafana:panel-targets".to_string(),
+            status: if targets.is_empty() {
+                Status::Fail
+            } else {
+                Status::Ok
+            },
+            summary: format!("{} query targets found", targets.len()),
+            details: json!({
+                "targets": targets.iter().map(|target| json!({
+                    "panel_id": target.panel_id,
+                    "panel_title": target.panel_title,
+                    "ref_id": target.ref_id,
+                    "datasource_uid": target.datasource_uid,
+                })).collect::<Vec<_>>()
+            }),
+        });
+        for target in &targets {
+            let rendered_query = render_query_vars(&target.query, &cli.host);
+            let query_result = query_panel_target(
+                &client,
+                &grafana_url,
+                user.as_deref(),
+                password.as_deref(),
+                target,
+                &rendered_query,
+            );
+            match query_result {
+                Ok((rows, first_number)) => {
+                    let freshness_failed = is_freshness_panel(&target.panel_title)
+                        && first_number
+                            .map(|value| value > cli.max_freshness_minutes)
+                            .unwrap_or(true);
+                    let status = if rows < cli.min_panel_rows || freshness_failed {
+                        Status::Fail
+                    } else {
+                        Status::Ok
+                    };
+                    let summary = if is_freshness_panel(&target.panel_title) {
+                        match first_number {
+                            Some(value) => format!(
+                                "panel '{}' returned {rows} rows, freshness {:.1} min",
+                                target.panel_title, value
+                            ),
+                            None => format!(
+                                "panel '{}' returned {rows} rows, but freshness value was absent",
+                                target.panel_title
+                            ),
+                        }
+                    } else {
+                        format!("panel '{}' returned {rows} rows", target.panel_title)
+                    };
+                    results.push(CheckResult {
+                        name: format!("grafana:panel-query:{}:{}", target.panel_id, target.ref_id),
+                        status,
+                        summary,
+                        details: json!({
+                            "panel_id": target.panel_id,
+                            "panel_title": target.panel_title,
+                            "ref_id": target.ref_id,
+                            "rows": rows,
+                            "first_number": first_number,
+                            "min_panel_rows": cli.min_panel_rows,
+                            "max_freshness_minutes": cli.max_freshness_minutes,
+                        }),
+                    });
+                }
+                Err(err) => results.push(CheckResult {
+                    name: format!("grafana:panel-query:{}:{}", target.panel_id, target.ref_id),
+                    status: Status::Fail,
+                    summary: format!("panel '{}' query failed: {err:#}", target.panel_title),
+                    details: json!({
+                        "panel_id": target.panel_id,
+                        "panel_title": target.panel_title,
+                        "ref_id": target.ref_id,
+                        "datasource_uid": target.datasource_uid,
+                    }),
+                }),
+            }
+        }
+    }
+
+    if cli.dashboard_file.exists() {
+        match read_json_file(&cli.dashboard_file) {
+            Ok(file_dashboard) => {
+                results.push(check_measurements(
+                    "grafana:provisioned-file-measurements",
+                    &file_dashboard,
+                ));
+            }
+            Err(err) => results.push(CheckResult {
+                name: "grafana:provisioned-file".to_string(),
+                status: Status::Warn,
+                summary: format!(
+                    "cannot read provisioned dashboard file {}: {err:#}",
+                    cli.dashboard_file.display()
+                ),
+                details: json!({ "path": cli.dashboard_file }),
+            }),
+        }
+    } else {
+        results.push(CheckResult {
+            name: "grafana:provisioned-file".to_string(),
+            status: Status::Warn,
+            summary: format!("dashboard file not found: {}", cli.dashboard_file.display()),
+            details: json!({ "path": cli.dashboard_file }),
+        });
+    }
+
+    let counts = count_statuses(&results);
+    Ok(Report {
+        ok: counts.fail == 0,
+        generated_at_utc: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
+        dashboard_uid: cli.dashboard_uid.clone(),
+        grafana_url,
+        counts,
+        results,
+    })
+}
+
+fn check_health(client: &Client, grafana_url: &str) -> CheckResult {
+    match get_json_auth(client, &format!("{grafana_url}/api/health"), None, None) {
+        Ok(value) => {
+            let database = value
+                .get("database")
+                .and_then(Value::as_str)
+                .unwrap_or("unknown");
+            CheckResult {
+                name: "grafana:health".to_string(),
+                status: if database.eq_ignore_ascii_case("ok") {
+                    Status::Ok
+                } else {
+                    Status::Warn
+                },
+                summary: format!("Grafana health database={database}"),
+                details: value,
+            }
+        }
+        Err(err) => CheckResult {
+            name: "grafana:health".to_string(),
+            status: Status::Fail,
+            summary: format!("Grafana health request failed: {err:#}"),
+            details: json!({ "url": grafana_url }),
+        },
+    }
+}
+
+fn check_dashboard_shape(dashboard: &Value, min_panels: usize) -> CheckResult {
+    let title = dashboard
+        .get("title")
+        .and_then(Value::as_str)
+        .unwrap_or("unknown");
+    let panels = collect_panels(dashboard);
+    CheckResult {
+        name: "grafana:dashboard-shape".to_string(),
+        status: if panels.len() >= min_panels {
+            Status::Ok
+        } else {
+            Status::Fail
+        },
+        summary: format!("dashboard '{title}' has {} panels", panels.len()),
+        details: json!({
+            "title": title,
+            "panel_count": panels.len(),
+            "min_panels": min_panels,
+        }),
+    }
+}
+
+fn check_measurements(name: &str, dashboard: &Value) -> CheckResult {
+    let text = dashboard.to_string();
+    let old = contains_any(&text, OLD_MEASUREMENTS);
+    let missing_required = REQUIRED_MEASUREMENTS
+        .iter()
+        .filter(|measurement| !text.contains(**measurement))
+        .copied()
+        .collect::<Vec<_>>();
+    let status = if old || !missing_required.is_empty() {
+        Status::Fail
+    } else {
+        Status::Ok
+    };
+    CheckResult {
+        name: name.to_string(),
+        status,
+        summary: if status == Status::Ok {
+            "dashboard uses current DetMir worktime measurements".to_string()
+        } else {
+            "dashboard measurement set is stale or incomplete".to_string()
+        },
+        details: json!({
+            "old_measurements_present": OLD_MEASUREMENTS
+                .iter()
+                .filter(|measurement| text.contains(**measurement))
+                .copied()
+                .collect::<Vec<_>>(),
+            "required_measurements_missing": missing_required,
+            "required_measurements": REQUIRED_MEASUREMENTS,
+        }),
+    }
+}
+
+fn query_panel_target(
+    client: &Client,
+    grafana_url: &str,
+    user: Option<&str>,
+    password: Option<&str>,
+    target: &PanelTarget,
+    query: &str,
+) -> Result<(usize, Option<f64>)> {
+    let body = json!({
+        "from": "now-48h",
+        "to": "now",
+        "queries": [{
+            "refId": target.ref_id,
+            "datasource": {
+                "type": target.datasource_type,
+                "uid": target.datasource_uid,
+            },
+            "query": query,
+            "rawQuery": true,
+            "format": "table",
+            "intervalMs": 60000,
+            "maxDataPoints": 1000,
+        }]
+    });
+    let response = post_json_auth(
+        client,
+        &format!("{grafana_url}/api/ds/query"),
+        user,
+        password,
+        &body,
+    )?;
+    let result = response
+        .pointer(&format!("/results/{}", target.ref_id))
+        .ok_or_else(|| anyhow!("missing result for refId {}", target.ref_id))?;
+    if let Some(error) = result.get("error").and_then(Value::as_str) {
+        return Err(anyhow!("{error}"));
+    }
+    if let Some(status) = result.get("status").and_then(Value::as_i64) {
+        if status >= 400 {
+            return Err(anyhow!("Grafana datasource status {status}"));
+        }
+    }
+    Ok((frame_row_count(result), first_numeric_value(result)))
+}
+
+fn get_json_auth(
+    client: &Client,
+    url: &str,
+    user: Option<&str>,
+    password: Option<&str>,
+) -> Result<Value> {
+    let mut request = client.get(url);
+    if let Some(user) = user {
+        request = request.basic_auth(user, password);
+    }
+    let response = request
+        .send()
+        .with_context(|| format!("GET {url}"))?
+        .error_for_status()
+        .with_context(|| format!("GET {url} returned non-success status"))?;
+    response
+        .json::<Value>()
+        .with_context(|| format!("decode JSON from {url}"))
+}
+
+fn post_json_auth(
+    client: &Client,
+    url: &str,
+    user: Option<&str>,
+    password: Option<&str>,
+    body: &Value,
+) -> Result<Value> {
+    let mut request = client.post(url).json(body);
+    if let Some(user) = user {
+        request = request.basic_auth(user, password);
+    }
+    let response = request
+        .send()
+        .with_context(|| format!("POST {url}"))?
+        .error_for_status()
+        .with_context(|| format!("POST {url} returned non-success status"))?;
+    response
+        .json::<Value>()
+        .with_context(|| format!("decode JSON from {url}"))
+}
+
+fn read_json_file(path: &PathBuf) -> Result<Value> {
+    let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
+    serde_json::from_str(&text).with_context(|| format!("parse {}", path.display()))
+}
+
+fn collect_panel_targets(dashboard: &Value) -> Vec<PanelTarget> {
+    let mut targets = Vec::new();
+    for panel in collect_panels(dashboard) {
+        let panel_id = panel.get("id").and_then(Value::as_i64).unwrap_or(0);
+        let panel_title = panel
+            .get("title")
+            .and_then(Value::as_str)
+            .unwrap_or("untitled")
+            .to_string();
+        let datasource_type = panel
+            .pointer("/datasource/type")
+            .and_then(Value::as_str)
+            .unwrap_or("influxdb")
+            .to_string();
+        let datasource_uid = panel
+            .pointer("/datasource/uid")
+            .and_then(Value::as_str)
+            .unwrap_or("")
+            .to_string();
+        let Some(items) = panel.get("targets").and_then(Value::as_array) else {
+            continue;
+        };
+        for item in items {
+            let Some(query) = item.get("query").and_then(Value::as_str) else {
+                continue;
+            };
+            targets.push(PanelTarget {
+                panel_id,
+                panel_title: panel_title.clone(),
+                ref_id: item
+                    .get("refId")
+                    .and_then(Value::as_str)
+                    .unwrap_or("A")
+                    .to_string(),
+                datasource_type: item
+                    .pointer("/datasource/type")
+                    .and_then(Value::as_str)
+                    .unwrap_or(&datasource_type)
+                    .to_string(),
+                datasource_uid: item
+                    .pointer("/datasource/uid")
+                    .and_then(Value::as_str)
+                    .unwrap_or(&datasource_uid)
+                    .to_string(),
+                query: query.to_string(),
+            });
+        }
+    }
+    targets
+}
+
+fn collect_panels(dashboard: &Value) -> Vec<&Value> {
+    let mut panels = Vec::new();
+    collect_panels_inner(dashboard, &mut panels);
+    panels
+}
+
+fn collect_panels_inner<'a>(value: &'a Value, panels: &mut Vec<&'a Value>) {
+    if value.get("targets").is_some() && value.get("type").is_some() {
+        panels.push(value);
+    }
+    if let Some(children) = value.get("panels").and_then(Value::as_array) {
+        for child in children {
+            collect_panels_inner(child, panels);
+        }
+    }
+}
+
+fn render_query_vars(query: &str, host: &str) -> String {
+    query.replace("${host}", host).replace("$host", host)
+}
+
+fn contains_any(text: &str, needles: &[&str]) -> bool {
+    needles.iter().any(|needle| text.contains(needle))
+}
+
+fn is_freshness_panel(title: &str) -> bool {
+    title.to_lowercase().contains("свеж")
+}
+
+fn frame_row_count(value: &Value) -> usize {
+    value
+        .get("frames")
+        .and_then(Value::as_array)
+        .map(|frames| {
+            frames
+                .iter()
+                .filter_map(|frame| frame.pointer("/data/values").and_then(Value::as_array))
+                .map(|columns| {
+                    columns
+                        .iter()
+                        .filter_map(Value::as_array)
+                        .map(Vec::len)
+                        .max()
+                        .unwrap_or(0)
+                })
+                .sum()
+        })
+        .unwrap_or(0)
+}
+
+fn first_numeric_value(value: &Value) -> Option<f64> {
+    match value {
+        Value::Number(number) => number.as_f64(),
+        Value::Array(items) => items.iter().find_map(first_numeric_value),
+        Value::Object(map) => map.values().find_map(first_numeric_value),
+        _ => None,
+    }
+}
+
+fn count_statuses(results: &[CheckResult]) -> Counts {
+    let mut counts = Counts::default();
+    for result in results {
+        match result.status {
+            Status::Ok => counts.ok += 1,
+            Status::Warn => counts.warn += 1,
+            Status::Fail => counts.fail += 1,
+        }
+    }
+    counts
+}
+
+fn env_nonempty(name: &str) -> Option<String> {
+    std::env::var(name).ok().filter(|value| !value.is_empty())
+}
+
+fn write_report_file(path: &PathBuf, content: &str) -> Result<()> {
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
+    }
+    fs::write(path, content).with_context(|| format!("write {}", path.display()))
+}
+
+fn render_text(report: &Report) -> String {
+    let mut out = String::new();
+    out.push_str("=== DetMir Grafana Check ===\n");
+    out.push_str(&format!("dashboard_uid={}\n", report.dashboard_uid));
+    out.push_str(&format!("grafana_url={}\n", report.grafana_url));
+    out.push_str(&format!("generated_at_utc={}\n\n", report.generated_at_utc));
+    for result in &report.results {
+        out.push_str(&format!(
+            "{:<5} {:<42} {}\n",
+            format!("{:?}", result.status).to_uppercase(),
+            result.name,
+            result.summary
+        ));
+    }
+    out.push('\n');
+    out.push_str(&format!(
+        "counts: ok={} warn={} fail={}",
+        report.counts.ok, report.counts.warn, report.counts.fail
+    ));
+    out.push('\n');
+    out.push_str(&format!("ok={}\n", report.ok));
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn query_vars_render_host_forms() {
+        assert_eq!(
+            render_query_vars(
+                "r.host == \"${host}\" or r.host == \"$host\"",
+                "SHARKON2025"
+            ),
+            "r.host == \"SHARKON2025\" or r.host == \"SHARKON2025\""
+        );
+    }
+
+    #[test]
+    fn frame_row_count_reads_grafana_frames() {
+        let value = json!({
+            "frames": [{
+                "data": {
+                    "values": [
+                        ["2026-06-02T10:00:00Z", "2026-06-02T11:00:00Z"],
+                        [1.0, 2.0]
+                    ]
+                }
+            }]
+        });
+        assert_eq!(frame_row_count(&value), 2);
+        assert_eq!(first_numeric_value(&value), Some(1.0));
+    }
+
+    #[test]
+    fn measurement_check_catches_old_and_missing() {
+        let dashboard = json!({
+            "panels": [{
+                "type": "stat",
+                "targets": [{"query": "from(bucket:\"aw_metrics\") |> filter(fn:(r)=>r._measurement == \"aw_window_event\")"}]
+            }]
+        });
+        let result = check_measurements("test", &dashboard);
+        assert_eq!(result.status, Status::Fail);
+    }
+
+    #[test]
+    fn collect_targets_from_dashboard_panels() {
+        let dashboard = json!({
+            "panels": [{
+                "id": 5,
+                "type": "stat",
+                "title": "Свежесть worktime данных",
+                "datasource": {"type": "influxdb", "uid": "influxdb_aw"},
+                "targets": [{"refId": "A", "query": "from(bucket:\"aw_metrics\")"}]
+            }]
+        });
+        let targets = collect_panel_targets(&dashboard);
+        assert_eq!(targets.len(), 1);
+        assert_eq!(targets[0].panel_id, 5);
+        assert_eq!(targets[0].datasource_uid, "influxdb_aw");
+    }
+}

ansible/deploy_grafana_check.yml

@@ -0,0 +1,163 @@
+---
+- name: Deploy DetMir Grafana correctness check into Grafana CT
+  hosts: proxmox
+  become: true
+  gather_facts: false
+
+  vars:
+    aw_repo_root: "{{ playbook_dir | dirname }}"
+    aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}"
+    grafana_check_ct_id: "{{ detmir_grafana_ct_id | default(201) }}"
+    grafana_check_url: "{{ detmir_grafana_check_url | default('http://127.0.0.1:3000') }}"
+    grafana_check_user: "{{ detmir_grafana_check_user | default(lookup('env', 'DETMIR_GRAFANA_USER') | default(lookup('env', 'GRAFANA_USER'), true), true) }}"
+    grafana_check_password: "{{ detmir_grafana_check_password | default(lookup('env', 'DETMIR_GRAFANA_PASSWORD') | default(lookup('env', 'GRAFANA_PASSWORD'), true), true) }}"
+    grafana_check_dashboard_uid: "{{ detmir_grafana_check_dashboard_uid | default('detmir-aw-main') }}"
+    grafana_check_host: "{{ detmir_grafana_check_host | default('SHARKON2025') }}"
+    grafana_check_max_freshness_minutes: "{{ detmir_grafana_check_max_freshness_minutes | default(360) }}"
+    grafana_check_on_calendar: "{{ detmir_grafana_check_on_calendar | default('*:0/15') }}"
+
+  tasks:
+    - name: Validate Grafana check credentials
+      ansible.builtin.assert:
+        that:
+          - grafana_check_user | length > 0
+          - grafana_check_password | length > 0
+        fail_msg: "Set DETMIR_GRAFANA_USER/DETMIR_GRAFANA_PASSWORD or detmir_grafana_check_user/password."
+      no_log: true
+
+    - name: Check local detmir-grafana-check binary
+      ansible.builtin.stat:
+        path: "{{ aw_rust_release_dir }}/detmir-grafana-check"
+      delegate_to: localhost
+      become: false
+      register: grafana_check_binary
+
+    - name: Fail when Rust binary is absent
+      ansible.builtin.fail:
+        msg: "Missing {{ aw_rust_release_dir }}/detmir-grafana-check. Build with cargo build --release -p detmir-grafana-check."
+      when: not (grafana_check_binary.stat.exists | default(false))
+
+    - name: Copy detmir-grafana-check binary to Proxmox staging
+      ansible.builtin.copy:
+        src: "{{ aw_rust_release_dir }}/detmir-grafana-check"
+        dest: /tmp/detmir-grafana-check
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install detmir-grafana-check into Grafana CT
+      ansible.builtin.command:
+        argv:
+          - pct
+          - push
+          - "{{ grafana_check_ct_id }}"
+          - /tmp/detmir-grafana-check
+          - /usr/local/bin/detmir-grafana-check
+      changed_when: true
+
+    - name: Set Grafana check binary permissions in CT
+      ansible.builtin.command:
+        argv:
+          - pct
+          - exec
+          - "{{ grafana_check_ct_id }}"
+          - --
+          - chmod
+          - "0755"
+          - /usr/local/bin/detmir-grafana-check
+      changed_when: true
+
+    - name: Create Grafana check runtime in CT
+      ansible.builtin.command:
+        argv:
+          - pct
+          - exec
+          - "{{ grafana_check_ct_id }}"
+          - --
+          - install
+          - "-d"
+          - "-m"
+          - "0755"
+          - /var/lib/detmir-grafana-check
+      changed_when: true
+
+    - name: Install Grafana check env in CT
+      ansible.builtin.shell: |
+        set -euo pipefail
+        pct exec {{ grafana_check_ct_id }} -- bash -lc 'umask 077; cat > /etc/detmir-grafana-check.env' <<'EOF'
+        DETMIR_GRAFANA_URL={{ grafana_check_url }}
+        DETMIR_GRAFANA_USER={{ grafana_check_user }}
+        DETMIR_GRAFANA_PASSWORD={{ grafana_check_password }}
+        DETMIR_GRAFANA_DASHBOARD_UID={{ grafana_check_dashboard_uid }}
+        DETMIR_GRAFANA_DASHBOARD_FILE=/etc/grafana/provisioning/dashboards/aw/detmir-aw-main.json
+        DETMIR_GRAFANA_HOST={{ grafana_check_host }}
+        DETMIR_GRAFANA_MAX_FRESHNESS_MINUTES={{ grafana_check_max_freshness_minutes }}
+        DETMIR_GRAFANA_OUTPUT_JSON=/var/lib/detmir-grafana-check/latest.json
+        DETMIR_GRAFANA_OUTPUT_TEXT=/var/lib/detmir-grafana-check/latest.txt
+        EOF
+      args:
+        executable: /bin/bash
+      changed_when: true
+      no_log: true
+
+    - name: Install Grafana check service in CT
+      ansible.builtin.shell: |
+        set -euo pipefail
+        pct exec {{ grafana_check_ct_id }} -- bash -lc 'cat > /etc/systemd/system/detmir-grafana-check.service' <<'EOF'
+        [Unit]
+        Description=DetMir Grafana dashboard data correctness check
+        After=grafana-server.service network-online.target
+        Wants=network-online.target
+
+        [Service]
+        Type=oneshot
+        EnvironmentFile=/etc/detmir-grafana-check.env
+        ExecStart=/usr/local/bin/detmir-grafana-check
+        EOF
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Install Grafana check timer in CT
+      ansible.builtin.shell: |
+        set -euo pipefail
+        pct exec {{ grafana_check_ct_id }} -- bash -lc 'cat > /etc/systemd/system/detmir-grafana-check.timer' <<'EOF'
+        [Unit]
+        Description=Run DetMir Grafana dashboard data correctness check
+
+        [Timer]
+        OnBootSec=2min
+        OnCalendar={{ grafana_check_on_calendar }}
+        Persistent=true
+        RandomizedDelaySec=2min
+
+        [Install]
+        WantedBy=timers.target
+        EOF
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Enable and run Grafana check timer in CT
+      ansible.builtin.command:
+        argv:
+          - pct
+          - exec
+          - "{{ grafana_check_ct_id }}"
+          - --
+          - bash
+          - "-lc"
+          - systemctl daemon-reload && systemctl enable --now detmir-grafana-check.timer && systemctl start detmir-grafana-check.service
+      changed_when: true
+
+    - name: Verify Grafana check result in CT
+      ansible.builtin.command:
+        argv:
+          - pct
+          - exec
+          - "{{ grafana_check_ct_id }}"
+          - --
+          - bash
+          - "-lc"
+          - test -s /var/lib/detmir-grafana-check/latest.json && jq -e '.ok == true and .counts.fail == 0' /var/lib/detmir-grafana-check/latest.json
+      changed_when: false

scripts/check_detmir_rust_release_artifacts.sh

@@ -36,6 +36,7 @@ required_bins=(
   aw-contour-smoke
   aw-browser-smoke
   diag-and-manual-restart
+  detmir-grafana-check
   dlp-health-check
   dlp-content-analyzer
   dlp-admin-cli