diff --git a/README.md b/README.md index 24a6f17..a9f25ee 100755 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ collectors. - [Pilot v1.0 acceptance checklist](docs/PILOT_V1_ACCEPTANCE_CHECKLIST_RU.md) - [Pilot v1.0 evidence](docs/PILOT_V1_EVIDENCE_RU.md) - [Итог production-расследования 2026-06-07](docs/PRODUCTION_INCIDENT_REPORT_2026-06-07_RU.md) +- [Runbook восстановления worktime reports](docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md) - [Позиционирование продукта](docs/PRODUCT_POSITIONING_RU.md) - [Экосистема сборщиков](docs/COLLECTOR_ECOSYSTEM_RU.md) - [Стратегия внедрения](docs/DEPLOYMENT_STRATEGY_RU.md) diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index 283ea5f..2dba3ed 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -3255,9 +3255,11 @@ dependencies = [ "anyhow", "chrono", "clap", + "regex", "reqwest", "serde", "serde_json", + "tempfile", "tiny_http", "url", "urlencoding", diff --git a/adk-rust/crates/detmir-portal/src/main.rs b/adk-rust/crates/detmir-portal/src/main.rs index 3bb59a2..8df47e2 100644 --- a/adk-rust/crates/detmir-portal/src/main.rs +++ b/adk-rust/crates/detmir-portal/src/main.rs @@ -2593,13 +2593,16 @@ fn http_json_source(name: &str, url: &str, timeout: Duration) -> SourceStatus { { Ok(response) => match response.error_for_status() { Ok(response) => match response.json::() { - Ok(payload) => SourceStatus { - ok: true, - status: status_from_payload(&payload), - summary: source_summary(name, &payload), - error: None, - payload: Some(payload), - }, + Ok(payload) => { + let status = status_from_payload(&payload); + SourceStatus { + ok: source_ok_from_payload(&payload, &status), + status, + summary: source_summary(name, &payload), + error: None, + payload: Some(payload), + } + } Err(err) => SourceStatus { ok: false, status: "FAIL".to_string(), @@ -9812,6 +9815,16 @@ fn status_from_payload(payload: &Value) -> String { .to_string() } +fn source_ok_from_payload(payload: &Value, status: &str) -> bool { + if payload_bool(payload, "/ok") == Some(false) { + return false; + } + !matches!( + status.trim().to_ascii_uppercase().as_str(), + "FAIL" | "FAILED" | "ERROR" | "CRITICAL" | "DEGRADED" + ) +} + fn source_summary(name: &str, payload: &Value) -> String { match name { "detmir_status" => format!( @@ -9861,7 +9874,19 @@ fn source_summary(name: &str, payload: &Value) -> String { .unwrap_or(0) ), "worktime_management" => format!( - "coverage={:.0}%, departments={}, owners={}", + "status={}, cache_hit={}, stale_served={}, coverage={:.0}%, departments={}, owners={}", + payload + .get("status") + .and_then(Value::as_str) + .unwrap_or("OK"), + payload + .pointer("/runtime/report_cache_hit") + .and_then(Value::as_bool) + .unwrap_or(false), + payload + .pointer("/runtime/report_stale_served") + .and_then(Value::as_bool) + .unwrap_or(false), payload .pointer("/summary/portfolio_coverage_pct") .and_then(Value::as_f64) @@ -10138,6 +10163,22 @@ mod tests { assert!(PortalRole::Admin.can_access("pfsense")); } + #[test] + fn degraded_worktime_payload_is_not_healthy_source() { + let payload = json!({ + "ok": false, + "status": "DEGRADED", + "runtime": { + "report_cache_hit": true, + "report_stale_served": true + } + }); + let status = status_from_payload(&payload); + assert_eq!(status, "DEGRADED"); + assert!(!source_ok_from_payload(&payload, &status)); + assert!(source_summary("worktime_management", &payload).contains("stale_served=true")); + } + #[test] fn role_filtered_reports_do_not_cross_default_scopes() { let report = json!({ diff --git a/adk-rust/crates/rebuild-install-kit/src/main.rs b/adk-rust/crates/rebuild-install-kit/src/main.rs index 0d58ad9..3a2d4a0 100644 --- a/adk-rust/crates/rebuild-install-kit/src/main.rs +++ b/adk-rust/crates/rebuild-install-kit/src/main.rs @@ -103,6 +103,7 @@ const SCRIPTS_FILES: &[&str] = &[ "scripts/rebuild_install_kit.sh", "scripts/validate_install_kit.sh", "scripts/verify_innosetup_installer.sh", + "scripts/worktime-degraded-smoke.mjs", ]; #[derive(Debug, Parser)] diff --git a/adk-rust/crates/worktime-api/Cargo.toml b/adk-rust/crates/worktime-api/Cargo.toml index c206f08..3b655a5 100644 --- a/adk-rust/crates/worktime-api/Cargo.toml +++ b/adk-rust/crates/worktime-api/Cargo.toml @@ -16,3 +16,7 @@ serde_json.workspace = true tiny_http.workspace = true url.workspace = true urlencoding.workspace = true +regex.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/adk-rust/crates/worktime-api/src/main.rs b/adk-rust/crates/worktime-api/src/main.rs index 243357b..de446d6 100644 --- a/adk-rust/crates/worktime-api/src/main.rs +++ b/adk-rust/crates/worktime-api/src/main.rs @@ -3,7 +3,10 @@ use std::{ fs, io::Cursor, path::{Path, PathBuf}, - sync::{Arc, Mutex}, + sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicU64, Ordering}, + }, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; @@ -13,6 +16,7 @@ use chrono::{ TimeZone, Utc, }; use clap::Parser; +use regex::Regex; use reqwest::{ blocking::Client, header::{CONNECTION, HeaderMap, HeaderValue}, @@ -137,12 +141,23 @@ type IdentitySamples = BTreeMap<(String, String), Vec<(DateTime, AwEvent)>> type DateBounds = (DateTime, DateTime); type EventsForDate = (DateBounds, Vec); +#[derive(Default)] +struct RuntimeMetrics { + aw_query_duration_ms: AtomicU64, + aw_query_timeout_count: AtomicU64, + report_build_error_count: AtomicU64, + report_cache_hit: AtomicU64, + report_stale_served: AtomicU64, + report_degraded: AtomicU64, +} + #[derive(Clone)] struct App { config: Arc, aw: Client, events_cache: EventsCache, report_cache: ReportCache, + metrics: Arc, } fn env(name: &str, fallback: &str) -> String { @@ -376,6 +391,7 @@ impl App { aw, events_cache: Arc::new(Mutex::new(HashMap::new())), report_cache: Arc::new(Mutex::new(HashMap::new())), + metrics: Arc::new(RuntimeMetrics::default()), }) } @@ -392,12 +408,17 @@ impl App { let url = format!("{}{}", self.config.aw_api_base, path); let mut last_error = None; for attempt in 1..=attempts.max(1) { + let started = Instant::now(); let mut request = self.aw.get(&url); if let Some(timeout) = timeout { request = request.timeout(timeout); } match request.send() { Ok(response) => { + self.metrics.aw_query_duration_ms.store( + started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + Ordering::Relaxed, + ); if !response.status().is_success() { return Err(anyhow!("AW {path} returned HTTP {}", response.status())); } @@ -406,7 +427,16 @@ impl App { .with_context(|| format!("decode AW JSON {path}")); } Err(error) => { + self.metrics.aw_query_duration_ms.store( + started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + Ordering::Relaxed, + ); let is_timeout = error.is_timeout(); + if is_timeout { + self.metrics + .aw_query_timeout_count + .fetch_add(1, Ordering::Relaxed); + } last_error = Some(error); if is_timeout { break; @@ -430,13 +460,135 @@ impl App { Ok(events) => events, Err(error) => { eprintln!( - "[aw-worktime-api-rust] events fetch failed bucket={bucket_id}: {error:#}" + "[aw-worktime-api-rust] events fetch failed bucket={} error={}", + sanitize_bucket_for_log(bucket_id), + sanitize_error_for_log(&format!("{error:#}")) ); Vec::new() } } } + fn health_payload(&self) -> Value { + let degraded = self.metrics.report_degraded.load(Ordering::Relaxed) > 0; + json!({ + "ok": !degraded, + "status": if degraded { "DEGRADED" } else { "OK" }, + "generated_at_utc": to_iso_utc(Utc::now()), + "report_timezone": "Europe/Moscow", + "default_host": self.config.default_host, + "aw_api_base": self.config.aw_api_base, + "runtime": self.runtime_metrics_payload(false, false), + }) + } + + fn runtime_metrics_payload(&self, report_cache_hit: bool, report_stale_served: bool) -> Value { + json!({ + "worktime_events_limit": self.config.worktime_events_limit, + "aw_http_timeout_seconds": self.config.aw_http_timeout_seconds, + "report_cache_hit": report_cache_hit, + "report_stale_served": report_stale_served, + "aw_query_duration_ms": self.metrics.aw_query_duration_ms.load(Ordering::Relaxed), + "aw_query_timeout_count": self.metrics.aw_query_timeout_count.load(Ordering::Relaxed), + "report_build_error_count": self.metrics.report_build_error_count.load(Ordering::Relaxed), + "report_cache_hit_count": self.metrics.report_cache_hit.load(Ordering::Relaxed), + "report_stale_served_count": self.metrics.report_stale_served.load(Ordering::Relaxed), + }) + } + + fn runtime_headers(&self, cache_state: &str) -> Vec<(String, String)> { + vec![ + ("X-AW-Worktime-Cache".into(), cache_state.to_string()), + ( + "X-AW-Worktime-Events-Limit".into(), + self.config.worktime_events_limit.to_string(), + ), + ( + "X-AW-Worktime-AW-Timeout-Seconds".into(), + format!("{:.3}", self.config.aw_http_timeout_seconds), + ), + ( + "X-AW-Worktime-AW-Query-Duration-Ms".into(), + self.metrics + .aw_query_duration_ms + .load(Ordering::Relaxed) + .to_string(), + ), + ( + "X-AW-Worktime-AW-Query-Timeout-Count".into(), + self.metrics + .aw_query_timeout_count + .load(Ordering::Relaxed) + .to_string(), + ), + ( + "X-AW-Worktime-Report-Build-Error-Count".into(), + self.metrics + .report_build_error_count + .load(Ordering::Relaxed) + .to_string(), + ), + ] + } + + fn log_report_runtime( + &self, + path: &str, + outcome: &str, + cache_state: &str, + report_cache_hit: bool, + report_stale_served: bool, + error: Option<&str>, + ) { + let error_suffix = error + .map(|value| format!(" error={}", sanitize_error_for_log(value))) + .unwrap_or_default(); + eprintln!( + "[aw-worktime-api-rust] report {outcome} path={path} cache={cache_state} worktime_events_limit={} aw_http_timeout_seconds={:.3} report_cache_hit={} report_stale_served={} aw_query_duration_ms={} aw_query_timeout_count={} report_build_error_count={}{}", + self.config.worktime_events_limit, + self.config.aw_http_timeout_seconds, + report_cache_hit, + report_stale_served, + self.metrics.aw_query_duration_ms.load(Ordering::Relaxed), + self.metrics.aw_query_timeout_count.load(Ordering::Relaxed), + self.metrics + .report_build_error_count + .load(Ordering::Relaxed), + error_suffix + ); + } + + fn add_runtime_to_json( + &self, + data: Vec, + content_type: &str, + report_cache_hit: bool, + report_stale_served: bool, + degraded_reason: Option<&str>, + ) -> Vec { + if !content_type.starts_with("application/json") { + return data; + } + let Ok(mut payload) = serde_json::from_slice::(&data) else { + return data; + }; + let Some(object) = payload.as_object_mut() else { + return data; + }; + object.insert( + "runtime".to_string(), + self.runtime_metrics_payload(report_cache_hit, report_stale_served), + ); + if let Some(reason) = degraded_reason { + object.insert("ok".to_string(), json!(false)); + object.insert("status".to_string(), json!("DEGRADED")); + object.insert("degraded".to_string(), json!(true)); + object.insert("stale".to_string(), json!(report_stale_served)); + object.insert("degraded_reason".to_string(), json!(reason)); + } + serde_json::to_vec_pretty(&payload).unwrap_or(data) + } + fn fetch_bucket_events_result( &self, bucket_id: &str, @@ -526,45 +678,103 @@ impl App { &department, ); if let Some(cached) = self.get_report_cache(&cache_key, false) { - return ( - cached.data, - cached.content_type, - vec![ - ("X-AW-Worktime-Cache".into(), "fresh".into()), - ("X-AW-Worktime-Cache-Reason".into(), "ttl".into()), - ], - ); + self.metrics + .report_cache_hit + .fetch_add(1, Ordering::Relaxed); + let data = + self.add_runtime_to_json(cached.data, &cached.content_type, true, false, None); + let mut headers = self.runtime_headers("fresh"); + headers.push(("X-AW-Worktime-Cache-Reason".into(), "ttl".into())); + self.log_report_runtime(path, "ok", "fresh", true, false, None); + return (data, cached.content_type, headers); } let built = self.build_report_response(path, params, &fmt, &host, report_date, &owner, &department); match built { Ok((data, content_type)) => { + self.metrics.report_degraded.store(0, Ordering::Relaxed); + let data = self.add_runtime_to_json(data, &content_type, false, false, None); self.save_report_cache(cache_key, data.clone(), content_type.clone()); - (data, content_type, Vec::new()) + self.log_report_runtime(path, "ok", "miss", false, false, None); + (data, content_type, self.runtime_headers("miss")) } Err(error) => { - eprintln!("[aw-worktime-api-rust] report build failed path={path}: {error:#}"); + self.metrics + .report_build_error_count + .fetch_add(1, Ordering::Relaxed); + self.metrics.report_degraded.store(1, Ordering::Relaxed); + let sanitized_error = sanitize_error_for_log(&format!("{error:#}")); if let Some(cached) = self.get_report_cache(&cache_key, true) { - return ( + self.metrics + .report_cache_hit + .fetch_add(1, Ordering::Relaxed); + self.metrics + .report_stale_served + .fetch_add(1, Ordering::Relaxed); + let data = self.add_runtime_to_json( cached.data, - cached.content_type, - vec![ - ("X-AW-Worktime-Cache".into(), "stale".into()), - ("X-AW-Worktime-Cache-Reason".into(), "build-error".into()), - ], + &cached.content_type, + true, + true, + Some("stale_cache_served_after_build_error"), ); + let mut headers = self.runtime_headers("stale"); + headers.push(("X-AW-Worktime-Cache-Reason".into(), "build-error".into())); + self.log_report_runtime( + path, + "degraded", + "stale", + true, + true, + Some(&sanitized_error), + ); + return (data, cached.content_type, headers); } + self.log_report_runtime( + path, + "degraded", + "degraded", + false, + false, + Some(&sanitized_error), + ); let data = serde_json::to_vec_pretty(&json!({ "ok": false, - "error": "report_unavailable", + "status": "DEGRADED", + "degraded": true, + "stale": false, + "reason": "report_unavailable", "message": "report build failed and no cached response is available", "generated_at_utc": to_iso_utc(Utc::now()), + "report_timezone": "Europe/Moscow", + "host": host, + "report_date": report_date.to_string(), + "rows": [], + "actions": [], + "sources": [], + "summary": { + "users_count": 0, + "active_users": 0, + "inactive_users": 0, + "portfolio_coverage_pct": 0.0, + "actions_count": 0, + "critical_actions_count": 0 + }, + "executive": { + "portfolio_state": "degraded", + "headline": "Данные рабочего времени временно недоступны; используется безопасный degraded-режим.", + "message": "ActivityWatch API не ответил в заданный лимит, тяжелые повторные запросы не выполняются.", + "focus_items": [], + "stale_sources": [], + "stale_sources_count": 0 + }, + "runtime": self.runtime_metrics_payload(false, false), })) .unwrap_or_default(); ( data, "application/json; charset=utf-8".to_string(), - Vec::new(), + self.runtime_headers("degraded"), ) } } @@ -925,6 +1135,39 @@ fn sessions_bucket(host: &str) -> String { format!("aw-worktime-sessions_{host}") } +fn sanitize_bucket_for_log(bucket_id: &str) -> String { + for prefix in [ + "aw-worktime-sessions_", + "aw-watcher-afk_", + "aw-watcher-window_", + "aw-rdp-afk_", + "aw-rdp-window_", + "aw-file-operations_", + "aw-dlp-endpoint-signals_", + "aw-watcher-web-chrome_", + "aw-watcher-web-edge_", + "aw-detmir-web-category_", + "aw-session-events_", + ] { + if bucket_id.starts_with(prefix) { + return format!("{prefix}"); + } + } + bucket_id.to_string() +} + +fn sanitize_error_for_log(value: &str) -> String { + static IP_RE: OnceLock = OnceLock::new(); + static BUCKET_HOST_RE: OnceLock = OnceLock::new(); + let ip_re = IP_RE.get_or_init(|| Regex::new(r"\b(?:\d{1,3}\.){3}\d{1,3}\b").unwrap()); + let bucket_re = + BUCKET_HOST_RE.get_or_init(|| Regex::new(r"(aw-[A-Za-z0-9-]+_)[A-Za-z0-9_.-]+").unwrap()); + let redacted_ips = ip_re.replace_all(value, "").into_owned(); + bucket_re + .replace_all(&redacted_ips, "$1") + .into_owned() +} + fn resolve_report_date(config: &Config, day: Option<&str>, date: Option<&str>) -> NaiveDate { if let Some(date) = date.and_then(|d| NaiveDate::parse_from_str(d, "%Y-%m-%d").ok()) { return date; @@ -3195,13 +3438,7 @@ fn handle(app: &App, request: Request) { let url = request.url().to_string(); let (path, query) = url.split_once('?').unwrap_or((&url, "")); if path == "/health" || path == "/api/health" { - let payload = json!({ - "ok": true, - "generated_at_utc": to_iso_utc(Utc::now()), - "report_timezone": "Europe/Moscow", - "default_host": app.config.default_host, - "aw_api_base": app.config.aw_api_base, - }); + let payload = app.health_payload(); respond( request, 200, @@ -3269,7 +3506,7 @@ fn handle(app: &App, request: Request) { let status = if content_type.starts_with("application/json") && serde_json::from_slice::(&data) .ok() - .and_then(|v| v.get("error").cloned()) + .and_then(|v| v.get("error").or_else(|| v.get("http_error")).cloned()) .is_some() { 503 @@ -3301,6 +3538,21 @@ mod tests { load_config() } + fn degraded_test_config(cache_dir: &Path) -> Config { + let mut cfg = test_config(); + cfg.aw_api_base = "http://127.0.0.1:9/api/0".to_string(); + cfg.default_host = DEFAULT_HOST.to_string(); + cfg.worktime_events_limit = 250; + cfg.aw_http_timeout_seconds = 0.5; + cfg.source_http_timeout_seconds = 0.25; + cfg.report_cache_ttl_seconds = 0; + cfg.report_stale_ttl_seconds = 600; + cfg.report_disk_stale_ttl_seconds = 600; + cfg.report_disk_cache_dir = cache_dir.to_path_buf(); + cfg.management_history_dir = cache_dir.join("history"); + cfg + } + fn event(ts: &str, username: &str, sid: i64, active: bool, extra: Value) -> AwEvent { let mut data = Map::new(); data.insert("username".into(), json!(username)); @@ -3531,6 +3783,90 @@ mod tests { )); } + #[test] + fn degraded_report_without_cache_returns_compact_response() { + let dir = tempfile::tempdir().unwrap(); + let app = App::new(degraded_test_config(dir.path())).unwrap(); + let started = Instant::now(); + let (data, content_type, headers) = app.report_response( + "/reports/worktime/management", + &Params::parse("format=json&host=HOST-EXAMPLE"), + "application/json", + ); + assert!( + started.elapsed() < Duration::from_secs(3), + "degraded response must be bounded" + ); + assert!(content_type.starts_with("application/json")); + assert!( + headers + .iter() + .any(|(key, value)| { key == "X-AW-Worktime-Cache" && value == "degraded" }) + ); + let payload: Value = serde_json::from_slice(&data).unwrap(); + assert_eq!(payload["ok"], false); + assert_eq!(payload["status"], "DEGRADED"); + assert_eq!(payload["stale"], false); + assert_eq!(payload["runtime"]["worktime_events_limit"], 250); + assert_eq!(payload["runtime"]["report_build_error_count"], 1); + let health = app.health_payload(); + assert_eq!(health["ok"], false); + assert_eq!(health["status"], "DEGRADED"); + } + + #[test] + fn stale_cache_is_served_after_report_build_error() { + let dir = tempfile::tempdir().unwrap(); + let cfg = degraded_test_config(dir.path()); + let report_date = resolve_report_date(&cfg, None, None); + let app = App::new(cfg).unwrap(); + let cache_key = make_report_cache_key( + "/reports/worktime/management", + "json", + "HOST-EXAMPLE", + report_date, + "", + "", + "", + ); + let cached_payload = json!({ + "generated_at_utc": to_iso_utc(Utc::now()), + "host": "HOST-EXAMPLE", + "report_date": report_date.to_string(), + "report_timezone": "Europe/Moscow", + "summary": {"portfolio_coverage_pct": 75.0}, + "rows": [{"user": "demo", "workday_active_seconds": 60}], + "actions": [], + "sources": [], + "department_rollups": [], + "owner_rollups": [] + }); + app.save_report_cache( + cache_key, + serde_json::to_vec_pretty(&cached_payload).unwrap(), + "application/json; charset=utf-8".to_string(), + ); + + let (data, _content_type, headers) = app.report_response( + "/reports/worktime/management", + &Params::parse("format=json&host=HOST-EXAMPLE"), + "application/json", + ); + assert!( + headers + .iter() + .any(|(key, value)| { key == "X-AW-Worktime-Cache" && value == "stale" }) + ); + let payload: Value = serde_json::from_slice(&data).unwrap(); + assert_eq!(payload["ok"], false); + assert_eq!(payload["status"], "DEGRADED"); + assert_eq!(payload["stale"], true); + assert_eq!(payload["runtime"]["report_cache_hit"], true); + assert_eq!(payload["runtime"]["report_stale_served"], true); + assert_eq!(payload["runtime"]["report_stale_served_count"], 1); + assert_eq!(app.health_payload()["ok"], false); + } + #[test] fn sanitize_trend_point_removes_legacy_action_only_rollups() { let point = sanitize_trend_point(json!({ diff --git a/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md b/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md new file mode 100644 index 0000000..529e1f3 --- /dev/null +++ b/docs/OPERATIONS_RUNBOOK_WORKTIME_RU.md @@ -0,0 +1,212 @@ +# Runbook восстановления worktime reports + +Документ описывает безопасную диагностику и восстановление цепочки +`ActivityWatch -> aw-worktime-api -> portal executive reports`. + +ClickHouse не является обязательной зависимостью worktime reports. Не +перезапускайте ClickHouse для восстановления отчетов рабочего времени, если нет +отдельного подтвержденного отказа ClickHouse. + +## Симптомы перегруза + +- `/portal/api/reports?role=executive` открывается медленно или отвечает + degraded/stale. +- `/portal/api/health` показывает degraded-состояние worktime source. +- `/reports/worktime/management` на `aw-worktime-api` возвращает + `status=DEGRADED`. +- В журнале `aw-worktime-api` растут `aw_query_timeout_count` или + `report_build_error_count`. +- ActivityWatch HTTP API отвечает медленно, не отвечает или держит SQLite под + высокой нагрузкой. + +## Быстрая диагностика + +Проверить состояние сервисов: + +```bash +systemctl status activitywatch-server aw-worktime-api --no-pager +systemctl status aw-worktime-ui-bridge.timer aw-worktime-autoheal.timer aw-rus-healthd.timer --no-pager +``` + +Проверить последние журналы: + +```bash +journalctl -u aw-worktime-api -n 80 --no-pager +journalctl -u activitywatch-server -n 80 --no-pager +``` + +Проверить health worktime API: + +```bash +curl -sS --max-time 5 http://:5610/health | jq +``` + +Проверить отчет в безопасном bounded-режиме: + +```bash +curl -sS --max-time 8 \ + "http://:5610/reports/worktime/management?format=json&host=HOST-EXAMPLE&allow_stale=1" \ + | jq '.status,.stale,.runtime' +``` + +Проверить portal health: + +```bash +curl -sS --max-time 8 http:///portal/api/health | jq +curl -sS --max-time 12 "http:///portal/api/reports?role=executive" | jq '.status,.sources' +``` + +## Проверка свежести bucket + +Проверить metadata конкретного worktime bucket: + +```bash +curl -sS --max-time 5 \ + http://:5600/api/0/buckets/aw-worktime-sessions_HOST-EXAMPLE \ + | jq '.metadata.end' +``` + +Проверить список bucket без чтения тяжелых событий: + +```bash +curl -sS --max-time 5 http://:5600/api/0/buckets | jq 'keys' +``` + +Если metadata свежая, а report degraded, вероятная причина - перегрузка чтения +events или временная недоступность ActivityWatch API. Не запускайте повторные +тяжелые запросы вручную без лимитов `--max-time`. + +## Проверка лимитов + +Проверить системные настройки: + +```bash +systemctl cat aw-worktime-api +grep '^AW_WORKTIME_' /etc/activitywatch/aw-server.env +``` + +Ключевые параметры: + +- `AW_WORKTIME_EVENTS_LIMIT` - верхний лимит чтения events из ActivityWatch. +- `AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS` - timeout запросов к ActivityWatch API. +- `AW_WORKTIME_SOURCE_HTTP_TIMEOUT_SECONDS` - timeout внешних source-запросов. +- `AW_WORKTIME_REPORT_CACHE_TTL_SECONDS` - TTL fresh report cache. +- `AW_WORKTIME_REPORT_STALE_TTL_SECONDS` - TTL stale cache для degraded path. + +Нормальная production-политика: bounded timeouts, ограниченный events limit, +stale cache включен. Нулевой stale TTL допустим только для специальных тестов, +но не для демонстрации или промышленного пилота. + +## Безопасный restart + +1. Зафиксировать текущий сигнал: + +```bash +systemctl status aw-worktime-api activitywatch-server --no-pager +journalctl -u aw-worktime-api -n 120 --no-pager +curl -sS --max-time 5 http://:5610/health | jq +``` + +2. Перезапустить только `aw-worktime-api`, если ActivityWatch отвечает, но + портал получает degraded report: + +```bash +systemctl restart aw-worktime-api +sleep 3 +curl -sS --max-time 5 http://:5610/health | jq +``` + +3. Перезапустить `activitywatch-server` только если ActivityWatch API не + отвечает или SQLite явно перегружен: + +```bash +systemctl restart activitywatch-server +sleep 5 +systemctl restart aw-worktime-api +``` + +4. Прогреть отчет один раз: + +```bash +curl -sS --max-time 12 \ + "http://:5610/reports/worktime/management?format=json&host=HOST-EXAMPLE&allow_stale=1" \ + | jq '.status,.stale,.runtime' +``` + +5. Проверить портал: + +```bash +curl -sS --max-time 8 http:///portal/api/health | jq +curl -sS --max-time 12 "http:///portal/api/reports?role=executive" | jq '.status' +``` + +## Rollback + +Rollback нужен, если после обновления бинарника или env-настроек: + +- fresh report не собирается; +- stale cache не отдается; +- `/health` не отражает degraded-состояние; +- портал зависает вместо bounded degraded response. + +Порядок: + +```bash +systemctl stop aw-worktime-api +cp /usr/local/bin/aw-worktime-api.prev /usr/local/bin/aw-worktime-api +systemctl daemon-reload +systemctl start aw-worktime-api +curl -sS --max-time 5 http://:5610/health | jq +``` + +Если rollback касается env/drop-in: + +```bash +cp /etc/systemd/system/aw-worktime-api.service.d/override.conf.prev \ + /etc/systemd/system/aw-worktime-api.service.d/override.conf +systemctl daemon-reload +systemctl restart aw-worktime-api +``` + +Перед rollback убедитесь, что backup-файлы действительно относятся к предыдущей +рабочей версии. + +## Признаки успешного восстановления + +- `/reports/worktime/management` отвечает HTTP 200 в bounded time. +- При свежей сборке `status` отсутствует или равен `OK`, `stale=false`. +- При временном отказе ActivityWatch API отдается `status=DEGRADED`, а не + timeout. +- Если stale cache доступен, response содержит `stale=true` и + `runtime.report_stale_served=true`. +- Если stale cache недоступен, response компактный, `stale=false`, + `reason=report_unavailable`. +- `/health` у `aw-worktime-api` и `/portal/api/health` не маркируют систему как + fully healthy при degraded reports. +- Счетчики `aw_query_timeout_count` и `report_build_error_count` перестают + расти после восстановления ActivityWatch API. + +## Smoke-тест degraded path + +Локально, без обращения к рабочему контуру: + +```bash +cd +cd adk-rust && cargo build -p worktime-api +cd .. +node scripts/worktime-degraded-smoke.mjs +``` + +Ожидаемый результат: + +```text +worktime degraded smoke OK +``` + +Smoke проверяет: + +- fresh report успевает построиться и прогреть cache; +- при недоступном ActivityWatch API отдается stale degraded response; +- при отсутствии stale cache отдается компактный degraded response; +- health отражает degraded-состояние; +- runtime-поля присутствуют в JSON. diff --git a/scripts/worktime-degraded-smoke.mjs b/scripts/worktime-degraded-smoke.mjs new file mode 100644 index 0000000..0195372 --- /dev/null +++ b/scripts/worktime-degraded-smoke.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node +import childProcess from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +function jsonString(value) { + return JSON.stringify(value, null, 2); +} + +async function randomLocalPort() { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close(() => resolve(port)); + }); + }); +} + +function cargoTargetDirectory() { + try { + const workspace = path.join(root, "adk-rust"); + const output = childProcess.execFileSync( + "cargo", + ["metadata", "--format-version", "1", "--no-deps", "--manifest-path", path.join(workspace, "Cargo.toml")], + { cwd: workspace, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, + ); + const metadata = JSON.parse(output); + return metadata.target_directory || ""; + } catch { + return ""; + } +} + +function resolveWorktimeBinary() { + const explicit = process.env.WORKTIME_API_BIN || ""; + const targetDir = cargoTargetDirectory(); + const candidates = [ + explicit, + targetDir ? path.join(targetDir, "debug/worktime-api") : "", + targetDir ? path.join(targetDir, "release/worktime-api") : "", + path.join(root, "adk-rust/target/debug/worktime-api"), + path.join(root, "adk-rust/target/release/worktime-api"), + ].filter(Boolean); + const found = candidates.find((candidate) => fs.existsSync(candidate) && fs.statSync(candidate).mode & 0o111); + if (!found) { + throw new Error("worktime-api binary not found; run: cd adk-rust && cargo build -p worktime-api"); + } + return found; +} + +function startActivityWatchStub() { + const now = new Date().toISOString(); + const sockets = new Set(); + const event = { + timestamp: now, + duration: 30, + data: { + username: "demo-user", + userId: "HOST-EXAMPLE\\demo-user", + sessionId: 1, + active: true, + state: "active", + sampleSeconds: 30, + }, + }; + const server = http.createServer((request, response) => { + const parsed = new URL(request.url || "/", "http://127.0.0.1"); + response.setHeader("Content-Type", "application/json; charset=utf-8"); + if (parsed.pathname === "/api/0/buckets") { + response.end(jsonString({ "aw-worktime-sessions_HOST-EXAMPLE": { metadata: { end: now } } })); + return; + } + if (parsed.pathname === "/api/0/buckets/aw-worktime-sessions_HOST-EXAMPLE/events") { + response.end(jsonString([event])); + return; + } + if (parsed.pathname === "/api/0/buckets/aw-worktime-sessions_HOST-EXAMPLE") { + response.end(jsonString({ metadata: { end: now } })); + return; + } + response.statusCode = 404; + response.end(jsonString({ error: "not_found" })); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resolve({ + server, + port: typeof address === "object" && address ? address.port : 0, + close: () => + new Promise((done) => { + for (const socket of sockets) socket.destroy(); + server.close(done); + }), + }); + }); + }); +} + +function spawnWorktimeApi(binary, port, awPort, stateDir) { + const env = { + ...process.env, + AW_SERVER_URL: `http://127.0.0.1:${awPort}`, + AW_WORKTIME_LISTEN_HOST: "127.0.0.1", + AW_WORKTIME_PORT: String(port), + AW_WORKTIME_HOST: "HOST-EXAMPLE", + AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS: "0.5", + AW_WORKTIME_SOURCE_HTTP_TIMEOUT_SECONDS: "0.25", + AW_WORKTIME_EVENTS_LIMIT: "250", + AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS: "0", + AW_WORKTIME_REPORT_CACHE_TTL_SECONDS: "0", + AW_WORKTIME_REPORT_STALE_TTL_SECONDS: "600", + AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS: "600", + AW_WORKTIME_REPORT_DISK_CACHE_DIR: path.join(stateDir, "cache"), + AW_WORKTIME_MANAGEMENT_HISTORY_DIR: path.join(stateDir, "history"), + }; + fs.mkdirSync(env.AW_WORKTIME_REPORT_DISK_CACHE_DIR, { recursive: true }); + fs.mkdirSync(env.AW_WORKTIME_MANAGEMENT_HISTORY_DIR, { recursive: true }); + + const child = childProcess.spawn(binary, [], { + cwd: root, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + const logs = []; + child.stdout.on("data", (chunk) => logs.push(String(chunk))); + child.stderr.on("data", (chunk) => logs.push(String(chunk))); + return { child, logs }; +} + +async function stopChild(child) { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + await new Promise((resolve) => { + const timeout = setTimeout(() => { + if (child.exitCode === null) child.kill("SIGKILL"); + resolve(); + }, 1500); + child.once("exit", () => { + clearTimeout(timeout); + resolve(); + }); + }); +} + +async function getJson(url, timeoutMs = 2500) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = Date.now(); + try { + const response = await fetch(url, { signal: controller.signal }); + const text = await response.text(); + let json = null; + try { + json = JSON.parse(text); + } catch { + // Keep the raw body in the error below. + } + return { + ok: response.ok, + status: response.status, + headers: response.headers, + json, + text, + elapsedMs: Date.now() - started, + }; + } finally { + clearTimeout(timeout); + } +} + +async function waitForHealth(port, timeoutMs = 5000) { + const deadline = Date.now() + timeoutMs; + let lastError = null; + while (Date.now() < deadline) { + try { + const response = await getJson(`http://127.0.0.1:${port}/health`, 500); + if (response.status === 200) return response; + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`worktime-api did not become ready: ${lastError?.message || "timeout"}`); +} + +async function staleFallbackCheck(binary, aw) { + const port = await randomLocalPort(); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-worktime-smoke-stale-")); + const api = spawnWorktimeApi(binary, port, aw.port, stateDir); + try { + await waitForHealth(port); + const reportUrl = `http://127.0.0.1:${port}/reports/worktime/management?format=json&host=HOST-EXAMPLE`; + const fresh = await getJson(reportUrl); + assert(fresh.status === 200, `fresh report status=${fresh.status} body=${fresh.text}`); + assert(fresh.json?.status !== "DEGRADED", `fresh report is unexpectedly degraded: ${fresh.text}`); + + await aw.close(); + const stale = await getJson(reportUrl); + assert(stale.status === 200, `stale report status=${stale.status} body=${stale.text}`); + assert(stale.elapsedMs < 2500, `stale report exceeded bounded response time: ${stale.elapsedMs}ms`); + assert(stale.json?.status === "DEGRADED", `stale report did not return DEGRADED: ${stale.text}`); + assert(stale.json?.stale === true, `stale report did not mark stale=true: ${stale.text}`); + assert(stale.json?.runtime?.report_stale_served === true, `runtime stale flag missing: ${stale.text}`); + assert(stale.headers.get("x-aw-worktime-cache") === "stale", "stale cache header missing"); + + const health = await getJson(`http://127.0.0.1:${port}/health`); + assert(health.status === 200, `health status=${health.status} body=${health.text}`); + assert(health.json?.ok === false, `health must not be fully healthy during degraded mode: ${health.text}`); + assert(health.json?.status === "DEGRADED", `health did not expose DEGRADED: ${health.text}`); + assert(typeof health.json?.runtime?.worktime_events_limit === "number", "health runtime events limit missing"); + assert(typeof health.json?.runtime?.aw_query_timeout_count === "number", "health timeout counter missing"); + + return { + freshStatus: fresh.json?.status || "OK", + staleStatus: stale.json?.status, + staleCacheHeader: stale.headers.get("x-aw-worktime-cache"), + staleElapsedMs: stale.elapsedMs, + healthStatus: health.json?.status, + }; + } finally { + await stopChild(api.child); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +} + +async function noCacheDegradedCheck(binary, closedAwPort) { + const port = await randomLocalPort(); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-worktime-smoke-degraded-")); + const api = spawnWorktimeApi(binary, port, closedAwPort, stateDir); + try { + await waitForHealth(port); + const reportUrl = `http://127.0.0.1:${port}/reports/worktime/management?format=json&host=HOST-EXAMPLE`; + const degraded = await getJson(reportUrl); + assert(degraded.status === 200, `degraded report status=${degraded.status} body=${degraded.text}`); + assert(degraded.elapsedMs < 2500, `degraded report exceeded bounded response time: ${degraded.elapsedMs}ms`); + assert(degraded.json?.status === "DEGRADED", `no-cache report did not return DEGRADED: ${degraded.text}`); + assert(degraded.json?.stale === false, `no-cache report must mark stale=false: ${degraded.text}`); + assert(degraded.json?.reason === "report_unavailable", `no-cache reason mismatch: ${degraded.text}`); + assert(degraded.headers.get("x-aw-worktime-cache") === "degraded", "degraded cache header missing"); + + const health = await getJson(`http://127.0.0.1:${port}/health`); + assert(health.json?.ok === false, `health must be degraded without cache: ${health.text}`); + assert(health.json?.status === "DEGRADED", `health status mismatch without cache: ${health.text}`); + return { + degradedStatus: degraded.json?.status, + stale: degraded.json?.stale, + cacheHeader: degraded.headers.get("x-aw-worktime-cache"), + degradedElapsedMs: degraded.elapsedMs, + healthStatus: health.json?.status, + }; + } finally { + await stopChild(api.child); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +} + +async function main() { + const binary = resolveWorktimeBinary(); + const aw = await startActivityWatchStub(); + const closedAwPort = aw.port; + const staleResult = await staleFallbackCheck(binary, aw); + const degradedResult = await noCacheDegradedCheck(binary, closedAwPort); + console.log( + jsonString({ + binary, + staleFallback: staleResult, + noCacheDegraded: degradedResult, + }), + ); + console.log("worktime degraded smoke OK"); +} + +main().catch((error) => { + console.error(`worktime degraded smoke FAILED: ${error.stack || error.message}`); + process.exit(1); +});