From 337da4bb5fc11ad79cecdc5be7edd0e8e030cb49 Mon Sep 17 00:00:00 2001 From: igor04091968 Date: Tue, 23 Jun 2026 21:52:19 +0300 Subject: [PATCH] feat(workforce): add ClickHouse workforce analytics contour --- README.md | 5 + adk-rust/Cargo.lock | 19 +- adk-rust/Cargo.toml | 1 + .../crates/aw-windows-telemetry/src/main.rs | 49 +- .../crates/aw-workforce-ingest/Cargo.toml | 18 + .../crates/aw-workforce-ingest/src/lib.rs | 1075 +++++++++++++++ .../crates/aw-workforce-ingest/src/main.rs | 10 + adk-rust/crates/quality-gate/src/main.rs | 4 + ansible/host_vars/rdp-prod.yml | 5 + .../users.d/99-aw-workforce-dict.xml | 13 + clickhouse-1c/docker-compose.yml | 1 + clickhouse-workforce/.env.example | 5 + clickhouse-workforce/.gitignore | 2 + clickhouse-workforce/README.md | 155 +++ .../admin/rebuild_aggregates.sql | 118 ++ clickhouse-workforce/catalog/README.md | 43 + .../catalog/application_categories.tsv | 97 ++ .../catalog/domain_categories.tsv | 128 ++ .../catalog/workstation_users.tsv | 8 + .../clickhouse/init/00_database.sql | 1 + .../clickhouse/init/01_raw_tables.sql | 32 + .../init/02_dimensions_dictionaries.sql | 145 ++ .../clickhouse/init/03_materialized_views.sql | 158 +++ .../clickhouse/init/04_quality_views.sql | 105 ++ .../users.d/99-aw-workforce-local.xml | 18 + clickhouse-workforce/docker-compose.yml | 17 + clickhouse-workforce/ops/apply_catalogs.sh | 169 +++ .../ops/aw-workforce-ingest.env.example | 14 + .../ops/aw-workforce-ingest.service | 17 + .../ops/aw-workforce-ingest.timer | 13 + clickhouse-workforce/ops/bootstrap_runtime.sh | 34 + clickhouse-workforce/ops/report_unknowns.sh | 48 + clickhouse-workforce/ops/run_smoke.sh | 136 ++ clickhouse-workforce/sample/seed_demo.sql | 37 + .../sample/seed_sharkon2025_p3.sql | 6 + docs/DETMIR_SUPPORT_SCRIPTS_RU.md | 75 +- docs/DETMIR_SUPPORT_TASKS_RU.md | 7 + docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md | 114 ++ docs/QUALITY_STATUS_RU.md | 13 + docs/REVIEW_CHECKLIST_RU.md | 13 +- .../AW_WORKFORCE_SOURCES_SHARKON2025_RU.md | 63 + .../DICTIONARIES_IMPLEMENTATION_PLAN_RU.md | 1222 +++++++++++------ scripts/detmir-support-run.sh | 204 ++- scripts/detmir-support.env.example | 17 +- 44 files changed, 3973 insertions(+), 461 deletions(-) create mode 100644 adk-rust/crates/aw-workforce-ingest/Cargo.toml create mode 100644 adk-rust/crates/aw-workforce-ingest/src/lib.rs create mode 100644 adk-rust/crates/aw-workforce-ingest/src/main.rs create mode 100644 ansible/host_vars/rdp-prod.yml create mode 100644 clickhouse-1c/clickhouse/users.d/99-aw-workforce-dict.xml create mode 100644 clickhouse-workforce/.env.example create mode 100644 clickhouse-workforce/.gitignore create mode 100644 clickhouse-workforce/README.md create mode 100644 clickhouse-workforce/admin/rebuild_aggregates.sql create mode 100644 clickhouse-workforce/catalog/README.md create mode 100644 clickhouse-workforce/catalog/application_categories.tsv create mode 100644 clickhouse-workforce/catalog/domain_categories.tsv create mode 100644 clickhouse-workforce/catalog/workstation_users.tsv create mode 100644 clickhouse-workforce/clickhouse/init/00_database.sql create mode 100644 clickhouse-workforce/clickhouse/init/01_raw_tables.sql create mode 100644 clickhouse-workforce/clickhouse/init/02_dimensions_dictionaries.sql create mode 100644 clickhouse-workforce/clickhouse/init/03_materialized_views.sql create mode 100644 clickhouse-workforce/clickhouse/init/04_quality_views.sql create mode 100644 clickhouse-workforce/clickhouse/users.d/99-aw-workforce-local.xml create mode 100644 clickhouse-workforce/docker-compose.yml create mode 100644 clickhouse-workforce/ops/apply_catalogs.sh create mode 100644 clickhouse-workforce/ops/aw-workforce-ingest.env.example create mode 100644 clickhouse-workforce/ops/aw-workforce-ingest.service create mode 100644 clickhouse-workforce/ops/aw-workforce-ingest.timer create mode 100644 clickhouse-workforce/ops/bootstrap_runtime.sh create mode 100644 clickhouse-workforce/ops/report_unknowns.sh create mode 100644 clickhouse-workforce/ops/run_smoke.sh create mode 100644 clickhouse-workforce/sample/seed_demo.sql create mode 100644 clickhouse-workforce/sample/seed_sharkon2025_p3.sql create mode 100644 docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md create mode 100644 docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md diff --git a/README.md b/README.md index 1a5ac15..d8341bc 100755 --- a/README.md +++ b/README.md @@ -206,6 +206,9 @@ collectors. - Ежедневная/еженедельная проверка эксплуатационного контура: [матрица проверки контура](docs/CONTOUR_CHECK_MATRIX_RU.md). +- Проверка после инженерных изменений: cargo/security gates, browser smoke и + production smoke: + [эксплуатационный validation runbook](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md). - Enterprise deployment documentation: [deployment guide](docs/ENTERPRISE_DEPLOYMENT_GUIDE_RU.md), @@ -360,6 +363,8 @@ collectors. - [Grafana and Prometheus Monitoring Stack](docs/wiki/Grafana-and-Prometheus-Monitoring-Stack.md) - [Grafana dashboards guide](docs/GRAFANA_DASHBOARDS_RU.md) +- [План внедрения ClickHouse Dictionaries для DetMir](docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md) +- [ClickHouse Workforce scaffold](clickhouse-workforce/README.md) - [Prometheus Exporter](docs/wiki/Prometheus-Exporter.md) Для сборщиков и интерфейса: diff --git a/adk-rust/Cargo.lock b/adk-rust/Cargo.lock index d74816f..a41e1f9 100644 --- a/adk-rust/Cargo.lock +++ b/adk-rust/Cargo.lock @@ -324,6 +324,21 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "aw-workforce-ingest" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "detmir-aw-client", + "reqwest", + "serde", + "serde_json", + "tempfile", + "urlencoding", +] + [[package]] name = "awatch-agent" version = "0.1.0" @@ -1824,9 +1839,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "getrandom 0.3.4", diff --git a/adk-rust/Cargo.toml b/adk-rust/Cargo.toml index 45d92ba..31a587e 100644 --- a/adk-rust/Cargo.toml +++ b/adk-rust/Cargo.toml @@ -32,6 +32,7 @@ members = [ "crates/detmir-readiness", "crates/detmir-portal", "crates/aw-slo-monitor", + "crates/aw-workforce-ingest", "crates/aw-rus-healthd", "crates/detmir-check", "crates/detmir-core", diff --git a/adk-rust/crates/aw-windows-telemetry/src/main.rs b/adk-rust/crates/aw-windows-telemetry/src/main.rs index 71e0ce0..8bc6692 100644 --- a/adk-rust/crates/aw-windows-telemetry/src/main.rs +++ b/adk-rust/crates/aw-windows-telemetry/src/main.rs @@ -29,7 +29,6 @@ const DEFAULT_DLP_STATE: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-sync-st const DEFAULT_DLP_TOKEN: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-upload-token.txt"; const DEFAULT_REMOTE_ROOT: &str = "/opt/activitywatch/clickhouse-1c/landing"; const DEFAULT_SSH_KEY: &str = r"C:\ProgramData\AWatch-rus\ssh\awops_ed25519"; -const DEFAULT_REGISTRY_WORKBOOK: &str = r"E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx"; #[derive(Parser)] #[command(about = "AWatch-rus Windows telemetry uploader without PowerShell runtime wrappers")] @@ -459,8 +458,7 @@ fn run_file1c_upload_inner(args: &mut File1cUpload, log_path: &Path) -> Result<( if args.registry_workbook_path.is_none() { args.registry_workbook_path = json_string(automation, &["registryWorkbookPath"]) .filter(|v| !v.trim().is_empty()) - .map(PathBuf::from) - .or_else(|| Some(PathBuf::from(DEFAULT_REGISTRY_WORKBOOK))); + .map(PathBuf::from); } let scp = system32_path("OpenSSH\\scp.exe"); @@ -1614,6 +1612,10 @@ struct ForegroundWindowContext { window_handle: isize, } +fn has_foreground_context(context: &ForegroundWindowContext) -> bool { + context.process_id != 0 || !context.app.trim().is_empty() || !context.title.trim().is_empty() +} + #[derive(Debug, Clone)] struct WebCategoryRule { name: String, @@ -1741,17 +1743,19 @@ fn run_browser_domains_collector(args: BrowserDomainsCollector) -> Result<()> { loop { let context = foreground_window_context(); let mut loop_failed = false; - match send_browser_window_event(&runtime, &context) { - Ok(()) => events_sent = events_sent.saturating_add(1), - Err(err) => { - loop_failed = true; - record_collector_send_failure( - &runtime, - &mut problems, - &mut send_failures, - "browser window heartbeat", - &err, - ); + if has_foreground_context(&context) { + match send_browser_window_event(&runtime, &context) { + Ok(()) => events_sent = events_sent.saturating_add(1), + Err(err) => { + loop_failed = true; + record_collector_send_failure( + &runtime, + &mut problems, + &mut send_failures, + "browser window heartbeat", + &err, + ); + } } } match send_browser_category_health(&runtime, &context, events_sent) { @@ -5977,7 +5981,7 @@ Connect=File="E:\Bases\Org\Base1"; ID=skip Connect=Srvr="srv";Ref="x"; "#; - let items = parse_v8i_text(text, "user1", Path::new("ibases.v8i")); + let items = parse_v8i_text(text, "fixture-user", Path::new("ibases.v8i")); assert_eq!(items.len(), 1); assert_eq!(items[0].infobase, "База 1"); assert_eq!(items[0].base_id.as_deref(), Some("abc-123")); @@ -6064,6 +6068,19 @@ Connect=Srvr="srv";Ref="x"; assert!(normalize_browser_url("new tab").is_none()); } + #[test] + fn foreground_context_requires_real_window_signal() { + assert!(!has_foreground_context(&ForegroundWindowContext::default())); + assert!(has_foreground_context(&ForegroundWindowContext { + process_id: 1000, + ..ForegroundWindowContext::default() + })); + assert!(has_foreground_context(&ForegroundWindowContext { + title: "1C".to_string(), + ..ForegroundWindowContext::default() + })); + } + #[test] fn dlp_block_is_suppressed_without_native_enforce() { let policy = dlp_policy_from_value( @@ -6183,7 +6200,7 @@ SERVICE_NAME: AWatchRusCollectorGuard #[test] fn file_operations_queue_token_is_filename_safe() { - let token = queue_name_token(r"DOMAIN\Администратор", 3); + let token = queue_name_token(r"DOMAIN\operator", 3); assert!(token.ends_with("-s3")); assert!(!token.contains('\\')); assert!( diff --git a/adk-rust/crates/aw-workforce-ingest/Cargo.toml b/adk-rust/crates/aw-workforce-ingest/Cargo.toml new file mode 100644 index 0000000..b7fd6cf --- /dev/null +++ b/adk-rust/crates/aw-workforce-ingest/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "aw-workforce-ingest" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +clap.workspace = true +detmir-aw-client.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +urlencoding.workspace = true diff --git a/adk-rust/crates/aw-workforce-ingest/src/lib.rs b/adk-rust/crates/aw-workforce-ingest/src/lib.rs new file mode 100644 index 0000000..8536977 --- /dev/null +++ b/adk-rust/crates/aw-workforce-ingest/src/lib.rs @@ -0,0 +1,1075 @@ +use std::collections::{BTreeMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use chrono::{DateTime, SecondsFormat, TimeDelta, Utc}; +use clap::Parser; +use detmir_aw_client::ActivityWatchClient; +use reqwest::blocking::Client; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +const DEFAULT_AW_URL: &str = "http://127.0.0.1:5600/api/0"; +const DEFAULT_CLICKHOUSE_URL: &str = "http://127.0.0.1:8123"; +const DEFAULT_CLICKHOUSE_DATABASE: &str = "aw_workforce"; +const DEFAULT_STATE_PATH: &str = "/var/lib/aw-workforce-ingest/state.json"; +const WINDOW_TABLE: &str = "aw_window_events"; +const BROWSER_TABLE: &str = "aw_browser_events"; + +const WINDOW_COLUMNS: &[&str] = &[ + "event_time", + "host_name", + "user_login", + "process_name", + "window_title", + "duration_sec", + "source_bucket", + "source_event_id", +]; + +const BROWSER_COLUMNS: &[&str] = &[ + "event_time", + "host_name", + "user_login", + "browser_name", + "url", + "title", + "duration_sec", + "source_bucket", + "source_event_id", +]; + +#[derive(Debug, Parser)] +#[command( + about = "Ingest ActivityWatch window/browser events into aw_workforce ClickHouse tables." +)] +pub struct Cli { + #[arg(long, default_value = DEFAULT_AW_URL, env = "AW_WORKFORCE_AW_URL")] + aw_url: String, + + #[arg( + long, + default_value = DEFAULT_CLICKHOUSE_URL, + env = "AW_WORKFORCE_CLICKHOUSE_URL" + )] + clickhouse_url: String, + + #[arg( + long, + default_value = DEFAULT_CLICKHOUSE_DATABASE, + env = "AW_WORKFORCE_CLICKHOUSE_DATABASE" + )] + clickhouse_database: String, + + #[arg(long, default_value = "", env = "AW_WORKFORCE_CLICKHOUSE_USER")] + clickhouse_user: String, + + #[arg(long, default_value = "", env = "AW_WORKFORCE_CLICKHOUSE_PASSWORD")] + clickhouse_password: String, + + #[arg(long, env = "AW_WORKFORCE_HOST")] + host: String, + + #[arg(long = "window-bucket")] + window_buckets: Vec, + + #[arg(long = "browser-bucket")] + browser_buckets: Vec, + + #[arg(long, default_value = DEFAULT_STATE_PATH, env = "AW_WORKFORCE_STATE_PATH")] + state_path: PathBuf, + + #[arg(long, default_value_t = 300, env = "AW_WORKFORCE_OVERLAP_SECONDS")] + overlap_seconds: i64, + + #[arg(long, env = "AW_WORKFORCE_NO_STATE")] + no_state: bool, + + #[arg(long)] + since: Option, + + #[arg(long)] + until: Option, + + #[arg(long, default_value_t = 24)] + hours: i64, + + #[arg(long, default_value_t = 10_000)] + limit: usize, + + #[arg(long, default_value_t = 30)] + timeout_seconds: u64, + + #[arg(long, default_value_t = 3, env = "AW_WORKFORCE_RETRY_ATTEMPTS")] + retry_attempts: usize, + + #[arg(long, default_value_t = 1000, env = "AW_WORKFORCE_RETRY_BACKOFF_MS")] + retry_backoff_ms: u64, + + #[arg(long, env = "AW_WORKFORCE_FAIL_ON_EMPTY")] + fail_on_empty: bool, + + #[arg(long)] + dry_run: bool, + + #[arg(long)] + json: bool, +} + +#[derive(Debug, Clone, Deserialize)] +struct Bucket { + #[serde(default)] + hostname: String, +} + +#[derive(Debug, Deserialize)] +struct RawAwEvent { + #[serde(default)] + id: Option, + timestamp: String, + #[serde(default)] + duration: f64, + #[serde(default)] + data: Value, +} + +#[derive(Debug)] +struct AwEvent { + bucket_id: String, + bucket_hostname: String, + event_id: String, + timestamp: String, + duration: f64, + data: Value, +} + +#[derive(Debug, Clone, Serialize)] +struct WindowRow { + event_time: String, + host_name: String, + user_login: String, + process_name: String, + window_title: String, + duration_sec: u32, + source_bucket: String, + source_event_id: String, +} + +#[derive(Debug, Clone, Serialize)] +struct BrowserRow { + event_time: String, + host_name: String, + user_login: String, + browser_name: String, + url: String, + title: String, + duration_sec: u32, + source_bucket: String, + source_event_id: String, +} + +trait SourceKey { + fn source_bucket(&self) -> &str; + fn source_event_id(&self) -> &str; +} + +impl SourceKey for WindowRow { + fn source_bucket(&self) -> &str { + &self.source_bucket + } + + fn source_event_id(&self) -> &str { + &self.source_event_id + } +} + +impl SourceKey for BrowserRow { + fn source_bucket(&self) -> &str { + &self.source_bucket + } + + fn source_event_id(&self) -> &str { + &self.source_event_id + } +} + +#[derive(Debug, Serialize)] +struct RunSummary { + ok: bool, + dry_run: bool, + aw_url: String, + clickhouse_url: String, + clickhouse_database: String, + host: String, + state_path: String, + state_enabled: bool, + state_loaded_last_end: Option, + state_saved_last_end: Option, + overlap_seconds: i64, + start: String, + end: String, + window_buckets: Vec, + browser_buckets: Vec, + missing_buckets: Vec, + window_events_fetched: usize, + browser_events_fetched: usize, + window_rows_filtered_invalid: usize, + window_rows_prepared: usize, + browser_rows_prepared: usize, + window_rows_inserted: usize, + browser_rows_inserted: usize, + window_rows_skipped_existing: usize, + browser_rows_skipped_existing: usize, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +struct State { + #[serde(default)] + last_end: Option, +} + +#[derive(Debug)] +struct ClickHouseClient { + client: Client, + base_url: String, + database: String, + username: String, + password: String, +} + +impl ClickHouseClient { + fn new( + base_url: &str, + database: &str, + username: &str, + password: &str, + timeout_seconds: u64, + ) -> Result { + let database = normalize_identifier(database, "ClickHouse database")?; + let client = Client::builder() + .timeout(Duration::from_secs(timeout_seconds)) + .no_proxy() + .build() + .context("build ClickHouse HTTP client")?; + Ok(Self { + client, + base_url: base_url.trim_end_matches('/').to_string(), + database, + username: username.to_string(), + password: password.to_string(), + }) + } + + fn execute(&self, sql: &str) -> Result { + if sql.trim().is_empty() { + return Ok(String::new()); + } + let mut req = self + .client + .post(&self.base_url) + .query(&[("database", self.database.as_str())]) + .body(sql.to_string()); + if !self.username.trim().is_empty() { + req = req.basic_auth(self.username.clone(), Some(self.password.clone())); + } + let response = req.send().context("ClickHouse request")?; + let status = response.status(); + let body = response.text().unwrap_or_default(); + if !status.is_success() { + bail!("ClickHouse HTTP {status}: {}", body.trim()); + } + Ok(body) + } + + fn query_json_each_row(&self, sql: &str) -> Result> { + let body = self.execute(sql)?; + body.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).with_context(|| "parse ClickHouse JSONEachRow")) + .collect() + } + + fn insert_json_each_row( + &self, + table: &str, + columns: &[&str], + rows: &[T], + ) -> Result { + if rows.is_empty() { + return Ok(0); + } + let table_name = self.table_name(table)?; + let mut payload = String::new(); + payload.push_str(&format!( + "INSERT INTO {table_name} ({}) FORMAT JSONEachRow\n", + columns.join(", ") + )); + for row in rows { + payload.push_str(&serde_json::to_string(row)?); + payload.push('\n'); + } + self.execute(&payload)?; + Ok(rows.len()) + } + + fn table_name(&self, table: &str) -> Result { + let table = normalize_identifier(table, "ClickHouse table")?; + Ok(format!("{}.{}", self.database, table)) + } +} + +#[derive(Debug, Deserialize)] +struct ExistingId { + source_event_id: String, +} + +pub fn run_from_args() -> Result<()> { + let cli = Cli::parse(); + let json_output = cli.json + || std::env::var("AW_WORKFORCE_JSON") + .map(|value| matches!(value.as_str(), "1" | "true" | "yes")) + .unwrap_or(false); + let summary = run_with_retries(cli)?; + if summary.dry_run || summary.ok { + if json_output || summary.dry_run { + println!("{}", serde_json::to_string_pretty(&summary)?); + } else { + println!( + "aw-workforce-ingest ok dry_run={} window_inserted={} browser_inserted={} window_existing={} browser_existing={} window_filtered_invalid={} missing_buckets={}", + summary.dry_run, + summary.window_rows_inserted, + summary.browser_rows_inserted, + summary.window_rows_skipped_existing, + summary.browser_rows_skipped_existing, + summary.window_rows_filtered_invalid, + summary.missing_buckets.len() + ); + } + } + Ok(()) +} + +fn run_with_retries(cli: Cli) -> Result { + let retry_attempts = cli.retry_attempts.max(1); + let retry_backoff_ms = cli.retry_backoff_ms; + let mut last_error = None; + for attempt in 1..=retry_attempts { + match run_once(&cli) { + Ok(summary) => return Ok(summary), + Err(err) => { + last_error = Some(err); + if attempt < retry_attempts { + std::thread::sleep(Duration::from_millis( + retry_backoff_ms.saturating_mul(attempt as u64), + )); + } + } + } + } + Err(last_error.expect("at least one retry attempt")) +} + +fn run_once(cli: &Cli) -> Result { + validate_cli(cli)?; + let host = cli.host.trim().to_string(); + let state_enabled = !cli.no_state; + let state = if state_enabled { + load_state(&cli.state_path)? + } else { + State::default() + }; + let state_loaded_last_end = state.last_end.clone(); + let end = match &cli.until { + Some(value) => parse_time(value)?, + None => Utc::now(), + }; + let start = match &cli.since { + Some(value) => parse_time(value)?, + None if state_enabled => match &state.last_end { + Some(last_end) => parse_time(last_end)? - TimeDelta::seconds(cli.overlap_seconds), + None => end - TimeDelta::hours(cli.hours), + }, + None => end - TimeDelta::hours(cli.hours), + }; + if start >= end { + bail!("--since must be earlier than --until"); + } + + let window_bucket_ids = default_window_buckets(&host, &cli.window_buckets); + let browser_bucket_ids = default_browser_buckets(&host, &cli.browser_buckets); + + let aw_client = + ActivityWatchClient::new(&cli.aw_url, Duration::from_secs(cli.timeout_seconds))?; + let buckets: BTreeMap = aw_client.get_json("/buckets")?; + let (window_buckets, mut missing_buckets) = resolve_buckets(&buckets, &window_bucket_ids); + let (browser_buckets, browser_missing) = resolve_buckets(&buckets, &browser_bucket_ids); + missing_buckets.extend(browser_missing); + if window_buckets.is_empty() && browser_buckets.is_empty() { + bail!("none of the requested ActivityWatch buckets exist"); + } + + let window_events = fetch_events(&aw_client, &window_buckets, start, end, cli.limit, "window")?; + let browser_events = fetch_events( + &aw_client, + &browser_buckets, + start, + end, + cli.limit, + "browser", + )?; + + let mut window_rows_filtered_invalid = 0usize; + let window_rows = window_events + .iter() + .filter_map(|event| match map_window_row(event, &host) { + Ok(row) if is_invalid_synthetic_window_event(event, &row) => { + window_rows_filtered_invalid = window_rows_filtered_invalid.saturating_add(1); + None + } + Ok(row) => Some(Ok(row)), + Err(err) => Some(Err(err)), + }) + .collect::>>()?; + let browser_rows = browser_events + .iter() + .map(|event| map_browser_row(event, &host)) + .collect::>>()?; + + let mut window_rows_inserted = 0; + let mut browser_rows_inserted = 0; + let mut window_rows_skipped_existing = 0; + let mut browser_rows_skipped_existing = 0; + + if !cli.dry_run { + let clickhouse = ClickHouseClient::new( + &cli.clickhouse_url, + &cli.clickhouse_database, + &cli.clickhouse_user, + &cli.clickhouse_password, + cli.timeout_seconds, + )?; + let (new_window_rows, skipped_window) = + filter_existing_rows(&clickhouse, WINDOW_TABLE, &window_rows)?; + let (new_browser_rows, skipped_browser) = + filter_existing_rows(&clickhouse, BROWSER_TABLE, &browser_rows)?; + window_rows_inserted = + clickhouse.insert_json_each_row(WINDOW_TABLE, WINDOW_COLUMNS, &new_window_rows)?; + browser_rows_inserted = + clickhouse.insert_json_each_row(BROWSER_TABLE, BROWSER_COLUMNS, &new_browser_rows)?; + window_rows_skipped_existing = skipped_window; + browser_rows_skipped_existing = skipped_browser; + } + if cli.fail_on_empty && window_rows.is_empty() && browser_rows.is_empty() { + bail!("no ActivityWatch events fetched for requested buckets and time range"); + } + + let mut state_saved_last_end = None; + if state_enabled && !cli.dry_run { + let next_state = State { + last_end: Some(format_aw_timestamp(end)), + }; + save_state(&cli.state_path, &next_state)?; + state_saved_last_end = next_state.last_end; + } + + Ok(RunSummary { + ok: true, + dry_run: cli.dry_run, + aw_url: cli.aw_url.clone(), + clickhouse_url: cli.clickhouse_url.clone(), + clickhouse_database: cli.clickhouse_database.clone(), + host, + state_path: cli.state_path.display().to_string(), + state_enabled, + state_loaded_last_end, + state_saved_last_end, + overlap_seconds: cli.overlap_seconds, + start: format_aw_timestamp(start), + end: format_aw_timestamp(end), + window_buckets: window_bucket_ids, + browser_buckets: browser_bucket_ids, + missing_buckets, + window_events_fetched: window_events.len(), + browser_events_fetched: browser_events.len(), + window_rows_filtered_invalid, + window_rows_prepared: window_rows.len(), + browser_rows_prepared: browser_rows.len(), + window_rows_inserted, + browser_rows_inserted, + window_rows_skipped_existing, + browser_rows_skipped_existing, + }) +} + +fn validate_cli(cli: &Cli) -> Result<()> { + if cli.host.trim().is_empty() { + bail!("--host or AW_WORKFORCE_HOST is required"); + } + if cli.hours <= 0 { + bail!("--hours must be positive"); + } + if cli.overlap_seconds < 0 { + bail!("--overlap-seconds must be zero or positive"); + } + if cli.limit == 0 { + bail!("--limit must be positive"); + } + if cli.timeout_seconds == 0 { + bail!("--timeout-seconds must be positive"); + } + if cli.retry_attempts == 0 { + bail!("--retry-attempts must be positive"); + } + Ok(()) +} + +fn load_state(path: &Path) -> Result { + if !path.exists() { + return Ok(State::default()); + } + let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&text).with_context(|| format!("parse {}", path.display())) +} + +fn save_state(path: &Path, state: &State) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + let mut temp = tempfile::NamedTempFile::new_in(parent) + .with_context(|| format!("create temp state in {}", parent.display()))?; + serde_json::to_writer_pretty(&mut temp, state)?; + std::io::Write::write_all(&mut temp, b"\n")?; + temp.persist(path) + .map_err(|err| anyhow::anyhow!("persist {}: {}", path.display(), err))?; + return Ok(()); + } + fs::write(path, serde_json::to_string_pretty(state)? + "\n") + .with_context(|| format!("write {}", path.display())) +} + +fn default_window_buckets(host: &str, requested: &[String]) -> Vec { + if requested.is_empty() { + return vec![format!("aw-watcher-window_{host}")]; + } + requested + .iter() + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn default_browser_buckets(host: &str, requested: &[String]) -> Vec { + if requested.is_empty() { + return vec![ + format!("aw-watcher-web-edge_{host}"), + format!("aw-watcher-web-chrome_{host}"), + ]; + } + requested + .iter() + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn resolve_buckets( + buckets: &BTreeMap, + requested: &[String], +) -> (Vec<(String, Bucket)>, Vec) { + let mut selected = Vec::new(); + let mut missing = Vec::new(); + for bucket_id in requested { + match buckets.get(bucket_id) { + Some(bucket) => selected.push((bucket_id.clone(), bucket.clone())), + None => missing.push(bucket_id.clone()), + } + } + (selected, missing) +} + +fn fetch_events( + client: &ActivityWatchClient, + buckets: &[(String, Bucket)], + start: DateTime, + end: DateTime, + limit: usize, + label: &str, +) -> Result> { + let mut events = Vec::new(); + for (bucket_id, bucket) in buckets { + let path = build_events_path(bucket_id, start, end, limit); + let raw_events: Vec = client + .get_json(&path) + .with_context(|| format!("fetch {label} events from {bucket_id}"))?; + events.extend(raw_events.into_iter().map(|raw| { + let data = normalized_data(raw.data); + let event_id = event_id(bucket_id, &raw.id, &raw.timestamp, raw.duration, &data); + AwEvent { + bucket_id: bucket_id.clone(), + bucket_hostname: bucket.hostname.clone(), + event_id, + timestamp: raw.timestamp, + duration: raw.duration, + data, + } + })); + } + Ok(events) +} + +fn build_events_path( + bucket_id: &str, + start: DateTime, + end: DateTime, + limit: usize, +) -> String { + format!( + "/buckets/{}/events?start={}&end={}&limit={}", + urlencoding::encode(bucket_id), + urlencoding::encode(&format_aw_timestamp(start)), + urlencoding::encode(&format_aw_timestamp(end)), + limit + ) +} + +fn map_window_row(event: &AwEvent, fallback_host: &str) -> Result { + Ok(WindowRow { + event_time: format_clickhouse_datetime(parse_time(&event.timestamp)?), + host_name: event_host(event, fallback_host), + user_login: event_user(event), + process_name: normalize_process_name( + &first_string( + &event.data, + &["app", "process", "process_name", "foregroundProcess"], + ) + .unwrap_or_else(|| "unknown".to_string()), + ), + window_title: first_string(&event.data, &["title", "window_title", "foregroundTitle"]) + .unwrap_or_default(), + duration_sec: duration_to_u32(event.duration), + source_bucket: event.bucket_id.clone(), + source_event_id: event.event_id.clone(), + }) +} + +fn is_invalid_synthetic_window_event(_event: &AwEvent, row: &WindowRow) -> bool { + row.user_login == "unknown" + || (row.process_name == "unknown" && row.window_title.trim().is_empty()) +} + +fn map_browser_row(event: &AwEvent, fallback_host: &str) -> Result { + Ok(BrowserRow { + event_time: format_clickhouse_datetime(parse_time(&event.timestamp)?), + host_name: event_host(event, fallback_host), + user_login: event_user(event), + browser_name: normalize_process_name( + &first_string(&event.data, &["browser", "app"]) + .unwrap_or_else(|| "unknown".to_string()), + ), + url: first_string(&event.data, &["url", "uri"]).unwrap_or_default(), + title: first_string(&event.data, &["title"]).unwrap_or_default(), + duration_sec: duration_to_u32(event.duration), + source_bucket: event.bucket_id.clone(), + source_event_id: event.event_id.clone(), + }) +} + +fn event_host(event: &AwEvent, fallback_host: &str) -> String { + first_string(&event.data, &["hostname", "host"]) + .or_else(|| { + let trimmed = event.bucket_hostname.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + }) + .unwrap_or_else(|| fallback_host.to_string()) +} + +fn event_user(event: &AwEvent) -> String { + let raw = first_string( + &event.data, + &["username", "user", "userName", "login", "userId"], + ); + normalize_user_login(raw.as_deref()) +} + +fn normalized_data(value: Value) -> Value { + if value.is_object() { + value + } else { + json!({ "raw": value }) + } +} + +fn first_string(data: &Value, keys: &[&str]) -> Option { + let object = data.as_object()?; + for key in keys { + if let Some(value) = object.get(*key) { + match value { + Value::Null => {} + Value::String(value) => { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + other => { + let rendered = other.to_string(); + if !rendered.is_empty() { + return Some(rendered); + } + } + } + } + } + None +} + +fn normalize_user_login(value: Option<&str>) -> String { + let Some(value) = value else { + return "unknown".to_string(); + }; + let mut value = value.trim(); + if value.is_empty() { + return "unknown".to_string(); + } + if let Some((_domain, user)) = value.rsplit_once('\\') { + value = user; + } else if let Some((_domain, user)) = value.rsplit_once('/') { + value = user; + } + let normalized = value.trim().to_ascii_lowercase(); + if normalized.is_empty() || normalized.ends_with('$') { + "unknown".to_string() + } else { + normalized + } +} + +fn normalize_process_name(value: &str) -> String { + let normalized = value.trim().to_ascii_lowercase(); + if normalized.is_empty() { + "unknown".to_string() + } else { + normalized + } +} + +fn duration_to_u32(value: f64) -> u32 { + if !value.is_finite() || value <= 0.0 { + return 0; + } + value.round().min(u32::MAX as f64) as u32 +} + +fn event_id( + bucket_id: &str, + raw_id: &Option, + timestamp: &str, + duration: f64, + data: &Value, +) -> String { + match raw_id { + Some(Value::String(value)) if !value.trim().is_empty() => value.trim().to_string(), + Some(Value::Number(value)) => value.to_string(), + Some(Value::Bool(value)) => value.to_string(), + _ => { + let payload = serde_json::to_string(data).unwrap_or_else(|_| "{}".to_string()); + format!("{bucket_id}|{timestamp}|{duration}|{payload}") + } + } +} + +fn filter_existing_rows( + clickhouse: &ClickHouseClient, + table: &str, + rows: &[T], +) -> Result<(Vec, usize)> { + if rows.is_empty() { + return Ok((Vec::new(), 0)); + } + let existing = existing_source_keys(clickhouse, table, rows)?; + let mut new_rows = Vec::new(); + let mut skipped = 0; + for row in rows { + let key = ( + row.source_bucket().to_string(), + row.source_event_id().to_string(), + ); + if existing.contains(&key) { + skipped += 1; + } else { + new_rows.push(row.clone()); + } + } + Ok((new_rows, skipped)) +} + +fn existing_source_keys( + clickhouse: &ClickHouseClient, + table: &str, + rows: &[T], +) -> Result> { + let table_name = clickhouse.table_name(table)?; + let mut by_bucket: BTreeMap> = BTreeMap::new(); + for row in rows { + by_bucket + .entry(row.source_bucket().to_string()) + .or_default() + .insert(row.source_event_id().to_string()); + } + + let mut existing = HashSet::new(); + for (bucket, ids) in by_bucket { + let ids: Vec = ids.into_iter().collect(); + for chunk in ids.chunks(500) { + let id_list = chunk + .iter() + .map(|id| clickhouse_string_literal(id)) + .collect::>() + .join(", "); + let sql = format!( + "SELECT source_event_id FROM {table_name} WHERE source_bucket = {} AND source_event_id IN ({id_list}) FORMAT JSONEachRow", + clickhouse_string_literal(&bucket) + ); + let found: Vec = clickhouse.query_json_each_row(&sql)?; + for item in found { + existing.insert((bucket.clone(), item.source_event_id)); + } + } + } + Ok(existing) +} + +fn parse_time(value: &str) -> Result> { + Ok(DateTime::parse_from_rfc3339(value) + .with_context(|| format!("parse timestamp {value}"))? + .with_timezone(&Utc)) +} + +fn format_aw_timestamp(value: DateTime) -> String { + value.to_rfc3339_opts(SecondsFormat::Micros, true) +} + +fn format_clickhouse_datetime(value: DateTime) -> String { + value.format("%Y-%m-%d %H:%M:%S").to_string() +} + +fn normalize_identifier(value: &str, label: &str) -> Result { + let value = value.trim(); + if value.is_empty() + || !value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') + { + bail!("{label} must contain only ASCII letters, digits and underscores"); + } + Ok(value.to_string()) +} + +fn clickhouse_string_literal(value: &str) -> String { + let mut out = String::from("'"); + for ch in value.chars() { + match ch { + '\'' => out.push_str("\\'"), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + _ => out.push(ch), + } + } + out.push('\''); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const TEST_HOST: &str = "HOST-EXAMPLE"; + const TEST_USER: &str = "Alice"; + const TEST_BUCKET: &str = "aw-watcher-window_HOST-EXAMPLE"; + + #[test] + fn normalizes_domain_user_login() { + assert_eq!(normalize_user_login(Some("HOST-EXAMPLE\\Alice")), "alice"); + assert_eq!(normalize_user_login(Some("Bob")), "bob"); + assert_eq!(normalize_user_login(Some("DOMAIN/Carol")), "carol"); + assert_eq!(normalize_user_login(Some("DOMAIN\\Оператор")), "Оператор"); + assert_eq!(normalize_user_login(Some("MACHINE$")), "unknown"); + assert_eq!(normalize_user_login(None), "unknown"); + } + + #[test] + fn builds_bounded_aw_events_path() { + let start = parse_time("2026-06-23T10:00:00Z").unwrap(); + let end = parse_time("2026-06-23T11:00:00Z").unwrap(); + let path = build_events_path(TEST_BUCKET, start, end, 500); + assert!(path.starts_with("/buckets/aw-watcher-window_HOST-EXAMPLE/events?")); + assert!(path.contains("limit=500")); + assert!(path.contains("start=2026-06-23T10%3A00%3A00.000000Z")); + assert!(path.contains("end=2026-06-23T11%3A00%3A00.000000Z")); + } + + #[test] + fn maps_window_event_to_clickhouse_row() { + let raw = RawAwEvent { + id: Some(Value::from(42)), + timestamp: "2026-06-23T12:17:13.853Z".to_string(), + duration: 1.6, + data: json!({ + "app": "1CV8C.EXE", + "hostname": TEST_HOST, + "title": "document", + "username": TEST_USER + }), + }; + let data = normalized_data(raw.data); + let event = AwEvent { + bucket_id: TEST_BUCKET.to_string(), + bucket_hostname: TEST_HOST.to_string(), + event_id: event_id(TEST_BUCKET, &raw.id, &raw.timestamp, raw.duration, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + }; + let row = map_window_row(&event, "fallback").unwrap(); + assert_eq!(row.event_time, "2026-06-23 12:17:13"); + assert_eq!(row.host_name, TEST_HOST); + assert_eq!(row.user_login, "alice"); + assert_eq!(row.process_name, "1cv8c.exe"); + assert_eq!(row.duration_sec, 2); + assert_eq!(row.source_event_id, "42"); + } + + #[test] + fn detects_invalid_synthetic_window_event() { + let raw = RawAwEvent { + id: Some(Value::from(43)), + timestamp: "2026-06-23T12:18:13.853Z".to_string(), + duration: 10.0, + data: json!({ + "app": "", + "hostname": TEST_HOST, + "processId": 0, + "source": "aw-windows-telemetry-rust", + "title": "", + "username": TEST_USER + }), + }; + let data = normalized_data(raw.data); + let event = AwEvent { + bucket_id: TEST_BUCKET.to_string(), + bucket_hostname: TEST_HOST.to_string(), + event_id: event_id(TEST_BUCKET, &raw.id, &raw.timestamp, raw.duration, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + }; + let row = map_window_row(&event, "fallback").unwrap(); + assert!(is_invalid_synthetic_window_event(&event, &row)); + } + + #[test] + fn detects_blank_window_event_without_source_marker() { + let raw = RawAwEvent { + id: Some(Value::from(45)), + timestamp: "2026-06-23T12:20:13.853Z".to_string(), + duration: 10.0, + data: json!({}), + }; + let data = normalized_data(raw.data); + let event = AwEvent { + bucket_id: TEST_BUCKET.to_string(), + bucket_hostname: TEST_HOST.to_string(), + event_id: event_id(TEST_BUCKET, &raw.id, &raw.timestamp, raw.duration, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + }; + let row = map_window_row(&event, "fallback").unwrap(); + assert!(is_invalid_synthetic_window_event(&event, &row)); + } + + #[test] + fn detects_window_event_without_user_identity() { + let raw = RawAwEvent { + id: Some(Value::from(46)), + timestamp: "2026-06-23T12:21:13.853Z".to_string(), + duration: 10.0, + data: json!({ + "app": "1cv8c.exe", + "title": "ТРАНСГАЗ" + }), + }; + let data = normalized_data(raw.data); + let event = AwEvent { + bucket_id: TEST_BUCKET.to_string(), + bucket_hostname: TEST_HOST.to_string(), + event_id: event_id(TEST_BUCKET, &raw.id, &raw.timestamp, raw.duration, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + }; + let row = map_window_row(&event, "fallback").unwrap(); + assert!(is_invalid_synthetic_window_event(&event, &row)); + } + + #[test] + fn keeps_real_window_event_from_rust_collector() { + let raw = RawAwEvent { + id: Some(Value::from(44)), + timestamp: "2026-06-23T12:19:13.853Z".to_string(), + duration: 0.0, + data: json!({ + "app": "EXPLORER.EXE", + "hostname": TEST_HOST, + "processId": 1000, + "source": "aw-windows-telemetry-rust", + "title": "Рабочий стол", + "username": TEST_USER + }), + }; + let data = normalized_data(raw.data); + let event = AwEvent { + bucket_id: TEST_BUCKET.to_string(), + bucket_hostname: TEST_HOST.to_string(), + event_id: event_id(TEST_BUCKET, &raw.id, &raw.timestamp, raw.duration, &data), + timestamp: raw.timestamp, + duration: raw.duration, + data, + }; + let row = map_window_row(&event, "fallback").unwrap(); + assert!(!is_invalid_synthetic_window_event(&event, &row)); + } + + #[test] + fn escapes_clickhouse_string_literals() { + assert_eq!( + clickhouse_string_literal("domain\\user'1\n"), + "'domain\\\\user\\'1\\n'" + ); + } + + #[test] + fn loads_and_saves_state_atomically() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.json"); + let state = State { + last_end: Some("2026-06-23T12:00:00.000000Z".to_string()), + }; + save_state(&path, &state).unwrap(); + let loaded = load_state(&path).unwrap(); + assert_eq!(loaded.last_end, state.last_end); + } + + #[test] + fn missing_state_is_empty() { + let dir = tempfile::tempdir().unwrap(); + let loaded = load_state(&dir.path().join("missing.json")).unwrap(); + assert_eq!(loaded.last_end, None); + } +} diff --git a/adk-rust/crates/aw-workforce-ingest/src/main.rs b/adk-rust/crates/aw-workforce-ingest/src/main.rs new file mode 100644 index 0000000..5f6b943 --- /dev/null +++ b/adk-rust/crates/aw-workforce-ingest/src/main.rs @@ -0,0 +1,10 @@ +fn main() { + let code = match aw_workforce_ingest::run_from_args() { + Ok(()) => 0, + Err(err) => { + eprintln!("{err:#}"); + 1 + } + }; + std::process::exit(code); +} diff --git a/adk-rust/crates/quality-gate/src/main.rs b/adk-rust/crates/quality-gate/src/main.rs index 1fc883a..ad46e6f 100644 --- a/adk-rust/crates/quality-gate/src/main.rs +++ b/adk-rust/crates/quality-gate/src/main.rs @@ -286,6 +286,7 @@ fn is_allowed_python_runtime_path(rel: &str) -> bool { || rel == "proxmox/tsj_guardian_bot.py" || rel == "proxmox/test_tsj_guardian_bot.py" || rel == "scripts/package_rust_release_binaries.py" + || rel == "scripts/public_secret_pattern_check.py" } fn is_detmir_retired_runtime_path(rel: &str) -> bool { @@ -348,6 +349,9 @@ mod tests { assert!(is_allowed_python_runtime_path( "scripts/package_rust_release_binaries.py" )); + assert!(is_allowed_python_runtime_path( + "scripts/public_secret_pattern_check.py" + )); } #[test] diff --git a/ansible/host_vars/rdp-prod.yml b/ansible/host_vars/rdp-prod.yml new file mode 100644 index 0000000..699a411 --- /dev/null +++ b/ansible/host_vars/rdp-prod.yml @@ -0,0 +1,5 @@ +--- +# SHARKON2025 uses aw-windows-telemetry browser-domains-collector as the +# per-user currentwindow source. The legacy aw-watcher-window process emits +# no-user duplicate rows in this RDP setup, so keep it disabled for this host. +aw_windows_window_enabled: false diff --git a/clickhouse-1c/clickhouse/users.d/99-aw-workforce-dict.xml b/clickhouse-1c/clickhouse/users.d/99-aw-workforce-dict.xml new file mode 100644 index 0000000..893c923 --- /dev/null +++ b/clickhouse-1c/clickhouse/users.d/99-aw-workforce-dict.xml @@ -0,0 +1,13 @@ + + + + + + 127.0.0.1 + ::1 + + readonly + default + + + diff --git a/clickhouse-1c/docker-compose.yml b/clickhouse-1c/docker-compose.yml index a940ff2..19f5151 100644 --- a/clickhouse-1c/docker-compose.yml +++ b/clickhouse-1c/docker-compose.yml @@ -13,6 +13,7 @@ services: volumes: - clickhouse_1c_data:/var/lib/clickhouse - ./clickhouse/init:/docker-entrypoint-initdb.d:ro + - ./clickhouse/users.d/99-aw-workforce-dict.xml:/etc/clickhouse-server/users.d/99-aw-workforce-dict.xml:ro volumes: clickhouse_1c_data: diff --git a/clickhouse-workforce/.env.example b/clickhouse-workforce/.env.example new file mode 100644 index 0000000..894d820 --- /dev/null +++ b/clickhouse-workforce/.env.example @@ -0,0 +1,5 @@ +CLICKHOUSE_DB=aw_workforce +CLICKHOUSE_HTTP_BIND=127.0.0.1 +CLICKHOUSE_PORT=8124 +CLICKHOUSE_NATIVE_BIND=127.0.0.1 +CLICKHOUSE_NATIVE_PORT=9001 diff --git a/clickhouse-workforce/.gitignore b/clickhouse-workforce/.gitignore new file mode 100644 index 0000000..789e73f --- /dev/null +++ b/clickhouse-workforce/.gitignore @@ -0,0 +1,2 @@ +.env +.local/ diff --git a/clickhouse-workforce/README.md b/clickhouse-workforce/README.md new file mode 100644 index 0000000..f69578b --- /dev/null +++ b/clickhouse-workforce/README.md @@ -0,0 +1,155 @@ +# ClickHouse Workforce analytics for AWatch-rus / DetMir + +Этот каталог содержит воспроизводимый ClickHouse-слой для привязки событий +AWatch-rus к оргструктуре, классификации приложений и доменов, а также для +быстрых агрегатов Grafana. + +Слой не заменяет `clickhouse-1c/`. Это отдельный контур для workforce/web +аналитики ActivityWatch-событий. + +## Состав + +- `docker-compose.yml` - локальный ClickHouse scaffold. +- `clickhouse/init/00_database.sql` - база `aw_workforce`. +- `clickhouse/init/01_raw_tables.sql` - нормализованные staging tables для + window/browser events. +- `clickhouse/init/02_dimensions_dictionaries.sql` - dimension tables и + ClickHouse Dictionaries. +- `clickhouse/init/03_materialized_views.sql` - агрегированная таблица и + materialized views для Grafana. +- `clickhouse/init/04_quality_views.sql` - views контроля unknown-зон. +- `sample/seed_demo.sql` - минимальные demo-данные для smoke-проверки. +- `sample/seed_sharkon2025_p3.sql` - первая реальная привязка + `SHARKON2025/sharkon2025/user1/tsj`. +- `ops/run_smoke.sh` - локальный smoke для DDL, dictionaries и агрегатов. +- `ops/aw-workforce-ingest.service` / `.timer` - production timer для + инкрементальной загрузки. +- `ops/aw-workforce-ingest.env.example` - переменные окружения loader-а. +- `catalog/*.tsv` - управляемые администратором справочники. +- `ops/apply_catalogs.sh` - полная загрузка справочников, reload dictionaries, + опциональный rebuild агрегатов. +- `ops/report_unknowns.sh` - быстрый отчет top unknown users/processes/domains. + +## Быстрый старт + +```bash +cd clickhouse-workforce +docker compose up -d +./ops/run_smoke.sh +``` + +Локальный scaffold не задает `CLICKHOUSE_USER/PASSWORD` через Docker entrypoint: +это оставляет штатный dev-доступ ClickHouse без пароля и не ломает +`SOURCE(CLICKHOUSE(...))` у dictionaries. Файл +`clickhouse/users.d/99-aw-workforce-local.xml` разрешает HTTP-запросы от Docker +host, а HTTP/native порты по умолчанию привязаны только к `127.0.0.1`. + +Скрипт применяет SQL в правильном порядке, загружает demo seed и проверяет: + +- статус dictionaries; +- наличие hourly aggregate rows; +- daily productivity view; +- unknown quality views. + +`sample/seed_demo.sql` добавляет демонстрационные строки. Для чистого повтора +локального smoke пересоздайте volume: + +```bash +docker compose down -v +docker compose up -d +./ops/run_smoke.sh +``` + +## Production порядок + +1. Реальные источники `aw_window_events` и `aw_browser_events` для + `SHARKON2025` подтверждены: + `docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md`. +2. Настроить ingest из ActivityWatch/exporter в staging tables. +3. Загрузить `dim_workstation_user`, `dim_application_category`, + `dim_domain_category`. +4. Проверить `system.dictionaries`. +5. Включить materialized views. +6. Перевести Grafana на `agg_workforce_productivity_hourly` и + `v_workforce_productivity_daily`. + +Исправление справочников не пересчитывает старые агрегаты автоматически. +Для исторических периодов нужен backfill по регламенту из +`docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md`. + +## Live ingest P2/P3 + +Rust loader находится в `adk-rust/crates/aw-workforce-ingest`. + +Пример загрузки bounded-окна из живого AW API в локальный ClickHouse: + +```bash +cargo run --manifest-path ../adk-rust/Cargo.toml -p aw-workforce-ingest -- \ + --aw-url http://10.10.10.13:5600/api/0 \ + --clickhouse-url http://127.0.0.1:8124 \ + --host SHARKON2025 \ + --hours 24 \ + --json +``` + +Применение первой привязки P3: + +```bash +docker exec -i aw-rus-workforce-clickhouse clickhouse-client --multiquery \ + < sample/seed_sharkon2025_p3.sql +``` + +## Production ingest P4 + +В штатном режиме loader запускается без `--since/--until`: он читает +`AW_WORKFORCE_STATE_PATH`, берет `last_end - AW_WORKFORCE_OVERLAP_SECONDS`, +загружает bounded range и атомарно сохраняет новый `last_end`. Повторная +загрузка overlap-окна не удваивает данные, потому что loader перед вставкой +проверяет `source_bucket + source_event_id`. + +Runtime-файлы: + +```bash +cd clickhouse-workforce +sudo bash ./ops/bootstrap_runtime.sh +sudo install -m 0755 ../adk-rust/target/release/aw-workforce-ingest \ + /usr/local/bin/aw-workforce-ingest +sudo editor /etc/activitywatch/aw-workforce-ingest.env +sudo systemctl enable --now aw-workforce-ingest.timer +``` + +Ручная production-проверка одного цикла: + +```bash +sudo systemctl start aw-workforce-ingest.service +sudo journalctl -u aw-workforce-ingest.service -n 80 --no-pager +``` + +## Admin workflow справочников P5 + +Справочники ведутся через `catalog/*.tsv`. Это полный source of truth: +`ops/apply_catalogs.sh` очищает dimension tables, загружает TSV, reload-ит +dictionaries и, если нужно, пересобирает агрегаты. + +Посмотреть слепые зоны: + +```bash +./ops/report_unknowns.sh +``` + +Добавить или изменить категорию: + +```bash +editor catalog/application_categories.tsv +REBUILD_AGGREGATES=1 ./ops/apply_catalogs.sh +``` + +Убрать запись из отчетов без потери аудита: поставить `is_active=0` в TSV и +запустить: + +```bash +REBUILD_AGGREGATES=1 ./ops/apply_catalogs.sh +``` + +Если менялись только future-facing справочники и старые агрегаты пересчитывать +не нужно, можно запустить без `REBUILD_AGGREGATES=1`. diff --git a/clickhouse-workforce/admin/rebuild_aggregates.sql b/clickhouse-workforce/admin/rebuild_aggregates.sql new file mode 100644 index 0000000..7f9ad14 --- /dev/null +++ b/clickhouse-workforce/admin/rebuild_aggregates.sql @@ -0,0 +1,118 @@ +TRUNCATE TABLE aw_workforce.agg_workforce_productivity_hourly; + +INSERT INTO aw_workforce.agg_workforce_productivity_hourly +SELECT + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'), + 'unknown' + ) AS branch, + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'), + 'unknown' + ) AS department, + + 'desktop' AS activity_type, + if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, 'unknown'), + 'unknown' + ) AS category, + if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, 'unknown'), + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '', + 0, + 1 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') NOT IN ('', 'unknown'), + 0, + 1 + ))) AS unknown_category_events +FROM aw_workforce.aw_window_events +GROUP BY + bucket_start, + event_date, + branch, + department, + activity_type, + category, + productivity_class; + +INSERT INTO aw_workforce.agg_workforce_productivity_hourly +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'), + 'unknown' + ) AS branch, + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'), + 'unknown' + ) AS department, + + 'browser' AS activity_type, + if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, 'unknown'), + 'unknown' + ) AS category, + if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, 'unknown'), + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '', + 0, + 1 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') NOT IN ('', 'unknown'), + 0, + 1 + ))) AS unknown_category_events +FROM aw_workforce.aw_browser_events +WHERE domain_name != '' +GROUP BY + bucket_start, + event_date, + branch, + department, + activity_type, + category, + productivity_class; diff --git a/clickhouse-workforce/catalog/README.md b/clickhouse-workforce/catalog/README.md new file mode 100644 index 0000000..0b59b08 --- /dev/null +++ b/clickhouse-workforce/catalog/README.md @@ -0,0 +1,43 @@ +# Workforce catalogs + +Эти TSV-файлы являются source of truth для справочников `aw_workforce`. + +## Операции администратора + +- Добавить категорию: добавить строку в соответствующий `*.tsv`, поставить + `is_active=1`, запустить `ops/apply_catalogs.sh`. +- Изменить категорию: изменить строку в `*.tsv`, запустить + `REBUILD_AGGREGATES=1 ops/apply_catalogs.sh`. +- Удалить категорию из отчетов: либо удалить строку из `*.tsv`, либо оставить + строку для аудита и поставить `is_active=0`, затем запустить + `REBUILD_AGGREGATES=1 ops/apply_catalogs.sh`. + +`is_active=0` трактуется отчетами как `unknown`: запись остается видимой в +каталоге, но не используется для обогащения. + +## Файлы + +- `workstation_users.tsv` - привязка `host_name + user_login` к оргструктуре. +- `application_categories.tsv` - классификация desktop processes. +- `domain_categories.tsv` - классификация browser domains. + +Формат: `TabSeparatedWithNames`, первая строка - имена колонок. Не используйте +tab-символы внутри значений. + +## Таксономия РФ baseline + +Baseline `catalog-ru-20260623` делит домены и приложения на рабочие для РФ +категории: `1c`, `edo_reporting`, `reporting`, `banking`, `government`, +`procurement`, `business_reference`, `legal_reference`, `mail`, `office`, +`pdf`, `browser`, `cloud_docs`, `communication`, `developer`, `admin_tool`, +`remote_admin`, `security_crypto`, `search`, `maps_reference`, `marketplace`, +`news`, `social`, `media`, `gaming`, `system`. + +Правило ведения: справочник хранит только точные ключи. Для приложений это +нормализованный `process_name` в нижнем регистре, например `1cv8c.exe`; для +web - точный host из URL, например `online.sbis.ru`. Wildcard-строки не +используются, потому что ClickHouse Dictionary выполняет точный lookup. + +`productivity_class` держите в одном из значений: `productive`, `neutral`, +`non_productive`, `unknown`. Для облаков, мессенджеров, AI и внешней почты +ставьте `risk_level=medium`, если нужна последующая DLP/policy проверка. diff --git a/clickhouse-workforce/catalog/application_categories.tsv b/clickhouse-workforce/catalog/application_categories.tsv new file mode 100644 index 0000000..8b87521 --- /dev/null +++ b/clickhouse-workforce/catalog/application_categories.tsv @@ -0,0 +1,97 @@ +process_name application_name vendor category productivity_class risk_level is_system source comment is_active +1cv8.exe 1C:Enterprise 1C 1c productive low 0 catalog-ru-20260623 1C thick client 1 +1cv8c.exe 1C:Enterprise client 1C 1c productive low 0 catalog-ru-20260623 1C current RDP workload 1 +1cv8s.exe 1C:Enterprise starter 1C 1c productive low 0 catalog-ru-20260623 1C server or launcher process 1 +1cestart.exe 1C:Enterprise launcher 1C 1c productive low 0 catalog-ru-20260623 1C launcher 1 +1cv8u.exe 1C:Enterprise updater 1C 1c productive low 0 catalog-ru-20260623 1C updater 1 +sbis.exe SBIS SBIS edo_reporting productive low 0 catalog-ru-20260623 EDO and reporting client 1 +sbisplugin.exe SBIS Plugin SBIS edo_reporting productive low 0 catalog-ru-20260623 SBIS browser integration 1 +saby.exe Saby SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby business client 1 +sabyapps.exe Saby Apps SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby desktop apps 1 +diadoc.exe Diadoc Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic document exchange client 1 +kontur.plugin.exe Kontur Plugin Kontur edo_reporting productive low 0 catalog-ru-20260623 Kontur browser integration 1 +extern.exe Kontur Extern Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic reporting client 1 +cryptoarm.exe CryptoARM Digital Technologies security_crypto productive low 0 catalog-ru-20260623 Crypto signing tool 1 +cryptcp.exe CryptoPro CryptCP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro command line signing tool 1 +cprocsp.exe CryptoPro CSP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro CSP utility 1 +certmgr.exe Certificate Manager Microsoft security_crypto productive low 0 catalog-ru-20260623 Certificate management 1 +cadesplugin.exe CAdES Plugin CryptoPro security_crypto productive low 0 catalog-ru-20260623 Browser crypto plugin 1 +chrome.exe Google Chrome Google browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1 +msedge.exe Microsoft Edge Microsoft browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1 +browser.exe Yandex Browser Yandex browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1 +firefox.exe Mozilla Firefox Mozilla browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1 +opera.exe Opera Opera browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1 +iexplore.exe Internet Explorer Microsoft browser neutral medium 0 catalog-ru-20260623 Legacy browser; domains classify productivity 1 +outlook.exe Microsoft Outlook Microsoft mail productive low 0 catalog-ru-20260623 Business mail client 1 +thunderbird.exe Thunderbird Mozilla mail productive low 0 catalog-ru-20260623 Mail client 1 +winword.exe Microsoft Word Microsoft office productive low 0 catalog-ru-20260623 Office document editor 1 +excel.exe Microsoft Excel Microsoft office productive low 0 catalog-ru-20260623 Spreadsheet editor 1 +powerpnt.exe Microsoft PowerPoint Microsoft office productive low 0 catalog-ru-20260623 Presentation editor 1 +onenote.exe Microsoft OneNote Microsoft office productive low 0 catalog-ru-20260623 Notes and documents 1 +msaccess.exe Microsoft Access Microsoft office productive low 0 catalog-ru-20260623 Database office tool 1 +soffice.bin LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite 1 +soffice.exe LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite launcher 1 +swriter.exe LibreOffice Writer The Document Foundation office productive low 0 catalog-ru-20260623 Office document editor 1 +scalc.exe LibreOffice Calc The Document Foundation office productive low 0 catalog-ru-20260623 Spreadsheet editor 1 +simpress.exe LibreOffice Impress The Document Foundation office productive low 0 catalog-ru-20260623 Presentation editor 1 +notepad.exe Notepad Microsoft office neutral low 0 catalog-ru-20260623 Text editor 1 +notepad++.exe Notepad++ Notepad++ office productive low 0 catalog-ru-20260623 Text and config editor 1 +acrord32.exe Adobe Acrobat Reader Adobe pdf productive low 0 catalog-ru-20260623 PDF reader 1 +acrobat.exe Adobe Acrobat Adobe pdf productive low 0 catalog-ru-20260623 PDF editor 1 +foxitpdfreader.exe Foxit PDF Reader Foxit pdf productive low 0 catalog-ru-20260623 PDF reader 1 +pdfxedit.exe PDF-XChange Editor Tracker Software pdf productive low 0 catalog-ru-20260623 PDF editor 1 +7zfm.exe 7-Zip File Manager 7-Zip archive productive low 0 catalog-ru-20260623 Archive manager 1 +7zg.exe 7-Zip GUI 7-Zip archive productive low 0 catalog-ru-20260623 Archive tool 1 +7z.exe 7-Zip CLI 7-Zip archive productive low 0 catalog-ru-20260623 Archive command line tool 1 +winrar.exe WinRAR RARLAB archive productive low 0 catalog-ru-20260623 Archive manager 1 +explorer.exe Windows Explorer Microsoft system neutral low 1 catalog-ru-20260623 Windows shell and file manager 1 +dwm.exe Desktop Window Manager Microsoft system neutral low 1 catalog-ru-20260623 Windows desktop compositor 1 +taskhostw.exe Task Host Microsoft system neutral low 1 catalog-ru-20260623 Windows task host 1 +applicationframehost.exe Application Frame Host Microsoft system neutral low 1 catalog-ru-20260623 Windows app frame host 1 +startmenuexperiencehost.exe Start Menu Experience Host Microsoft system neutral low 1 catalog-ru-20260623 Windows Start menu 1 +searchapp.exe Windows Search Microsoft system neutral low 1 catalog-ru-20260623 Windows search UI 1 +systemsettings.exe Windows Settings Microsoft system neutral low 1 catalog-ru-20260623 Windows settings 1 +rdpclip.exe RDP Clipboard Microsoft system neutral low 1 catalog-ru-20260623 RDP clipboard process 1 +conhost.exe Console Host Microsoft system neutral low 1 catalog-ru-20260623 Windows console host 1 +taskmgr.exe Task Manager Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1 +mmc.exe Microsoft Management Console Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console 1 +eventvwr.exe Event Viewer Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1 +services.exe Windows Services Microsoft admin_tool productive low 1 catalog-ru-20260623 Service control manager 1 +compmgmtlauncher.exe Computer Management Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console launcher 1 +regedit.exe Registry Editor Microsoft admin_tool productive medium 1 catalog-ru-20260623 Registry editor 1 +cmd.exe Command Prompt Microsoft admin_tool productive medium 1 catalog-ru-20260623 Command shell 1 +powershell.exe Windows PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1 +pwsh.exe PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1 +windowsterminal.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal 1 +wt.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal launcher 1 +mstsc.exe Remote Desktop Connection Microsoft remote_admin productive low 0 catalog-ru-20260623 RDP client 1 +putty.exe PuTTY PuTTY remote_admin productive low 0 catalog-ru-20260623 SSH client 1 +winscp.exe WinSCP WinSCP remote_admin productive low 0 catalog-ru-20260623 SFTP and SCP client 1 +filezilla.exe FileZilla FileZilla remote_admin productive low 0 catalog-ru-20260623 File transfer client 1 +procexp.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1 +procexp64.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1 +procmon.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1 +procmon64.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1 +telegram.exe Telegram Telegram communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1 +whatsapp.exe WhatsApp Meta communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1 +teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1 +ms-teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1 +zoom.exe Zoom Zoom communication productive low 0 catalog-ru-20260623 Business communication 1 +skype.exe Skype Microsoft communication neutral medium 0 catalog-ru-20260623 Messenger 1 +discord.exe Discord Discord social non_productive medium 0 catalog-ru-20260623 Social and gaming communication 1 +yandexdisk.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +yandexdisk2.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +onedrive.exe OneDrive Microsoft cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +dropbox.exe Dropbox Dropbox cloud_docs neutral medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +code.exe Visual Studio Code Microsoft developer productive low 0 catalog-ru-20260623 Code editor and admin scripting 1 +devenv.exe Visual Studio Microsoft developer productive low 0 catalog-ru-20260623 Development IDE 1 +git.exe Git Git developer productive low 0 catalog-ru-20260623 Version control CLI 1 +docker desktop.exe Docker Desktop Docker developer productive low 0 catalog-ru-20260623 Container development tool 1 +vlc.exe VLC VideoLAN media non_productive medium 0 catalog-ru-20260623 Media player 1 +wmplayer.exe Windows Media Player Microsoft media non_productive medium 0 catalog-ru-20260623 Media player 1 +spotify.exe Spotify Spotify media non_productive medium 0 catalog-ru-20260623 Music streaming 1 +steam.exe Steam Valve gaming non_productive high 0 catalog-ru-20260623 Gaming platform 1 +gamebar.exe Xbox Game Bar Microsoft gaming non_productive high 0 catalog-ru-20260623 Gaming overlay 1 +securityhealthsystray.exe Windows Security Microsoft security_crypto neutral low 1 catalog-ru-20260623 Windows security UI 1 +avgui.exe AVG Antivirus AVG security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1 +avpui.exe Kaspersky Kaspersky security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1 diff --git a/clickhouse-workforce/catalog/domain_categories.tsv b/clickhouse-workforce/catalog/domain_categories.tsv new file mode 100644 index 0000000..1b855d2 --- /dev/null +++ b/clickhouse-workforce/catalog/domain_categories.tsv @@ -0,0 +1,128 @@ +domain site_name category productivity_class risk_level business_allowed source comment is_active +intranet.local Internal portal internal_service productive low 1 catalog-ru-20260623 Internal work portal 1 +dbo.sevnb.ru SEVNB online banking banking productive low 1 catalog-ru-20260623 Banking workflow seen in RDP title stream 1 +sberbank.ru Sberbank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +online.sberbank.ru Sberbank Online banking productive low 1 catalog-ru-20260623 Russian banking web client 1 +sbi.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking 1 +business-online.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking legacy host 1 +vtb.ru VTB banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +dbo.vtb.ru VTB Business Online banking productive low 1 catalog-ru-20260623 VTB business banking 1 +alfabank.ru Alfa-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +ibank.alfabank.ru Alfa-Bank Business banking productive low 1 catalog-ru-20260623 Alfa business banking 1 +tbank.ru T-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +business.tbank.ru T-Business banking productive low 1 catalog-ru-20260623 T-Bank business banking 1 +tinkoff.ru Tinkoff banking productive low 1 catalog-ru-20260623 T-Bank legacy domain 1 +business.tinkoff.ru Tinkoff Business banking productive low 1 catalog-ru-20260623 T-Bank business legacy domain 1 +tochka.com Tochka Bank banking productive low 1 catalog-ru-20260623 Business banking 1 +bspb.ru Bank Saint Petersburg banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +dbo.bspb.ru Bank Saint Petersburg Business banking productive low 1 catalog-ru-20260623 Business banking 1 +gazprombank.ru Gazprombank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +bankuralsib.ru Uralsib banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +rshb.ru Russian Agricultural Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1 +modulbank.ru Modulbank banking productive low 1 catalog-ru-20260623 Business banking 1 +gosuslugi.ru Gosuslugi government productive low 1 catalog-ru-20260623 Russian government services 1 +esia.gosuslugi.ru ESIA government productive low 1 catalog-ru-20260623 Government authentication 1 +lk.gosuslugi.ru Gosuslugi account government productive low 1 catalog-ru-20260623 Government services account 1 +nalog.gov.ru FNS government productive low 1 catalog-ru-20260623 Russian tax service 1 +service.nalog.ru FNS services government productive low 1 catalog-ru-20260623 Russian tax service portals 1 +egrul.nalog.ru FNS EGRUL government productive low 1 catalog-ru-20260623 Company registry lookup 1 +pb.nalog.ru FNS transparent business government productive low 1 catalog-ru-20260623 Tax counterparty lookup 1 +lkfl2.nalog.ru FNS personal account government productive low 1 catalog-ru-20260623 Tax account 1 +lkul.nalog.ru FNS legal entity account government productive low 1 catalog-ru-20260623 Tax account for legal entities 1 +fssp.gov.ru FSSP government productive low 1 catalog-ru-20260623 Bailiff service lookup 1 +rosreestr.gov.ru Rosreestr government productive low 1 catalog-ru-20260623 Real estate registry 1 +kad.arbitr.ru Arbitration cases government productive low 1 catalog-ru-20260623 Arbitration case database 1 +my.arbitr.ru Arbitration e-filing government productive low 1 catalog-ru-20260623 Arbitration electronic filing 1 +sudrf.ru Courts of Russia government productive low 1 catalog-ru-20260623 Court information 1 +zakupki.gov.ru Unified procurement government productive low 1 catalog-ru-20260623 State procurement 1 +torgi.gov.ru Torgi.gov government productive low 1 catalog-ru-20260623 State auctions and property tenders 1 +mos.ru Mos.ru government productive low 1 catalog-ru-20260623 Moscow city services 1 +cbr.ru Bank of Russia government productive low 1 catalog-ru-20260623 Central bank reference data 1 +rosstat.gov.ru Rosstat government productive low 1 catalog-ru-20260623 Statistics and reporting 1 +sfr.gov.ru Social Fund government productive low 1 catalog-ru-20260623 Social fund services 1 +trudvsem.ru Trudvsem government productive low 1 catalog-ru-20260623 Labor and vacancies portal 1 +kontur.ru Kontur edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business checks 1 +extern.kontur.ru Kontur Extern edo_reporting productive low 1 catalog-ru-20260623 Electronic reporting 1 +diadoc.kontur.ru Diadoc edo_reporting productive low 1 catalog-ru-20260623 Electronic document exchange 1 +focus.kontur.ru Kontur Focus business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1 +normativ.kontur.ru Kontur Normativ business_reference productive low 1 catalog-ru-20260623 Legal and accounting reference 1 +sbis.ru SBIS edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business services 1 +online.sbis.ru SBIS Online edo_reporting productive low 1 catalog-ru-20260623 EDO reporting portal 1 +saby.ru Saby edo_reporting productive low 1 catalog-ru-20260623 SBIS/Saby services 1 +ofd.ru OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1 +ofd.yandex.ru Yandex OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1 +1-ofd.ru 1-OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1 +sberbank-ast.ru Sberbank AST procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1 +roseltorg.ru Roseltorg procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1 +rts-tender.ru RTS Tender procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1 +fabrikant.ru Fabrikant procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1 +spark-interfax.ru SPARK Interfax business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1 +rusprofile.ru Rusprofile business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1 +checko.ru Checko business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1 +zachestnyibiznes.ru Za Chestny Biznes business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1 +list-org.com List-Org business_reference neutral medium 1 catalog-ru-20260623 Counterparty checks with mixed data quality 1 +garant.ru Garant legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1 +consultant.ru ConsultantPlus legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1 +mail.yandex.ru Yandex Mail mail productive low 1 catalog-ru-20260623 Business mail when used with org account 1 +360.yandex.ru Yandex 360 mail productive low 1 catalog-ru-20260623 Business mail and documents 1 +mail.ru Mail.ru mail neutral medium 1 catalog-ru-20260623 Mail portal; classify account policy separately if needed 1 +e.mail.ru Mail.ru webmail mail neutral medium 1 catalog-ru-20260623 Mail.ru webmail 1 +biz.mail.ru Mail.ru business mail productive low 1 catalog-ru-20260623 Business mail 1 +corp.mail.ru Mail.ru corporate mail productive low 1 catalog-ru-20260623 Corporate services 1 +gmail.com Gmail mail neutral medium 1 catalog-ru-20260623 External mail 1 +outlook.office.com Outlook Web mail productive low 1 catalog-ru-20260623 Business mail 1 +yandex.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1 +ya.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1 +google.com Google Search search neutral low 1 catalog-ru-20260623 Search portal 1 +2gis.ru 2GIS maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1 +maps.yandex.ru Yandex Maps maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1 +dadata.ru DaData business_reference productive low 1 catalog-ru-20260623 Address and company reference 1 +disk.yandex.ru Yandex Disk cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1 +docs.yandex.ru Yandex Documents cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1 +cloud.mail.ru Cloud Mail.ru cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +docs.google.com Google Docs cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1 +drive.google.com Google Drive cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1 +office.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1 +microsoft365.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1 +web.telegram.org Telegram Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1 +t.me Telegram links communication neutral medium 1 catalog-ru-20260623 Messenger links 1 +telegram.org Telegram communication neutral medium 1 catalog-ru-20260623 Messenger 1 +web.whatsapp.com WhatsApp Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1 +whatsapp.com WhatsApp communication neutral medium 1 catalog-ru-20260623 Messenger 1 +teams.microsoft.com Microsoft Teams communication productive low 1 catalog-ru-20260623 Business communication 1 +meet.google.com Google Meet communication productive low 1 catalog-ru-20260623 Business communication 1 +zoom.us Zoom communication productive low 1 catalog-ru-20260623 Business communication 1 +github.com GitHub developer productive low 1 catalog-ru-20260623 Development and documentation workflow 1 +gitlab.com GitLab developer productive low 1 catalog-ru-20260623 Development workflow 1 +stackoverflow.com Stack Overflow developer productive low 1 catalog-ru-20260623 Technical reference 1 +learn.microsoft.com Microsoft Learn developer productive low 1 catalog-ru-20260623 Technical documentation 1 +docs.microsoft.com Microsoft Docs developer productive low 1 catalog-ru-20260623 Technical documentation legacy host 1 +microsoft.com Microsoft developer neutral low 1 catalog-ru-20260623 Vendor documentation and downloads 1 +docker.com Docker developer productive low 1 catalog-ru-20260623 Developer tooling documentation 1 +software.qnap.com QNAP Software work_service neutral low 1 catalog-ru-20260623 Vendor software/download portal for admin work 1 +cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1 +www.cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1 +chatgpt.com ChatGPT ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1 +chat.openai.com ChatGPT legacy ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1 +giga.chat GigaChat ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1 +alice.yandex.ru Yandex Alice ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1 +wildberries.ru Wildberries marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1 +ozon.ru Ozon marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1 +market.yandex.ru Yandex Market marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1 +avito.ru Avito marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business use depends on policy 1 +dns-shop.ru DNS Shop marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1 +citilink.ru Citilink marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1 +vk.com VK social non_productive medium 0 catalog-ru-20260623 Social network; whitelist business communities separately if needed 1 +ok.ru Odnoklassniki social non_productive medium 0 catalog-ru-20260623 Social network 1 +dzen.ru Dzen news non_productive medium 0 catalog-ru-20260623 News/feed portal 1 +rbc.ru RBC news neutral medium 1 catalog-ru-20260623 Business news; review by department policy 1 +ria.ru RIA Novosti news neutral medium 1 catalog-ru-20260623 News; review by department policy 1 +lenta.ru Lenta news non_productive medium 0 catalog-ru-20260623 News/feed portal 1 +youtube.com YouTube media non_productive medium 0 catalog-ru-20260623 Video platform; business use requires URL-level exception 1 +rutube.ru Rutube media non_productive medium 0 catalog-ru-20260623 Video platform 1 +vkvideo.ru VK Video media non_productive medium 0 catalog-ru-20260623 Video platform 1 +music.yandex.ru Yandex Music media non_productive medium 0 catalog-ru-20260623 Music streaming 1 +kinopoisk.ru Kinopoisk media non_productive medium 0 catalog-ru-20260623 Entertainment streaming 1 +twitch.tv Twitch media non_productive high 0 catalog-ru-20260623 Entertainment streaming 1 +example.com Example Domain test_reference neutral low 0 catalog-ru-20260623 Test/reference domain 1 +unknown.example Unknown example test_reference neutral low 0 catalog-ru-20260623 Test/reference domain from demo data 1 diff --git a/clickhouse-workforce/catalog/workstation_users.tsv b/clickhouse-workforce/catalog/workstation_users.tsv new file mode 100644 index 0000000..3a35426 --- /dev/null +++ b/clickhouse-workforce/catalog/workstation_users.tsv @@ -0,0 +1,8 @@ +host_name user_login user_domain employee_id employee_name department branch position source is_active +SHARKON2025 user1 sharkon2025 sharkon2025\\user1 user1 tsj tsj RDP user catalog-p5 1 +SHARKON2025 user4 sharkon2025 sharkon2025\\user4 user4 tsj tsj RDP user catalog-p5 1 +SHARKON2025 user5 sharkon2025 sharkon2025\\user5 user5 tsj tsj RDP user catalog-p5 1 +SHARKON2025 Администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator catalog-p5 1 +SHARKON2025 администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator legacy lowercase alias catalog-p5 1 +ws-001 ivanov corp E001 Иванов И.И. Бухгалтерия Филиал 1 Бухгалтер demo 1 +ws-002 petrova corp E002 Петрова П.П. Операционный отдел Филиал 1 Оператор demo 1 diff --git a/clickhouse-workforce/clickhouse/init/00_database.sql b/clickhouse-workforce/clickhouse/init/00_database.sql new file mode 100644 index 0000000..e8fc5e7 --- /dev/null +++ b/clickhouse-workforce/clickhouse/init/00_database.sql @@ -0,0 +1 @@ +CREATE DATABASE IF NOT EXISTS aw_workforce; diff --git a/clickhouse-workforce/clickhouse/init/01_raw_tables.sql b/clickhouse-workforce/clickhouse/init/01_raw_tables.sql new file mode 100644 index 0000000..772e944 --- /dev/null +++ b/clickhouse-workforce/clickhouse/init/01_raw_tables.sql @@ -0,0 +1,32 @@ +CREATE TABLE IF NOT EXISTS aw_workforce.aw_window_events +( + event_time DateTime, + host_name String, + user_login String, + process_name String, + window_title String, + duration_sec UInt32, + source_bucket LowCardinality(String), + source_event_id String, + ingested_at DateTime DEFAULT now() +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_time) +ORDER BY (event_time, host_name, user_login, process_name, source_event_id); + +CREATE TABLE IF NOT EXISTS aw_workforce.aw_browser_events +( + event_time DateTime, + host_name String, + user_login String, + browser_name String, + url String, + title String, + duration_sec UInt32, + source_bucket LowCardinality(String), + source_event_id String, + ingested_at DateTime DEFAULT now() +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_time) +ORDER BY (event_time, host_name, user_login, browser_name, source_event_id); diff --git a/clickhouse-workforce/clickhouse/init/02_dimensions_dictionaries.sql b/clickhouse-workforce/clickhouse/init/02_dimensions_dictionaries.sql new file mode 100644 index 0000000..736408b --- /dev/null +++ b/clickhouse-workforce/clickhouse/init/02_dimensions_dictionaries.sql @@ -0,0 +1,145 @@ +CREATE TABLE IF NOT EXISTS aw_workforce.dim_workstation_user +( + host_name String, + user_login String, + user_domain String, + + employee_id String, + employee_name String, + department String, + branch String, + position String, + + source LowCardinality(String), + is_active UInt8 DEFAULT 1, + updated_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(updated_at) +ORDER BY (host_name, user_login); + +CREATE TABLE IF NOT EXISTS aw_workforce.dim_application_category +( + process_name String, + + application_name String, + vendor String, + category LowCardinality(String), + productivity_class LowCardinality(String), + risk_level LowCardinality(String), + + is_system UInt8 DEFAULT 0, + is_active UInt8 DEFAULT 1, + source LowCardinality(String), + comment String, + updated_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(updated_at) +ORDER BY process_name; + +CREATE TABLE IF NOT EXISTS aw_workforce.dim_domain_category +( + domain String, + + site_name String, + category LowCardinality(String), + productivity_class LowCardinality(String), + risk_level LowCardinality(String), + business_allowed UInt8 DEFAULT 0, + + source LowCardinality(String), + comment String, + is_active UInt8 DEFAULT 1, + updated_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(updated_at) +ORDER BY domain; + +CREATE TABLE IF NOT EXISTS aw_workforce.dim_url_rule +( + rule_id String, + domain String, + path_pattern String, + + category LowCardinality(String), + productivity_class LowCardinality(String), + risk_level LowCardinality(String), + + priority UInt16 DEFAULT 100, + is_active UInt8 DEFAULT 1, + comment String, + updated_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(updated_at) +ORDER BY (domain, priority, rule_id); + +DROP DICTIONARY IF EXISTS aw_workforce.dict_workstation_user; + +CREATE DICTIONARY aw_workforce.dict_workstation_user +( + host_name String, + user_login String, + user_domain String, + + employee_id String, + employee_name String, + department String, + branch String, + position String, + is_active UInt8 +) +PRIMARY KEY host_name, user_login +SOURCE(CLICKHOUSE( + USER 'aw_workforce_dict' + PASSWORD '' + DB 'aw_workforce' + TABLE 'dim_workstation_user' +)) +LAYOUT(COMPLEX_KEY_HASHED()) +LIFETIME(MIN 3600 MAX 86400); + +DROP DICTIONARY IF EXISTS aw_workforce.dict_application_category; + +CREATE DICTIONARY aw_workforce.dict_application_category +( + process_name String, + + application_name String, + vendor String, + category String, + productivity_class String, + risk_level String, + is_system UInt8, + is_active UInt8 +) +PRIMARY KEY process_name +SOURCE(CLICKHOUSE( + USER 'aw_workforce_dict' + PASSWORD '' + DB 'aw_workforce' + TABLE 'dim_application_category' +)) +LAYOUT(HASHED()) +LIFETIME(MIN 3600 MAX 86400); + +DROP DICTIONARY IF EXISTS aw_workforce.dict_domain_category; + +CREATE DICTIONARY aw_workforce.dict_domain_category +( + domain String, + + site_name String, + category String, + productivity_class String, + risk_level String, + business_allowed UInt8, + is_active UInt8 +) +PRIMARY KEY domain +SOURCE(CLICKHOUSE( + USER 'aw_workforce_dict' + PASSWORD '' + DB 'aw_workforce' + TABLE 'dim_domain_category' +)) +LAYOUT(HASHED()) +LIFETIME(MIN 3600 MAX 86400); diff --git a/clickhouse-workforce/clickhouse/init/03_materialized_views.sql b/clickhouse-workforce/clickhouse/init/03_materialized_views.sql new file mode 100644 index 0000000..4ce0c51 --- /dev/null +++ b/clickhouse-workforce/clickhouse/init/03_materialized_views.sql @@ -0,0 +1,158 @@ +CREATE TABLE IF NOT EXISTS aw_workforce.agg_workforce_productivity_hourly +( + bucket_start DateTime, + event_date Date, + + branch LowCardinality(String), + department LowCardinality(String), + + activity_type LowCardinality(String), + category LowCardinality(String), + productivity_class LowCardinality(String), + + duration_sec UInt64, + event_count UInt64, + unknown_subject_events UInt64, + unknown_category_events UInt64 +) +ENGINE = SummingMergeTree(( + duration_sec, + event_count, + unknown_subject_events, + unknown_category_events +)) +PARTITION BY toYYYYMM(event_date) +ORDER BY ( + event_date, + bucket_start, + branch, + department, + activity_type, + productivity_class, + category +); + +DROP VIEW IF EXISTS aw_workforce.mv_desktop_productivity_hourly; + +CREATE MATERIALIZED VIEW aw_workforce.mv_desktop_productivity_hourly +TO aw_workforce.agg_workforce_productivity_hourly +AS +SELECT + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'), + 'unknown' + ) AS branch, + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'), + 'unknown' + ) AS department, + + 'desktop' AS activity_type, + if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, 'unknown'), + 'unknown' + ) AS category, + if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, 'unknown'), + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '', + 0, + 1 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') NOT IN ('', 'unknown'), + 0, + 1 + ))) AS unknown_category_events +FROM aw_workforce.aw_window_events +GROUP BY + bucket_start, + event_date, + branch, + department, + activity_type, + category, + productivity_class; + +DROP VIEW IF EXISTS aw_workforce.mv_browser_productivity_hourly; + +CREATE MATERIALIZED VIEW aw_workforce.mv_browser_productivity_hourly +TO aw_workforce.agg_workforce_productivity_hourly +AS +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'), + 'unknown' + ) AS branch, + if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '', + dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'), + 'unknown' + ) AS department, + + 'browser' AS activity_type, + if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, 'unknown'), + 'unknown' + ) AS category, + if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, '') != '', + dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, 'unknown'), + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '', + 0, + 1 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1 + AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') NOT IN ('', 'unknown'), + 0, + 1 + ))) AS unknown_category_events +FROM aw_workforce.aw_browser_events +WHERE domain_name != '' +GROUP BY + bucket_start, + event_date, + branch, + department, + activity_type, + category, + productivity_class; diff --git a/clickhouse-workforce/clickhouse/init/04_quality_views.sql b/clickhouse-workforce/clickhouse/init/04_quality_views.sql new file mode 100644 index 0000000..b9bc9af --- /dev/null +++ b/clickhouse-workforce/clickhouse/init/04_quality_views.sql @@ -0,0 +1,105 @@ +CREATE OR REPLACE VIEW aw_workforce.v_workforce_productivity_daily AS +SELECT + event_date, + branch, + department, + activity_type, + category, + productivity_class, + sum(duration_sec) AS duration_sec, + sum(event_count) AS event_count, + sum(unknown_subject_events) AS unknown_subject_events, + sum(unknown_category_events) AS unknown_category_events +FROM aw_workforce.agg_workforce_productivity_hourly +GROUP BY + event_date, + branch, + department, + activity_type, + category, + productivity_class; + +CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_subjects AS +SELECT + host_name, + user_login, + count() AS events, + sum(duration_sec) AS duration_sec +FROM aw_workforce.aw_window_events +WHERE dictGetUInt8OrDefault( + 'aw_workforce.dict_workstation_user', + 'is_active', + (host_name, user_login), + 0 + ) != 1 + OR dictGetStringOrDefault( + 'aw_workforce.dict_workstation_user', + 'employee_name', + (host_name, user_login), + '' + ) = '' +GROUP BY + host_name, + user_login +ORDER BY duration_sec DESC; + +CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_processes AS +SELECT + process_name, + count() AS events, + sum(duration_sec) AS duration_sec +FROM aw_workforce.aw_window_events +WHERE dictGetUInt8OrDefault( + 'aw_workforce.dict_application_category', + 'is_active', + process_name, + 0 + ) != 1 + OR dictGetStringOrDefault( + 'aw_workforce.dict_application_category', + 'category', + process_name, + '' + ) IN ('', 'unknown') +GROUP BY process_name +ORDER BY duration_sec DESC; + +CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_domains AS +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + domain_name, + count() AS events, + sum(duration_sec) AS duration_sec +FROM aw_workforce.aw_browser_events +WHERE domain_name != '' + AND ( + dictGetUInt8OrDefault( + 'aw_workforce.dict_domain_category', + 'is_active', + domain_name, + 0 + ) != 1 + OR dictGetStringOrDefault( + 'aw_workforce.dict_domain_category', + 'category', + domain_name, + '' + ) IN ('', 'unknown') + ) +GROUP BY domain_name +ORDER BY duration_sec DESC; + +CREATE OR REPLACE VIEW aw_workforce.v_workforce_unknown_quality_daily AS +SELECT + event_date, + sum(event_count) AS events, + sum(unknown_subject_events) AS unknown_subject_events, + round(unknown_subject_events / nullIf(events, 0), 4) AS unknown_subject_ratio, + sum(unknown_category_events) AS unknown_category_events, + round(unknown_category_events / nullIf(events, 0), 4) AS unknown_category_ratio +FROM aw_workforce.agg_workforce_productivity_hourly +GROUP BY event_date +ORDER BY event_date DESC; diff --git a/clickhouse-workforce/clickhouse/users.d/99-aw-workforce-local.xml b/clickhouse-workforce/clickhouse/users.d/99-aw-workforce-local.xml new file mode 100644 index 0000000..51b4362 --- /dev/null +++ b/clickhouse-workforce/clickhouse/users.d/99-aw-workforce-local.xml @@ -0,0 +1,18 @@ + + + + + ::/0 + + + + + + 127.0.0.1 + ::1 + + readonly + default + + + diff --git a/clickhouse-workforce/docker-compose.yml b/clickhouse-workforce/docker-compose.yml new file mode 100644 index 0000000..bf14130 --- /dev/null +++ b/clickhouse-workforce/docker-compose.yml @@ -0,0 +1,17 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server:24.8 + container_name: aw-rus-workforce-clickhouse + restart: unless-stopped + environment: + CLICKHOUSE_DB: ${CLICKHOUSE_DB:-aw_workforce} + ports: + - "${CLICKHOUSE_HTTP_BIND:-127.0.0.1}:${CLICKHOUSE_PORT:-8124}:8123" + - "${CLICKHOUSE_NATIVE_BIND:-127.0.0.1}:${CLICKHOUSE_NATIVE_PORT:-9001}:9000" + volumes: + - clickhouse_workforce_data:/var/lib/clickhouse + - ./clickhouse/init:/docker-entrypoint-initdb.d:ro + - ./clickhouse/users.d/99-aw-workforce-local.xml:/etc/clickhouse-server/users.d/99-aw-workforce-local.xml:ro + +volumes: + clickhouse_workforce_data: diff --git a/clickhouse-workforce/ops/apply_catalogs.sh b/clickhouse-workforce/ops/apply_catalogs.sh new file mode 100644 index 0000000..8266a32 --- /dev/null +++ b/clickhouse-workforce/ops/apply_catalogs.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +CATALOG_DIR="${CATALOG_DIR:-$ROOT_DIR/catalog}" + +CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}" +CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-}" +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}" +CLICKHOUSE_READY_TIMEOUT_SEC="${CLICKHOUSE_READY_TIMEOUT_SEC:-60}" +REBUILD_AGGREGATES="${REBUILD_AGGREGATES:-0}" + +client_auth_args=() +if [[ -n "$CLICKHOUSE_USER" ]]; then + client_auth_args+=(--user "$CLICKHOUSE_USER") +fi +if [[ -n "$CLICKHOUSE_PASSWORD" ]]; then + client_auth_args+=(--password "$CLICKHOUSE_PASSWORD") +fi + +run_query() { + local query="$1" + + if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \ + --database "$CLICKHOUSE_DATABASE" --query "$query" + return + fi + + if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then + "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" \ + --database "$CLICKHOUSE_DATABASE" --query "$query" + return + fi + + printf 'No running ClickHouse container "%s" and no %s in PATH\n' \ + "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2 + return 127 +} + +run_query_file() { + local query_file="$1" + + if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \ + --multiquery <"$query_file" + return + fi + + if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then + "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --multiquery <"$query_file" + return + fi + + printf 'No running ClickHouse container "%s" and no %s in PATH\n' \ + "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2 + return 127 +} + +run_insert_file() { + local query="$1" + local data_file="$2" + + if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" \ + --database "$CLICKHOUSE_DATABASE" --query "$query" <"$data_file" + return + fi + + if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then + "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" \ + --database "$CLICKHOUSE_DATABASE" --query "$query" <"$data_file" + return + fi + + printf 'No running ClickHouse container "%s" and no %s in PATH\n' \ + "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2 + return 127 +} + +wait_for_clickhouse() { + local deadline + deadline=$((SECONDS + CLICKHOUSE_READY_TIMEOUT_SEC)) + while (( SECONDS < deadline )); do + if run_query "SELECT 1" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + printf 'ClickHouse is not ready after %s seconds\n' "$CLICKHOUSE_READY_TIMEOUT_SEC" >&2 + return 1 +} + +require_file() { + local path="$1" + if [[ ! -f "$path" ]]; then + printf 'Required catalog file is missing: %s\n' "$path" >&2 + return 1 + fi +} + +wait_for_clickhouse + +require_file "$CATALOG_DIR/workstation_users.tsv" +require_file "$CATALOG_DIR/application_categories.tsv" +require_file "$CATALOG_DIR/domain_categories.tsv" + +printf '[catalog] truncate dimension tables\n' +run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_workstation_user" +run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_application_category" +run_query "TRUNCATE TABLE $CLICKHOUSE_DATABASE.dim_domain_category" + +printf '[catalog] load workstation users\n' +run_insert_file " +INSERT INTO $CLICKHOUSE_DATABASE.dim_workstation_user + (host_name, user_login, user_domain, employee_id, employee_name, department, branch, position, source, is_active) +FORMAT TabSeparatedWithNames +" "$CATALOG_DIR/workstation_users.tsv" + +printf '[catalog] load application categories\n' +run_insert_file " +INSERT INTO $CLICKHOUSE_DATABASE.dim_application_category + (process_name, application_name, vendor, category, productivity_class, risk_level, is_system, source, comment, is_active) +FORMAT TabSeparatedWithNames +" "$CATALOG_DIR/application_categories.tsv" + +printf '[catalog] load domain categories\n' +run_insert_file " +INSERT INTO $CLICKHOUSE_DATABASE.dim_domain_category + (domain, site_name, category, productivity_class, risk_level, business_allowed, source, comment, is_active) +FORMAT TabSeparatedWithNames +" "$CATALOG_DIR/domain_categories.tsv" + +printf '[catalog] reload dictionaries\n' +run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_workstation_user" +run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_application_category" +run_query "SYSTEM RELOAD DICTIONARY $CLICKHOUSE_DATABASE.dict_domain_category" + +if [[ "$REBUILD_AGGREGATES" == "1" ]]; then + printf '[catalog] rebuild aggregates\n' + run_query_file "$ROOT_DIR/admin/rebuild_aggregates.sql" +fi + +printf '[catalog] dictionary status\n' +run_query " +SELECT name, status, last_exception +FROM system.dictionaries +WHERE database = '$CLICKHOUSE_DATABASE' + AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category') +ORDER BY name +FORMAT PrettyCompact +" + +printf '[catalog] raw unknown summary\n' +run_query " +SELECT 'subjects' AS area, count() AS rows +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_subjects +UNION ALL +SELECT 'processes' AS area, count() AS rows +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_processes +UNION ALL +SELECT 'domains' AS area, count() AS rows +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_domains +FORMAT PrettyCompact +" diff --git a/clickhouse-workforce/ops/aw-workforce-ingest.env.example b/clickhouse-workforce/ops/aw-workforce-ingest.env.example new file mode 100644 index 0000000..d9995a6 --- /dev/null +++ b/clickhouse-workforce/ops/aw-workforce-ingest.env.example @@ -0,0 +1,14 @@ +AW_WORKFORCE_AW_URL=http://10.10.10.13:5600/api/0 +AW_WORKFORCE_CLICKHOUSE_URL=http://10.10.10.2:8123 +AW_WORKFORCE_CLICKHOUSE_DATABASE=aw_workforce +AW_WORKFORCE_CLICKHOUSE_USER= +AW_WORKFORCE_CLICKHOUSE_PASSWORD= +AW_WORKFORCE_HOST=SHARKON2025 +AW_WORKFORCE_STATE_PATH=/var/lib/aw-workforce-ingest/state.json +AW_WORKFORCE_OVERLAP_SECONDS=300 +AW_WORKFORCE_RETRY_ATTEMPTS=3 +AW_WORKFORCE_RETRY_BACKOFF_MS=1000 +AW_WORKFORCE_FAIL_ON_EMPTY=false +AW_WORKFORCE_JSON=true +no_proxy=localhost,127.0.0.1,10.10.10.13,10.10.10.2,10.10.10.0/24 +NO_PROXY=localhost,127.0.0.1,10.10.10.13,10.10.10.2,10.10.10.0/24 diff --git a/clickhouse-workforce/ops/aw-workforce-ingest.service b/clickhouse-workforce/ops/aw-workforce-ingest.service new file mode 100644 index 0000000..4f4adc3 --- /dev/null +++ b/clickhouse-workforce/ops/aw-workforce-ingest.service @@ -0,0 +1,17 @@ +[Unit] +Description=AW-rus workforce ClickHouse ingest +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/opt/activitywatch/clickhouse-workforce +EnvironmentFile=-/etc/activitywatch/aw-workforce-ingest.env +ExecStart=/usr/local/bin/aw-workforce-ingest +TimeoutStartSec=20min +User=root +Group=root +Nice=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=aw-workforce-ingest diff --git a/clickhouse-workforce/ops/aw-workforce-ingest.timer b/clickhouse-workforce/ops/aw-workforce-ingest.timer new file mode 100644 index 0000000..1baafcb --- /dev/null +++ b/clickhouse-workforce/ops/aw-workforce-ingest.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Run AW-rus workforce ClickHouse ingest every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +AccuracySec=30s +RandomizedDelaySec=30s +Persistent=true +Unit=aw-workforce-ingest.service + +[Install] +WantedBy=timers.target diff --git a/clickhouse-workforce/ops/bootstrap_runtime.sh b/clickhouse-workforce/ops/bootstrap_runtime.sh new file mode 100644 index 0000000..48a5a99 --- /dev/null +++ b/clickhouse-workforce/ops/bootstrap_runtime.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="${AW_WORKFORCE_ROOT:-/opt/activitywatch/clickhouse-workforce}" +ENV_DIR="${AW_WORKFORCE_ENV_DIR:-/etc/activitywatch}" +STATE_DIR="${AW_WORKFORCE_STATE_DIR:-/var/lib/aw-workforce-ingest}" +SYSTEMD_DIR="${AW_WORKFORCE_SYSTEMD_DIR:-/etc/systemd/system}" + +install -d -m 0755 "$ROOT" "$ENV_DIR" "$STATE_DIR" "$SYSTEMD_DIR" + +if [[ ! -f "$ENV_DIR/aw-workforce-ingest.env" ]]; then + install -m 0640 "$SCRIPT_DIR/aw-workforce-ingest.env.example" \ + "$ENV_DIR/aw-workforce-ingest.env" +fi + +install -m 0644 "$SCRIPT_DIR/aw-workforce-ingest.service" \ + "$SYSTEMD_DIR/aw-workforce-ingest.service" +install -m 0644 "$SCRIPT_DIR/aw-workforce-ingest.timer" \ + "$SYSTEMD_DIR/aw-workforce-ingest.timer" + +if command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload +fi + +cat < /usr/local/bin/aw-workforce-ingest + edit $ENV_DIR/aw-workforce-ingest.env + systemctl enable --now aw-workforce-ingest.timer + systemctl start aw-workforce-ingest.service +EOF diff --git a/clickhouse-workforce/ops/report_unknowns.sh b/clickhouse-workforce/ops/report_unknowns.sh new file mode 100644 index 0000000..b9db120 --- /dev/null +++ b/clickhouse-workforce/ops/report_unknowns.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}" +CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}" +CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}" +LIMIT="${LIMIT:-50}" + +run_query() { + local query="$1" + + if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client \ + --database "$CLICKHOUSE_DATABASE" --query "$query" + return + fi + + "$CLICKHOUSE_CLIENT_BIN" --database "$CLICKHOUSE_DATABASE" --query "$query" +} + +cd "$ROOT_DIR" + +printf '\n[unknown subjects]\n' +run_query " +SELECT * +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_subjects +LIMIT $LIMIT +FORMAT PrettyCompact +" + +printf '\n[unknown processes]\n' +run_query " +SELECT * +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_processes +LIMIT $LIMIT +FORMAT PrettyCompact +" + +printf '\n[unknown domains]\n' +run_query " +SELECT * +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_domains +LIMIT $LIMIT +FORMAT PrettyCompact +" diff --git a/clickhouse-workforce/ops/run_smoke.sh b/clickhouse-workforce/ops/run_smoke.sh new file mode 100644 index 0000000..878a1f8 --- /dev/null +++ b/clickhouse-workforce/ops/run_smoke.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +CLICKHOUSE_CONTAINER="${CLICKHOUSE_CONTAINER:-aw-rus-workforce-clickhouse}" +CLICKHOUSE_DATABASE="${CLICKHOUSE_DATABASE:-aw_workforce}" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-}" +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +CLICKHOUSE_CLIENT_BIN="${CLICKHOUSE_CLIENT_BIN:-clickhouse-client}" +LOAD_DEMO_SEED="${LOAD_DEMO_SEED:-1}" +CLICKHOUSE_READY_TIMEOUT_SEC="${CLICKHOUSE_READY_TIMEOUT_SEC:-60}" + +client_auth_args=() +if [[ -n "$CLICKHOUSE_USER" ]]; then + client_auth_args+=(--user "$CLICKHOUSE_USER") +fi +if [[ -n "$CLICKHOUSE_PASSWORD" ]]; then + client_auth_args+=(--password "$CLICKHOUSE_PASSWORD") +fi + +run_client() { + local query_file="${1:-}" + local query="${2:-}" + + if docker ps --format '{{.Names}}' | grep -Fxq "$CLICKHOUSE_CONTAINER"; then + if [[ -n "$query_file" ]]; then + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" --multiquery <"$query_file" + else + docker exec -i "$CLICKHOUSE_CONTAINER" clickhouse-client "${client_auth_args[@]}" --database "$CLICKHOUSE_DATABASE" --query "$query" + fi + return + fi + + if command -v "$CLICKHOUSE_CLIENT_BIN" >/dev/null 2>&1; then + if [[ -n "$query_file" ]]; then + "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --multiquery <"$query_file" + else + "$CLICKHOUSE_CLIENT_BIN" "${client_auth_args[@]}" --database "$CLICKHOUSE_DATABASE" --query "$query" + fi + return + fi + + printf 'No running ClickHouse container "%s" and no %s in PATH\n' \ + "$CLICKHOUSE_CONTAINER" "$CLICKHOUSE_CLIENT_BIN" >&2 + return 127 +} + +wait_for_clickhouse() { + local deadline + deadline=$((SECONDS + CLICKHOUSE_READY_TIMEOUT_SEC)) + while (( SECONDS < deadline )); do + if run_client "" "SELECT 1" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + printf 'ClickHouse is not ready after %s seconds\n' "$CLICKHOUSE_READY_TIMEOUT_SEC" >&2 + return 1 +} + +apply_sql_dir() { + local sql_file + for sql_file in "$ROOT_DIR"/clickhouse/init/*.sql; do + printf '[sql] %s\n' "${sql_file#$ROOT_DIR/}" + run_client "$sql_file" "" + done +} + +assert_scalar_nonzero() { + local name="$1" + local query="$2" + local value + value="$(run_client "" "$query" | tr -d '[:space:]')" + if [[ ! "$value" =~ ^[0-9]+$ ]] || (( value < 1 )); then + printf '[FAIL] %s: expected positive integer, got "%s"\n' "$name" "$value" >&2 + return 1 + fi + printf '[OK] %s: %s\n' "$name" "$value" +} + +assert_no_dictionary_errors() { + local errors + errors="$(run_client "" " +SELECT count() +FROM system.dictionaries +WHERE database = '$CLICKHOUSE_DATABASE' + AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category') + AND (status != 'LOADED' OR last_exception != '') +")" + errors="$(printf '%s' "$errors" | tr -d '[:space:]')" + if [[ "$errors" != "0" ]]; then + printf '[FAIL] dictionaries have load errors\n' >&2 + run_client "" " +SELECT database, name, status, last_exception +FROM system.dictionaries +WHERE database = '$CLICKHOUSE_DATABASE' + AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category') +FORMAT Vertical +" + return 1 + fi + printf '[OK] dictionaries loaded\n' +} + +wait_for_clickhouse +apply_sql_dir + +if [[ "$LOAD_DEMO_SEED" == "1" ]]; then + printf '[sql] sample/seed_demo.sql\n' + run_client "$ROOT_DIR/sample/seed_demo.sql" "" +fi + +assert_no_dictionary_errors +assert_scalar_nonzero "dictionary count" " +SELECT count() +FROM system.dictionaries +WHERE database = '$CLICKHOUSE_DATABASE' + AND name IN ('dict_workstation_user', 'dict_application_category', 'dict_domain_category') +" +assert_scalar_nonzero "hourly aggregate rows" " +SELECT count() +FROM $CLICKHOUSE_DATABASE.agg_workforce_productivity_hourly +" +assert_scalar_nonzero "daily productivity rows" " +SELECT count() +FROM $CLICKHOUSE_DATABASE.v_workforce_productivity_daily +" +assert_scalar_nonzero "unknown quality rows" " +SELECT count() +FROM $CLICKHOUSE_DATABASE.v_workforce_unknown_quality_daily +" + +printf '[OK] ClickHouse workforce smoke completed\n' diff --git a/clickhouse-workforce/sample/seed_demo.sql b/clickhouse-workforce/sample/seed_demo.sql new file mode 100644 index 0000000..a0c5d64 --- /dev/null +++ b/clickhouse-workforce/sample/seed_demo.sql @@ -0,0 +1,37 @@ +INSERT INTO aw_workforce.dim_workstation_user + (host_name, user_login, user_domain, employee_id, employee_name, department, branch, position, source) +VALUES + ('ws-001', 'ivanov', 'corp', 'E001', 'Иванов И.И.', 'Бухгалтерия', 'Филиал 1', 'Бухгалтер', 'demo'), + ('ws-002', 'petrova', 'corp', 'E002', 'Петрова П.П.', 'Операционный отдел', 'Филиал 1', 'Оператор', 'demo'); + +INSERT INTO aw_workforce.dim_application_category + (process_name, application_name, vendor, category, productivity_class, risk_level, is_system, source, comment) +VALUES + ('1cv8.exe', '1C:Enterprise', '1C', '1c', 'productive', 'low', 0, 'demo', 'core business app'), + ('chrome.exe', 'Google Chrome', 'Google', 'browser', 'neutral', 'low', 0, 'demo', 'domain classified separately'), + ('soffice.bin', 'LibreOffice', 'The Document Foundation', 'office', 'productive', 'low', 0, 'demo', 'office suite'); + +INSERT INTO aw_workforce.dim_domain_category + (domain, site_name, category, productivity_class, risk_level, business_allowed, source, comment) +VALUES + ('intranet.local', 'Internal portal', 'internal_service', 'productive', 'low', 1, 'demo', 'internal work portal'), + ('github.com', 'GitHub', 'developer', 'productive', 'low', 1, 'demo', 'developer workflow'), + ('youtube.com', 'YouTube', 'media', 'neutral', 'medium', 0, 'demo', 'context-dependent media'); + +SYSTEM RELOAD DICTIONARY aw_workforce.dict_workstation_user; +SYSTEM RELOAD DICTIONARY aw_workforce.dict_application_category; +SYSTEM RELOAD DICTIONARY aw_workforce.dict_domain_category; + +INSERT INTO aw_workforce.aw_window_events + (event_time, host_name, user_login, process_name, window_title, duration_sec, source_bucket, source_event_id) +VALUES + (now() - INTERVAL 20 MINUTE, 'ws-001', 'ivanov', '1cv8.exe', '1C - документы', 900, 'demo-window', 'w-001'), + (now() - INTERVAL 15 MINUTE, 'ws-002', 'petrova', 'soffice.bin', 'Отчет', 600, 'demo-window', 'w-002'), + (now() - INTERVAL 10 MINUTE, 'ws-unknown', 'unknown', 'unknown.exe', 'Unknown tool', 120, 'demo-window', 'w-003'); + +INSERT INTO aw_workforce.aw_browser_events + (event_time, host_name, user_login, browser_name, url, title, duration_sec, source_bucket, source_event_id) +VALUES + (now() - INTERVAL 9 MINUTE, 'ws-001', 'ivanov', 'chrome.exe', 'https://intranet.local/tasks', 'Tasks', 300, 'demo-browser', 'b-001'), + (now() - INTERVAL 8 MINUTE, 'ws-002', 'petrova', 'chrome.exe', 'https://github.com/igor04091968/AWatch-rus', 'AWatch-rus', 240, 'demo-browser', 'b-002'), + (now() - INTERVAL 7 MINUTE, 'ws-002', 'petrova', 'chrome.exe', 'https://unknown.example/path', 'Unknown', 90, 'demo-browser', 'b-003'); diff --git a/clickhouse-workforce/sample/seed_sharkon2025_p3.sql b/clickhouse-workforce/sample/seed_sharkon2025_p3.sql new file mode 100644 index 0000000..9358854 --- /dev/null +++ b/clickhouse-workforce/sample/seed_sharkon2025_p3.sql @@ -0,0 +1,6 @@ +INSERT INTO aw_workforce.dim_workstation_user + (host_name, user_login, user_domain, employee_id, employee_name, department, branch, position, source) +VALUES + ('SHARKON2025', 'user1', 'sharkon2025', 'sharkon2025\\user1', 'user1', 'tsj', 'tsj', 'RDP user', 'manual-p3'); + +SYSTEM RELOAD DICTIONARY aw_workforce.dict_workstation_user; diff --git a/docs/DETMIR_SUPPORT_SCRIPTS_RU.md b/docs/DETMIR_SUPPORT_SCRIPTS_RU.md index 901664f..784cdf0 100644 --- a/docs/DETMIR_SUPPORT_SCRIPTS_RU.md +++ b/docs/DETMIR_SUPPORT_SCRIPTS_RU.md @@ -10,13 +10,16 @@ ## Быстрый старт -1. Скопировать шаблон окружения: +1. Создать приватный файл окружения вне репозитория: ```bash -cp scripts/detmir-support.env.example scripts/detmir-support.env +mkdir -p "$HOME/.config/awatch-rus" +cp scripts/detmir-support.env.example "$HOME/.config/awatch-rus/detmir-support.env" +chmod 600 "$HOME/.config/awatch-rus/detmir-support.env" ``` 2. Внести фактические IP/имена хостов, сервисы, пути бэкапов, VM IDs и SSH-данные. + Реальные пароли и приватные ключи не должны храниться в репозитории. 3. Запустить нужный режим: @@ -32,6 +35,38 @@ cp scripts/detmir-support.env.example scripts/detmir-support.env ./scripts/detmir-support-run.sh --scope daily --output-dir /var/log/detmir-support ``` +## Файл окружения + +Скрипт загружает параметры в таком порядке: + +1. файл из переменной `DETMIR_SUPPORT_ENV_FILE`, если она задана; +2. `scripts/detmir-support.env`, если он существует; +3. `$HOME/.config/awatch-rus/detmir-support.env`, если локального файла в + репозитории нет. + +Рекомендуемый промышленный вариант — хранить секреты в +`$HOME/.config/awatch-rus/detmir-support.env`, а в репозитории держать только +`scripts/detmir-support.env.example`. + +Минимальные параметры для текущего контура DetMir: + +```bash +DETMIR_SUPPORT_PVE_HOST=10.10.10.2 +DETMIR_SUPPORT_AW_HOST=10.10.10.13 +DETMIR_SUPPORT_WEB_HOST=10.10.10.2 +DETMIR_SUPPORT_WEB_TLS_HOST=10.10.10.2 +DETMIR_SUPPORT_WINDOWS_HOST=192.168.100.18 +DETMIR_SUPPORT_SURICATA_HOST=10.10.10.2 + +DETMIR_SUPPORT_SSH_USER=igor +DETMIR_SUPPORT_AW_SSH_USER=igor +DETMIR_SUPPORT_PVE_BACKUP_DIRS=/var/lib/pve/local-btrfs/dump +``` + +Парольные переменные вида `DETMIR_SUPPORT_AW_SSH_PASSWORD` допускаются только в +локальном приватном env-файле. В документации, Git и отчетах пароли не +фиксируются. + ## Что делает скрипт Собираются проверки по режиму: @@ -40,6 +75,20 @@ cp scripts/detmir-support.env.example scripts/detmir-support.env - **weekly**: всё из `daily` + расширенные проверки логов и журналов. - **monthly**: всё из `weekly` + проверка апдейтов и базовая фиксация документов/DR-процесса. +Актуальные особенности текущего контура: + +- операторский gateway находится на `10.10.10.2`, а не на историческом + `10.10.10.11`; +- web health endpoint: `https://10.10.10.2/healthz`, ожидаемый ответ `200`; +- защищенный корень gateway `https://10.10.10.2/` штатно отвечает `401`; +- AW API health проверяется через + `http://10.10.10.13:5600/api/0/settings/`; +- актуальный каталог Proxmox backup storage: + `/var/lib/pve/local-btrfs/dump`; +- если `suricata.service` не активен, проверка процесса Suricata + пропускается как следствие состояния сервиса, а не как отдельный сбой + процесса. + ## Результаты В каталоге отчётов создаются файлы: @@ -53,3 +102,25 @@ cp scripts/detmir-support.env.example scripts/detmir-support.env - `0` — без ошибок и предупреждений - `1` — есть WARN - `2` — есть FAIL + +`SKIP` означает, что проверка не могла быть выполнена в текущих условиях: +например, нет SSH-аутентификации, сервис намеренно выключен или отсутствует +проверяемый компонент. `SKIP` не должен маскировать причину: в строке отчета +должна быть указана диагностическая причина, например +`Permission denied (publickey,password)` или +`Skipped because suricata.service state=inactive`. + +## Текущие ожидаемые предупреждения + +На момент актуализации документации для контура DetMir допустимо увидеть: + +- `aw-server-rust` на `10.10.10.13` в состоянии `inactive`, если фактический + production service — `activitywatch-server`; +- предупреждение по резервным копиям, если последний файл в + `/var/lib/pve/local-btrfs/dump` старше установленного порога; +- `suricata.service state=inactive`, если IDS/IPS на данном узле не введен в + штатную эксплуатацию. + +Эти предупреждения нужно фиксировать в отчете и отдельно согласовывать: +включать сервис, менять список ожидаемых сервисов или отмечать компонент как +неиспользуемый. diff --git a/docs/DETMIR_SUPPORT_TASKS_RU.md b/docs/DETMIR_SUPPORT_TASKS_RU.md index abccf29..710d92c 100644 --- a/docs/DETMIR_SUPPORT_TASKS_RU.md +++ b/docs/DETMIR_SUPPORT_TASKS_RU.md @@ -82,6 +82,10 @@ DetMir: серверы доступны, виртуальные машины з reachability внутренних сетей, ошибки TLS/auth/route push. - Проверять Suricata: запущен ли сервис, нет ли массовых блокировок рабочего трафика, критичных алертов и переполнения логов. +- Если Suricata на конкретном узле не введена в эксплуатацию и + `suricata.service` находится в состоянии `inactive`, фиксировать это как + контролируемое предупреждение или `SKIP` с явной причиной, а не как + самостоятельный сбой процесса. - Проверять, что критичные systemd services, timers и cron jobs находятся в ожидаемом состоянии. - Проверять наличие свежих резервных копий по базовым системам. @@ -228,6 +232,9 @@ DetMir: серверы доступны, виртуальные машины з - административный доступ к Proxmox, pfSense, Linux-серверам, web-серверу, VPN и другим компонентам, входящим в контур; - безопасный способ хранения и передачи учетных данных; +- локальный приватный файл параметров для автоматизированных проверок, например + `$HOME/.config/awatch-rus/detmir-support.env`, с правами доступа только для + владельца и без помещения секретов в Git; - контакт ответственного лица для согласования рискованных изменений; - список критичных сервисов и допустимые окна обслуживания; - сведения о провайдерах, доменах, сертификатах, внешних адресах и каналах diff --git a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md new file mode 100644 index 0000000..3aa4bc8 --- /dev/null +++ b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @@ -0,0 +1,114 @@ +# Эксплуатационная проверка контура + +Дата актуализации: 2026-06-23 + +Документ фиксирует минимальный профессиональный контур проверки после +существенных изменений Rust-кода, сборщиков telemetry, ClickHouse workforce +аналитики, gateway или Grafana dashboards. + +## Rust / cargo gate + +Выполнять из репозитория. `CARGO_TARGET_DIR` должен быть вне рабочей копии, чтобы +не загрязнять diff. + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust +export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target + +cargo fmt --all --check +cargo test --workspace --all-targets --locked +cargo test --workspace --doc --locked +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo audit --deny warnings +``` + +Проверка политики зависимостей: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +cargo deny --manifest-path adk-rust/Cargo.toml check \ + --config deny.toml \ + --hide-inclusion-graph \ + --show-stats +``` + +Windows/RDP collector дополнительно проверяется под целевой ABI: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust +export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target + +cargo check --target x86_64-pc-windows-gnu -p aw-windows-telemetry --locked +cargo clippy --target x86_64-pc-windows-gnu \ + -p aw-windows-telemetry \ + --all-targets \ + --locked \ + -- -D warnings +``` + +Repository-specific gate: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +python3 scripts/public_secret_pattern_check.py + +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust +export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target +cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian +``` + +## Browser smoke + +Browser smoke не заменяет API/CLI проверки. Он подтверждает, что операторский +контур реально открывается в браузере и dashboards рендерят панели. + +Минимальный набор страниц: + +- `http://10.10.10.13:5600/` - ActivityWatch WebUI. +- `http://10.10.10.13:5610/reports/worktime/today` - дневной RDP отчет. +- `http://10.10.10.13:5610/reports/worktime/management` - управленческий RDP + отчет. +- `http://10.10.10.2:8710/manager/brief` - 1C executive brief. +- `http://10.10.10.2:8710/manager/actions` - очередь управленческих действий. +- `http://10.10.10.2:8710/manager/recovery` - recovery brief. +- `http://10.10.10.2:8710/manager/digest/weekly` - weekly digest. +- `https://dm.iri1968.dpdns.org/` - gateway index через Basic Auth. +- `https://dm.iri1968.dpdns.org/d/detmir-rdp-user-activity/detmir3a-rabota-pol-zovatelej-v-rdp?orgId=1&from=now-7d&to=now&timezone=browser&var-host=SHARKON2025&refresh=5m` + - RDP user activity dashboard. +- `https://dm.iri1968.dpdns.org/d/detmir-aw-main/detmir3a-activitywatch-overview?orgId=1&from=now-24h&to=now&timezone=browser&refresh=5m` + - main ActivityWatch dashboard. + +Правила: + +- Basic Auth читать с `pve-detmir:/etc/detmir/proxmox-web-gateway.credentials`. + Пароль нельзя печатать в логах, документации, commit messages или final report. +- Скриншоты для диагностики хранить в `/tmp/aw-browser-smoke-*`; не коммитить. +- Ошибки `404` по Grafana endpoint `/api/dashboards/uid/*/public-dashboards` + не считаются отказом панели: это metadata public dashboard, не datasource. + Пустые panels, 5xx, ошибки datasource или отсутствие данных в body/screenshot + считаются регрессией. + +## Production smoke + +После deploy или изменения telemetry/workforce выполнить: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +NO_PROXY=localhost,127.0.0.1,10.10.10.13,10.10.10.2,192.168.100.18,10.10.10.0/24 \ +no_proxy=localhost,127.0.0.1,10.10.10.13,10.10.10.2,192.168.100.18,10.10.10.0/24 \ +./check-aw-full.sh +``` + +Для workforce ClickHouse дополнительно проверить quality views: + +```sql +SELECT + (SELECT count() FROM aw_workforce.v_workforce_unknown_subjects) AS unknown_subjects, + (SELECT count() FROM aw_workforce.v_workforce_unknown_processes) AS unknown_processes, + (SELECT count() FROM aw_workforce.v_workforce_unknown_domains) AS unknown_domains, + (SELECT countIf(user_login = 'unknown' + OR (process_name = 'unknown' AND lengthUTF8(window_title) = 0)) + FROM aw_workforce.aw_window_events) AS no_user_window_rows; +``` + +Ожидаемое состояние после cleanup/normalization: все четыре значения равны `0`. diff --git a/docs/QUALITY_STATUS_RU.md b/docs/QUALITY_STATUS_RU.md index 304d616..818f8f3 100644 --- a/docs/QUALITY_STATUS_RU.md +++ b/docs/QUALITY_STATUS_RU.md @@ -45,6 +45,19 @@ evidence. Russian build-runner still required for registry release evidence. Public security checks are advisory/public validation. Registry release security evidence must be generated in the Russian build contour. +Current local engineering gate for product changes is documented in +`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md` and includes: + +- full Rust workspace tests with `--workspace --all-targets --locked`; +- doc tests; +- `clippy` with `-D warnings`; +- RustSec check through `cargo audit --deny warnings`; +- dependency policy through `cargo deny`; +- repository `quality-gate`; +- public secret-pattern check; +- browser smoke through ActivityWatch UI, worktime reports, 1C manager pages, + gateway and Grafana dashboards. + ## Conservative positioning The quality layer does not claim certification, does not position AWatch-rus as diff --git a/docs/REVIEW_CHECKLIST_RU.md b/docs/REVIEW_CHECKLIST_RU.md index 63255ad..a8332a1 100644 --- a/docs/REVIEW_CHECKLIST_RU.md +++ b/docs/REVIEW_CHECKLIST_RU.md @@ -26,7 +26,12 @@ release evidence должен производиться на российско ## Rust code quality - Rust-код форматируется `cargo fmt --all --check`. -- Для затронутого Rust workspace ожидаются relevant `cargo test` checks. +- Для существенных Rust-изменений ожидается полный workspace gate: + `cargo test --workspace --all-targets --locked`, + `cargo test --workspace --doc --locked` и + `cargo clippy --workspace --all-targets --locked -- -D warnings`. +- Для Windows/RDP collector дополнительно проверяется target + `x86_64-pc-windows-gnu` через `cargo check` и `cargo clippy`. - Ошибки обрабатываются явно; нет silent fallback для security-sensitive paths. - Timeouts, retries and bounds are explicit for network or long-running work. - Новые dependencies justified and license-compatible. @@ -95,7 +100,11 @@ release evidence должен производиться на российско `bash scripts/registry_readiness_check.sh`, `git diff --check`. - For shell changes, `bash -n` is mandatory for changed shell scripts. -- For Rust/product changes, Rust checks are required by risk and scope. +- For Rust/product changes, use + `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md` as the default local validation + contour. +- For operator-facing web, gateway, worktime reports or Grafana dashboards, + browser smoke through the rendered pages is required in addition to API checks. ## Evidence requirements diff --git a/docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md b/docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md new file mode 100644 index 0000000..5851693 --- /dev/null +++ b/docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md @@ -0,0 +1,63 @@ +# Источники ActivityWatch для workforce ingest: SHARKON2025 + +Дата проверки: `2026-06-23` + +AW API: `http://10.10.10.13:5600/api/0` + +Хост: `SHARKON2025` + +## Итоговое решение P1 + +Основные источники для `aw_workforce`: + +| Поток | Bucket | Решение | Поля | +|---|---|---|---| +| Desktop/window facts | `aw-watcher-window_SHARKON2025` | Загружать в `aw_window_events` | `app`, `hostname`, `processId`, `sessionId`, `source`, `title`, `username` | +| Browser facts, Edge | `aw-watcher-web-edge_SHARKON2025` | Загружать в `aw_browser_events` | `app`, `browser`, `hostname`, `sessionId`, `source`, `title`, `url`, `username` | +| Browser facts, Chrome | `aw-watcher-web-chrome_SHARKON2025` | Загружать как исторический browser source; если `username` отсутствует, писать `unknown` | `app`, `browser`, `sessionId`, `source`, `title`, `url` | + +Не использовать как основной fact-source продуктивности: + +| Bucket | Причина | +|---|---| +| `aw-rdp-window_SHARKON2025` | Есть активное окно RDP bridge, но нет `username` в событии; не годится для точной per-user привязки. | +| `aw-detmir-web-category_SHARKON2025` | Это health/category signal с `signalType=collector_health` и нулевой длительностью; полезен для диагностики collector/user presence, но не для длительности продуктивности. | +| `aw-worktime-sessions_SHARKON2025` | Авторитетный источник сессий и пользователей RDP, но это presence/session facts, а не window/browser usage facts. | + +## Подтвержденные факты + +`aw-watcher-window_SHARKON2025`: + +- type: `currentwindow`; +- client: `aw-watcher-window`; +- hostname: `SHARKON2025`; +- за последние 24 часа есть события с пользователями `USER1`, `USER4`, `USER5`, `Администратор`; +- `USER1` подтвержден в этом bucket и будет нормализован loader-ом в `user1`. + +`aw-watcher-web-edge_SHARKON2025`: + +- type: `web.tab.current`; +- client: `aw-watcher-web-edge`; +- hostname: `SHARKON2025`; +- события за последние 30 дней содержат `username`; +- на момент проверки событий за последние 24 часа не было, поэтому источник включается как основной browser source, но freshness контролируется отдельно. + +`aw-watcher-web-chrome_SHARKON2025`: + +- type: `web.tab.current`; +- client: `aw-watcher-web-chrome`; +- hostname: `SHARKON2025`; +- последние найденные события исторические и не содержат `username`; +- loader загружает их с `user_login='unknown'`, пока нет надежной session correlation. + +## Первая привязка P3 + +Файл загрузки: `clickhouse-workforce/sample/seed_sharkon2025_p3.sql`. + +| host_name | user_domain | user_login | department | branch | +|---|---|---|---|---| +| `SHARKON2025` | `sharkon2025` | `user1` | `tsj` | `tsj` | + +Ключ словаря остается `(host_name, user_login)`. Домен хранится как атрибут +`user_domain`, потому что текущие raw-события ActivityWatch дают `username`, а не +стабильный `DOMAIN\user` в window/browser facts. diff --git a/docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md b/docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md index a2a4f7f..97eeeba 100644 --- a/docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md +++ b/docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md @@ -1,32 +1,48 @@ -# План внедрения ClickHouse Dictionaries и агрегатов для AWatch-rus / DetMir +# План внедрения ClickHouse Dictionaries для AWatch-rus / DetMir Дата: `2026-06-23` -Статус: рабочий архитектурный план. +Статус: рабочий архитектурный план. Спринты 1-4 зафиксированы как +последовательность внедрения. Для `SHARKON2025` источники P1 сверены с живым +ActivityWatch API и зафиксированы в +`docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md`. -## 1. Общая цель +Реализация первого воспроизводимого слоя в репозитории: +`clickhouse-workforce/`. В нем зафиксированы DDL, локальный ClickHouse scaffold, +demo seed и smoke-проверка для dictionaries, materialized views и агрегатов. -Цель внедрения ClickHouse Dictionaries и Materialized Views - подготовить -AWatch-rus / DetMir к росту объема данных и числу пользователей отчетности. +P2 реализуется Rust loader-ом `adk-rust/crates/aw-workforce-ingest`, который +читает bounded ranges из ActivityWatch и пишет `JSONEachRow` в +`aw_window_events` / `aw_browser_events`. -Сырые события ActivityWatch остаются в fact-таблицах. Отчеты и Grafana не -должны постоянно выполнять тяжелые `JOIN` и агрегации по миллионам или -миллиардам строк. Обогащение и схлопывание данных нужно переносить на этап -записи, backfill или scheduled aggregation. +P4 добавляет production-режим loader-а: state file с `last_end`, overlap-окно +для защиты от поздних событий, retry/backoff и systemd timer +`clickhouse-workforce/ops/aw-workforce-ingest.timer`. -Базовая схема: +P5 добавляет администрируемый source of truth для справочников: +`clickhouse-workforce/catalog/*.tsv`. Загрузка выполняется через +`clickhouse-workforce/ops/apply_catalogs.sh`; удаление/отключение строки +делается либо удалением из TSV, либо `is_active=0`. Для уже материализованных +агрегатов после изменения категорий используется +`clickhouse-workforce/admin/rebuild_aggregates.sql`. + +## 1. Общий принцип + +ClickHouse Dictionaries внедряются как слой быстрого обогащения событий, а не +как новый источник истины. + +Источник истины: ```text -raw ActivityWatch events - -> dimension tables - -> ClickHouse Dictionaries - -> Materialized Views / aggregate tables - -> Grafana / reports +исходные справочники / NetBox / XLS / инвентаризация / ручная классификация + -> dimension tables в ClickHouse + -> ClickHouse Dictionaries + -> отчеты, materialized views, Grafana ``` -Dictionaries не являются источником истины. Источник истины - обычные таблицы -измерений, импорт из NetBox/XLS/инвентаризации или утвержденные ручные -справочники. +Сырые события AWatch-rus остаются fact-таблицами. Справочники и dictionaries +используются для привязки событий к оргструктуре, приложениям, категориям и +рискам. ## 2. Спринт 1. Оргструктура и слепые зоны @@ -35,14 +51,19 @@ Dictionaries не являются источником истины. Источ Связать сырые события AWatch-rus с оргструктурой компании и выявить слепые зоны в привязке рабочих мест, пользователей и подразделений. -### Решение +### Принятое архитектурное решение Создаются: -- `dim_workstation_user` - dimension/current snapshot; -- `dict_workstation_user` - словарь для lookup по паре `host_name + user_login`. +- `dim_workstation_user` - dimension/current snapshot, а не таблица фактов; +- `dict_workstation_user` - ClickHouse Dictionary для lookup по паре + `host_name + user_login`. -Так как ключ составной, используется `COMPLEX_KEY_HASHED()`. +Ключ словаря комплексный, поэтому используется `COMPLEX_KEY_HASHED()`, а не +обычный `HASHED()`. + +`CACHE()` на первом этапе не используется: справочник рабочих мест должен быть +достаточно небольшим, чтобы держать его целиком в памяти. ### Source table @@ -51,7 +72,9 @@ CREATE TABLE IF NOT EXISTS dim_workstation_user ( host_name String, user_login String, + user_domain String, + employee_id String, employee_name String, department String, branch String, @@ -72,7 +95,9 @@ CREATE DICTIONARY IF NOT EXISTS dict_workstation_user ( host_name String, user_login String, + user_domain String, + employee_id String, employee_name String, department String, branch String, @@ -80,11 +105,41 @@ CREATE DICTIONARY IF NOT EXISTS dict_workstation_user is_active UInt8 ) PRIMARY KEY host_name, user_login -SOURCE(CLICKHOUSE(TABLE 'dim_workstation_user')) +SOURCE(CLICKHOUSE( + TABLE 'dim_workstation_user' +)) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(MIN 3600 MAX 86400); ``` +### Использование + +```sql +SELECT + event_time, + host_name, + user_login, + dictGetStringOrDefault( + 'dict_workstation_user', + 'employee_name', + (host_name, user_login), + 'unknown' + ) AS employee_name, + dictGetStringOrDefault( + 'dict_workstation_user', + 'department', + (host_name, user_login), + 'unknown' + ) AS department, + dictGetStringOrDefault( + 'dict_workstation_user', + 'branch', + (host_name, user_login), + 'unknown' + ) AS branch +FROM aw_raw_events; +``` + ### Отчет по слепым зонам ```sql @@ -99,39 +154,98 @@ WHERE dictGetStringOrDefault( (host_name, user_login), '' ) = '' -GROUP BY host_name, user_login +GROUP BY + host_name, + user_login ORDER BY events DESC LIMIT 100; ``` ### Definition of Done +Спринт 1 считается завершенным, когда: + - создана таблица `dim_workstation_user`; - создан словарь `dict_workstation_user`; - словарь находится в статусе `LOADED`; - есть стартовая загрузка данных; -- есть отчет unknown `host_name + user_login`; -- есть первый обогащенный запрос по сырым событиям; +- есть SQL-отчет по неизвестным `host_name + user_login`; +- есть хотя бы один обогащенный запрос по сырым событиям; +- есть проверка состояния словаря через `system.dictionaries`; - документирован источник истины для оргструктуры. -## 3. Спринт 2. Продуктивность и классификация desktop software +## 3. Спринт 2. Продуктивность и классификация софта ### Цель Уйти от анализа миллионов строк процессов к понятным бизнес-метрикам по -использованию ПО на рабочих местах. +использованию программного обеспечения на рабочих местах. -Сырые события должны обогащаться не только `process_name`, а нормализованным -именем приложения, категорией, признаком продуктивности и уровнем риска. +События рабочего стола должны обогащаться не только именем процесса, а +категорией приложения, признаком продуктивности, уровнем риска и нормализованным +именем приложения. -### Решение +### Что должно появиться Создаются: -- `dim_application_category`; -- `dict_application_category`; -- отчет top unknown processes; -- первый отчет продуктивности по отделам и сотрудникам. +- `dim_application_category` - справочник классификации процессов и приложений; +- `dict_application_category` - ClickHouse Dictionary для lookup по + `process_name`; +- отчет по неизвестным процессам; +- первый отчет/витрина продуктивности по отделам и сотрудникам. + +### Бизнес-метрики спринта 2 + +Минимальный набор метрик: + +- время в продуктивных приложениях; +- время в условно нейтральных приложениях; +- время в непродуктивных приложениях; +- время в административных/технических инструментах; +- время в неизвестных процессах; +- top unknown processes по числу событий и длительности; +- top risky processes; +- разрез по отделу, филиалу, сотруднику, рабочей станции. + +### Категории приложений + +Начальный набор категорий: + +| Категория | Смысл | +|---|---| +| `office` | офисные приложения, документы, таблицы | +| `browser` | браузеры без оценки домена | +| `1c` | 1C и связанные клиенты | +| `banking` | банковские и внутренние рабочие приложения | +| `messenger` | корпоративные и внешние мессенджеры | +| `mail` | почтовые клиенты | +| `admin_tool` | администрирование, диагностика, удаленный доступ | +| `development` | разработка, скрипты, IDE | +| `security_tool` | средства защиты, DLP, AV, мониторинг | +| `media` | аудио/видео/плееры | +| `archive` | архиваторы, файловые утилиты | +| `system` | системные процессы ОС | +| `unknown` | не классифицировано | +| `risky` | потенциально рискованный или нежелательный софт | + +### Признак продуктивности + +Поле `productivity_class` используется отдельно от категории. + +Начальные значения: + +| Значение | Смысл | +|---|---| +| `productive` | рабочее приложение | +| `neutral` | системное или вспомогательное приложение | +| `non_productive` | явно нерабочее использование | +| `risky` | потенциальный риск ИБ или нежелательное ПО | +| `unknown` | нет классификации | + +Важно: продуктивность не должна автоматически считаться дисциплинарным выводом. +Это аналитическая классификация для управленческого отчета и выявления слепых +зон. ### Source table @@ -158,6 +272,9 @@ ORDER BY process_name; ### Dictionary +Для спринта 2 используется обычный `HASHED()`, потому что ключ один: +`process_name`. + ```sql CREATE DICTIONARY IF NOT EXISTS dict_application_category ( @@ -172,20 +289,47 @@ CREATE DICTIONARY IF NOT EXISTS dict_application_category is_active UInt8 ) PRIMARY KEY process_name -SOURCE(CLICKHOUSE(TABLE 'dim_application_category')) +SOURCE(CLICKHOUSE( + TABLE 'dim_application_category' +)) LAYOUT(HASHED()) LIFETIME(MIN 3600 MAX 86400); ``` -### Базовые классы продуктивности +### Использование в запросах -| Значение | Смысл | -|---|---| -| `productive` | рабочее приложение | -| `neutral` | системное или вспомогательное приложение | -| `non_productive` | явно нерабочее использование | -| `risky` | потенциальный риск ИБ или нежелательное ПО | -| `unknown` | нет классификации | +```sql +SELECT + event_time, + host_name, + user_login, + process_name, + dictGetStringOrDefault( + 'dict_application_category', + 'application_name', + process_name, + process_name + ) AS application_name, + dictGetStringOrDefault( + 'dict_application_category', + 'category', + process_name, + 'unknown' + ) AS app_category, + dictGetStringOrDefault( + 'dict_application_category', + 'productivity_class', + process_name, + 'unknown' + ) AS productivity_class, + dictGetStringOrDefault( + 'dict_application_category', + 'risk_level', + process_name, + 'unknown' + ) AS risk_level +FROM aw_window_events; +``` ### Отчет unknown processes @@ -206,52 +350,133 @@ ORDER BY duration_sec DESC LIMIT 100; ``` +### Пример витрины продуктивности + +```sql +SELECT + toDate(event_time) AS date, + dictGetStringOrDefault( + 'dict_workstation_user', + 'department', + (host_name, user_login), + 'unknown' + ) AS department, + dictGetStringOrDefault( + 'dict_workstation_user', + 'employee_name', + (host_name, user_login), + 'unknown' + ) AS employee_name, + dictGetStringOrDefault( + 'dict_application_category', + 'productivity_class', + process_name, + 'unknown' + ) AS productivity_class, + sum(duration_sec) AS duration_sec +FROM aw_window_events +GROUP BY + date, + department, + employee_name, + productivity_class +ORDER BY + date, + department, + employee_name, + productivity_class; +``` + ### Definition of Done +Спринт 2 считается завершенным, когда: + - создана таблица `dim_application_category`; - создан словарь `dict_application_category`; +- словарь находится в статусе `LOADED`; - загружен стартовый список известных процессов; -- есть top unknown processes; -- есть отчет продуктивности в разрезе отдела/сотрудника; +- есть отчет top unknown processes; +- есть отчет по продуктивности в разрезе отдела/сотрудника; +- неизвестные процессы видны как отдельная зона качества данных; - классификация не используется как дисциплинарный вывод без ручной проверки; -- назначен владелец справочника классификации ПО. +- документирован владелец справочника классификации ПО. ## 4. Спринт 3. Веб-аналитика и глубинная фильтрация ### Цель -Перейти от анализа заголовков окон браузера и полных URL к нормальной -веб-аналитике: домены, категории сайтов, рабочая/нерабочая активность, -рисковые ресурсы и unknown domains. +Перейти от анализа сырых заголовков окон браузера и полных URL к нормальной +веб-аналитике: домены, категории сайтов, признаки рабочей/нерабочей активности, +рисковые ресурсы и слепые зоны классификации. -### Решение +Для современного сотрудника значительная часть работы идет в браузере, поэтому +без доменной классификации отчеты по продуктивности будут неполными: браузер +сам по себе не показывает, была ли активность рабочей, нейтральной или +нежелательной. + +### Что должно появиться Создаются: -- нормализация URL в `domain_name`; -- `dim_domain_category`; -- `dict_domain_category`; -- top unknown domains; -- browser productivity report; -- задел под path-level правила. +- нормализованное поле `domain` в запросах, view или materialized view; +- `dim_domain_category` - справочник классификации доменов; +- `dict_domain_category` - ClickHouse Dictionary для lookup по `domain`; +- отчет top unknown domains; +- отчет browser activity по категориям сайтов; +- правила глубинной фильтрации для URL/path, если одного домена недостаточно. -### Нормализация URL +### Нормализация URL в домен + +Источник может содержать: + +- полный URL: `https://github.com/org/repo/issues`; +- адрес без схемы: `github.com/org/repo`; +- заголовок окна с доменом внутри строки; +- browser bucket с отдельным URL-полем; +- пустые или неполные значения. + +Базовое правило: сначала извлекать чистый домен, потом классифицировать его +через dictionary. + +Пример для поля с полным URL: + +```sql +SELECT + url, + lowerUTF8(domain(url)) AS domain_name +FROM aw_browser_events; +``` + +Если `domain()` недостаточно из-за строк без схемы, использовать нормализацию: ```sql SELECT url, lowerUTF8( - parseURL( - if(position(url, '://') = 0, concat('http://', url), url), - 'host' + domain( + if( + position(url, '://') = 0, + concat('http://', url), + url + ) ) ) AS domain_name FROM aw_browser_events; ``` -Если в данных уже есть корректный полный URL, можно использовать `domain(url)`. -Если URL спрятан только в title, regex parsing допускается как fallback, но не -как основной путь. +На ClickHouse `24.8` в локальном scaffold проверена функция `domain()`: + +```sql +SELECT + url, + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +FROM aw_browser_events; +``` + +Для строк, где URL спрятан в title, допустим отдельный regex-pass, но его не +нужно делать основным путем, если в событиях уже есть URL. ### Source table @@ -277,6 +502,8 @@ ORDER BY domain; ### Dictionary +Для домена используется один ключ, поэтому достаточно `HASHED()`. + ```sql CREATE DICTIONARY IF NOT EXISTS dict_domain_category ( @@ -290,7 +517,9 @@ CREATE DICTIONARY IF NOT EXISTS dict_domain_category is_active UInt8 ) PRIMARY KEY domain -SOURCE(CLICKHOUSE(TABLE 'dim_domain_category')) +SOURCE(CLICKHOUSE( + TABLE 'dim_domain_category' +)) LAYOUT(HASHED()) LIFETIME(MIN 3600 MAX 86400); ``` @@ -303,7 +532,7 @@ LIFETIME(MIN 3600 MAX 86400); | `banking` | банковские и финансовые ресурсы | | `government` | государственные сервисы | | `work_service` | рабочие SaaS/порталы/документация | -| `developer` | Git, документация, package registry | +| `developer` | Git, документация, package registry, developer tools | | `mail` | почтовые сервисы | | `messenger` | web-мессенджеры | | `cloud_storage` | облачные хранилища | @@ -317,12 +546,150 @@ LIFETIME(MIN 3600 MAX 86400); | `unknown` | не классифицировано | | `risky` | рискованный или нежелательный ресурс | -### Path-level правила +### Признак продуктивности для доменов -Классификация по домену не всегда достаточна. Для `github.com`, `youtube.com`, -`docs.google.com`, `mail.ru` может потребоваться учет path или контекста. +Используется тот же подход, что в спринте 2: -Задел под будущую детализацию: +| Значение | Смысл | +|---|---| +| `productive` | рабочий ресурс | +| `neutral` | вспомогательный или неоднозначный ресурс | +| `non_productive` | явно нерабочий ресурс | +| `risky` | потенциальный ИБ-риск или нежелательный ресурс | +| `unknown` | нет классификации | + +Важно: продуктивность домена зависит от контекста. Например, `youtube.com` для +одного подразделения может быть учебным ресурсом, а для другого - +непродуктивной активностью. В спринте 3 это фиксируется как ограничение модели. +Тонкую контекстную политику по подразделениям лучше выносить в следующий этап. + +### Использование в запросах + +```sql +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + event_time, + host_name, + user_login, + url, + domain_name, + dictGetStringOrDefault( + 'dict_domain_category', + 'site_name', + domain_name, + domain_name + ) AS site_name, + dictGetStringOrDefault( + 'dict_domain_category', + 'category', + domain_name, + 'unknown' + ) AS domain_category, + dictGetStringOrDefault( + 'dict_domain_category', + 'productivity_class', + domain_name, + 'unknown' + ) AS productivity_class, + dictGetStringOrDefault( + 'dict_domain_category', + 'risk_level', + domain_name, + 'unknown' + ) AS risk_level +FROM aw_browser_events; +``` + +### Отчет unknown domains + +```sql +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + domain_name, + count() AS events, + sum(duration_sec) AS duration_sec +FROM aw_browser_events +WHERE domain_name != '' + AND dictGetStringOrDefault( + 'dict_domain_category', + 'category', + domain_name, + '' + ) = '' +GROUP BY domain_name +ORDER BY duration_sec DESC +LIMIT 100; +``` + +### Витрина browser productivity + +```sql +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + toDate(event_time) AS date, + dictGetStringOrDefault( + 'dict_workstation_user', + 'department', + (host_name, user_login), + 'unknown' + ) AS department, + dictGetStringOrDefault( + 'dict_workstation_user', + 'employee_name', + (host_name, user_login), + 'unknown' + ) AS employee_name, + dictGetStringOrDefault( + 'dict_domain_category', + 'category', + domain_name, + 'unknown' + ) AS domain_category, + dictGetStringOrDefault( + 'dict_domain_category', + 'productivity_class', + domain_name, + 'unknown' + ) AS productivity_class, + sum(duration_sec) AS duration_sec +FROM aw_browser_events +WHERE domain_name != '' +GROUP BY + date, + department, + employee_name, + domain_category, + productivity_class +ORDER BY + date, + department, + employee_name, + duration_sec DESC; +``` + +### Глубинная фильтрация + +Классификация только по домену не всегда достаточна. + +Примеры: + +- `github.com` может быть рабочим ресурсом, но конкретный path может относиться + к личным репозиториям; +- `youtube.com` может быть обучением или развлечением; +- `docs.google.com` может быть рабочим или личным ресурсом; +- `mail.ru` может быть почтой, новостями или облаком. + +В спринте 3 нужно заложить второй уровень правил, но не усложнять основную +модель. Для этого создается отдельная таблица правил URL/path. ```sql CREATE TABLE IF NOT EXISTS dim_url_rule @@ -344,419 +711,452 @@ ENGINE = ReplacingMergeTree(updated_at) ORDER BY (domain, priority, rule_id); ``` -На спринте 3 `dim_url_rule` можно вести как backlog правил и применять точечно. +На первом этапе `dim_url_rule` можно не подключать как dictionary. Достаточно +вести ее как backlog правил и применять точечно в offline-отчетах или +materialized view, если появится необходимость. ### Definition of Done +Спринт 3 считается завершенным, когда: + - создана таблица `dim_domain_category`; - создан словарь `dict_domain_category`; -- есть нормализация URL в `domain_name`; +- словарь находится в статусе `LOADED`; +- есть нормализация URL в `domain_name` через `domain()`; - есть стартовая классификация top domains; -- есть top unknown domains; -- есть browser productivity report; +- есть отчет top unknown domains; +- есть отчет browser productivity по отделу/сотруднику; +- известные рабочие ресурсы классифицированы отдельно от неизвестных; +- risky/non-productive домены не используются как дисциплинарный вывод без + ручной проверки; - ограничения доменной модели и необходимость path-level правил задокументированы. +### Риски спринта 3 + +| Риск | Митигация | +|---|---| +| URL не всегда есть в событии | использовать URL-поле, а title parsing оставить fallback | +| Домен не отражает реальное назначение страницы | добавить `dim_url_rule` для path-level правил | +| Ошибочная классификация домена | review справочника и поле `comment` | +| Большая доля unknown domains | ежедневный top unknown отчет | +| Внешние AI/cloud сервисы требуют отдельной политики | вынести `ai_service` и `cloud_storage` в отдельные категории | +| Продуктивность зависит от подразделения | оставить как ограничение спринта 3, контекстную политику вынести позже | + ## 5. Спринт 4. Enterprise-масштабирование и стабильность ### Цель -Гарантировать, что система не ляжет при росте компании и объема событий. -Grafana не должна при каждом открытии dashboard на лету агрегировать миллиарды -сырых строк. +Снять нагрузку с Grafana и ClickHouse при росте объема событий: dashboard'ы не +должны каждый раз агрегировать миллионы или миллиарды строк сырых событий. -### Ключевое архитектурное правило - -Materialized Views должны сразу складывать в агрегированные таблицы уже -обогащенные измерения, полученные через dictionaries. - -То есть Grafana должна читать не `user_id` и не `process_name`, требующие -дальнейшего JOIN, а готовые бизнес-срезы: +Для этого создается слой materialized views, который при вставке новых событий +сразу складывает в агрегированные таблицы уже обогащенные метрики: ```text -Дата / Час -Отделение -Отдел -Сотрудник -Категория продуктивности -Категория приложения или домена -Сумма секунд -Количество событий +raw AWatch-rus events + -> dictionaries lookup during INSERT + -> hourly aggregate tables + -> Grafana queries over small aggregates ``` -Это переносит CPU-нагрузку с момента открытия dashboard на момент insert/backfill -и практически убирает тяжелые вычисления из пользовательских запросов Grafana. +Ключевое решение: materialized views должны записывать в агрегированную таблицу +не только технические `host_name`, `user_login`, `process_name` или `domain`, а +готовые управленческие разрезы: -### Общая схема +- дата/час; +- филиал; +- отделение/подразделение; +- тип активности: `desktop` или `browser`; +- категория приложения или домена; +- класс продуктивности; +- сумма секунд; +- количество событий; +- счетчики unknown-зон. -```text -raw events - -> Materialized View с dictGet* enrichment - -> SummingMergeTree / AggregatingMergeTree aggregate table - -> Grafana читает готовые агрегаты -``` +Это снижает CPU-нагрузку на ClickHouse и делает Grafana предсказуемой по +времени ответа. -Raw tables остаются для расследований и drill-down. Штатные dashboards должны -читать агрегаты. +### Принятое архитектурное решение -### Выбор движка +Для первого промышленного слоя используется `SummingMergeTree`, потому что +базовые метрики аддитивны: -| Движок | Где использовать | -|---|---| -| `SummingMergeTree` | суммы duration/count, основные dashboards | -| `AggregatingMergeTree` | uniq, quantile, topK, сложные агрегатные состояния | +- `duration_sec`; +- `event_count`; +- `unknown_subject_events`; +- `unknown_category_events`. -Рекомендация: начинать с `SummingMergeTree`. `AggregatingMergeTree` подключать -только при доказанной необходимости. +`AggregatingMergeTree` нужен позже, если появятся сложные состояния: +`uniqState`, `quantileState`, percentile latency, distinct users/workstations +или rolling windows. Для Спринта 4 он не является обязательным. -### Desktop hourly aggregate +Важно: dictionary lookup внутри materialized view выполняется в момент вставки. +Это хорошо для скорости Grafana, но означает, что исправление справочника не +пересчитает старые агрегаты автоматически. Для этого нужен явный backfill. + +### Target table для Grafana ```sql -CREATE TABLE IF NOT EXISTS agg_aw_desktop_hourly +CREATE TABLE IF NOT EXISTS agg_workforce_productivity_hourly ( - date Date, - hour DateTime, + bucket_start DateTime, + event_date Date, - branch String, - department String, - employee_name String, - host_name String, - user_login String, + branch LowCardinality(String), + department LowCardinality(String), - application_name String, - app_category String, - productivity_class String, - risk_level String, + activity_type LowCardinality(String), + category LowCardinality(String), + productivity_class LowCardinality(String), duration_sec UInt64, - event_count UInt64 + event_count UInt64, + unknown_subject_events UInt64, + unknown_category_events UInt64 ) -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(date) -ORDER BY -( - date, - hour, +ENGINE = SummingMergeTree(( + duration_sec, + event_count, + unknown_subject_events, + unknown_category_events +)) +PARTITION BY toYYYYMM(event_date) +ORDER BY ( + event_date, + bucket_start, branch, department, - employee_name, - productivity_class, - app_category, - application_name, - host_name, - user_login -); -``` - -```sql -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_aw_desktop_hourly -TO agg_aw_desktop_hourly -AS -SELECT - toDate(event_time) AS date, - toStartOfHour(event_time) AS hour, - - dictGetStringOrDefault('dict_workstation_user', 'branch', (host_name, user_login), 'unknown') AS branch, - dictGetStringOrDefault('dict_workstation_user', 'department', (host_name, user_login), 'unknown') AS department, - dictGetStringOrDefault('dict_workstation_user', 'employee_name', (host_name, user_login), 'unknown') AS employee_name, - - host_name, - user_login, - - dictGetStringOrDefault('dict_application_category', 'application_name', process_name, process_name) AS application_name, - dictGetStringOrDefault('dict_application_category', 'category', process_name, 'unknown') AS app_category, - dictGetStringOrDefault('dict_application_category', 'productivity_class', process_name, 'unknown') AS productivity_class, - dictGetStringOrDefault('dict_application_category', 'risk_level', process_name, 'unknown') AS risk_level, - - sum(duration_sec) AS duration_sec, - count() AS event_count -FROM aw_window_events -GROUP BY - date, - hour, - branch, - department, - employee_name, - host_name, - user_login, - application_name, - app_category, - productivity_class, - risk_level; -``` - -### Browser hourly aggregate - -```sql -CREATE TABLE IF NOT EXISTS agg_aw_browser_hourly -( - date Date, - hour DateTime, - - branch String, - department String, - employee_name String, - host_name String, - user_login String, - - domain_name String, - site_name String, - domain_category String, - productivity_class String, - risk_level String, - - duration_sec UInt64, - event_count UInt64 -) -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(date) -ORDER BY -( - date, - hour, - branch, - department, - employee_name, - productivity_class, - domain_category, - domain_name, - host_name, - user_login -); -``` - -```sql -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_aw_browser_hourly -TO agg_aw_browser_hourly -AS -WITH - lowerUTF8( - parseURL( - if(position(url, '://') = 0, concat('http://', url), url), - 'host' - ) - ) AS domain_name -SELECT - toDate(event_time) AS date, - toStartOfHour(event_time) AS hour, - - dictGetStringOrDefault('dict_workstation_user', 'branch', (host_name, user_login), 'unknown') AS branch, - dictGetStringOrDefault('dict_workstation_user', 'department', (host_name, user_login), 'unknown') AS department, - dictGetStringOrDefault('dict_workstation_user', 'employee_name', (host_name, user_login), 'unknown') AS employee_name, - - host_name, - user_login, - domain_name, - - dictGetStringOrDefault('dict_domain_category', 'site_name', domain_name, domain_name) AS site_name, - dictGetStringOrDefault('dict_domain_category', 'category', domain_name, 'unknown') AS domain_category, - dictGetStringOrDefault('dict_domain_category', 'productivity_class', domain_name, 'unknown') AS productivity_class, - dictGetStringOrDefault('dict_domain_category', 'risk_level', domain_name, 'unknown') AS risk_level, - - sum(duration_sec) AS duration_sec, - count() AS event_count -FROM aw_browser_events -WHERE domain_name != '' -GROUP BY - date, - hour, - branch, - department, - employee_name, - host_name, - user_login, - domain_name, - site_name, - domain_category, - productivity_class, - risk_level; -``` - -### Daily aggregate для управленческих отчетов - -Daily layer лучше строить из hourly aggregates, чтобы не дублировать enrichment -логику и не перечитывать raw events. - -```sql -CREATE TABLE IF NOT EXISTS agg_aw_activity_daily -( - date Date, - branch String, - department String, - source_type LowCardinality(String), - productivity_class String, - category String, - duration_sec UInt64, - event_count UInt64 -) -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(date) -ORDER BY -( - date, - branch, - department, - source_type, + activity_type, productivity_class, category ); ``` -Пример загрузки из desktop hourly: +`bucket_start` хранит часовую гранулярность. Дневные отчеты строятся простой +агрегацией поверх этой таблицы, без чтения raw-событий. + +### Materialized view для desktop events ```sql -INSERT INTO agg_aw_activity_daily +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_desktop_productivity_hourly +TO agg_workforce_productivity_hourly +AS SELECT - date, - branch, - department, - 'desktop' AS source_type, - productivity_class, - app_category AS category, - sum(duration_sec) AS duration_sec, - sum(event_count) AS event_count -FROM agg_aw_desktop_hourly -WHERE date = yesterday() + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + dictGetStringOrDefault( + 'dict_workstation_user', + 'branch', + (host_name, user_login), + 'unknown' + ) AS branch, + dictGetStringOrDefault( + 'dict_workstation_user', + 'department', + (host_name, user_login), + 'unknown' + ) AS department, + + 'desktop' AS activity_type, + dictGetStringOrDefault( + 'dict_application_category', + 'category', + process_name, + 'unknown' + ) AS category, + dictGetStringOrDefault( + 'dict_application_category', + 'productivity_class', + process_name, + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetStringOrDefault( + 'dict_workstation_user', + 'employee_name', + (host_name, user_login), + '' + ) = '', + 1, + 0 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetStringOrDefault( + 'dict_application_category', + 'category', + process_name, + '' + ) = '', + 1, + 0 + ))) AS unknown_category_events +FROM aw_window_events GROUP BY - date, + bucket_start, + event_date, branch, department, - source_type, - productivity_class, - category; + activity_type, + category, + productivity_class; ``` -### Почему обогащение нужно делать до Grafana - -Если Grafana читает raw tables и выполняет `dictGet*` или `JOIN` при каждом -открытии панели, CPU ClickHouse будет расходоваться на одни и те же вычисления. -При росте компании это станет узким местом. - -Если Materialized View уже положила в агрегат строки вида: - -```text -2026-06-23 / Сыктывкар / ОТ / productive / office / 18420 sec -``` - -то Grafana выполняет простой `SELECT sum(duration_sec) ... GROUP BY ...` по -маленькой таблице. Это дает кратный выигрыш и делает dashboard стабильным. - -### Backfill и rebuild - -Materialized View обрабатывает только новые вставки после создания MV. -Исторические данные нужно пересчитать отдельно. - -Порядок: - -1. создать aggregate table; -2. создать materialized view для новых данных; -3. выполнить `INSERT INTO aggregate SELECT ... FROM raw WHERE ...` за историю; -4. сверить суммы raw vs aggregate; -5. переключить Grafana на aggregate; -6. документировать rebuild-период, если справочники изменились. - -Важно: если словари поменялись после агрегации, старые агрегаты сами не -пересчитаются. Для строгой актуальности нужен rebuild затронутого периода или -snapshot/history dimensions. - -### Coverage aggregate для качества справочников +### Materialized view для browser events ```sql -CREATE TABLE IF NOT EXISTS agg_aw_dictionary_coverage_daily -( - date Date, - source_type LowCardinality(String), - unknown_type LowCardinality(String), - unknown_key String, - events UInt64, - duration_sec UInt64 -) -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(date) -ORDER BY (date, source_type, unknown_type, unknown_key); +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_browser_productivity_hourly +TO agg_workforce_productivity_hourly +AS +WITH + lowerUTF8( + domain(if(position(url, '://') = 0, concat('http://', url), url)) + ) AS domain_name +SELECT + toStartOfHour(event_time) AS bucket_start, + toDate(event_time) AS event_date, + + dictGetStringOrDefault( + 'dict_workstation_user', + 'branch', + (host_name, user_login), + 'unknown' + ) AS branch, + dictGetStringOrDefault( + 'dict_workstation_user', + 'department', + (host_name, user_login), + 'unknown' + ) AS department, + + 'browser' AS activity_type, + dictGetStringOrDefault( + 'dict_domain_category', + 'category', + domain_name, + 'unknown' + ) AS category, + dictGetStringOrDefault( + 'dict_domain_category', + 'productivity_class', + domain_name, + 'unknown' + ) AS productivity_class, + + toUInt64(sum(duration_sec)) AS duration_sec, + toUInt64(count()) AS event_count, + toUInt64(sum(if( + dictGetStringOrDefault( + 'dict_workstation_user', + 'employee_name', + (host_name, user_login), + '' + ) = '', + 1, + 0 + ))) AS unknown_subject_events, + toUInt64(sum(if( + dictGetStringOrDefault( + 'dict_domain_category', + 'category', + domain_name, + '' + ) = '', + 1, + 0 + ))) AS unknown_category_events +FROM aw_browser_events +WHERE domain_name != '' +GROUP BY + bucket_start, + event_date, + branch, + department, + activity_type, + category, + productivity_class; ``` -Эта таблица нужна, чтобы видеть: +### Базовый запрос Grafana -- unknown workstations; -- unknown processes; -- unknown domains; -- динамику качества справочников. - -### Grafana policy - -После спринта 4 штатные dashboards должны читать: - -- `agg_aw_desktop_hourly`; -- `agg_aw_browser_hourly`; -- `agg_aw_activity_daily`; -- `agg_aw_dictionary_coverage_daily`. - -Raw tables допустимы только для расследований, drill-down и технической -диагностики. - -### Monitoring - -Минимальные проверки: +Grafana должна читать агрегированную таблицу, а не raw events: ```sql SELECT - table, - sum(rows) AS rows, - formatReadableSize(sum(bytes_on_disk)) AS size -FROM system.parts -WHERE active -GROUP BY table -ORDER BY rows DESC; + event_date, + branch, + department, + productivity_class, + sum(duration_sec) AS duration_sec +FROM agg_workforce_productivity_hourly +WHERE $__timeFilter(bucket_start) +GROUP BY + event_date, + branch, + department, + productivity_class +ORDER BY + event_date, + branch, + department, + productivity_class; ``` +Для детализации по desktop/browser: + +```sql +SELECT + event_date, + department, + activity_type, + category, + productivity_class, + sum(duration_sec) AS duration_sec +FROM agg_workforce_productivity_hourly +WHERE $__timeFilter(bucket_start) +GROUP BY + event_date, + department, + activity_type, + category, + productivity_class +ORDER BY duration_sec DESC; +``` + +### Backfill и пересчет + +Так как materialized views фиксируют результат dictionary lookup в момент +вставки, для исправления старых данных нужен регламент: + +1. Исправить source dimension table. +2. Выполнить `SYSTEM RELOAD DICTIONARY`. +3. Проверить `system.dictionaries`. +4. Пересобрать нужный период агрегатов из raw-таблиц. + +Безопасный промышленный вариант для пересчета: + +- создать новую агрегированную таблицу с тем же schema; +- выполнить `INSERT INTO new_agg SELECT ... FROM raw ... WHERE event_date ...`; +- сверить контрольные суммы и строки; +- атомарно переключить имя таблицы через `RENAME TABLE`. + +Удаление и повторная вставка по partition допустимы только после отдельной +проверки на тестовом контуре. + +### Definition of Done + +Спринт 4 считается завершенным, когда: + +- создана агрегированная таблица для Grafana; +- materialized view для desktop events пишет в агрегат; +- materialized view для browser events пишет в агрегат; +- dictionary enrichment выполняется при вставке; +- Grafana dashboard'ы читают агрегат, а не raw-события; +- есть запрос контроля unknown-долей по оргструктуре, приложениям и доменам; +- есть регламент backfill после исправления справочников; +- есть smoke-запрос, подтверждающий, что новые raw-события попали в агрегат; +- зафиксировано, что исправление справочника не пересчитывает исторические + агрегаты без backfill. + +### Риски спринта 4 + +| Риск | Митигация | +|---|---| +| Старые агрегаты не обновляются после правки dictionary | формальный backfill-регламент | +| Grafana продолжает читать raw-события | review dashboard JSON и запрет тяжелых raw-запросов | +| Неверная гранулярность агрегата | хранить hourly, дневные отчеты строить поверх hourly | +| Слишком много cardinality в ORDER BY | не включать `employee_name` в основной агрегат, держать его в drilldown | +| Ошибочная классификация попадает в агрегат | unknown counters, review справочников, backfill | +| Разные raw-схемы desktop/browser | держать отдельные MV, но общий target aggregate | + +## 6. Проверка состояния dictionaries + ```sql SELECT name, status, element_count, - last_exception + last_exception, + last_successful_update_time FROM system.dictionaries -WHERE name LIKE 'dict_%'; +WHERE name IN ( + 'dict_workstation_user', + 'dict_application_category', + 'dict_domain_category' +); ``` -### Retention +Ожидаемое состояние перед включением materialized views: -Начальная рекомендация: +- все dictionaries имеют статус `LOADED`; +- `last_exception` пустой; +- `element_count` не равен нулю для загруженных справочников; +- время `last_successful_update_time` соответствует последней загрузке. -| Слой | Retention | -|---|---| -| raw events | 90-180 дней или по политике проекта | -| hourly aggregates | 12-24 месяца | -| daily aggregates | 3-5 лет или по требованиям отчетности | -| dictionary coverage | 12 месяцев | +## 7. Ручная перезагрузка -TTL включать только после согласования требований отчетности и хранения. +После обновления справочников: -### Definition of Done +```sql +SYSTEM RELOAD DICTIONARY dict_workstation_user; +SYSTEM RELOAD DICTIONARY dict_application_category; +SYSTEM RELOAD DICTIONARY dict_domain_category; +``` -Спринт 4 считается завершенным, когда: +После reload обязательно проверить `system.dictionaries`. Для materialized views +reload влияет только на новые вставки и ручной backfill, а не на уже записанные +агрегаты. -- создана `agg_aw_desktop_hourly`; -- создана `mv_aw_desktop_hourly`; -- создана `agg_aw_browser_hourly`; -- создана `mv_aw_browser_hourly`; -- создан daily aggregate layer; -- создан coverage aggregate для unknown значений; -- выполнен backfill хотя бы за один исторический период; -- сверены суммы raw vs aggregate; -- минимум один Grafana dashboard переключен на aggregate table; -- Grafana не выполняет штатные отчеты по raw events; -- документирован rebuild/backfill runbook; -- есть мониторинг таблиц, словарей и unknown coverage. +## 8. Контроль качества данных -### Риски +Минимальный набор контрольных запросов: + +```sql +SELECT + event_date, + sum(event_count) AS events, + sum(unknown_subject_events) AS unknown_subject_events, + round(unknown_subject_events / nullIf(events, 0), 4) AS unknown_subject_ratio, + sum(unknown_category_events) AS unknown_category_events, + round(unknown_category_events / nullIf(events, 0), 4) AS unknown_category_ratio +FROM agg_workforce_productivity_hourly +GROUP BY event_date +ORDER BY event_date DESC +LIMIT 14; +``` + +```sql +SELECT + event_date, + activity_type, + productivity_class, + sum(duration_sec) AS duration_sec +FROM agg_workforce_productivity_hourly +GROUP BY + event_date, + activity_type, + productivity_class +ORDER BY + event_date DESC, + duration_sec DESC; +``` + +Эти проверки нужны не для дисциплинарных выводов, а для качества данных: +находить незаведенные рабочие станции, неизвестные процессы, неизвестные +домены и ошибки классификации. + +## 9. Общие риски | Риск | Митигация | |---|---| -| MV не обработала исторические данные | обязательный backfill | -| Ошибка классификации попала в агрегаты | rebuild затронутого периода | -| Слишком высокая кардинальность aggregate | не тащить URL/title в основные агрегаты | -| Grafana продолжает читать raw tables | dashboard review и policy запрета raw для штатных панелей | -| Словари изменились после агрегации | rebuild или snapshot/history dimensions | -| Слишком много parts | контролировать batch insert, partitioning, merge health | -| Сложность `AggregatingMergeTree` | начинать с `SummingMergeTree` | +| Ошибочная привязка пользователя к оргструктуре | отчет unknown и ручная сверка источника | +| Ошибочная классификация приложения | поле `comment`, владелец справочника, review изменений | +| Ошибочная классификация домена | отдельный review доменного справочника и path-level backlog | +| Потеря историчности | на пилоте принимается current snapshot; для регламентных отчетов проектировать history отдельно | +| Unknown processes/domains растут | регулярный top unknown отчет | +| Dictionary stale | контроль `system.dictionaries` и reload после загрузки | +| Исправления справочников не попадают в старые агрегаты | формальный backfill | +| Использование классификации как наказательного инструмента | явно фиксировать аналитический, а не дисциплинарный характер метрик | diff --git a/scripts/detmir-support-run.sh b/scripts/detmir-support-run.sh index 944e880..b78deaa 100644 --- a/scripts/detmir-support-run.sh +++ b/scripts/detmir-support-run.sh @@ -4,8 +4,15 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DEFAULT_ENV_FILE="$SCRIPT_DIR/detmir-support.env" -ENV_FILE="${DETMIR_SUPPORT_ENV_FILE:-$DEFAULT_ENV_FILE}" +LOCAL_ENV_FILE="$SCRIPT_DIR/detmir-support.env" +USER_ENV_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/awatch-rus/detmir-support.env" +if [[ -n "${DETMIR_SUPPORT_ENV_FILE:-}" ]]; then + ENV_FILE="$DETMIR_SUPPORT_ENV_FILE" +elif [[ -f "$LOCAL_ENV_FILE" ]]; then + ENV_FILE="$LOCAL_ENV_FILE" +else + ENV_FILE="$USER_ENV_FILE" +fi if [[ -f "$ENV_FILE" ]]; then set -a @@ -71,7 +78,7 @@ fi DETMIR_SUPPORT_PVE_HOST="${DETMIR_SUPPORT_PVE_HOST:-10.10.10.2}" DETMIR_SUPPORT_AW_HOST="${DETMIR_SUPPORT_AW_HOST:-10.10.10.13}" -DETMIR_SUPPORT_WEB_HOST="${DETMIR_SUPPORT_WEB_HOST:-10.10.10.11}" +DETMIR_SUPPORT_WEB_HOST="${DETMIR_SUPPORT_WEB_HOST:-$DETMIR_SUPPORT_PVE_HOST}" DETMIR_SUPPORT_WINDOWS_HOST="${DETMIR_SUPPORT_WINDOWS_HOST:-192.168.100.18}" DETMIR_SUPPORT_PFSENSE_HOST="${DETMIR_SUPPORT_PFSENSE_HOST:-}" DETMIR_SUPPORT_PFSENSE_URL="${DETMIR_SUPPORT_PFSENSE_URL:-}" @@ -80,17 +87,30 @@ DETMIR_SUPPORT_OPENVPN_WEB_URL="${DETMIR_SUPPORT_OPENVPN_WEB_URL:-}" DETMIR_SUPPORT_SURICATA_HOST="${DETMIR_SUPPORT_SURICATA_HOST:-$DETMIR_SUPPORT_PVE_HOST}" DETMIR_SUPPORT_WEB_TLS_HOST="${DETMIR_SUPPORT_WEB_TLS_HOST:-$DETMIR_SUPPORT_WEB_HOST}" -DETMIR_SUPPORT_SSH_USER="${DETMIR_SUPPORT_SSH_USER:-root}" +DETMIR_SUPPORT_SSH_USER="${DETMIR_SUPPORT_SSH_USER:-igor}" +DETMIR_SUPPORT_PVE_SSH_USER="${DETMIR_SUPPORT_PVE_SSH_USER:-$DETMIR_SUPPORT_SSH_USER}" +DETMIR_SUPPORT_AW_SSH_USER="${DETMIR_SUPPORT_AW_SSH_USER:-$DETMIR_SUPPORT_SSH_USER}" +DETMIR_SUPPORT_WEB_SSH_USER="${DETMIR_SUPPORT_WEB_SSH_USER:-$DETMIR_SUPPORT_SSH_USER}" +DETMIR_SUPPORT_SURICATA_SSH_USER="${DETMIR_SUPPORT_SURICATA_SSH_USER:-$DETMIR_SUPPORT_PVE_SSH_USER}" +DETMIR_SUPPORT_SSH_PASSWORD="${DETMIR_SUPPORT_SSH_PASSWORD:-}" +DETMIR_SUPPORT_PVE_SSH_PASSWORD="${DETMIR_SUPPORT_PVE_SSH_PASSWORD:-$DETMIR_SUPPORT_SSH_PASSWORD}" +DETMIR_SUPPORT_AW_SSH_PASSWORD="${DETMIR_SUPPORT_AW_SSH_PASSWORD:-$DETMIR_SUPPORT_SSH_PASSWORD}" +DETMIR_SUPPORT_WEB_SSH_PASSWORD="${DETMIR_SUPPORT_WEB_SSH_PASSWORD:-$DETMIR_SUPPORT_SSH_PASSWORD}" +DETMIR_SUPPORT_SURICATA_SSH_PASSWORD="${DETMIR_SUPPORT_SURICATA_SSH_PASSWORD:-$DETMIR_SUPPORT_PVE_SSH_PASSWORD}" DETMIR_SUPPORT_SSH_IDENTITY="${DETMIR_SUPPORT_SSH_IDENTITY:-}" +DETMIR_SUPPORT_PVE_SSH_IDENTITY="${DETMIR_SUPPORT_PVE_SSH_IDENTITY:-$DETMIR_SUPPORT_SSH_IDENTITY}" +DETMIR_SUPPORT_AW_SSH_IDENTITY="${DETMIR_SUPPORT_AW_SSH_IDENTITY:-$DETMIR_SUPPORT_SSH_IDENTITY}" +DETMIR_SUPPORT_WEB_SSH_IDENTITY="${DETMIR_SUPPORT_WEB_SSH_IDENTITY:-$DETMIR_SUPPORT_SSH_IDENTITY}" +DETMIR_SUPPORT_SURICATA_SSH_IDENTITY="${DETMIR_SUPPORT_SURICATA_SSH_IDENTITY:-$DETMIR_SUPPORT_PVE_SSH_IDENTITY}" DETMIR_SUPPORT_SKIP_REMOTE="${DETMIR_SUPPORT_SKIP_REMOTE:-0}" DETMIR_SUPPORT_CONNECT_TIMEOUT="${DETMIR_SUPPORT_CONNECT_TIMEOUT:-8}" DETMIR_SUPPORT_PVE_SERVICES="${DETMIR_SUPPORT_PVE_SERVICES:-pve-cluster pvedaemon pvestatd pveproxy pvedaemon.service pvestatd.service}" DETMIR_SUPPORT_AW_SERVICES="${DETMIR_SUPPORT_AW_SERVICES:-activitywatch-server aw-server-rust}" -DETMIR_SUPPORT_WEB_SERVICES="${DETMIR_SUPPORT_WEB_SERVICES:-nginx apache2}" +DETMIR_SUPPORT_WEB_SERVICES="${DETMIR_SUPPORT_WEB_SERVICES:-nginx}" DETMIR_SUPPORT_SURICATA_SERVICES="${DETMIR_SUPPORT_SURICATA_SERVICES:-suricata}" DETMIR_SUPPORT_PVE_VM_IDS="${DETMIR_SUPPORT_PVE_VM_IDS:-}" -DETMIR_SUPPORT_PVE_BACKUP_DIRS="${DETMIR_SUPPORT_PVE_BACKUP_DIRS:-/var/lib/vz/dump}" +DETMIR_SUPPORT_PVE_BACKUP_DIRS="${DETMIR_SUPPORT_PVE_BACKUP_DIRS:-/var/lib/pve/local-btrfs/dump}" DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS="${DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS:-14}" DETMIR_SUPPORT_DISK_WARN_PERCENT="${DETMIR_SUPPORT_DISK_WARN_PERCENT:-85}" @@ -115,6 +135,7 @@ WARN_COUNT=0 FAIL_COUNT=0 SKIP_COUNT=0 REMOTE_OUTPUT="" +REMOTE_ERROR="" printf '%s\n' "timestamp;scope;section;status;check;result;action" > "$CSV_FILE" @@ -170,6 +191,68 @@ require_command() { fi } +remote_failure_summary() { + local fallback="${1:-Remote command failed}" + local detail="" + if [[ -n "$REMOTE_ERROR" ]]; then + detail="$REMOTE_ERROR" + elif [[ -n "$REMOTE_OUTPUT" ]]; then + detail="$REMOTE_OUTPUT" + else + detail="$fallback" + fi + detail="$(printf '%s\n' "$detail" | grep -Ev '^(Warning: Permanently added|$)' | head -n 1 || true)" + if [[ -z "$detail" ]]; then + detail="$fallback" + fi + sanitize "$detail" +} + +remote_user_for_host() { + local host="$1" + if [[ "$host" == "$DETMIR_SUPPORT_PVE_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_PVE_SSH_USER" + elif [[ "$host" == "$DETMIR_SUPPORT_AW_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_AW_SSH_USER" + elif [[ "$host" == "$DETMIR_SUPPORT_WEB_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_WEB_SSH_USER" + elif [[ "$host" == "$DETMIR_SUPPORT_SURICATA_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_SURICATA_SSH_USER" + else + printf '%s' "$DETMIR_SUPPORT_SSH_USER" + fi +} + +remote_password_for_host() { + local host="$1" + if [[ "$host" == "$DETMIR_SUPPORT_PVE_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_PVE_SSH_PASSWORD" + elif [[ "$host" == "$DETMIR_SUPPORT_AW_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_AW_SSH_PASSWORD" + elif [[ "$host" == "$DETMIR_SUPPORT_WEB_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_WEB_SSH_PASSWORD" + elif [[ "$host" == "$DETMIR_SUPPORT_SURICATA_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_SURICATA_SSH_PASSWORD" + else + printf '%s' "$DETMIR_SUPPORT_SSH_PASSWORD" + fi +} + +remote_identity_for_host() { + local host="$1" + if [[ "$host" == "$DETMIR_SUPPORT_PVE_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_PVE_SSH_IDENTITY" + elif [[ "$host" == "$DETMIR_SUPPORT_AW_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_AW_SSH_IDENTITY" + elif [[ "$host" == "$DETMIR_SUPPORT_WEB_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_WEB_SSH_IDENTITY" + elif [[ "$host" == "$DETMIR_SUPPORT_SURICATA_HOST" ]]; then + printf '%s' "$DETMIR_SUPPORT_SURICATA_SSH_IDENTITY" + else + printf '%s' "$DETMIR_SUPPORT_SSH_IDENTITY" + fi +} + run_remote() { local host="$1" shift @@ -179,20 +262,51 @@ run_remote() { return 125 fi + local ssh_user ssh_password ssh_identity + ssh_user="$(remote_user_for_host "$host")" + ssh_password="$(remote_password_for_host "$host")" + ssh_identity="$(remote_identity_for_host "$host")" + local -a ssh_opts=( - -o BatchMode=yes -o ConnectTimeout="$DETMIR_SUPPORT_CONNECT_TIMEOUT" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ) - if [[ -n "$DETMIR_SUPPORT_SSH_IDENTITY" ]]; then - ssh_opts=("-i" "$DETMIR_SUPPORT_SSH_IDENTITY" "${ssh_opts[@]}") + if [[ -n "$ssh_password" ]]; then + ssh_opts=(-o BatchMode=no "${ssh_opts[@]}") + else + ssh_opts=(-o BatchMode=yes "${ssh_opts[@]}") + fi + if [[ -n "$ssh_identity" ]]; then + ssh_opts=("-i" "$ssh_identity" "${ssh_opts[@]}") fi - if REMOTE_OUTPUT="$(printf '%s\n' "$remote_cmd" | ssh "${ssh_opts[@]}" "${DETMIR_SUPPORT_SSH_USER}@${host}" bash -s 2>&1)"; then + local stderr_file rc + stderr_file="$(mktemp)" + REMOTE_OUTPUT="" + REMOTE_ERROR="" + local -a ssh_cmd=(ssh "${ssh_opts[@]}" "${ssh_user}@${host}" bash -s) + if [[ -n "$ssh_password" ]]; then + if ! command -v sshpass >/dev/null 2>&1; then + REMOTE_ERROR="sshpass is required for password authentication" + rm -f "$stderr_file" + return 127 + fi + ssh_cmd=(sshpass -e "${ssh_cmd[@]}") + fi + + if REMOTE_OUTPUT="$(printf '%s\n' "$remote_cmd" | SSHPASS="$ssh_password" "${ssh_cmd[@]}" 2>"$stderr_file")"; then + REMOTE_ERROR="$(<"$stderr_file")" + rm -f "$stderr_file" + if echo "$REMOTE_ERROR" | grep -Eiq "Permission denied|Connection refused|No route to host|Connection timed out|Could not resolve hostname"; then + return 255 + fi return 0 fi - return "$?" + rc=$? + REMOTE_ERROR="$(<"$stderr_file")" + rm -f "$stderr_file" + return "$rc" } check_tcp() { @@ -271,14 +385,26 @@ check_remote_service_status() { local service for service in $services; do - if run_remote "$host" "systemctl is-active --quiet \"$service\""; then - record "$section" OK "$service" "$host active" "" + if run_remote "$host" "systemctl is-active \"$service\""; then + local state + state="$(echo "$REMOTE_OUTPUT" | tr -d '\r' | awk 'NF {print $1; exit}')" + if [[ "$state" == "active" ]]; then + record "$section" OK "$service" "$host active" "" + elif [[ -n "$state" ]]; then + record "$section" WARN "$service" "$host state=$state" "Check unit status" + else + local detail + detail="$(remote_failure_summary "Empty systemctl output")" + record "$section" SKIP "$service" "$host empty systemctl output; ${detail}" "Check remote SSH/auth and unit name" + fi else local rc=$? if (( rc == 125 )); then record "$section" SKIP "$service" "SSH skipped" "Provide SSH access or run locally" else - record "$section" WARN "$service" "$host inactive or unit missing" "Check unit status" + local detail + detail="$(remote_failure_summary "systemctl failed")" + record "$section" WARN "$service" "$host systemctl check failed; ${detail}" "Check remote SSH/auth and unit status" fi fi done @@ -290,6 +416,8 @@ check_node_load() { local load_line="$REMOTE_OUTPUT" if [[ -z "$load_line" ]]; then record "$section" SKIP "Node load" "Empty loadavg output" "Check /proc on remote" + elif ! [[ "$load_line" =~ ^[0-9]+([.][0-9]+)?[[:space:]][0-9]+([.][0-9]+)?[[:space:]][0-9]+([.][0-9]+)?$ ]]; then + record "$section" SKIP "Node load" "Unable to parse loadavg output ($(sanitize "$load_line"))" "Check remote shell output" else local avg1 avg5 avg15 avg1="${load_line%% *}" @@ -302,7 +430,9 @@ check_node_load() { if (( rc == 125 )); then record "$section" SKIP "Node load" "SSH skipped" "Provide SSH access or run locally" else - record "$section" WARN "Node load" "Cannot read /proc/loadavg" "Check remote shell access" + local detail + detail="$(remote_failure_summary "Cannot read /proc/loadavg")" + record "$section" WARN "Node load" "$host load check failed; ${detail}" "Check remote SSH/auth and /proc access" fi fi } @@ -315,7 +445,9 @@ check_disk() { raw_total="$(echo "$REMOTE_OUTPUT" | tr '\\t' ' ')" used="${raw_total%%\%*}" if ! [[ "$used" =~ ^[0-9]+$ ]]; then - record "$section" SKIP "disk $mount" "Unable to parse df output ($raw_total)" "Check mount and fs output" + local detail + detail="$(remote_failure_summary "Empty or invalid df output")" + record "$section" SKIP "disk $mount" "Unable to parse df output on ${host}:${mount}; ${detail}" "Check remote SSH/auth, mount and fs output" return fi if (( used >= DETMIR_SUPPORT_DISK_CRIT_PERCENT )); then @@ -330,7 +462,9 @@ check_disk() { if (( rc == 125 )); then record "$section" SKIP "disk $mount" "SSH skipped" "Provide SSH access" else - record "$section" FAIL "disk $mount" "df failed" "Check permissions and mount" + local detail + detail="$(remote_failure_summary "df failed")" + record "$section" FAIL "disk $mount" "$host df failed; ${detail}" "Check remote SSH/auth, permissions and mount" fi fi } @@ -356,6 +490,10 @@ EOF record "$section" WARN "backup" "No backup files found in ${path}" "Check backup schedule" return fi + if ! [[ "$REMOTE_OUTPUT" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + record "$section" SKIP "backup" "Unable to parse backup timestamp ($(sanitize "$REMOTE_OUTPUT"))" "Check remote SSH output and backup path" + return + fi local epoch_now age_days epoch_now="$(date -u +%s)" age_days=$(( (epoch_now - ${REMOTE_OUTPUT%.*}) / 86400 )) @@ -369,7 +507,9 @@ EOF if (( rc == 125 )); then record "$section" SKIP "backup" "SSH skipped" "Provide SSH access" else - record "$section" FAIL "backup" "Check failed" "Inspect permissions/path" + local detail + detail="$(remote_failure_summary "Backup check failed")" + record "$section" FAIL "backup" "$host backup check failed; ${detail}" "Inspect remote SSH/auth, permissions and path" fi fi } @@ -408,7 +548,7 @@ check_vms() { check_task_log() { local section="$1" host="$2" - if run_remote "$host" "journalctl -p 3 --no-pager -n 50 2>/dev/null | grep -Eci '(error|fail|critical|timeout)'"; then + if run_remote "$host" "journalctl -p 3 --no-pager -n 50 2>/dev/null | grep -Eci '(error|fail|critical|timeout)' || true"; then local errors="${REMOTE_OUTPUT//[$'\n']/ }" if ! [[ "$errors" =~ ^[0-9]+$ ]]; then record "$section" SKIP "task log" "Cannot parse journal count" "Check journald" @@ -487,14 +627,29 @@ check_suricata_service() { local section="$1" host="$2" check_remote_service_status "$section" "$host" "$DETMIR_SUPPORT_SURICATA_SERVICES" - if run_remote "$host" "command -v suricata >/dev/null 2>&1 && pgrep -af suricata >/dev/null"; then - record "$section" OK "Suricata process" "running" + if run_remote "$host" "systemctl is-active suricata 2>/dev/null || true"; then + local service_state + service_state="$(echo "$REMOTE_OUTPUT" | tr -d '\r' | awk 'NF {print $1; exit}')" + if [[ -n "$service_state" && "$service_state" != "active" ]]; then + record "$section" SKIP "Suricata process" "Skipped because suricata.service state=${service_state}" "Start Suricata only if IDS/IPS is required on this host" + return + fi + fi + + if run_remote "$host" "command -v suricata >/dev/null 2>&1 && pgrep -af suricata >/dev/null && echo running"; then + if [[ "$(echo "$REMOTE_OUTPUT" | awk 'NF {print $1; exit}')" == "running" ]]; then + record "$section" OK "Suricata process" "running" + else + record "$section" SKIP "Suricata process" "No running process reported" "Service is inactive or process is absent" + fi else local rc=$? if (( rc == 125 )); then record "$section" SKIP "Suricata process" "SSH skipped" "Provide SSH access" else - record "$section" WARN "Suricata process" "No running process or suricata missing" "Verify IDS/IPS host" + local detail + detail="$(remote_failure_summary "No running process or suricata missing")" + record "$section" WARN "Suricata process" "$host process check failed; ${detail}" "Verify IDS/IPS host and remote SSH/auth" fi fi } @@ -507,8 +662,9 @@ run_daily() { check_tcp "Доступность" "$DETMIR_SUPPORT_WEB_HOST" 443 "Web HTTPS" [[ -n "$DETMIR_SUPPORT_WINDOWS_HOST" ]] && check_tcp "Удаленный доступ" "$DETMIR_SUPPORT_WINDOWS_HOST" 3389 "Windows RDP" - check_http "Ключевые сервисы" "https://$DETMIR_SUPPORT_AW_HOST:5600" '^2[0-9][0-9]$' "AW API health" - check_http "Web" "https://$DETMIR_SUPPORT_WEB_TLS_HOST/" '^2[0-9][0-9]$' "Public HTTPS" + check_http "Ключевые сервисы" "http://$DETMIR_SUPPORT_AW_HOST:5600/api/0/settings/" '^2[0-9][0-9]$' "AW API health" + check_http "Web" "https://$DETMIR_SUPPORT_WEB_TLS_HOST/healthz" '^200$' "Gateway healthz" + check_http "Web" "https://$DETMIR_SUPPORT_WEB_TLS_HOST/" '^401$' "Gateway protected root" check_tls "TLS" "$DETMIR_SUPPORT_WEB_TLS_HOST" 443 "Web certificate" check_pfsense "Сеть" diff --git a/scripts/detmir-support.env.example b/scripts/detmir-support.env.example index 01f000a..b0ba9b9 100644 --- a/scripts/detmir-support.env.example +++ b/scripts/detmir-support.env.example @@ -1,7 +1,7 @@ ## Базовые узлы DETMIR_SUPPORT_PVE_HOST=10.10.10.2 DETMIR_SUPPORT_AW_HOST=10.10.10.13 -DETMIR_SUPPORT_WEB_HOST=10.10.10.11 +DETMIR_SUPPORT_WEB_HOST=10.10.10.2 DETMIR_SUPPORT_WINDOWS_HOST=192.168.100.18 ## Сетевые компоненты @@ -13,7 +13,16 @@ DETMIR_SUPPORT_SURICATA_HOST=${DETMIR_SUPPORT_PVE_HOST} DETMIR_SUPPORT_WEB_TLS_HOST=${DETMIR_SUPPORT_WEB_HOST} ## SSH/доступ -DETMIR_SUPPORT_SSH_USER=root +DETMIR_SUPPORT_SSH_USER=igor +# Для парольного SSH задайте переменные в локальном scripts/detmir-support.env. +# Не коммитьте реальные пароли в Git. +#DETMIR_SUPPORT_SSH_PASSWORD=change-me +#DETMIR_SUPPORT_PVE_SSH_USER=igor +#DETMIR_SUPPORT_PVE_SSH_PASSWORD=change-me +#DETMIR_SUPPORT_AW_SSH_USER=igor +#DETMIR_SUPPORT_AW_SSH_PASSWORD=change-me +#DETMIR_SUPPORT_WEB_SSH_USER=igor +#DETMIR_SUPPORT_WEB_SSH_PASSWORD=change-me #DETMIR_SUPPORT_SSH_IDENTITY=/path/to/key DETMIR_SUPPORT_SKIP_REMOTE=0 DETMIR_SUPPORT_CONNECT_TIMEOUT=8 @@ -21,12 +30,12 @@ DETMIR_SUPPORT_CONNECT_TIMEOUT=8 ## Сервисы и параметры DETMIR_SUPPORT_PVE_SERVICES=pve-cluster pvedaemon pvestatd pveproxy DETMIR_SUPPORT_AW_SERVICES=activitywatch-server aw-server-rust -DETMIR_SUPPORT_WEB_SERVICES=nginx apache2 +DETMIR_SUPPORT_WEB_SERVICES=nginx DETMIR_SUPPORT_SURICATA_SERVICES=suricata ## VM и резервные копии DETMIR_SUPPORT_PVE_VM_IDS="101 102 103" -DETMIR_SUPPORT_PVE_BACKUP_DIRS=/var/lib/vz/dump +DETMIR_SUPPORT_PVE_BACKUP_DIRS=/var/lib/pve/local-btrfs/dump DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS=14 ## Пороги