Document and harden DetMir production maintenance
This commit is contained in:
igor04091968
@@ -279,6 +279,8 @@ collectors.
|
||||
[docs/REVIEW_CHECKLIST_RU.md](docs/REVIEW_CHECKLIST_RU.md).
|
||||
- Operational validation runbook:
|
||||
[docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md).
|
||||
- Daily maintenance runbook with optional Pollinations AI evidence assistant:
|
||||
[docs/DAILY_MAINTENANCE_RU.md](docs/DAILY_MAINTENANCE_RU.md).
|
||||
- Operational maturity harness:
|
||||
[docs/OPERATIONAL_MATURITY_RU.md](docs/OPERATIONAL_MATURITY_RU.md).
|
||||
- Residual risks register:
|
||||
|
||||
Generated
+6
-57
@@ -169,12 +169,12 @@ name = "aw-1c-ingest"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"calamine",
|
||||
"chrono",
|
||||
"clap",
|
||||
"csv",
|
||||
"filetime",
|
||||
"fs2",
|
||||
"quick-xml",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"serde",
|
||||
@@ -182,6 +182,7 @@ dependencies = [
|
||||
"serde_yaml",
|
||||
"sha1",
|
||||
"tempfile",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -397,33 +398,12 @@ version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
||||
|
||||
[[package]]
|
||||
name = "calamine"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a3a315226fdc5b1c3e33521073e1712a05944bc0664d665ff1f6ff0396334da"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"codepage",
|
||||
"encoding_rs",
|
||||
"log",
|
||||
"quick-xml",
|
||||
"serde",
|
||||
"zip 0.6.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.63"
|
||||
@@ -540,15 +520,6 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "codepage"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
@@ -949,15 +920,6 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
@@ -1801,11 +1763,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.31.0"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
|
||||
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
@@ -1936,7 +1897,7 @@ dependencies = [
|
||||
"sha2",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"zip 2.4.2",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2609,7 +2570,7 @@ dependencies = [
|
||||
"sha2",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"zip 2.4.2",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3455,18 +3416,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"flate2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.4.2"
|
||||
|
||||
+2
-1
@@ -71,7 +71,6 @@ publish = false
|
||||
adk-rust = { version = "0.9.1", default-features = false }
|
||||
anyhow = "1"
|
||||
base64 = "0.22"
|
||||
calamine = "=0.24.0"
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] }
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
csv = "1"
|
||||
@@ -81,6 +80,7 @@ detmir-state = { path = "crates/detmir-state" }
|
||||
filetime = "0.2"
|
||||
fs2 = "0.4"
|
||||
notify = "6"
|
||||
quick-xml = "0.41"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
|
||||
regex = "1"
|
||||
rusqlite = "0.32"
|
||||
@@ -93,4 +93,5 @@ tempfile = "3"
|
||||
tiny_http = "0.12"
|
||||
url = "2"
|
||||
urlencoding = "2"
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
windows-sys = "0.59"
|
||||
|
||||
@@ -8,12 +8,12 @@ publish = false
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
calamine.workspace = true
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
csv.workspace = true
|
||||
filetime.workspace = true
|
||||
fs2.workspace = true
|
||||
quick-xml.workspace = true
|
||||
regex.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
@@ -21,3 +21,4 @@ serde_json.workspace = true
|
||||
serde_yaml.workspace = true
|
||||
sha1.workspace = true
|
||||
tempfile.workspace = true
|
||||
zip.workspace = true
|
||||
|
||||
@@ -6,16 +6,19 @@ use std::process::Command;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use calamine::{Data, Reader, open_workbook_auto};
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use clap::Parser;
|
||||
use filetime::{FileTime, set_file_times};
|
||||
use fs2::FileExt;
|
||||
use quick_xml::Reader as XmlReader;
|
||||
use quick_xml::XmlVersion;
|
||||
use quick_xml::events::{BytesStart, Event};
|
||||
use regex::Regex;
|
||||
use reqwest::blocking::Client;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
use sha1::{Digest, Sha1};
|
||||
use zip::ZipArchive;
|
||||
|
||||
const DATASETS: &[&str] = &[
|
||||
"documents",
|
||||
@@ -1103,15 +1106,14 @@ fn load_registry(
|
||||
}
|
||||
|
||||
fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
|
||||
let mut workbook =
|
||||
open_workbook_auto(path).with_context(|| format!("open xlsx {}", path.display()))?;
|
||||
let workbook = read_xlsx_workbook(path)?;
|
||||
let now = Utc::now()
|
||||
.naive_utc()
|
||||
.format("%Y-%m-%d %H:%M:%S")
|
||||
.to_string();
|
||||
let mut tax_map: HashMap<String, (String, String)> = HashMap::new();
|
||||
if let Ok(range) = workbook.worksheet_range("Лист2") {
|
||||
for row in range.rows().skip(2) {
|
||||
if let Some(rows) = workbook.get("Лист2") {
|
||||
for row in rows.iter().skip(2) {
|
||||
let company_name = cell_text(row.get(1));
|
||||
if company_name.is_empty() {
|
||||
continue;
|
||||
@@ -1123,10 +1125,10 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let Ok(range) = workbook.worksheet_range("ОСНОВНОЙ") else {
|
||||
let Some(sheet_rows) = workbook.get("ОСНОВНОЙ") else {
|
||||
return Ok(out);
|
||||
};
|
||||
let rows: Vec<_> = range.rows().collect();
|
||||
let rows: Vec<_> = sheet_rows.iter().collect();
|
||||
if rows.len() < 3 {
|
||||
return Ok(out);
|
||||
}
|
||||
@@ -1134,7 +1136,8 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
|
||||
let manager_headers: Vec<String> = rows[1].iter().map(|v| cell_text(Some(v))).collect();
|
||||
let mut specs = Vec::<(usize, String, Option<usize>, String, String)>::new();
|
||||
let mut current: Option<(usize, String, String)> = None;
|
||||
for idx in 1..manager_headers.len() {
|
||||
let header_len = top_headers.len().max(manager_headers.len());
|
||||
for idx in 1..header_len {
|
||||
let manager = manager_headers.get(idx).cloned().unwrap_or_default();
|
||||
let top = top_headers.get(idx).cloned().unwrap_or_default();
|
||||
if !manager.is_empty() {
|
||||
@@ -1194,26 +1197,291 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn cell_text(value: Option<&Data>) -> String {
|
||||
match value {
|
||||
Some(Data::String(v)) => v.trim().to_string(),
|
||||
Some(Data::Float(v)) => {
|
||||
if v.fract() == 0.0 {
|
||||
format!("{}", *v as i64)
|
||||
type XlsxWorkbook = HashMap<String, Vec<Vec<String>>>;
|
||||
|
||||
fn read_xlsx_workbook(path: &Path) -> Result<XlsxWorkbook> {
|
||||
let file = File::open(path).with_context(|| format!("open xlsx {}", path.display()))?;
|
||||
let mut archive =
|
||||
ZipArchive::new(file).with_context(|| format!("read xlsx {}", path.display()))?;
|
||||
let shared_strings = read_xlsx_shared_strings(&mut archive)?;
|
||||
let sheets = read_xlsx_sheet_paths(&mut archive)?;
|
||||
let mut workbook = XlsxWorkbook::new();
|
||||
for (name, sheet_path) in sheets {
|
||||
let xml = read_zip_text(&mut archive, &sheet_path)
|
||||
.with_context(|| format!("read sheet {name} from {}", path.display()))?;
|
||||
let rows = parse_xlsx_sheet(&xml, &shared_strings)
|
||||
.with_context(|| format!("parse sheet {name} from {}", path.display()))?;
|
||||
workbook.insert(name, rows);
|
||||
}
|
||||
Ok(workbook)
|
||||
}
|
||||
|
||||
fn read_xlsx_shared_strings<R: Read + std::io::Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
) -> Result<Vec<String>> {
|
||||
let Some(xml) = read_zip_text_optional(archive, "xl/sharedStrings.xml")? else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let mut reader = XmlReader::from_str(&xml);
|
||||
reader.config_mut().trim_text(false);
|
||||
let mut buf = Vec::new();
|
||||
let mut strings = Vec::new();
|
||||
let mut in_si = false;
|
||||
let mut in_t = false;
|
||||
let mut current = String::new();
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf)? {
|
||||
Event::Start(e) => match local_name(e.name().as_ref()) {
|
||||
b"si" => {
|
||||
in_si = true;
|
||||
current.clear();
|
||||
}
|
||||
b"t" if in_si => in_t = true,
|
||||
_ => {}
|
||||
},
|
||||
Event::Text(e) if in_si && in_t => current.push_str(e.decode()?.as_ref()),
|
||||
Event::End(e) => match local_name(e.name().as_ref()) {
|
||||
b"t" => in_t = false,
|
||||
b"si" => {
|
||||
strings.push(current.clone());
|
||||
in_si = false;
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Event::Eof => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(strings)
|
||||
}
|
||||
|
||||
fn read_xlsx_sheet_paths<R: Read + std::io::Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
) -> Result<Vec<(String, String)>> {
|
||||
let workbook_xml = read_zip_text(archive, "xl/workbook.xml")?;
|
||||
let rels_xml = read_zip_text(archive, "xl/_rels/workbook.xml.rels")?;
|
||||
let rels = parse_xlsx_relationships(&rels_xml)?;
|
||||
let mut reader = XmlReader::from_str(&workbook_xml);
|
||||
reader.config_mut().trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut sheets = Vec::new();
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf)? {
|
||||
Event::Empty(e) | Event::Start(e) if local_name(e.name().as_ref()) == b"sheet" => {
|
||||
let name = xml_attr(&reader, &e, b"name")?.unwrap_or_default();
|
||||
let mut rel_id = xml_attr(&reader, &e, b"r:id")?.unwrap_or_default();
|
||||
if rel_id.is_empty() {
|
||||
rel_id = xml_attr(&reader, &e, b"id")?.unwrap_or_default();
|
||||
}
|
||||
if !name.is_empty() && !rel_id.is_empty() {
|
||||
let target = rels
|
||||
.get(&rel_id)
|
||||
.with_context(|| format!("missing workbook relationship {rel_id}"))?;
|
||||
sheets.push((name, normalize_xlsx_target(target)?));
|
||||
}
|
||||
}
|
||||
Event::Eof => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(sheets)
|
||||
}
|
||||
|
||||
fn parse_xlsx_relationships(xml: &str) -> Result<HashMap<String, String>> {
|
||||
let mut reader = XmlReader::from_str(xml);
|
||||
reader.config_mut().trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut rels = HashMap::new();
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf)? {
|
||||
Event::Empty(e) | Event::Start(e)
|
||||
if local_name(e.name().as_ref()) == b"Relationship" =>
|
||||
{
|
||||
let id = xml_attr(&reader, &e, b"Id")?.unwrap_or_default();
|
||||
let target = xml_attr(&reader, &e, b"Target")?.unwrap_or_default();
|
||||
if !id.is_empty() && !target.is_empty() {
|
||||
rels.insert(id, target);
|
||||
}
|
||||
}
|
||||
Event::Eof => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(rels)
|
||||
}
|
||||
|
||||
fn parse_xlsx_sheet(xml: &str, shared_strings: &[String]) -> Result<Vec<Vec<String>>> {
|
||||
let mut reader = XmlReader::from_str(xml);
|
||||
reader.config_mut().trim_text(false);
|
||||
let mut buf = Vec::new();
|
||||
let mut rows = Vec::<Vec<String>>::new();
|
||||
let mut current_row = Vec::<String>::new();
|
||||
let mut current_cell: Option<(usize, String, String)> = None;
|
||||
let mut in_value = false;
|
||||
let mut in_inline_text = false;
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf)? {
|
||||
Event::Empty(e) => match local_name(e.name().as_ref()) {
|
||||
b"row" => rows.push(Vec::new()),
|
||||
b"c" => {
|
||||
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
|
||||
let column = cell_ref
|
||||
.chars()
|
||||
.take_while(|ch| ch.is_ascii_alphabetic())
|
||||
.collect::<String>();
|
||||
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
|
||||
if current_row.len() <= col {
|
||||
current_row.resize(col + 1, String::new());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Event::Start(e) => match local_name(e.name().as_ref()) {
|
||||
b"row" => current_row.clear(),
|
||||
b"c" => {
|
||||
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
|
||||
let cell_type = xml_attr(&reader, &e, b"t")?.unwrap_or_default();
|
||||
let column = cell_ref
|
||||
.chars()
|
||||
.take_while(|ch| ch.is_ascii_alphabetic())
|
||||
.collect::<String>();
|
||||
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
|
||||
current_cell = Some((col, cell_type, String::new()));
|
||||
}
|
||||
b"v" => in_value = true,
|
||||
b"t" if current_cell
|
||||
.as_ref()
|
||||
.is_some_and(|(_, kind, _)| kind == "inlineStr") =>
|
||||
{
|
||||
in_inline_text = true;
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Event::Text(e) if in_value || in_inline_text => {
|
||||
if let Some((_, _, value)) = current_cell.as_mut() {
|
||||
value.push_str(e.decode()?.as_ref());
|
||||
}
|
||||
}
|
||||
Event::End(e) => match local_name(e.name().as_ref()) {
|
||||
b"v" => in_value = false,
|
||||
b"t" => in_inline_text = false,
|
||||
b"c" => {
|
||||
if let Some((col, cell_type, raw)) = current_cell.take() {
|
||||
let value = xlsx_cell_value(&cell_type, &raw, shared_strings);
|
||||
if current_row.len() <= col {
|
||||
current_row.resize(col + 1, String::new());
|
||||
}
|
||||
current_row[col] = value;
|
||||
}
|
||||
}
|
||||
b"row" => rows.push(current_row.clone()),
|
||||
_ => {}
|
||||
},
|
||||
Event::Eof => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
fn read_zip_text<R: Read + std::io::Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
path: &str,
|
||||
) -> Result<String> {
|
||||
read_zip_text_optional(archive, path)?.with_context(|| format!("missing xlsx entry {path}"))
|
||||
}
|
||||
|
||||
fn read_zip_text_optional<R: Read + std::io::Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
path: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let Ok(mut file) = archive.by_name(path) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut text = String::new();
|
||||
file.read_to_string(&mut text)
|
||||
.with_context(|| format!("read xlsx entry {path}"))?;
|
||||
Ok(Some(text))
|
||||
}
|
||||
|
||||
fn normalize_xlsx_target(target: &str) -> Result<String> {
|
||||
let normalized = target.replace('\\', "/");
|
||||
if normalized.split('/').any(|part| part == "..") {
|
||||
bail!("unsafe xlsx relationship target: {target}");
|
||||
}
|
||||
let path = normalized.trim_start_matches('/');
|
||||
if path.starts_with("xl/") {
|
||||
Ok(path.to_string())
|
||||
} else {
|
||||
Ok(format!("xl/{path}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn xml_attr(reader: &XmlReader<&[u8]>, e: &BytesStart<'_>, key: &[u8]) -> Result<Option<String>> {
|
||||
for attr in e.attributes() {
|
||||
let attr = attr?;
|
||||
if attr.key.as_ref() == key {
|
||||
return Ok(Some(
|
||||
attr.decoded_and_normalized_value(XmlVersion::Implicit1_0, reader.decoder())?
|
||||
.into_owned(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn local_name(name: &[u8]) -> &[u8] {
|
||||
name.rsplit(|b| *b == b':').next().unwrap_or(name)
|
||||
}
|
||||
|
||||
fn xlsx_column_index(column: &str) -> Option<usize> {
|
||||
let mut value = 0usize;
|
||||
for byte in column.bytes() {
|
||||
if !byte.is_ascii_alphabetic() {
|
||||
return None;
|
||||
}
|
||||
value = value * 26 + usize::from(byte.to_ascii_uppercase() - b'A' + 1);
|
||||
}
|
||||
value.checked_sub(1)
|
||||
}
|
||||
|
||||
fn xlsx_cell_value(cell_type: &str, raw: &str, shared_strings: &[String]) -> String {
|
||||
match cell_type {
|
||||
"s" => raw
|
||||
.parse::<usize>()
|
||||
.ok()
|
||||
.and_then(|idx| shared_strings.get(idx))
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
"b" => {
|
||||
if raw.trim() == "1" {
|
||||
"true".to_string()
|
||||
} else {
|
||||
v.to_string()
|
||||
"false".to_string()
|
||||
}
|
||||
}
|
||||
Some(Data::Int(v)) => v.to_string(),
|
||||
Some(Data::Bool(v)) => v.to_string(),
|
||||
Some(Data::DateTime(v)) => v.to_string(),
|
||||
Some(Data::DateTimeIso(v)) => v.trim().to_string(),
|
||||
Some(Data::DurationIso(v)) => v.trim().to_string(),
|
||||
Some(Data::Empty) | None => String::new(),
|
||||
Some(other) => other.to_string(),
|
||||
_ => raw.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn cell_text(value: Option<&String>) -> String {
|
||||
value
|
||||
.map(|v| {
|
||||
let trimmed = v.trim();
|
||||
if let Ok(number) = trimmed.parse::<f64>() {
|
||||
if number.fract() == 0.0 {
|
||||
return format!("{}", number as i64);
|
||||
}
|
||||
}
|
||||
trimmed.to_string()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn archive_or_delete(
|
||||
config: &RawConfig,
|
||||
dataset: &str,
|
||||
@@ -1507,4 +1775,81 @@ mod tests {
|
||||
fn split_sql_ignores_empty_chunks() {
|
||||
assert_eq!(split_sql_statements("SELECT 1; ; SELECT 2;").len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_registry_xlsx_with_inline_strings() -> Result<()> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("registry.xlsx");
|
||||
let file = File::create(&path)?;
|
||||
let mut zip = zip::ZipWriter::new(file);
|
||||
let options = zip::write::SimpleFileOptions::default()
|
||||
.compression_method(zip::CompressionMethod::Stored);
|
||||
for (name, xml) in [
|
||||
(
|
||||
"xl/workbook.xml",
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets>
|
||||
<sheet name="Лист2" sheetId="1" r:id="rId1"/>
|
||||
<sheet name="ОСНОВНОЙ" sheetId="2" r:id="rId2"/>
|
||||
</sheets>
|
||||
</workbook>"#,
|
||||
),
|
||||
(
|
||||
"xl/_rels/workbook.xml.rels",
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Target="worksheets/sheet1.xml"/>
|
||||
<Relationship Id="rId2" Target="worksheets/sheet2.xml"/>
|
||||
</Relationships>"#,
|
||||
),
|
||||
(
|
||||
"xl/worksheets/sheet1.xml",
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="1"/>
|
||||
<row r="2"/>
|
||||
<row r="3">
|
||||
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
|
||||
<c r="C3" t="inlineStr"><is><t>1101000000</t></is></c>
|
||||
<c r="D3" t="inlineStr"><is><t>110101001</t></is></c>
|
||||
</row>
|
||||
</sheetData>
|
||||
</worksheet>"#,
|
||||
),
|
||||
(
|
||||
"xl/worksheets/sheet2.xml",
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="1">
|
||||
<c r="C1" t="inlineStr"><is><t>Ключевой контур</t></is></c>
|
||||
</row>
|
||||
<row r="2">
|
||||
<c r="B2" t="inlineStr"><is><t>Иванов И.И.</t></is></c>
|
||||
</row>
|
||||
<row r="3">
|
||||
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
|
||||
<c r="C3" t="inlineStr"><is><t>ЕСТЬ</t></is></c>
|
||||
</row>
|
||||
</sheetData>
|
||||
</worksheet>"#,
|
||||
),
|
||||
] {
|
||||
zip.start_file(name, options)?;
|
||||
zip.write_all(xml.as_bytes())?;
|
||||
}
|
||||
zip.finish()?;
|
||||
|
||||
let rows = parse_registry_xlsx(&path)?;
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(rows[0]["company_name"], "ООО Ромашка");
|
||||
assert_eq!(rows[0]["company_key"], "ООО РОМАШКА");
|
||||
assert_eq!(rows[0]["assignee_name"], "Иванов И.И.");
|
||||
assert_eq!(rows[0]["key_contour"], 1);
|
||||
assert_eq!(rows[0]["inn"], "1101000000");
|
||||
assert_eq!(rows[0]["kpp"], "110101001");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4037,6 +4037,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
|
||||
.unwrap_or_else(|| "powershell_primary".to_string());
|
||||
let worktime_legacy_fallback_enabled =
|
||||
json_bool(&config, &["collectors", "worktimeLegacyFallbackEnabled"]).unwrap_or(true);
|
||||
let window_enabled = json_bool(&config, &["collectors", "windowEnabled"]).unwrap_or(true);
|
||||
let file_ops_enabled = json_bool(&config, &["collectors", "fileOpsEnabled"]).unwrap_or(true);
|
||||
let file_ops_mode = json_string(&config, &["collectors", "fileOpsMode"])
|
||||
.unwrap_or_else(|| "powershell_primary".to_string());
|
||||
@@ -4047,7 +4048,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
|
||||
args.interactive_max_age_seconds,
|
||||
15,
|
||||
);
|
||||
let bucket_checks = vec![
|
||||
let mut bucket_checks = vec![
|
||||
worktime_bucket.clone(),
|
||||
get_bucket_health(
|
||||
&api_base,
|
||||
@@ -4055,19 +4056,21 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
|
||||
args.interactive_max_age_seconds,
|
||||
15,
|
||||
),
|
||||
get_bucket_health(
|
||||
];
|
||||
if window_enabled {
|
||||
bucket_checks.push(get_bucket_health(
|
||||
&api_base,
|
||||
&format!("aw-watcher-window_{aw_hostname}"),
|
||||
args.interactive_max_age_seconds,
|
||||
15,
|
||||
),
|
||||
get_bucket_health(
|
||||
&api_base,
|
||||
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
|
||||
args.interactive_max_age_seconds,
|
||||
15,
|
||||
),
|
||||
];
|
||||
));
|
||||
}
|
||||
bucket_checks.push(get_bucket_health(
|
||||
&api_base,
|
||||
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
|
||||
args.interactive_max_age_seconds,
|
||||
15,
|
||||
));
|
||||
let interactive_stale = bucket_checks
|
||||
.iter()
|
||||
.skip(1)
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
# Автономная проверка AWatch-rus
|
||||
|
||||
Документ описывает единый validation entry point для длительной
|
||||
эксплуатационной проверки AWatch-rus/DetMir без ручного надзора.
|
||||
|
||||
Цель framework - не менять production runtime, а регулярно доказывать состояние
|
||||
репозитория, release artifacts, operational maturity, runtime health и evidence.
|
||||
|
||||
## Запуск
|
||||
|
||||
Из корня репозитория:
|
||||
|
||||
```bash
|
||||
scripts/run_full_validation.sh --profile full
|
||||
```
|
||||
|
||||
Профили:
|
||||
|
||||
- `quick` - repository, JSON/YAML/Markdown, retention/recovery docs, secret and
|
||||
contract checks.
|
||||
- `standard` - `quick` плюс operational maturity, deployment readiness, pilot
|
||||
validation, release artifact checks, binary parity self-test.
|
||||
- `full` - `standard` плюс Rust, Cargo и dependency hygiene gates.
|
||||
- `runtime` - `standard` с акцентом на runtime diagnostics и health endpoints.
|
||||
|
||||
Для unattended запуска можно задать каталог reports:
|
||||
|
||||
```bash
|
||||
AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation \
|
||||
AW_VALIDATION_RETENTION_DAYS=45 \
|
||||
scripts/run_full_validation.sh --profile standard
|
||||
```
|
||||
|
||||
## Production evidence inputs
|
||||
|
||||
Framework не собирает production binary parity evidence самостоятельно. Это
|
||||
намеренное ограничение: оператор сначала формирует approved evidence JSON, затем
|
||||
gate проверяет его в репозитории.
|
||||
|
||||
```bash
|
||||
PRODUCTION_BINARY_PARITY_EVIDENCE=/var/lib/activitywatch/evidence/production-binary-parity.json \
|
||||
scripts/run_full_validation.sh --profile standard
|
||||
```
|
||||
|
||||
Release evidence проверяется только когда явно указан каталог evidence package:
|
||||
|
||||
```bash
|
||||
AW_RELEASE_EVIDENCE_DIR=/var/lib/activitywatch/release-evidence/rc-1 \
|
||||
scripts/run_full_validation.sh --profile standard
|
||||
```
|
||||
|
||||
## Runtime diagnostics
|
||||
|
||||
Каждый запуск собирает machine-readable диагностику:
|
||||
|
||||
- CPU/load average;
|
||||
- RAM/swap;
|
||||
- disk usage;
|
||||
- размеры известных runtime paths;
|
||||
- queue directories;
|
||||
- systemd unit state, restart count, exit status;
|
||||
- ActivityWatch, Worktime, Grafana, Prometheus, ClickHouse endpoints.
|
||||
- TLS certificate expiry for hosts listed in `AW_VALIDATION_TLS_HOSTS`.
|
||||
|
||||
Endpoint defaults безопасны для локального запуска:
|
||||
|
||||
- `AW_VALIDATION_ACTIVITYWATCH_URL`, default
|
||||
`http://127.0.0.1:5600/api/0/info`;
|
||||
- `AW_VALIDATION_WORKTIME_URL`, default `http://127.0.0.1:5610/healthz`;
|
||||
- `AW_VALIDATION_GRAFANA_URL`;
|
||||
- `AW_VALIDATION_PROMETHEUS_URL`;
|
||||
- `AW_VALIDATION_CLICKHOUSE_URL`.
|
||||
- `AW_VALIDATION_TLS_HOSTS`, comma-separated `host:port` list.
|
||||
|
||||
ActivityWatch и Worktime по умолчанию проверяются на localhost. Grafana,
|
||||
Prometheus, ClickHouse и TLS hosts проверяются только если заданы. Если заданный
|
||||
endpoint недоступен, создается machine-readable alert.
|
||||
|
||||
## Reports
|
||||
|
||||
Каждый запуск пишет reports в:
|
||||
|
||||
```text
|
||||
output/validation/history/<timestamp>/
|
||||
output/validation/latest/
|
||||
```
|
||||
|
||||
Файлы:
|
||||
|
||||
- `validation-report.json`;
|
||||
- `validation-report.md`;
|
||||
- `production-health.json`;
|
||||
- `deployment-health.json`;
|
||||
- `runtime-health.json`;
|
||||
- `release-evidence.json`;
|
||||
- `operational-evidence.json`;
|
||||
- `recovery-evidence.json`;
|
||||
- `build-evidence.json`;
|
||||
- `operational-dashboard.json`;
|
||||
- `operational-dashboard.md`.
|
||||
|
||||
Все JSON reports содержат timestamp, Git SHA, host identifier, profile,
|
||||
validation outcome и evidence summary.
|
||||
|
||||
## Regression detection
|
||||
|
||||
Framework сравнивает текущий запуск с предыдущим report из history и фиксирует:
|
||||
|
||||
- новые failures;
|
||||
- восстановленные failures;
|
||||
- ухудшение длительности checks;
|
||||
- изменение `Cargo.lock`;
|
||||
- configuration drift;
|
||||
- unexpected release artifact changes;
|
||||
- disk/database/log path growth above threshold;
|
||||
- новые alerts.
|
||||
|
||||
Первый запуск создает baseline. Дальнейшие запуски пишут regression summary в
|
||||
`validation-report.json` и `validation-report.md`.
|
||||
|
||||
## Alert rules
|
||||
|
||||
Alerts создаются при:
|
||||
|
||||
- failed validation;
|
||||
- недоступном настроенном health endpoint;
|
||||
- превышении disk threshold;
|
||||
- превышении memory threshold;
|
||||
- failed systemd unit;
|
||||
- restart count больше нуля;
|
||||
- queue backlog выше threshold;
|
||||
- отсутствии production binary parity evidence;
|
||||
- отсутствии release evidence directory;
|
||||
- expired или missing backup evidence path, если он задан;
|
||||
- secret pattern check failure;
|
||||
- configuration drift;
|
||||
- unexpected artifact changes;
|
||||
- certificate expiry within `AW_VALIDATION_CERT_EXPIRY_WARN_DAYS`.
|
||||
|
||||
Thresholds:
|
||||
|
||||
```bash
|
||||
AW_VALIDATION_DISK_USED_WARN_PCT=85
|
||||
AW_VALIDATION_DISK_USED_CRIT_PCT=95
|
||||
AW_VALIDATION_MEMORY_USED_WARN_PCT=90
|
||||
AW_VALIDATION_MEMORY_USED_CRIT_PCT=97
|
||||
AW_VALIDATION_CERT_EXPIRY_WARN_DAYS=14
|
||||
AW_VALIDATION_PATH_GROWTH_WARN_BYTES=1073741824
|
||||
AW_VALIDATION_PATH_GROWTH_WARN_RATIO=1.25
|
||||
AW_VALIDATION_QUEUE_FILES_WARN=1000
|
||||
AW_VALIDATION_BACKUP_EVIDENCE_PATH=/var/lib/activitywatch/evidence/latest-backup.json
|
||||
AW_VALIDATION_BACKUP_EVIDENCE_MAX_AGE_DAYS=7
|
||||
```
|
||||
|
||||
## Scheduling
|
||||
|
||||
Пример systemd timer должен выполняться вне hot path и не должен включать DLP,
|
||||
Loki или always-on Velociraptor:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/mnt/usb_hdd2/Projects/ActivityWatch-Russian
|
||||
Environment=AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation
|
||||
Environment=AW_VALIDATION_RETENTION_DAYS=45
|
||||
ExecStart=/mnt/usb_hdd2/Projects/ActivityWatch-Russian/scripts/run_full_validation.sh --profile standard
|
||||
```
|
||||
|
||||
Для weekly deep validation использовать `--profile full`, потому что Rust и
|
||||
dependency gates могут быть дорогими.
|
||||
|
||||
## Failure investigation
|
||||
|
||||
1. Открыть `output/validation/latest/validation-report.md`.
|
||||
2. Проверить `Alerts`.
|
||||
3. Для failed command смотреть `validation-report.json`: `stdout`, `stderr`,
|
||||
`exit_code`, `timed_out`.
|
||||
4. Если alert относится к runtime endpoint, проверить соответствующий service
|
||||
через runbook [эксплуатационной проверки](OPERATIONS_VALIDATION_RUNBOOK_RU.md).
|
||||
5. Если alert относится к binary parity, обновить production evidence JSON и
|
||||
повторить запуск.
|
||||
6. Если alert относится к release evidence, проверить controlled release package
|
||||
через registry runbook.
|
||||
|
||||
## History retention
|
||||
|
||||
История хранится в `history/<timestamp>/`. Старые запуски удаляются по
|
||||
`AW_VALIDATION_RETENTION_DAYS` или `--retention-days`.
|
||||
|
||||
Минимальное значение retention: 1 день. Для production рекомендуется 30-45
|
||||
дней, чтобы видеть недельные тренды по runtime failures, restart frequency,
|
||||
disk growth и validation score.
|
||||
@@ -0,0 +1,310 @@
|
||||
# Ежедневное обслуживание AWatch-rus
|
||||
|
||||
Дата актуализации: 2026-07-02
|
||||
|
||||
Документ описывает ежедневный операторский цикл обслуживания AWatch-rus /
|
||||
DetMir и безопасное использование Pollinations AI как вспомогательного
|
||||
аналитика evidence-артефактов. Pollinations не является production runtime
|
||||
dependency, не принимает решения автоматически и не имеет прямого доступа к
|
||||
customer contour.
|
||||
|
||||
## Цели
|
||||
|
||||
- Подтвердить, что сервисы отвечают и не находятся в скрытой деградации.
|
||||
- Проверить свежесть данных, состояние очередей, timers, retention и readiness
|
||||
evidence.
|
||||
- Сформировать короткий дневной maintenance summary.
|
||||
- Использовать Pollinations только для сжатия и первичного анализа локальных
|
||||
JSON/Markdown evidence без секретов.
|
||||
|
||||
## Guardrails
|
||||
|
||||
- Не включать heavy DLP, Loki или always-on Velociraptor в рамках ежедневного
|
||||
обслуживания.
|
||||
- DLP runtime для DetMir production считать нормальным в light/disabled profile,
|
||||
если оператор отдельно не принял решение о включении.
|
||||
- Не удалять ClickHouse, Grafana, DLP evidence, Hayabusa archives, Windows
|
||||
queues, release evidence и diagnostic bundles вручную без отдельного change.
|
||||
- Pollinations failure не является отказом AWatch-rus. Если AI недоступен,
|
||||
ежедневное обслуживание продолжается по обычному runbook.
|
||||
- Не отправлять в Pollinations secrets, private env files, bearer tokens,
|
||||
пароли, raw customer evidence, user identifiers, скриншоты с ПДн или полные
|
||||
журналы production.
|
||||
|
||||
## Ежедневный цикл
|
||||
|
||||
### 1. Быстрый статус сервисов
|
||||
|
||||
Проверить локальный health/readiness и failed units:
|
||||
|
||||
```bash
|
||||
curl -fsS http://127.0.0.1:5600/api/0/info >/dev/null
|
||||
curl -fsS http://127.0.0.1:5610/healthz >/dev/null
|
||||
systemctl --failed --no-pager
|
||||
systemctl list-timers \
|
||||
aw-prune-local-state.timer \
|
||||
aw-db-maintenance.timer \
|
||||
aw-db-vacuum.timer \
|
||||
detmir-readiness.timer
|
||||
```
|
||||
|
||||
Критерий: ActivityWatch и Worktime API отвечают, нет неожиданных failed units,
|
||||
ежедневные timers присутствуют в расписании.
|
||||
|
||||
### 2. Readiness bundle
|
||||
|
||||
Проверить, что ежедневный bundle сформирован и проверяется:
|
||||
|
||||
```bash
|
||||
cd /var/lib/activitywatch/health/readiness-bundle
|
||||
sha256sum -c sha256sums.txt
|
||||
openssl dgst -sha256 -verify public-key.pem \
|
||||
-signature sha256sums.txt.sig sha256sums.txt
|
||||
jq -r '.status // .overall_status // empty' detmir-readiness-status.json
|
||||
```
|
||||
|
||||
Критерий: checksum/signature проходят, статус не `FAIL`, дата bundle относится
|
||||
к текущему maintenance window.
|
||||
|
||||
### 3. Свежесть telemetry и reports
|
||||
|
||||
Проверить штатные smoke/validation команды из репозитория:
|
||||
|
||||
```bash
|
||||
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
|
||||
node scripts/operational-maturity-check.mjs --json
|
||||
node scripts/pilot-validation-smoke.mjs
|
||||
node scripts/deployment-readiness-smoke.mjs
|
||||
```
|
||||
|
||||
Если включён автономный validation orchestrator, использовать его как единый
|
||||
источник daily evidence:
|
||||
|
||||
```bash
|
||||
./scripts/run_full_validation.sh --profile standard
|
||||
```
|
||||
|
||||
Критерий: нет новых hard-fail проверок. Warning допустим только при известном
|
||||
и задокументированном operational constraint.
|
||||
|
||||
### 4. Очереди, backlog и storage growth
|
||||
|
||||
Проверить отсутствие неконтролируемого роста:
|
||||
|
||||
```bash
|
||||
du -sh /var/lib/activitywatch /var/log/activitywatch 2>/dev/null || true
|
||||
docker system df 2>/dev/null || true
|
||||
docker exec aw-rus-1c-clickhouse clickhouse-client --query \
|
||||
"SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" \
|
||||
2>/dev/null || true
|
||||
```
|
||||
|
||||
Для Windows queues не выполнять ручную очистку. Если backlog растёт, сначала
|
||||
проверить доступность AW server, scheduled tasks и upload logs.
|
||||
|
||||
### 5. Retention safety
|
||||
|
||||
Проверить dry-run cleanup, если есть подозрение на рост диска или после
|
||||
изменения retention-конфигурации:
|
||||
|
||||
```bash
|
||||
sudo AW_DATA_DIR=/var/lib/activitywatch \
|
||||
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache \
|
||||
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400 \
|
||||
/usr/local/bin/aw-prune-local-state-rust --json
|
||||
```
|
||||
|
||||
Apply не является ежедневной обязательной операцией. Apply выполнять только
|
||||
после просмотра planned items и подтверждения, что cleanup не затрагивает
|
||||
configuration, dashboards, forensic/security evidence, release evidence или
|
||||
Windows queues.
|
||||
|
||||
### 6. Короткая запись результата
|
||||
|
||||
Сохранить дневной summary в operator journal или internal change log:
|
||||
|
||||
```text
|
||||
date:
|
||||
operator:
|
||||
health:
|
||||
readiness_bundle:
|
||||
telemetry_freshness:
|
||||
storage:
|
||||
retention:
|
||||
new_alerts:
|
||||
known_constraints:
|
||||
actions_taken:
|
||||
```
|
||||
|
||||
## Pollinations AI assistant
|
||||
|
||||
### Роль
|
||||
|
||||
Pollinations используется только как локально запускаемый помощник для:
|
||||
|
||||
- сжатия daily evidence в операторский summary;
|
||||
- выделения новых failures/warnings из JSON/Markdown reports;
|
||||
- подготовки списка вопросов для ручной проверки;
|
||||
- сравнения текущего maintenance summary с предыдущим summary.
|
||||
|
||||
Pollinations не должен:
|
||||
|
||||
- запускать production commands самостоятельно;
|
||||
- получать secrets/private env/raw logs;
|
||||
- менять конфигурацию;
|
||||
- принимать решение о cleanup apply, DLP enablement, restart или rollback.
|
||||
|
||||
### Проверка доступности CLI
|
||||
|
||||
```bash
|
||||
/home/igor/.local/bin/polli-chat --help
|
||||
/home/igor/.local/bin/polli-agent --help
|
||||
/home/igor/.local/bin/polli-chat --max-tokens 16 'ping'
|
||||
```
|
||||
|
||||
Для `polli-agent` использовать `--collection`, не `--db`:
|
||||
|
||||
```bash
|
||||
/home/igor/.local/bin/polli-agent index \
|
||||
--collection /tmp/awatch-maintenance-index \
|
||||
/path/to/sanitized-evidence.md \
|
||||
/path/to/sanitized-validation-report.json
|
||||
|
||||
/home/igor/.local/bin/polli-agent ask \
|
||||
--collection /tmp/awatch-maintenance-index \
|
||||
--context-only \
|
||||
'Какие новые failures или warnings есть в сегодняшнем evidence?'
|
||||
```
|
||||
|
||||
### Sanitization перед AI-анализом
|
||||
|
||||
Перед передачей в Pollinations оставить только технические поля:
|
||||
|
||||
- status/outcome;
|
||||
- timestamps;
|
||||
- Git SHA/version;
|
||||
- service names;
|
||||
- check names;
|
||||
- counters;
|
||||
- sizes;
|
||||
- non-sensitive error class;
|
||||
- documented operational constraints.
|
||||
|
||||
Удалить или заменить placeholders:
|
||||
|
||||
- usernames and user identifiers;
|
||||
- tokens/passwords/API keys;
|
||||
- Basic Auth credentials;
|
||||
- private IP details, если summary планируется публиковать;
|
||||
- screenshots and raw event payloads;
|
||||
- full journal lines with command arguments.
|
||||
|
||||
### Prompt для daily summary
|
||||
|
||||
```text
|
||||
Ты анализируешь sanitized AWatch-rus daily maintenance evidence.
|
||||
Не придумывай факты. Не предлагай включать DLP/Loki/always-on Velociraptor.
|
||||
Раздели вывод на:
|
||||
1. OK
|
||||
2. New warnings
|
||||
3. New failures
|
||||
4. Evidence gaps
|
||||
5. Required operator actions
|
||||
Если evidence недостаточно, напиши "не доказано".
|
||||
```
|
||||
|
||||
### Fail-closed правило
|
||||
|
||||
Если Pollinations summary расходится с raw evidence, считать источником истины
|
||||
raw evidence и runbook. AI-вывод можно использовать только как черновик для
|
||||
операторского анализа.
|
||||
|
||||
## Журнал выполненного обслуживания
|
||||
|
||||
### 2026-07-03: production cleanup, collector guard и dependency hardening
|
||||
|
||||
Выполнено:
|
||||
|
||||
- очищены старые ActivityWatch SQLite backup files на AW server с сохранением
|
||||
последних двух rollback-точек;
|
||||
- `/` на AW server снижен с `85%` до `40%`, свободно `19G`;
|
||||
- заменен Windows production binary
|
||||
`C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe`;
|
||||
- заменен gateway/ClickHouse production binary
|
||||
`/usr/local/bin/aw-1c-ingest-rust`;
|
||||
- закрыт RustSec риск `quick-xml` через удаление `calamine` из
|
||||
`aw-1c-ingest` и переход на прямое чтение XLSX через `zip` +
|
||||
`quick-xml 0.41.0`;
|
||||
- добавлен regression test на разбор registry XLSX листов `Лист2` и
|
||||
`ОСНОВНОЙ`.
|
||||
|
||||
Production checksums:
|
||||
|
||||
| Binary | SHA256 |
|
||||
|---|---|
|
||||
| `aw-windows-telemetry.exe` | `A8F517AB636C8537413201B84EA6540CB198045F388FECE6C8D6810537FBDDF1` |
|
||||
| `aw-1c-ingest-rust` | `e0c4e665ea8946ba9c243127a65fd69554cacdc4e3191ec798b8ca19acf89623` |
|
||||
|
||||
Rollback files:
|
||||
|
||||
- `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe.bak.20260702T202626Z`;
|
||||
- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T211637Z`;
|
||||
- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T212258Z`.
|
||||
|
||||
Post-change production evidence:
|
||||
|
||||
- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`;
|
||||
- `aw-rus-healthd`: `ok=True`;
|
||||
- AW server `systemctl --failed`: `0 loaded units listed`;
|
||||
- `pve-detmir systemctl --failed`: `0 loaded units listed`;
|
||||
- `aw-1c-ingest.service`: `Result=success`, `ExecMainStatus=0`,
|
||||
`files_loaded=8`, `rows_loaded=561`;
|
||||
- ClickHouse container `aw-rus-1c-clickhouse`: `healthy`;
|
||||
- gateway `/healthz`: `ok`.
|
||||
|
||||
Validation executed before documenting:
|
||||
|
||||
```bash
|
||||
cargo fmt --all --check
|
||||
cargo test --workspace --locked
|
||||
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
|
||||
cargo audit --no-fetch
|
||||
cargo deny --manifest-path adk-rust/Cargo.toml check --config deny.toml --hide-inclusion-graph --show-stats
|
||||
cargo metadata --locked --format-version 1
|
||||
cargo tree --duplicates --locked
|
||||
cargo machete --with-metadata
|
||||
python3 scripts/public_secret_pattern_check.py
|
||||
node scripts/operational-maturity-check.mjs --json
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Cargo/RustSec network note: if direct `https://index.crates.io` is unavailable
|
||||
from the laptop, use the local HTTP CONNECT proxy:
|
||||
|
||||
```bash
|
||||
export HTTP_PROXY=http://127.0.0.1:10808
|
||||
export HTTPS_PROXY=http://127.0.0.1:10808
|
||||
export http_proxy=http://127.0.0.1:10808
|
||||
export https_proxy=http://127.0.0.1:10808
|
||||
unset ALL_PROXY all_proxy
|
||||
```
|
||||
|
||||
## Эскалация
|
||||
|
||||
Эскалировать как production issue, если:
|
||||
|
||||
- `/healthz`, `/readyz` или Worktime reports не отвечают;
|
||||
- readiness bundle не сформирован или не проходит checksum/signature;
|
||||
- есть неожиданные failed systemd units;
|
||||
- storage growth продолжается после штатного retention;
|
||||
- Windows queues растут и upload не восстанавливается;
|
||||
- DLP/security evidence удалено, повреждено или изменилось без change;
|
||||
- Pollinations обнаружил warning, который подтверждается raw evidence.
|
||||
|
||||
## Связанные документы
|
||||
|
||||
- [Operations Runbook](OPERATIONS_RUNBOOK_RU.md)
|
||||
- [Эксплуатационная проверка контура](OPERATIONS_VALIDATION_RUNBOOK_RU.md)
|
||||
- [Retention and Cleanup Policy](RETENTION_POLICY_RU.md)
|
||||
- [Production readiness](PRODUCTION_READINESS_RU.md)
|
||||
- [Autonomous validation](AUTONOMOUS_VALIDATION_RU.md)
|
||||
@@ -2,6 +2,10 @@
|
||||
|
||||
Документ описывает базовые эксплуатационные проверки AWatch-rus.
|
||||
|
||||
Ежедневный операторский цикл обслуживания, retention safety и безопасное
|
||||
использование Pollinations AI для анализа sanitized evidence описаны отдельно:
|
||||
[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md).
|
||||
|
||||
## Быстрая проверка
|
||||
|
||||
Проверить доступность:
|
||||
|
||||
@@ -61,6 +61,14 @@ cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian
|
||||
Подробности эксплуатационного maturity harness: [эксплуатационная зрелость
|
||||
DetMir/AWatch-rus](OPERATIONAL_MATURITY_RU.md).
|
||||
|
||||
Единый автономный entry point для unattended validation, runtime diagnostics,
|
||||
history, regression detection, alerts и reports описан отдельно:
|
||||
[автономная проверка AWatch-rus](AUTONOMOUS_VALIDATION_RU.md).
|
||||
|
||||
Ежедневный maintenance-порядок с optional Pollinations AI assistant для
|
||||
sanitized evidence summaries описан в отдельном runbook:
|
||||
[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md).
|
||||
|
||||
## Production binary parity gate
|
||||
|
||||
Production binary parity gate доказывает, что production unit/timer/task
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
if ! command -v node >/dev/null 2>&1; then
|
||||
echo "run_full_validation=fail node_not_found" >&2
|
||||
exit 127
|
||||
fi
|
||||
|
||||
run_preflight_self_test=1
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--self-test" || "$arg" == "--help" || "$arg" == "-h" ]]; then
|
||||
run_preflight_self_test=0
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${AW_VALIDATION_SKIP_SELF_TEST:-0}" != "1" && "$run_preflight_self_test" == "1" ]]; then
|
||||
node scripts/full-validation-orchestrator.mjs --self-test
|
||||
fi
|
||||
|
||||
exec node scripts/full-validation-orchestrator.mjs "$@"
|
||||
@@ -594,7 +594,15 @@ $watcherCountsOk = $true
|
||||
$endpointProcessOk = $true
|
||||
$fileProcessOk = $true
|
||||
$browserProcessOk = $true
|
||||
$sessionCollectorOk = (@($sessionCollectorProcesses).Count -eq 1)
|
||||
$sessionCollectorOk = if (@($sessionCollectorProcesses).Count -eq 1) {
|
||||
$true
|
||||
}
|
||||
elseif (-not $liveSessionScopedCollectorsRequired) {
|
||||
$true
|
||||
}
|
||||
else {
|
||||
$false
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
Reference in New Issue
Block a user