Compare commits

..
Author SHA1 Message Date
igor04091968 518a501dee docs: switch Gitea runbooks to git.iri1968.ru
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-08-04 12:03:28 +03:00
igor04091968 af735ec1ea Respect disabled DLP in contour smoke
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-11 10:19:17 +03:00
igor04091968 cd25edc4f3 Fix DetMir light DLP readiness and collector guard
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-09 23:38:57 +03:00
igor04091968 597b462d39 Stabilize Hayabusa drop autoprocess 2026-07-09 00:26:40 +03:00
igor04091968 93f517fbf7 Stabilize DetMir contour checks 2026-07-05 23:18:25 +03:00
igor04091968 234afdd184 Fix DetMir portal gateway API mapping 2026-07-03 10:07:41 +03:00
igor04091968 08c67bde1a Add DetMir primary recovery guard 2026-07-03 08:51:11 +03:00
igor04091968 130d4cd94f Document and harden DetMir production maintenance 2026-07-03 00:31:28 +03:00
igor04091968 77c054dcad Document DetMir portal telemetry ingest repair 2026-07-02 10:33:11 +03:00
igor04091968 ce27a8170d Add production binary parity gate 2026-07-01 19:56:15 +03:00
igor04091968 d79b37fff5 Add production evidence gaps 2026-07-01 17:18:06 +03:00
igor04091968 e6378e87f5 Add build provenance to release evidence 2026-07-01 16:49:35 +03:00
igor04091968 b9dca66d1b Add disaster recovery proof 2026-07-01 16:33:04 +03:00
igor04091968 0c91d56a57 Add operational recovery audit 2026-07-01 15:23:36 +03:00
igor04091968 82691d7d92 Remove ClickHouse password from process argv 2026-07-01 14:41:59 +03:00
igor04091968 bb86939179 Add production retention and cleanup policy 2026-07-01 14:16:36 +03:00
igor04091968 0bc7186bbc Add production readiness audit 2026-07-01 12:08:19 +03:00
igor04091968 0ce831d56f Add next development plan audit 2026-07-01 09:14:18 +03:00
igor04091968 dcf14eae67 Codify production-first engineering governance
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 08:48:35 +03:00
igor04091968 121d66cfcf Add DetMir operational maturity harness
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 07:34:43 +03:00
80 changed files with 8622 additions and 236 deletions
+4
View File
@@ -31,6 +31,7 @@
/.github/workflows/ci.yml @igor04091968 @rachkovii68-bisness
/.github/workflows/security.yml @igor04091968 @rachkovii68-bisness
/.github/workflows/coverage.yml @igor04091968 @rachkovii68-bisness
/.github/GOVERNANCE.md @igor04091968 @rachkovii68-bisness
/.github/pull_request_template.md @igor04091968 @rachkovii68-bisness
/.github/ISSUE_TEMPLATE/ @igor04091968 @rachkovii68-bisness
/.github/CODEOWNERS @igor04091968 @rachkovii68-bisness
@@ -38,6 +39,9 @@
/docs/BRANCH_PROTECTION_EVIDENCE_RU.md @igor04091968 @rachkovii68-bisness
/docs/PR_REVIEW_WORKFLOW_RU.md @igor04091968 @rachkovii68-bisness
/docs/PR_REVIEW_EVIDENCE_RU.md @igor04091968 @rachkovii68-bisness
/docs/ENTERPRISE_QUALITY_STANDARD_RU.md @igor04091968 @rachkovii68-bisness
/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @igor04091968 @rachkovii68-bisness
/docs/OPERATIONAL_MATURITY_RU.md @igor04091968 @rachkovii68-bisness
# Ansible deployment automation.
/ansible/ @igor04091968 @rachkovii68-bisness
+41
View File
@@ -0,0 +1,41 @@
# AWatch-rus Governance
GitHub is the public mirror validation surface. Primary registry release
evidence must be produced on the Russian build-runner and primary operational
context remains the private Gitea contour.
## Production-first standard
AWatch-rus is already deployed in a real company. Production stability has
absolute priority. The canonical engineering standard is:
- [Enterprise quality standard](../docs/ENTERPRISE_QUALITY_STANDARD_RU.md)
- [Review checklist](../docs/REVIEW_CHECKLIST_RU.md)
- [Operational validation runbook](../docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md)
- [Operational maturity harness](../docs/OPERATIONAL_MATURITY_RU.md)
## Required PR evidence
Every PR must state:
- Purpose.
- Operational impact.
- Risk assessment.
- Rollback strategy.
- Validation steps.
- Documentation changes.
- Acceptance criteria.
Documentation-only or governance-only PRs must explicitly state that runtime,
API and UI behavior are unchanged.
## Guardrails
- Prefer additive, backward-compatible changes.
- Do not redesign working subsystems without measured benefit.
- Do not add dependencies without justification and validation.
- Do not weaken authentication, authorization, audit logging, secret handling,
dependency hygiene or configuration validation.
- Do not enable heavy DLP, Loki or always-on Velociraptor during routine
recovery, validation or public CI.
- Keep blocking CI fast; keep heavy/load/nightly checks scheduled or advisory.
+45
View File
@@ -2,6 +2,12 @@
Describe what changed and why.
## Purpose
State the production-quality purpose of the change. Prefer reliability,
operational maturity, security, maintainability, reproducibility, performance
or simplicity over new functionality.
## Impact
- Runtime impact: `none / changed / not applicable`
@@ -11,14 +17,45 @@ Describe what changed and why.
- Rollback impact: `none / documented / not applicable`
- Evidence impact: `none / registry docs updated / release evidence required`
## Operational Impact
Describe deployment, upgrade, rollback, observability, diagnostics,
configuration, recovery, performance or dependency-hygiene impact.
## Risk Assessment
List production risks and why the change is backward-compatible. For
documentation-only or governance-only changes, state that runtime/API/UI behavior
is unchanged.
## Rollback Strategy
State how to revert the change. Runtime, automation, config and dependency
changes need an explicit rollback path.
## Validation
List commands executed. Use `skipped: <reason>` when a check requires a live
stand or unavailable tool.
## Documentation Changes
List README/runbook/architecture/governance updates, or state `not applicable`
with a reason.
## Acceptance Criteria
List concrete conditions that make the PR safe to merge.
## Review Checklist
- [ ] Linked issue is provided, or the PR explains why no issue is applicable.
- [ ] Purpose, operational impact, risk assessment, rollback strategy,
validation steps, documentation changes and acceptance criteria are stated.
- [ ] Change is additive/backward-compatible, or breaking impact is explicitly
blocked for this stage.
- [ ] Production stability is preserved for existing deployments.
- [ ] No working subsystem is redesigned without measured benefit.
- [ ] Runtime/API/UI impact is stated.
- [ ] Registry claims are checked and remain conservative.
- [ ] Secrets, PII, employee logs and customer identifiers are absent.
@@ -33,6 +70,8 @@ stand or unavailable tool.
- [ ] I checked registry claims: no completed registry submission, no
FSTEC/FSB certification claim, no SIEM/DLP replacement claim.
- [ ] I ran relevant checks or documented why a check was skipped.
- [ ] I checked dependency impact: no unnecessary dependency was added, and no
unused dependency remains in touched crates.
- [ ] I stated runtime/API/UI impact.
- [ ] I stated documentation impact.
- [ ] I stated smoke-test result or why smoke testing is not applicable.
@@ -48,6 +87,12 @@ stand or unavailable tool.
build-runner.
- Update `docs/registry/` when registry-readiness behavior or evidence changes.
## Governance
- Production-first standard: `.github/GOVERNANCE.md`.
- Canonical review checklist: `docs/REVIEW_CHECKLIST_RU.md`.
- Canonical validation runbook: `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
## Safety
- No secrets, tokens, passwords or private keys.
@@ -0,0 +1,73 @@
name: Operational maturity
# GitHub Actions is public mirror validation only.
# Live DetMir production checks must be run explicitly by an operator with
# private network access and must not enable heavy DLP, Loki or always-on
# Velociraptor.
on:
push:
pull_request:
workflow_dispatch:
inputs:
live:
description: "Run live endpoint contract if AWATCH_OPS_LIVE_URL is configured"
required: false
default: "false"
schedule:
- cron: "41 3 * * 2"
permissions:
contents: read
jobs:
offline-operational-maturity:
name: Offline operational maturity
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Script syntax
run: node --check scripts/operational-maturity-check.mjs
- name: Operational maturity harness
run: node scripts/operational-maturity-check.mjs --json
live-operational-contract:
name: Live operational contract
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && inputs.live == 'true'
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Run live contract when URL is available
env:
AWATCH_OPS_LIVE_URL: ${{ secrets.AWATCH_OPS_LIVE_URL }}
run: |
if [[ -z "${AWATCH_OPS_LIVE_URL}" ]]; then
echo "skipped: AWATCH_OPS_LIVE_URL secret is not configured"
exit 0
fi
node scripts/operational-maturity-check.mjs --json --live
+11 -1
View File
@@ -4,6 +4,11 @@ GitHub is public mirror validation only. Primary registry release evidence is
produced separately on the Russian build-runner and documented under
`docs/registry/`.
All contributions must follow the production-first governance entrypoint:
`.github/GOVERNANCE.md`. AWatch-rus is already deployed in a real company, so
reliability, operational maturity, security and backward compatibility take
priority over new functionality.
## Branching
- Работайте в feature-ветке, не пушьте напрямую в `main`.
@@ -49,6 +54,11 @@ produced separately on the Russian build-runner and documented under
## PR content
- Изменения и обоснование.
- Риск и rollback.
- Purpose.
- Operational impact.
- Risk assessment.
- Rollback strategy.
- Какие команды валидации были выполнены.
- Documentation changes.
- Acceptance criteria.
- Какие проверки были пропущены и почему, если пропуск был необходим.
+826
View File
@@ -0,0 +1,826 @@
# TASK_065: Production 1.0 development plan
Audit date: 2026-07-01.
Scope: AWatch-rus repository readiness for Production 1.0. This plan is based
only on repository evidence: tracked files, documentation, CI workflows,
Cargo metadata, dependency checks, scripts, deployment assets and operational
maturity contracts.
This is not a feature roadmap. Every task below exists because the current
repository still has a confirmed blocker or material risk for a durable
Production 1.0 release.
## Guardrails
- Do not redesign working subsystems.
- Do not add new product functionality as part of Production 1.0 readiness.
- Preserve backward compatibility for deployed DetMir production.
- Keep DLP runtime conservative: `core_only/disabled` by default; `light` only
after explicit operator resource preflight.
- Do not enable Loki or always-on Velociraptor as part of Production 1.0.
- Keep PowerShell fallback until Rust parity, canary evidence and rollback
criteria are proven.
- GitHub remains public mirror validation. Release evidence must be produced in
the controlled release contour documented under `docs/registry/`.
## Evidence Base
Reviewed repository areas:
- Rust workspace: `adk-rust/`, 58 crates from `adk-rust/crates/*/Cargo.toml`.
- Cargo state: `cargo metadata --locked --format-version 1` returned 58
workspace members and 349 packages.
- Dependency hygiene: `cargo audit --deny warnings` passed; `cargo machete
--with-metadata` found no unused dependencies; `cargo deny` passed with
non-blocking policy warnings; `cargo tree --duplicates --locked` found
duplicate roots including `bitflags`, `getrandom`, `hashbrown`, `mio`, `zip`.
- CI: `.github/workflows/ci.yml`, `security.yml`, `coverage.yml`,
`dependency-hygiene.yml`, `operational-maturity.yml`,
`rust-workspace.yml`, `rust-binary-build.yml`, `release-assets.yml`.
- Operational maturity: `configs/operational-maturity-contract.json`,
`scripts/operational-maturity-check.mjs`,
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
- Registry and release evidence docs: `docs/registry/*`,
`docs/PROJECT_STATUS_RU.md`, `docs/RESIDUAL_RISKS_RU.md`,
`docs/QUALITY_STATUS_RU.md`, `ROADMAP.md`.
- Runtime/deployment: `ansible/`, `aw-server/`, `windows/`, `proxmox/`,
`clickhouse-1c/`, `grafana/`, `pfsense/`, root operational wrappers.
Confirmed current strengths:
- Rust-first runtime and wrappers exist.
- Public mirror CI exists.
- Dependency hygiene automation exists.
- Operational maturity offline harness exists.
- DLP resource guardrails and conservative runtime profile exist.
- Security Finding Inbox / Hayabusa / Velociraptor are optional and separated
from Workforce hot path.
- Production runbooks and DetMir guardrails are documented.
Confirmed Production 1.0 blockers and material risks are organized below.
## P0 - Critical before Production 1.0
### P0-1. Production binary parity gate
Status: repository gate addressed by TASK_072. Live production execution remains
External Production Validation: operators must collect current production
SHA256 evidence and run the gate before Release Candidate approval.
Purpose: prove that every binary actually running in production matches the
reviewed release artifact.
Reason: `scripts/check_detmir_rust_release_artifacts.sh` verifies local release
artifacts, but Production 1.0 also needs a verified mapping from deployed unit,
timer or Windows task to exact production SHA256 and source crate.
Operational impact: prevents stale binary drift and makes rollback decisions
deterministic.
Risk: high. A stale production binary can invalidate test results and hide
regressions.
Estimated effort: 3-5 days.
Affected modules:
- `scripts/check_detmir_rust_release_artifacts.sh`
- `scripts/check_production_binary_parity.py`
- `scripts/package_rust_release_binaries.py`
- `scripts/detmir-full-diagnostics/`
- `adk-rust/crates/detmir-readiness/`
- `adk-rust/crates/aw-windows-telemetry/`
- `windows/validate-deployment.ps1`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Report maps `service/timer/task -> binary path -> crate -> runtime role`.
- Report includes local release SHA256, production SHA256 and git SHA.
- Missing or mismatched binaries fail the gate.
- DLP/Loki/Velociraptor heavy runtime is not enabled by the check.
- Repository validator accepts inactive optional contours only with explicit
`skip_reason`.
Validation steps:
- Run the parity gate against current release artifacts.
- Run diagnostics for `10.10.10.2`, `10.10.10.13` and Windows RDP host.
- Confirm stale binary simulation fails.
Expected benefit: release and production state become auditable.
### P0-2. Controlled release evidence build
Status: addressed by TASK_070. Existing release evidence tooling now records
resolved Git SHA, deterministic `SOURCE_DATE_EPOCH` / `BUILD_TIME`, runner
environment and artifact checksums, and the evidence checker enforces the
required provenance fields.
Purpose: produce Production 1.0 release evidence outside GitHub public mirror.
Reason: `docs/PROJECT_STATUS_RU.md`, `docs/QUALITY_STATUS_RU.md` and
`docs/registry/registry-evidence-manifest.json` state that GitHub Actions are
public validation only and that the Russian build-runner/release evidence path
is still pending.
Operational impact: separates mirror CI from release authority.
Risk: high. Without controlled release evidence, Production 1.0 cannot be
treated as reproducibly built.
Estimated effort: 3-6 days plus infrastructure window.
Affected modules:
- `docs/registry/RU_BUILD_RUNNER_READINESS_RU.md`
- `docs/registry/BUILD_RUNNER_SETUP_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_MANIFEST_RU.md`
- `scripts/build_release_evidence.sh`
- `scripts/check_release_evidence.sh`
- `scripts/verify_release_assets.sh`
Acceptance criteria:
- Controlled runner has documented OS, toolchain and access model.
- Release evidence includes source archive, binary archive, SBOM if available,
SHA256SUMS, Cargo metadata/tree, smoke logs and release manifest.
- `scripts/check_release_evidence.sh` passes on produced artifacts.
- GitHub CI is not described as release evidence.
Validation steps:
- Run release evidence script on controlled runner.
- Verify manifest and checksums.
- Record runner environment and commit SHA.
Expected benefit: Production 1.0 release becomes reproducible and auditable.
### P0-3. Disaster recovery restore proof
Purpose: prove that repository backup can be restored before Production 1.0.
Reason: `docs/registry/registry-evidence-manifest.json` records
`restore_tested=false`, and `docs/RESIDUAL_RISKS_RU.md` lists Gitea restore
test as open.
Operational impact: validates recovery from source repository loss.
Risk: high. Untested backup is not a recovery capability.
Estimated effort: 2-4 days.
Affected modules:
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md`
- `docs/registry/registry-evidence-manifest.json`
- `scripts/registry_readiness_check.sh`
- `docs/PROJECT_STATUS_RU.md`
- `docs/RESIDUAL_RISKS_RU.md`
Acceptance criteria:
- Restore is performed on a separate host.
- SHA256 verification, logs, restored repository access and rollback notes are
captured.
- Manifest is updated only after evidence exists.
- No secrets are copied into repository documentation.
Validation steps:
- Execute restore runbook.
- Run registry readiness check after manifest update.
- Confirm restored repository clone and log evidence.
Expected benefit: repository DR becomes proven, not only documented.
### P0-4. Bounded retention for operational state and evidence
Purpose: prevent disk exhaustion from state, queues, diagnostics, evidence and
forensics artifacts.
Reason: DLP optional runtime docs explicitly state that historical DLP buckets
and artifacts may remain until a separate retention/cleanup procedure. Scripts
and diagnostics also create durable output.
Operational impact: reduces outage risk on Proxmox/AW server/Windows state
paths.
Risk: high. Disk exhaustion can stop ingestion, portal, ClickHouse or
ActivityWatch services.
Estimated effort: 4-7 days.
Affected modules:
- `adk-rust/crates/aw-prune-local-state/`
- `scripts/detmir-full-diagnostics/`
- `scripts/detmir_dlp_warehouse_sync.sh`
- `aw-server/logrotate.conf`
- `windows/validate-deployment.ps1`
- `adk-rust/crates/aw-windows-telemetry/`
- `docs/DLP_OPTIONAL_RUNTIME_RU.md`
- `docs/DLP_RESOURCE_PROFILES_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Retention matrix lists roots, owners, max age, max size and dry-run behavior.
- Cleanup only touches allowlisted roots and refuses traversal/symlink escape.
- Active state is preserved.
- Disabled DLP buckets remain `SKIPPED`, not failure.
Validation steps:
- Run dry-run cleanup on fixture tree.
- Run apply mode on controlled temporary tree.
- Run operational smoke after cleanup.
Expected benefit: long-running production operation has bounded disk behavior.
### P0-5. Remove ClickHouse password exposure from process arguments
Status: addressed by TASK_068. Runtime ClickHouse/1C wrappers keep
`CLICKHOUSE_PASSWORD` in the environment/config path and no longer pass it in
process arguments.
Purpose: keep production credentials out of `ps`/process argv.
Reason: `rg` confirmed `clickhouse-1c/ops/run_*.sh` wrappers pass
`--password "${CLICKHOUSE_PASSWORD}"`.
Operational impact: improves secret handling for the 1C/ClickHouse contour.
Risk: high. Local process listing can reveal ClickHouse credentials.
Estimated effort: 2-4 days.
Affected modules:
- `clickhouse-1c/ops/run_ingest_cycle.sh`
- `clickhouse-1c/ops/run_manager_brief.sh`
- `clickhouse-1c/ops/run_recovery_brief.sh`
- `clickhouse-1c/ops/run_company_registry_bindings_refresh.sh`
- `clickhouse-1c/ops/run_company_intelligence_refresh.sh`
- `clickhouse-1c/ops/check_ingest_freshness.sh`
- `clickhouse-1c/ai/*.py`
- `clickhouse-1c/etl/*.py`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Runtime wrappers no longer pass password through argv.
- Existing environment-based deployment remains backward compatible.
- Logs redact authentication failures.
- `ps` smoke proves password absence.
Validation steps:
- `bash -n clickhouse-1c/ops/*.sh`
- Run affected wrapper against a test or dry-run configuration.
- Verify `ps` output during execution.
- Run secret-pattern scan.
Expected benefit: production secret exposure surface is reduced.
### P0-6. Portal/worktime hot-path load gate
Purpose: prevent Production 1.0 from regressing under report/prewarm load.
Reason: `docs/PROJECT_STATUS_RU.md` records that full report/snapshot prewarm
can still be CPU/IO expensive.
Operational impact: protects owner/operator portal, worktime reports and AW
query path.
Risk: high. More users or more history can produce slow portal, stale data or
AW datastore pressure.
Estimated effort: 1-2 weeks.
Affected modules:
- `scripts/operational-maturity-check.mjs`
- `scripts/awatch-production-hardening-smoke.mjs`
- `adk-rust/crates/detmir-portal/`
- `adk-rust/crates/worktime-api/`
- `adk-rust/crates/worktime-prewarm/`
- `adk-rust/crates/aw-contour-smoke/`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Synthetic 5/20/50 user fixtures exist without production data.
- Gate records p95 latency, max RSS, query count and stale-cache behavior.
- Disconnected RDP sessions do not false-fail.
- `AW_DLP_ENABLED=false` semantics remain valid.
- Heavy load job is scheduled/advisory; blocking smoke remains fast.
Validation steps:
- Run offline load harness.
- Confirm configured p95/RSS ceilings.
- Run existing operational maturity smoke.
Expected benefit: Production 1.0 has measurable performance safety.
## P1 - Strongly recommended for the first Production 1.0 release train
### P1-1. Align Rust toolchain across CI
Purpose: remove compiler drift from blocking workflows.
Reason: `rust-toolchain.toml` pins `1.94.0`, but several workflows install
floating `stable`.
Operational impact: improves reproducibility between local, CI and release
contours.
Risk: medium. Toolchain drift can create inconsistent warnings or binaries.
Estimated effort: 1-2 days.
Affected modules:
- `rust-toolchain.toml`
- `.github/workflows/ci.yml`
- `.github/workflows/security.yml`
- `.github/workflows/coverage.yml`
- `.github/workflows/dependency-hygiene.yml`
- `.github/workflows/release-assets.yml`
- `docs/QUALITY_STATUS_RU.md`
Acceptance criteria:
- Blocking Rust workflows use the pinned toolchain.
- Nightly remains limited to advisory `cargo udeps`.
- Required check names do not change.
Validation steps:
- YAML syntax validation.
- Affected workflow dry review.
- Run relevant Rust checks if workflow commands change.
Expected benefit: CI becomes more deterministic.
### P1-2. Dependency warning baseline and future block policy
Purpose: make dependency hygiene fail closed for new risk while preserving
current compatibility.
Reason: `cargo deny` passes but reports 36 non-blocking `bans` warnings;
`cargo tree --duplicates --locked` reports duplicate roots; `serde_yaml` is
documented as deprecated in third-party license docs.
Operational impact: reduces future supply-chain drift.
Risk: medium. Uncontrolled duplicate/deprecated dependency growth increases
maintenance and security load.
Estimated effort: 4-8 days.
Affected modules:
- `adk-rust/Cargo.toml`
- `adk-rust/Cargo.lock`
- `deny.toml`
- `.github/workflows/dependency-hygiene.yml`
- `docs/THIRD_PARTY_LICENSES_RU.md`
- `docs/QUALITY_STATUS_RU.md`
Acceptance criteria:
- Each existing warning is classified: keep, update, remove or defer.
- New duplicate/deprecated dependencies require documented exception.
- `cargo audit`, `cargo deny`, `cargo machete`, `cargo metadata` pass.
Validation steps:
- Run dependency hygiene pipeline.
- Verify policy failure on synthetic unapproved duplicate where practical.
Expected benefit: dependency hygiene remains controlled after 1.0.
### P1-3. Windows Rust validation parity
Purpose: prove Rust validation is equivalent to current PowerShell validation
before reducing fallback reliance.
Reason: `docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md` lists remaining fallback
and runtime PowerShell paths, including validation and Hayabusa upload.
Operational impact: keeps Windows/RDP production recoverable while reducing
runtime drift.
Risk: medium. Premature fallback removal can break localized Windows Server
2019 recovery paths.
Estimated effort: 1-2 weeks.
Affected modules:
- `adk-rust/crates/aw-windows-telemetry/`
- `windows/validate-deployment.ps1`
- `windows/ActivityWatch.Windows.Common.psm1`
- `windows/export-upload-hayabusa-to-aw-server.ps1`
- `ansible/deploy_aw_windows.yml`
- `docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md`
- `docs/POWERSHELL_TO_RUST_ROADMAP_RU.md`
Acceptance criteria:
- Rust validation covers all current production validation sections.
- Localized Windows user/session handling is tested.
- Canary comparison between Rust and PowerShell reports is recorded.
- PowerShell remains documented rollback.
Validation steps:
- Run Rust validation against fixture and live canary.
- Run PowerShell validation against same host.
- Compare normalized reports.
Expected benefit: Windows runtime maturity improves without breaking rollback.
### P1-4. Install kit reproducibility and stale payload gate
Purpose: ensure Windows installer payloads match repository and release
artifacts.
Reason: repository contains install-kit tooling and large installer artifacts;
stale payloads can deploy old collectors while CI is green.
Operational impact: safer Windows upgrades and rollback.
Risk: medium. Mismatched install kit can create production drift.
Estimated effort: 5-8 days.
Affected modules:
- `windows/installkit/innosetup/`
- `adk-rust/crates/check-install-kit-vs-repo/`
- `adk-rust/crates/rebuild-install-kit/`
- `adk-rust/crates/validate-install-kit/`
- `adk-rust/crates/verify-innosetup-installer/`
- `scripts/rebuild_install_kit.sh`
- `docs/INSTALL_KIT_RUNBOOK_RU.md`
Acceptance criteria:
- Installer manifest contains source commit and payload SHA256 values.
- Validation fails on stale collector payload.
- Existing Windows task names and config schema remain compatible.
Validation steps:
- Rebuild install kit on controlled runner.
- Run install-kit validators.
- Compare payload manifest with repository state.
Expected benefit: Windows deployments become reproducible.
### P1-5. Current-state documentation cleanup
Purpose: prevent operators and reviewers from following stale status text.
Reason: current docs include historical statuses, older required check names
and explicit pending sections that must be reconciled with active branch
protection and current workflows.
Operational impact: reduces release and support mistakes.
Risk: medium. Wrong runbook/status interpretation can cause incorrect release
decisions.
Estimated effort: 3-6 days.
Affected modules:
- `docs/PROJECT_STATUS_RU.md`
- `docs/QUALITY_STATUS_RU.md`
- `docs/ROADMAP_CONFORMANCE_AUDIT_RU.md`
- `docs/BRANCH_PROTECTION_POLICY_RU.md`
- `docs/BRANCH_PROTECTION_EVIDENCE_RU.md`
- `README.md`
- `ROADMAP.md`
Acceptance criteria:
- Current docs list actual required check names.
- Historical docs are clearly marked historical.
- Registry/release claims remain conservative.
- No stale instructions contradict Production 1.0 guardrails.
Validation steps:
- Run registry readiness check.
- Run docs smoke/link validation.
- Run secret scan.
Expected benefit: release process is less error-prone.
### P1-6. Reviewed PR and release governance evidence
Purpose: prove review discipline before Production 1.0.
Reason: `docs/RESIDUAL_RISKS_RU.md` records first reviewed PR evidence as
pending even though CODEOWNERS, PR template and ruleset are present.
Operational impact: improves change control for production releases.
Risk: medium. Lack of review evidence weakens release governance.
Estimated effort: 1-3 days after reviewer availability.
Affected modules:
- `.github/CODEOWNERS`
- `.github/pull_request_template.md`
- `docs/PR_REVIEW_WORKFLOW_RU.md`
- `docs/PR_REVIEW_EVIDENCE_RU.md`
- `docs/REVIEW_CHECKLIST_RU.md`
- `docs/RESIDUAL_RISKS_RU.md`
Acceptance criteria:
- At least one PR is reviewed and merged without bypass.
- Evidence records checks, reviewer, approval and merge path.
- Release branch review policy is documented.
Validation steps:
- Verify PR history and ruleset evidence.
- Run registry readiness check after evidence update.
Expected benefit: Production 1.0 has visible governance evidence.
## P2 - Engineering improvements for post-1.0 hardening
### P2-1. Incremental decomposition of large modules
Purpose: reduce review risk in the largest files without behavior changes.
Reason: confirmed hotspots include `detmir-portal/src/main.rs` at 14200 lines,
`aw-windows-telemetry/src/main.rs` at 6411 lines,
`proxmox/tsj_guardian_bot.py` at 4610 lines,
`worktime-api/src/main.rs` at 3988 lines and
`ansible/deploy_aw_server.yml` at 3099 lines.
Operational impact: easier reviews and lower regression risk.
Risk: medium. Large files increase accidental coupling.
Estimated effort: 2-4 weeks in small PRs.
Affected modules:
- `adk-rust/crates/detmir-portal/`
- `adk-rust/crates/aw-windows-telemetry/`
- `adk-rust/crates/worktime-api/`
- `proxmox/tsj_guardian_bot.py`
- `ansible/deploy_aw_server.yml`
Acceptance criteria:
- Only extract bounded domains.
- Public API, config, unit and task names remain unchanged.
- Tests before and after remain equivalent.
Validation steps:
- Targeted tests per extracted module.
- Full Rust pipeline for Rust changes.
- Ansible syntax/list-tasks parity for playbook changes.
Expected benefit: maintainability improves without architecture rewrite.
### P2-2. Consolidate exact duplicate diagnostic scripts
Purpose: prevent script drift.
Reason: SHA256 confirms exact duplicates:
`scripts/aw-contour-diag.sh` equals
`scripts/detmir-full-diagnostics/aw-contour-diag.sh`; and
`scripts/check_production_inventory_placeholders.sh` equals
`scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`.
Operational impact: diagnostics remain consistent.
Risk: low-medium. Future fixes may land in one copy only.
Estimated effort: 1-2 days.
Affected modules:
- `scripts/aw-contour-diag.sh`
- `scripts/detmir-full-diagnostics/aw-contour-diag.sh`
- `scripts/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh`
Acceptance criteria:
- Existing paths continue to work.
- One implementation is canonical or drift check is enforced.
- Shell syntax and shellcheck pass.
Validation steps:
- `bash -n` on affected scripts.
- Run diagnostic wrapper in dry-run/smoke mode.
Expected benefit: lower maintenance overhead.
### P2-3. Bound 1C ingest memory profile
Purpose: make 1C ingest safer for larger files.
Reason: `adk-rust/crates/aw-1c-ingest/src/main.rs` reads rows into `Vec` and
builds large JSON batches; acceptable now, but risky as export volume grows.
Operational impact: improves ClickHouse ingestion predictability.
Risk: medium. Large files can create memory spikes and long insert windows.
Estimated effort: 1-2 weeks.
Affected modules:
- `adk-rust/crates/aw-1c-ingest/src/main.rs`
- `clickhouse-1c/etl/config.yml`
- `clickhouse-1c/etl/config.example.yml`
- `clickhouse-1c/sql/`
Acceptance criteria:
- Oversized input fails closed with clear diagnostic.
- Batch size is bounded and configurable.
- Existing small DetMir files produce identical output.
Validation steps:
- Add synthetic large CSV/XLSX fixture.
- Run targeted Rust tests.
- Measure max RSS on fixture.
Expected benefit: safer scaling of 1C analytics.
### P2-4. Command execution boundary audit
Purpose: standardize shell/command execution safety.
Reason: `detmir-portal` already validates shell probe commands and tests
process-tree timeout cleanup, while other operational tools also execute
commands.
Operational impact: prevents future command injection or timeout regressions.
Risk: medium. Config-driven command execution must remain fail-closed.
Estimated effort: 4-7 days.
Affected modules:
- `adk-rust/crates/detmir-portal/src/main.rs`
- `adk-rust/crates/detmir-portal/src/production/limits.rs`
- `adk-rust/crates/aw-slo-monitor/src/main.rs`
- `adk-rust/crates/diag-and-manual-restart/src/main.rs`
- `adk-rust/crates/quality-gate/src/main.rs`
Acceptance criteria:
- Runtime command sources are classified.
- Config-driven commands reject shell control operators where applicable.
- Timeout tests cover child/grandchild cleanup.
- Logs do not expose secrets.
Validation steps:
- Targeted Rust tests.
- Clippy for affected crates.
- Secret scan.
Expected benefit: stronger fail-closed security posture.
## P3 - Long-term improvements after Production 1.0
### P3-1. Russian OS compatibility matrix
Purpose: document supported and unsupported target OS combinations.
Reason: `ROADMAP.md` lists Russian OS compatibility validation as planned.
Operational impact: reduces deployment surprises for new customers.
Risk: low-medium for current DetMir, higher for wider distribution.
Estimated effort: 2-4 weeks depending on test hosts.
Affected modules:
- `docs/registry/`
- `docs/INSTALLATION.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `ansible/`
- `windows/`
- `windows/installkit/`
Acceptance criteria:
- Matrix lists OS, role, test date, result and limitations.
- Unsupported combinations are explicit.
- No production defaults are changed just for claims.
Validation steps:
- Run install/deploy smoke per OS.
- Record evidence paths.
Expected benefit: clearer deployment support boundary.
### P3-2. Coverage threshold after baseline review
Purpose: prevent coverage decline after baseline stabilizes.
Reason: `docs/QUALITY_STATUS_RU.md` and `ROADMAP.md` state coverage threshold
is not enforced yet.
Operational impact: improves long-term regression resistance.
Risk: low for current production if kept advisory first.
Estimated effort: 1-2 weeks.
Affected modules:
- `.github/workflows/coverage.yml`
- `docs/QUALITY_STATUS_RU.md`
- `docs/REVIEW_CHECKLIST_RU.md`
- selected `adk-rust/` crates
Acceptance criteria:
- Initial threshold is based on measured baseline.
- Threshold starts advisory and becomes blocking only after stable history.
- Generated/fixture code exclusions are documented.
Validation steps:
- Run coverage workflow locally or in CI.
- Compare summary to baseline.
Expected benefit: gradual improvement in test discipline.
### P3-3. Capacity sizing guide from measured data
Purpose: provide measured sizing guidance beyond the current 5-user DetMir
deployment.
Reason: current production is small; future deployments need measured guidance
for AW SQLite, ClickHouse, Grafana, DLP light profile and Windows collector
load.
Operational impact: safer planning for larger deployments.
Risk: low for current production, medium for growth.
Estimated effort: 2-4 weeks after P0 load gate data exists.
Affected modules:
- `docs/SIZING_GUIDE_RU.md`
- `docs/DLP_RESOURCE_PROFILES_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `grafana/`
- `scripts/operational-maturity-check.mjs`
Acceptance criteria:
- Profiles exist for 5, 20, 50 and 100 users.
- Optional DLP/Hayabusa/Velociraptor resource costs are explicit.
- Guidance is based on measured harness output.
Validation steps:
- Run synthetic capacity scenarios.
- Update sizing doc with observed p95/RSS/storage data.
Expected benefit: production planning becomes evidence-based.
## Not Proposed Because Already Implemented
- Rust-first runtime direction.
- Public mirror CI, security scan and dependency review.
- `cargo audit`, `cargo deny`, `cargo machete`, `cargo tree --duplicates` and
advisory `cargo udeps` workflow coverage.
- DLP `core_only/disabled` production guardrails and load guard.
- Optional Security Finding Inbox / Hayabusa / Velociraptor separation from
Workforce hot path.
- Operational maturity offline harness.
- Branch protection, CODEOWNERS, PR template and review checklist mechanisms.
- Unused dependency cleanup: current `cargo machete --with-metadata` reports no
unused dependencies.
+246
View File
@@ -0,0 +1,246 @@
# Disaster Recovery Proof
Date: 2026-07-01
Scope: evidence-backed disaster recovery validation for the current
AWatch-rus / DetMir production deployment, based only on repository contents.
This document does not introduce new recovery procedures and does not change
production behavior.
## Result
Current proof level: partially proven.
Recovery confidence: 48 / 100.
The repository contains deployable components, restart/health checks, Windows
collector recovery automation, ActivityWatch DB merge support, retention guard
rails, and validation scripts. The repository does not yet contain evidence of
a completed separate-host restore for the primary repository/Gitea backup, and
does not implement fully automated restore for all runtime data stores.
Production 1.0 release implication: DR is not blocked by missing documentation,
but remains blocked by missing restore evidence for the repository/Gitea backup
and by missing automated restore coverage for several runtime stores.
## Status Definitions
- Implemented: repository contains executable automation or a concrete
validation script for the step.
- Partially implemented: repository contains part of the mechanism, but not an
end-to-end proven path.
- Manual: repository documents the step, but operator execution and evidence
capture are required.
- Missing: repository evidence shows the step is not implemented.
- Unsupported: repository code explicitly rejects or excludes the operation.
## Evidence Inventory
| Area | Status | Evidence |
| --- | --- | --- |
| General backup/recovery model | Manual | `docs/BACKUP_AND_RECOVERY_RU.md:59-71` defines the generic stop, preserve state, restore, start, health, smoke, and record-result sequence. |
| Production health endpoints | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:5-13` defines `/healthz`, `/readyz`, and `/metrics` checks; `docs/OPERATIONS_RUNBOOK_RU.md:18-47` defines expected semantics. |
| Post-recovery smoke | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:49-64` lists smoke scripts and requires them after recovery. |
| Operational maturity validation | Implemented | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59` requires `scripts/public_secret_pattern_check.py`, `scripts/operational-maturity-check.mjs`, and `quality-gate`. |
| Browser/operator validation | Manual | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:64-93` lists required operator pages and secret-handling rules. |
| DetMir production smoke | Implemented | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119` defines `check-aw-full.sh`, `check-aw-data.sh`, and stable `SHARKON2025` host id handling. |
| ActivityWatch service startup | Implemented | `aw-server/activitywatch-server.service:6-14` defines env file, DB path, web path, restart policy, and restart delay. |
| ActivityWatch DB maintenance | Implemented | `aw-server/aw-db-maintenance.service:29-32` and `aw-server/aw-db-maintenance.timer:34-42` define guarded maintenance. |
| Readiness evidence bundle | Implemented | `aw-server/detmir-readiness.service:48-59` and `aw-server/detmir-readiness.timer:60-71` generate daily readiness bundles. |
| Server deploy and API startup validation | Implemented | `ansible/deploy_aw_server.yml:2365-2380` restarts `aw-server/activitywatch-server.service` and waits for `/api/0/info`. |
| ActivityWatch DB merge backup | Partially implemented | `ansible/deploy_aw_server.yml:2222-2350` checks/install merge binary, backs up target and legacy DBs, merges, and installs merged DB when enabled. |
| Rust restore planner | Partially implemented | `scripts/prod_backup_restore.sh:14-29` requires the Rust planner; `adk-rust/crates/prod-backup-restore/src/main.rs:100-105` explicitly rejects `--apply`. |
| Windows package rollback backup | Implemented | `windows/ActivityWatch.Windows.Common.psm1:148-247` backs up install root before replacement and cleans old install backups. |
| Windows recovery loop | Implemented | `windows/ActivityWatch.Windows.Common.psm1:1576-1597` writes recovery script; `windows/ActivityWatch.Windows.Common.psm1:2114-2167` runs the recovery loop. |
| Windows recovery scheduled task | Implemented | `windows/ActivityWatch.Windows.Common.psm1:2447-2495` registers `ActivityWatch Recovery`; `windows/ActivityWatch.Windows.Common.psm1:2656-2672` starts launch tasks and recovery task. |
| Windows deployment validation | Implemented | `ansible/deploy_aw_windows.yml:501-545` checks ActivityWatch API buckets; `ansible/deploy_aw_windows.yml:547-565` runs and fetches endpoint validation. |
| Windows post-deploy validation | Implemented | `ansible/post_validate_aw_windows.yml:55-120` starts recovery/launch tasks and waits for fresh worktime events. |
| Gitea backup | Manual / partially implemented | `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:7-23` documents path, script, timer, format, checksum, retention, and restore-tested status. |
| Gitea restore proof | Missing | `docs/registry/registry-evidence-manifest.json:26-40` records `restore_tested=false` and `production_ready=false`. |
| ClickHouse 1C runtime store | Partially implemented | `clickhouse-1c/docker-compose.yml:1-19` defines the service and persistent Docker volume; no restore automation is present in the repository. |
| ClickHouse Workforce runtime store | Partially implemented | `clickhouse-workforce/docker-compose.yml:1-17` defines the service and persistent Docker volume; no restore automation is present in the repository. |
| Grafana/Prometheus runtime store | Partially implemented | `grafana-1c/docker-compose.yml:36-79` defines Prometheus/Grafana volumes and Prometheus retention; no Grafana data restore automation is present in the repository. |
| Retention boundaries | Implemented for documented cleanup scopes | `docs/RETENTION_POLICY_RU.md:65-102` documents retention and recovery impact for persistent stores. |
## End-to-End Recovery Chain
| Step | Status | Current evidence | Proof conclusion |
| --- | --- | --- | --- |
| Repository | Partially implemented | Primary self-hosted Gitea repository is recorded in `docs/registry/registry-evidence-manifest.json:6-10`; backup configuration is recorded in `docs/registry/registry-evidence-manifest.json:26-40`. | Source repository is identified and backup metadata exists, but restore proof is missing. |
| Configuration | Manual | Git stores sanitized templates and explicitly excludes secrets/live DBs in `docs/BACKUP_AND_RECOVERY_RU.md:29-46`. | Configuration can be reconstructed from repository templates plus customer secret store, but secrets are intentionally external. |
| Deployment | Implemented | Server deployment restarts ActivityWatch and waits for API readiness in `ansible/deploy_aw_server.yml:2365-2380`; Windows deployment runs package install, recovery tasks, and smoke checks in `ansible/deploy_aw_windows.yml:279-565`. | Server and Windows deployment are executable from repository playbooks. |
| Restore | Partially implemented | AW DB merge path exists in `ansible/deploy_aw_server.yml:2222-2350`; Rust planner emits a plan but rejects `--apply` in `adk-rust/crates/prod-backup-restore/src/main.rs:100-105`; Gitea restore is manual and untested per `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`. | Restore is only partially proven. Several runtime stores require manual or future automation. |
| Startup | Implemented | `aw-server/activitywatch-server.service` has restart policy in `aw-server/activitywatch-server.service:6-14`; Windows recovery task and loop are implemented in `windows/ActivityWatch.Windows.Common.psm1:2114-2167` and `:2447-2495`. | Repository supports service startup and collector recovery startup. |
| Health | Implemented | `/healthz`, `/readyz`, `/metrics` are documented in `docs/OPERATIONS_RUNBOOK_RU.md:5-47`; server deployment waits for `/api/0/info` in `ansible/deploy_aw_server.yml:2372-2380`. | Health checks are present and part of deploy/recovery validation. |
| Operational validation | Implemented | Operational gates are documented in `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59`; production smoke is documented in `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`. | Repository has repeatable operational validation commands. |
| Pilot validation | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:49-64` lists pilot/demo/deployment smoke scripts after recovery; `scripts/pilot-validation-smoke.mjs` and `scripts/deployment-readiness-smoke.mjs` exist. | Pilot/deployment smoke validation is implemented as repository scripts. |
| Ready | Manual / conditional | General recovery procedure requires result recording in `docs/BACKUP_AND_RECOVERY_RU.md:59-71`; Gitea manifest still says `production_ready=false` in `docs/registry/registry-evidence-manifest.json:37-40`. | Ready can only be claimed after external restore evidence is captured and manifest gaps are closed. |
## Current Recovery Capability
### Server
Implemented:
- ActivityWatch server restart and API readiness wait through Ansible.
- Systemd restart-on-failure for `aw-server/activitywatch-server.service`.
- Guarded SQLite maintenance and daily readiness bundle timers.
- Limited AW DB merge/recovery-like flow with pre-merge backups.
Partially implemented:
- Generic production restore planning through `prod-backup-restore`, because the
Rust binary builds a plan but rejects apply mode.
- ActivityWatch DB restore, because the repository implements legacy DB merge,
but not a generic restore-selected-backup command.
Missing:
- Evidence of a completed end-to-end AW DB restore drill.
- Automated rollback from a failed DB merge to the pre-merge backup.
### Windows/RDP
Implemented:
- Package replacement with install-root backup.
- Recovery script generation.
- Scheduled `ActivityWatch Recovery` task.
- Long-running recovery loop that restarts collectors/tasks for live sessions.
- Post-deploy validation that waits for fresh ActivityWatch worktime events.
Partially implemented:
- Full Windows state restore, because install backups exist but
`C:\ProgramData\AWatch-rus` restore as a whole is not implemented.
### Repository/Gitea
Manual / partially implemented:
- Backup target, script name, systemd unit/timer names, format, checksum, and
retention are documented.
- Restore procedure is documented as an outline for a separate server.
Missing:
- Actual separate-host restore evidence.
- Manifest update proving `restore_tested=true`.
- Offsite copy evidence.
### ClickHouse, Grafana, Prometheus
Partially implemented:
- Docker Compose files define persistent volumes and restart policies.
- Prometheus retention is configurable through compose.
- Retention policy documents recovery impact for ClickHouse, Grafana, and
Prometheus stores.
Missing:
- Automated backup and restore for ClickHouse 1C volume.
- Automated backup and restore for ClickHouse Workforce volume.
- Automated Grafana data restore.
- End-to-end restore evidence for these stores.
## Current Limitations
1. Gitea backup cannot be called production-ready until a separate-host restore
is performed and recorded. Evidence: `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:3-5`,
`docs/registry/registry-evidence-manifest.json:37-40`.
2. `prod-backup-restore --apply` is unsupported by code. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
3. ActivityWatch DB recovery is limited to legacy-root merge and does not prove
generic restore from a selected backup file. Evidence:
`ansible/deploy_aw_server.yml:2222-2350`.
4. ClickHouse/Grafana/Prometheus data volumes are declared, but repository
restore automation is not present. Evidence:
`clickhouse-1c/docker-compose.yml:13-19`,
`clickhouse-workforce/docker-compose.yml:11-17`,
`grafana-1c/docker-compose.yml:36-79`.
5. Customer secrets are intentionally outside Git. Recovery therefore requires
access to the customer secret store and cannot be proven from repository
contents alone. Evidence: `docs/BACKUP_AND_RECOVERY_RU.md:29-46`.
6. Evidence/customer data must not be placed in the public repository. Evidence:
`docs/BACKUP_AND_RECOVERY_RU.md:91-98`.
## Estimated Recovery Sequence
This sequence is the current evidence-backed chain. Steps marked manual require
operator execution and external evidence capture.
1. Repository availability: clone the primary Gitea repository or validated
mirror. Status: partially implemented. Evidence:
`docs/registry/registry-evidence-manifest.json:6-10`.
2. Repository restore, if primary Gitea is lost: follow the Gitea restore
outline on a separate host, verify checksum, run Gitea checks, and record
evidence. Status: manual / missing proof. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`.
3. Configuration recovery: restore sanitized templates from Git and secrets
from the approved customer secret store. Status: manual. Evidence:
`docs/BACKUP_AND_RECOVERY_RU.md:29-46`.
4. Server deployment: run server deployment playbook and wait for
`/api/0/info`. Status: implemented. Evidence:
`ansible/deploy_aw_server.yml:2365-2380`.
5. AW DB merge path, only when legacy-root merge is explicitly enabled: back up
target and legacy DBs, merge, install merged DB. Status: partially
implemented. Evidence: `ansible/deploy_aw_server.yml:2222-2350`.
6. Windows/RDP deployment: run Windows deployment playbook, deploy package,
configure recovery tasks, and validate bucket events. Status: implemented.
Evidence: `ansible/deploy_aw_windows.yml:279-565`.
7. Windows post-restore validation: start recovery/launch tasks and wait for
worktime events. Status: implemented. Evidence:
`ansible/post_validate_aw_windows.yml:55-120`.
8. Service startup validation: check systemd status, `/healthz`, `/readyz`,
`/metrics`, and `/api/0/info`. Status: implemented. Evidence:
`docs/OPERATIONS_RUNBOOK_RU.md:5-47`,
`ansible/deploy_aw_server.yml:2372-2380`.
9. Production smoke: run `check-aw-full.sh`, `check-aw-data.sh`, and contour
smoke with stable `SHARKON2025` logical host id. Status: implemented.
Evidence: `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`.
10. Operational maturity validation: run secret scan, operational maturity
check, and quality gate. Status: implemented. Evidence:
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59`.
11. Pilot/deployment readiness validation: run pilot and deployment smoke
scripts after recovery. Status: implemented. Evidence:
`docs/OPERATIONS_RUNBOOK_RU.md:49-64`.
12. Ready decision: record recovery result and do not claim production-ready DR
until missing restore evidence is closed. Status: manual / conditional.
Evidence: `docs/BACKUP_AND_RECOVERY_RU.md:59-71`,
`docs/registry/registry-evidence-manifest.json:37-40`.
## Evidence Gaps
| Gap | Status | Required evidence before closing |
| --- | --- | --- |
| Gitea separate-host restore | Missing | Backup filename, SHA256 verification output, Gitea version, restore duration, post-restore checks, clone/access proof, and manifest update. |
| Generic AW DB restore | Missing | Tested command or runbook restoring a selected backup into active DB with checksum and rollback evidence. |
| ClickHouse 1C restore | Missing | Tested volume/table backup and restore evidence for `clickhouse_1c_data`. |
| ClickHouse Workforce restore | Missing | Tested volume/table backup and restore evidence for `clickhouse_workforce_data`. |
| Grafana data restore | Missing | Tested restore evidence for `grafana-data` or explicit proof that provisioned dashboards plus documented credentials are sufficient. |
| Off-host backup copy | Missing | Repository evidence of destination, retention, checksum policy, access policy, and restore test. |
## Recommended Future Automation
These are future tasks, not implemented by this document:
1. Add a non-destructive restore drill checklist artifact that records exact
backup filename, checksum, host, duration, and post-restore checks.
2. Add a separate-host Gitea restore evidence template and only then update
`restore_tested` in `docs/registry/registry-evidence-manifest.json`.
3. Add tested backup/restore automation for ClickHouse 1C and Workforce volumes
or explicitly document that those stores are rebuilt from source exports.
4. Add a generic ActivityWatch DB restore runbook with rollback command and
checksum verification.
5. Add a single post-restore validation command that runs health, operational
maturity, pilot validation, and deployment readiness in the documented order.
## Release Decision
DR proof is sufficient to show that the repository contains significant
recoverability mechanisms and validation gates.
DR proof is not sufficient to claim full Production 1.0 disaster recovery until
the missing separate-host restore evidence and runtime store restore gaps are
closed.
+311
View File
@@ -0,0 +1,311 @@
# Production Evidence Gaps
Date: 2026-07-01
Scope: final evidence still missing before Release Candidate readiness for the
current AWatch-rus / DetMir production deployment.
This document does not add functionality, redesign recovery, or define new
tests. It records only evidence gaps confirmed by existing repository backlog,
runbooks, audit reports, validation scripts, and registry-readiness documents.
## Summary
The repository already contains operational validation scripts, pilot/deployment
smoke scripts, retention policy, recovery audit, disaster recovery proof, and
release evidence tooling. The remaining gap is not lack of plans. The remaining
gap is live or controlled-environment evidence proving that the current
production deployment and release candidate can be recovered, traced, rebuilt,
and operated under expected load.
## Missing Evidence Items
| ID | Evidence gap | RC priority | Requires live production environment |
| --- | --- | --- | --- |
| PEG-001 | Production binary parity evidence | P0 | Yes |
| PEG-002 | Separate-host Gitea restore proof | P0 | Separate restore host; production backup input |
| PEG-003 | Runtime data restore proof for production stores | P0 | Yes or isolated restore clone |
| PEG-004 | First controlled release evidence package from `awatch-build-01` | P0 | Controlled build-runner, not production runtime |
| PEG-005 | Portal/worktime hot-path load evidence | P0 | Prefer production-like or live low-risk window |
| PEG-006 | Capacity baseline and headroom evidence | P1 | Yes |
| PEG-007 | Backup verification and off-host backup evidence | P1 | Yes |
| PEG-008 | Install kit reproducibility and stale payload evidence | P1 | Controlled build/staging environment |
| PEG-009 | Reviewed PR / release governance evidence | P1 | GitHub/Gitea governance environment |
## PEG-001: Production Binary Parity Evidence
Why it matters: Release Candidate confidence depends on proving that binaries
actually running in production match reviewed release artifacts. Otherwise test
results can refer to one binary while production runs another.
Current state: repository gate implemented by TASK_072; live production
evidence still missing. The existing local artifact check verifies release
artifacts, and `scripts/check_production_binary_parity.py` now validates
deployed unit/timer/task to production SHA256 and source crate mapping against
local release artifacts and repository Git SHA. Release Candidate approval still
requires an operator-collected production evidence JSON from the current
production deployment.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:65-105`
- `PRODUCTION_READINESS_REPORT.md:209-212`
- `scripts/check_detmir_rust_release_artifacts.sh`
- `scripts/check_production_binary_parity.py`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
How it could be verified: collect production binary paths from actual
systemd/timer/Windows scheduled task inventory, compute production SHA256,
save evidence JSON, then run `scripts/check_production_binary_parity.py` to
compare with local release artifact SHA256 and record
`service/timer/task -> binary path -> crate -> runtime role -> production sha256
-> release sha256 -> git sha`.
Expected operational benefit: removes stale-binary ambiguity during incident
response, rollback, and RC approval.
Risk if left unverified: production may run stale or locally patched binaries
while release evidence and tests refer to different artifacts.
## PEG-002: Separate-Host Gitea Restore Proof
Why it matters: Source repository backup is not a recovery capability until a
restore has been proven on a separate host.
Current state: Missing. Registry evidence explicitly records
`restore_tested=false` and `production_ready=false`; the restore runbook states
that backup cannot be production-ready until a separate test restore is
completed.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:154-188`
- `RECOVERY_AUDIT.md:250-263`
- `DISASTER_RECOVERY_PROOF.md`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`
- `docs/registry/registry-evidence-manifest.json:26-40`
How it could be verified: execute the existing Gitea restore runbook on a
separate host, verify backup SHA256, run `gitea doctor check`, verify restored
repository access/clone, record backup filename, checksum, Gitea version,
restore duration, post-restore output, and rollback notes.
Expected operational benefit: proves the source repository can be recovered
after primary Git/Gitea loss.
Risk if left unverified: repository loss recovery remains assumed rather than
proven.
## PEG-003: Runtime Data Restore Proof for Production Stores
Why it matters: Service restart is not enough for disaster recovery if primary
runtime data stores cannot be restored after corruption or loss.
Current state: Missing / partially implemented. Recovery automation exists for
service restart and Windows collector restart. Generic restore evidence is still
missing for ActivityWatch active SQLite DB, ClickHouse 1C, ClickHouse Workforce,
Grafana data, Prometheus TSDB, DLP state/evidence, Hayabusa archive, Windows
state root, diagnostic bundles, and release evidence.
Repository evidence:
- `RECOVERY_AUDIT.md:272-281`
- `RECOVERY_AUDIT.md:379-428`
- `DISASTER_RECOVERY_PROOF.md`
- `docs/RETENTION_POLICY_RU.md:65-102`
How it could be verified: for each production store, use existing backup or
restore documentation where present, perform restore into an isolated target,
then run existing health and validation scripts. Where the repository says
restore is not currently implemented, record the gap rather than inventing a
procedure.
Expected operational benefit: identifies which production data can actually be
recovered and which data still depends on external/customer backup handling.
Risk if left unverified: an outage may be recoverable at the service level but
not at the data level.
## PEG-004: First Controlled Release Evidence Package from `awatch-build-01`
Why it matters: GitHub Actions are public validation only. RC release authority
requires a controlled build-runner evidence package with source archive, binary
archive, checksums, metadata, logs, and manifest.
Current state: Tooling exists and was strengthened by TASK_070, but the
registry-readiness documents still require the first real release evidence build
on the Russian build-runner. The build-runner status is planned, not proven
production-ready.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:107-149`
- `docs/registry/RU_BUILD_RUNNER_READINESS_RU.md`
- `docs/registry/BUILD_RUNNER_SETUP_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_MANIFEST_RU.md`
- `docs/registry/registry-evidence-manifest.json:41-75`
- `scripts/build_release_evidence.sh`
- `scripts/check_release_evidence.sh`
How it could be verified: run `scripts/build_release_evidence.sh` on the
controlled runner for the RC commit, then run
`scripts/check_release_evidence.sh <evidence-dir>` and preserve the generated
manifest, logs, source archive, binary archive, SHA256SUMS, cargo metadata/tree,
and documented skips.
Expected operational benefit: provides auditable release provenance independent
of the public mirror.
Risk if left unverified: RC could be validated only by public mirror CI rather
than by the release authority expected by the registry-readiness contour.
## PEG-005: Portal/Worktime Hot-Path Load Evidence
Why it matters: Production 1.0 should not regress under report, prewarm,
ActivityWatch query, cache, and operator portal load.
Current state: Missing. The backlog still lists `P0-6. Portal/worktime hot-path
load gate`. Existing operational maturity checks include offline bounded-load
validation, but repository evidence still requires an explicit portal/worktime
hot-path gate before Production 1.0.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:281-319`
- `scripts/operational-maturity-check.mjs`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`
How it could be verified: run the existing operational maturity bounded-load
harness and production smoke sequence against a production-like or controlled
live window, record p95 latency, memory growth, cache/stale behavior, AW query
duration, and failure semantics. Keep any heavy job advisory/scheduled rather
than blocking fast smoke.
Expected operational benefit: proves that the current 5-user production
deployment has headroom for normal operator/report usage.
Risk if left unverified: RC may pass functional checks while still being fragile
under repeated report/prewarm/operator access.
## PEG-006: Capacity Baseline and Headroom Evidence
Why it matters: Operators need a measured baseline for CPU, RAM, disk, queue,
ClickHouse, Grafana, and ActivityWatch behavior before scaling beyond the
current small deployment.
Current state: Missing / P1. The roadmap explicitly places capacity monitoring
after P0 load gate data. Retention policy documents storage areas and cleanup
impact, but not production headroom evidence.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:778-805`
- `docs/RETENTION_POLICY_RU.md:65-102`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:187-209`
How it could be verified: collect production baseline snapshots for disk usage,
service health, ClickHouse table sizes, queue/backlog state, and operational
smoke output over an agreed observation window.
Expected operational benefit: gives operators thresholds for safe growth and
early warning before disk or load incidents.
Risk if left unverified: capacity problems may be discovered only after
operator-visible degradation.
## PEG-007: Backup Verification and Off-Host Backup Evidence
Why it matters: Retention prevents uncontrolled growth, but backup verification
proves recoverability. A local backup without checksum/off-host evidence can
fail during actual disaster recovery.
Current state: Missing / partial. Gitea backup is documented with checksum and
timer metadata. The recovery audit records missing scheduled full AW DB backup,
off-host backup copy, and restore ownership for several runtime stores.
Repository evidence:
- `RECOVERY_AUDIT.md:128-149`
- `RECOVERY_AUDIT.md:306-321`
- `RECOVERY_AUDIT.md:326-338`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:25-40`
- `docs/RETENTION_POLICY_RU.md:65-102`
How it could be verified: use existing backup locations/runbooks where present,
verify checksums, confirm backup age/retention, confirm off-host copy status if
owned outside the repository, and record components that remain without a repo
backup/restore path.
Expected operational benefit: separates "data is retained" from "data can be
restored".
Risk if left unverified: backups may be absent, stale, local-only, or
unreadable when needed.
## PEG-008: Install Kit Reproducibility and Stale Payload Evidence
Why it matters: Windows installer payload drift can deploy old collectors even
when repository scripts and CI are current.
Current state: Missing / P1. The backlog lists install kit reproducibility and
stale payload gate as open.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:441-475`
- `scripts/rebuild_install_kit.sh`
- `scripts/validate_install_kit.sh`
- `scripts/check_install_kit_vs_repo.sh`
- `adk-rust/crates/rebuild-install-kit/`
How it could be verified: rebuild the install kit on the controlled runner,
generate or verify payload manifest with source commit and SHA256 values, then
compare packaged collector payloads with repository state and release artifacts.
Expected operational benefit: prevents deploying stale Windows collector
payloads during RC rollout.
Risk if left unverified: endpoint deployment can silently diverge from the
reviewed release.
## PEG-009: Reviewed PR / Release Governance Evidence
Why it matters: Production 1.0 closure requires visible evidence that protected
branch, CODEOWNERS, and PR review processes are not only documented but used for
release-quality changes.
Current state: Missing / P1. Branch protection evidence exists, but the backlog
still lists reviewed PR and release governance evidence as open.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:519-552`
- `docs/BRANCH_PROTECTION_EVIDENCE_RU.md`
- `docs/PR_REVIEW_EVIDENCE_RU.md`
- `scripts/registry_readiness_check.sh`
How it could be verified: record a reviewed PR flow with required checks,
CODEOWNERS/reviewer approval, no admin bypass, and final merge evidence for the
RC branch or release preparation branch.
Expected operational benefit: proves that Production 1.0 changes pass the
intended governance workflow.
Risk if left unverified: release governance remains documented but not proven
on the actual RC workflow.
## Release Candidate Evidence Gate
Release Candidate readiness should not be declared until the P0 evidence gaps
are closed or explicitly accepted by the operator with written risk acceptance:
1. PEG-001 production binary parity evidence.
2. PEG-002 separate-host Gitea restore proof.
3. PEG-003 runtime data restore proof or documented operator acceptance for
stores without repository restore support.
4. PEG-004 controlled release evidence package from `awatch-build-01`.
5. PEG-005 portal/worktime hot-path load evidence.
P1 evidence can remain after RC only if it is explicitly tracked as release
follow-up and does not invalidate the operator's Production 1.0 risk decision.
+332
View File
@@ -0,0 +1,332 @@
# TASK_065: Production readiness report
Audit date: 2026-07-01.
Scope: repository-level engineering and operational audit for AWatch-rus
Production 1.0 readiness.
Production context: the project is already deployed in a small company
environment with approximately 5 RDP users. Stability, rollback, diagnostics
and maintainability have priority over new functionality.
## Method
The audit used repository evidence only. No readiness claim in this report is
based on an unverified assumption.
Reviewed evidence:
- `README.md`, `CONTRIBUTING.md`, `SECURITY.md`, `ROADMAP.md`.
- `docs/PROJECT_STATUS_RU.md`, `docs/RESIDUAL_RISKS_RU.md`,
`docs/QUALITY_STATUS_RU.md`, `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`,
`docs/DLP_OPTIONAL_RUNTIME_RU.md`, `docs/DLP_RESOURCE_PROFILES_RU.md`,
`docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md`.
- `docs/registry/*` including `registry-evidence-manifest.json`.
- `.github/workflows/*`.
- `scripts/*`, `scripts/detmir-full-diagnostics/*`.
- `adk-rust/` workspace manifests and crate tree.
- `ansible/`, `aw-server/`, `windows/`, `proxmox/`, `clickhouse-1c/`,
`grafana/`, `pfsense/`.
- `configs/operational-maturity-contract.json`.
Commands used for evidence gathering:
```bash
git ls-files | wc -l
find adk-rust/crates -maxdepth 2 -name Cargo.toml
cargo metadata --locked --format-version 1
cargo audit --deny warnings
cargo machete --with-metadata
cargo tree --duplicates --locked
cargo deny check --config ../deny.toml --hide-inclusion-graph --show-stats
wc -l adk-rust/crates/detmir-portal/src/main.rs adk-rust/crates/aw-windows-telemetry/src/main.rs proxmox/tsj_guardian_bot.py adk-rust/crates/worktime-api/src/main.rs ansible/deploy_aw_server.yml
sha256sum scripts/aw-contour-diag.sh scripts/detmir-full-diagnostics/aw-contour-diag.sh scripts/check_production_inventory_placeholders.sh scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh
rg -n "CLICKHOUSE_PASSWORD|--password|password" clickhouse-1c/ops clickhouse-1c/ai clickhouse-1c/etl
```
## Implemented Capability Inventory
### Core Runtime
- Rust workspace under `adk-rust/` with 58 crates.
- ActivityWatch checks and wrappers: `check-aw-data`, `check-aw-full`,
`aw-health-check`, `aw-rus-healthd`, `aw-slo-monitor`.
- DetMir portal: Rust HTML/API portal with contracts, metrics, readiness,
reports, operator/manager/security views and role-based access logic.
- Worktime stack: `worktime-api`, `worktime-prewarm`,
`worktime-influx-exporter`, `worktime-autoheal`,
`worktime-ui-bridge`, `rdp-worktime-report`.
- Windows telemetry: `aw-windows-telemetry` for collectors, validation,
file-1C upload, DLP evidence sync and collector guard paths.
- Release/install tooling: `rebuild-install-kit`, `validate-install-kit`,
`verify-innosetup-installer`, `check-install-kit-vs-repo`.
### Workforce / 1C / ClickHouse
- `aw-workforce-ingest` and `aw-1c-ingest` exist for workforce and 1C
ingestion paths.
- `clickhouse-1c/` includes ETL, SQL, Grafana provisioning and operational
wrappers.
- Grafana dashboards are version-controlled under `grafana/` and related
ClickHouse/Grafana directories.
### Security / DLP / Forensics
- DLP server-side helpers exist as Rust crates:
`dlp-policy-engine`, `dlp-case-management`, `dlp-compliance`,
`dlp-aggregator`, `dlp-health-check`, `dlp-content-analyzer`,
exporters/senders.
- DLP production runtime is intentionally conservative:
`core_only/disabled` default with documented `light` profile and load guard.
- Security Finding Inbox and Hayabusa/Velociraptor findings paths are optional
and separated from Workforce hot path.
- Hayabusa tooling exists under `hayabusa-tools` and `aw-server/hayabusa/`.
### Deployment / Operations
- Ansible deployment exists for server and Windows contours.
- Windows deployment and recovery scripts exist under `windows/`.
- Proxmox/pfSense support assets exist under `proxmox/` and `pfsense/`.
- Operational wrappers exist at root and under `scripts/`.
- `scripts/operational-maturity-check.mjs` validates API compatibility,
fixtures, fault injection, bounded load, config, systemd, ClickHouse
migration and observability contracts.
### Governance / Release
- Public CI workflows exist.
- Security workflow includes cargo audit, cargo deny, secret pattern scan and
dependency review.
- Dependency hygiene workflow includes cargo metadata, machete, duplicates,
audit, deny and advisory udeps.
- Registry-readiness documentation exists under `docs/registry/`.
- CODEOWNERS, PR template, review checklist and branch protection evidence docs
exist.
## Confirmed Gap Analysis
### Technical Debt
- Large modules increase review risk:
- `adk-rust/crates/detmir-portal/src/main.rs`: 14200 lines.
- `adk-rust/crates/aw-windows-telemetry/src/main.rs`: 6411 lines.
- `proxmox/tsj_guardian_bot.py`: 4610 lines.
- `adk-rust/crates/worktime-api/src/main.rs`: 3988 lines.
- `ansible/deploy_aw_server.yml`: 3099 lines.
- PowerShell fallback remains necessary and documented; it is not dead code, but
it increases parity and validation burden.
### Duplicated Logic
- Exact duplicate scripts confirmed by SHA256:
- `scripts/aw-contour-diag.sh`
- `scripts/detmir-full-diagnostics/aw-contour-diag.sh`
- `scripts/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`
### Outdated Or Pending Documentation
- `docs/PROJECT_STATUS_RU.md` still records first reviewed PR evidence as
pending and contains historical required check names.
- `docs/RESIDUAL_RISKS_RU.md` records Gitea restore test, build-runner,
release evidence and legal package as open.
- `ROADMAP.md` records coverage threshold and Russian OS compatibility as
planned, not complete.
### Obsolete Or Deprecated Components
- Some Ansible DLP roles are explicitly marked deprecated because they deployed
old service paths.
- `serde_yaml 0.9.34+deprecated` is documented as a medium third-party risk.
- Legacy scripts remain as fallback/reference; they should not be removed
without parity gates.
### Missing Operational Checks
- No confirmed repository check currently proves production binary SHA parity
across all actually running units/timers/tasks and local release artifacts.
- Retention/cleanup policy for long-lived state/evidence/diagnostic output is
not yet complete.
- Existing operational maturity bounded load is useful, but does not yet cover
Production 1.0 scale scenarios for 5/20/50 users and portal/worktime prewarm.
### Missing Tests
- Load regression tests for portal/worktime full report and prewarm hot paths
are not yet sufficient for Production 1.0 scale confidence.
- Windows Rust validation parity still needs canary evidence against the
PowerShell validation path.
### Security Gaps
- ClickHouse/1C ops wrappers pass `CLICKHOUSE_PASSWORD` via `--password`, which
exposes secrets in process argv.
- `cargo deny` passes but currently permits non-blocking duplicate/wildcard
dependency warnings. This is not an immediate vulnerability, but it needs a
Production 1.0 baseline.
## Readiness Scores
Scoring scale:
- 90-100: production-ready with evidence.
- 75-89: strong, but with bounded gaps.
- 60-74: usable in current production, but not yet 1.0 release-grade.
- below 60: material blocker.
| Category | Score | Justification |
|---|---:|---|
| Architecture status | 82 | Rust-first runtime, documented boundaries and conservative DLP separation exist. Large modules remain maintainability risk. |
| Repository health | 80 | 910 tracked files, clear ownership areas and runbooks. Some duplicate scripts and historical docs remain. |
| Dependency health | 78 | Audit and machete pass; 349 packages; cargo deny exits 0. Remaining deny warnings and duplicate roots need baseline policy. |
| CI health | 82 | CI/security/coverage/dependency/operational workflows exist. Toolchain drift remains between pinned `1.94.0` and floating `stable`. |
| Operational maturity | 84 | Offline operational maturity harness covers compatibility, fixtures, fault injection, bounded load, config and observability. Production-scale load gate still needed. |
| Documentation status | 78 | Extensive docs and runbooks exist. Current-state docs need cleanup around historical statuses and pending evidence. |
| Security status | 76 | Secret scan, audit, deny and conservative claims exist. ClickHouse password-in-argv is a direct hygiene gap. |
| Testing status | 80 | Full Rust pipeline recently passed and operational smokes exist. Missing scale/load and Windows parity evidence remain. |
| Deployment readiness | 76 | Ansible, Windows install kit, runbooks and wrappers exist. Install kit stale-payload gate and production binary parity still missing. |
| Upgrade readiness | 72 | Release scripts and rollback docs exist, but controlled release evidence and binary parity are not proven. |
| Recovery readiness | 70 | Recovery runbooks and backups exist; Gitea restore test is not done. |
| Configuration validation | 82 | Operational maturity validates JSON/YAML/systemd/ClickHouse files; production inventory placeholder checks exist. Coverage must be extended to retention/binary parity. |
| Observability | 84 | Metrics contract and operational maturity observability checks exist; capacity metrics need scale scenarios. |
| Support readiness | 78 | Many runbooks exist and DetMir guardrails are strong; docs need current-state cleanup for 1.0. |
| Maintainability | 72 | Strong tests and Rust-first direction, but large files and fallback parity increase maintenance cost. |
Overall Production Readiness Score: **78 / 100**.
Recommended release decision: **not yet Production 1.0**. The project is fit
for the current small production/pilot environment with conservative runtime
guardrails, but Production 1.0 should wait until P0 blockers in
`DEVELOPMENT_PLAN_NEXT.md` are closed and evidenced.
## Risk Assessment
### High Risk
1. Production binary drift
- Probability: medium.
- Impact: high.
- Description: running binaries may not match reviewed release artifacts.
- Mitigation: implement production binary parity gate.
- Complexity: medium.
2. Missing controlled release evidence
- Probability: high.
- Impact: high.
- Description: GitHub mirror validation is not release evidence.
- Mitigation: controlled runner release evidence build.
- Complexity: medium.
3. Untested repository restore
- Probability: medium.
- Impact: high.
- Description: backup exists, but restore is not proven.
- Mitigation: separate-host restore drill.
- Complexity: medium.
4. Unbounded operational artifacts
- Probability: medium.
- Impact: high.
- Description: state/evidence/diagnostic artifacts can accumulate and fill
disks.
- Mitigation: allowlisted retention and cleanup policy.
- Complexity: medium.
5. ClickHouse password in process argv
- Probability: high on affected scripts.
- Impact: high.
- Description: local process listing can expose credentials.
- Mitigation: remove password from argv and verify with `ps`.
- Complexity: low-medium.
6. Portal/worktime hot-path overload
- Probability: medium.
- Impact: high.
- Description: full report/snapshot prewarm remains documented as CPU/IO
expensive.
- Mitigation: Production 1.0 load gate with synthetic datasets.
- Complexity: medium-high.
### Medium Risk
1. CI toolchain drift
- Probability: medium.
- Impact: medium.
- Mitigation: align workflows to pinned toolchain.
- Complexity: low.
2. Dependency warning drift
- Probability: medium.
- Impact: medium.
- Mitigation: baseline current warnings and block new unapproved warnings.
- Complexity: medium.
3. Windows validation parity gap
- Probability: medium.
- Impact: medium-high.
- Mitigation: Rust/PowerShell canary comparison.
- Complexity: medium.
4. Install kit stale payload
- Probability: medium.
- Impact: medium-high.
- Mitigation: install kit manifest and validation gate.
- Complexity: medium.
5. Documentation drift
- Probability: high.
- Impact: medium.
- Mitigation: current-state cleanup and historical labeling.
- Complexity: low-medium.
6. Large module maintainability
- Probability: high.
- Impact: medium.
- Mitigation: incremental extraction with tests.
- Complexity: medium-high.
### Low Risk
1. Exact duplicate diagnostic scripts
- Probability: medium.
- Impact: low-medium.
- Mitigation: canonical implementation or drift check.
- Complexity: low.
2. Coverage threshold not yet enforced
- Probability: medium.
- Impact: low for current production, medium long-term.
- Mitigation: baseline review, advisory threshold, later blocking policy.
- Complexity: medium.
3. Russian OS compatibility not yet matrixed
- Probability: low for current DetMir, medium for wider distribution.
- Impact: medium for new deployments.
- Mitigation: compatibility matrix with evidence.
- Complexity: medium-high.
## Known Limitations
- The audit did not claim legal readiness, certification or registry
submission completion.
- The audit did not validate live production services during this documentation
update.
- Heavy DLP, Loki and always-on Velociraptor are intentionally not required for
Production 1.0.
- PowerShell fallback remains a supported rollback/support path until parity
evidence allows retirement.
## Release Readiness Summary
Production 1.0 should be blocked on:
1. production binary parity evidence;
2. controlled release evidence build;
3. Gitea restore drill;
4. bounded retention/cleanup;
5. ClickHouse password argv fix;
6. portal/worktime hot-path load gate.
After these are complete, the project can reasonably move from current small
production/pilot readiness to Production 1.0 readiness, assuming validation
passes and no new runtime regressions are introduced.
+10
View File
@@ -271,8 +271,18 @@ collectors.
### Engineering governance and residual risks
- Enterprise quality standard:
[docs/ENTERPRISE_QUALITY_STANDARD_RU.md](docs/ENTERPRISE_QUALITY_STANDARD_RU.md).
- GitHub governance entrypoint:
[.github/GOVERNANCE.md](.github/GOVERNANCE.md).
- Review checklist:
[docs/REVIEW_CHECKLIST_RU.md](docs/REVIEW_CHECKLIST_RU.md).
- Operational validation runbook:
[docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md).
- Daily maintenance runbook with optional Pollinations AI evidence assistant:
[docs/DAILY_MAINTENANCE_RU.md](docs/DAILY_MAINTENANCE_RU.md).
- Operational maturity harness:
[docs/OPERATIONAL_MATURITY_RU.md](docs/OPERATIONAL_MATURITY_RU.md).
- Residual risks register:
[docs/RESIDUAL_RISKS_RU.md](docs/RESIDUAL_RISKS_RU.md).
- Public issues plan:
+443
View File
@@ -0,0 +1,443 @@
# Operational Recovery Audit
Date: 2026-07-01
Scope: repository-first audit of the current AWatch-rus / DetMir operational
recovery capability. This document describes only mechanisms already present in
the repository. It does not define new backup or recovery mechanisms.
## Evidence Sources
The audit reviewed these repository sources:
- `docs/BACKUP_AND_RECOVERY_RU.md`
- `docs/RETENTION_POLICY_RU.md`
- `docs/OPERATIONS_RUNBOOK_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `docs/DETMIR_RESTORE_BASELINE_2026-06-29_RU.md`
- `scripts/prod_backup_restore.sh`
- `adk-rust/crates/prod-backup-restore/src/main.rs`
- `ansible/deploy_aw_server.yml`
- `ansible/deploy_aw_windows.yml`
- `ansible/post_validate_aw_windows.yml`
- `aw-server/*.service`, `aw-server/*.timer`, `aw-server/*.path`
- `windows/ActivityWatch.Windows.Common.psm1`
- `windows/hardening-recovery.ps1`
- `windows/rebuild-worktime-tasks.ps1`
- `windows/fix-session-watchers.ps1`
- `windows/cleanup-disc-sessions.ps1`
- `clickhouse-1c/docker-compose.yml`
- `clickhouse-workforce/docker-compose.yml`
- `grafana-1c/docker-compose.yml`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md`
- `docs/registry/registry-evidence-manifest.json`
## Current Recovery Procedure Actually Supported
### Server-side service recovery
The repository supports service restart and health validation through systemd,
Ansible deployment, and smoke checks:
- `activitywatch-server.service` restarts on failure and runs with the configured
AW server env file. Evidence: `aw-server/activitywatch-server.service`.
- `ansible/deploy_aw_server.yml` installs and restarts
`activitywatch-server.service`, then waits for `/api/0/info`. Evidence:
`ansible/deploy_aw_server.yml:2365-2375`.
- The generic operations runbook requires checking `/healthz`, `/readyz`,
`/metrics`, service status, journal, and smoke scripts after recovery.
Evidence: `docs/OPERATIONS_RUNBOOK_RU.md:59-71`.
- The production validation runbook defines the DetMir smoke sequence through
`check-aw-data.sh`, `check-aw-full.sh`, and local contour smoke. Evidence:
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
### ActivityWatch SQLite maintenance and recovery support
The repository supports guarded maintenance and limited DB merge/migration
flows, not a complete automated restore:
- Weekly guarded SQLite maintenance is installed as
`aw-db-maintenance.service` / `aw-db-maintenance.timer`. Evidence:
`aw-server/aw-db-maintenance.service`, `aw-server/aw-db-maintenance.timer`.
- Optional SQLite vacuum is defined as `aw-db-vacuum.service` /
`aw-db-vacuum.timer`. It is opt-in in Ansible through
`aw_db_vacuum_timer_enabled`. Evidence:
`aw-server/aw-db-vacuum.service`, `aw-server/aw-db-vacuum.timer`,
`ansible/deploy_aw_server.yml:613-649`.
- Legacy root DB merge is implemented in `ansible/deploy_aw_server.yml` when
`aw_legacy_db_merge_enabled` is true. It stops the service, backs up target
and legacy DB files, merges them with `merge-aw-server-dbs`, installs the
merged DB, restarts the service, and waits for API readiness. Evidence:
`ansible/deploy_aw_server.yml:2248-2375`.
- A separate Rust `prod-backup-restore` binary exists only as a plan/checker.
It explicitly rejects `--apply`. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
### Windows collector recovery
The Windows side has the strongest implemented recovery automation:
- Deployment writes `deployment-config.json`, launcher scripts, and
`recovery-loop.ps1`. Evidence:
`windows/deploy-domain-users.ps1`, `windows/hardening-recovery.ps1`.
- `Write-ActivityWatchRecoveryScript` generates a script that imports
`ActivityWatch.Windows.Common.psm1` and calls
`Invoke-ActivityWatchRecoveryLoop`. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:1576-1597`.
- `Register-ActivityWatchRecoveryTask` creates the scheduled task
`ActivityWatch Recovery`, using an interactive user when possible and SYSTEM
fallback otherwise. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:2447-2488`.
- `Invoke-ActivityWatchRecoveryLoop` is an actual loop: it uses a lock file,
cleans non-live session processes, starts the worktime session collector when
allowed, starts configured live user launch tasks, and uses console fallback.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:2114-2168`.
- `Start-ActivityWatchTasks` starts launch tasks for live users and starts the
recovery task. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:2656-2672`.
- `rebuild-worktime-tasks.ps1`, `fix-session-watchers.ps1`, and
`cleanup-disc-sessions.ps1` provide manual repair paths for task/script
regeneration, stale recovery loop restart, and disconnected-session cleanup.
### Hayabusa intake recovery
The repository supports event-driven reprocessing of uploaded Hayabusa packages:
- `aw-hayabusa-drop.path` watches `/opt/activitywatch/aw-rus-ops/drop` for zip
packages and triggers `aw-hayabusa-drop.service`. Evidence:
`aw-server/aw-hayabusa-drop.path`.
- `aw-hayabusa-drop.service` runs `/usr/local/bin/aw-hayabusa-autoprocess`.
Evidence: `aw-server/aw-hayabusa-drop.service`.
- `ansible/deploy_aw_server.yml` creates Hayabusa directories, installs pinned
Hayabusa, installs Rust helpers, creates the drop zone, enables
`aw-hayabusa-drop.path`, and runs `aw-hayabusa doctor`. Evidence:
`ansible/deploy_aw_server.yml:2780-3070`.
### Readiness evidence recovery support
The repository supports periodic readiness bundle generation:
- `detmir-readiness.service` writes readiness output to
`/var/lib/activitywatch/health/readiness-bundle`. Evidence:
`aw-server/detmir-readiness.service`.
- `detmir-readiness.timer` runs daily with persistent timer behavior. Evidence:
`aw-server/detmir-readiness.timer`.
## Current Backup Procedure Actually Supported
### ActivityWatch server local backup artifacts
Implemented:
- `ansible/deploy_aw_server.yml` creates `/var/lib/activitywatch/backups` and
`/var/lib/activitywatch/backups/db`. Evidence:
`ansible/deploy_aw_server.yml:80-128`, `ansible/deploy_aw_server.yml:2263-2269`.
- Before legacy DB merge, Ansible copies the target DB and legacy root DB into
timestamped files under `{{ aw_server_data_dir }}/backups/db`. Evidence:
`ansible/deploy_aw_server.yml:2274-2294`.
- Before applying server-side settings/views/classes, Ansible copies current
payloads to timestamped JSON backups under `{{ aw_server_data_dir }}/backups`.
Evidence: `ansible/deploy_aw_server.yml:2406-2420`.
- `aw-prune-local-state.timer` prunes old local backups using configured
retention and keep-last values. Evidence:
`ansible/deploy_aw_server.yml:236-281`, `docs/RETENTION_POLICY_RU.md:70-71`.
Not currently implemented:
- A general scheduled full backup of the active ActivityWatch SQLite DB.
- A repository-defined off-host backup copy for AW DB, configs, Grafana,
ClickHouse, DLP evidence, Hayabusa archives, or Windows collector state.
### Windows package rollback backup
Implemented:
- `Install-ActivityWatchPackage` backs up the existing install root into
`install-<timestamp>` before replacing it, keeps only the latest two install
backups, checks free space, and cleans temporary extraction directories.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:80-95`,
`windows/ActivityWatch.Windows.Common.psm1:148-245`.
- `hardening-recovery.ps1` can run `Install-ActivityWatchPackage` when
`-RepairPackage` is supplied. Evidence: `windows/hardening-recovery.ps1`.
Not currently implemented:
- Automatic restore from the saved Windows `install-*` backup directory.
- Backup of `C:\ProgramData\AWatch-rus` as a whole before repair.
### Gitea registry backup
Implemented as registry-readiness support, not DetMir runtime recovery:
- Registry docs define Gitea backup path, script, systemd service/timer,
`gitea dump` ZIP format, SHA256 checksum, daily schedule, and 14-day
retention. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:7-23`,
`docs/registry/registry-evidence-manifest.json:26-39`.
Not currently implemented:
- Tested Gitea restore. The manifest explicitly says `restore_tested=false` and
`production_ready=false`. Evidence:
`docs/registry/registry-evidence-manifest.json:37-39`.
## Current Restore Procedure
### ActivityWatch DB merge/restore-like flow
Supported:
1. Enable `aw_legacy_db_merge_enabled`.
2. Deploy server playbook.
3. Playbook checks legacy root DB and target DB.
4. Playbook stops `activitywatch-server.service`.
5. Playbook backs up target and legacy DB files.
6. Playbook runs `/usr/local/bin/merge-aw-server-dbs`.
7. Playbook installs the merged DB as active target DB.
8. Playbook restarts `activitywatch-server.service`.
9. Playbook waits for `/api/0/info`.
Evidence: `ansible/deploy_aw_server.yml:2248-2375`.
Not currently implemented:
- A generic "restore selected backup file to active AW DB" command.
- A tested end-to-end AW DB restore runbook.
- Automated checksum verification for AW DB backup files.
- Automated rollback from a failed DB merge to the backup file.
### `prod-backup-restore` plan-only flow
Supported:
1. `scripts/prod_backup_restore.sh` locates the Rust planner binary or exits
with build instructions. Evidence: `scripts/prod_backup_restore.sh:14-29`.
2. The Rust planner reads `private-config/runtime.env` if available, checks
required env vars, checks `sshpass`, `ansible-playbook`, inventory, and
`merge-aw-server-dbs`. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:111-131`,
`adk-rust/crates/prod-backup-restore/src/main.rs:175-209`.
3. The planner prints planned commands including remote backup directory
creation, DB copies, service stop, merge, install, and Ansible validation.
Evidence: `adk-rust/crates/prod-backup-restore/src/main.rs:211-280`.
Not currently implemented:
- Execution of the planned restore. `--apply` fails by design. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
### Windows collector recovery
Supported:
1. Run `ActivityWatch Recovery` scheduled task or let it run on its configured
trigger.
2. Recovery loop enforces a single lock.
3. Recovery loop stops collectors/watchers in non-live sessions.
4. Recovery loop starts the global worktime collector when allowed.
5. Recovery loop starts configured live-user launch tasks.
6. Recovery loop attempts console fallback when no configured live task starts.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:2114-2168`,
`windows/ActivityWatch.Windows.Common.psm1:2447-2488`,
`windows/ActivityWatch.Windows.Common.psm1:2656-2672`.
Not currently implemented:
- Automatic reconstruction of lost Windows state from an external backup.
- Automatic restore of Windows collector queues after corruption or deletion.
### Gitea restore
Supported:
- Manual outline exists: prepare isolated test server, install same Gitea
version, stop Gitea, verify checksum, unpack dump, restore app/data/repos/db
according to Gitea official procedure, fix ownership, start Gitea, run
`gitea doctor check`, regenerate hooks if needed, and run post-restore
checks. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-90`.
Not currently implemented:
- Tested Gitea restore. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:92-100`.
## Components That Cannot Yet Be Restored Automatically
The following are confirmed by repository inspection:
| Component | Current state | Evidence |
| --- | --- | --- |
| ActivityWatch active SQLite DB | Backups exist around merge/settings operations, but generic restore is Not currently implemented. | `ansible/deploy_aw_server.yml:2274-2294`, `adk-rust/crates/prod-backup-restore/src/main.rs:100-105` |
| ClickHouse 1C data | Docker volume exists; no backup/restore automation found. Not currently implemented. | `clickhouse-1c/docker-compose.yml:13-19` |
| ClickHouse Workforce data | Docker volume exists; no backup/restore automation found. Not currently implemented. | `clickhouse-workforce/docker-compose.yml:11-17` |
| Grafana data volume | Docker volume exists; no repo cleanup and no restore automation. Not currently implemented. | `grafana-1c/docker-compose.yml:67-79`, `docs/RETENTION_POLICY_RU.md:99` |
| Prometheus TSDB | Retention configured, backup/restore not documented. Not currently implemented. | `grafana-1c/docker-compose.yml:40-49` |
| DLP policy/case/warehouse DBs | Retention doc says no automatic deletion; backup/restore not implemented. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:43-47`, `docs/RETENTION_POLICY_RU.md:87-90` |
| DLP evidence and compliance reports | Cleanup disabled; restore depends on customer backup if manually deleted. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:221-223` |
| Hayabusa reports/archive | Processing is automated; restore of archive/reports is not automated. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:48`, `docs/RETENTION_POLICY_RU.md:91` |
| Windows collector state and queues | Recovery restarts collectors; external backup/restore of state is not implemented. | `docs/RETENTION_POLICY_RU.md:40`, `docs/RETENTION_POLICY_RU.md:85` |
| Diagnostic bundles and release evidence | No automatic backup/restore found. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:100-101` |
## Components Requiring Manual Intervention
- AW DB merge/recovery: operator must enable `aw_legacy_db_merge_enabled`, run
Ansible, review backup files, and verify API. Evidence:
`ansible/deploy_aw_server.yml:2248-2375`.
- `prod-backup-restore`: operator can only review a plan; execution is manual
because `--apply` is disabled. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
- Windows collector recovery after severe state loss: operator must use
`hardening-recovery.ps1`, `rebuild-worktime-tasks.ps1`,
`fix-session-watchers.ps1`, or redeploy. Evidence: `windows/*.ps1`.
- Hayabusa stuck path/service: operator may need to reset failed units, repair
drop-zone permissions, and rerun processing. Evidence:
`aw-server/aw-hayabusa-drop.path`, `aw-server/aw-hayabusa-drop.service`.
- Gitea restore: manual isolated test restore is required; tested restore is
not yet recorded. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`.
- ClickHouse/Grafana/Prometheus/DLP data restore: Not currently implemented.
## Missing Documentation
Critical and high-confidence gaps only:
- Exact AW DB restore runbook from `/var/lib/activitywatch/backups/db` to the
active DB path. Not currently implemented.
- Post-restore verification checklist for AW DB backup restore, including
checksum, ownership, service restart, bucket freshness, and worktime report
checks. Not currently implemented.
- Backup inventory mapping each persistent component to an actual backup owner,
schedule, storage location, retention, and restore command. Partially covered
by `docs/BACKUP_AND_RECOVERY_RU.md` and `docs/RETENTION_POLICY_RU.md`, but
operational restore ownership is Not currently implemented.
- ClickHouse 1C and ClickHouse Workforce backup/restore runbooks. Not currently
implemented.
- Grafana volume restore runbook. Not currently implemented.
- DLP evidence/case/policy restore runbook. Not currently implemented.
- Windows `C:\ProgramData\AWatch-rus` state backup/restore runbook. Not
currently implemented.
- Gitea restore evidence result. The runbook exists, but restore test is marked
false. Evidence: `docs/registry/registry-evidence-manifest.json:37-39`.
## Missing Automation
- Automated AW DB restore from a selected backup file. Not currently
implemented.
- Automated AW DB backup with checksum on a schedule independent of merge
operations. Not currently implemented.
- Off-host/offline copy for AW DB, ClickHouse volumes, Grafana data, DLP
evidence, Hayabusa archives, Windows state, and release evidence. Not
currently implemented.
- ClickHouse backup and restore automation. Not currently implemented.
- Grafana data volume backup and restore automation. Not currently implemented.
- DLP evidence/case/policy backup and restore automation. Not currently
implemented.
- Windows state backup and restore automation. Not currently implemented.
- Automated restore drill evidence generation. Not currently implemented.
## Operational Risks
| Risk | Severity | Evidence | Impact |
| --- | --- | --- | --- |
| Restore is partially plan-only for AW DB | Critical | `prod-backup-restore` rejects `--apply` | Operator can plan but cannot run a deterministic automated restore through this tool |
| AW DB backups are created around specific operations, not as a general scheduled full backup | Critical | `ansible/deploy_aw_server.yml:2274-2294` | A recent recovery point may be unavailable if no merge/settings operation occurred |
| ClickHouse data has no repo-defined restore path | Critical | ClickHouse Docker volumes only | Loss/corruption of 1C or workforce ClickHouse data requires ad hoc operator recovery |
| DLP/Hayabusa evidence has no automated restore | High | `docs/RETENTION_POLICY_RU.md:87-91`, `docs/RETENTION_POLICY_RU.md:221-223` | Forensic/case continuity depends on external/customer backup |
| Grafana data volume has no repo-defined backup/restore | High | `grafana-1c/docker-compose.yml:67-79` | Dashboard DB/users/session state may require manual reconstruction even though provisioned dashboards exist |
| Gitea restore is documented but untested | High | `restore_tested=false` | Source-control recovery confidence remains limited |
| Windows recovery restarts collectors but does not restore deleted state | Medium | Windows recovery loop evidence | Collector state/queues/logs can be lost if state root is deleted |
| Generic backup document is intentionally high-level | Medium | `docs/BACKUP_AND_RECOVERY_RU.md:5-6` | Operators need component-specific procedures during incidents |
| Hayabusa path recovery depends on service/path health and permissions | Medium | `aw-hayabusa-drop.path`, `aw-hayabusa-drop.service` | Drop backlog or permission drift can stall forensic intake |
| Recovery smoke is documented but not tied to a single recovery command | Low | `docs/OPERATIONS_RUNBOOK_RU.md`, `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md` | Operator can validate, but command sequencing remains manual |
## Recovery Confidence Score
Overall score: **48 / 100**
Justification:
- Windows collector process recovery is mature: scheduled task, lock, live
session handling, cleanup of non-live sessions, and task restart are
implemented.
- Server service recovery and smoke validation are present.
- Local maintenance and retention are present.
- AW DB merge has safety backups and API validation, but generic restore is not
implemented.
- `prod-backup-restore` is explicitly plan-only.
- ClickHouse, Grafana, Prometheus, DLP evidence/cases, Hayabusa archive, Windows
state, diagnostic bundles, and release evidence do not have automated restore
procedures in the repository.
- Gitea backup is documented with checksum and timer metadata, but restore is
marked untested.
## Prioritized Confirmed Gaps
### Critical
1. Generic AW DB restore from backup is Not currently implemented.
Evidence: backup files are created by Ansible, but `prod-backup-restore`
refuses `--apply`.
2. Scheduled full AW DB backup independent of merge/settings changes is Not
currently implemented.
Evidence: backups are tied to merge/settings operations and pruning exists,
but no scheduled full DB backup unit is present.
3. ClickHouse 1C and Workforce backup/restore are Not currently implemented.
Evidence: both stacks persist to Docker volumes; no backup/restore runbook or
automation is present.
### High
4. DLP evidence/case/policy restore is Not currently implemented.
Evidence: retention policy explicitly leaves DLP evidence/cases without
automatic cleanup and says recovery depends on customer backup after manual
deletion.
5. Hayabusa reports/archive restore is Not currently implemented.
Evidence: intake processing is automated, but archive/report restore is not.
6. Grafana data volume backup/restore is Not currently implemented.
Evidence: Grafana uses `grafana-data`; no repo restore procedure exists.
7. Gitea restore test is Not currently implemented.
Evidence: registry manifest has `restore_tested=false`.
### Medium
8. Windows state root backup/restore is Not currently implemented.
Evidence: Windows recovery restarts collectors and repairs tasks, but does
not restore `C:\ProgramData\AWatch-rus` from backup.
9. Component-specific recovery ownership matrix is Not currently implemented.
Evidence: generic backup doc says component list must be refined by release
profile and customer infrastructure.
10. Restore drill evidence generation is Not currently implemented.
Evidence: validation commands exist, but no restore-drill artifact workflow
is present.
### Low
11. Prometheus TSDB restore is Not currently implemented.
Evidence: compose config bounds retention, but no backup/restore procedure
is present.
12. Diagnostic bundle and release evidence restore is Not currently
implemented.
Evidence: retention policy intentionally avoids pruning, but does not define
backup or restore automation.
## Release-Relevant Conclusion
The repository currently supports operational restart, Windows collector
self-healing, local maintenance, retention, readiness evidence, and limited
backup-before-mutation behavior.
It does not yet support deterministic full production recovery for all persisted
state. The most important gap is not service restart; it is data restore:
ActivityWatch DB, ClickHouse volumes, Grafana state, DLP/Hayabusa evidence, and
Windows state are not covered by a tested automated restore process in the
repository.
+7 -57
View File
@@ -169,12 +169,12 @@ name = "aw-1c-ingest"
version = "0.1.0"
dependencies = [
"anyhow",
"calamine",
"chrono",
"clap",
"csv",
"filetime",
"fs2",
"quick-xml",
"regex",
"reqwest",
"serde",
@@ -182,6 +182,7 @@ dependencies = [
"serde_yaml",
"sha1",
"tempfile",
"zip",
]
[[package]]
@@ -397,33 +398,12 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "calamine"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a3a315226fdc5b1c3e33521073e1712a05944bc0664d665ff1f6ff0396334da"
dependencies = [
"byteorder",
"codepage",
"encoding_rs",
"log",
"quick-xml",
"serde",
"zip 0.6.6",
]
[[package]]
name = "cc"
version = "1.2.63"
@@ -540,15 +520,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "codepage"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
dependencies = [
"encoding_rs",
]
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -949,15 +920,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -1249,6 +1211,7 @@ dependencies = [
"serde",
"serde_json",
"tempfile",
"zip",
]
[[package]]
@@ -1801,11 +1764,10 @@ dependencies = [
[[package]]
name = "quick-xml"
version = "0.31.0"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"encoding_rs",
"memchr",
]
@@ -1936,7 +1898,7 @@ dependencies = [
"sha2",
"tar",
"tempfile",
"zip 2.4.2",
"zip",
]
[[package]]
@@ -2609,7 +2571,7 @@ dependencies = [
"sha2",
"tar",
"tempfile",
"zip 2.4.2",
"zip",
]
[[package]]
@@ -3455,18 +3417,6 @@ dependencies = [
"syn",
]
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"byteorder",
"crc32fast",
"crossbeam-utils",
"flate2",
]
[[package]]
name = "zip"
version = "2.4.2"
+2 -1
View File
@@ -71,7 +71,6 @@ publish = false
adk-rust = { version = "0.9.1", default-features = false }
anyhow = "1"
base64 = "0.22"
calamine = "=0.24.0"
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] }
clap = { version = "4", features = ["derive", "env"] }
csv = "1"
@@ -81,6 +80,7 @@ detmir-state = { path = "crates/detmir-state" }
filetime = "0.2"
fs2 = "0.4"
notify = "6"
quick-xml = "0.41"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
regex = "1"
rusqlite = "0.32"
@@ -93,4 +93,5 @@ tempfile = "3"
tiny_http = "0.12"
url = "2"
urlencoding = "2"
zip = { version = "2", default-features = false, features = ["deflate"] }
windows-sys = "0.59"
+2 -1
View File
@@ -8,12 +8,12 @@ publish = false
[dependencies]
anyhow.workspace = true
calamine.workspace = true
chrono.workspace = true
clap.workspace = true
csv.workspace = true
filetime.workspace = true
fs2.workspace = true
quick-xml.workspace = true
regex.workspace = true
reqwest.workspace = true
serde.workspace = true
@@ -21,3 +21,4 @@ serde_json.workspace = true
serde_yaml.workspace = true
sha1.workspace = true
tempfile.workspace = true
zip.workspace = true
+367 -22
View File
@@ -6,16 +6,19 @@ use std::process::Command;
use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use calamine::{Data, Reader, open_workbook_auto};
use chrono::{DateTime, NaiveDateTime, Utc};
use clap::Parser;
use filetime::{FileTime, set_file_times};
use fs2::FileExt;
use quick_xml::Reader as XmlReader;
use quick_xml::XmlVersion;
use quick_xml::events::{BytesStart, Event};
use regex::Regex;
use reqwest::blocking::Client;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use sha1::{Digest, Sha1};
use zip::ZipArchive;
const DATASETS: &[&str] = &[
"documents",
@@ -1103,15 +1106,14 @@ fn load_registry(
}
fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
let mut workbook =
open_workbook_auto(path).with_context(|| format!("open xlsx {}", path.display()))?;
let workbook = read_xlsx_workbook(path)?;
let now = Utc::now()
.naive_utc()
.format("%Y-%m-%d %H:%M:%S")
.to_string();
let mut tax_map: HashMap<String, (String, String)> = HashMap::new();
if let Ok(range) = workbook.worksheet_range("Лист2") {
for row in range.rows().skip(2) {
if let Some(rows) = workbook.get("Лист2") {
for row in rows.iter().skip(2) {
let company_name = cell_text(row.get(1));
if company_name.is_empty() {
continue;
@@ -1123,10 +1125,10 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
}
}
let mut out = Vec::new();
let Ok(range) = workbook.worksheet_range("ОСНОВНОЙ") else {
let Some(sheet_rows) = workbook.get("ОСНОВНОЙ") else {
return Ok(out);
};
let rows: Vec<_> = range.rows().collect();
let rows: Vec<_> = sheet_rows.iter().collect();
if rows.len() < 3 {
return Ok(out);
}
@@ -1134,7 +1136,8 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
let manager_headers: Vec<String> = rows[1].iter().map(|v| cell_text(Some(v))).collect();
let mut specs = Vec::<(usize, String, Option<usize>, String, String)>::new();
let mut current: Option<(usize, String, String)> = None;
for idx in 1..manager_headers.len() {
let header_len = top_headers.len().max(manager_headers.len());
for idx in 1..header_len {
let manager = manager_headers.get(idx).cloned().unwrap_or_default();
let top = top_headers.get(idx).cloned().unwrap_or_default();
if !manager.is_empty() {
@@ -1194,26 +1197,291 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
Ok(out)
}
fn cell_text(value: Option<&Data>) -> String {
match value {
Some(Data::String(v)) => v.trim().to_string(),
Some(Data::Float(v)) => {
if v.fract() == 0.0 {
format!("{}", *v as i64)
type XlsxWorkbook = HashMap<String, Vec<Vec<String>>>;
fn read_xlsx_workbook(path: &Path) -> Result<XlsxWorkbook> {
let file = File::open(path).with_context(|| format!("open xlsx {}", path.display()))?;
let mut archive =
ZipArchive::new(file).with_context(|| format!("read xlsx {}", path.display()))?;
let shared_strings = read_xlsx_shared_strings(&mut archive)?;
let sheets = read_xlsx_sheet_paths(&mut archive)?;
let mut workbook = XlsxWorkbook::new();
for (name, sheet_path) in sheets {
let xml = read_zip_text(&mut archive, &sheet_path)
.with_context(|| format!("read sheet {name} from {}", path.display()))?;
let rows = parse_xlsx_sheet(&xml, &shared_strings)
.with_context(|| format!("parse sheet {name} from {}", path.display()))?;
workbook.insert(name, rows);
}
Ok(workbook)
}
fn read_xlsx_shared_strings<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
) -> Result<Vec<String>> {
let Some(xml) = read_zip_text_optional(archive, "xl/sharedStrings.xml")? else {
return Ok(Vec::new());
};
let mut reader = XmlReader::from_str(&xml);
reader.config_mut().trim_text(false);
let mut buf = Vec::new();
let mut strings = Vec::new();
let mut in_si = false;
let mut in_t = false;
let mut current = String::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Start(e) => match local_name(e.name().as_ref()) {
b"si" => {
in_si = true;
current.clear();
}
b"t" if in_si => in_t = true,
_ => {}
},
Event::Text(e) if in_si && in_t => current.push_str(e.decode()?.as_ref()),
Event::End(e) => match local_name(e.name().as_ref()) {
b"t" => in_t = false,
b"si" => {
strings.push(current.clone());
in_si = false;
}
_ => {}
},
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(strings)
}
fn read_xlsx_sheet_paths<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
) -> Result<Vec<(String, String)>> {
let workbook_xml = read_zip_text(archive, "xl/workbook.xml")?;
let rels_xml = read_zip_text(archive, "xl/_rels/workbook.xml.rels")?;
let rels = parse_xlsx_relationships(&rels_xml)?;
let mut reader = XmlReader::from_str(&workbook_xml);
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut sheets = Vec::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) | Event::Start(e) if local_name(e.name().as_ref()) == b"sheet" => {
let name = xml_attr(&reader, &e, b"name")?.unwrap_or_default();
let mut rel_id = xml_attr(&reader, &e, b"r:id")?.unwrap_or_default();
if rel_id.is_empty() {
rel_id = xml_attr(&reader, &e, b"id")?.unwrap_or_default();
}
if !name.is_empty() && !rel_id.is_empty() {
let target = rels
.get(&rel_id)
.with_context(|| format!("missing workbook relationship {rel_id}"))?;
sheets.push((name, normalize_xlsx_target(target)?));
}
}
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(sheets)
}
fn parse_xlsx_relationships(xml: &str) -> Result<HashMap<String, String>> {
let mut reader = XmlReader::from_str(xml);
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut rels = HashMap::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) | Event::Start(e)
if local_name(e.name().as_ref()) == b"Relationship" =>
{
let id = xml_attr(&reader, &e, b"Id")?.unwrap_or_default();
let target = xml_attr(&reader, &e, b"Target")?.unwrap_or_default();
if !id.is_empty() && !target.is_empty() {
rels.insert(id, target);
}
}
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(rels)
}
fn parse_xlsx_sheet(xml: &str, shared_strings: &[String]) -> Result<Vec<Vec<String>>> {
let mut reader = XmlReader::from_str(xml);
reader.config_mut().trim_text(false);
let mut buf = Vec::new();
let mut rows = Vec::<Vec<String>>::new();
let mut current_row = Vec::<String>::new();
let mut current_cell: Option<(usize, String, String)> = None;
let mut in_value = false;
let mut in_inline_text = false;
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) => match local_name(e.name().as_ref()) {
b"row" => rows.push(Vec::new()),
b"c" => {
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
let column = cell_ref
.chars()
.take_while(|ch| ch.is_ascii_alphabetic())
.collect::<String>();
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
if current_row.len() <= col {
current_row.resize(col + 1, String::new());
}
}
_ => {}
},
Event::Start(e) => match local_name(e.name().as_ref()) {
b"row" => current_row.clear(),
b"c" => {
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
let cell_type = xml_attr(&reader, &e, b"t")?.unwrap_or_default();
let column = cell_ref
.chars()
.take_while(|ch| ch.is_ascii_alphabetic())
.collect::<String>();
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
current_cell = Some((col, cell_type, String::new()));
}
b"v" => in_value = true,
b"t" if current_cell
.as_ref()
.is_some_and(|(_, kind, _)| kind == "inlineStr") =>
{
in_inline_text = true;
}
_ => {}
},
Event::Text(e) if in_value || in_inline_text => {
if let Some((_, _, value)) = current_cell.as_mut() {
value.push_str(e.decode()?.as_ref());
}
}
Event::End(e) => match local_name(e.name().as_ref()) {
b"v" => in_value = false,
b"t" => in_inline_text = false,
b"c" => {
if let Some((col, cell_type, raw)) = current_cell.take() {
let value = xlsx_cell_value(&cell_type, &raw, shared_strings);
if current_row.len() <= col {
current_row.resize(col + 1, String::new());
}
current_row[col] = value;
}
}
b"row" => rows.push(current_row.clone()),
_ => {}
},
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(rows)
}
fn read_zip_text<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
path: &str,
) -> Result<String> {
read_zip_text_optional(archive, path)?.with_context(|| format!("missing xlsx entry {path}"))
}
fn read_zip_text_optional<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
path: &str,
) -> Result<Option<String>> {
let Ok(mut file) = archive.by_name(path) else {
return Ok(None);
};
let mut text = String::new();
file.read_to_string(&mut text)
.with_context(|| format!("read xlsx entry {path}"))?;
Ok(Some(text))
}
fn normalize_xlsx_target(target: &str) -> Result<String> {
let normalized = target.replace('\\', "/");
if normalized.split('/').any(|part| part == "..") {
bail!("unsafe xlsx relationship target: {target}");
}
let path = normalized.trim_start_matches('/');
if path.starts_with("xl/") {
Ok(path.to_string())
} else {
Ok(format!("xl/{path}"))
}
}
fn xml_attr(reader: &XmlReader<&[u8]>, e: &BytesStart<'_>, key: &[u8]) -> Result<Option<String>> {
for attr in e.attributes() {
let attr = attr?;
if attr.key.as_ref() == key {
return Ok(Some(
attr.decoded_and_normalized_value(XmlVersion::Implicit1_0, reader.decoder())?
.into_owned(),
));
}
}
Ok(None)
}
fn local_name(name: &[u8]) -> &[u8] {
name.rsplit(|b| *b == b':').next().unwrap_or(name)
}
fn xlsx_column_index(column: &str) -> Option<usize> {
let mut value = 0usize;
for byte in column.bytes() {
if !byte.is_ascii_alphabetic() {
return None;
}
value = value * 26 + usize::from(byte.to_ascii_uppercase() - b'A' + 1);
}
value.checked_sub(1)
}
fn xlsx_cell_value(cell_type: &str, raw: &str, shared_strings: &[String]) -> String {
match cell_type {
"s" => raw
.parse::<usize>()
.ok()
.and_then(|idx| shared_strings.get(idx))
.cloned()
.unwrap_or_default(),
"b" => {
if raw.trim() == "1" {
"true".to_string()
} else {
v.to_string()
"false".to_string()
}
}
Some(Data::Int(v)) => v.to_string(),
Some(Data::Bool(v)) => v.to_string(),
Some(Data::DateTime(v)) => v.to_string(),
Some(Data::DateTimeIso(v)) => v.trim().to_string(),
Some(Data::DurationIso(v)) => v.trim().to_string(),
Some(Data::Empty) | None => String::new(),
Some(other) => other.to_string(),
_ => raw.to_string(),
}
}
fn cell_text(value: Option<&String>) -> String {
value
.map(|v| {
let trimmed = v.trim();
if let Ok(number) = trimmed.parse::<f64>() {
if number.fract() == 0.0 {
return format!("{}", number as i64);
}
}
trimmed.to_string()
})
.unwrap_or_default()
}
fn archive_or_delete(
config: &RawConfig,
dataset: &str,
@@ -1507,4 +1775,81 @@ mod tests {
fn split_sql_ignores_empty_chunks() {
assert_eq!(split_sql_statements("SELECT 1; ; SELECT 2;").len(), 2);
}
#[test]
fn parses_registry_xlsx_with_inline_strings() -> Result<()> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("registry.xlsx");
let file = File::create(&path)?;
let mut zip = zip::ZipWriter::new(file);
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Stored);
for (name, xml) in [
(
"xl/workbook.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<sheets>
<sheet name="Лист2" sheetId="1" r:id="rId1"/>
<sheet name="ОСНОВНОЙ" sheetId="2" r:id="rId2"/>
</sheets>
</workbook>"#,
),
(
"xl/_rels/workbook.xml.rels",
r#"<?xml version="1.0" encoding="UTF-8"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Target="worksheets/sheet1.xml"/>
<Relationship Id="rId2" Target="worksheets/sheet2.xml"/>
</Relationships>"#,
),
(
"xl/worksheets/sheet1.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1"/>
<row r="2"/>
<row r="3">
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
<c r="C3" t="inlineStr"><is><t>1101000000</t></is></c>
<c r="D3" t="inlineStr"><is><t>110101001</t></is></c>
</row>
</sheetData>
</worksheet>"#,
),
(
"xl/worksheets/sheet2.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1">
<c r="C1" t="inlineStr"><is><t>Ключевой контур</t></is></c>
</row>
<row r="2">
<c r="B2" t="inlineStr"><is><t>Иванов И.И.</t></is></c>
</row>
<row r="3">
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
<c r="C3" t="inlineStr"><is><t>ЕСТЬ</t></is></c>
</row>
</sheetData>
</worksheet>"#,
),
] {
zip.start_file(name, options)?;
zip.write_all(xml.as_bytes())?;
}
zip.finish()?;
let rows = parse_registry_xlsx(&path)?;
assert_eq!(rows.len(), 1);
assert_eq!(rows[0]["company_name"], "ООО Ромашка");
assert_eq!(rows[0]["company_key"], "ООО РОМАШКА");
assert_eq!(rows[0]["assignee_name"], "Иванов И.И.");
assert_eq!(rows[0]["key_contour"], 1);
assert_eq!(rows[0]["inn"], "1101000000");
assert_eq!(rows[0]["kpp"], "110101001");
Ok(())
}
}
@@ -7,6 +7,7 @@ use clap::Parser;
use serde::Serialize;
const DEFAULT_DATA_DIR: &str = "/var/lib/activitywatch";
const DEFAULT_WORKTIME_REPORT_CACHE_RETENTION_SECONDS: u64 = 86_400;
const TMP_ARCHIVE_PATTERNS: &[NamePattern] = &[
NamePattern::PrefixSuffix("activitywatch-", ".zip"),
NamePattern::PrefixSuffix("hayabusa-", ".zip"),
@@ -30,6 +31,9 @@ struct Cli {
#[arg(long)]
browser_smoke_dir: Option<PathBuf>,
#[arg(long)]
worktime_report_cache_dir: Option<PathBuf>,
#[arg(long)]
tmp_dir: Option<PathBuf>,
@@ -48,6 +52,9 @@ struct Cli {
#[arg(long, default_value_t = 1)]
browser_smoke_retention_days: u64,
#[arg(long, default_value_t = DEFAULT_WORKTIME_REPORT_CACHE_RETENTION_SECONDS)]
worktime_report_cache_retention_seconds: u64,
#[arg(long, default_value_t = 1)]
tmp_archive_retention_days: u64,
@@ -66,12 +73,14 @@ struct Config {
data_dir: PathBuf,
backup_dir: PathBuf,
browser_smoke_dir: PathBuf,
worktime_report_cache_dir: PathBuf,
tmp_dir: PathBuf,
backup_retention_days: u64,
backup_keep_last_db: usize,
backup_keep_last_json: usize,
browser_smoke_keep_runs: usize,
browser_smoke_retention_days: u64,
worktime_report_cache_retention_seconds: u64,
tmp_archive_retention_days: u64,
tmp_webui_retention_days: u64,
apply: bool,
@@ -154,6 +163,7 @@ fn run() -> Result<i32> {
&mut items,
)?;
plan_browser_smoke(&cfg, &mut items)?;
plan_worktime_report_cache(&cfg, &mut items)?;
plan_tmp(
&cfg.tmp_dir,
TMP_ARCHIVE_PATTERNS,
@@ -200,6 +210,10 @@ impl Config {
.browser_smoke_dir
.or_else(|| env_path("AW_BROWSER_SMOKE_OUTPUT_DIR"))
.unwrap_or_else(|| data_dir.join("browser-smoke"));
let worktime_report_cache_dir = cli
.worktime_report_cache_dir
.or_else(|| env_path("AW_WORKTIME_REPORT_DISK_CACHE_DIR"))
.unwrap_or_else(|| data_dir.join("worktime-report-cache"));
let tmp_dir = cli
.tmp_dir
.or_else(|| env_path("AW_TMP_DIR"))
@@ -208,6 +222,7 @@ impl Config {
data_dir,
backup_dir,
browser_smoke_dir,
worktime_report_cache_dir,
tmp_dir,
backup_retention_days: env_u64("AW_BACKUP_RETENTION_DAYS", cli.backup_retention_days),
backup_keep_last_db: env_usize("AW_BACKUP_KEEP_LAST_DB", cli.backup_keep_last_db),
@@ -220,6 +235,10 @@ impl Config {
"AW_BROWSER_SMOKE_RETENTION_DAYS",
cli.browser_smoke_retention_days,
),
worktime_report_cache_retention_seconds: env_u64(
"AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS",
cli.worktime_report_cache_retention_seconds,
),
tmp_archive_retention_days: env_u64(
"AW_TMP_ARCHIVE_RETENTION_DAYS",
cli.tmp_archive_retention_days,
@@ -267,6 +286,41 @@ fn plan_browser_smoke(cfg: &Config, items: &mut Vec<PruneItem>) -> Result<()> {
Ok(())
}
fn plan_worktime_report_cache(cfg: &Config, items: &mut Vec<PruneItem>) -> Result<()> {
if cfg.worktime_report_cache_retention_seconds == 0 || !cfg.worktime_report_cache_dir.exists() {
return Ok(());
}
let cutoff = cutoff_seconds(cfg.worktime_report_cache_retention_seconds);
for entry in fs::read_dir(&cfg.worktime_report_cache_dir)
.with_context(|| format!("read {}", cfg.worktime_report_cache_dir.display()))?
{
let entry = entry?;
if !entry.file_type()?.is_file() {
continue;
}
let name = entry.file_name();
let name = name.to_string_lossy();
if !name.ends_with(".json") {
continue;
}
let meta = entry.metadata()?;
let modified = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH);
if modified >= cutoff {
continue;
}
items.push(
Candidate {
path: entry.path(),
kind: ItemKind::File,
modified,
size_bytes: meta.len(),
}
.into_item("worktime_report_disk_cache"),
);
}
Ok(())
}
fn plan_tmp(
dir: &Path,
patterns: &[NamePattern],
@@ -395,6 +449,9 @@ fn validate_plan(cfg: &Config, items: &[PruneItem]) -> Result<()> {
"backup_db" => is_under_or_equal(&item.path, &cfg.backup_dir.join("db")),
"backup_root" => is_under_or_equal(&item.path, &cfg.backup_dir),
"browser_smoke_run" => is_under_or_equal(&item.path, &cfg.browser_smoke_dir),
"worktime_report_disk_cache" => {
is_under_or_equal(&item.path, &cfg.worktime_report_cache_dir)
}
"tmp_archive" | "tmp_webui" => is_under_or_equal(&item.path, &cfg.tmp_dir),
_ => false,
};
@@ -410,6 +467,7 @@ fn validate_plan(cfg: &Config, items: &[PruneItem]) -> Result<()> {
if item.path == cfg.data_dir
|| item.path == cfg.backup_dir
|| item.path == cfg.browser_smoke_dir
|| item.path == cfg.worktime_report_cache_dir
{
bail!("refusing to delete root directory {}", item.path.display());
}
@@ -486,8 +544,12 @@ fn print_summary(summary: &Summary, json: bool) -> Result<()> {
}
fn cutoff(days: u64) -> SystemTime {
cutoff_seconds(days.saturating_mul(86_400))
}
fn cutoff_seconds(seconds: u64) -> SystemTime {
SystemTime::now()
.checked_sub(Duration::from_secs(days.saturating_mul(86_400)))
.checked_sub(Duration::from_secs(seconds))
.unwrap_or(SystemTime::UNIX_EPOCH)
}
@@ -598,12 +660,14 @@ mod tests {
data_dir: tmp.path().to_path_buf(),
backup_dir: tmp.path().join("backups"),
browser_smoke_dir: tmp.path().join("browser-smoke"),
worktime_report_cache_dir: tmp.path().join("worktime-report-cache"),
tmp_dir: tmp.path().join("tmp"),
backup_retention_days: 1,
backup_keep_last_db: 1,
backup_keep_last_json: 1,
browser_smoke_keep_runs: 1,
browser_smoke_retention_days: 1,
worktime_report_cache_retention_seconds: 1,
tmp_archive_retention_days: 1,
tmp_webui_retention_days: 1,
apply: false,
@@ -618,4 +682,43 @@ mod tests {
};
assert!(validate_plan(&cfg, &[item]).is_err());
}
#[test]
fn worktime_report_cache_prunes_only_json_files_inside_cache_dir() {
let tmp = tempfile::tempdir().expect("tempdir");
let cache_dir = tmp.path().join("worktime-report-cache");
fs::create_dir_all(&cache_dir).expect("cache dir");
fs::write(cache_dir.join("old-cache.json"), b"{}").expect("cache json");
fs::write(cache_dir.join("keep.txt"), b"keep").expect("non-json");
fs::create_dir_all(cache_dir.join("nested")).expect("nested dir");
fs::write(cache_dir.join("nested").join("nested-cache.json"), b"{}")
.expect("nested cache json");
let cfg = Config {
data_dir: tmp.path().to_path_buf(),
backup_dir: tmp.path().join("backups"),
browser_smoke_dir: tmp.path().join("browser-smoke"),
worktime_report_cache_dir: cache_dir.clone(),
tmp_dir: tmp.path().join("tmp"),
backup_retention_days: 1,
backup_keep_last_db: 1,
backup_keep_last_json: 1,
browser_smoke_keep_runs: 1,
browser_smoke_retention_days: 1,
worktime_report_cache_retention_seconds: 1,
tmp_archive_retention_days: 1,
tmp_webui_retention_days: 1,
apply: false,
json: false,
};
std::thread::sleep(Duration::from_secs(2));
let mut items = Vec::new();
plan_worktime_report_cache(&cfg, &mut items).expect("plan cache");
assert_eq!(items.len(), 1);
assert_eq!(items[0].reason, "worktime_report_disk_cache");
assert_eq!(items[0].path, cache_dir.join("old-cache.json"));
validate_plan(&cfg, &items).expect("valid cache plan");
}
}
+378 -14
View File
@@ -3985,9 +3985,47 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
let server_port = json_i64(&config, &["server", "port"]).unwrap_or(5600);
let api_base = format!("{server_scheme}://{server_host}:{server_port}/api/0");
let mut process_snapshot = collect_process_snapshot();
let session_snapshot = collect_session_snapshot();
let live_session_ids = live_session_ids(&session_snapshot.sessions);
let mut problems = Vec::new();
let mut actions = Vec::new();
let non_live_stop_plan =
non_live_session_collectors(&process_snapshot.processes, &live_session_ids);
if !non_live_stop_plan.is_empty() {
if args.mode == "enforce" {
for process in &non_live_stop_plan {
let ok = process.pid.is_some_and(terminate_process);
actions.push(json!({
"action": "stop-non-live-session-collector",
"kind": session_scoped_collector_kind(process).unwrap_or("unknown"),
"sessionId": process.session_id,
"pid": process.pid,
"applied": true,
"ok": ok
}));
if !ok {
problems.push(format!(
"failed to stop collector pid {:?} in non-live session {:?}",
process.pid, process.session_id
));
}
}
process_snapshot = collect_process_snapshot();
} else {
for process in &non_live_stop_plan {
actions.push(json!({
"action": "stop-non-live-session-collector",
"kind": session_scoped_collector_kind(process).unwrap_or("unknown"),
"sessionId": process.session_id,
"pid": process.pid,
"applied": false,
"mode": "shadow"
}));
}
}
}
let duplicate_plan = duplicate_legacy_collectors(&process_snapshot.processes);
if !duplicate_plan.is_empty() {
if args.mode == "enforce" {
@@ -4037,6 +4075,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
.unwrap_or_else(|| "powershell_primary".to_string());
let worktime_legacy_fallback_enabled =
json_bool(&config, &["collectors", "worktimeLegacyFallbackEnabled"]).unwrap_or(true);
let window_enabled = json_bool(&config, &["collectors", "windowEnabled"]).unwrap_or(true);
let file_ops_enabled = json_bool(&config, &["collectors", "fileOpsEnabled"]).unwrap_or(true);
let file_ops_mode = json_string(&config, &["collectors", "fileOpsMode"])
.unwrap_or_else(|| "powershell_primary".to_string());
@@ -4047,7 +4086,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
args.interactive_max_age_seconds,
15,
);
let bucket_checks = vec![
let mut bucket_checks = vec![
worktime_bucket.clone(),
get_bucket_health(
&api_base,
@@ -4055,19 +4094,21 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
args.interactive_max_age_seconds,
15,
),
get_bucket_health(
];
if window_enabled {
bucket_checks.push(get_bucket_health(
&api_base,
&format!("aw-watcher-window_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
),
get_bucket_health(
&api_base,
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
),
];
));
}
bucket_checks.push(get_bucket_health(
&api_base,
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
));
let interactive_stale = bucket_checks
.iter()
.skip(1)
@@ -4085,7 +4126,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
}
let task_defs = guard_task_definitions(&config);
let missing_fileops_sessions =
let mut missing_fileops_sessions =
if file_ops_enabled && file_ops_mode.eq_ignore_ascii_case("rust_primary") {
missing_rust_collector_sessions(
&process_snapshot.processes,
@@ -4095,10 +4136,13 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
} else {
Vec::new()
};
let launch_needed = interactive_stale || !missing_fileops_sessions.is_empty();
missing_fileops_sessions.retain(|session_id| live_session_ids.contains(session_id));
let has_live_sessions = !live_session_ids.is_empty();
let effective_interactive_stale = interactive_stale && has_live_sessions;
let launch_needed = effective_interactive_stale || !missing_fileops_sessions.is_empty();
if launch_needed {
let active_legacy_collectors = active_legacy_collector_count(&process_snapshot.processes);
if interactive_stale
if effective_interactive_stale
&& missing_fileops_sessions.is_empty()
&& active_legacy_collectors > 0
&& process_snapshot.command_line_query_ok
@@ -4127,6 +4171,16 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
problems.push(format!("refuse non-allowlisted task {}", task.task_name));
continue;
}
if !task_has_live_session(&task.user_id, &session_snapshot.sessions) {
actions.push(json!({
"action": "run-task",
"target": task.task_name,
"applied": false,
"reason": "no-live-session-for-user",
"userId": task.user_id
}));
continue;
}
let key = format!("task:{}", task.task_name);
let allowed = runtime.action_allowed(
&key,
@@ -4165,7 +4219,8 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
}
}
let status = if problems.is_empty() && (args.mode == "enforce" || !interactive_stale) {
let status = if problems.is_empty() && (args.mode == "enforce" || !effective_interactive_stale)
{
"ok"
} else {
"warn"
@@ -4184,12 +4239,25 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
"rustWorktimeAgentRunning": rust_agent_running,
"powerShellRuntimeByKind": power_shell_by_kind
},
"sessions": {
"queryOk": session_snapshot.query_ok,
"source": session_snapshot.source,
"error": session_snapshot.error,
"liveSessionIds": live_session_ids.iter().copied().collect::<Vec<_>>(),
"records": session_snapshot.sessions.iter().map(|session| json!({
"sessionId": session.session_id,
"userName": session.user_name,
"state": session.state,
"isLive": session.is_live
})).collect::<Vec<_>>()
},
"buckets": bucket_checks,
"tasks": task_defs.iter().map(|task| json!({
"taskName": task.task_name,
"userId": task.user_id
})).collect::<Vec<_>>(),
"interactiveStale": interactive_stale,
"effectiveInteractiveStale": effective_interactive_stale,
"fileOperationsPresence": {
"enabled": file_ops_enabled,
"mode": file_ops_mode,
@@ -4282,6 +4350,51 @@ fn guard_task_definitions(config: &Value) -> Vec<GuardTaskDefinition> {
out
}
fn live_session_ids(sessions: &[SessionInfo]) -> HashSet<u32> {
sessions
.iter()
.filter(|session| session.is_live)
.map(|session| session.session_id)
.collect()
}
fn task_has_live_session(user_id: &str, sessions: &[SessionInfo]) -> bool {
let candidates = user_candidates(user_id);
sessions.iter().any(|session| {
session.is_live
&& session
.user_name
.as_deref()
.is_some_and(|user| user_matches_candidates(user, &candidates))
})
}
fn user_candidates(user_id: &str) -> HashSet<String> {
let normalized = user_id.trim().to_ascii_lowercase();
let mut out = HashSet::new();
if normalized.is_empty() {
return out;
}
out.insert(normalized.clone());
if let Some((_, leaf)) = normalized.rsplit_once('\\') {
out.insert(leaf.to_string());
}
out
}
fn user_matches_candidates(user_name: &str, candidates: &HashSet<String>) -> bool {
let normalized = user_name.trim().to_ascii_lowercase();
if normalized.is_empty() {
return false;
}
if candidates.contains(&normalized) {
return true;
}
normalized
.rsplit_once('\\')
.is_some_and(|(_, leaf)| candidates.contains(leaf))
}
fn run_scheduled_task(task_name: &str) -> bool {
if !task_name.starts_with("ActivityWatch Launch ") {
return false;
@@ -4293,6 +4406,54 @@ fn run_scheduled_task(task_name: &str) -> bool {
.unwrap_or(false)
}
fn non_live_session_collectors<'a>(
processes: &'a [ProcessInfo],
live_session_ids: &HashSet<u32>,
) -> Vec<&'a ProcessInfo> {
processes
.iter()
.filter(|process| {
let Some(session_id) = process.session_id else {
return false;
};
session_id > 0
&& !live_session_ids.contains(&session_id)
&& session_scoped_collector_kind(process).is_some()
&& process.pid.is_some()
})
.collect()
}
fn session_scoped_collector_kind(process: &ProcessInfo) -> Option<&'static str> {
if let Some(kind) = legacy_collector_kind(process) {
return Some(kind);
}
let name = process.name.as_deref().unwrap_or_default();
if name.eq_ignore_ascii_case("aw-watcher-afk.exe") {
return Some("afk");
}
if name.eq_ignore_ascii_case("aw-watcher-window.exe") {
return Some("window");
}
if name.eq_ignore_ascii_case("aw-windows-telemetry.exe") {
let command_line = process
.command_line
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
if command_line.contains("browser-domains-collector") {
return Some("browser");
}
if command_line.contains("dlp-endpoint-collector") {
return Some("dlp_endpoint");
}
if command_line.contains("file-operations-collector") {
return Some("fileops");
}
}
None
}
#[derive(Debug, Clone, Serialize)]
struct LegacyCollectorDuplicate {
kind: &'static str,
@@ -5011,6 +5172,22 @@ struct ProcessInfo {
command_line: Option<String>,
}
#[derive(Debug, Default)]
struct SessionSnapshot {
query_ok: bool,
source: String,
error: Option<String>,
sessions: Vec<SessionInfo>,
}
#[derive(Debug, Clone)]
struct SessionInfo {
session_id: u32,
user_name: Option<String>,
state: String,
is_live: bool,
}
fn validate_files(paths: &[PathBuf]) -> Value {
let mut list = Vec::new();
let mut missing = Vec::new();
@@ -5202,6 +5379,118 @@ fn collect_process_snapshot() -> ProcessSnapshot {
}
}
fn collect_session_snapshot() -> SessionSnapshot {
if let Some(raw) = command_output_utf16le("cmd", &["/U", "/C", "query user"])
.or_else(|| command_output_utf16le("cmd", &["/U", "/C", "quser"]))
{
let sessions = parse_query_user_sessions(&raw);
if !sessions.is_empty() {
return SessionSnapshot {
query_ok: true,
source: "quser_utf16".to_string(),
error: None,
sessions,
};
}
}
if let Some(raw) = command_output_lossy_combined("cmd", &["/C", "query user"])
.or_else(|| command_output_lossy_combined("cmd", &["/C", "quser"]))
{
let sessions = parse_query_user_sessions(&raw);
if !sessions.is_empty() {
return SessionSnapshot {
query_ok: true,
source: "quser_lossy".to_string(),
error: None,
sessions,
};
}
}
SessionSnapshot {
query_ok: false,
source: "unavailable".to_string(),
error: Some("query user and quser returned no sessions".to_string()),
sessions: Vec::new(),
}
}
fn command_output_utf16le(program: &str, args: &[&str]) -> Option<String> {
let output = Command::new(program).args(args).output().ok()?;
if !output.status.success() {
return None;
}
let mut bytes = output.stdout;
bytes.extend_from_slice(&output.stderr);
if bytes.is_empty() {
return None;
}
let words = bytes
.chunks_exact(2)
.map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
.collect::<Vec<_>>();
String::from_utf16(&words)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn command_output_lossy_combined(program: &str, args: &[&str]) -> Option<String> {
let output = Command::new(program).args(args).output().ok()?;
if !output.status.success() {
return None;
}
let mut bytes = output.stdout;
bytes.extend_from_slice(&output.stderr);
Some(String::from_utf8_lossy(&bytes).trim().to_string()).filter(|value| !value.is_empty())
}
fn parse_query_user_sessions(raw: &str) -> Vec<SessionInfo> {
raw.lines()
.filter_map(parse_query_user_line)
.collect::<Vec<_>>()
}
fn parse_query_user_line(line: &str) -> Option<SessionInfo> {
let cleaned = line.trim().trim_start_matches('>').trim();
if cleaned.is_empty()
|| cleaned.to_ascii_lowercase().starts_with("username")
|| cleaned.starts_with("ПОЛЬЗОВАТЕЛЬ")
{
return None;
}
let parts = cleaned.split_whitespace().collect::<Vec<_>>();
if parts.len() < 3 {
return None;
}
let username = parts.first()?.trim();
let (session_id, state, has_session_name) =
if parts.get(1)?.chars().all(|ch| ch.is_ascii_digit()) {
(*parts.get(1)?, *parts.get(2)?, false)
} else {
(*parts.get(2)?, *parts.get(3).unwrap_or(&"Unknown"), true)
};
let session_id = session_id.parse::<u32>().ok()?;
Some(SessionInfo {
session_id,
user_name: (!username.is_empty()).then(|| username.to_string()),
state: state.to_string(),
is_live: session_state_is_live(state)
|| (has_session_name && !session_state_is_disconnected(state)),
})
}
fn session_state_is_live(state: &str) -> bool {
let lower = state.to_lowercase();
lower.contains("active") || lower.contains("conn") || lower.contains("актив")
}
fn session_state_is_disconnected(state: &str) -> bool {
let lower = state.to_lowercase();
lower.contains("disc") || lower.contains("диск")
}
#[cfg(windows)]
fn collect_native_process_snapshot() -> Option<ProcessSnapshot> {
use std::mem::{MaybeUninit, size_of};
@@ -6368,6 +6657,81 @@ SERVICE_NAME: AWatchRusCollectorGuard
);
}
#[test]
fn collector_guard_parses_live_and_disconnected_sessions() {
let sessions = parse_query_user_sessions(
r#"
USERNAME SESSIONNAME ID STATE IDLE TIME LOGON TIME
user1 rdp-tcp#5 5 Active none 09.07.2026 10:00
user2 2 Disc 3:14 07.07.2026 8:34
"#,
);
assert_eq!(sessions.len(), 2);
assert_eq!(live_session_ids(&sessions), HashSet::from([5]));
assert!(task_has_live_session(r"SHARKON2025\user1", &sessions));
assert!(!task_has_live_session(r"SHARKON2025\user2", &sessions));
}
#[test]
fn collector_guard_treats_named_rdp_session_as_live_when_state_is_localized() {
let sessions = parse_query_user_sessions(
r#"
USERNAME SESSIONNAME ID STATE IDLE TIME LOGON TIME
user1 rdp-tcp#12 12 ????? none 09.07.2026 10:00
user2 2 ????? 3:14 07.07.2026 8:34
"#,
);
assert_eq!(sessions.len(), 2);
assert_eq!(live_session_ids(&sessions), HashSet::from([12]));
assert!(task_has_live_session(r"SHARKON2025\user1", &sessions));
assert!(!task_has_live_session(r"SHARKON2025\user2", &sessions));
}
#[test]
fn collector_guard_stops_session_collectors_outside_live_sessions() {
let processes = vec![
ProcessInfo {
name: Some("aw-watcher-afk.exe".to_string()),
pid: Some(100),
session_id: Some(2),
created_unix_seconds: Some(10),
command_line: Some("aw-watcher-afk.exe --host 10.10.10.13".to_string()),
},
ProcessInfo {
name: Some("aw-windows-telemetry.exe".to_string()),
pid: Some(101),
session_id: Some(2),
created_unix_seconds: Some(11),
command_line: Some(
"aw-windows-telemetry.exe browser-domains-collector --mode enforce".to_string(),
),
},
ProcessInfo {
name: Some("aw-watcher-window.exe".to_string()),
pid: Some(200),
session_id: Some(5),
created_unix_seconds: Some(20),
command_line: Some("aw-watcher-window.exe --host 10.10.10.13".to_string()),
},
ProcessInfo {
name: Some("awatch-agent-rs.exe".to_string()),
pid: Some(300),
session_id: Some(0),
created_unix_seconds: Some(30),
command_line: Some("awatch-agent-rs.exe --config x".to_string()),
},
];
let stop_plan = non_live_session_collectors(&processes, &HashSet::from([5]));
let pids = stop_plan
.iter()
.filter_map(|process| process.pid)
.collect::<Vec<_>>();
assert_eq!(pids, vec![100, 101]);
}
#[test]
fn validate_deployment_parses_wmic_process_csv() {
let csv = br#"Node,CommandLine,Name,ProcessId,SessionId
+60 -13
View File
@@ -221,15 +221,17 @@ fn run(cli: &Cli) -> Result<Report> {
let worktime = influx_config(&aw_env, "AW_WORKTIME_INFLUX");
let dlp = influx_config(&aw_env, "AW_DLP_INFLUX");
let dlp_enabled = env_bool(&aw_env, "AW_DLP_ENABLED", true);
let dlp_profile = env_value(&aw_env, "AW_DLP_PROFILE", "full");
let dlp_influx_required = dlp_influx_required_for_profile(dlp_enabled, &dlp_profile);
checks.push(check_influx_env(&worktime, cli.allow_disabled_influx));
if dlp_enabled {
if dlp_influx_required {
checks.push(check_influx_env(&dlp, cli.allow_disabled_influx));
} else {
checks.push(warn(
checks.push(ok(
"env:AW_DLP_INFLUX",
"DLP Influx runtime disabled by AW_DLP_ENABLED=false",
json!({"enabled": false, "mode": "disabled"}),
"DLP Influx runtime is not required by the current DLP profile",
json!({"enabled": dlp_enabled, "profile": dlp_profile.as_str()}),
));
}
@@ -238,7 +240,7 @@ fn run(cli: &Cli) -> Result<Report> {
} else {
checks.extend(check_systemd_services(&systemd_services_for_mode(
&cli.systemd_services,
dlp_enabled,
dlp_influx_required,
)));
}
@@ -250,13 +252,13 @@ fn run(cli: &Cli) -> Result<Report> {
));
} else {
checks.push(check_influx_write(&client, "worktime", &worktime));
if dlp_enabled {
if dlp_influx_required {
checks.push(check_influx_write(&client, "dlp", &dlp));
} else {
checks.push(warn(
checks.push(ok(
"influx:write:dlp",
"DLP write probe skipped because DLP is disabled",
json!({"enabled": false, "mode": "disabled"}),
"DLP write probe skipped because DLP Influx is not required by the current profile",
json!({"enabled": dlp_enabled, "profile": dlp_profile.as_str()}),
));
}
}
@@ -292,7 +294,7 @@ fn run(cli: &Cli) -> Result<Report> {
git_commit: cli.git_commit.clone(),
counts,
checks,
limitations: build_limitations(cli, dlp_enabled),
limitations: build_limitations(cli, dlp_enabled, dlp_influx_required),
})
}
@@ -360,9 +362,19 @@ fn split_csv(value: &str) -> Vec<String> {
.collect()
}
fn systemd_services_for_mode(csv: &str, dlp_enabled: bool) -> String {
fn dlp_influx_required_for_profile(dlp_enabled: bool, profile: &str) -> bool {
if !dlp_enabled {
return false;
}
!matches!(
profile.trim().to_ascii_lowercase().as_str(),
"light" | "core_only" | "disabled" | "off" | "on_demand"
)
}
fn systemd_services_for_mode(csv: &str, dlp_influx_required: bool) -> String {
let mut services = split_csv(csv);
if dlp_enabled {
if dlp_influx_required {
for service in split_csv(DEFAULT_DLP_SYSTEMD_SERVICES) {
if !services.iter().any(|item| item == &service) {
services.push(service);
@@ -384,7 +396,7 @@ fn hostname() -> String {
.unwrap_or_else(|| "unknown".to_string())
}
fn build_limitations(cli: &Cli, dlp_enabled: bool) -> Vec<String> {
fn build_limitations(cli: &Cli, dlp_enabled: bool, dlp_influx_required: bool) -> Vec<String> {
let mut limitations = Vec::new();
limitations.push(
"Проверка подтверждает состояние runtime на момент формирования акта и не заменяет аудит конфигурации, нагрузочное тестирование или приемочные испытания заказчика.".to_string(),
@@ -417,6 +429,10 @@ fn build_limitations(cli: &Cli, dlp_enabled: bool) -> Vec<String> {
limitations.push(
"DLP runtime отключен штатно через AW_DLP_ENABLED=false; readiness не считает DLP services/timers и DLP Influx write обязательными.".to_string(),
);
} else if !dlp_influx_required {
limitations.push(
"DLP runtime включен в лёгком профиле; readiness не считает DLP Influx timer и DLP Influx write обязательными.".to_string(),
);
}
limitations
}
@@ -1348,6 +1364,37 @@ mod tests {
assert_eq!(counts.fail, 1);
}
#[test]
fn dlp_influx_is_required_only_for_full_profile() {
for profile in ["full", "enabled", "on"] {
assert!(
dlp_influx_required_for_profile(true, profile),
"{profile} should require DLP Influx"
);
}
for profile in ["light", "core_only", "disabled", "off", "on_demand"] {
assert!(
!dlp_influx_required_for_profile(true, profile),
"{profile} should not require DLP Influx"
);
}
assert!(!dlp_influx_required_for_profile(false, "full"));
assert!(dlp_influx_required_for_profile(true, "unexpected"));
}
#[test]
fn light_profile_excludes_dlp_influx_timer() {
let services = systemd_services_for_mode(DEFAULT_SYSTEMD_SERVICES, false);
assert!(services.contains("activitywatch-server"));
assert!(services.contains("aw-worktime-influx-exporter.timer"));
assert!(!services.contains("aw-dlp-influx-exporter.timer"));
let services = systemd_services_for_mode(DEFAULT_SYSTEMD_SERVICES, true);
assert!(services.contains("aw-dlp-influx-exporter.timer"));
}
#[test]
fn renders_readiness_act_without_secrets() {
let checks = vec![ok(
@@ -31,6 +31,7 @@ regex.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
zip.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -1,12 +1,16 @@
use std::fs::{self, File};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::thread;
use std::time::{Duration, Instant, SystemTime};
use anyhow::{Context, Result, bail};
use chrono::Utc;
use clap::Parser;
use fs2::FileExt;
use hayabusa_tools::{guess_host_from_filename, read_json_file};
use serde_json::{Value, json};
use zip::ZipArchive;
const LOCK_PATH: &str = "/opt/hayabusa/state/aw-hayabusa-autoprocess.lock";
const WRAPPER: &str = "/usr/local/bin/aw-hayabusa";
@@ -20,6 +24,9 @@ struct Cli {
#[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/drop")]
drop_dir: PathBuf,
#[arg(long, default_value = "/opt/hayabusa/quarantine/drop")]
quarantine_dir: PathBuf,
#[arg(long, default_value_t = true)]
once: bool,
}
@@ -66,15 +73,29 @@ fn run() -> Result<i32> {
return Ok(0);
}
for zip_path in zips {
let result = process_one(&zip_path)?;
println!(
"{}",
serde_json::to_string_pretty(&json!({
"processed": zip_path.display().to_string(),
"latest_intake": result.latest_intake,
"case_alert": result.case_alert,
}))?
);
match process_one(&zip_path) {
Ok(result) => {
println!(
"{}",
serde_json::to_string_pretty(&json!({
"processed": zip_path.display().to_string(),
"latest_intake": result.latest_intake,
"case_alert": result.case_alert,
}))?
);
}
Err(err) => {
let quarantined = quarantine_drop_package(&cli.quarantine_dir, &zip_path, &err)?;
println!(
"{}",
serde_json::to_string_pretty(&json!({
"quarantined": zip_path.display().to_string(),
"quarantine_dir": quarantined.display().to_string(),
"reason": err.to_string(),
}))?
);
}
}
}
Ok(0)
}
@@ -97,6 +118,7 @@ fn list_zips(drop_dir: &Path) -> Result<Vec<PathBuf>> {
}
fn process_one(zip_path: &Path) -> Result<ProcessResult> {
wait_for_stable_zip(zip_path)?;
let sidecars = load_sidecars(zip_path)?;
let host = guess_host(zip_path, &sidecars);
let mode = if sidecars.mode.is_empty() {
@@ -174,6 +196,85 @@ fn process_one(zip_path: &Path) -> Result<ProcessResult> {
})
}
fn wait_for_stable_zip(zip_path: &Path) -> Result<()> {
wait_for_stable_zip_with(
zip_path,
Duration::from_secs(60),
Duration::from_secs(1),
Duration::from_secs(2),
2,
)
}
fn wait_for_stable_zip_with(
zip_path: &Path,
max_wait: Duration,
interval: Duration,
min_modified_age: Duration,
required_stable_checks: u32,
) -> Result<()> {
let started = Instant::now();
let mut last_len = None;
let mut stable_checks = 0;
let mut last_zip_error = None;
loop {
let metadata =
fs::metadata(zip_path).with_context(|| format!("stat {}", zip_path.display()))?;
let len = metadata.len();
let modified_age = metadata
.modified()
.ok()
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
.unwrap_or_default();
if len > 0 && last_len == Some(len) && modified_age >= min_modified_age {
stable_checks += 1;
} else {
stable_checks = 0;
}
if stable_checks >= required_stable_checks {
match verify_zip_readable(zip_path) {
Ok(()) => return Ok(()),
Err(err) => {
last_zip_error = Some(err.to_string());
stable_checks = 0;
}
}
}
if started.elapsed() >= max_wait {
if let Some(err) = last_zip_error {
bail!(
"drop zip {} did not become readable: {err}",
zip_path.display()
);
}
bail!(
"drop zip {} did not stabilize within {}s",
zip_path.display(),
max_wait.as_secs()
);
}
last_len = Some(len);
thread::sleep(interval);
}
}
fn verify_zip_readable(zip_path: &Path) -> Result<()> {
let file = File::open(zip_path).with_context(|| format!("open {}", zip_path.display()))?;
let mut archive =
ZipArchive::new(file).with_context(|| format!("read zip {}", zip_path.display()))?;
for idx in 0..archive.len() {
let _entry = archive
.by_index(idx)
.with_context(|| format!("read zip entry {idx}"))?;
}
Ok(())
}
fn load_sidecars(zip_path: &Path) -> Result<Sidecars> {
let base = zip_path.with_extension("");
let caseid_path = base.with_extension("caseid");
@@ -214,6 +315,49 @@ fn load_sidecars(zip_path: &Path) -> Result<Sidecars> {
})
}
fn quarantine_drop_package(
quarantine_dir: &Path,
zip_path: &Path,
err: &anyhow::Error,
) -> Result<PathBuf> {
fs::create_dir_all(quarantine_dir)
.with_context(|| format!("create {}", quarantine_dir.display()))?;
let timestamp = Utc::now().format("%Y%m%dT%H%M%SZ");
let zip_name = zip_path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("package.zip");
let target_dir = quarantine_dir.join(format!("{timestamp}_{zip_name}"));
fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?;
let base = zip_path.with_extension("");
for path in [
zip_path.to_path_buf(),
base.with_extension("caseid"),
base.with_extension("meta.json"),
zip_path.with_extension("zip.sha256"),
] {
if path.is_file() {
let target = target_dir.join(path.file_name().context("quarantine file name")?);
fs::rename(&path, &target)
.with_context(|| format!("move {} to {}", path.display(), target.display()))?;
}
}
let reason = json!({
"quarantined_at": Utc::now().to_rfc3339(),
"package": zip_path.display().to_string(),
"reason": err.to_string(),
"error_chain": format!("{err:#}"),
});
fs::write(
target_dir.join("reason.json"),
serde_json::to_vec_pretty(&reason).context("serialize quarantine reason")?,
)
.with_context(|| format!("write {}", target_dir.join("reason.json").display()))?;
Ok(target_dir)
}
fn archive_sidecars(report_dir: &Path, sidecars: &Sidecars) -> Result<()> {
let target_dir = report_dir.join("input-sidecars");
fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?;
@@ -302,3 +446,54 @@ fn run_capture(program: &Path, args: &[String]) -> Result<Captured> {
stderr: String::from_utf8_lossy(&output.stderr).to_string(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use tempfile::tempdir;
use zip::write::SimpleFileOptions;
#[test]
fn wait_for_stable_zip_accepts_complete_archive() {
let dir = tempdir().expect("tempdir");
let zip_path = dir.path().join("HOST-20260709-000001.zip");
write_test_zip(&zip_path);
wait_for_stable_zip_with(
&zip_path,
Duration::from_secs(1),
Duration::from_millis(1),
Duration::from_secs(0),
1,
)
.expect("complete zip should be accepted");
}
#[test]
fn wait_for_stable_zip_rejects_unreadable_archive() {
let dir = tempdir().expect("tempdir");
let zip_path = dir.path().join("HOST-20260709-000001.zip");
fs::write(&zip_path, b"not a zip").expect("write partial zip");
let err = wait_for_stable_zip_with(
&zip_path,
Duration::from_millis(20),
Duration::from_millis(1),
Duration::from_secs(0),
1,
)
.expect_err("invalid zip should be rejected");
assert!(err.to_string().contains("did not become readable"));
}
fn write_test_zip(path: &Path) {
let file = File::create(path).expect("create zip");
let mut zip = zip::ZipWriter::new(file);
zip.start_file("manifest.json", SimpleFileOptions::default())
.expect("start manifest");
zip.write_all(br#"{"host":"SHARKON2025"}"#)
.expect("write manifest");
zip.finish().expect("finish zip");
}
}
+3
View File
@@ -249,6 +249,8 @@
Environment=AW_BACKUP_RETENTION_DAYS={{ aw_server_backup_retention_days }}
Environment=AW_BACKUP_KEEP_LAST_DB={{ aw_server_backup_keep_last_db }}
Environment=AW_BACKUP_KEEP_LAST_JSON={{ aw_server_backup_keep_last_json }}
Environment=AW_WORKTIME_REPORT_DISK_CACHE_DIR={{ aw_worktime_report_disk_cache_dir | default(aw_server_data_dir ~ '/worktime-report-cache') }}
Environment=AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS={{ aw_worktime_report_disk_stale_ttl_seconds | default(86400) }}
ExecStart=/usr/local/bin/aw-prune-local-state.sh
- name: Установить systemd timer prune локального state
@@ -792,6 +794,7 @@
AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS={{ aw_worktime_events_cache_ttl_seconds | default(300) }}
AW_WORKTIME_REPORT_CACHE_TTL_SECONDS={{ aw_worktime_report_cache_ttl_seconds | default(300) }}
AW_WORKTIME_REPORT_STALE_TTL_SECONDS={{ aw_worktime_report_stale_ttl_seconds | default(3600) }}
AW_WORKTIME_REPORT_DISK_CACHE_DIR={{ aw_worktime_report_disk_cache_dir | default(aw_server_data_dir ~ '/worktime-report-cache') }}
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS={{ aw_worktime_report_disk_stale_ttl_seconds | default(86400) }}
AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output
AW_DLP_POLICY_ENGINE_BIND_HOST={{ aw_dlp_policy_engine_bind_host }}
+1
View File
@@ -17,6 +17,7 @@ aw_worktime_aw_http_timeout_seconds: 6
aw_worktime_events_cache_ttl_seconds: 300
aw_worktime_report_cache_ttl_seconds: 300
aw_worktime_report_stale_ttl_seconds: 3600
aw_worktime_report_disk_cache_dir: "{{ aw_server_data_dir }}/worktime-report-cache"
aw_worktime_report_disk_stale_ttl_seconds: 86400
aw_worktime_influx_enabled: false
aw_worktime_influx_url: "http://<INFLUXDB_HOST>:8086"
+1
View File
@@ -17,6 +17,7 @@ aw_worktime_aw_http_timeout_seconds: 6
aw_worktime_events_cache_ttl_seconds: 300
aw_worktime_report_cache_ttl_seconds: 300
aw_worktime_report_stale_ttl_seconds: 3600
aw_worktime_report_disk_cache_dir: "{{ aw_server_data_dir }}/worktime-report-cache"
aw_worktime_report_disk_stale_ttl_seconds: 86400
aw_worktime_influx_enabled: true
aw_worktime_influx_url: "http://192.0.2.10:8086"
+14 -3
View File
@@ -250,6 +250,10 @@ server {
}
{% for route in proxmox_web_gateway_routes %}
location = /go/{{ route.slug }} {
return 302 {{ route.target_url }};
}
{% if route.proxy_target_url is defined %}
location = {{ route.proxy_path }} {
proxy_set_header Authorization "";
@@ -267,14 +271,21 @@ server {
location ^~ /portal/api/dlp/evidence {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://192.0.2.13:8721/api/dlp/evidence;
proxy_pass http://127.0.0.1:8720/api/dlp/evidence;
proxy_redirect off;
}
location ^~ /portal/api/readiness {
location = /portal/api/readiness {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://192.0.2.13:8721/api/readiness;
proxy_pass http://127.0.0.1:8720/readyz;
proxy_redirect off;
}
location ^~ /portal/api/readiness/ {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://127.0.0.1:8720/api/readiness/;
proxy_redirect off;
}
+13
View File
@@ -22,6 +22,7 @@ AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS=6
AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS=300
AW_WORKTIME_REPORT_CACHE_TTL_SECONDS=300
AW_WORKTIME_REPORT_STALE_TTL_SECONDS=3600
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400
AW_SERVER_URL=http://127.0.0.1:5600
AW_DLP_AW_API_BASE=http://127.0.0.1:5600/api/0
@@ -80,10 +81,22 @@ AW_BROWSER_SMOKE_WORKTIME_BASE=http://127.0.0.1:5610
AW_BROWSER_SMOKE_HOST=HOST-EXAMPLE
AW_BROWSER_SMOKE_OUTPUT_DIR=/var/lib/activitywatch/browser-smoke
AW_BROWSER_SMOKE_KEEP_RUNS=24
AW_BROWSER_SMOKE_RETENTION_DAYS=1
AW_BROWSER_SMOKE_ENGINE=chromium-cli
AW_BROWSER_SMOKE_TIMEOUT_MS=20000
AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS=15000
# Retention / cleanup policy. Cleanup must stay allowlisted and dry-run tested
# before changing these values in production.
AW_BACKUP_RETENTION_DAYS=7
AW_BACKUP_KEEP_LAST_DB=2
AW_BACKUP_KEEP_LAST_JSON=2
AW_TMP_ARCHIVE_RETENTION_DAYS=1
AW_TMP_WEBUI_RETENTION_DAYS=2
AW_DB_MAINTENANCE_RETENTION_DAYS=7
AW_DB_MAINTENANCE_CHUNK_SIZE=1000
DETMIR_READINESS_RETENTION_DAYS=30
# Hayabusa auto-case / alerting
AW_HAYABUSA_AUTO_CASE_ENABLED=true
AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY=medium
+1 -1
View File
@@ -118,4 +118,4 @@ Production scheduled task on `SHARKON2025`:
Do not switch this task back to `SYSTEM` on the current RDP host: Task Scheduler starts `powershell.exe` under `SYSTEM`, but the process exits with `0xC0000142` before the upload script starts.
Server-side processing accepts Windows zip packages with backslash path separators and UTF-8 BOM in sidecar JSON. `aw-hayabusa-autoprocess` processes the full incoming queue after accepting a drop package, so stale incoming files from an earlier failed run are drained before the latest intake is recorded.
Server-side processing accepts Windows zip packages with backslash path separators and UTF-8 BOM in sidecar JSON. `aw-hayabusa-autoprocess` waits for a dropped ZIP to become size-stable and readable before accepting it, then processes the full incoming queue. This prevents `systemd.path` from quarantining an archive while SCP is still writing it. ZIPs that remain unreadable after the bounded wait are moved to `/opt/hayabusa/quarantine/drop` with `reason.json`.
+22
View File
@@ -3,6 +3,28 @@
# Сервер: http://10.10.10.13:5600
# RDP-хост: 192.168.100.19 (logical host id SHARKON2025)
normalize_http_base() {
local value="${1:-}"
value="${value%/}"
case "$value" in
"") return 1 ;;
http://*|https://*) printf '%s' "$value" ;;
*) printf 'http://%s' "$value" ;;
esac
}
normalize_url_env() {
local name="$1"
local value="${!name:-}"
if [ -n "$value" ]; then
export "$name=$(normalize_http_base "$value")"
fi
}
normalize_url_env CHECK_AW_FULL_SERVER
normalize_url_env AW_SMOKE_AW_SERVER
normalize_url_env AW_SERVER
if [[ "${CHECK_AW_FULL_FORCE_LEGACY:-0}" != "1" ]]; then
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
for candidate in \
+3 -4
View File
@@ -18,13 +18,12 @@ fi
# shellcheck disable=SC1090
. "${ENV_FILE}"
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
query_max_age() {
local table="$1"
docker exec "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
-q "SELECT if(count()=0, -1, dateDiff('hour', max(ts), now())) FROM ${table}"
}
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
aw_1c_xml_escape() {
local value="$1"
value="${value//&/&amp;}"
value="${value//</&lt;}"
value="${value//>/&gt;}"
value="${value//\"/&quot;}"
value="${value//\'/&apos;}"
printf '%s' "${value}"
}
aw_1c_clickhouse_client() {
local container="${CH_CONTAINER:-${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}}"
local local_cfg remote_cfg status
if [[ -z "${CLICKHOUSE_USER:-}" ]]; then
echo "CLICKHOUSE_USER is required" >&2
return 1
fi
if [[ -z "${CLICKHOUSE_DB:-}" ]]; then
echo "CLICKHOUSE_DB is required" >&2
return 1
fi
if [[ -z "${CLICKHOUSE_PASSWORD+x}" ]]; then
echo "CLICKHOUSE_PASSWORD is required" >&2
return 1
fi
local_cfg="$(mktemp "${TMPDIR:-/tmp}/aw-1c-clickhouse-client.XXXXXX.xml")"
chmod 0600 "${local_cfg}"
remote_cfg="/tmp/aw-1c-clickhouse-client.$(date +%s).$$.xml"
{
printf '<config>\n'
printf ' <user>%s</user>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_USER}")"
printf ' <password>%s</password>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_PASSWORD}")"
printf ' <database>%s</database>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_DB}")"
printf '</config>\n'
} > "${local_cfg}"
if ! docker exec -i "${container}" sh -c 'umask 077 && cat > "$1"' sh "${remote_cfg}" < "${local_cfg}"; then
rm -f "${local_cfg}"
echo "failed to stage ClickHouse client config in container" >&2
return 1
fi
rm -f "${local_cfg}"
status=0
docker exec -i "${container}" clickhouse-client --config-file "${remote_cfg}" "$@" || status=$?
docker exec "${container}" rm -f "${remote_cfg}" >/dev/null 2>&1 || true
return "${status}"
}
+7
View File
@@ -8,4 +8,11 @@
- core tables: `365` дней
- detections/cases/timeline: `365` дней или по регламенту ИБ
Важно: это policy target, а не заявление о текущем автоматическом TTL. В
`clickhouse/init/*.sql` сейчас нет TTL clauses, поэтому production cleanup для
ClickHouse должен внедряться отдельной staged migration после backup, dry-run
оценки объема и operator/customer approval.
Сводная политика хранения всего контура: `../../docs/RETENTION_POLICY_RU.md`.
Если регуляторика требует больше, меняется отдельно от Grafana UI.
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# shellcheck disable=SC2119
set -euo pipefail
ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
@@ -21,18 +22,20 @@ if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_registry_bindings_refresh.sh"
@@ -41,5 +44,4 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"
@@ -15,17 +15,19 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
"${VENV}/bin/python" "${ROOT}/ai/refresh_company_registry_bindings.py" \
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"
+7 -16
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# shellcheck disable=SC2119
set -euo pipefail
ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
@@ -40,6 +41,8 @@ fi
# shellcheck disable=SC1090
. "${ENV_FILE}"
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
if [[ "${RUN_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST}" == "1" ]]; then
"${VENV}/bin/python" "${ROOT}/etl/extract_1c_mcp_toolkit.py" --config "${CONFIG}"
@@ -49,31 +52,19 @@ fi
"${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}"
"${VENV}/bin/python" "${ROOT}/etl/load_company_registry_xlsx.py" --config "${CONFIG}" --landing "${ROOT}/landing/registry"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/build_entity_timeline.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_registry_bindings_refresh.sh"
"${ROOT}/ops/run_company_intelligence_refresh.sh"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/insert_detections.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/open_cases_from_detections.sql"
if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then
+2 -2
View File
@@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
@@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}"
@@ -43,7 +44,6 @@ while (( attempt <= RETRIES )); do
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"; then
exit 0
fi
+2 -2
View File
@@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
@@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
exec 9>"${LOCK_FILE}"
if ! flock -w "${LOCK_WAIT_SEC}" 9; then
@@ -41,7 +42,6 @@ while (( attempt <= RETRIES )); do
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"; then
exit 0
fi
+143
View File
@@ -0,0 +1,143 @@
{
"version": "2026-07-01.operational-maturity-v1",
"apiCompatibility": {
"openapiPath": "adk-rust/crates/detmir-portal/src/contracts/openapi.json",
"requiredOpenApiPaths": {
"/contracts": ["get"],
"/contracts/openapi.json": ["get"],
"/health": ["get"],
"/operator": ["get"],
"/reports": ["get"],
"/workforce/kpi/explain": ["get"],
"/ueba": ["get"],
"/risk/narrative": ["get"],
"/actions": ["get"],
"/readiness/latest": ["get"],
"/telemetry": ["post"]
},
"requiredSchemas": [
"ContractIndex",
"JsonObject",
"ActionCenterResponse",
"RiskNarrative",
"UebaPayload",
"WorkforceKpiExplain"
],
"runtimeEndpoints": [
"/healthz",
"/readyz",
"/version",
"/metrics"
],
"runtimeSourcePath": "adk-rust/crates/detmir-portal/src/main.rs"
},
"integrationFixtures": [
{
"path": "/healthz",
"fixture": "docs/fixtures/operational-maturity/portal-healthz.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["status", "service", "schema_version"],
"equals": { "status": "ok", "schema_version": "pilot-v1" }
},
{
"path": "/readyz",
"fixture": "docs/fixtures/operational-maturity/portal-readyz.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["status", "checks.portal", "checks.dlp_runtime"],
"equals": { "status": "ready", "checks.dlp_runtime.status": "skipped" }
},
{
"path": "/version",
"fixture": "docs/fixtures/operational-maturity/portal-version.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["app_version", "schema_version", "environment"],
"equals": { "schema_version": "pilot-v1" }
},
{
"path": "/reports/worktime/management",
"fixture": "docs/fixtures/operational-maturity/worktime-management-ok.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["status", "stale", "report_cache_hit", "aw_query_duration_ms"],
"equals": { "status": "OK", "stale": false }
},
{
"path": "/reports/worktime/management?fault=aw_timeout",
"fixture": "docs/fixtures/operational-maturity/worktime-management-degraded.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["status", "reason", "stale", "report_stale_served", "aw_query_timeout_count"],
"equals": { "status": "DEGRADED", "reason": "report_unavailable", "stale": false }
},
{
"path": "/security/finding-inbox",
"fixture": "docs/fixtures/operational-maturity/security-finding-inbox-shadow.json",
"status": 200,
"contentType": "application/json",
"requiredFields": ["mode", "auto_apply", "workflow"],
"equals": { "mode": "shadow", "auto_apply": false }
}
],
"observability": {
"metricsFixture": "docs/fixtures/operational-maturity/portal-metrics.prom",
"requiredMetrics": [
"awatch_http_requests_total",
"awatch_http_request_duration_seconds_sum",
"awatch_http_request_duration_seconds_count",
"awatch_report_requests_total",
"awatch_report_cache_hits_total",
"awatch_report_cache_misses_total",
"awatch_report_cache_stale_hits_total",
"awatch_reports_generated_total",
"awatch_ingestion_records_total",
"awatch_ingestion_rejected_total",
"awatch_role_denied_total",
"awatch_readyz_status"
],
"sourcePath": "adk-rust/crates/detmir-portal/src/production/metrics.rs",
"diagnosticHeaders": ["x-request-id", "x-correlation-id"]
},
"configValidation": {
"jsonFiles": [
"configs/worktime-interpretation-policy.example.json",
"configs/detmir-workforce-policy.example.json",
"adk-rust/crates/detmir-portal/src/contracts/openapi.json",
"docs/fixtures/operational-maturity/portal-healthz.json",
"docs/fixtures/operational-maturity/portal-readyz.json",
"docs/fixtures/operational-maturity/portal-version.json",
"docs/fixtures/operational-maturity/worktime-management-ok.json",
"docs/fixtures/operational-maturity/worktime-management-degraded.json",
"docs/fixtures/operational-maturity/security-finding-inbox-shadow.json"
],
"systemdDirs": [
"ops/systemd",
"clickhouse-1c/ops"
],
"clickhouseInitDir": "clickhouse-1c/clickhouse/init",
"yamlFiles": [
"configs/detmir-ueba-risk-policy.example.yaml",
"clickhouse-1c/etl/config.example.yml",
"clickhouse-1c/grafana/provisioning/datasources/clickhouse.yml",
"clickhouse-1c/grafana/provisioning/dashboards/dashboards.yml"
]
},
"faultInjection": {
"clientTimeoutMs": 350,
"maxFailureClassificationMs": 1200
},
"boundedLoad": {
"requests": 60,
"concurrency": 8,
"p95MaxMs": 750,
"heapGrowthMaxBytes": 20971520,
"paths": [
"/healthz",
"/readyz",
"/version",
"/reports/worktime/management"
]
}
}
+192
View File
@@ -0,0 +1,192 @@
# Автономная проверка AWatch-rus
Документ описывает единый validation entry point для длительной
эксплуатационной проверки AWatch-rus/DetMir без ручного надзора.
Цель framework - не менять production runtime, а регулярно доказывать состояние
репозитория, release artifacts, operational maturity, runtime health и evidence.
## Запуск
Из корня репозитория:
```bash
scripts/run_full_validation.sh --profile full
```
Профили:
- `quick` - repository, JSON/YAML/Markdown, retention/recovery docs, secret and
contract checks.
- `standard` - `quick` плюс operational maturity, deployment readiness, pilot
validation, release artifact checks, binary parity self-test.
- `full` - `standard` плюс Rust, Cargo и dependency hygiene gates.
- `runtime` - `standard` с акцентом на runtime diagnostics и health endpoints.
Для unattended запуска можно задать каталог reports:
```bash
AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation \
AW_VALIDATION_RETENTION_DAYS=45 \
scripts/run_full_validation.sh --profile standard
```
## Production evidence inputs
Framework не собирает production binary parity evidence самостоятельно. Это
намеренное ограничение: оператор сначала формирует approved evidence JSON, затем
gate проверяет его в репозитории.
```bash
PRODUCTION_BINARY_PARITY_EVIDENCE=/var/lib/activitywatch/evidence/production-binary-parity.json \
scripts/run_full_validation.sh --profile standard
```
Release evidence проверяется только когда явно указан каталог evidence package:
```bash
AW_RELEASE_EVIDENCE_DIR=/var/lib/activitywatch/release-evidence/rc-1 \
scripts/run_full_validation.sh --profile standard
```
## Runtime diagnostics
Каждый запуск собирает machine-readable диагностику:
- CPU/load average;
- RAM/swap;
- disk usage;
- размеры известных runtime paths;
- queue directories;
- systemd unit state, restart count, exit status;
- ActivityWatch, Worktime, Grafana, Prometheus, ClickHouse endpoints.
- TLS certificate expiry for hosts listed in `AW_VALIDATION_TLS_HOSTS`.
Endpoint defaults безопасны для локального запуска:
- `AW_VALIDATION_ACTIVITYWATCH_URL`, default
`http://127.0.0.1:5600/api/0/info`;
- `AW_VALIDATION_WORKTIME_URL`, default `http://127.0.0.1:5610/healthz`;
- `AW_VALIDATION_GRAFANA_URL`;
- `AW_VALIDATION_PROMETHEUS_URL`;
- `AW_VALIDATION_CLICKHOUSE_URL`.
- `AW_VALIDATION_TLS_HOSTS`, comma-separated `host:port` list.
ActivityWatch и Worktime по умолчанию проверяются на localhost. Grafana,
Prometheus, ClickHouse и TLS hosts проверяются только если заданы. Если заданный
endpoint недоступен, создается machine-readable alert.
## Reports
Каждый запуск пишет reports в:
```text
output/validation/history/<timestamp>/
output/validation/latest/
```
Файлы:
- `validation-report.json`;
- `validation-report.md`;
- `production-health.json`;
- `deployment-health.json`;
- `runtime-health.json`;
- `release-evidence.json`;
- `operational-evidence.json`;
- `recovery-evidence.json`;
- `build-evidence.json`;
- `operational-dashboard.json`;
- `operational-dashboard.md`.
Все JSON reports содержат timestamp, Git SHA, host identifier, profile,
validation outcome и evidence summary.
## Regression detection
Framework сравнивает текущий запуск с предыдущим report из history и фиксирует:
- новые failures;
- восстановленные failures;
- ухудшение длительности checks;
- изменение `Cargo.lock`;
- configuration drift;
- unexpected release artifact changes;
- disk/database/log path growth above threshold;
- новые alerts.
Первый запуск создает baseline. Дальнейшие запуски пишут regression summary в
`validation-report.json` и `validation-report.md`.
## Alert rules
Alerts создаются при:
- failed validation;
- недоступном настроенном health endpoint;
- превышении disk threshold;
- превышении memory threshold;
- failed systemd unit;
- restart count больше нуля;
- queue backlog выше threshold;
- отсутствии production binary parity evidence;
- отсутствии release evidence directory;
- expired или missing backup evidence path, если он задан;
- secret pattern check failure;
- configuration drift;
- unexpected artifact changes;
- certificate expiry within `AW_VALIDATION_CERT_EXPIRY_WARN_DAYS`.
Thresholds:
```bash
AW_VALIDATION_DISK_USED_WARN_PCT=85
AW_VALIDATION_DISK_USED_CRIT_PCT=95
AW_VALIDATION_MEMORY_USED_WARN_PCT=90
AW_VALIDATION_MEMORY_USED_CRIT_PCT=97
AW_VALIDATION_CERT_EXPIRY_WARN_DAYS=14
AW_VALIDATION_PATH_GROWTH_WARN_BYTES=1073741824
AW_VALIDATION_PATH_GROWTH_WARN_RATIO=1.25
AW_VALIDATION_QUEUE_FILES_WARN=1000
AW_VALIDATION_BACKUP_EVIDENCE_PATH=/var/lib/activitywatch/evidence/latest-backup.json
AW_VALIDATION_BACKUP_EVIDENCE_MAX_AGE_DAYS=7
```
## Scheduling
Пример systemd timer должен выполняться вне hot path и не должен включать DLP,
Loki или always-on Velociraptor:
```ini
[Service]
Type=oneshot
WorkingDirectory=/mnt/usb_hdd2/Projects/ActivityWatch-Russian
Environment=AW_VALIDATION_OUTPUT_DIR=/var/lib/activitywatch/validation
Environment=AW_VALIDATION_RETENTION_DAYS=45
ExecStart=/mnt/usb_hdd2/Projects/ActivityWatch-Russian/scripts/run_full_validation.sh --profile standard
```
Для weekly deep validation использовать `--profile full`, потому что Rust и
dependency gates могут быть дорогими.
## Failure investigation
1. Открыть `output/validation/latest/validation-report.md`.
2. Проверить `Alerts`.
3. Для failed command смотреть `validation-report.json`: `stdout`, `stderr`,
`exit_code`, `timed_out`.
4. Если alert относится к runtime endpoint, проверить соответствующий service
через runbook [эксплуатационной проверки](OPERATIONS_VALIDATION_RUNBOOK_RU.md).
5. Если alert относится к binary parity, обновить production evidence JSON и
повторить запуск.
6. Если alert относится к release evidence, проверить controlled release package
через registry runbook.
## History retention
История хранится в `history/<timestamp>/`. Старые запуски удаляются по
`AW_VALIDATION_RETENTION_DAYS` или `--retention-days`.
Минимальное значение retention: 1 день. Для production рекомендуется 30-45
дней, чтобы видеть недельные тренды по runtime failures, restart frequency,
disk growth и validation score.
+327
View File
@@ -0,0 +1,327 @@
# Ежедневное обслуживание AWatch-rus
Дата актуализации: 2026-07-03
Документ описывает ежедневный операторский цикл обслуживания AWatch-rus /
DetMir и безопасное использование Pollinations AI как вспомогательного
аналитика evidence-артефактов. Pollinations не является production runtime
dependency, не принимает решения автоматически и не имеет прямого доступа к
customer contour.
## Цели
- Подтвердить, что сервисы отвечают и не находятся в скрытой деградации.
- Проверить свежесть данных, состояние очередей, timers, retention и readiness
evidence.
- Сформировать короткий дневной maintenance summary.
- Использовать Pollinations только для сжатия и первичного анализа локальных
JSON/Markdown evidence без секретов.
## Guardrails
- Не включать heavy DLP, Loki или always-on Velociraptor в рамках ежедневного
обслуживания.
- DLP runtime для DetMir production считать нормальным в light/disabled profile,
если оператор отдельно не принял решение о включении.
- Не удалять ClickHouse, Grafana, DLP evidence, Hayabusa archives, Windows
queues, release evidence и diagnostic bundles вручную без отдельного change.
- Pollinations failure не является отказом AWatch-rus. Если AI недоступен,
ежедневное обслуживание продолжается по обычному runbook.
- Не отправлять в Pollinations secrets, private env files, bearer tokens,
пароли, raw customer evidence, user identifiers, скриншоты с ПДн или полные
журналы production.
## Ежедневный цикл
### 1. Быстрый статус сервисов
Проверить локальный health/readiness и failed units:
```bash
curl -fsS http://127.0.0.1:5600/api/0/info >/dev/null
curl -fsS http://127.0.0.1:5610/healthz >/dev/null
systemctl --failed --no-pager
systemctl list-timers \
aw-prune-local-state.timer \
aw-db-maintenance.timer \
aw-db-vacuum.timer \
detmir-readiness.timer
```
Критерий: ActivityWatch и Worktime API отвечают, нет неожиданных failed units,
ежедневные timers присутствуют в расписании.
### 2. Readiness bundle
Проверить, что ежедневный bundle сформирован и проверяется:
```bash
cd /var/lib/activitywatch/health/readiness-bundle
sha256sum -c sha256sums.txt
openssl dgst -sha256 -verify public-key.pem \
-signature sha256sums.txt.sig sha256sums.txt
jq -r '.status // .overall_status // empty' detmir-readiness-status.json
```
Критерий: checksum/signature проходят, статус не `FAIL`, дата bundle относится
к текущему maintenance window.
### 3. Свежесть telemetry и reports
Проверить штатные smoke/validation команды из репозитория:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
node scripts/operational-maturity-check.mjs --json
node scripts/pilot-validation-smoke.mjs
node scripts/deployment-readiness-smoke.mjs
```
Если включён автономный validation orchestrator, использовать его как единый
источник daily evidence:
```bash
./scripts/run_full_validation.sh --profile standard
```
Критерий: нет новых hard-fail проверок. Warning допустим только при известном
и задокументированном operational constraint.
### 3.1. Primary recovery guard
Проверить, что автоматический recovery первичного AW API включен и не скрывает
новые incident:
```bash
systemctl is-active detmir-aw-primary-recovery.timer
systemctl status detmir-aw-primary-recovery.timer --no-pager
sudo /usr/local/bin/detmir-aw-primary-recovery --check-only
sudo jq . /var/lib/detmir-aw-primary-recovery/latest.json 2>/dev/null || true
```
Критерий: timer активен, `--check-only` возвращает `status=ok`, новые incident
не имеют `outcome=failed`. Если latest incident отсутствует, это нормально для
контура, где recovery еще ни разу не требовался.
### 4. Очереди, backlog и storage growth
Проверить отсутствие неконтролируемого роста:
```bash
du -sh /var/lib/activitywatch /var/log/activitywatch 2>/dev/null || true
docker system df 2>/dev/null || true
docker exec aw-rus-1c-clickhouse clickhouse-client --query \
"SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" \
2>/dev/null || true
```
Для Windows queues не выполнять ручную очистку. Если backlog растёт, сначала
проверить доступность AW server, scheduled tasks и upload logs.
### 5. Retention safety
Проверить dry-run cleanup, если есть подозрение на рост диска или после
изменения retention-конфигурации:
```bash
sudo AW_DATA_DIR=/var/lib/activitywatch \
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache \
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400 \
/usr/local/bin/aw-prune-local-state-rust --json
```
Apply не является ежедневной обязательной операцией. Apply выполнять только
после просмотра planned items и подтверждения, что cleanup не затрагивает
configuration, dashboards, forensic/security evidence, release evidence или
Windows queues.
### 6. Короткая запись результата
Сохранить дневной summary в operator journal или internal change log:
```text
date:
operator:
health:
readiness_bundle:
telemetry_freshness:
storage:
retention:
new_alerts:
known_constraints:
actions_taken:
```
## Pollinations AI assistant
### Роль
Pollinations используется только как локально запускаемый помощник для:
- сжатия daily evidence в операторский summary;
- выделения новых failures/warnings из JSON/Markdown reports;
- подготовки списка вопросов для ручной проверки;
- сравнения текущего maintenance summary с предыдущим summary.
Pollinations не должен:
- запускать production commands самостоятельно;
- получать secrets/private env/raw logs;
- менять конфигурацию;
- принимать решение о cleanup apply, DLP enablement, restart или rollback.
### Проверка доступности CLI
```bash
/home/igor/.local/bin/polli-chat --help
/home/igor/.local/bin/polli-agent --help
/home/igor/.local/bin/polli-chat --max-tokens 16 'ping'
```
Для `polli-agent` использовать `--collection`, не `--db`:
```bash
/home/igor/.local/bin/polli-agent index \
--collection /tmp/awatch-maintenance-index \
/path/to/sanitized-evidence.md \
/path/to/sanitized-validation-report.json
/home/igor/.local/bin/polli-agent ask \
--collection /tmp/awatch-maintenance-index \
--context-only \
'Какие новые failures или warnings есть в сегодняшнем evidence?'
```
### Sanitization перед AI-анализом
Перед передачей в Pollinations оставить только технические поля:
- status/outcome;
- timestamps;
- Git SHA/version;
- service names;
- check names;
- counters;
- sizes;
- non-sensitive error class;
- documented operational constraints.
Удалить или заменить placeholders:
- usernames and user identifiers;
- tokens/passwords/API keys;
- Basic Auth credentials;
- private IP details, если summary планируется публиковать;
- screenshots and raw event payloads;
- full journal lines with command arguments.
### Prompt для daily summary
```text
Ты анализируешь sanitized AWatch-rus daily maintenance evidence.
Не придумывай факты. Не предлагай включать DLP/Loki/always-on Velociraptor.
Раздели вывод на:
1. OK
2. New warnings
3. New failures
4. Evidence gaps
5. Required operator actions
Если evidence недостаточно, напиши "не доказано".
```
### Fail-closed правило
Если Pollinations summary расходится с raw evidence, считать источником истины
raw evidence и runbook. AI-вывод можно использовать только как черновик для
операторского анализа.
## Журнал выполненного обслуживания
### 2026-07-03: production cleanup, collector guard и dependency hardening
Выполнено:
- очищены старые ActivityWatch SQLite backup files на AW server с сохранением
последних двух rollback-точек;
- `/` на AW server снижен с `85%` до `40%`, свободно `19G`;
- заменен Windows production binary
`C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe`;
- заменен gateway/ClickHouse production binary
`/usr/local/bin/aw-1c-ingest-rust`;
- закрыт RustSec риск `quick-xml` через удаление `calamine` из
`aw-1c-ingest` и переход на прямое чтение XLSX через `zip` +
`quick-xml 0.41.0`;
- добавлен regression test на разбор registry XLSX листов `Лист2` и
`ОСНОВНОЙ`.
Production checksums:
| Binary | SHA256 |
|---|---|
| `aw-windows-telemetry.exe` | `A8F517AB636C8537413201B84EA6540CB198045F388FECE6C8D6810537FBDDF1` |
| `aw-1c-ingest-rust` | `e0c4e665ea8946ba9c243127a65fd69554cacdc4e3191ec798b8ca19acf89623` |
Rollback files:
- `C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe.bak.20260702T202626Z`;
- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T211637Z`;
- `/usr/local/bin/aw-1c-ingest-rust.bak.20260702T212258Z`.
Post-change production evidence:
- `check-aw-full.sh`: `FRESH=8`, `STALE=0`, `DEAD=0`;
- `aw-rus-healthd`: `ok=True`;
- AW server `systemctl --failed`: `0 loaded units listed`;
- `pve-detmir systemctl --failed`: `0 loaded units listed`;
- `aw-1c-ingest.service`: `Result=success`, `ExecMainStatus=0`,
`files_loaded=8`, `rows_loaded=561`;
- ClickHouse container `aw-rus-1c-clickhouse`: `healthy`;
- gateway `/healthz`: `ok`.
Validation executed before documenting:
```bash
cargo fmt --all --check
cargo test --workspace --locked
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
cargo audit --no-fetch
cargo deny --manifest-path adk-rust/Cargo.toml check --config deny.toml --hide-inclusion-graph --show-stats
cargo metadata --locked --format-version 1
cargo tree --duplicates --locked
cargo machete --with-metadata
python3 scripts/public_secret_pattern_check.py
node scripts/operational-maturity-check.mjs --json
git diff --check
```
Cargo/RustSec network note: if direct `https://index.crates.io` is unavailable
from the laptop, use the local HTTP CONNECT proxy:
```bash
export HTTP_PROXY=http://127.0.0.1:10808
export HTTPS_PROXY=http://127.0.0.1:10808
export http_proxy=http://127.0.0.1:10808
export https_proxy=http://127.0.0.1:10808
unset ALL_PROXY all_proxy
```
## Эскалация
Эскалировать как production issue, если:
- `/healthz`, `/readyz` или Worktime reports не отвечают;
- readiness bundle не сформирован или не проходит checksum/signature;
- есть неожиданные failed systemd units;
- storage growth продолжается после штатного retention;
- Windows queues растут и upload не восстанавливается;
- DLP/security evidence удалено, повреждено или изменилось без change;
- Pollinations обнаружил warning, который подтверждается raw evidence.
## Связанные документы
- [Operations Runbook](OPERATIONS_RUNBOOK_RU.md)
- [Эксплуатационная проверка контура](OPERATIONS_VALIDATION_RUNBOOK_RU.md)
- [DetMir service reliability runbook](DETMIR_SERVICE_RELIABILITY_RUNBOOK_RU.md)
- [Retention and Cleanup Policy](RETENTION_POLICY_RU.md)
- [Production readiness](PRODUCTION_READINESS_RU.md)
- [Autonomous validation](AUTONOMOUS_VALIDATION_RU.md)
+16
View File
@@ -7,6 +7,22 @@ production IP Windows/RDP host теперь `192.168.100.19`; stable ActivityWat
logical host id остаётся `SHARKON2025`. Подробный post-restore baseline:
`docs/DETMIR_RESTORE_BASELINE_2026-06-29_RU.md`.
Обновление 2026-07-02 после ремонта portal telemetry ingest: Windows
`awatch-agent-rs` был запущен и отправлял `POST /api/telemetry`, но
`detmir-portal.service` на gateway работал без production
`DETMIR_PORTAL_TELEMETRY_API_KEY` и принимал default `change-me`. Поэтому
запросы telemetry получали `401`, `/var/lib/detmir-portal/telemetry.jsonl`
оставался stale с `2026-06-25`, а portal ошибочно показывал
`SHARKON2025` как `MISSING node`, `agent_coverage_pct=0` и высокий риск.
Исправление: API key синхронизирован из production agent config
`C:\ProgramData\AWatch-rus\agent\awatch-agent.toml` в
`/etc/detmir-portal.env`, предварительно создан backup
`/etc/detmir-portal.env.bak.telemetry-key.<timestamp>`, затем перезапущен
только `detmir-portal.service`. После проверки: telemetry свежая
`2026-07-02T07:20:09Z SHARKON2025`, `/portal/api/summary` вернул
`severity=OK`, `operator_ok=true`, executive report показал
`trust_kpi_score=100`, `agent_coverage_pct=100`, `critical_candidates=0`.
Документ фиксирует фактическое состояние DetMir/AWatch-rus и первую границу
переработки горячего пути портала. Это не release evidence для реестра
российского ПО, не заявление о сертификации и не claim замены DLP/SIEM/EDR.
@@ -0,0 +1,233 @@
# DetMir Service Reliability Runbook
Дата актуализации: 2026-07-03
Документ описывает production-safe контур снижения повторных выпадений сервисов
AWatch-rus/DetMir. Главная цель — отделять первичный отказ ActivityWatch API от
вторичных падений ingest/report/health jobs и выполнять ограниченное
автовосстановление только для подтвержденного безопасного сценария.
## План реализации
1. Считать `activitywatch-server` первичным сервисом для AW API.
2. Считать `aw-workforce-ingest`, `aw-worktime-autoheal`, `aw-worktime-prewarm`,
`aw-worktime-ui-bridge`, `aw-worktime-influx-exporter`, `aw-rus-healthd` и
`aw-slo-monitor` вторичными зависимыми jobs.
3. Проверять AW API через `/api/0/settings/` и `/api/0/buckets/`.
4. Автоматически восстанавливать только подтвержденный класс отказа:
`poisoned datastore lock` или неактивный `activitywatch-server`.
5. Перед restart первичного сервиса временно остановить вторичные jobs, чтобы
они не усиливали нагрузку на AW SQLite.
6. Перезапустить только `activitywatch-server` внутри CT `203`.
7. Дождаться восстановления AW API.
8. Вернуть вторичные timers и записать incident JSON.
## Что реализовано
Реализован script:
```text
scripts/detmir-aw-primary-recovery.sh
```
Systemd units для Proxmox host:
```text
ops/systemd/detmir-aw-primary-recovery.service
ops/systemd/detmir-aw-primary-recovery.timer
```
Контур предназначен для запуска на Proxmox host `10.10.10.2`, потому что там
доступны одновременно:
- управление PVE jobs через локальный `systemctl`;
- управление AW CT через `pct exec 203`;
- проверка внешнего AW API `http://10.10.10.13:5600`.
## Safety Rules
Recovery script:
- не удаляет SQLite database, lock files или journal files;
- не включает DLP, Loki или always-on Velociraptor;
- не перезапускает Windows host и Windows collectors;
- не меняет конфигурацию AW server;
- не выполняет recovery при произвольном HTTP timeout без подтвержденной
причины;
- имеет cooldown между restart попытками;
- пишет incident evidence в JSON.
## Установка
На Proxmox host:
```bash
sudo install -m 0755 scripts/detmir-aw-primary-recovery.sh \
/usr/local/bin/detmir-aw-primary-recovery
sudo install -m 0644 ops/systemd/detmir-aw-primary-recovery.service \
/etc/systemd/system/detmir-aw-primary-recovery.service
sudo install -m 0644 ops/systemd/detmir-aw-primary-recovery.timer \
/etc/systemd/system/detmir-aw-primary-recovery.timer
sudo systemctl daemon-reload
sudo systemctl enable --now detmir-aw-primary-recovery.timer
```
Опциональная конфигурация:
```bash
sudo install -d -m 0755 /etc/detmir
sudoedit /etc/detmir/aw-primary-recovery.env
```
Пример:
```bash
DETMIR_AW_RECOVERY_URL=http://10.10.10.13:5600
DETMIR_AW_RECOVERY_CT_ID=203
DETMIR_AW_RECOVERY_CONFIRM_ATTEMPTS=2
DETMIR_AW_RECOVERY_COOLDOWN_SECONDS=900
DETMIR_AW_RECOVERY_HTTP_TIMEOUT_SECONDS=8
```
## Проверка без изменений
```bash
/usr/local/bin/detmir-aw-primary-recovery --self-test
/usr/local/bin/detmir-aw-primary-recovery --check-only
systemctl start detmir-aw-primary-recovery.service
journalctl -u detmir-aw-primary-recovery.service -n 80 --no-pager
```
Ожидаемо для здорового контура:
```text
event=probe attempt=1 status=ok
```
## Evidence
Incident files:
```text
/var/lib/detmir-aw-primary-recovery/incidents/*.json
/var/lib/detmir-aw-primary-recovery/latest.json
/var/lib/detmir-aw-primary-recovery/status.json
```
Проверка последнего incident:
```bash
sudo jq . /var/lib/detmir-aw-primary-recovery/latest.json
```
Ключевые поля:
- `outcome`: `recovered`, `failed`, `skipped`, `observed_no_action`, `dry_run`;
- `reason`: `poisoned_lock`, `service_inactive`, `cooldown_active_after_*`;
- `recovered`: boolean;
- `pve_pause_units`;
- `ct_pause_units`;
- `details`.
## Rollback
Отключить automation:
```bash
sudo systemctl disable --now detmir-aw-primary-recovery.timer
sudo systemctl reset-failed detmir-aw-primary-recovery.service
```
Удаление установленных файлов не требуется для rollback. Если нужно убрать
полностью:
```bash
sudo rm -f /etc/systemd/system/detmir-aw-primary-recovery.service
sudo rm -f /etc/systemd/system/detmir-aw-primary-recovery.timer
sudo rm -f /usr/local/bin/detmir-aw-primary-recovery
sudo systemctl daemon-reload
```
## Manual Recovery Sequence
Если automation disabled или recovery не помог:
```bash
sudo systemctl stop aw-workforce-ingest.timer aw-workforce-ingest.service
sudo pct exec 203 -- systemctl stop \
aw-worktime-autoheal.timer aw-worktime-autoheal.service \
aw-worktime-prewarm.timer aw-worktime-prewarm.service \
aw-worktime-ui-bridge.timer aw-worktime-ui-bridge.service \
aw-worktime-influx-exporter.timer aw-worktime-influx-exporter.service \
aw-rus-healthd.timer aw-rus-healthd.service \
aw-slo-monitor.timer aw-slo-monitor.service
sudo pct exec 203 -- systemctl restart activitywatch-server
curl -fsS http://10.10.10.13:5600/api/0/settings/ >/dev/null
curl -fsS http://10.10.10.13:5600/api/0/buckets/ >/dev/null
sudo pct exec 203 -- systemctl start \
aw-worktime-autoheal.timer \
aw-worktime-prewarm.timer \
aw-worktime-ui-bridge.timer \
aw-worktime-influx-exporter.timer \
aw-rus-healthd.timer \
aw-slo-monitor.timer
sudo systemctl start aw-workforce-ingest.timer
```
## Post-Recovery Validation
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
./check-aw-full.sh
ssh igor@10.10.10.2 'systemctl --failed --no-legend || true'
ssh igor@10.10.10.2 'sudo pct exec 203 -- systemctl --failed --no-legend || true'
curl -fsS http://10.10.10.13:5610/health | jq .
curl -fsS http://10.10.10.2:8720/readyz | jq .
```
Критерий:
- `check-aw-full.sh`: `DEAD=0`, `STALE=0`;
- failed units на Proxmox host: `0`;
- failed units внутри CT `203`: `0`;
- Worktime health: `status=OK`;
- Portal readiness: `status=ready`.
## Gateway Contract
Операторские shortcuts должны соответствовать runbook:
```text
/go/proxmox-gui
/go/file1c-brief
/go/file1c-actions
```
Они формируются из `proxmox_web_gateway_routes` в nginx template и должны
возвращать `302` на целевой внутренний URL после Basic Auth.
Readiness compatibility endpoint:
```text
/portal/api/readiness -> http://127.0.0.1:8720/readyz
```
## Known Limitations
- Recovery intentionally limited to AW API poisoned lock and inactive primary
service. Other failures remain operator-visible incidents.
- If restart does not restore AW API before timeout, secondary timers remain
paused to avoid failure amplification. Operator must inspect
`/var/lib/detmir-aw-primary-recovery/latest.json`.
- Windows credential or WinRM authentication problems are outside this recovery
loop.
- This does not prove backup/restore. It is runtime availability recovery only.
+122
View File
@@ -0,0 +1,122 @@
# AWatch-rus: enterprise quality standard
Дата актуализации: 2026-07-01.
AWatch-rus уже работает в production-контуре заказчика. Основная цель
разработки - не добавление функций, а повышение надежности, эксплуатационной
зрелости, безопасности, сопровождаемости и воспроизводимости.
Приоритеты изменений, по убыванию:
1. Reliability.
2. Operational maturity.
3. Security.
4. Maintainability.
5. Reproducibility.
6. Performance.
7. Simplicity.
Новые функции не должны иметь приоритет над стабильностью production.
## Production-first правила
- Предполагать, что production deployment существует и пользователи зависят от
непрерывной работы.
- Предпочитать additive/backward-compatible изменения.
- Не перепроектировать работающие подсистемы без измеримой пользы.
- Не включать heavy DLP, Loki или always-on Velociraptor без отдельного
operator-approved решения.
- Не менять ActivityWatch logical host ids, bucket suffixes, Grafana variables
или ClickHouse workforce keys без отдельной compatibility-процедуры.
- Fail closed для security-sensitive и deployment-sensitive paths.
- Сохранять rollback path для runtime, automation, config и dependency changes.
## Порядок принятия решений
Перед реализацией любого изменения ответ должен быть положительным хотя бы на
один вопрос:
1. Улучшает ли это production stability?
2. Снижает ли это operational risk?
3. Улучшает ли это diagnostics или observability?
4. Улучшает ли это maintainability?
5. Уменьшает ли это технический долг с низким regression risk?
Если ответ отрицательный на все пять вопросов, изменение не должно попадать в
production-oriented PR.
## Dependencies
Перед добавлением зависимости нужно явно обосновать:
- почему стандартной библиотеки недостаточно;
- почему существующий workspace crate не решает задачу;
- operational cost зависимости;
- maintenance cost зависимости;
- license/security impact.
Неиспользуемые зависимости удаляются отдельными низкорисковыми PR после
targeted tests. Обновление `Cargo.lock` без причины не допускается.
## Required PR sections
Каждый PR обязан содержать:
- Purpose.
- Operational impact.
- Risk assessment.
- Rollback strategy.
- Validation steps.
- Documentation changes.
- Acceptance criteria.
Для documentation-only/governance-only PR нужно явно указать, что runtime, API и
UI impact отсутствуют.
## Validation baseline
Канонический набор проверок описан в
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`. Минимальный baseline для
существенных Rust/runtime changes:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust
export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
cargo fmt --all --check
cargo test --workspace --all-targets --locked
cargo test --workspace --doc --locked
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo audit --deny warnings
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
cargo deny --manifest-path adk-rust/Cargo.toml check \
--config deny.toml \
--hide-inclusion-graph \
--show-stats
python3 scripts/public_secret_pattern_check.py
node scripts/operational-maturity-check.mjs --json
```
Targeted checks are acceptable for small docs/governance or isolated changes,
but skipped checks must have a concrete reason.
## Technical debt policy
Fix technical debt only when all conditions are true:
- change is isolated;
- regression risk is low;
- tests or operational smoke cover the touched behavior;
- documentation remains accurate.
Otherwise create a follow-up task or document the residual risk instead of
mixing broad cleanup into a functional PR.
## Canonical references
- Review checklist: `docs/REVIEW_CHECKLIST_RU.md`.
- Validation runbook: `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
- Operational maturity harness: `docs/OPERATIONAL_MATURITY_RU.md`.
- PR workflow: `docs/PR_REVIEW_WORKFLOW_RU.md`.
- GitHub governance entrypoint: `.github/GOVERNANCE.md`.
+58
View File
@@ -0,0 +1,58 @@
# Эксплуатационная зрелость DetMir/AWatch-rus
Дата актуализации: 2026-07-01.
Этот контур добавляет автоматическую проверку эксплуатационной зрелости без
нагрузки на production. Public CI запускает только offline checks: fixtures,
локальный mock HTTP, статическую проверку конфигураций, ClickHouse DDL и
контракт наблюдаемости. Live checks запускаются только вручную оператором с
private access.
## Что проверяется
- Integration harness: локальный mock обслуживает ключевые endpoints
`/healthz`, `/readyz`, `/version`, Worktime management и Security Finding
Inbox shadow payload.
- Fault injection: клиент должен быстро классифицировать `503`, timeout и
connection reset, не зависая сверх заданного бюджета.
- API compatibility: DetMir Portal OpenAPI обязан сохранять ключевые paths,
schemas и runtime endpoints.
- Config/migration validation: JSON/YAML examples, systemd units/timers и
ClickHouse init SQL проверяются на базовую пригодность и idempotency.
- Bounded load: короткий локальный load smoke с concurrency и p95 budget,
без sizing claims.
- Observability: обязательные health/version/readiness fields, diagnostic
headers и Prometheus metric names закреплены manifest/fixtures/source check.
## Команды
Offline PR/CI gate:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
node --check scripts/operational-maturity-check.mjs
node scripts/operational-maturity-check.mjs --json
```
Live contract, только оператором и только если контур доступен:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
AWATCH_OPS_LIVE_URL=http://127.0.0.1:8720 \
node scripts/operational-maturity-check.mjs --json --live
```
Live check не включает DLP, Loki или Velociraptor. Он только читает
health/readiness/version/metrics endpoints и принимает controlled statuses
`200` или `503`.
## Governance
Контракт расположен в `configs/operational-maturity-contract.json`.
Fixture payloads лежат в `docs/fixtures/operational-maturity/`.
CI workflow: `.github/workflows/operational-maturity.yml`.
Правило изменений: если endpoint, metric, schema, systemd unit или ClickHouse
migration меняется, сначала обновляется manifest/fixture, затем код. Удаление
полей или paths считается breaking change, если нет отдельного operator-approved
major contract change.
+22
View File
@@ -2,6 +2,14 @@
Документ описывает базовые эксплуатационные проверки AWatch-rus.
Ежедневный операторский цикл обслуживания, retention safety и безопасное
использование Pollinations AI для анализа sanitized evidence описаны отдельно:
[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md).
Автоматическое восстановление первичного AW API от подтвержденного
`poisoned datastore lock` описано отдельно:
[DetMir service reliability runbook](DETMIR_SERVICE_RELIABILITY_RUNBOOK_RU.md).
## Быстрая проверка
Проверить доступность:
@@ -104,6 +112,20 @@ systemctl --failed --no-pager
- coverage;
- errors in logs.
### ActivityWatch API возвращает `503 poisoned lock`
Проверить, сработал ли primary recovery guard:
```bash
systemctl status detmir-aw-primary-recovery.timer --no-pager
journalctl -u detmir-aw-primary-recovery.service -n 80 --no-pager
sudo jq . /var/lib/detmir-aw-primary-recovery/latest.json
```
Если guard отключен, использовать manual sequence из
`DETMIR_SERVICE_RELIABILITY_RUNBOOK_RU.md`. Не удалять SQLite, lock или journal
файлы ActivityWatch вручную.
### Нет данных
Проверить:
+214
View File
@@ -51,12 +51,94 @@ Repository-specific gate:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
python3 scripts/public_secret_pattern_check.py
node scripts/operational-maturity-check.mjs --json
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust
export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian
```
Подробности эксплуатационного maturity harness: [эксплуатационная зрелость
DetMir/AWatch-rus](OPERATIONAL_MATURITY_RU.md).
Единый автономный entry point для unattended validation, runtime diagnostics,
history, regression detection, alerts и reports описан отдельно:
[автономная проверка AWatch-rus](AUTONOMOUS_VALIDATION_RU.md).
Ежедневный maintenance-порядок с optional Pollinations AI assistant для
sanitized evidence summaries описан в отдельном runbook:
[ежедневное обслуживание AWatch-rus](DAILY_MAINTENANCE_RU.md).
## Production binary parity gate
Production binary parity gate доказывает, что production unit/timer/task
запускает тот же бинарник, который собран из проверяемого Git commit. Gate не
собирает данные с production самостоятельно и не включает DLP/Loki/Velociraptor:
оператор подготавливает evidence JSON, затем репозиторий проверяет SHA256.
Формат evidence: [пример production-binary-parity.example.json](fixtures/production-binary-parity.example.json).
Минимальные поля для каждого активного бинарника:
- `unit_or_task` - systemd unit/timer или Windows scheduled task.
- `binary_path` - фактический путь production executable.
- `crate` - Rust crate, из которого собран бинарник.
- `release_artifact` - имя файла в release directory.
- `runtime_role` - роль в production runtime.
- `production_sha256` - SHA256 production executable.
- `git_sha` - Git commit, из которого собран release.
Опциональные отключенные контуры, включая DLP runtime, указывать как
`"active": false` с `skip_reason`. Это фиксирует намеренное отключение без
ложного отказа gate.
Пример сбора SHA256 на Linux-хосте:
```bash
sha256sum /usr/local/bin/detmir-readiness-rust
systemctl show -p FragmentPath -p ExecStart detmir-readiness.service
```
Пример сбора SHA256 на Windows RDP host:
```powershell
Get-FileHash 'C:\Program Files\AWatch-rus\windows\aw-windows-telemetry.exe' -Algorithm SHA256
Get-ScheduledTask | Where-Object {$_.TaskName -like '*AWatch*'}
```
Проверка evidence против локальных release artifacts:
```bash
cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
python3 scripts/check_production_binary_parity.py \
--evidence /path/to/production-binary-parity.json \
--output-json /path/to/production-binary-parity-report.json
```
Та же проверка может быть включена в существующий gate локальных release
artifacts:
```bash
PRODUCTION_BINARY_PARITY_EVIDENCE=/path/to/production-binary-parity.json \
scripts/check_detmir_rust_release_artifacts.sh
```
Ожидаемый результат:
```text
production_binary_parity=ok
```
Gate должен падать при:
- отсутствии активных production binaries;
- отсутствии обязательных полей;
- несовпадении `git_sha` с текущим repository HEAD;
- отсутствии локального release artifact;
- несовпадении production SHA256 и release SHA256.
## Browser smoke
Browser smoke не заменяет API/CLI проверки. Он подтверждает, что операторский
@@ -127,3 +209,135 @@ SELECT
```
Ожидаемое состояние после cleanup/normalization: все четыре значения равны `0`.
## Retention / cleanup validation
Политика хранения описана в [Retention and Cleanup Policy](RETENTION_POLICY_RU.md).
Перед изменением сроков хранения или включением нового cleanup scope сначала
выполнить dry-run и сохранить вывод в change evidence.
Проверить активные timers:
```bash
systemctl list-timers \
aw-prune-local-state.timer \
aw-db-maintenance.timer \
aw-db-vacuum.timer \
detmir-readiness.timer
```
Dry-run локальной очистки:
```bash
sudo AW_DATA_DIR=/var/lib/activitywatch \
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache \
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400 \
/usr/local/bin/aw-prune-local-state-rust --json
```
Проверить, что в planned items нет:
- production database outside `/var/lib/activitywatch/backups/db`;
- config/env files;
- dashboards;
- Hayabusa forensic archive;
- DLP evidence/cases/compliance reports;
- release evidence;
- Windows queues.
Apply разрешен только после dry-run review:
```bash
sudo /usr/local/bin/aw-prune-local-state-rust --apply --json
```
Проверить журналы cleanup и DB maintenance:
```bash
journalctl \
-u aw-prune-local-state.service \
-u aw-db-maintenance.service \
-u aw-db-vacuum.service \
-u detmir-readiness.service \
-n 160 --no-pager
```
Оценить disk usage до и после:
```bash
du -sh \
/var/lib/activitywatch \
/var/log/activitywatch \
/opt/hayabusa \
/opt/activitywatch/clickhouse-1c \
/opt/activitywatch/clickhouse-workforce 2>/dev/null || true
docker system df 2>/dev/null || true
```
Для ClickHouse 1C/workforce проверить размер таблиц без удаления данных:
```bash
docker exec aw-rus-1c-clickhouse clickhouse-client --query \
"SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" \
2>/dev/null || true
docker exec aw-rus-workforce-clickhouse clickhouse-client --query \
"SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" \
2>/dev/null || true
```
Проверить, что cleanup не повлиял на running services:
```bash
systemctl status activitywatch-server aw-worktime-api --no-pager
curl -fsS http://127.0.0.1:5600/api/0/info >/dev/null
curl -fsS http://127.0.0.1:5610/healthz >/dev/null
```
Windows EVTX retention проверяется отдельно, потому что выполняется на RDP host:
```powershell
powershell.exe -ExecutionPolicy Bypass `
-File C:\ProgramData\AWatch-rus\export-evtx-for-hayabusa.ps1 `
-RetentionDays 14
```
Запрещено вручную удалять Windows collector queues, incident artifacts, DLP
evidence, Hayabusa archives, Grafana data или ClickHouse tables без отдельного
operator approval и backup/restore plan.
## Проверка отсутствия ClickHouse-пароля в argv
Цель проверки - убедиться, что ClickHouse/1C runtime wrappers не передают
`CLICKHOUSE_PASSWORD` через аргументы процессов. Пароль должен поступать из
`/opt/activitywatch/clickhouse-1c/.env` в окружение или временный client config,
а не через `--password`.
Статическая проверка wrappers:
```bash
rg -n -- '--password[= ]+"?\$[{]?CLICKHOUSE_PASSWORD' \
/opt/activitywatch/clickhouse-1c/ops
```
Ожидаемый результат: команда не выводит совпадений.
Runtime smoke во время ingest/brief refresh:
```bash
set -a
. /opt/activitywatch/clickhouse-1c/.env
set +a
ps -eo args= | grep -E 'clickhouse-client|generate_.*brief|refresh_company' |
while IFS= read -r line; do
if printf '%s' "${line}" | grep -F -- "${CLICKHOUSE_PASSWORD}" >/dev/null; then
echo "FAIL: ClickHouse password is visible in process argv" >&2
exit 1
fi
done
```
Ожидаемый результат: команда завершается с кодом `0` и не печатает секрет.
Если проверка падает, остановить rollout и вернуть предыдущий release artifact.
+235
View File
@@ -0,0 +1,235 @@
# Retention and Cleanup Policy
Дата актуализации: 2026-07-01
Документ фиксирует production-политику хранения и очистки для AWatch-rus /
DetMir. Цель политики - остановить неконтролируемый рост диска без неожиданного
удаления operational, audit, forensic или rollback-данных.
## Принципы
- Cleanup должен быть конфигурируемым через env, Ansible vars, compose vars или
customer config без recompilation.
- Dry-run обязателен перед изменением сроков хранения или включением нового
cleanup scope.
- Очистка не удаляет configuration, dashboards, rollback backups, release
manifests, private env files и secrets.
- Forensic/security evidence не удаляется автоматически, если в репозитории нет
явной безопасной политики и подтвержденного customer retention решения.
- DLP runtime в DetMir production остается light/disabled по текущим guardrails;
retention policy не включает heavy DLP, Loki или always-on Velociraptor.
## Inventory persistent storage
Фактические источники найдены по Rust defaults, systemd/Ansible, compose,
PowerShell deployment config и существующим runbooks.
| Component | Data type | Path / storage | Existing cleanup |
| --- | --- | --- | --- |
| ActivityWatch server | primary event DB | `aw_server_db_path`, `/var/lib/activitywatch/aw-server-rust/sqlite.db` in Ansible | `aw-db-maintenance` removes only old process-level session events; no broad bucket deletion |
| ActivityWatch DB backups | SQLite backup files | `/var/lib/activitywatch/backups/db` | `aw-prune-local-state` via Ansible timer |
| ActivityWatch local backup metadata | JSON/root backup files | `/var/lib/activitywatch/backups` | `aw-prune-local-state` via Ansible timer |
| Worktime report cache | generated report cache | `/var/lib/activitywatch/worktime-report-cache` | TTL on read in `worktime-api`; proactive prune in `aw-prune-local-state` |
| Worktime management history | daily aggregate trend points | `/var/lib/activitywatch/worktime-management-history` | `worktime-api` prunes by date |
| Browser smoke artifacts | screenshots/result dirs/cache | `/var/lib/activitywatch/browser-smoke` | `aw-prune-local-state` |
| Temporary release/webui artifacts | temporary archives/scripts | `/tmp/activitywatch-*.zip`, `/tmp/hayabusa-*.zip`, webui temp files | `aw-prune-local-state` |
| Service logs | file logs | `/var/log/activitywatch/*.log` | `aw-server/logrotate.conf` |
| Journald | service journal | journald persistent/runtime storage | Ansible journald size caps and `journalctl --vacuum-size` |
| DetMir readiness bundle | signed readiness archives | `/var/lib/activitywatch/health/readiness-bundle` | `detmir-readiness` archive retention |
| DetMir auto/check state | check/report runs | `/var/lib/detmir-ai` | `detmir-auto` retains generated runs/files by `DETMIR_AI_RETAIN_DAYS` |
| Windows collector state | deployment config, queues, markers, logs | `C:\ProgramData\AWatch-rus` | no generic cleanup; queues are health-checked, not pruned |
| Windows EVTX exports | zipped forensic EVTX batches | `C:\ProgramData\AWatch-rus\forensics\evtx-exports` | `export-evtx-for-hayabusa.ps1` prunes by `retentionDays` |
| Windows incident artifacts | screenshots/evidence files | `C:\ProgramData\AWatch-rus\incident-artifacts`, per-user LocalAppData fallback | no automatic deletion in repo |
| DLP policy DB | policy/audit SQLite | `/var/lib/activitywatch/dlp-policy-engine.sqlite` | no automatic deletion |
| DLP case DB | case/audit SQLite | `/opt/activitywatch/dlp-case-management/cases.db` | no automatic deletion |
| DLP warehouse/evidence | evidence metadata and screenshots | `/var/lib/activitywatch/dlp_warehouse.sqlite`, `/var/lib/detmir-portal/evidence` | no automatic deletion |
| DLP aggregator DB | endpoint/fileops event warehouse | `data/dlp-events.sqlite3` unless overridden | no automatic deletion |
| DLP compliance reports | generated compliance artifacts | `/opt/activitywatch/dlp-compliance/reports` | no automatic deletion |
| Hayabusa server bundle | reports, archived input packages/payloads | `/opt/hayabusa/reports`, `/opt/hayabusa/archive` | archive only; no deletion |
| ClickHouse 1C | raw/core/security/business tables | Docker volume `clickhouse_1c_data` | policy doc exists; init SQL has no TTL |
| ClickHouse 1C landing/archive | exported raw files | `/opt/activitywatch/clickhouse-1c/landing`, `/archive` | ETL moves to archive; no automatic age prune |
| ClickHouse Workforce | raw/aggregate workforce tables | Docker volume `clickhouse_workforce_data` | no TTL in init SQL |
| Workforce ingest state | incremental loader state | `/var/lib/aw-workforce-ingest/state.json` | no cleanup needed; single state file |
| Prometheus | metrics TSDB | Docker volume `prometheus-data` | compose retention time, default `30d` |
| Grafana | SQLite/plugins/session state | Docker volume `grafana-data` | no repo cleanup; dashboards are provisioned read-only |
| Diagnostic bundles | operator support output | `/var/log/detmir-full-diagnostics` by example | no automatic deletion |
| Release evidence / rollout logs | build/release evidence and rollout logs | configured output dirs, `.rollout-logs` | no automatic deletion |
| Gitea registry backup | registry support backup | `/var/backups/gitea` | registry manifest expects `14` days |
## Retention matrix
Значения ниже взяты из текущего репозитория. Если срок в репозитории не задан,
cleanup остается disabled; recommended value фиксируется как operator decision,
а не как придуманное число.
| Component | Data type | Default retention | Minimum | Maximum | Cleanup method | Recovery impact | Disk usage impact | Recommended value |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| ActivityWatch primary DB | primary buckets/events | not globally configured | not configured | customer policy | no broad cleanup; only targeted `aw-db-maintenance` | broad deletion can break reports/audit | highest growth source | do not delete globally before customer data policy |
| AW process session events | process start/stop noise in session bucket | `7` days | `1` day in Rust validation | not enforced | `aw-db-maintenance --apply --json`, weekly timer | low; targeted process-level data only | medium on busy RDP hosts | `7` days until measured otherwise |
| AW SQLite vacuum | DB free pages | nightly | n/a | n/a | `aw-db-maintenance --vacuum --apply --json`, stops service via guard | short maintenance window | reduces reclaimed DB space | keep nightly low-load window |
| AW DB backups | SQLite backup files | `7` days, keep last `2` | keep last protects latest even with `0` days | not enforced | `aw-prune-local-state` | old rollback points removed; latest protected | medium | `7` days and keep last `2` |
| AW root backup JSON | backup metadata/root files | `7` days, keep last `2` | keep last protects latest even with `0` days | not enforced | `aw-prune-local-state` | old metadata removed | low | `7` days and keep last `2` |
| Worktime report disk cache | generated report JSON cache | `86400` seconds | `0` disables disk cache prune | not enforced | `aw-prune-local-state`; files under configured cache dir only | none; cache regenerates | medium under report load | `86400` seconds |
| Worktime in-memory events cache | process memory cache | `300` seconds in Ansible | `0` disables | not enforced | in-process TTL | none | RAM bounded by TTL/query limits | `300` seconds |
| Worktime in-memory report cache | process memory cache | `300` seconds in Ansible | `0` disables | not enforced | in-process TTL | none | RAM bounded by TTL/query limits | `300` seconds |
| Worktime stale report cache | stale fallback response | `3600` seconds in Ansible | at least report cache TTL in code | not enforced | in-process TTL | improves degraded-mode availability | low/medium | `3600` seconds |
| Worktime management history | aggregate daily trend points | `120` days | `1` day | `3660` days in code clamp | `worktime-api` date prune | old trend depth removed; raw events untouched | low | `120` days |
| Browser smoke artifacts | smoke runs/screenshots/cache | `1` day, keep `24` runs | `0` days plus keep last | not enforced | `aw-prune-local-state` | none; diagnostic artifacts only | low/medium | `1` day, keep `24` runs |
| Temporary archives | release/Hayabusa tmp archives | `1` day | `0` days | not enforced | `aw-prune-local-state` allowlisted names in `/tmp` | none if release already installed | medium | `1` day |
| Temporary webui files | webui patch temp files | `2` days | `0` days | not enforced | `aw-prune-local-state` allowlisted names in `/tmp` | none | low | `2` days |
| ActivityWatch file logs | `/var/log/activitywatch/*.log` | `30` daily rotations | logrotate policy | logrotate policy | `aw-server/logrotate.conf` | old logs compressed/removed | medium | keep `30` rotations |
| Journald | service journal | size caps `100M` system, `50M` runtime, keep free `500M` | journald policy | journald policy | Ansible drop-in and `journalctl --vacuum-size` | old journal lines removed | high on noisy hosts | keep current caps for small production |
| DetMir readiness archives | signed readiness evidence | `30` days | not enforced by doc; Rust accepts integer days | not enforced | `detmir-readiness`, daily timer | old readiness archives removed; latest files remain | low/medium | `30` days |
| DetMir auto/check state | generated run/report files | `14` days | `0` accepted by code | not enforced | `detmir-auto` cleanup | old diagnostic reports removed | low | `14` days if service is used |
| Windows EVTX exports | forensic EVTX zip/batch dirs | `14` days | `1` day via script max guard | not enforced | `export-evtx-for-hayabusa.ps1` | older EVTX export batches removed | high during incidents | `14` days unless legal hold |
| Windows queues | collector queue JSONL/locks | no retention | n/a | n/a | health/backlog detection only | deletion could lose unsent events | medium when AW unreachable | do not prune automatically |
| Windows incident artifacts | screenshots/evidence | no retention | n/a | n/a | disabled cleanup | deleting can break evidence chain | potentially high | operator/legal decision before cleanup |
| DLP policy DB | SQLite policy/audit | no retention | n/a | n/a | disabled cleanup | audit history loss if deleted | low/medium | retain until DLP policy decision |
| DLP case DB | SQLite cases/comments/audit | no retention | n/a | n/a | disabled cleanup | case/audit loss if deleted | low/medium | retain until case retention policy |
| DLP warehouse/evidence root | metadata/screenshots | no retention | n/a | n/a | disabled cleanup | evidence loss if deleted | high | retain until customer evidence policy |
| DLP compliance reports | generated compliance files | no retention | n/a | n/a | disabled cleanup | compliance evidence loss | medium | retain until compliance policy |
| Hayabusa reports/archive | timelines, packages, sidecars | no retention | n/a | n/a | archive only | forensic evidence loss if deleted | high after incident uploads | retain until operator/legal decision |
| ClickHouse 1C landing files | raw exported files before load | policy doc says `30` days | not enforced | not enforced | ETL archive/delete flags; no age prune | source replay loss if deleted | medium/high | `30` days only after implementing safe prune |
| ClickHouse 1C archived raw files | archived loaded files | policy doc says `90` days | not enforced | not enforced | no age prune in repo | replay/debug loss if deleted | high | `90` days only after implementing safe prune |
| ClickHouse 1C raw tables | `raw_*` tables | policy doc says `30` days | not enforced | not enforced | no TTL in init SQL | raw replay loss if TTL applied | high | `30` days after explicit ClickHouse TTL migration |
| ClickHouse 1C core/security tables | normalized tables/cases/timeline | policy doc says `365` days | not enforced | not enforced | no TTL in init SQL | audit/history loss if TTL applied | high | `365` days after customer approval |
| ClickHouse Workforce tables | raw and aggregate workforce tables | no retention | n/a | n/a | no TTL in init SQL | workforce history loss if TTL applied | high | define after production growth measurement |
| Workforce ingest state | one JSON state file | latest only | n/a | n/a | overwrite/atomic state | reset causes overlap/backfill | negligible | keep latest |
| Prometheus TSDB | metrics samples | `30d` | Prometheus config value | Prometheus config value | `--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME:-30d}` | old metrics removed | high but bounded | `30d` |
| Grafana data volume | dashboards state, users, sqlite | no repo cleanup | n/a | n/a | disabled cleanup | can break dashboards/users | low/medium | backup, do not prune automatically |
| Diagnostic bundles | support logs/reports | no retention | n/a | n/a | disabled cleanup | old diagnostic evidence removed | medium | operator decision per support package |
| Release evidence / rollout logs | release proof and rollout logs | no retention | n/a | n/a | disabled cleanup | auditability loss | low/medium | retain through release/support window |
| Gitea backup | registry backup dump | `14` days | policy-defined | policy-defined | `awatch-gitea-backup.timer` per registry docs | old registry backups removed | medium | `14` days until restore tests define otherwise |
## Cleanup implementation
### Active automatic cleanup
- `aw-prune-local-state.timer` is installed/enabled by
`ansible/deploy_aw_server.yml` and runs daily at `04:40`.
- `aw-prune-local-state` prunes only allowlisted paths:
AW backups, browser smoke runs, worktime report disk cache and selected `/tmp`
artifacts.
- `aw-db-maintenance.timer` deletes only old process-level session events.
- `aw-db-vacuum.timer` performs SQLite vacuum in a low-load window.
- `detmir-readiness.timer` writes signed bundles and prunes old dated archives.
- `export-evtx-for-hayabusa.ps1` prunes old EVTX export folders/zips after a
new export run.
- Prometheus TSDB is bounded by compose retention time.
### Disabled cleanup by design
No automatic deletion is implemented for ClickHouse 1C/workforce data, Hayabusa
archives, DLP evidence, DLP cases, DLP compliance reports, Windows queues,
Grafana data, release evidence or support diagnostic bundles. These areas can
contain audit, forensic, replay or rollback value. Cleanup must be added only
after operator/customer decision and dry-run validation.
## Configuration examples
Server env example: `aw-server/aw-server.env.example`.
```bash
AW_BACKUP_RETENTION_DAYS=7
AW_BACKUP_KEEP_LAST_DB=2
AW_BACKUP_KEEP_LAST_JSON=2
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400
AW_BROWSER_SMOKE_RETENTION_DAYS=1
AW_BROWSER_SMOKE_KEEP_RUNS=24
AW_TMP_ARCHIVE_RETENTION_DAYS=1
AW_TMP_WEBUI_RETENTION_DAYS=2
AW_DB_MAINTENANCE_RETENTION_DAYS=7
DETMIR_READINESS_RETENTION_DAYS=30
```
Ansible vars:
```yaml
aw_server_backup_retention_days: 7
aw_server_backup_keep_last_db: 2
aw_server_backup_keep_last_json: 2
aw_worktime_report_disk_cache_dir: "{{ aw_server_data_dir }}/worktime-report-cache"
aw_worktime_report_disk_stale_ttl_seconds: 86400
aw_worktime_management_history_retention_days: 120
aw_windows_evtx_retention_days: 14
```
Prometheus:
```bash
PROMETHEUS_RETENTION_TIME=30d docker compose up -d prometheus
```
Windows EVTX:
```powershell
powershell.exe -ExecutionPolicy Bypass `
-File C:\ProgramData\AWatch-rus\export-evtx-for-hayabusa.ps1 `
-RetentionDays 14
```
## Production validation
Dry-run cleanup:
```bash
sudo AW_DATA_DIR=/var/lib/activitywatch \
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache \
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400 \
/usr/local/bin/aw-prune-local-state-rust --json
```
Apply only after dry-run review:
```bash
sudo /usr/local/bin/aw-prune-local-state-rust --apply --json
```
Timer and logs:
```bash
systemctl list-timers aw-prune-local-state.timer aw-db-maintenance.timer aw-db-vacuum.timer detmir-readiness.timer
journalctl -u aw-prune-local-state.service -u aw-db-maintenance.service -u aw-db-vacuum.service -n 120 --no-pager
```
Disk estimate before/after:
```bash
du -sh /var/lib/activitywatch /var/log/activitywatch /opt/hayabusa 2>/dev/null || true
docker system df 2>/dev/null || true
docker exec aw-rus-1c-clickhouse clickhouse-client --query \
"SELECT database, table, formatReadableSize(sum(bytes_on_disk)) AS size FROM system.parts WHERE active GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC" 2>/dev/null || true
```
Service safety after cleanup:
```bash
systemctl status activitywatch-server aw-worktime-api --no-pager
curl -fsS http://127.0.0.1:5600/api/0/info >/dev/null
curl -fsS http://127.0.0.1:5610/healthz >/dev/null
```
## Recovery notes
- AW DB backups are under `/var/lib/activitywatch/backups/db`; latest backup
files are protected by keep-last.
- Worktime disk cache does not need restore; reports regenerate from AW data.
- Browser smoke and `/tmp` artifacts do not need restore.
- Readiness latest files remain in the root of `readiness-bundle`; old dated
archive directories are non-critical after the retention window.
- Forensic evidence, Hayabusa archives, DLP cases and compliance reports are not
cleaned automatically and must be recovered from customer backup if operator
deletes them manually.
## Known limitations
- ClickHouse 1C retention periods exist in `clickhouse-1c/ops/retention-policy.md`,
but repository init SQL currently has no TTL clauses. Applying TTL to existing
production tables requires a separate staged migration and customer approval.
- ClickHouse Workforce has no retention policy in repo; do not infer one from 1C
retention.
- Windows incident screenshots and DLP evidence can grow during incidents.
Cleanup is intentionally disabled until legal/operator retention is defined.
- Diagnostic bundles and release evidence are intentionally not pruned because
they are often needed for support and audit.
+19 -1
View File
@@ -2,7 +2,7 @@
Дата: 2026-06-22
Статус: advisory checklist for public review process.
Статус: canonical checklist for production-first PR review.
Этот документ описывает проверочный чеклист для pull requests и внешнего
инженерного review. Он не утверждает, что внешний peer review уже выполняется
@@ -13,7 +13,15 @@ release evidence должен производиться на российско
## Общая безопасность изменений
- Изменение соответствует production-first стандарту:
`docs/ENTERPRISE_QUALITY_STANDARD_RU.md`.
- Цель PR относится к reliability, operational maturity, security,
maintainability, reproducibility, performance или simplicity. Если PR добавляет
функциональность, он также объясняет measurable operational benefit.
- Изменение имеет понятную цель, ограниченный scope and documented impact.
- Изменение additive/backward-compatible, если нет отдельного
operator-approved breaking-change решения.
- Не перепроектируются работающие подсистемы без измеримой пользы.
- Нет секретов, токенов, паролей, приватных ключей, recovery codes or live
credentials.
- Нет персональных данных сотрудников, реальных employee logs or customer
@@ -52,6 +60,8 @@ release evidence должен производиться на российско
- No runtime behavior is changed by documentation/governance-only PRs.
- No service restart, migration or production config change is implied unless
explicitly documented.
- Existing production deployment remains compatible, including logical host ids,
bucket suffixes, Grafana variables and ClickHouse workforce keys.
## Registry-readiness impact
@@ -80,6 +90,9 @@ release evidence должен производиться на российско
- README, `docs/PROJECT_STATUS_RU.md`, registry docs and operational runbooks
are updated when claims, checks, workflows or procedures change.
- Behavior changes update the relevant runbook and validation commands.
- Architecture docs are updated when an interface, data flow, deployment
boundary or security boundary changes.
- New claims are conservative and evidence-backed.
- Pending work remains marked as planned/pending until evidence exists.
- Public mirror wording remains separate from registry release evidence.
@@ -88,6 +101,8 @@ release evidence должен производиться на российско
- PR states whether deployment action is required.
- Rollback path is documented for runtime or automation changes.
- Dependency changes are isolated, justified and include rollback by reverting
the dependency/lockfile PR.
- Documentation-only PRs state that runtime/API/UI impact is unchanged.
- Changes to scripts include syntax checks and a clear operator failure mode.
@@ -103,6 +118,9 @@ release evidence должен производиться на российско
- For Rust/product changes, use
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md` as the default local validation
contour.
- Dependency hygiene and operational maturity checks are expected when touched
files can affect Rust dependencies, runtime contracts, configuration,
migrations, observability or CI governance.
- For operator-facing web, gateway, worktime reports or Grafana dashboards,
browser smoke through the rendered pages is required in addition to API checks.
@@ -0,0 +1,6 @@
{
"status": "ok",
"service": "detmir-portal",
"schema_version": "pilot-v1",
"timestamp": "2026-07-01T00:00:00Z"
}
@@ -0,0 +1,26 @@
# HELP awatch_http_requests_total HTTP requests handled by AWatch-rus portal
# TYPE awatch_http_requests_total counter
awatch_http_requests_total{method="GET",route="/healthz",status="200",module="portal"} 1
# HELP awatch_http_request_duration_seconds HTTP request duration in seconds
# TYPE awatch_http_request_duration_seconds summary
awatch_http_request_duration_seconds_sum{method="GET",route="/healthz",status="200",module="portal"} 0.001
awatch_http_request_duration_seconds_count{method="GET",route="/healthz",status="200",module="portal"} 1
# TYPE awatch_report_requests_total counter
awatch_report_requests_total 1
# TYPE awatch_report_cache_hits_total counter
awatch_report_cache_hits_total 0
# TYPE awatch_report_cache_misses_total counter
awatch_report_cache_misses_total 1
# TYPE awatch_report_cache_stale_hits_total counter
awatch_report_cache_stale_hits_total 0
# TYPE awatch_reports_generated_total counter
awatch_reports_generated_total 1
# TYPE awatch_ingestion_records_total counter
awatch_ingestion_records_total 0
# TYPE awatch_ingestion_rejected_total counter
awatch_ingestion_rejected_total 0
# TYPE awatch_role_denied_total counter
awatch_role_denied_total 0
# HELP awatch_readyz_status Portal readiness status, 1=ready, 0=not_ready
# TYPE awatch_readyz_status gauge
awatch_readyz_status 1
@@ -0,0 +1,26 @@
{
"status": "ready",
"checks": {
"portal": {
"status": "ok",
"latency_ms": 1
},
"worktime_api": {
"status": "ok",
"stale": false
},
"clickhouse": {
"status": "ok",
"mode": "external"
},
"dlp_runtime": {
"status": "skipped",
"profile": "core_only",
"reason": "disabled_by_production_default"
},
"security_finding_inbox": {
"status": "ok",
"mode": "shadow"
}
}
}
@@ -0,0 +1,10 @@
{
"app_name": "AWatch-rus DetMir Portal",
"app_version": "0.1.0",
"schema_version": "pilot-v1",
"environment": "test",
"build": {
"profile": "ci",
"source": "fixture"
}
}
@@ -0,0 +1,18 @@
{
"mode": "shadow",
"auto_apply": false,
"workflow": ["decide", "plan", "approve", "apply", "verify"],
"findings": [
{
"id": "fixture-001",
"source": "hayabusa",
"severity": "medium",
"status": "new",
"containment": {
"recommended": true,
"approved": false,
"applied": false
}
}
]
}
@@ -0,0 +1,11 @@
{
"status": "DEGRADED",
"stale": false,
"reason": "report_unavailable",
"report_cache_hit": false,
"report_stale_served": false,
"aw_query_duration_ms": 350,
"aw_query_timeout_count": 1,
"report_build_error_count": 1,
"users": []
}
@@ -0,0 +1,17 @@
{
"status": "OK",
"stale": false,
"reason": null,
"report_cache_hit": false,
"report_stale_served": false,
"aw_query_duration_ms": 42,
"aw_query_timeout_count": 0,
"report_build_error_count": 0,
"users": [
{
"user": "demo-user",
"active_seconds": 1800,
"apps_count": 3
}
]
}
@@ -0,0 +1,44 @@
{
"schema_version": 1,
"generated_at_utc": "2026-07-01T00:00:00Z",
"git_sha": "0123456789abcdef0123456789abcdef01234567",
"items": [
{
"id": "aw-server:detmir-readiness.service",
"host": "aw-server-prod",
"kind": "systemd_service",
"unit_or_task": "detmir-readiness.service",
"binary_path": "/usr/local/bin/detmir-readiness-rust",
"crate": "detmir-readiness",
"release_artifact": "detmir-readiness",
"runtime_role": "readiness check",
"production_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"active": true
},
{
"id": "rdp:aw-windows-telemetry",
"host": "rdp-host-prod",
"kind": "windows_scheduled_task",
"unit_or_task": "AWatch-rus telemetry collector",
"binary_path": "C:\\Program Files\\AWatch-rus\\windows\\aw-windows-telemetry.exe",
"crate": "aw-windows-telemetry",
"release_artifact": "aw-windows-telemetry.exe",
"runtime_role": "Windows telemetry collector",
"production_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"platform": "windows",
"active": true
},
{
"id": "optional:dlp-aggregator",
"host": "aw-server-prod",
"kind": "systemd_service",
"unit_or_task": "dlp-aggregator.service",
"binary_path": "/usr/local/bin/dlp-aggregator-rust",
"crate": "dlp-aggregator",
"release_artifact": "dlp-aggregator",
"runtime_role": "optional DLP aggregator",
"active": false,
"skip_reason": "DLP runtime intentionally disabled for production load control"
}
]
}
@@ -96,7 +96,7 @@ usermod -aG sudo build
HTTPS clone:
```bash
git clone https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus.git
git clone https://git.iri1968.ru/awatch-rus/AWatch-rus.git
```
SSH clone может быть добавлен отдельным шагом после настройки ключей Gitea.
@@ -73,14 +73,14 @@ chmod 640 /etc/gitea/app.ini
```bash
systemctl status gitea --no-pager
curl -L https://git.iri1968.dpdns.org | head
curl -L https://git.iri1968.ru | head
gitea doctor check
```
Также открыть в браузере:
```text
https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus
https://git.iri1968.ru/awatch-rus/AWatch-rus
```
Если менялся путь установки или сервер переносился, выполнить:
+3 -3
View File
@@ -10,7 +10,7 @@ Primary registry-readiness remote / Gitea duplicate:
```text
ru-origin:
https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus.git
https://git.iri1968.ru/awatch-rus/AWatch-rus.git
```
GitHub public mirror:
@@ -54,7 +54,7 @@ git remote -v
Добавить российский remote:
```bash
git remote add ru-origin https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus.git
git remote add ru-origin https://git.iri1968.ru/awatch-rus/AWatch-rus.git
```
Запушить основную ветку:
@@ -103,7 +103,7 @@ git push github --tags
Ожидаемый публичный URL:
```text
https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus
https://git.iri1968.ru/awatch-rus/AWatch-rus
```
Gitea должна обслуживаться через Nginx reverse proxy с HTTPS. После проверки
@@ -58,6 +58,9 @@ checks only. They are not primary registry release evidence.
- `rustc` / `cargo` versions;
- git commit;
- tag/version;
- resolved 40-character Git commit SHA;
- `SOURCE_DATE_EPOCH`;
- deterministic `BUILD_TIME` derived from `SOURCE_DATE_EPOCH`;
- checks;
- artifacts;
- SBOM;
@@ -34,6 +34,19 @@ produced on the Russian build-runner.
17. Сохранить logs.
18. Сохранить final report.
Build provenance requirements:
- `RELEASE_COMMIT` may be a SHA or tag, but manifest `release_commit` must be
the resolved full 40-character Git SHA.
- `GIT_COMMIT` must be exported during Rust build so `/version` and other
build/version probes can report the source revision.
- `SOURCE_DATE_EPOCH` defaults to the resolved commit timestamp and must be
recorded in release evidence.
- `BUILD_TIME` must be derived from `SOURCE_DATE_EPOCH`, not from wall-clock
build time.
- Wall-clock `generated_at` is evidence generation time only; it is not the
embedded build timestamp.
Если smoke требует live stand, не удалять и не скрывать его. В release report
фиксировать: `skipped: requires live stand`.
@@ -63,6 +76,8 @@ product/runtime release candidate.
- `RELEASE_EVIDENCE_REPORT_RU.md`.
- `SHA256SUMS`.
- `logs/`.
- `logs/source-date-epoch.log`.
- `logs/build-time-utc.log`.
- Source archive.
- Binary artifacts archive или documented skip только для `DOCS_ONLY=1`.
- `cargo-metadata.json` или documented skip.
+1 -1
View File
@@ -45,7 +45,7 @@ services:
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=30d"
- "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME:-30d}"
- "--web.enable-lifecycle"
ports:
- "${PROMETHEUS_PORT:-9090}:9090"
@@ -0,0 +1,18 @@
[Unit]
Description=DetMir ActivityWatch primary recovery guard
Documentation=https://github.com/igor04091968/AWatch-rus/blob/main/docs/DETMIR_SERVICE_RELIABILITY_RUNBOOK_RU.md
Wants=network-online.target
After=network-online.target
ConditionPathExists=/usr/sbin/pct
[Service]
Type=oneshot
EnvironmentFile=-/etc/detmir/aw-primary-recovery.env
ExecStart=/usr/local/bin/detmir-aw-primary-recovery --once
User=root
Group=root
Nice=5
TimeoutStartSec=5min
StandardOutput=journal
StandardError=journal
SyslogIdentifier=detmir-aw-primary-recovery
@@ -0,0 +1,13 @@
[Unit]
Description=Run DetMir ActivityWatch primary recovery guard every 2 minutes
[Timer]
OnBootSec=3min
OnUnitActiveSec=2min
AccuracySec=30s
RandomizedDelaySec=15s
Persistent=false
Unit=detmir-aw-primary-recovery.service
[Install]
WantedBy=timers.target
+132 -54
View File
@@ -5,6 +5,7 @@ set -uo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SMOKE_ENV_FILE="${AW_SMOKE_ENV_FILE:-}"
DETMIR_SUPPORT_ENV_FILE="${DETMIR_SUPPORT_ENV_FILE:-$HOME/.config/awatch-rus/detmir-support.env}"
for env_candidate in "$REPO_ROOT/private-config/runtime.env" "$HOME/.config/aw-contour-smoke.env" "$SMOKE_ENV_FILE"; do
if [ -n "$env_candidate" ] && [ -f "$env_candidate" ]; then
# Load local credentials and site-specific overrides without committing them.
@@ -14,6 +15,51 @@ for env_candidate in "$REPO_ROOT/private-config/runtime.env" "$HOME/.config/aw-c
set +a
fi
done
if [ -f "$DETMIR_SUPPORT_ENV_FILE" ]; then
set -a
. "$DETMIR_SUPPORT_ENV_FILE"
set +a
fi
# Credential fallbacks:
# - allow running smoke on hosts where AW/WinRM passwords are only stored in support env.
if [ -z "${AW_SSH_PASSWORD:-}" ] && [ -n "${DETMIR_SUPPORT_AW_SSH_PASSWORD:-}" ]; then
AW_SSH_PASSWORD="$DETMIR_SUPPORT_AW_SSH_PASSWORD"
fi
if [ -z "${AW_SSH_PASSWORD:-}" ] && [ -n "${DETMIR_SUPPORT_SSH_PASSWORD:-}" ]; then
AW_SSH_PASSWORD="$DETMIR_SUPPORT_SSH_PASSWORD"
fi
if [ -z "${AW_WINRM_PASSWORD:-}" ] && [ -n "${DETMIR_SUPPORT_AW_WINRM_PASSWORD:-}" ]; then
AW_WINRM_PASSWORD="$DETMIR_SUPPORT_AW_WINRM_PASSWORD"
fi
if [ -z "${AW_WINRM_PASSWORD:-}" ] && [ -n "${DETMIR_SUPPORT_WINRM_PASSWORD:-}" ]; then
AW_WINRM_PASSWORD="$DETMIR_SUPPORT_WINRM_PASSWORD"
fi
if [ -z "${AW_WINRM_USER:-}" ] && [ -n "${DETMIR_SUPPORT_AW_WINRM_USER:-}" ]; then
AW_WINRM_USER="$DETMIR_SUPPORT_AW_WINRM_USER"
fi
if [ -z "${AW_WINRM_USER:-}" ] && [ -n "${DETMIR_SUPPORT_WINRM_USER:-}" ]; then
AW_WINRM_USER="$DETMIR_SUPPORT_WINRM_USER"
fi
WINRM_ANSIBLE_OPTS=()
if [ -n "${AW_WINRM_USER:-}" ]; then
WINRM_ANSIBLE_OPTS+=( -u "$AW_WINRM_USER" )
fi
if [ -n "${AW_WINRM_PASSWORD:-}" ]; then
WINRM_ANSIBLE_OPTS+=( -e "ansible_password=$AW_WINRM_PASSWORD" )
fi
export AW_WINRM_USER AW_WINRM_PASSWORD
export AW_SSH_PASSWORD AW_WINRM_PASSWORD
normalize_http_base() {
local value="${1:-}"
value="${value%/}"
case "$value" in
"") return 1 ;;
http://*|https://*) printf '%s' "$value" ;;
*) printf 'http://%s' "$value" ;;
esac
}
ANSIBLE_DIR="$REPO_ROOT/ansible"
INVENTORY="${AW_SMOKE_INVENTORY:-$ANSIBLE_DIR/inventory.ini}"
@@ -32,9 +78,9 @@ do
done
REMOTE_RUST_SRC="${AW_SMOKE_REMOTE_RUST_SRC:-$DEFAULT_REMOTE_RUST_SRC}"
REMOTE_RUST_DST="${AW_SMOKE_REMOTE_RUST_BIN:-/usr/local/sbin/aw-contour-smoke}"
AW_SERVER="${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}"
WORKTIME_API="${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}"
GRAFANA_URL="${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}"
AW_SERVER="$(normalize_http_base "${AW_SMOKE_AW_SERVER:-http://10.10.10.13:5600}")"
WORKTIME_API="$(normalize_http_base "${AW_SMOKE_WORKTIME_API:-http://10.10.10.13:5610}")"
GRAFANA_URL="$(normalize_http_base "${AW_SMOKE_GRAFANA_URL:-http://10.10.10.11:3000}")"
GRAFANA_USER="${GRAFANA_USER:-igor}"
GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-}"
PROXMOX_HOST="${AW_SMOKE_PROXMOX_HOST:-10.10.10.2}"
@@ -46,7 +92,11 @@ LOG_DIR="${AW_SMOKE_LOG_DIR:-$REPO_ROOT/output/smoke}"
RUN_REMOTE="${AW_SMOKE_RUN_REMOTE:-1}"
RUN_WINRM="${AW_SMOKE_RUN_WINRM:-1}"
RUN_SERVER_SYSTEMD="${AW_SMOKE_RUN_SERVER_SYSTEMD:-1}"
LOKI_ENABLED="${AW_SMOKE_LOKI_ENABLED:-0}"
DLP_ENABLED="${DETMIR_DLP_ENABLED:-${AW_DLP_ENABLED:-false}}"
case "${DLP_ENABLED,,}" in
1|true|yes|on) DLP_ENABLED=true ;;
*) DLP_ENABLED=false ;;
esac
NO_PROXY_REQUIRED="localhost,127.0.0.1,$PROXMOX_HOST,$AW_HOST,$GRAFANA_HOST,$WINDOWS_HOST,10.10.10.0/24,192.168.100.0/24"
if [ -n "${no_proxy:-}" ]; then
@@ -64,6 +114,7 @@ OK_COUNT=0
WARN_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0
HOST_INACTIVE=0
if [ -t 1 ]; then
RED='\033[0;31m'
@@ -87,8 +138,9 @@ Environment overrides:
AW_SMOKE_AW_SERVER=http://10.10.10.13:5600
AW_SMOKE_WORKTIME_API=http://10.10.10.13:5610
AW_SMOKE_GRAFANA_URL=http://10.10.10.11:3000
AW_WINRM_USER=Администртор
AW_WINRM_PASSWORD=...
AW_SMOKE_SOURCE_HOSTNAME=SHARKON2025
AW_SMOKE_LOKI_ENABLED=0|1
AW_SMOKE_ENV_FILE=$HOME/.config/aw-contour-smoke.env
AW_SMOKE_LOG_DIR=$REPO_ROOT/output/smoke
GRAFANA_USER/GRAFANA_PASSWORD via env or a local env file
@@ -218,39 +270,6 @@ check_http_code_basic_auth() {
rm -f "$tmp" "$tmp.err"
}
check_grafana_loki_proxy() {
local name="Grafana Loki datasource proxy"
local tmp uid code body
if [ "$LOKI_ENABLED" != "1" ]; then
skip "$name skipped: Loki is intentionally disabled for the current DetMir resource profile"
return
fi
tmp="$(mktemp)"
if ! curl -k -fsS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 "$GRAFANA_URL/api/datasources" -o "$tmp" 2>"$tmp.err"; then
fail "$name cannot list datasources"
sed 's/^/ /' "$tmp.err" "$tmp" 2>/dev/null | head -40
rm -f "$tmp" "$tmp.err"
return
fi
uid="$(jq -r '.[] | select(.type == "loki") | .uid' "$tmp" 2>/dev/null | head -1)"
if [ -z "$uid" ]; then
fail "$name no loki datasource found"
jq -r '.[] | "\(.name) \(.type) \(.uid)"' "$tmp" 2>/dev/null | sed 's/^/ /' | head -20
rm -f "$tmp" "$tmp.err"
return
fi
body="$(mktemp)"
code="$(curl -k -sS -u "$GRAFANA_USER:$GRAFANA_PASSWORD" --connect-timeout 5 --max-time 20 -o "$body" -w '%{http_code}' "$GRAFANA_URL/api/datasources/proxy/uid/$uid/loki/api/v1/labels" 2>"$tmp.err")"
if [ "$code" = "200" ]; then
pass "$name HTTP $code uid=$uid"
jq -r '.status // .data[0] // .' "$body" 2>/dev/null | sed 's/^/ /' | head -5
else
fail "$name HTTP $code uid=$uid"
sed 's/^/ /' "$tmp.err" "$body" 2>/dev/null | head -40
fi
rm -f "$tmp" "$tmp.err" "$body"
}
check_grafana_influx_health() {
local name="Grafana InfluxDB-AW datasource health"
local tmp
@@ -353,7 +372,7 @@ check_ansible_win_shell() {
local command="$3"
local tmp
tmp="$(mktemp)"
if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m win_shell -a "$command" "${WINRM_ANSIBLE_OPTS[@]}" >"$tmp" 2>&1; then
pass "$name"
sed 's/^/ /' "$tmp" | head -80
else
@@ -369,7 +388,7 @@ check_ansible_win_shell_warn() {
local command="$3"
local tmp
tmp="$(mktemp)"
if ansible_win_shell "$group" "$command" >"$tmp" 2>&1; then
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m win_shell -a "$command" "${WINRM_ANSIBLE_OPTS[@]}" >"$tmp" 2>&1; then
pass "$name"
sed 's/^/ /' "$tmp" | head -80
else
@@ -386,7 +405,7 @@ check_ansible_module() {
local args="${4:-}"
local tmp
tmp="$(mktemp)"
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} >"$tmp" 2>&1; then
if ANSIBLE_NOCOLOR=1 ansible "$group" -i "$INVENTORY" -m "$module" ${args:+-a "$args"} "${WINRM_ANSIBLE_OPTS[@]}" >"$tmp" 2>&1; then
pass "$name"
sed 's/^/ /' "$tmp" | head -80
else
@@ -396,9 +415,42 @@ check_ansible_module() {
rm -f "$tmp"
}
read_activitywatch_context() {
local bucket_id tmp last_ts active event_epoch now age_sec
bucket_id="aw-worktime-sessions_${AW_SOURCE_HOSTNAME}"
tmp="$(mktemp)"
if curl -fsS --connect-timeout 5 --max-time 20 "$AW_SERVER/api/0/buckets/$bucket_id/events?limit=1" -o "$tmp" 2>"$tmp.err"; then
last_ts="$(jq -r '.[0].timestamp // empty' "$tmp" 2>/dev/null)"
active="$(jq -r '.[0].data.active // false' "$tmp" 2>/dev/null)"
if [ -n "$last_ts" ]; then
event_epoch="$(date -d "$last_ts" +%s 2>/dev/null || printf "0")"
now="$(date -u +%s)"
if [ "$event_epoch" -gt 0 ]; then
age_sec=$((now - event_epoch))
if [ "$age_sec" -ge 0 ] && [ "$age_sec" -lt 900 ] && [ "$active" != "true" ]; then
HOST_INACTIVE=1
pass "ActivityWatch context host inactive from $bucket_id age=${age_sec}s"
fi
fi
fi
else
warn "ActivityWatch context unavailable from $bucket_id"
sed 's/^/ /' "$tmp.err" | head -20
fi
rm -f "$tmp" "$tmp.err"
}
classify_bucket_age() {
local bucket="$1"
local age_sec="$2"
if [ "$bucket" = "aw-dlp-endpoint-signals" ] && [ "$DLP_ENABLED" = "false" ]; then
printf "inactive"
return
fi
if [ "$bucket" = "aw-watcher-window" ] && [ "$HOST_INACTIVE" = "1" ]; then
printf "inactive"
return
fi
case "$bucket" in
aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
if [ "$age_sec" -lt 86400 ]; then
@@ -443,6 +495,20 @@ check_bucket_freshness() {
if [ -z "$last_ts" ]; then
case "$bucket" in
aw-dlp-endpoint-signals)
if [ "$DLP_ENABLED" = "false" ]; then
pass "bucket $bucket_id inactive/empty while DLP runtime disabled"
else
fail "bucket $bucket_id empty"
fi
;;
aw-watcher-window)
if [ "$HOST_INACTIVE" = "1" ]; then
pass "bucket $bucket_id inactive/empty while host inactive"
else
fail "bucket $bucket_id empty"
fi
;;
aw-dlp-incidents|aw-dlp-review|aw-dlp-rules|aw-session-events)
warn "bucket $bucket_id empty/event-driven"
;;
@@ -462,8 +528,12 @@ check_bucket_freshness() {
age_sec=$((now - event_epoch))
status="$(classify_bucket_age "$bucket" "$age_sec")"
case "$status" in
fresh|event-driven)
pass "bucket $bucket_id $status age=${age_sec}s id=$last_id"
fresh|event-driven|inactive)
if [ "$bucket" = "aw-dlp-endpoint-signals" ] && [ "$DLP_ENABLED" = "false" ]; then
pass "bucket $bucket_id inactive while DLP runtime disabled age=${age_sec}s id=$last_id"
else
pass "bucket $bucket_id $status age=${age_sec}s id=$last_id"
fi
;;
stale)
warn "bucket $bucket_id stale age=${age_sec}s id=$last_id"
@@ -530,6 +600,7 @@ exec > >(tee "$LOG_FILE") 2>&1
section "Run Context"
printf "repo=%s\ninventory=%s\nlog=%s\n" "$REPO_ROOT" "$INVENTORY" "$LOG_FILE" | sed 's/^/ /'
printf "dlp_enabled=%s\n" "$DLP_ENABLED" | sed 's/^/ /'
date -Is | sed 's/^/ /'
section "Local Prerequisites"
@@ -563,6 +634,7 @@ check_http_code "AW WebUI" "$AW_SERVER/" '^200$'
check_http_json_key "AW buckets list" "$AW_SERVER/api/0/buckets/" 'keys | length'
section "ActivityWatch Buckets"
read_activitywatch_context
for bucket in \
aw-watcher-afk \
aw-watcher-window \
@@ -584,9 +656,9 @@ check_http_code "worktime management html" "$WORKTIME_API/reports/worktime/manag
section "Gateway And 1C HTTP"
check_http_code "gateway healthz" "https://$PROXMOX_HOST/healthz" '^200$'
check_http_code "gateway proxmox protected" "https://$PROXMOX_HOST/go/proxmox-gui" '^401$'
check_http_code "gateway file1c brief protected" "https://$PROXMOX_HOST/go/file1c-brief" '^401$'
check_http_code "gateway file1c actions protected" "https://$PROXMOX_HOST/go/file1c-actions" '^401$'
check_http_code "gateway proxmox redirect" "https://$PROXMOX_HOST/go/proxmox-gui" '^302$'
check_http_code "gateway file1c brief redirect" "https://$PROXMOX_HOST/go/file1c-brief" '^302$'
check_http_code "gateway file1c actions redirect" "https://$PROXMOX_HOST/go/file1c-actions" '^302$'
check_http_code "1C /health" "http://$PROXMOX_HOST:8710/health" '^200$'
check_http_code "1C /api/health" "http://$PROXMOX_HOST:8710/api/health" '^200$'
check_http_code "1C manager brief" "http://$PROXMOX_HOST:8710/manager/brief" '^200$'
@@ -597,12 +669,10 @@ check_http_code "1C weekly digest" "http://$PROXMOX_HOST:8710/manager/digest/wee
section "Grafana HTTP"
check_http_json_key "Grafana health" "$GRAFANA_URL/api/health" '.database // .version // .commit'
check_http_code "Grafana dashboards page" "$GRAFANA_URL/dashboards" '^200$|^302$'
check_http_code "Grafana pfSense dashboard" "$GRAFANA_URL/d/pfsense-loki-dashboard/pfsense-firewall-overview?orgId=1&from=now-1h&to=now&timezone=browser" '^200$|^302$'
if [ -n "${GRAFANA_USER:-}" ] && [ -n "${GRAFANA_PASSWORD:-}" ]; then
section "Grafana Authenticated API"
check_http_json_key_basic_auth "Grafana datasources" "$GRAFANA_URL/api/datasources" 'length'
check_grafana_loki_proxy
check_grafana_influx_health
check_grafana_aw_main_dashboard_queries
else
@@ -610,20 +680,28 @@ else
fi
if [ "$RUN_SERVER_SYSTEMD" = "1" ] && have ansible; then
if [ -z "${AW_SSH_PASSWORD:-}" ] && [ -z "${AW_SUDO_PASSWORD:-}" ] && [ -z "${AW_SSH_KEY_PATH:-}" ]; then
fail "AW server systemd checks blocked: missing AW_SSH_PASSWORD or AW_SUDO_PASSWORD (set AW_SSH_PASSWORD / AW_SUDO_PASSWORD or AW_SSH_KEY_PATH)."
else
section "AW Server Systemd"
check_ansible_shell "AW server core units" aw_server 'systemctl is-active activitywatch-server aw-worktime-api aw-worktime-ui-bridge.timer aw-rus-healthd.timer aw-worktime-influx-exporter.timer aw-dlp-influx-exporter.timer aw-worktime-autoheal.timer'
check_ansible_shell "AW server failed units" aw_server 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|influx|grafana|prometheus|loki" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
check_ansible_shell "AW server failed units" aw_server 'failed=$(systemctl --failed --no-legend | awk "{print \$1}" | grep -E "activitywatch|aw-|influx|grafana|prometheus" || true); test -z "$failed" && echo "no AW-related failed units" || { echo "$failed"; exit 1; }'
check_ansible_shell "AW server local health script" aw_server 'test -x /opt/activitywatch/health-check.sh && /opt/activitywatch/health-check.sh || test -x /usr/local/bin/health-check.sh && /usr/local/bin/health-check.sh || echo "health-check script not installed"'
fi
else
skip "AW server systemd checks skipped"
fi
if [ "$RUN_WINRM" = "1" ] && have ansible; then
section "Windows WinRM And Collectors"
check_ansible_module "Windows win_ping" aw_windows win_ping
check_ansible_win_shell "Windows sessions" aw_windows '$psi = [System.Diagnostics.ProcessStartInfo]::new(); $psi.FileName = "$env:SystemRoot\System32\query.exe"; $psi.Arguments = "user"; $psi.UseShellExecute = $false; $psi.RedirectStandardOutput = $true; $psi.RedirectStandardError = $true; $psi.StandardOutputEncoding = [System.Text.Encoding]::GetEncoding(866); $psi.StandardErrorEncoding = [System.Text.Encoding]::GetEncoding(866); $p = [System.Diagnostics.Process]::Start($psi); $out = $p.StandardOutput.ReadToEnd(); $err = $p.StandardError.ReadToEnd(); $p.WaitForExit(); [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $out; if ($err) { $err }; if ($out -match "USERNAME|ПОЛЬЗОВАТЕЛЬ|администратор|Администратор") { exit 0 } else { exit $p.ExitCode }'
check_ansible_win_shell_warn "Windows collector processes" aw_windows '$p = Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue; if ($p) { $p | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize } else { "no aw-watcher-afk/window process visible to this WinRM session" }'
check_ansible_win_shell "Windows ActivityWatch tasks" aw_windows 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V; Get-Content -Raw "C:\ProgramData\AWatch-rus\deployment-config.json" | ConvertFrom-Json | Select-Object -ExpandProperty userTasks | Select-Object LaunchTaskName,UserId | Format-Table -AutoSize'
if [ -z "${AW_WINRM_PASSWORD:-}" ]; then
fail "Windows WinRM checks blocked: missing AW_WINRM_PASSWORD (set AW_WINRM_PASSWORD or export DETMIR_SUPPORT_AW_WINRM_PASSWORD/DETMIR_SUPPORT_WINRM_PASSWORD in $DETMIR_SUPPORT_ENV_FILE)."
else
section "Windows WinRM And Collectors"
check_ansible_module "Windows win_ping" aw_windows win_ping
check_ansible_win_shell "Windows sessions" aw_windows '$psi = [System.Diagnostics.ProcessStartInfo]::new(); $psi.FileName = "$env:SystemRoot\System32\query.exe"; $psi.Arguments = "user"; $psi.UseShellExecute = $false; $psi.RedirectStandardOutput = $true; $psi.RedirectStandardError = $true; $psi.StandardOutputEncoding = [System.Text.Encoding]::GetEncoding(866); $psi.StandardErrorEncoding = [System.Text.Encoding]::GetEncoding(866); $p = [System.Diagnostics.Process]::Start($psi); $out = $p.StandardOutput.ReadToEnd(); $err = $p.StandardError.ReadToEnd(); $p.WaitForExit(); [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $out; if ($err) { $err }; if ($out -match "USERNAME|ПОЛЬЗОВАТЕЛЬ|администратор|Администратор") { exit 0 } else { exit $p.ExitCode }'
check_ansible_win_shell_warn "Windows collector processes" aw_windows '$p = Get-Process aw-watcher-afk,aw-watcher-window -ErrorAction SilentlyContinue; if ($p) { $p | Select-Object Name,Id,SessionId,StartTime | Format-Table -AutoSize } else { "no aw-watcher-afk/window process visible to this WinRM session" }'
check_ansible_win_shell "Windows ActivityWatch tasks" aw_windows 'schtasks /Query /TN "ActivityWatch Recovery" /FO LIST /V; Get-Content -Raw "C:\ProgramData\AWatch-rus\deployment-config.json" | ConvertFrom-Json | Select-Object -ExpandProperty userTasks | Select-Object LaunchTaskName,UserId | Format-Table -AutoSize'
fi
else
skip "Windows WinRM checks skipped"
fi
+26 -1
View File
@@ -1,11 +1,36 @@
#!/usr/bin/env node
const baseUrl = (process.env.AWATCH_PORTAL_SMOKE_URL || "http://127.0.0.1:8720").replace(/\/+$/, "");
if (["1", "true", "yes"].includes(String(process.env.DETMIR_PORTAL_SMOKE_INSECURE_TLS || "").toLowerCase())) {
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
}
function env(name, fallback = "") {
const value = process.env[name];
return value && value.trim() ? value.trim() : fallback;
}
function normalizeBaseUrl(raw) {
const trimmed = String(raw || "").trim().replace(/\/+$/, "");
if (!trimmed) return "http://127.0.0.1:8720";
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) return trimmed;
return `http://${trimmed}`;
}
function authHeaders() {
const explicit = env("DETMIR_PORTAL_SMOKE_AUTH_HEADER", env("DETMIR_PORTAL_AUTH_HEADER"));
if (explicit) return { Authorization: explicit };
const basic = env("DETMIR_PORTAL_SMOKE_BASIC_AUTH", env("DETMIR_BASIC_AUTH"));
if (basic) return { Authorization: `Basic ${basic}` };
return {};
}
const baseUrl = normalizeBaseUrl(env("AWATCH_PORTAL_SMOKE_URL", env("DETMIR_PORTAL_URL", "http://127.0.0.1:8720")));
async function request(path, options = {}) {
const response = await fetch(`${baseUrl}${path}`, {
...options,
headers: {
...authHeaders(),
"X-AWatch-Role": "executive",
"X-Request-Id": "smoke-production-hardening",
...(options.headers || {}),
+10 -1
View File
@@ -129,7 +129,16 @@ trap cleanup_on_failure EXIT
mkdir -p "$OUT_DIR"
git rev-parse HEAD > "$OUT_DIR/git-commit.txt"
GIT_COMMIT="$(git rev-parse HEAD)"
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git show -s --format=%ct "$GIT_COMMIT")}"
BUILD_TIME="$(date -u -d "@${SOURCE_DATE_EPOCH}" +%Y-%m-%dT%H:%M:%SZ)"
export GIT_COMMIT
export SOURCE_DATE_EPOCH
export BUILD_TIME
printf '%s\n' "$GIT_COMMIT" > "$OUT_DIR/git-commit.txt"
printf '%s\n' "$SOURCE_DATE_EPOCH" > "$OUT_DIR/source-date-epoch.txt"
printf '%s\n' "$BUILD_TIME" > "$OUT_DIR/build-time-utc.txt"
cargo fmt --manifest-path adk-rust/Cargo.toml --all -- --check
cargo test --manifest-path adk-rust/Cargo.toml --workspace
+45 -6
View File
@@ -118,11 +118,21 @@ if [[ "$RELEASE_COMMIT" != "$(git rev-parse HEAD)" ]]; then
git -c filter.lfs.smudge= -c filter.lfs.process= -c filter.lfs.required=false checkout --detach "$RELEASE_COMMIT"
fi
RELEASE_COMMIT_INPUT="$RELEASE_COMMIT"
RELEASE_COMMIT="$(git rev-parse HEAD)"
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git show -s --format=%ct "$RELEASE_COMMIT")}"
BUILD_TIME_UTC="$(date -u -d "@${SOURCE_DATE_EPOCH}" +%Y-%m-%dT%H:%M:%SZ)"
export GIT_COMMIT="$RELEASE_COMMIT"
export BUILD_TIME="$BUILD_TIME_UTC"
export SOURCE_DATE_EPOCH
{
git remote -v
} > "$LOG_DIR/git-remotes.log" 2>&1
git status --short > "$LOG_DIR/git-status-short.log" 2>&1
git rev-parse HEAD > "$LOG_DIR/git-rev-parse-head.log" 2>&1
printf '%s\n' "$SOURCE_DATE_EPOCH" > "$LOG_DIR/source-date-epoch.log"
printf '%s\n' "$BUILD_TIME_UTC" > "$LOG_DIR/build-time-utc.log"
git log --oneline -20 > "$LOG_DIR/git-log-oneline-20.log" 2>&1
capture_command rustc-version rustc --version
capture_command cargo-version cargo --version
@@ -198,14 +208,21 @@ git -c filter.lfs.smudge= -c filter.lfs.process= -c filter.lfs.required=false ar
record_check "source_archive" "ok" "$SOURCE_ARCHIVE"
BINARY_ARCHIVE="$ARTIFACT_DIR/${RELEASE_VERSION}-binaries.tar.gz"
if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then
BINARY_RELEASE_DIR="${CARGO_TARGET_DIR%/}/release"
elif [[ -n "$CARGO_WORKSPACE_DIR" ]]; then
BINARY_RELEASE_DIR="$CARGO_WORKSPACE_DIR/target/release"
else
BINARY_RELEASE_DIR="$ROOT/target/release"
fi
if [[ "$DOCS_ONLY" == "1" ]]; then
printf 'skipped: DOCS_ONLY=1\n' > "$BINARY_ARCHIVE.skip"
skip_check "binary_archive" "DOCS_ONLY=1"
elif [[ -d target/release ]]; then
tar -czf "$BINARY_ARCHIVE" target/release
elif [[ -d "$BINARY_RELEASE_DIR" ]]; then
tar -czf "$BINARY_ARCHIVE" -C "$BINARY_RELEASE_DIR" .
record_check "binary_archive" "ok" "$BINARY_ARCHIVE"
else
skip_check "binary_archive" "target/release missing"
skip_check "binary_archive" "$BINARY_RELEASE_DIR missing"
fi
if command -v cargo-cyclonedx >/dev/null 2>&1; then
@@ -226,12 +243,22 @@ GENERATED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
BUILD_RUNNER_HOST="$(hostname 2>/dev/null || printf unknown)"
PRIMARY_SOURCE_REPOSITORY="https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus"
python3 - "$OUTPUT_DIR/release-evidence-manifest.json" "$RELEASE_VERSION" "$RELEASE_COMMIT" "$GENERATED_AT" "$BUILD_RUNNER_HOST" "$PRIMARY_SOURCE_REPOSITORY" <<'PY'
python3 - "$OUTPUT_DIR/release-evidence-manifest.json" "$RELEASE_VERSION" "$RELEASE_COMMIT_INPUT" "$RELEASE_COMMIT" "$SOURCE_DATE_EPOCH" "$BUILD_TIME_UTC" "$GENERATED_AT" "$BUILD_RUNNER_HOST" "$PRIMARY_SOURCE_REPOSITORY" <<'PY'
import json
import sys
from pathlib import Path
manifest_path, release_version, release_commit, generated_at, build_runner, primary_source = sys.argv[1:]
(
manifest_path,
release_version,
release_commit_input,
release_commit,
source_date_epoch,
build_time_utc,
generated_at,
build_runner,
primary_source,
) = sys.argv[1:]
root = Path(manifest_path).parent
checks = []
checks_file = root / "checks.tsv"
@@ -248,7 +275,10 @@ for path in sorted((root / "artifacts").glob("*")):
data = {
"product": "AWatch-rus",
"release_version": release_version,
"release_commit_input": release_commit_input,
"release_commit": release_commit,
"source_date_epoch": int(source_date_epoch),
"build_time_utc": build_time_utc,
"build_runner": build_runner,
"primary_source_repository": primary_source,
"github_role": "public_mirror_only",
@@ -269,6 +299,12 @@ Release version: ${RELEASE_VERSION}
Release commit: ${RELEASE_COMMIT}
Release commit input: ${RELEASE_COMMIT_INPUT}
SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH}
Build time UTC: ${BUILD_TIME_UTC}
Generated at: ${GENERATED_AT}
Build runner: ${BUILD_RUNNER_HOST}
@@ -300,7 +336,10 @@ EOF
(
cd "$OUTPUT_DIR"
find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS
sums_tmp="$(mktemp)"
trap 'rm -f "$sums_tmp"' EXIT
find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > "$sums_tmp"
mv "$sums_tmp" SHA256SUMS
)
if [[ -f scripts/check_release_evidence.sh ]]; then
@@ -125,3 +125,9 @@ EOF
fi
echo "detmir rust release artifacts: OK scope=$SCOPE ($RELEASE_DIR)"
if [[ -n "${PRODUCTION_BINARY_PARITY_EVIDENCE:-}" ]]; then
python3 "$ROOT_DIR/scripts/check_production_binary_parity.py" \
--evidence "$PRODUCTION_BINARY_PARITY_EVIDENCE" \
--release-dir "$RELEASE_DIR"
fi
+440
View File
@@ -0,0 +1,440 @@
#!/usr/bin/env python3
"""Validate DetMir production binary parity evidence.
The script compares SHA256 hashes collected from running production binaries
with locally built release artifacts from the same Git revision. It intentionally
does not collect live production data: operators provide evidence produced from
the approved production contour.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any
HEX40 = re.compile(r"^[0-9a-f]{40}$")
HEX64 = re.compile(r"^[0-9a-f]{64}$")
WINDOWS_TARGET = "x86_64-pc-windows-gnu"
REQUIRED_FIELDS = (
"id",
"host",
"kind",
"unit_or_task",
"binary_path",
"crate",
"release_artifact",
"runtime_role",
"production_sha256",
)
def repo_root() -> Path:
return Path(__file__).resolve().parents[1]
def default_target_root(root: Path) -> Path:
return Path(os.environ.get("CARGO_TARGET_DIR", root / "adk-rust" / "target"))
def default_release_dir(root: Path) -> Path:
return default_target_root(root) / "release"
def default_windows_release_dir(root: Path) -> Path:
return default_target_root(root) / WINDOWS_TARGET / "release"
def git_head(root: Path) -> str:
completed = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=root,
check=True,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
return completed.stdout.strip().lower()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def load_json(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
data = json.load(handle)
if not isinstance(data, dict):
raise ValueError("evidence root must be a JSON object")
return data
def is_windows_item(item: dict[str, Any]) -> bool:
platform = str(item.get("platform", "")).lower()
artifact = str(item.get("release_artifact", "")).lower()
binary_path = str(item.get("binary_path", ""))
return platform == "windows" or artifact.endswith(".exe") or "\\" in binary_path
def artifact_path(
item: dict[str, Any],
release_dir: Path,
windows_release_dir: Path,
) -> Path:
artifact = str(item.get("release_artifact", ""))
if not artifact or "/" in artifact or "\\" in artifact:
raise ValueError("release_artifact must be a file name, not a path")
base = windows_release_dir if is_windows_item(item) else release_dir
return base / artifact
def normalize_sha(value: Any) -> str:
return str(value).strip().lower()
def validate_evidence(
evidence: dict[str, Any],
*,
expected_git_sha: str,
release_dir: Path,
windows_release_dir: Path,
) -> dict[str, Any]:
errors: list[str] = []
rows: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
schema_version = evidence.get("schema_version")
if schema_version != 1:
errors.append("schema_version must be 1")
git_sha = normalize_sha(evidence.get("git_sha", ""))
if not HEX40.match(git_sha):
errors.append("top-level git_sha must be a 40-character lowercase hex SHA")
elif git_sha != expected_git_sha:
errors.append(
f"top-level git_sha {git_sha} does not match expected {expected_git_sha}"
)
items = evidence.get("items")
if not isinstance(items, list) or not items:
errors.append("items must be a non-empty array")
items = []
seen_ids: set[str] = set()
active_count = 0
for index, raw_item in enumerate(items):
if not isinstance(raw_item, dict):
errors.append(f"items[{index}] must be an object")
continue
item = raw_item
item_id = str(item.get("id", f"items[{index}]"))
if item_id in seen_ids:
errors.append(f"{item_id}: duplicate id")
seen_ids.add(item_id)
raw_active = item.get("active", True)
if not isinstance(raw_active, bool):
errors.append(f"{item_id}: active must be boolean")
continue
active = raw_active
if not active:
skip_reason = str(item.get("skip_reason", "")).strip()
if not skip_reason:
errors.append(f"{item_id}: inactive item must include skip_reason")
skipped.append(
{
"id": item_id,
"host": item.get("host", ""),
"unit_or_task": item.get("unit_or_task", ""),
"crate": item.get("crate", ""),
"runtime_role": item.get("runtime_role", ""),
"skip_reason": skip_reason,
}
)
continue
active_count += 1
for field in REQUIRED_FIELDS:
value = item.get(field)
if value is None or str(value).strip() == "":
errors.append(f"{item_id}: missing required field {field}")
item_git_sha = item.get("git_sha")
if item_git_sha is not None and normalize_sha(item_git_sha) != expected_git_sha:
errors.append(
f"{item_id}: item git_sha {normalize_sha(item_git_sha)} "
f"does not match expected {expected_git_sha}"
)
production_sha = normalize_sha(item.get("production_sha256", ""))
if not HEX64.match(production_sha):
errors.append(f"{item_id}: production_sha256 must be 64 lowercase hex chars")
continue
try:
local_artifact = artifact_path(item, release_dir, windows_release_dir)
except ValueError as exc:
errors.append(f"{item_id}: {exc}")
continue
if not local_artifact.is_file():
errors.append(f"{item_id}: release artifact not found: {local_artifact}")
continue
release_sha = sha256_file(local_artifact)
parity = release_sha == production_sha
if not parity:
errors.append(
f"{item_id}: production SHA {production_sha} does not match "
f"release SHA {release_sha} for {local_artifact}"
)
rows.append(
{
"id": item_id,
"host": item.get("host", ""),
"kind": item.get("kind", ""),
"unit_or_task": item.get("unit_or_task", ""),
"binary_path": item.get("binary_path", ""),
"crate": item.get("crate", ""),
"release_artifact": item.get("release_artifact", ""),
"runtime_role": item.get("runtime_role", ""),
"production_sha256": production_sha,
"release_sha256": release_sha,
"git_sha": expected_git_sha,
"parity": parity,
}
)
if active_count == 0:
errors.append("at least one active production binary must be present")
return {
"status": "ok" if not errors else "fail",
"schema_version": schema_version,
"git_sha": expected_git_sha,
"release_dir": str(release_dir),
"windows_release_dir": str(windows_release_dir),
"active_count": active_count,
"skipped_count": len(skipped),
"items": rows,
"skipped": skipped,
"errors": errors,
}
def print_report(report: dict[str, Any]) -> None:
if report["status"] == "ok":
print(
"production_binary_parity=ok "
f"active={report['active_count']} skipped={report['skipped_count']} "
f"git_sha={report['git_sha']}"
)
for item in report["items"]:
print(
"OK "
f"{item['unit_or_task']} -> {item['binary_path']} -> "
f"{item['crate']} role={item['runtime_role']} "
f"sha256={item['production_sha256']}"
)
for item in report["skipped"]:
print(
"SKIP "
f"{item['unit_or_task']} crate={item['crate']} "
f"reason={item['skip_reason']}"
)
return
print(
"production_binary_parity=fail "
f"active={report['active_count']} skipped={report['skipped_count']} "
f"git_sha={report['git_sha']}",
file=sys.stderr,
)
for error in report["errors"]:
print(f"ERROR {error}", file=sys.stderr)
def write_json_report(path: Path, report: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
json.dump(report, handle, indent=2, sort_keys=True)
handle.write("\n")
def run_self_test() -> int:
root = Path(tempfile.mkdtemp(prefix="aw-parity-selftest-"))
try:
release_dir = root / "release"
windows_release_dir = root / WINDOWS_TARGET / "release"
release_dir.mkdir(parents=True)
windows_release_dir.mkdir(parents=True)
linux_artifact = release_dir / "detmir-readiness"
windows_artifact = windows_release_dir / "aw-windows-telemetry.exe"
linux_artifact.write_bytes(b"linux-release-artifact\n")
windows_artifact.write_bytes(b"windows-release-artifact\n")
git_sha = "0123456789abcdef0123456789abcdef01234567"
evidence = {
"schema_version": 1,
"generated_at_utc": "2026-07-01T00:00:00Z",
"git_sha": git_sha,
"items": [
{
"id": "server:detmir-readiness.service",
"host": "10.10.10.13",
"kind": "systemd_service",
"unit_or_task": "detmir-readiness.service",
"binary_path": "/usr/local/bin/detmir-readiness-rust",
"crate": "detmir-readiness",
"release_artifact": "detmir-readiness",
"runtime_role": "readiness check",
"production_sha256": sha256_file(linux_artifact),
"active": True,
},
{
"id": "rdp:aw-windows-telemetry",
"host": "192.168.100.19",
"kind": "windows_scheduled_task",
"unit_or_task": "AWatch-rus telemetry collector",
"binary_path": (
r"C:\Program Files\AWatch-rus\windows"
r"\aw-windows-telemetry.exe"
),
"crate": "aw-windows-telemetry",
"release_artifact": "aw-windows-telemetry.exe",
"runtime_role": "Windows telemetry collector",
"production_sha256": sha256_file(windows_artifact),
"platform": "windows",
"active": True,
},
{
"id": "optional:dlp-aggregator",
"host": "10.10.10.13",
"kind": "systemd_service",
"unit_or_task": "dlp-aggregator.service",
"binary_path": "/usr/local/bin/dlp-aggregator-rust",
"crate": "dlp-aggregator",
"release_artifact": "dlp-aggregator",
"runtime_role": "optional DLP aggregator",
"active": False,
"skip_reason": "DLP runtime intentionally disabled",
},
],
}
ok_report = validate_evidence(
evidence,
expected_git_sha=git_sha,
release_dir=release_dir,
windows_release_dir=windows_release_dir,
)
if ok_report["status"] != "ok":
print(json.dumps(ok_report, indent=2), file=sys.stderr)
return 1
mismatch = json.loads(json.dumps(evidence))
mismatch["items"][0]["production_sha256"] = "0" * 64
fail_report = validate_evidence(
mismatch,
expected_git_sha=git_sha,
release_dir=release_dir,
windows_release_dir=windows_release_dir,
)
if fail_report["status"] != "fail":
print(json.dumps(fail_report, indent=2), file=sys.stderr)
return 1
print("check_production_binary_parity self-test: OK")
return 0
finally:
shutil.rmtree(root)
def parse_args(argv: list[str]) -> argparse.Namespace:
root = repo_root()
parser = argparse.ArgumentParser(
description="Validate DetMir production binary parity evidence."
)
parser.add_argument("--evidence", type=Path, help="production evidence JSON")
parser.add_argument(
"--release-dir",
type=Path,
default=default_release_dir(root),
help="Linux release artifact directory",
)
parser.add_argument(
"--windows-release-dir",
type=Path,
default=default_windows_release_dir(root),
help="Windows release artifact directory",
)
parser.add_argument(
"--git-sha",
default=None,
help="expected source Git SHA; defaults to repository HEAD",
)
parser.add_argument("--output-json", type=Path, help="write validation report")
parser.add_argument(
"--self-test",
action="store_true",
help="run internal validator regression test",
)
args = parser.parse_args(argv)
if not args.self_test and args.evidence is None:
parser.error("--evidence is required unless --self-test is used")
return args
def main(argv: list[str]) -> int:
args = parse_args(argv)
if args.self_test:
return run_self_test()
root = repo_root()
expected_git_sha = normalize_sha(args.git_sha or git_head(root))
if not HEX40.match(expected_git_sha):
print(
f"expected git SHA must be a 40-character lowercase hex SHA: {expected_git_sha}",
file=sys.stderr,
)
return 2
try:
evidence = load_json(args.evidence)
report = validate_evidence(
evidence,
expected_git_sha=expected_git_sha,
release_dir=args.release_dir,
windows_release_dir=args.windows_release_dir,
)
except (OSError, ValueError, json.JSONDecodeError, subprocess.CalledProcessError) as exc:
print(f"production_binary_parity=fail error={exc}", file=sys.stderr)
return 1
if args.output_json:
write_json_report(args.output_json, report)
print_report(report)
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
+9 -2
View File
@@ -48,7 +48,10 @@ if [[ -s "$EVIDENCE_DIR/release-evidence-manifest.json" ]]; then
jq -e '
.product == "AWatch-rus"
and (.release_version | type == "string" and length > 0)
and (.release_commit | type == "string" and length > 0)
and (.release_commit | type == "string" and test("^[0-9a-f]{40}$"))
and (.release_commit_input | type == "string" and length > 0)
and (.source_date_epoch | type == "number")
and (.build_time_utc | type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"))
and (.build_runner | type == "string" and length > 0)
and .primary_source_repository == "https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus"
and .github_role == "public_mirror_only"
@@ -72,9 +75,13 @@ required = {
for key, value in required.items():
if data.get(key) != value:
raise SystemExit(f"{key} mismatch")
for key in ("release_version", "release_commit", "build_runner", "generated_at"):
for key in ("release_version", "release_commit", "release_commit_input", "build_time_utc", "build_runner", "generated_at"):
if not isinstance(data.get(key), str) or not data[key]:
raise SystemExit(f"{key} missing")
if len(data["release_commit"]) != 40 or any(char not in "0123456789abcdef" for char in data["release_commit"]):
raise SystemExit("release_commit must be a full lowercase git SHA")
if not isinstance(data.get("source_date_epoch"), int):
raise SystemExit("source_date_epoch missing")
if not isinstance(data.get("checks"), list):
raise SystemExit("checks missing")
if not isinstance(data.get("artifacts"), list):
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env bash
set -euo pipefail
# Production-safe primary recovery loop for DetMir ActivityWatch.
#
# Scope:
# - Detects ActivityWatch API poisoned datastore lock.
# - Pauses dependent secondary jobs before restarting the primary AW service.
# - Restarts only activitywatch-server inside the configured Proxmox CT.
# - Writes structured incident evidence.
#
# Non-goals:
# - Does not delete SQLite, journal or lock files.
# - Does not enable DLP, Loki or Velociraptor.
# - Does not restart Windows collectors.
AW_URL="${DETMIR_AW_RECOVERY_URL:-http://10.10.10.13:5600}"
AW_CT_ID="${DETMIR_AW_RECOVERY_CT_ID:-203}"
AW_SERVICE="${DETMIR_AW_RECOVERY_SERVICE:-activitywatch-server}"
STATE_DIR="${DETMIR_AW_RECOVERY_STATE_DIR:-/var/lib/detmir-aw-primary-recovery}"
HTTP_TIMEOUT_SECONDS="${DETMIR_AW_RECOVERY_HTTP_TIMEOUT_SECONDS:-8}"
CONFIRM_ATTEMPTS="${DETMIR_AW_RECOVERY_CONFIRM_ATTEMPTS:-2}"
CONFIRM_SLEEP_SECONDS="${DETMIR_AW_RECOVERY_CONFIRM_SLEEP_SECONDS:-5}"
STARTUP_TIMEOUT_SECONDS="${DETMIR_AW_RECOVERY_STARTUP_TIMEOUT_SECONDS:-120}"
STARTUP_SLEEP_SECONDS="${DETMIR_AW_RECOVERY_STARTUP_SLEEP_SECONDS:-5}"
COOLDOWN_SECONDS="${DETMIR_AW_RECOVERY_COOLDOWN_SECONDS:-900}"
DRY_RUN="${DETMIR_AW_RECOVERY_DRY_RUN:-0}"
PVE_PAUSE_UNITS="${DETMIR_AW_RECOVERY_PVE_PAUSE_UNITS:-aw-workforce-ingest.timer aw-workforce-ingest.service}"
PVE_RESUME_UNITS="${DETMIR_AW_RECOVERY_PVE_RESUME_UNITS:-aw-workforce-ingest.timer}"
CT_PAUSE_UNITS="${DETMIR_AW_RECOVERY_CT_PAUSE_UNITS:-aw-worktime-autoheal.timer aw-worktime-autoheal.service aw-worktime-prewarm.timer aw-worktime-prewarm.service aw-worktime-ui-bridge.timer aw-worktime-ui-bridge.service aw-worktime-influx-exporter.timer aw-worktime-influx-exporter.service aw-rus-healthd.timer aw-rus-healthd.service aw-slo-monitor.timer aw-slo-monitor.service}"
CT_RESUME_UNITS="${DETMIR_AW_RECOVERY_CT_RESUME_UNITS:-aw-worktime-autoheal.timer aw-worktime-prewarm.timer aw-worktime-ui-bridge.timer aw-worktime-influx-exporter.timer aw-rus-healthd.timer aw-slo-monitor.timer}"
usage() {
cat <<'EOF'
Usage:
detmir-aw-primary-recovery.sh [--once|--check-only|--self-test]
Environment:
DETMIR_AW_RECOVERY_URL default: http://10.10.10.13:5600
DETMIR_AW_RECOVERY_CT_ID default: 203
DETMIR_AW_RECOVERY_SERVICE default: activitywatch-server
DETMIR_AW_RECOVERY_STATE_DIR default: /var/lib/detmir-aw-primary-recovery
DETMIR_AW_RECOVERY_CONFIRM_ATTEMPTS default: 2
DETMIR_AW_RECOVERY_COOLDOWN_SECONDS default: 900
DETMIR_AW_RECOVERY_DRY_RUN default: 0
DETMIR_AW_RECOVERY_PVE_PAUSE_UNITS space-separated units on Proxmox host
DETMIR_AW_RECOVERY_PVE_RESUME_UNITS space-separated units on Proxmox host
DETMIR_AW_RECOVERY_CT_PAUSE_UNITS space-separated units inside AW CT
DETMIR_AW_RECOVERY_CT_RESUME_UNITS space-separated units inside AW CT
The script never removes ActivityWatch SQLite, lock or journal files.
EOF
}
log() {
printf 'ts=%s component=detmir-aw-primary-recovery %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
}
require_command() {
local cmd="$1"
if ! command -v "$cmd" >/dev/null 2>&1; then
log "level=error event=missing_command command=$cmd"
exit 127
fi
}
json_string_array() {
jq -Rsc 'split(" ") | map(select(length > 0))' <<<"$1"
}
record_incident() {
local outcome="$1"
local reason="$2"
local recovered="$3"
local started_at="$4"
local finished_at="$5"
local details="$6"
local incident_id
incident_id="$(date -u +%Y%m%dT%H%M%SZ)-$$"
local incident_path="${STATE_DIR}/incidents/${incident_id}.json"
install -d -m 0750 "${STATE_DIR}/incidents"
jq -n \
--arg generated_at_utc "$finished_at" \
--arg started_at_utc "$started_at" \
--arg outcome "$outcome" \
--arg reason "$reason" \
--arg recovered "$recovered" \
--arg aw_url "$AW_URL" \
--arg aw_ct_id "$AW_CT_ID" \
--arg aw_service "$AW_SERVICE" \
--arg details "$details" \
--argjson pve_pause_units "$(json_string_array "$PVE_PAUSE_UNITS")" \
--argjson ct_pause_units "$(json_string_array "$CT_PAUSE_UNITS")" \
'{
generated_at_utc: $generated_at_utc,
started_at_utc: $started_at_utc,
outcome: $outcome,
reason: $reason,
recovered: ($recovered == "true"),
aw_url: $aw_url,
aw_ct_id: ($aw_ct_id | tonumber? // $aw_ct_id),
aw_service: $aw_service,
details: $details,
pve_pause_units: $pve_pause_units,
ct_pause_units: $ct_pause_units
}' >"$incident_path"
ln -sfn "$incident_path" "${STATE_DIR}/latest.json"
log "level=info event=incident_written path=$incident_path outcome=$outcome reason=$reason recovered=$recovered"
}
classify_http_response() {
local code="$1"
local body_file="$2"
if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
printf 'ok'
return 0
fi
if grep -Eiq 'poisoned lock|Taking datastore lock failed|datastore lock failed' "$body_file"; then
printf 'poisoned_lock'
return 0
fi
printf 'http_%s' "$code"
}
probe_path() {
local path="$1"
local body_file
body_file="$(mktemp)"
local code rc
code="$(curl -sS --max-time "$HTTP_TIMEOUT_SECONDS" -o "$body_file" -w '%{http_code}' "${AW_URL}${path}" 2>/dev/null)" || rc=$?
rc="${rc:-0}"
if [[ "$rc" -ne 0 ]]; then
rm -f "$body_file"
printf 'timeout'
return 0
fi
local status
status="$(classify_http_response "$code" "$body_file")"
rm -f "$body_file"
printf '%s' "$status"
}
ct_systemctl() {
pct exec "$AW_CT_ID" -- systemctl "$@"
}
activitywatch_service_status() {
if ! pct status "$AW_CT_ID" >/dev/null 2>&1; then
printf 'ct_unavailable'
return 0
fi
if ct_systemctl is-active --quiet "$AW_SERVICE"; then
printf 'active'
else
printf 'inactive'
fi
}
probe_aw() {
local service_state settings_status buckets_status
service_state="$(activitywatch_service_status)"
if [[ "$service_state" != "active" ]]; then
printf 'service_%s' "$service_state"
return 0
fi
settings_status="$(probe_path '/api/0/settings/')"
buckets_status="$(probe_path '/api/0/buckets/')"
if [[ "$settings_status" == "ok" && "$buckets_status" == "ok" ]]; then
printf 'ok'
elif [[ "$settings_status" == "poisoned_lock" || "$buckets_status" == "poisoned_lock" ]]; then
printf 'poisoned_lock'
else
printf '%s,%s' "$settings_status" "$buckets_status"
fi
}
stop_units_on_host() {
local units="$1"
local unit
for unit in $units; do
log "level=info event=stop_unit scope=pve unit=$unit"
systemctl stop "$unit" >/dev/null 2>&1 || true
done
}
start_units_on_host() {
local units="$1"
local unit
for unit in $units; do
log "level=info event=start_unit scope=pve unit=$unit"
systemctl start "$unit" >/dev/null 2>&1 || true
done
}
stop_units_in_ct() {
local units="$1"
local unit
for unit in $units; do
log "level=info event=stop_unit scope=ct ct=$AW_CT_ID unit=$unit"
ct_systemctl stop "$unit" >/dev/null 2>&1 || true
done
}
start_units_in_ct() {
local units="$1"
local unit
for unit in $units; do
log "level=info event=start_unit scope=ct ct=$AW_CT_ID unit=$unit"
ct_systemctl start "$unit" >/dev/null 2>&1 || true
done
}
cooldown_active() {
local last_file="${STATE_DIR}/last_restart_epoch"
[[ -f "$last_file" ]] || return 1
local now last
now="$(date -u +%s)"
last="$(cat "$last_file" 2>/dev/null || printf '0')"
[[ "$last" =~ ^[0-9]+$ ]] || return 1
(( now - last < COOLDOWN_SECONDS ))
}
wait_for_aw_ok() {
local deadline now status
deadline=$(( $(date -u +%s) + STARTUP_TIMEOUT_SECONDS ))
while true; do
status="$(probe_aw)"
if [[ "$status" == "ok" ]]; then
return 0
fi
now="$(date -u +%s)"
if (( now >= deadline )); then
log "level=error event=wait_for_aw_timeout last_status=$status"
return 1
fi
log "level=info event=wait_for_aw status=$status"
sleep "$STARTUP_SLEEP_SECONDS"
done
}
run_recovery() {
local reason="$1"
local started_at="$2"
if cooldown_active; then
local finished_at
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
record_incident "skipped" "cooldown_active_after_${reason}" "false" "$started_at" "$finished_at" "restart suppressed by cooldown"
return 0
fi
if [[ "$DRY_RUN" == "1" ]]; then
local finished_at
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
record_incident "dry_run" "$reason" "false" "$started_at" "$finished_at" "dry run requested"
return 0
fi
log "level=warn event=recovery_start reason=$reason ct=$AW_CT_ID service=$AW_SERVICE"
stop_units_on_host "$PVE_PAUSE_UNITS"
stop_units_in_ct "$CT_PAUSE_UNITS"
log "level=warn event=restart_primary ct=$AW_CT_ID service=$AW_SERVICE"
ct_systemctl restart "$AW_SERVICE"
printf '%s\n' "$(date -u +%s)" >"${STATE_DIR}/last_restart_epoch"
local finished_at
if wait_for_aw_ok; then
start_units_in_ct "$CT_RESUME_UNITS"
start_units_on_host "$PVE_RESUME_UNITS"
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
record_incident "recovered" "$reason" "true" "$started_at" "$finished_at" "primary restarted and AW API returned ok"
return 0
fi
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
record_incident "failed" "$reason" "false" "$started_at" "$finished_at" "primary restart did not restore AW API; secondary timers left paused"
return 1
}
check_once() {
install -d -m 0750 "$STATE_DIR"
local started_at status attempt reason
started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
reason=""
for attempt in $(seq 1 "$CONFIRM_ATTEMPTS"); do
status="$(probe_aw)"
log "level=info event=probe attempt=$attempt status=$status"
case "$status" in
ok)
jq -n \
--arg generated_at_utc "$started_at" \
--arg status "ok" \
--arg aw_url "$AW_URL" \
'{generated_at_utc: $generated_at_utc, status: $status, aw_url: $aw_url}' \
>"${STATE_DIR}/status.json"
return 0
;;
poisoned_lock|service_inactive|service_ct_unavailable)
reason="$status"
;;
*)
local finished_at
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
record_incident "observed_no_action" "$status" "false" "$started_at" "$finished_at" "unhealthy state is not in automatic recovery allowlist"
return 0
;;
esac
if [[ "$attempt" -lt "$CONFIRM_ATTEMPTS" ]]; then
sleep "$CONFIRM_SLEEP_SECONDS"
fi
done
run_recovery "$reason" "$started_at"
}
self_test() {
local tmp
tmp="$(mktemp)"
printf '{"message":"Taking datastore lock failed, returning 504: poisoned lock: another task failed inside"}' >"$tmp"
[[ "$(classify_http_response 503 "$tmp")" == "poisoned_lock" ]]
printf '{"ok":true}' >"$tmp"
[[ "$(classify_http_response 200 "$tmp")" == "ok" ]]
printf '{"message":"other"}' >"$tmp"
[[ "$(classify_http_response 503 "$tmp")" == "http_503" ]]
rm -f "$tmp"
log "level=info event=self_test status=ok"
}
main() {
local mode="${1:---once}"
case "$mode" in
--help|-h)
usage
;;
--self-test)
require_command jq
self_test
;;
--check-only)
require_command curl
require_command jq
require_command pct
status="$(probe_aw)"
log "level=info event=check_only status=$status"
[[ "$status" == "ok" ]]
;;
--once)
require_command curl
require_command jq
require_command pct
require_command flock
install -d -m 0750 "$STATE_DIR"
exec 9>"${STATE_DIR}/lock"
if ! flock -n 9; then
log "level=warn event=lock_busy"
exit 0
fi
check_once
;;
*)
usage >&2
exit 2
;;
esac
}
main "$@"
+3 -1
View File
@@ -24,7 +24,9 @@ function authHeaders() {
}
function normalizeBaseUrl(raw) {
const value = raw.endsWith("/") ? raw : `${raw}/`;
const trimmed = String(raw || "").trim();
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `http://${trimmed}`;
const value = withScheme.endsWith("/") ? withScheme : `${withScheme}/`;
return new URL(value);
}
+23 -2
View File
@@ -31,6 +31,12 @@ function env(name, fallback = "") {
return value && value.trim() ? value.trim() : fallback;
}
function normalizeBaseUrl(raw) {
const trimmed = String(raw || "").trim();
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `http://${trimmed}`;
return withScheme.endsWith("/") ? withScheme : `${withScheme}/`;
}
function authHeaders() {
const explicit = env("DETMIR_PORTAL_SMOKE_AUTH_HEADER");
if (explicit) return { Authorization: explicit };
@@ -61,6 +67,16 @@ function expectedSecurityEventsText(mode) {
return "";
}
function isExpectedProtectedResponse(response) {
if (response.status() !== 403) return false;
try {
const { pathname } = new URL(response.url());
return pathname.endsWith("/api/security/findings");
} catch {
return false;
}
}
function forbiddenExecutiveTerms(text) {
const forbidden = [
"Trust KPI",
@@ -126,7 +142,7 @@ async function main() {
}
const { chromium } = loadPlaywright();
const url = env("DETMIR_PORTAL_SMOKE_URL", "http://127.0.0.1:8720/portal/");
const url = normalizeBaseUrl(env("DETMIR_PORTAL_SMOKE_URL", "http://127.0.0.1:8720/portal/"));
const timeout = Number(env("DETMIR_PORTAL_SMOKE_TIMEOUT_MS", "30000"));
const browser = await chromium.launch({
headless: true,
@@ -142,9 +158,14 @@ async function main() {
const badResponses = [];
page.on("pageerror", (error) => errors.push(error.message));
page.on("console", (message) => {
if (message.type() === "error") errors.push(message.text());
if (message.type() !== "error") return;
const text = message.text();
if (text.includes("/api/security/findings") && text.includes("403")) return;
if (text === "Failed to load resource: the server responded with a status of 403 ()") return;
errors.push(text);
});
page.on("response", (response) => {
if (isExpectedProtectedResponse(response)) return;
if (response.status() >= 400) badResponses.push({ url: response.url(), status: response.status() });
});
+2 -2
View File
@@ -106,9 +106,9 @@ DETMIR_SUPPORT_SKIP_REMOTE="${DETMIR_SUPPORT_SKIP_REMOTE:-0}"
DETMIR_SUPPORT_CONNECT_TIMEOUT="${DETMIR_SUPPORT_CONNECT_TIMEOUT:-8}"
DETMIR_SUPPORT_PVE_SERVICES="${DETMIR_SUPPORT_PVE_SERVICES:-pve-cluster pvedaemon pvestatd pveproxy pvedaemon.service pvestatd.service}"
DETMIR_SUPPORT_AW_SERVICES="${DETMIR_SUPPORT_AW_SERVICES:-activitywatch-server aw-server-rust}"
DETMIR_SUPPORT_AW_SERVICES="${DETMIR_SUPPORT_AW_SERVICES:-activitywatch-server}"
DETMIR_SUPPORT_WEB_SERVICES="${DETMIR_SUPPORT_WEB_SERVICES:-nginx}"
DETMIR_SUPPORT_SURICATA_SERVICES="${DETMIR_SUPPORT_SURICATA_SERVICES:-suricata}"
DETMIR_SUPPORT_SURICATA_SERVICES="${DETMIR_SUPPORT_SURICATA_SERVICES:-}"
DETMIR_SUPPORT_PVE_VM_IDS="${DETMIR_SUPPORT_PVE_VM_IDS:-}"
DETMIR_SUPPORT_PVE_BACKUP_DIRS="${DETMIR_SUPPORT_PVE_BACKUP_DIRS:-/var/lib/pve/local-btrfs/dump}"
DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS="${DETMIR_SUPPORT_BACKUP_MAX_AGE_DAYS:-14}"
+3 -2
View File
@@ -29,9 +29,10 @@ DETMIR_SUPPORT_CONNECT_TIMEOUT=8
## Сервисы и параметры
DETMIR_SUPPORT_PVE_SERVICES=pve-cluster pvedaemon pvestatd pveproxy
DETMIR_SUPPORT_AW_SERVICES=activitywatch-server aw-server-rust
DETMIR_SUPPORT_AW_SERVICES=activitywatch-server
DETMIR_SUPPORT_WEB_SERVICES=nginx
DETMIR_SUPPORT_SURICATA_SERVICES=suricata
# Suricata is optional for the current light DetMir profile.
DETMIR_SUPPORT_SURICATA_SERVICES=
## VM и резервные копии
DETMIR_SUPPORT_PVE_VM_IDS="101 102 103"
File diff suppressed because it is too large Load Diff
+487
View File
@@ -0,0 +1,487 @@
#!/usr/bin/env node
import fs from "node:fs";
import http from "node:http";
import net from "node:net";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const manifestPath = "configs/operational-maturity-contract.json";
function readText(relativePath) {
return fs.readFileSync(path.join(root, relativePath), "utf8");
}
function readJson(relativePath) {
return JSON.parse(readText(relativePath));
}
function exists(relativePath) {
return fs.existsSync(path.join(root, relativePath));
}
function assert(condition, message) {
if (!condition) throw new Error(message);
}
function getPath(value, dotted) {
return dotted.split(".").reduce((current, key) => {
if (current === null || current === undefined) return undefined;
return current[key];
}, value);
}
function result(name, fn) {
const started = Date.now();
try {
const details = fn() || {};
return { name, ok: true, duration_ms: Date.now() - started, ...details };
} catch (error) {
return { name, ok: false, duration_ms: Date.now() - started, error: error.message };
}
}
async function asyncResult(name, fn) {
const started = Date.now();
try {
const details = (await fn()) || {};
return { name, ok: true, duration_ms: Date.now() - started, ...details };
} catch (error) {
return { name, ok: false, duration_ms: Date.now() - started, error: error.message };
}
}
function walk(dir, predicate = () => true) {
const base = path.join(root, dir);
if (!fs.existsSync(base)) return [];
const files = [];
for (const entry of fs.readdirSync(base, { withFileTypes: true })) {
const absolute = path.join(base, entry.name);
const relative = path.relative(root, absolute).replaceAll(path.sep, "/");
if (entry.isDirectory()) {
files.push(...walk(relative, predicate));
} else if (predicate(relative)) {
files.push(relative);
}
}
return files.sort();
}
function parseArgs() {
const args = new Set(process.argv.slice(2));
return {
json: args.has("--json"),
live: args.has("--live"),
failOnDuplicateDebt: args.has("--fail-on-duplicate-debt"),
};
}
function randomLocalPort() {
return new Promise((resolve, reject) => {
const server = net.createServer();
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
const port = typeof address === "object" && address ? address.port : 0;
server.close(() => resolve(port));
});
});
}
function fixtureResponse(fixture) {
if (fixture.contentType === "text/plain") {
return readText(fixture.fixture);
}
return JSON.stringify(readJson(fixture.fixture));
}
async function startFixtureServer(manifest) {
const fixturesByPath = new Map();
for (const fixture of manifest.integrationFixtures) {
fixturesByPath.set(fixture.path, fixture);
}
fixturesByPath.set("/metrics", {
fixture: manifest.observability.metricsFixture,
status: 200,
contentType: "text/plain",
});
const sockets = new Set();
const server = http.createServer((request, response) => {
const parsed = new URL(request.url || "/", "http://127.0.0.1");
const fixture = fixturesByPath.get(`${parsed.pathname}${parsed.search}`) || fixturesByPath.get(parsed.pathname);
if (!fixture) {
response.statusCode = 404;
response.setHeader("Content-Type", "application/json; charset=utf-8");
response.end(JSON.stringify({ error: "not_found", path: parsed.pathname }));
return;
}
response.statusCode = fixture.status || 200;
response.setHeader("Content-Type", fixture.contentType === "text/plain" ? "text/plain; charset=utf-8" : "application/json; charset=utf-8");
response.setHeader("X-Request-Id", request.headers["x-request-id"] || "operational-maturity-fixture");
response.setHeader("X-Correlation-Id", request.headers["x-request-id"] || "operational-maturity-fixture");
response.end(fixtureResponse(fixture));
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
});
const port = await randomLocalPort();
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, "127.0.0.1", resolve);
});
return {
port,
close: () =>
new Promise((resolve) => {
for (const socket of sockets) socket.destroy();
server.close(resolve);
}),
};
}
async function fetchWithTimeout(url, timeoutMs, options = {}) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
const started = Date.now();
try {
const response = await fetch(url, { ...options, signal: controller.signal });
const text = await response.text();
let json = null;
try {
json = text ? JSON.parse(text) : null;
} catch {
json = null;
}
return {
status: response.status,
ok: response.ok,
headers: response.headers,
text,
json,
elapsed_ms: Date.now() - started,
};
} finally {
clearTimeout(timeout);
}
}
function checkApiCompatibility(manifest) {
const spec = readJson(manifest.apiCompatibility.openapiPath);
const missing = [];
for (const [apiPath, methods] of Object.entries(manifest.apiCompatibility.requiredOpenApiPaths)) {
for (const method of methods) {
if (!spec.paths?.[apiPath]?.[method]) missing.push(`${method.toUpperCase()} ${apiPath}`);
}
}
for (const schema of manifest.apiCompatibility.requiredSchemas) {
if (!spec.components?.schemas?.[schema]) missing.push(`schema ${schema}`);
}
const runtimeSource = readText(manifest.apiCompatibility.runtimeSourcePath);
for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) {
if (!runtimeSource.includes(`"${endpoint}"`)) missing.push(`runtime endpoint ${endpoint}`);
}
assert(missing.length === 0, `missing API compatibility anchors: ${missing.join(", ")}`);
return {
required_paths: Object.keys(manifest.apiCompatibility.requiredOpenApiPaths).length,
required_schemas: manifest.apiCompatibility.requiredSchemas.length,
runtime_endpoints: manifest.apiCompatibility.runtimeEndpoints.length,
};
}
function validateFixtureContract(manifest) {
const checked = [];
for (const fixture of manifest.integrationFixtures) {
const payload = readJson(fixture.fixture);
for (const field of fixture.requiredFields || []) {
assert(getPath(payload, field) !== undefined, `${fixture.fixture} missing field ${field}`);
}
for (const [field, expected] of Object.entries(fixture.equals || {})) {
assert(getPath(payload, field) === expected, `${fixture.fixture} expected ${field}=${JSON.stringify(expected)}`);
}
checked.push(fixture.path);
}
return { fixtures: checked.length };
}
async function checkIntegrationHarness(manifest) {
const server = await startFixtureServer(manifest);
const checked = [];
try {
for (const fixture of manifest.integrationFixtures) {
const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${fixture.path}`, 1500, {
headers: { "X-Request-Id": "operational-maturity" },
});
assert(response.status === fixture.status, `${fixture.path} status ${response.status}, expected ${fixture.status}`);
if (fixture.contentType === "application/json") {
assert(response.json && typeof response.json === "object", `${fixture.path} did not return JSON`);
}
assert(response.headers.get("x-correlation-id") === "operational-maturity", `${fixture.path} did not echo correlation id`);
checked.push({ path: fixture.path, elapsed_ms: response.elapsed_ms });
}
return { endpoints: checked.length, max_elapsed_ms: Math.max(...checked.map((item) => item.elapsed_ms)) };
} finally {
await server.close();
}
}
async function startFaultServer() {
const sockets = new Set();
const server = http.createServer((request, response) => {
const parsed = new URL(request.url || "/", "http://127.0.0.1");
if (parsed.pathname === "/fault/503") {
response.statusCode = 503;
response.end("unavailable");
return;
}
if (parsed.pathname === "/fault/slow") {
setTimeout(() => {
response.statusCode = 200;
response.end("slow-ok");
}, 1200);
return;
}
if (parsed.pathname === "/fault/reset") {
request.socket.destroy();
return;
}
response.statusCode = 200;
response.end("ok");
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
});
const port = await randomLocalPort();
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(port, "127.0.0.1", resolve);
});
return {
port,
close: () =>
new Promise((resolve) => {
for (const socket of sockets) socket.destroy();
server.close(resolve);
}),
};
}
async function checkFaultInjection(manifest) {
const server = await startFaultServer();
const timeoutMs = manifest.faultInjection.clientTimeoutMs;
const maxClassifyMs = manifest.faultInjection.maxFailureClassificationMs;
const classifications = [];
try {
const unavailable = await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/503`, timeoutMs);
assert(unavailable.status === 503, "503 fault was not classified as HTTP 503");
classifications.push("http_503");
const slowStart = Date.now();
try {
await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/slow`, timeoutMs);
throw new Error("slow fault did not timeout");
} catch (error) {
const elapsed = Date.now() - slowStart;
assert(elapsed < maxClassifyMs, `timeout classification exceeded ${maxClassifyMs}ms`);
classifications.push("timeout");
}
try {
await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/reset`, timeoutMs);
throw new Error("connection reset fault did not fail");
} catch {
classifications.push("connection_reset");
}
return { classifications };
} finally {
await server.close();
}
}
function percentile(values, pct) {
const sorted = [...values].sort((a, b) => a - b);
const index = Math.min(sorted.length - 1, Math.ceil((pct / 100) * sorted.length) - 1);
return sorted[index] || 0;
}
async function mapLimit(items, limit, fn) {
const results = [];
let next = 0;
async function worker() {
while (next < items.length) {
const index = next;
next += 1;
results[index] = await fn(items[index], index);
}
}
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
return results;
}
async function checkBoundedLoad(manifest) {
const server = await startFixtureServer(manifest);
const cfg = manifest.boundedLoad;
const heapBefore = process.memoryUsage().heapUsed;
try {
const work = Array.from({ length: cfg.requests }, (_, index) => cfg.paths[index % cfg.paths.length]);
const responses = await mapLimit(work, cfg.concurrency, async (apiPath) => {
const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${apiPath}`, 1500);
assert(response.status >= 200 && response.status < 300, `${apiPath} failed with ${response.status}`);
return response.elapsed_ms;
});
const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore);
const p95 = percentile(responses, 95);
assert(p95 <= cfg.p95MaxMs, `bounded load p95 ${p95}ms exceeds ${cfg.p95MaxMs}ms`);
assert(heapGrowth <= cfg.heapGrowthMaxBytes, `heap growth ${heapGrowth} exceeds ${cfg.heapGrowthMaxBytes}`);
return { requests: cfg.requests, concurrency: cfg.concurrency, p95_ms: p95, heap_growth_bytes: heapGrowth };
} finally {
await server.close();
}
}
function validateJsonAndYaml(manifest) {
const checked = [];
for (const file of manifest.configValidation.jsonFiles) {
assert(exists(file), `missing JSON config/fixture ${file}`);
readJson(file);
checked.push(file);
}
for (const file of manifest.configValidation.yamlFiles) {
assert(exists(file), `missing YAML config ${file}`);
const text = readText(file);
assert(!text.includes("\t"), `${file} contains tabs`);
if (!/(\.example\.ya?ml|example\.yml)$/i.test(file)) {
assert(!/password:\s*(change-me|changeme|password)\b/i.test(text), `${file} contains unsafe password placeholder`);
}
checked.push(file);
}
return { files: checked.length };
}
function validateSystemdUnits(manifest) {
const files = manifest.configValidation.systemdDirs.flatMap((dir) =>
walk(dir, (file) => file.endsWith(".service") || file.endsWith(".timer")),
);
assert(files.length > 0, "no systemd files found");
const findings = [];
for (const file of files) {
const text = readText(file);
if (!text.includes("[Unit]")) findings.push(`${file}: missing [Unit]`);
if (file.endsWith(".service")) {
if (!text.includes("[Service]")) findings.push(`${file}: missing [Service]`);
if (!/^ExecStart=/m.test(text)) findings.push(`${file}: missing ExecStart`);
if (/^Type=oneshot$/m.test(text) && /^Restart=always$/m.test(text)) {
findings.push(`${file}: oneshot service must not use Restart=always`);
}
}
if (file.endsWith(".timer")) {
if (!text.includes("[Timer]")) findings.push(`${file}: missing [Timer]`);
if (!/^Unit=.*\.service$/m.test(text)) findings.push(`${file}: missing service Unit`);
if (!text.includes("[Install]")) findings.push(`${file}: missing [Install]`);
}
}
assert(findings.length === 0, findings.join("; "));
return { files: files.length };
}
function validateClickHouseMigrations(manifest) {
const dir = manifest.configValidation.clickhouseInitDir;
const files = walk(dir, (file) => file.endsWith(".sql"));
assert(files.length > 0, "no ClickHouse init SQL files found");
const findings = [];
const order = files.map((file) => path.basename(file));
const sorted = [...order].sort();
if (order.join("|") !== sorted.join("|")) findings.push("ClickHouse init filenames are not sorted");
for (const file of files) {
const text = readText(file);
if (/\b(DROP|TRUNCATE)\s+(DATABASE|TABLE)\b/i.test(text)) {
findings.push(`${file}: destructive DDL is not allowed in init validation`);
}
const createStatements = text.match(/\bCREATE\s+(DATABASE|TABLE|VIEW|OR\s+REPLACE\s+VIEW)\b/gi) || [];
for (const create of createStatements) {
if (/CREATE\s+(DATABASE|TABLE)/i.test(create) && !/IF\s+NOT\s+EXISTS/i.test(text.slice(text.indexOf(create), text.indexOf(create) + 140))) {
findings.push(`${file}: CREATE DATABASE/TABLE must use IF NOT EXISTS`);
}
if (/CREATE\s+VIEW/i.test(create) && !/CREATE\s+(OR\s+REPLACE\s+)?VIEW/i.test(create)) {
findings.push(`${file}: CREATE VIEW must be OR REPLACE or otherwise idempotent`);
}
}
}
assert(findings.length === 0, findings.join("; "));
return { files: files.length };
}
function checkObservability(manifest) {
const metricsText = readText(manifest.observability.metricsFixture);
const sourceText = readText(manifest.observability.sourcePath);
const missing = [];
for (const metric of manifest.observability.requiredMetrics) {
if (!metricsText.includes(metric)) missing.push(`fixture metric ${metric}`);
if (!sourceText.includes(metric)) missing.push(`source metric ${metric}`);
}
const hardeningSmoke = readText("scripts/awatch-production-hardening-smoke.mjs");
for (const header of manifest.observability.diagnosticHeaders) {
if (!hardeningSmoke.includes(header)) missing.push(`diagnostic header ${header}`);
}
assert(missing.length === 0, `missing observability anchors: ${missing.join(", ")}`);
return { metrics: manifest.observability.requiredMetrics.length };
}
async function checkLive(manifest) {
const baseUrl = (process.env.AWATCH_OPS_LIVE_URL || "").replace(/\/+$/, "");
assert(baseUrl, "AWATCH_OPS_LIVE_URL is required for --live");
const checked = [];
for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) {
const response = await fetchWithTimeout(`${baseUrl}${endpoint}`, 3000, {
headers: { "X-Request-Id": "operational-maturity-live" },
});
assert([200, 503].includes(response.status), `${endpoint} returned uncontrolled status ${response.status}`);
checked.push({ endpoint, status: response.status, elapsed_ms: response.elapsed_ms });
}
return { baseUrl, checked };
}
async function main() {
const args = parseArgs();
const manifest = readJson(manifestPath);
const checks = [
result("api_compatibility", () => checkApiCompatibility(manifest)),
result("fixture_contracts", () => validateFixtureContract(manifest)),
await asyncResult("integration_harness", () => checkIntegrationHarness(manifest)),
await asyncResult("fault_injection", () => checkFaultInjection(manifest)),
await asyncResult("bounded_load", () => checkBoundedLoad(manifest)),
result("json_yaml_config_validation", () => validateJsonAndYaml(manifest)),
result("systemd_config_validation", () => validateSystemdUnits(manifest)),
result("clickhouse_migration_validation", () => validateClickHouseMigrations(manifest)),
result("observability_contract", () => checkObservability(manifest)),
];
if (args.live) {
checks.push(await asyncResult("live_operational_contract", () => checkLive(manifest)));
}
const report = {
ok: checks.every((check) => check.ok),
mode: args.live ? "live" : "offline",
manifest: manifest.version,
checks,
};
if (args.json) {
console.log(JSON.stringify(report, null, 2));
} else {
for (const check of checks) {
console.log(`${check.ok ? "ok" : "fail"} ${check.name} ${check.duration_ms}ms`);
if (!check.ok) console.log(` ${check.error}`);
}
}
if (!report.ok) process.exit(1);
}
main().catch((error) => {
console.error(JSON.stringify({ ok: false, error: error.message }, null, 2));
process.exit(1);
});
+15 -1
View File
@@ -6,6 +6,7 @@ from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import stat
import tarfile
@@ -84,6 +85,18 @@ def create_compatibility_aliases(out_dir: Path, archive: Path) -> None:
write_archive_checksum(archive_alias)
def build_time_utc() -> str:
source_date_epoch = os.environ.get("SOURCE_DATE_EPOCH")
if source_date_epoch:
try:
stamp = datetime.fromtimestamp(int(source_date_epoch), timezone.utc)
except ValueError as exc:
raise SystemExit("SOURCE_DATE_EPOCH must be an integer Unix timestamp") from exc
else:
stamp = datetime.now(timezone.utc)
return stamp.isoformat(timespec="seconds").replace("+00:00", "Z")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--release-dir", type=Path, required=True)
@@ -127,7 +140,8 @@ def main() -> None:
"commit": args.commit,
"ref": args.ref,
"run_id": args.run_id,
"build_time_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"source_date_epoch": os.environ.get("SOURCE_DATE_EPOCH", ""),
"build_time_utc": build_time_utc(),
"binaries": manifest_binaries,
}
write(out_dir / "BUILD_MANIFEST.json", json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")
+76
View File
@@ -66,6 +66,81 @@ require_live_value() {
esac
}
normalize_http_base() {
local value="${1:-}"
value="${value%/}"
case "${value}" in
"") return 1 ;;
http://*|https://*) printf '%s' "${value}" ;;
*) printf 'http://%s' "${value}" ;;
esac
}
normalize_url_env() {
local name="$1"
local value="${!name:-}"
if [[ -n "${value}" ]]; then
export "${name}=$(normalize_http_base "${value}")"
fi
}
portal_smoke_url_from_portal_url() {
local value="${1%/}"
case "${value}" in
*/portal) printf '%s/\n' "${value}" ;;
*) printf '%s/portal/\n' "${value}" ;;
esac
}
configure_smoke_env() {
normalize_url_env DETMIR_AW_API
normalize_url_env DETMIR_WORKTIME_URL
normalize_url_env DETMIR_ONE_C_URL
normalize_url_env DETMIR_PORTAL_URL
export \
DETMIR_AW_API \
DETMIR_WORKTIME_URL \
DETMIR_ONE_C_URL \
DETMIR_RDP_HOST \
DETMIR_HOSTNAME \
DETMIR_GATEWAY_HOST \
DETMIR_PORTAL_URL \
DETMIR_DLP_ENABLED \
DETMIR_DISABLE_PORTAL_CHECK \
DETMIR_DISABLE_DLP_HEALTH_CHECK \
DETMIR_PCT_BIN \
DETMIR_GRAFANA_CHECK_JSON \
DETMIR_PORTAL_SMOKE_BASIC_AUTH \
DETMIR_BASIC_AUTH \
DETMIR_PORTAL_SMOKE_AUTH_HEADER \
DETMIR_PORTAL_AUTH_HEADER
case "${AWATCH_PORTAL_SMOKE_URL:-}" in
""|http://127.0.0.1:8720*)
export AWATCH_PORTAL_SMOKE_URL="${DETMIR_PORTAL_URL}"
;;
esac
case "${DETMIR_PORTAL_SMOKE_URL:-}" in
""|http://127.0.0.1:8720*)
DETMIR_PORTAL_SMOKE_URL="$(portal_smoke_url_from_portal_url "${DETMIR_PORTAL_URL}")"
;;
esac
export AWATCH_PORTAL_SMOKE_URL DETMIR_PORTAL_SMOKE_URL
if [[ "${DETMIR_PORTAL_URL}" == https://* && -z "${DETMIR_PORTAL_SMOKE_INSECURE_TLS:-}" ]]; then
export DETMIR_PORTAL_SMOKE_INSECURE_TLS=1
fi
export DETMIR_PORTAL_SMOKE_INSECURE_TLS
if [[ -z "${DETMIR_PORTAL_AUTH_HEADER:-}" && -n "${DETMIR_PORTAL_SMOKE_BASIC_AUTH:-}" ]]; then
export DETMIR_PORTAL_AUTH_HEADER="Basic ${DETMIR_PORTAL_SMOKE_BASIC_AUTH}"
fi
case "${DETMIR_DLP_ENABLED,,}" in
0|false|no|off)
export DETMIR_DISABLE_DLP_HEALTH_CHECK="${DETMIR_DISABLE_DLP_HEALTH_CHECK:-1}"
;;
esac
}
write_summary() {
{
printf '# AWatch-rus contour check\n\n'
@@ -89,6 +164,7 @@ write_summary() {
}
configure_detmir_env
configure_smoke_env
find_detmir_check() {
if [[ -n "${DETMIR_CHECK_BIN:-}" ]]; then
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
if ! command -v node >/dev/null 2>&1; then
echo "run_full_validation=fail node_not_found" >&2
exit 127
fi
run_preflight_self_test=1
for arg in "$@"; do
if [[ "$arg" == "--self-test" || "$arg" == "--help" || "$arg" == "-h" ]]; then
run_preflight_self_test=0
fi
done
if [[ "${AW_VALIDATION_SKIP_SELF_TEST:-0}" != "1" && "$run_preflight_self_test" == "1" ]]; then
node scripts/full-validation-orchestrator.mjs --self-test
fi
exec node scripts/full-validation-orchestrator.mjs "$@"
+9 -1
View File
@@ -594,7 +594,15 @@ $watcherCountsOk = $true
$endpointProcessOk = $true
$fileProcessOk = $true
$browserProcessOk = $true
$sessionCollectorOk = (@($sessionCollectorProcesses).Count -eq 1)
$sessionCollectorOk = if (@($sessionCollectorProcesses).Count -eq 1) {
$true
}
elseif (-not $liveSessionScopedCollectorsRequired) {
$true
}
else {
$false
}
$result = [ordered]@{
generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o')