Compare commits

..
Author SHA1 Message Date
igor04091968 518a501dee docs: switch Gitea runbooks to git.iri1968.ru
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-08-04 12:03:28 +03:00
igor04091968 af735ec1ea Respect disabled DLP in contour smoke
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-11 10:19:17 +03:00
igor04091968 cd25edc4f3 Fix DetMir light DLP readiness and collector guard
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-09 23:38:57 +03:00
igor04091968 597b462d39 Stabilize Hayabusa drop autoprocess 2026-07-09 00:26:40 +03:00
igor04091968 93f517fbf7 Stabilize DetMir contour checks 2026-07-05 23:18:25 +03:00
igor04091968 234afdd184 Fix DetMir portal gateway API mapping 2026-07-03 10:07:41 +03:00
igor04091968 08c67bde1a Add DetMir primary recovery guard 2026-07-03 08:51:11 +03:00
igor04091968 130d4cd94f Document and harden DetMir production maintenance 2026-07-03 00:31:28 +03:00
igor04091968 77c054dcad Document DetMir portal telemetry ingest repair 2026-07-02 10:33:11 +03:00
igor04091968 ce27a8170d Add production binary parity gate 2026-07-01 19:56:15 +03:00
igor04091968 d79b37fff5 Add production evidence gaps 2026-07-01 17:18:06 +03:00
igor04091968 e6378e87f5 Add build provenance to release evidence 2026-07-01 16:49:35 +03:00
igor04091968 b9dca66d1b Add disaster recovery proof 2026-07-01 16:33:04 +03:00
igor04091968 0c91d56a57 Add operational recovery audit 2026-07-01 15:23:36 +03:00
igor04091968 82691d7d92 Remove ClickHouse password from process argv 2026-07-01 14:41:59 +03:00
igor04091968 bb86939179 Add production retention and cleanup policy 2026-07-01 14:16:36 +03:00
igor04091968 0bc7186bbc Add production readiness audit 2026-07-01 12:08:19 +03:00
igor04091968 0ce831d56f Add next development plan audit 2026-07-01 09:14:18 +03:00
igor04091968 dcf14eae67 Codify production-first engineering governance
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 08:48:35 +03:00
igor04091968 121d66cfcf Add DetMir operational maturity harness
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Operational maturity / Offline operational maturity (push) Waiting to run
Operational maturity / Live operational contract (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 07:34:43 +03:00
igor04091968 2e1fe8618c Add Rust dependency hygiene CI
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Dependency hygiene / Unused dependency check (push) Waiting to run
Dependency hygiene / Dependency duplicate report (push) Waiting to run
Dependency hygiene / Dependency security policy (push) Waiting to run
Dependency hygiene / Cargo udeps nightly advisory (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 07:19:07 +03:00
igor04091968 6e99676011 Remove unused Rust dependencies
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 07:06:45 +03:00
igor04091968 388b61d903 Normalize DetMir Rust lockfile
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-07-01 06:23:18 +03:00
igor04091968 fe87c85a31 Harden DetMir DLP production runtime
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
- default DetMir DLP runtime to core_only/disabled with load-guard protection

- add fail-closed placeholder validation and runtime-scoped artifact checks

- document operator re-enable flow for light profile and guard rollback

- update prod docs, env examples, and Ansible DLP defaults
2026-07-01 00:05:23 +03:00
igor04091968 1149f5dfbd fix(grafana): restore worktime application details panels 2026-06-30 10:13:12 +03:00
IgorRachkovandGitHub 5ac7b694c6 Merge pull request #51 from igor04091968/docs/detmir-production-baseline-20260629
release-assets / sbom-and-release-assets (push) Waiting to run
rust-workspace / rust-workspace (push) Waiting to run
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
docs(detmir): record production restore baseline
2026-06-29 13:56:17 +03:00
igor04091968 c1e470ce4d docs(detmir): clarify optional dlp and velociraptor boundaries 2026-06-29 13:33:13 +03:00
igor04091968 f4bdcee459 fix(detmir): format check-aw-full defaults 2026-06-29 13:24:48 +03:00
igor04091968 583cfe7003 docs(detmir): record production restore baseline 2026-06-29 13:20:54 +03:00
igor04091968 34e017673a docs(governance): record protected PR workflow evidence
CI / Rust checks (push) Waiting to run
CI / Docs and registry checks (push) Waiting to run
CI / Smoke checks (push) Waiting to run
Coverage / Coverage baseline (push) Waiting to run
Security / Cargo audit (push) Waiting to run
Security / Cargo deny (push) Waiting to run
Security / Secret pattern check (push) Waiting to run
Security / Dependency review (push) Waiting to run
2026-06-24 01:11:54 +03:00
igor04091968 7e6c067adf docs(governance): prepare branch protection and PR review evidence 2026-06-23 23:00:16 +03:00
igor04091968 54129d82b6 docs(governance): record public issue tracker evidence 2026-06-23 22:49:13 +03:00
igor04091968 c2e0afae10 docs(governance): prepare public issue creation package 2026-06-23 22:10:51 +03:00
igor04091968 337da4bb5f feat(workforce): add ClickHouse workforce analytics contour 2026-06-23 21:52:57 +03:00
igor04091968 341799ff9d docs(clickhouse): plan dictionaries and aggregate sprints 2026-06-23 13:33:59 +03:00
igor04091968 77c705a649 docs(governance): add review process residual risks and issue plan 2026-06-22 21:42:43 +03:00
igor04091968 f310aeff49 docs(risk): document residual risks and public issue plan 2026-06-22 21:08:33 +03:00
igor04091968 44f5d6ecc2 docs(readme): update maturity assessment status 2026-06-22 21:00:11 +03:00
igor04091968 a0d268b416 docs(maturity): update public validation weaknesses 2026-06-22 20:51:22 +03:00
IgorRachkov a03829ba25 docs: add professional maturity assessment (Copilot review 2026-06-22) 2026-06-22 20:38:36 +03:00
igor04091968 d412f35644 docs(status): record public CI security coverage validation 2026-06-22 20:35:06 +03:00
igor04091968 cc0b2ff53d feat(detmir-check): add portal and DLP health checks 2026-06-22 01:02:52 +03:00
igor04091968 e893c296ac feat(contour): add local-friendly contour run visibility 2026-06-22 00:59:42 +03:00
igor04091968 18da4e7cfd docs: document gitea duplicate status 2026-06-22 00:47:32 +03:00
igor04091968 4f90aba2a1 chore(security): harden public secret scan and document policy 2026-06-21 14:17:05 +03:00
igor04091968 9f3278f0dc docs(status): freeze registry readiness project status 2026-06-21 13:57:00 +03:00
igor04091968 4970d31a81 chore(public): add CI coverage security and OSS process visibility 2026-06-21 08:54:53 +03:00
igor04091968 f5e9c81c5b docs(registry): add Russian build runner and release evidence plan 2026-06-21 07:10:43 +03:00
igor04091968 6861df9be4 docs(registry): document Russian Gitea contour and registry docs policy 2026-06-21 05:56:11 +03:00
igor04091968 4ba055fcdf docs(registry): document Russian Gitea contour and backup evidence 2026-06-21 05:11:42 +03:00
IgorRachkovandGitHub 8b25ec345e Update README.md 2026-06-20 16:51:08 +03:00
IgorRachkovandGitHub 09dd01f502 Update README.md 2026-06-20 16:36:54 +03:00
IgorRachkovandGitHub c241e49b5a Update README.md 2026-06-20 16:32:10 +03:00
IgorRachkovandGitHub 401d9612f0 Update README.md 2026-06-20 16:23:19 +03:00
IgorRachkovandGitHub 830f622627 Update README.md 2026-06-20 16:08:19 +03:00
igor04091968 cfeaf772ea fix(aw-db): add guarded sqlite vacuum maintenance 2026-06-20 11:04:10 +03:00
igor04091968 f800c676ce docs(pilot): freeze readiness and demo safety guardrails 2026-06-20 09:57:34 +03:00
IgorRachkovandGitHub 73578e8388 Update PILOT_DEMO_SCENARIO_RU.md 2026-06-19 09:47:43 +03:00
igor04091968 9b4b847723 feat(detmir): add support scripts docs and daily/weekly/monthly runners 2026-06-18 22:26:28 +03:00
igor04091968 c9ad5ecca7 chore(scripts): add full diagnostics package for detmir 2026-06-18 22:11:38 +03:00
igor04091968 ced3b0fb20 docs: clarify DetMir support responsibility and SLA 2026-06-18 17:33:00 +03:00
igor04091968 5f8a25e056 docs: update support scope with web server 2026-06-17 23:33:27 +03:00
igor04091968 66dc0a09b1 docs: add network and virtualization support tasks 2026-06-17 23:21:36 +03:00
igor04091968 4c96879915 docs: remove 1C from DetMir support scope 2026-06-17 23:16:42 +03:00
igor04091968 371a1e2728 docs: narrow DetMir support scope 2026-06-17 23:06:14 +03:00
igor04091968 b7209771f6 docs: add DetMir support task list 2026-06-17 23:01:54 +03:00
IgorRachkovandGitHub 525f45e0f7 Merge pull request #37 from igor04091968/refactor/portal-telemetry-ingest
refactor(portal): move telemetry ingest into module
2026-06-15 15:31:34 +03:00
igor04091968 66b0b6cffb refactor(portal): move telemetry ingest into module 2026-06-15 14:24:10 +03:00
IgorRachkovandGitHub 57ef5abf7e Merge pull request #36 from igor04091968/refactor/portal-http-response
refactor(portal): move HTTP response helpers into module
2026-06-15 12:04:21 +03:00
IgorRachkovandGitHub ae1909a28a Update README.md 2026-06-15 09:58:03 +03:00
IgorRachkovandGitHub 064fbd05fb Update README.md 2026-06-15 09:57:24 +03:00
igor04091968 fed7cc2eaa refactor(portal): move HTTP response helpers into module 2026-06-15 07:54:19 +03:00
IgorRachkovandGitHub cd61a530d6 Merge pull request #35 from igor04091968/refactor/portal-static-assets
refactor(portal): move static assets into module
2026-06-15 07:15:25 +03:00
igor04091968 58535fa5c6 refactor(portal): move static assets into module 2026-06-15 07:01:13 +03:00
IgorRachkovandGitHub 7f1abdb9a7 Merge pull request #34 from igor04091968/refactor/portal-snapshot-cache
refactor(portal): move snapshot cache helpers into module
2026-06-15 01:43:58 +03:00
igor04091968 5ca325034f refactor(portal): move snapshot cache helpers into module 2026-06-15 01:36:23 +03:00
IgorRachkovandGitHub 5312da175d Merge pull request #33 from igor04091968/refactor/portal-role-access
refactor(portal): move role access helpers into module
2026-06-15 00:43:06 +03:00
igor04091968 fa1ddf64b4 refactor(portal): wire role access module 2026-06-15 00:03:44 +03:00
igor04091968 643d5d2d69 refactor(portal): move role access helpers into module 2026-06-15 00:00:04 +03:00
IgorRachkovandGitHub b629879958 Merge pull request #32 from igor04091968/refactor/portal-readiness-api
refactor(portal): move readiness API helpers into module
2026-06-14 23:56:40 +03:00
igor04091968 78a560dc3e refactor(portal): move readiness API helpers into module 2026-06-14 23:50:25 +03:00
IgorRachkovandGitHub 803c3169d7 Merge pull request #31 from igor04091968/refactor/portal-api-contracts
refactor(portal): move API contract summary into module
2026-06-14 23:25:49 +03:00
igor04091968 03f10435ce refactor(portal): move API contract summary into module 2026-06-14 23:14:30 +03:00
IgorRachkovandGitHub 2f8193e7b3 Merge pull request #30 from igor04091968/refactor/portal-path-query-helpers
refactor(portal): move path and query helpers into module
2026-06-14 23:05:35 +03:00
igor04091968 e76fa5a5c2 refactor(portal): move path and query helpers into module 2026-06-14 22:17:12 +03:00
IgorRachkovandGitHub 68c0fd1a37 Merge pull request #29 from igor04091968/refactor/portal-command-runner
refactor(portal): move command runner into module
2026-06-14 22:11:39 +03:00
igor04091968 d19b3d478f refactor(portal): move command runner into module 2026-06-14 22:00:44 +03:00
IgorRachkovandGitHub 0cd6e4f856 Merge pull request #28 from igor04091968/refactor/portal-roles-module
refactor(portal): move role model into module
2026-06-14 21:43:48 +03:00
igor04091968 acf767360f refactor(portal): move role model into module 2026-06-14 21:25:30 +03:00
IgorRachkovandGitHub 9ad5b2fc34 Merge pull request #27 from igor04091968/release/github-release-assets
rust-binary-build / build-linux-x86_64 (push) Waiting to run
ci: publish Rust binary package to GitHub Releases on tags
2026-06-14 21:00:59 +03:00
243 changed files with 28934 additions and 1226 deletions
+52 -1
View File
@@ -1 +1,52 @@
* @igor04091968
# AWatch-rus CODEOWNERS
#
# Ownership is used for review routing and engineering accountability.
# It does not certify security, legal readiness, performance or fitness for
# production use. Contributors remain responsible for the safety of their
# submissions.
# Repository default owner.
* @igor04091968 @rachkovii68-bisness
# Rust workspace and crates.
/adk-rust/ @igor04091968 @rachkovii68-bisness
**/Cargo.toml @igor04091968 @rachkovii68-bisness
**/Cargo.lock @igor04091968 @rachkovii68-bisness
**/*.rs @igor04091968 @rachkovii68-bisness
# Operational scripts and automation.
/scripts/ @igor04091968 @rachkovii68-bisness
# Product documentation, demo material and screenshots.
/docs/ @igor04091968 @rachkovii68-bisness
/docs/demo/ @igor04091968 @rachkovii68-bisness
/docs/screenshots/ @igor04091968 @rachkovii68-bisness
/docs/assets/screenshots/ @igor04091968 @rachkovii68-bisness
# Registry-readiness evidence and policy documents.
/docs/registry/ @igor04091968 @rachkovii68-bisness
# GitHub public mirror validation metadata and governance controls.
/.github/workflows/ @igor04091968 @rachkovii68-bisness
/.github/workflows/ci.yml @igor04091968 @rachkovii68-bisness
/.github/workflows/security.yml @igor04091968 @rachkovii68-bisness
/.github/workflows/coverage.yml @igor04091968 @rachkovii68-bisness
/.github/GOVERNANCE.md @igor04091968 @rachkovii68-bisness
/.github/pull_request_template.md @igor04091968 @rachkovii68-bisness
/.github/ISSUE_TEMPLATE/ @igor04091968 @rachkovii68-bisness
/.github/CODEOWNERS @igor04091968 @rachkovii68-bisness
/docs/BRANCH_PROTECTION_POLICY_RU.md @igor04091968 @rachkovii68-bisness
/docs/BRANCH_PROTECTION_EVIDENCE_RU.md @igor04091968 @rachkovii68-bisness
/docs/PR_REVIEW_WORKFLOW_RU.md @igor04091968 @rachkovii68-bisness
/docs/PR_REVIEW_EVIDENCE_RU.md @igor04091968 @rachkovii68-bisness
/docs/ENTERPRISE_QUALITY_STANDARD_RU.md @igor04091968 @rachkovii68-bisness
/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @igor04091968 @rachkovii68-bisness
/docs/OPERATIONAL_MATURITY_RU.md @igor04091968 @rachkovii68-bisness
# Ansible deployment automation.
/ansible/ @igor04091968 @rachkovii68-bisness
# Security and contribution governance.
/SECURITY.md @igor04091968 @rachkovii68-bisness
/CONTRIBUTING.md @igor04091968 @rachkovii68-bisness
/deny.toml @igor04091968 @rachkovii68-bisness
+41
View File
@@ -0,0 +1,41 @@
# AWatch-rus Governance
GitHub is the public mirror validation surface. Primary registry release
evidence must be produced on the Russian build-runner and primary operational
context remains the private Gitea contour.
## Production-first standard
AWatch-rus is already deployed in a real company. Production stability has
absolute priority. The canonical engineering standard is:
- [Enterprise quality standard](../docs/ENTERPRISE_QUALITY_STANDARD_RU.md)
- [Review checklist](../docs/REVIEW_CHECKLIST_RU.md)
- [Operational validation runbook](../docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md)
- [Operational maturity harness](../docs/OPERATIONAL_MATURITY_RU.md)
## Required PR evidence
Every PR must state:
- Purpose.
- Operational impact.
- Risk assessment.
- Rollback strategy.
- Validation steps.
- Documentation changes.
- Acceptance criteria.
Documentation-only or governance-only PRs must explicitly state that runtime,
API and UI behavior are unchanged.
## Guardrails
- Prefer additive, backward-compatible changes.
- Do not redesign working subsystems without measured benefit.
- Do not add dependencies without justification and validation.
- Do not weaken authentication, authorization, audit logging, secret handling,
dependency hygiene or configuration validation.
- Do not enable heavy DLP, Loki or always-on Velociraptor during routine
recovery, validation or public CI.
- Keep blocking CI fast; keep heavy/load/nightly checks scheduled or advisory.
+42
View File
@@ -0,0 +1,42 @@
name: Bug report
description: Report a reproducible defect with sanitized evidence.
title: "fix: "
labels: ["bug"]
body:
- type: markdown
attributes:
value: |
Do not publish secrets, tokens, private keys, personal data, real employee logs or customer evidence. Use demo/anonymized evidence.
- type: textarea
id: summary
attributes:
label: Summary
description: What is broken?
validations:
required: true
- type: textarea
id: steps
attributes:
label: Reproduction steps
description: Minimal steps using sanitized data.
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected behavior
validations:
required: true
- type: textarea
id: actual
attributes:
label: Actual behavior
validations:
required: true
- type: textarea
id: evidence
attributes:
label: Sanitized evidence
description: Logs, screenshots or output with secrets and personal data removed.
validations:
required: false
@@ -0,0 +1,42 @@
name: Feature request
description: Request a capability without using customer or employee data.
title: "feat: "
labels: ["enhancement"]
body:
- type: markdown
attributes:
value: |
Do not publish secrets, personal data, real employee logs or non-anonymized customer evidence.
- type: textarea
id: problem
attributes:
label: Problem
description: What user or operator problem should this solve?
validations:
required: true
- type: textarea
id: proposal
attributes:
label: Proposed behavior
description: Describe the desired behavior conservatively.
validations:
required: true
- type: dropdown
id: area
attributes:
label: Area
options:
- Workforce
- Security analytics
- Forensics
- Registry-readiness
- Install/deployment
- Documentation
validations:
required: true
- type: textarea
id: evidence
attributes:
label: Demo/anonymized evidence
validations:
required: false
@@ -0,0 +1,29 @@
name: Registry-readiness task
description: Track registry-readiness documentation, evidence or process gaps.
title: "docs(registry): "
labels: ["registry-readiness", "documentation"]
body:
- type: markdown
attributes:
value: |
Do not claim legal completion of registry registration. Do not publish secrets, personal data, real employee logs or private infrastructure credentials.
- type: textarea
id: scope
attributes:
label: Scope
description: What registry-readiness gap is being tracked?
validations:
required: true
- type: textarea
id: evidence
attributes:
label: Required evidence
description: List sanitized evidence or documents needed.
validations:
required: true
- type: textarea
id: remaining
attributes:
label: Remaining gaps
validations:
required: false
@@ -0,0 +1,30 @@
name: Security hardening task
description: Track hardening without exposing sensitive data.
title: "chore(security): "
labels: ["security", "hardening"]
body:
- type: markdown
attributes:
value: |
Do not publish exploits, secrets, private keys, tokens, personal data, real employee logs or customer evidence.
- type: textarea
id: hardening
attributes:
label: Hardening target
description: What should be hardened?
validations:
required: true
- type: textarea
id: risk
attributes:
label: Risk addressed
description: Explain the risk without sensitive details.
validations:
required: true
- type: textarea
id: validation
attributes:
label: Validation
description: Checks or evidence required.
validations:
required: true
+99 -9
View File
@@ -1,13 +1,103 @@
## Summary
- what changed
- why it changed
- risk and rollback notes
Describe what changed and why.
## Checklist
## Purpose
- [ ] No real secrets or credentials committed
- [ ] Server-side scripts validated (`bash -n`)
- [ ] PowerShell scripts validated (`Invoke-ScriptAnalyzer`)
- [ ] Docs updated (full paths and runbook steps)
- [ ] Rollback steps documented
State the production-quality purpose of the change. Prefer reliability,
operational maturity, security, maintainability, reproducibility, performance
or simplicity over new functionality.
## Impact
- Runtime impact: `none / changed / not applicable`
- API impact: `none / changed / not applicable`
- UI impact: `none / changed / not applicable`
- Documentation impact: `none / changed / not applicable`
- Rollback impact: `none / documented / not applicable`
- Evidence impact: `none / registry docs updated / release evidence required`
## Operational Impact
Describe deployment, upgrade, rollback, observability, diagnostics,
configuration, recovery, performance or dependency-hygiene impact.
## Risk Assessment
List production risks and why the change is backward-compatible. For
documentation-only or governance-only changes, state that runtime/API/UI behavior
is unchanged.
## Rollback Strategy
State how to revert the change. Runtime, automation, config and dependency
changes need an explicit rollback path.
## Validation
List commands executed. Use `skipped: <reason>` when a check requires a live
stand or unavailable tool.
## Documentation Changes
List README/runbook/architecture/governance updates, or state `not applicable`
with a reason.
## Acceptance Criteria
List concrete conditions that make the PR safe to merge.
## Review Checklist
- [ ] Linked issue is provided, or the PR explains why no issue is applicable.
- [ ] Purpose, operational impact, risk assessment, rollback strategy,
validation steps, documentation changes and acceptance criteria are stated.
- [ ] Change is additive/backward-compatible, or breaking impact is explicitly
blocked for this stage.
- [ ] Production stability is preserved for existing deployments.
- [ ] No working subsystem is redesigned without measured benefit.
- [ ] Runtime/API/UI impact is stated.
- [ ] Registry claims are checked and remain conservative.
- [ ] Secrets, PII, employee logs and customer identifiers are absent.
- [ ] Tests/checks executed are listed, or skipped checks have reasons.
- [ ] Evidence docs are updated when the change affects governance, registry
readiness or release evidence.
- [ ] GitHub Actions are public validation only, not registry release evidence.
- [ ] I checked that this PR does not publish secrets, tokens, passwords,
private keys, recovery codes or live credentials.
- [ ] I checked that this PR does not publish personal data, real employee data,
customer logs or customer infrastructure identifiers.
- [ ] I checked registry claims: no completed registry submission, no
FSTEC/FSB certification claim, no SIEM/DLP replacement claim.
- [ ] I ran relevant checks or documented why a check was skipped.
- [ ] I checked dependency impact: no unnecessary dependency was added, and no
unused dependency remains in touched crates.
- [ ] I stated runtime/API/UI impact.
- [ ] I stated documentation impact.
- [ ] I stated smoke-test result or why smoke testing is not applicable.
- [ ] I stated rollback and evidence impact.
- [ ] I checked that GitHub Actions remains public mirror validation only.
- [ ] I checked that registry release evidence still requires the Russian
build-runner.
## Registry / Public Mirror Scope
- GitHub is public mirror validation only.
- Primary registry release evidence must be produced on the Russian
build-runner.
- Update `docs/registry/` when registry-readiness behavior or evidence changes.
## Governance
- Production-first standard: `.github/GOVERNANCE.md`.
- Canonical review checklist: `docs/REVIEW_CHECKLIST_RU.md`.
- Canonical validation runbook: `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
## Safety
- No secrets, tokens, passwords or private keys.
- No personal data.
- No real employee logs.
- No customer evidence unless anonymized.
- No unsupported claims about certification, DLP/SIEM replacement or legal
registry completion.
+84 -40
View File
@@ -1,70 +1,114 @@
name: shell-and-powershell-ci
name: CI
# GitHub Actions is public mirror validation only.
# Primary registry release evidence must be produced on Russian build-runner.
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
workflow_dispatch:
permissions:
contents: read
jobs:
shell-check:
rust-checks:
name: Rust checks
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install shellcheck
run: sudo apt-get update && sudo apt-get install -y shellcheck
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- name: Run shellcheck
run: |
find . -type f -name "*.sh" -print0 | xargs -0 -r shellcheck -S error -e SC1007,SC1090,SC2016
- name: cargo fmt
run: cargo fmt --all --check
- name: Run production inventory placeholder guard self-test
run: bash scripts/check_production_inventory_placeholders.sh --self-test
- name: cargo test
run: cargo test --workspace
- name: Run private-config guard
run: bash scripts/check_private_config_guard.sh
- name: cargo clippy
run: cargo clippy --workspace --all-targets -- -D warnings
- name: Run portal contract sync guard
run: node scripts/check_portal_contract_sync.mjs
- name: cargo build
run: cargo build --workspace
rust-runtime-guard:
docs-registry-checks:
name: Docs and registry checks
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Test detmir-core runtime guard
run: cargo test --manifest-path adk-rust/Cargo.toml -p detmir-core runtime_guard
- name: Registry script syntax
run: bash -n scripts/registry_readiness_check.sh
- name: Test detmir readiness crate
run: cargo test --manifest-path adk-rust/Cargo.toml -p detmir-readiness
- name: Registry readiness
run: bash scripts/registry_readiness_check.sh
powershell-analyzer:
- name: Release evidence script syntax
run: |
if [[ -f scripts/build_release_evidence.sh ]]; then
bash -n scripts/build_release_evidence.sh
fi
if [[ -f scripts/check_release_evidence.sh ]]; then
bash -n scripts/check_release_evidence.sh
fi
- name: Whitespace diff check
run: git diff --check
smoke-checks:
name: Smoke checks
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install PSScriptAnalyzer
shell: pwsh
run: |
Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
Install-Module PSScriptAnalyzer -Scope CurrentUser -Force
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Analyze PowerShell scripts
shell: pwsh
- name: Deployment readiness smoke
run: |
$targets = @(
"windows/*.ps1",
"windows/*.psm1",
"windows/*.psd1"
)
$issues = $targets | ForEach-Object {
Invoke-ScriptAnalyzer -Path $_ -Recurse -Severity Error
}
if ($issues) {
$issues | Format-Table -AutoSize
throw "PSScriptAnalyzer detected issues."
}
if command -v node >/dev/null 2>&1 && [[ -f scripts/deployment-readiness-smoke.mjs ]]; then
node scripts/deployment-readiness-smoke.mjs
else
echo "skipped: node or scripts/deployment-readiness-smoke.mjs missing"
fi
- name: Pilot validation smoke
run: |
if command -v node >/dev/null 2>&1 && [[ -f scripts/pilot-validation-smoke.mjs ]]; then
node scripts/pilot-validation-smoke.mjs
else
echo "skipped: node or scripts/pilot-validation-smoke.mjs missing"
fi
- name: Browser conformance smoke
run: |
if [[ -f scripts/browser-conformance-smoke.mjs ]]; then
echo "skipped: requires live stand unless explicitly run by operator"
else
echo "skipped: scripts/browser-conformance-smoke.mjs missing"
fi
+47
View File
@@ -0,0 +1,47 @@
name: Coverage
# GitHub Actions is public mirror validation only.
# Primary registry release evidence must be produced on Russian build-runner.
on:
push:
pull_request:
workflow_dispatch:
permissions:
contents: read
jobs:
coverage-baseline:
name: Coverage baseline
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Generate coverage summary
run: |
mkdir -p ../coverage
cargo llvm-cov --workspace --summary-only | tee ../coverage/coverage-summary.txt
- name: Upload coverage summary
uses: actions/upload-artifact@v4
with:
name: coverage-summary
path: coverage/coverage-summary.txt
- name: Future threshold placeholder
run: |
echo "Coverage threshold is not enforced yet; this workflow tracks baseline and regressions."
+149
View File
@@ -0,0 +1,149 @@
name: Dependency hygiene
# GitHub Actions is public mirror validation only.
# Primary registry release evidence must be produced on Russian build-runner.
on:
push:
pull_request:
workflow_dispatch:
schedule:
- cron: "17 2 * * 1"
permissions:
contents: read
pull-requests: read
jobs:
unused-dependencies:
name: Unused dependency check
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-machete
run: cargo install cargo-machete --locked
- name: cargo metadata
run: cargo metadata --locked --format-version 1 > /tmp/aw-rus-cargo-metadata.json
- name: cargo machete
run: cargo machete --with-metadata
- name: Ensure cargo-machete metadata did not rewrite manifests
working-directory: .
run: git diff --exit-code -- adk-rust/Cargo.lock adk-rust/Cargo.toml adk-rust/crates
- name: Require explicit justification for cargo-machete ignores
working-directory: .
run: |
python3 - <<'PY'
from pathlib import Path
failures = []
for path in Path("adk-rust").rglob("Cargo.toml"):
lines = path.read_text(encoding="utf-8").splitlines()
in_machete = False
for idx, line in enumerate(lines):
stripped = line.strip()
if stripped.startswith("[") and stripped.endswith("]"):
in_machete = stripped == "[package.metadata.cargo-machete]"
continue
if not in_machete or not stripped.startswith("ignored"):
continue
same_line_comment = "#" in line and line.split("#", 1)[1].strip()
prev_comment = idx > 0 and lines[idx - 1].strip().startswith("#")
if not same_line_comment and not prev_comment:
failures.append(f"{path}:{idx + 1}")
if failures:
print("cargo-machete ignored entries require an adjacent TOML comment explaining why the dependency is intentionally kept:")
for item in failures:
print(f" {item}")
raise SystemExit(1)
PY
dependency-tree:
name: Dependency duplicate report
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: cargo tree duplicates
run: cargo tree --duplicates --locked
dependency-security:
name: Dependency security policy
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-audit
uses: taiki-e/install-action@cargo-audit
- name: Install cargo-deny
uses: taiki-e/install-action@cargo-deny
- name: cargo audit
working-directory: adk-rust
run: cargo audit --deny warnings
- name: cargo deny
run: |
cargo deny --manifest-path adk-rust/Cargo.toml check \
--config deny.toml \
--hide-inclusion-graph \
--show-stats
cargo-udeps-nightly:
name: Cargo udeps nightly advisory
runs-on: ubuntu-latest
continue-on-error: true
if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install nightly Rust
uses: dtolnay/rust-toolchain@nightly
- name: Install cargo-udeps
run: cargo install cargo-udeps --locked
- name: cargo udeps
run: cargo +nightly udeps --workspace --all-targets
@@ -0,0 +1,73 @@
name: Operational maturity
# GitHub Actions is public mirror validation only.
# Live DetMir production checks must be run explicitly by an operator with
# private network access and must not enable heavy DLP, Loki or always-on
# Velociraptor.
on:
push:
pull_request:
workflow_dispatch:
inputs:
live:
description: "Run live endpoint contract if AWATCH_OPS_LIVE_URL is configured"
required: false
default: "false"
schedule:
- cron: "41 3 * * 2"
permissions:
contents: read
jobs:
offline-operational-maturity:
name: Offline operational maturity
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Script syntax
run: node --check scripts/operational-maturity-check.mjs
- name: Operational maturity harness
run: node scripts/operational-maturity-check.mjs --json
live-operational-contract:
name: Live operational contract
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' && inputs.live == 'true'
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Run live contract when URL is available
env:
AWATCH_OPS_LIVE_URL: ${{ secrets.AWATCH_OPS_LIVE_URL }}
run: |
if [[ -z "${AWATCH_OPS_LIVE_URL}" ]]; then
echo "skipped: AWATCH_OPS_LIVE_URL secret is not configured"
exit 0
fi
node scripts/operational-maturity-check.mjs --json --live
+86
View File
@@ -0,0 +1,86 @@
name: Security
# GitHub Actions is public mirror validation only.
# Primary registry release security evidence must be produced on Russian build-runner.
on:
push:
pull_request:
workflow_dispatch:
permissions:
contents: read
pull-requests: read
jobs:
cargo-audit:
name: Cargo audit
runs-on: ubuntu-latest
defaults:
run:
shell: bash
working-directory: adk-rust
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-audit
uses: taiki-e/install-action@cargo-audit
- name: cargo audit
run: cargo audit
cargo-deny:
name: Cargo deny
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Install stable Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-deny
uses: taiki-e/install-action@cargo-deny
- name: cargo deny
run: cargo deny check advisories licenses sources --config ../deny.toml
working-directory: adk-rust
secret-pattern-check:
name: Secret pattern check
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Scan for obvious secret patterns
run: python3 scripts/public_secret_pattern_check.py
dependency-review:
name: Dependency review
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout
uses: actions/checkout@v4
with:
lfs: false
- name: Dependency Review
uses: actions/dependency-review-action@v4
+40 -3
View File
@@ -1,5 +1,14 @@
# Contributing
GitHub is public mirror validation only. Primary registry release evidence is
produced separately on the Russian build-runner and documented under
`docs/registry/`.
All contributions must follow the production-first governance entrypoint:
`.github/GOVERNANCE.md`. AWatch-rus is already deployed in a real company, so
reliability, operational maturity, security and backward compatibility take
priority over new functionality.
## Branching
- Работайте в feature-ветке, не пушьте напрямую в `main`.
@@ -15,13 +24,41 @@
## Required checks before PR
- `bash -n` для всех `*.sh`.
- `cargo fmt --all --check` from `adk-rust/`.
- `cargo test --workspace` from `adk-rust/`, unless the PR is documentation-only
and the skip is documented.
- `cargo clippy --workspace --all-targets -- -D warnings` from `adk-rust/`.
- `bash -n` для всех changed `*.sh`.
- `bash scripts/registry_readiness_check.sh` when registry docs/process changes.
- `node scripts/deployment-readiness-smoke.mjs` when Node.js is available.
- `node scripts/pilot-validation-smoke.mjs` when Node.js is available.
- `Invoke-ScriptAnalyzer` для `windows/*.ps1`, `windows/*.psm1`, `windows/*.psd1`.
- Проверка, что нет секретов (`secrets/deploy.secrets.env` не должен быть в индексе git).
- Проверка, что нет секретов (`secrets/deploy.secrets.env` не должен быть в
индексе git).
- Обновлены инструкции и runbook при изменении поведения.
## Registry-readiness docs
- Registry-readiness documents live in `docs/registry/`.
- Public GitHub CI is not registry release evidence.
- Registry release evidence must be generated on the Russian build-runner.
- GitHub remains public mirror validation only.
## Secrets and personal data
- Do not commit secrets, tokens, passwords, cookies or private keys.
- Do not commit personal data.
- Do not commit real employee logs.
- Use demo/anonymized evidence for issues, PRs, docs and screenshots.
## PR content
- Изменения и обоснование.
- Риск и rollback.
- Purpose.
- Operational impact.
- Risk assessment.
- Rollback strategy.
- Какие команды валидации были выполнены.
- Documentation changes.
- Acceptance criteria.
- Какие проверки были пропущены и почему, если пропуск был необходим.
+826
View File
@@ -0,0 +1,826 @@
# TASK_065: Production 1.0 development plan
Audit date: 2026-07-01.
Scope: AWatch-rus repository readiness for Production 1.0. This plan is based
only on repository evidence: tracked files, documentation, CI workflows,
Cargo metadata, dependency checks, scripts, deployment assets and operational
maturity contracts.
This is not a feature roadmap. Every task below exists because the current
repository still has a confirmed blocker or material risk for a durable
Production 1.0 release.
## Guardrails
- Do not redesign working subsystems.
- Do not add new product functionality as part of Production 1.0 readiness.
- Preserve backward compatibility for deployed DetMir production.
- Keep DLP runtime conservative: `core_only/disabled` by default; `light` only
after explicit operator resource preflight.
- Do not enable Loki or always-on Velociraptor as part of Production 1.0.
- Keep PowerShell fallback until Rust parity, canary evidence and rollback
criteria are proven.
- GitHub remains public mirror validation. Release evidence must be produced in
the controlled release contour documented under `docs/registry/`.
## Evidence Base
Reviewed repository areas:
- Rust workspace: `adk-rust/`, 58 crates from `adk-rust/crates/*/Cargo.toml`.
- Cargo state: `cargo metadata --locked --format-version 1` returned 58
workspace members and 349 packages.
- Dependency hygiene: `cargo audit --deny warnings` passed; `cargo machete
--with-metadata` found no unused dependencies; `cargo deny` passed with
non-blocking policy warnings; `cargo tree --duplicates --locked` found
duplicate roots including `bitflags`, `getrandom`, `hashbrown`, `mio`, `zip`.
- CI: `.github/workflows/ci.yml`, `security.yml`, `coverage.yml`,
`dependency-hygiene.yml`, `operational-maturity.yml`,
`rust-workspace.yml`, `rust-binary-build.yml`, `release-assets.yml`.
- Operational maturity: `configs/operational-maturity-contract.json`,
`scripts/operational-maturity-check.mjs`,
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
- Registry and release evidence docs: `docs/registry/*`,
`docs/PROJECT_STATUS_RU.md`, `docs/RESIDUAL_RISKS_RU.md`,
`docs/QUALITY_STATUS_RU.md`, `ROADMAP.md`.
- Runtime/deployment: `ansible/`, `aw-server/`, `windows/`, `proxmox/`,
`clickhouse-1c/`, `grafana/`, `pfsense/`, root operational wrappers.
Confirmed current strengths:
- Rust-first runtime and wrappers exist.
- Public mirror CI exists.
- Dependency hygiene automation exists.
- Operational maturity offline harness exists.
- DLP resource guardrails and conservative runtime profile exist.
- Security Finding Inbox / Hayabusa / Velociraptor are optional and separated
from Workforce hot path.
- Production runbooks and DetMir guardrails are documented.
Confirmed Production 1.0 blockers and material risks are organized below.
## P0 - Critical before Production 1.0
### P0-1. Production binary parity gate
Status: repository gate addressed by TASK_072. Live production execution remains
External Production Validation: operators must collect current production
SHA256 evidence and run the gate before Release Candidate approval.
Purpose: prove that every binary actually running in production matches the
reviewed release artifact.
Reason: `scripts/check_detmir_rust_release_artifacts.sh` verifies local release
artifacts, but Production 1.0 also needs a verified mapping from deployed unit,
timer or Windows task to exact production SHA256 and source crate.
Operational impact: prevents stale binary drift and makes rollback decisions
deterministic.
Risk: high. A stale production binary can invalidate test results and hide
regressions.
Estimated effort: 3-5 days.
Affected modules:
- `scripts/check_detmir_rust_release_artifacts.sh`
- `scripts/check_production_binary_parity.py`
- `scripts/package_rust_release_binaries.py`
- `scripts/detmir-full-diagnostics/`
- `adk-rust/crates/detmir-readiness/`
- `adk-rust/crates/aw-windows-telemetry/`
- `windows/validate-deployment.ps1`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Report maps `service/timer/task -> binary path -> crate -> runtime role`.
- Report includes local release SHA256, production SHA256 and git SHA.
- Missing or mismatched binaries fail the gate.
- DLP/Loki/Velociraptor heavy runtime is not enabled by the check.
- Repository validator accepts inactive optional contours only with explicit
`skip_reason`.
Validation steps:
- Run the parity gate against current release artifacts.
- Run diagnostics for `10.10.10.2`, `10.10.10.13` and Windows RDP host.
- Confirm stale binary simulation fails.
Expected benefit: release and production state become auditable.
### P0-2. Controlled release evidence build
Status: addressed by TASK_070. Existing release evidence tooling now records
resolved Git SHA, deterministic `SOURCE_DATE_EPOCH` / `BUILD_TIME`, runner
environment and artifact checksums, and the evidence checker enforces the
required provenance fields.
Purpose: produce Production 1.0 release evidence outside GitHub public mirror.
Reason: `docs/PROJECT_STATUS_RU.md`, `docs/QUALITY_STATUS_RU.md` and
`docs/registry/registry-evidence-manifest.json` state that GitHub Actions are
public validation only and that the Russian build-runner/release evidence path
is still pending.
Operational impact: separates mirror CI from release authority.
Risk: high. Without controlled release evidence, Production 1.0 cannot be
treated as reproducibly built.
Estimated effort: 3-6 days plus infrastructure window.
Affected modules:
- `docs/registry/RU_BUILD_RUNNER_READINESS_RU.md`
- `docs/registry/BUILD_RUNNER_SETUP_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_MANIFEST_RU.md`
- `scripts/build_release_evidence.sh`
- `scripts/check_release_evidence.sh`
- `scripts/verify_release_assets.sh`
Acceptance criteria:
- Controlled runner has documented OS, toolchain and access model.
- Release evidence includes source archive, binary archive, SBOM if available,
SHA256SUMS, Cargo metadata/tree, smoke logs and release manifest.
- `scripts/check_release_evidence.sh` passes on produced artifacts.
- GitHub CI is not described as release evidence.
Validation steps:
- Run release evidence script on controlled runner.
- Verify manifest and checksums.
- Record runner environment and commit SHA.
Expected benefit: Production 1.0 release becomes reproducible and auditable.
### P0-3. Disaster recovery restore proof
Purpose: prove that repository backup can be restored before Production 1.0.
Reason: `docs/registry/registry-evidence-manifest.json` records
`restore_tested=false`, and `docs/RESIDUAL_RISKS_RU.md` lists Gitea restore
test as open.
Operational impact: validates recovery from source repository loss.
Risk: high. Untested backup is not a recovery capability.
Estimated effort: 2-4 days.
Affected modules:
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md`
- `docs/registry/registry-evidence-manifest.json`
- `scripts/registry_readiness_check.sh`
- `docs/PROJECT_STATUS_RU.md`
- `docs/RESIDUAL_RISKS_RU.md`
Acceptance criteria:
- Restore is performed on a separate host.
- SHA256 verification, logs, restored repository access and rollback notes are
captured.
- Manifest is updated only after evidence exists.
- No secrets are copied into repository documentation.
Validation steps:
- Execute restore runbook.
- Run registry readiness check after manifest update.
- Confirm restored repository clone and log evidence.
Expected benefit: repository DR becomes proven, not only documented.
### P0-4. Bounded retention for operational state and evidence
Purpose: prevent disk exhaustion from state, queues, diagnostics, evidence and
forensics artifacts.
Reason: DLP optional runtime docs explicitly state that historical DLP buckets
and artifacts may remain until a separate retention/cleanup procedure. Scripts
and diagnostics also create durable output.
Operational impact: reduces outage risk on Proxmox/AW server/Windows state
paths.
Risk: high. Disk exhaustion can stop ingestion, portal, ClickHouse or
ActivityWatch services.
Estimated effort: 4-7 days.
Affected modules:
- `adk-rust/crates/aw-prune-local-state/`
- `scripts/detmir-full-diagnostics/`
- `scripts/detmir_dlp_warehouse_sync.sh`
- `aw-server/logrotate.conf`
- `windows/validate-deployment.ps1`
- `adk-rust/crates/aw-windows-telemetry/`
- `docs/DLP_OPTIONAL_RUNTIME_RU.md`
- `docs/DLP_RESOURCE_PROFILES_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Retention matrix lists roots, owners, max age, max size and dry-run behavior.
- Cleanup only touches allowlisted roots and refuses traversal/symlink escape.
- Active state is preserved.
- Disabled DLP buckets remain `SKIPPED`, not failure.
Validation steps:
- Run dry-run cleanup on fixture tree.
- Run apply mode on controlled temporary tree.
- Run operational smoke after cleanup.
Expected benefit: long-running production operation has bounded disk behavior.
### P0-5. Remove ClickHouse password exposure from process arguments
Status: addressed by TASK_068. Runtime ClickHouse/1C wrappers keep
`CLICKHOUSE_PASSWORD` in the environment/config path and no longer pass it in
process arguments.
Purpose: keep production credentials out of `ps`/process argv.
Reason: `rg` confirmed `clickhouse-1c/ops/run_*.sh` wrappers pass
`--password "${CLICKHOUSE_PASSWORD}"`.
Operational impact: improves secret handling for the 1C/ClickHouse contour.
Risk: high. Local process listing can reveal ClickHouse credentials.
Estimated effort: 2-4 days.
Affected modules:
- `clickhouse-1c/ops/run_ingest_cycle.sh`
- `clickhouse-1c/ops/run_manager_brief.sh`
- `clickhouse-1c/ops/run_recovery_brief.sh`
- `clickhouse-1c/ops/run_company_registry_bindings_refresh.sh`
- `clickhouse-1c/ops/run_company_intelligence_refresh.sh`
- `clickhouse-1c/ops/check_ingest_freshness.sh`
- `clickhouse-1c/ai/*.py`
- `clickhouse-1c/etl/*.py`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Runtime wrappers no longer pass password through argv.
- Existing environment-based deployment remains backward compatible.
- Logs redact authentication failures.
- `ps` smoke proves password absence.
Validation steps:
- `bash -n clickhouse-1c/ops/*.sh`
- Run affected wrapper against a test or dry-run configuration.
- Verify `ps` output during execution.
- Run secret-pattern scan.
Expected benefit: production secret exposure surface is reduced.
### P0-6. Portal/worktime hot-path load gate
Purpose: prevent Production 1.0 from regressing under report/prewarm load.
Reason: `docs/PROJECT_STATUS_RU.md` records that full report/snapshot prewarm
can still be CPU/IO expensive.
Operational impact: protects owner/operator portal, worktime reports and AW
query path.
Risk: high. More users or more history can produce slow portal, stale data or
AW datastore pressure.
Estimated effort: 1-2 weeks.
Affected modules:
- `scripts/operational-maturity-check.mjs`
- `scripts/awatch-production-hardening-smoke.mjs`
- `adk-rust/crates/detmir-portal/`
- `adk-rust/crates/worktime-api/`
- `adk-rust/crates/worktime-prewarm/`
- `adk-rust/crates/aw-contour-smoke/`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
Acceptance criteria:
- Synthetic 5/20/50 user fixtures exist without production data.
- Gate records p95 latency, max RSS, query count and stale-cache behavior.
- Disconnected RDP sessions do not false-fail.
- `AW_DLP_ENABLED=false` semantics remain valid.
- Heavy load job is scheduled/advisory; blocking smoke remains fast.
Validation steps:
- Run offline load harness.
- Confirm configured p95/RSS ceilings.
- Run existing operational maturity smoke.
Expected benefit: Production 1.0 has measurable performance safety.
## P1 - Strongly recommended for the first Production 1.0 release train
### P1-1. Align Rust toolchain across CI
Purpose: remove compiler drift from blocking workflows.
Reason: `rust-toolchain.toml` pins `1.94.0`, but several workflows install
floating `stable`.
Operational impact: improves reproducibility between local, CI and release
contours.
Risk: medium. Toolchain drift can create inconsistent warnings or binaries.
Estimated effort: 1-2 days.
Affected modules:
- `rust-toolchain.toml`
- `.github/workflows/ci.yml`
- `.github/workflows/security.yml`
- `.github/workflows/coverage.yml`
- `.github/workflows/dependency-hygiene.yml`
- `.github/workflows/release-assets.yml`
- `docs/QUALITY_STATUS_RU.md`
Acceptance criteria:
- Blocking Rust workflows use the pinned toolchain.
- Nightly remains limited to advisory `cargo udeps`.
- Required check names do not change.
Validation steps:
- YAML syntax validation.
- Affected workflow dry review.
- Run relevant Rust checks if workflow commands change.
Expected benefit: CI becomes more deterministic.
### P1-2. Dependency warning baseline and future block policy
Purpose: make dependency hygiene fail closed for new risk while preserving
current compatibility.
Reason: `cargo deny` passes but reports 36 non-blocking `bans` warnings;
`cargo tree --duplicates --locked` reports duplicate roots; `serde_yaml` is
documented as deprecated in third-party license docs.
Operational impact: reduces future supply-chain drift.
Risk: medium. Uncontrolled duplicate/deprecated dependency growth increases
maintenance and security load.
Estimated effort: 4-8 days.
Affected modules:
- `adk-rust/Cargo.toml`
- `adk-rust/Cargo.lock`
- `deny.toml`
- `.github/workflows/dependency-hygiene.yml`
- `docs/THIRD_PARTY_LICENSES_RU.md`
- `docs/QUALITY_STATUS_RU.md`
Acceptance criteria:
- Each existing warning is classified: keep, update, remove or defer.
- New duplicate/deprecated dependencies require documented exception.
- `cargo audit`, `cargo deny`, `cargo machete`, `cargo metadata` pass.
Validation steps:
- Run dependency hygiene pipeline.
- Verify policy failure on synthetic unapproved duplicate where practical.
Expected benefit: dependency hygiene remains controlled after 1.0.
### P1-3. Windows Rust validation parity
Purpose: prove Rust validation is equivalent to current PowerShell validation
before reducing fallback reliance.
Reason: `docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md` lists remaining fallback
and runtime PowerShell paths, including validation and Hayabusa upload.
Operational impact: keeps Windows/RDP production recoverable while reducing
runtime drift.
Risk: medium. Premature fallback removal can break localized Windows Server
2019 recovery paths.
Estimated effort: 1-2 weeks.
Affected modules:
- `adk-rust/crates/aw-windows-telemetry/`
- `windows/validate-deployment.ps1`
- `windows/ActivityWatch.Windows.Common.psm1`
- `windows/export-upload-hayabusa-to-aw-server.ps1`
- `ansible/deploy_aw_windows.yml`
- `docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md`
- `docs/POWERSHELL_TO_RUST_ROADMAP_RU.md`
Acceptance criteria:
- Rust validation covers all current production validation sections.
- Localized Windows user/session handling is tested.
- Canary comparison between Rust and PowerShell reports is recorded.
- PowerShell remains documented rollback.
Validation steps:
- Run Rust validation against fixture and live canary.
- Run PowerShell validation against same host.
- Compare normalized reports.
Expected benefit: Windows runtime maturity improves without breaking rollback.
### P1-4. Install kit reproducibility and stale payload gate
Purpose: ensure Windows installer payloads match repository and release
artifacts.
Reason: repository contains install-kit tooling and large installer artifacts;
stale payloads can deploy old collectors while CI is green.
Operational impact: safer Windows upgrades and rollback.
Risk: medium. Mismatched install kit can create production drift.
Estimated effort: 5-8 days.
Affected modules:
- `windows/installkit/innosetup/`
- `adk-rust/crates/check-install-kit-vs-repo/`
- `adk-rust/crates/rebuild-install-kit/`
- `adk-rust/crates/validate-install-kit/`
- `adk-rust/crates/verify-innosetup-installer/`
- `scripts/rebuild_install_kit.sh`
- `docs/INSTALL_KIT_RUNBOOK_RU.md`
Acceptance criteria:
- Installer manifest contains source commit and payload SHA256 values.
- Validation fails on stale collector payload.
- Existing Windows task names and config schema remain compatible.
Validation steps:
- Rebuild install kit on controlled runner.
- Run install-kit validators.
- Compare payload manifest with repository state.
Expected benefit: Windows deployments become reproducible.
### P1-5. Current-state documentation cleanup
Purpose: prevent operators and reviewers from following stale status text.
Reason: current docs include historical statuses, older required check names
and explicit pending sections that must be reconciled with active branch
protection and current workflows.
Operational impact: reduces release and support mistakes.
Risk: medium. Wrong runbook/status interpretation can cause incorrect release
decisions.
Estimated effort: 3-6 days.
Affected modules:
- `docs/PROJECT_STATUS_RU.md`
- `docs/QUALITY_STATUS_RU.md`
- `docs/ROADMAP_CONFORMANCE_AUDIT_RU.md`
- `docs/BRANCH_PROTECTION_POLICY_RU.md`
- `docs/BRANCH_PROTECTION_EVIDENCE_RU.md`
- `README.md`
- `ROADMAP.md`
Acceptance criteria:
- Current docs list actual required check names.
- Historical docs are clearly marked historical.
- Registry/release claims remain conservative.
- No stale instructions contradict Production 1.0 guardrails.
Validation steps:
- Run registry readiness check.
- Run docs smoke/link validation.
- Run secret scan.
Expected benefit: release process is less error-prone.
### P1-6. Reviewed PR and release governance evidence
Purpose: prove review discipline before Production 1.0.
Reason: `docs/RESIDUAL_RISKS_RU.md` records first reviewed PR evidence as
pending even though CODEOWNERS, PR template and ruleset are present.
Operational impact: improves change control for production releases.
Risk: medium. Lack of review evidence weakens release governance.
Estimated effort: 1-3 days after reviewer availability.
Affected modules:
- `.github/CODEOWNERS`
- `.github/pull_request_template.md`
- `docs/PR_REVIEW_WORKFLOW_RU.md`
- `docs/PR_REVIEW_EVIDENCE_RU.md`
- `docs/REVIEW_CHECKLIST_RU.md`
- `docs/RESIDUAL_RISKS_RU.md`
Acceptance criteria:
- At least one PR is reviewed and merged without bypass.
- Evidence records checks, reviewer, approval and merge path.
- Release branch review policy is documented.
Validation steps:
- Verify PR history and ruleset evidence.
- Run registry readiness check after evidence update.
Expected benefit: Production 1.0 has visible governance evidence.
## P2 - Engineering improvements for post-1.0 hardening
### P2-1. Incremental decomposition of large modules
Purpose: reduce review risk in the largest files without behavior changes.
Reason: confirmed hotspots include `detmir-portal/src/main.rs` at 14200 lines,
`aw-windows-telemetry/src/main.rs` at 6411 lines,
`proxmox/tsj_guardian_bot.py` at 4610 lines,
`worktime-api/src/main.rs` at 3988 lines and
`ansible/deploy_aw_server.yml` at 3099 lines.
Operational impact: easier reviews and lower regression risk.
Risk: medium. Large files increase accidental coupling.
Estimated effort: 2-4 weeks in small PRs.
Affected modules:
- `adk-rust/crates/detmir-portal/`
- `adk-rust/crates/aw-windows-telemetry/`
- `adk-rust/crates/worktime-api/`
- `proxmox/tsj_guardian_bot.py`
- `ansible/deploy_aw_server.yml`
Acceptance criteria:
- Only extract bounded domains.
- Public API, config, unit and task names remain unchanged.
- Tests before and after remain equivalent.
Validation steps:
- Targeted tests per extracted module.
- Full Rust pipeline for Rust changes.
- Ansible syntax/list-tasks parity for playbook changes.
Expected benefit: maintainability improves without architecture rewrite.
### P2-2. Consolidate exact duplicate diagnostic scripts
Purpose: prevent script drift.
Reason: SHA256 confirms exact duplicates:
`scripts/aw-contour-diag.sh` equals
`scripts/detmir-full-diagnostics/aw-contour-diag.sh`; and
`scripts/check_production_inventory_placeholders.sh` equals
`scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`.
Operational impact: diagnostics remain consistent.
Risk: low-medium. Future fixes may land in one copy only.
Estimated effort: 1-2 days.
Affected modules:
- `scripts/aw-contour-diag.sh`
- `scripts/detmir-full-diagnostics/aw-contour-diag.sh`
- `scripts/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/detmir-full-diagnostics.sh`
Acceptance criteria:
- Existing paths continue to work.
- One implementation is canonical or drift check is enforced.
- Shell syntax and shellcheck pass.
Validation steps:
- `bash -n` on affected scripts.
- Run diagnostic wrapper in dry-run/smoke mode.
Expected benefit: lower maintenance overhead.
### P2-3. Bound 1C ingest memory profile
Purpose: make 1C ingest safer for larger files.
Reason: `adk-rust/crates/aw-1c-ingest/src/main.rs` reads rows into `Vec` and
builds large JSON batches; acceptable now, but risky as export volume grows.
Operational impact: improves ClickHouse ingestion predictability.
Risk: medium. Large files can create memory spikes and long insert windows.
Estimated effort: 1-2 weeks.
Affected modules:
- `adk-rust/crates/aw-1c-ingest/src/main.rs`
- `clickhouse-1c/etl/config.yml`
- `clickhouse-1c/etl/config.example.yml`
- `clickhouse-1c/sql/`
Acceptance criteria:
- Oversized input fails closed with clear diagnostic.
- Batch size is bounded and configurable.
- Existing small DetMir files produce identical output.
Validation steps:
- Add synthetic large CSV/XLSX fixture.
- Run targeted Rust tests.
- Measure max RSS on fixture.
Expected benefit: safer scaling of 1C analytics.
### P2-4. Command execution boundary audit
Purpose: standardize shell/command execution safety.
Reason: `detmir-portal` already validates shell probe commands and tests
process-tree timeout cleanup, while other operational tools also execute
commands.
Operational impact: prevents future command injection or timeout regressions.
Risk: medium. Config-driven command execution must remain fail-closed.
Estimated effort: 4-7 days.
Affected modules:
- `adk-rust/crates/detmir-portal/src/main.rs`
- `adk-rust/crates/detmir-portal/src/production/limits.rs`
- `adk-rust/crates/aw-slo-monitor/src/main.rs`
- `adk-rust/crates/diag-and-manual-restart/src/main.rs`
- `adk-rust/crates/quality-gate/src/main.rs`
Acceptance criteria:
- Runtime command sources are classified.
- Config-driven commands reject shell control operators where applicable.
- Timeout tests cover child/grandchild cleanup.
- Logs do not expose secrets.
Validation steps:
- Targeted Rust tests.
- Clippy for affected crates.
- Secret scan.
Expected benefit: stronger fail-closed security posture.
## P3 - Long-term improvements after Production 1.0
### P3-1. Russian OS compatibility matrix
Purpose: document supported and unsupported target OS combinations.
Reason: `ROADMAP.md` lists Russian OS compatibility validation as planned.
Operational impact: reduces deployment surprises for new customers.
Risk: low-medium for current DetMir, higher for wider distribution.
Estimated effort: 2-4 weeks depending on test hosts.
Affected modules:
- `docs/registry/`
- `docs/INSTALLATION.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `ansible/`
- `windows/`
- `windows/installkit/`
Acceptance criteria:
- Matrix lists OS, role, test date, result and limitations.
- Unsupported combinations are explicit.
- No production defaults are changed just for claims.
Validation steps:
- Run install/deploy smoke per OS.
- Record evidence paths.
Expected benefit: clearer deployment support boundary.
### P3-2. Coverage threshold after baseline review
Purpose: prevent coverage decline after baseline stabilizes.
Reason: `docs/QUALITY_STATUS_RU.md` and `ROADMAP.md` state coverage threshold
is not enforced yet.
Operational impact: improves long-term regression resistance.
Risk: low for current production if kept advisory first.
Estimated effort: 1-2 weeks.
Affected modules:
- `.github/workflows/coverage.yml`
- `docs/QUALITY_STATUS_RU.md`
- `docs/REVIEW_CHECKLIST_RU.md`
- selected `adk-rust/` crates
Acceptance criteria:
- Initial threshold is based on measured baseline.
- Threshold starts advisory and becomes blocking only after stable history.
- Generated/fixture code exclusions are documented.
Validation steps:
- Run coverage workflow locally or in CI.
- Compare summary to baseline.
Expected benefit: gradual improvement in test discipline.
### P3-3. Capacity sizing guide from measured data
Purpose: provide measured sizing guidance beyond the current 5-user DetMir
deployment.
Reason: current production is small; future deployments need measured guidance
for AW SQLite, ClickHouse, Grafana, DLP light profile and Windows collector
load.
Operational impact: safer planning for larger deployments.
Risk: low for current production, medium for growth.
Estimated effort: 2-4 weeks after P0 load gate data exists.
Affected modules:
- `docs/SIZING_GUIDE_RU.md`
- `docs/DLP_RESOURCE_PROFILES_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `grafana/`
- `scripts/operational-maturity-check.mjs`
Acceptance criteria:
- Profiles exist for 5, 20, 50 and 100 users.
- Optional DLP/Hayabusa/Velociraptor resource costs are explicit.
- Guidance is based on measured harness output.
Validation steps:
- Run synthetic capacity scenarios.
- Update sizing doc with observed p95/RSS/storage data.
Expected benefit: production planning becomes evidence-based.
## Not Proposed Because Already Implemented
- Rust-first runtime direction.
- Public mirror CI, security scan and dependency review.
- `cargo audit`, `cargo deny`, `cargo machete`, `cargo tree --duplicates` and
advisory `cargo udeps` workflow coverage.
- DLP `core_only/disabled` production guardrails and load guard.
- Optional Security Finding Inbox / Hayabusa / Velociraptor separation from
Workforce hot path.
- Operational maturity offline harness.
- Branch protection, CODEOWNERS, PR template and review checklist mechanisms.
- Unused dependency cleanup: current `cargo machete --with-metadata` reports no
unused dependencies.
+246
View File
@@ -0,0 +1,246 @@
# Disaster Recovery Proof
Date: 2026-07-01
Scope: evidence-backed disaster recovery validation for the current
AWatch-rus / DetMir production deployment, based only on repository contents.
This document does not introduce new recovery procedures and does not change
production behavior.
## Result
Current proof level: partially proven.
Recovery confidence: 48 / 100.
The repository contains deployable components, restart/health checks, Windows
collector recovery automation, ActivityWatch DB merge support, retention guard
rails, and validation scripts. The repository does not yet contain evidence of
a completed separate-host restore for the primary repository/Gitea backup, and
does not implement fully automated restore for all runtime data stores.
Production 1.0 release implication: DR is not blocked by missing documentation,
but remains blocked by missing restore evidence for the repository/Gitea backup
and by missing automated restore coverage for several runtime stores.
## Status Definitions
- Implemented: repository contains executable automation or a concrete
validation script for the step.
- Partially implemented: repository contains part of the mechanism, but not an
end-to-end proven path.
- Manual: repository documents the step, but operator execution and evidence
capture are required.
- Missing: repository evidence shows the step is not implemented.
- Unsupported: repository code explicitly rejects or excludes the operation.
## Evidence Inventory
| Area | Status | Evidence |
| --- | --- | --- |
| General backup/recovery model | Manual | `docs/BACKUP_AND_RECOVERY_RU.md:59-71` defines the generic stop, preserve state, restore, start, health, smoke, and record-result sequence. |
| Production health endpoints | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:5-13` defines `/healthz`, `/readyz`, and `/metrics` checks; `docs/OPERATIONS_RUNBOOK_RU.md:18-47` defines expected semantics. |
| Post-recovery smoke | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:49-64` lists smoke scripts and requires them after recovery. |
| Operational maturity validation | Implemented | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59` requires `scripts/public_secret_pattern_check.py`, `scripts/operational-maturity-check.mjs`, and `quality-gate`. |
| Browser/operator validation | Manual | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:64-93` lists required operator pages and secret-handling rules. |
| DetMir production smoke | Implemented | `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119` defines `check-aw-full.sh`, `check-aw-data.sh`, and stable `SHARKON2025` host id handling. |
| ActivityWatch service startup | Implemented | `aw-server/activitywatch-server.service:6-14` defines env file, DB path, web path, restart policy, and restart delay. |
| ActivityWatch DB maintenance | Implemented | `aw-server/aw-db-maintenance.service:29-32` and `aw-server/aw-db-maintenance.timer:34-42` define guarded maintenance. |
| Readiness evidence bundle | Implemented | `aw-server/detmir-readiness.service:48-59` and `aw-server/detmir-readiness.timer:60-71` generate daily readiness bundles. |
| Server deploy and API startup validation | Implemented | `ansible/deploy_aw_server.yml:2365-2380` restarts `aw-server/activitywatch-server.service` and waits for `/api/0/info`. |
| ActivityWatch DB merge backup | Partially implemented | `ansible/deploy_aw_server.yml:2222-2350` checks/install merge binary, backs up target and legacy DBs, merges, and installs merged DB when enabled. |
| Rust restore planner | Partially implemented | `scripts/prod_backup_restore.sh:14-29` requires the Rust planner; `adk-rust/crates/prod-backup-restore/src/main.rs:100-105` explicitly rejects `--apply`. |
| Windows package rollback backup | Implemented | `windows/ActivityWatch.Windows.Common.psm1:148-247` backs up install root before replacement and cleans old install backups. |
| Windows recovery loop | Implemented | `windows/ActivityWatch.Windows.Common.psm1:1576-1597` writes recovery script; `windows/ActivityWatch.Windows.Common.psm1:2114-2167` runs the recovery loop. |
| Windows recovery scheduled task | Implemented | `windows/ActivityWatch.Windows.Common.psm1:2447-2495` registers `ActivityWatch Recovery`; `windows/ActivityWatch.Windows.Common.psm1:2656-2672` starts launch tasks and recovery task. |
| Windows deployment validation | Implemented | `ansible/deploy_aw_windows.yml:501-545` checks ActivityWatch API buckets; `ansible/deploy_aw_windows.yml:547-565` runs and fetches endpoint validation. |
| Windows post-deploy validation | Implemented | `ansible/post_validate_aw_windows.yml:55-120` starts recovery/launch tasks and waits for fresh worktime events. |
| Gitea backup | Manual / partially implemented | `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:7-23` documents path, script, timer, format, checksum, retention, and restore-tested status. |
| Gitea restore proof | Missing | `docs/registry/registry-evidence-manifest.json:26-40` records `restore_tested=false` and `production_ready=false`. |
| ClickHouse 1C runtime store | Partially implemented | `clickhouse-1c/docker-compose.yml:1-19` defines the service and persistent Docker volume; no restore automation is present in the repository. |
| ClickHouse Workforce runtime store | Partially implemented | `clickhouse-workforce/docker-compose.yml:1-17` defines the service and persistent Docker volume; no restore automation is present in the repository. |
| Grafana/Prometheus runtime store | Partially implemented | `grafana-1c/docker-compose.yml:36-79` defines Prometheus/Grafana volumes and Prometheus retention; no Grafana data restore automation is present in the repository. |
| Retention boundaries | Implemented for documented cleanup scopes | `docs/RETENTION_POLICY_RU.md:65-102` documents retention and recovery impact for persistent stores. |
## End-to-End Recovery Chain
| Step | Status | Current evidence | Proof conclusion |
| --- | --- | --- | --- |
| Repository | Partially implemented | Primary self-hosted Gitea repository is recorded in `docs/registry/registry-evidence-manifest.json:6-10`; backup configuration is recorded in `docs/registry/registry-evidence-manifest.json:26-40`. | Source repository is identified and backup metadata exists, but restore proof is missing. |
| Configuration | Manual | Git stores sanitized templates and explicitly excludes secrets/live DBs in `docs/BACKUP_AND_RECOVERY_RU.md:29-46`. | Configuration can be reconstructed from repository templates plus customer secret store, but secrets are intentionally external. |
| Deployment | Implemented | Server deployment restarts ActivityWatch and waits for API readiness in `ansible/deploy_aw_server.yml:2365-2380`; Windows deployment runs package install, recovery tasks, and smoke checks in `ansible/deploy_aw_windows.yml:279-565`. | Server and Windows deployment are executable from repository playbooks. |
| Restore | Partially implemented | AW DB merge path exists in `ansible/deploy_aw_server.yml:2222-2350`; Rust planner emits a plan but rejects `--apply` in `adk-rust/crates/prod-backup-restore/src/main.rs:100-105`; Gitea restore is manual and untested per `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`. | Restore is only partially proven. Several runtime stores require manual or future automation. |
| Startup | Implemented | `aw-server/activitywatch-server.service` has restart policy in `aw-server/activitywatch-server.service:6-14`; Windows recovery task and loop are implemented in `windows/ActivityWatch.Windows.Common.psm1:2114-2167` and `:2447-2495`. | Repository supports service startup and collector recovery startup. |
| Health | Implemented | `/healthz`, `/readyz`, `/metrics` are documented in `docs/OPERATIONS_RUNBOOK_RU.md:5-47`; server deployment waits for `/api/0/info` in `ansible/deploy_aw_server.yml:2372-2380`. | Health checks are present and part of deploy/recovery validation. |
| Operational validation | Implemented | Operational gates are documented in `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59`; production smoke is documented in `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`. | Repository has repeatable operational validation commands. |
| Pilot validation | Implemented | `docs/OPERATIONS_RUNBOOK_RU.md:49-64` lists pilot/demo/deployment smoke scripts after recovery; `scripts/pilot-validation-smoke.mjs` and `scripts/deployment-readiness-smoke.mjs` exist. | Pilot/deployment smoke validation is implemented as repository scripts. |
| Ready | Manual / conditional | General recovery procedure requires result recording in `docs/BACKUP_AND_RECOVERY_RU.md:59-71`; Gitea manifest still says `production_ready=false` in `docs/registry/registry-evidence-manifest.json:37-40`. | Ready can only be claimed after external restore evidence is captured and manifest gaps are closed. |
## Current Recovery Capability
### Server
Implemented:
- ActivityWatch server restart and API readiness wait through Ansible.
- Systemd restart-on-failure for `aw-server/activitywatch-server.service`.
- Guarded SQLite maintenance and daily readiness bundle timers.
- Limited AW DB merge/recovery-like flow with pre-merge backups.
Partially implemented:
- Generic production restore planning through `prod-backup-restore`, because the
Rust binary builds a plan but rejects apply mode.
- ActivityWatch DB restore, because the repository implements legacy DB merge,
but not a generic restore-selected-backup command.
Missing:
- Evidence of a completed end-to-end AW DB restore drill.
- Automated rollback from a failed DB merge to the pre-merge backup.
### Windows/RDP
Implemented:
- Package replacement with install-root backup.
- Recovery script generation.
- Scheduled `ActivityWatch Recovery` task.
- Long-running recovery loop that restarts collectors/tasks for live sessions.
- Post-deploy validation that waits for fresh ActivityWatch worktime events.
Partially implemented:
- Full Windows state restore, because install backups exist but
`C:\ProgramData\AWatch-rus` restore as a whole is not implemented.
### Repository/Gitea
Manual / partially implemented:
- Backup target, script name, systemd unit/timer names, format, checksum, and
retention are documented.
- Restore procedure is documented as an outline for a separate server.
Missing:
- Actual separate-host restore evidence.
- Manifest update proving `restore_tested=true`.
- Offsite copy evidence.
### ClickHouse, Grafana, Prometheus
Partially implemented:
- Docker Compose files define persistent volumes and restart policies.
- Prometheus retention is configurable through compose.
- Retention policy documents recovery impact for ClickHouse, Grafana, and
Prometheus stores.
Missing:
- Automated backup and restore for ClickHouse 1C volume.
- Automated backup and restore for ClickHouse Workforce volume.
- Automated Grafana data restore.
- End-to-end restore evidence for these stores.
## Current Limitations
1. Gitea backup cannot be called production-ready until a separate-host restore
is performed and recorded. Evidence: `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:3-5`,
`docs/registry/registry-evidence-manifest.json:37-40`.
2. `prod-backup-restore --apply` is unsupported by code. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
3. ActivityWatch DB recovery is limited to legacy-root merge and does not prove
generic restore from a selected backup file. Evidence:
`ansible/deploy_aw_server.yml:2222-2350`.
4. ClickHouse/Grafana/Prometheus data volumes are declared, but repository
restore automation is not present. Evidence:
`clickhouse-1c/docker-compose.yml:13-19`,
`clickhouse-workforce/docker-compose.yml:11-17`,
`grafana-1c/docker-compose.yml:36-79`.
5. Customer secrets are intentionally outside Git. Recovery therefore requires
access to the customer secret store and cannot be proven from repository
contents alone. Evidence: `docs/BACKUP_AND_RECOVERY_RU.md:29-46`.
6. Evidence/customer data must not be placed in the public repository. Evidence:
`docs/BACKUP_AND_RECOVERY_RU.md:91-98`.
## Estimated Recovery Sequence
This sequence is the current evidence-backed chain. Steps marked manual require
operator execution and external evidence capture.
1. Repository availability: clone the primary Gitea repository or validated
mirror. Status: partially implemented. Evidence:
`docs/registry/registry-evidence-manifest.json:6-10`.
2. Repository restore, if primary Gitea is lost: follow the Gitea restore
outline on a separate host, verify checksum, run Gitea checks, and record
evidence. Status: manual / missing proof. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`.
3. Configuration recovery: restore sanitized templates from Git and secrets
from the approved customer secret store. Status: manual. Evidence:
`docs/BACKUP_AND_RECOVERY_RU.md:29-46`.
4. Server deployment: run server deployment playbook and wait for
`/api/0/info`. Status: implemented. Evidence:
`ansible/deploy_aw_server.yml:2365-2380`.
5. AW DB merge path, only when legacy-root merge is explicitly enabled: back up
target and legacy DBs, merge, install merged DB. Status: partially
implemented. Evidence: `ansible/deploy_aw_server.yml:2222-2350`.
6. Windows/RDP deployment: run Windows deployment playbook, deploy package,
configure recovery tasks, and validate bucket events. Status: implemented.
Evidence: `ansible/deploy_aw_windows.yml:279-565`.
7. Windows post-restore validation: start recovery/launch tasks and wait for
worktime events. Status: implemented. Evidence:
`ansible/post_validate_aw_windows.yml:55-120`.
8. Service startup validation: check systemd status, `/healthz`, `/readyz`,
`/metrics`, and `/api/0/info`. Status: implemented. Evidence:
`docs/OPERATIONS_RUNBOOK_RU.md:5-47`,
`ansible/deploy_aw_server.yml:2372-2380`.
9. Production smoke: run `check-aw-full.sh`, `check-aw-data.sh`, and contour
smoke with stable `SHARKON2025` logical host id. Status: implemented.
Evidence: `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`.
10. Operational maturity validation: run secret scan, operational maturity
check, and quality gate. Status: implemented. Evidence:
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:49-59`.
11. Pilot/deployment readiness validation: run pilot and deployment smoke
scripts after recovery. Status: implemented. Evidence:
`docs/OPERATIONS_RUNBOOK_RU.md:49-64`.
12. Ready decision: record recovery result and do not claim production-ready DR
until missing restore evidence is closed. Status: manual / conditional.
Evidence: `docs/BACKUP_AND_RECOVERY_RU.md:59-71`,
`docs/registry/registry-evidence-manifest.json:37-40`.
## Evidence Gaps
| Gap | Status | Required evidence before closing |
| --- | --- | --- |
| Gitea separate-host restore | Missing | Backup filename, SHA256 verification output, Gitea version, restore duration, post-restore checks, clone/access proof, and manifest update. |
| Generic AW DB restore | Missing | Tested command or runbook restoring a selected backup into active DB with checksum and rollback evidence. |
| ClickHouse 1C restore | Missing | Tested volume/table backup and restore evidence for `clickhouse_1c_data`. |
| ClickHouse Workforce restore | Missing | Tested volume/table backup and restore evidence for `clickhouse_workforce_data`. |
| Grafana data restore | Missing | Tested restore evidence for `grafana-data` or explicit proof that provisioned dashboards plus documented credentials are sufficient. |
| Off-host backup copy | Missing | Repository evidence of destination, retention, checksum policy, access policy, and restore test. |
## Recommended Future Automation
These are future tasks, not implemented by this document:
1. Add a non-destructive restore drill checklist artifact that records exact
backup filename, checksum, host, duration, and post-restore checks.
2. Add a separate-host Gitea restore evidence template and only then update
`restore_tested` in `docs/registry/registry-evidence-manifest.json`.
3. Add tested backup/restore automation for ClickHouse 1C and Workforce volumes
or explicitly document that those stores are rebuilt from source exports.
4. Add a generic ActivityWatch DB restore runbook with rollback command and
checksum verification.
5. Add a single post-restore validation command that runs health, operational
maturity, pilot validation, and deployment readiness in the documented order.
## Release Decision
DR proof is sufficient to show that the repository contains significant
recoverability mechanisms and validation gates.
DR proof is not sufficient to claim full Production 1.0 disaster recovery until
the missing separate-host restore evidence and runtime store restore gaps are
closed.
+311
View File
@@ -0,0 +1,311 @@
# Production Evidence Gaps
Date: 2026-07-01
Scope: final evidence still missing before Release Candidate readiness for the
current AWatch-rus / DetMir production deployment.
This document does not add functionality, redesign recovery, or define new
tests. It records only evidence gaps confirmed by existing repository backlog,
runbooks, audit reports, validation scripts, and registry-readiness documents.
## Summary
The repository already contains operational validation scripts, pilot/deployment
smoke scripts, retention policy, recovery audit, disaster recovery proof, and
release evidence tooling. The remaining gap is not lack of plans. The remaining
gap is live or controlled-environment evidence proving that the current
production deployment and release candidate can be recovered, traced, rebuilt,
and operated under expected load.
## Missing Evidence Items
| ID | Evidence gap | RC priority | Requires live production environment |
| --- | --- | --- | --- |
| PEG-001 | Production binary parity evidence | P0 | Yes |
| PEG-002 | Separate-host Gitea restore proof | P0 | Separate restore host; production backup input |
| PEG-003 | Runtime data restore proof for production stores | P0 | Yes or isolated restore clone |
| PEG-004 | First controlled release evidence package from `awatch-build-01` | P0 | Controlled build-runner, not production runtime |
| PEG-005 | Portal/worktime hot-path load evidence | P0 | Prefer production-like or live low-risk window |
| PEG-006 | Capacity baseline and headroom evidence | P1 | Yes |
| PEG-007 | Backup verification and off-host backup evidence | P1 | Yes |
| PEG-008 | Install kit reproducibility and stale payload evidence | P1 | Controlled build/staging environment |
| PEG-009 | Reviewed PR / release governance evidence | P1 | GitHub/Gitea governance environment |
## PEG-001: Production Binary Parity Evidence
Why it matters: Release Candidate confidence depends on proving that binaries
actually running in production match reviewed release artifacts. Otherwise test
results can refer to one binary while production runs another.
Current state: repository gate implemented by TASK_072; live production
evidence still missing. The existing local artifact check verifies release
artifacts, and `scripts/check_production_binary_parity.py` now validates
deployed unit/timer/task to production SHA256 and source crate mapping against
local release artifacts and repository Git SHA. Release Candidate approval still
requires an operator-collected production evidence JSON from the current
production deployment.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:65-105`
- `PRODUCTION_READINESS_REPORT.md:209-212`
- `scripts/check_detmir_rust_release_artifacts.sh`
- `scripts/check_production_binary_parity.py`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
How it could be verified: collect production binary paths from actual
systemd/timer/Windows scheduled task inventory, compute production SHA256,
save evidence JSON, then run `scripts/check_production_binary_parity.py` to
compare with local release artifact SHA256 and record
`service/timer/task -> binary path -> crate -> runtime role -> production sha256
-> release sha256 -> git sha`.
Expected operational benefit: removes stale-binary ambiguity during incident
response, rollback, and RC approval.
Risk if left unverified: production may run stale or locally patched binaries
while release evidence and tests refer to different artifacts.
## PEG-002: Separate-Host Gitea Restore Proof
Why it matters: Source repository backup is not a recovery capability until a
restore has been proven on a separate host.
Current state: Missing. Registry evidence explicitly records
`restore_tested=false` and `production_ready=false`; the restore runbook states
that backup cannot be production-ready until a separate test restore is
completed.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:154-188`
- `RECOVERY_AUDIT.md:250-263`
- `DISASTER_RECOVERY_PROOF.md`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`
- `docs/registry/registry-evidence-manifest.json:26-40`
How it could be verified: execute the existing Gitea restore runbook on a
separate host, verify backup SHA256, run `gitea doctor check`, verify restored
repository access/clone, record backup filename, checksum, Gitea version,
restore duration, post-restore output, and rollback notes.
Expected operational benefit: proves the source repository can be recovered
after primary Git/Gitea loss.
Risk if left unverified: repository loss recovery remains assumed rather than
proven.
## PEG-003: Runtime Data Restore Proof for Production Stores
Why it matters: Service restart is not enough for disaster recovery if primary
runtime data stores cannot be restored after corruption or loss.
Current state: Missing / partially implemented. Recovery automation exists for
service restart and Windows collector restart. Generic restore evidence is still
missing for ActivityWatch active SQLite DB, ClickHouse 1C, ClickHouse Workforce,
Grafana data, Prometheus TSDB, DLP state/evidence, Hayabusa archive, Windows
state root, diagnostic bundles, and release evidence.
Repository evidence:
- `RECOVERY_AUDIT.md:272-281`
- `RECOVERY_AUDIT.md:379-428`
- `DISASTER_RECOVERY_PROOF.md`
- `docs/RETENTION_POLICY_RU.md:65-102`
How it could be verified: for each production store, use existing backup or
restore documentation where present, perform restore into an isolated target,
then run existing health and validation scripts. Where the repository says
restore is not currently implemented, record the gap rather than inventing a
procedure.
Expected operational benefit: identifies which production data can actually be
recovered and which data still depends on external/customer backup handling.
Risk if left unverified: an outage may be recoverable at the service level but
not at the data level.
## PEG-004: First Controlled Release Evidence Package from `awatch-build-01`
Why it matters: GitHub Actions are public validation only. RC release authority
requires a controlled build-runner evidence package with source archive, binary
archive, checksums, metadata, logs, and manifest.
Current state: Tooling exists and was strengthened by TASK_070, but the
registry-readiness documents still require the first real release evidence build
on the Russian build-runner. The build-runner status is planned, not proven
production-ready.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:107-149`
- `docs/registry/RU_BUILD_RUNNER_READINESS_RU.md`
- `docs/registry/BUILD_RUNNER_SETUP_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_RUNBOOK_RU.md`
- `docs/registry/RELEASE_EVIDENCE_MANIFEST_RU.md`
- `docs/registry/registry-evidence-manifest.json:41-75`
- `scripts/build_release_evidence.sh`
- `scripts/check_release_evidence.sh`
How it could be verified: run `scripts/build_release_evidence.sh` on the
controlled runner for the RC commit, then run
`scripts/check_release_evidence.sh <evidence-dir>` and preserve the generated
manifest, logs, source archive, binary archive, SHA256SUMS, cargo metadata/tree,
and documented skips.
Expected operational benefit: provides auditable release provenance independent
of the public mirror.
Risk if left unverified: RC could be validated only by public mirror CI rather
than by the release authority expected by the registry-readiness contour.
## PEG-005: Portal/Worktime Hot-Path Load Evidence
Why it matters: Production 1.0 should not regress under report, prewarm,
ActivityWatch query, cache, and operator portal load.
Current state: Missing. The backlog still lists `P0-6. Portal/worktime hot-path
load gate`. Existing operational maturity checks include offline bounded-load
validation, but repository evidence still requires an explicit portal/worktime
hot-path gate before Production 1.0.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:281-319`
- `scripts/operational-maturity-check.mjs`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:95-119`
How it could be verified: run the existing operational maturity bounded-load
harness and production smoke sequence against a production-like or controlled
live window, record p95 latency, memory growth, cache/stale behavior, AW query
duration, and failure semantics. Keep any heavy job advisory/scheduled rather
than blocking fast smoke.
Expected operational benefit: proves that the current 5-user production
deployment has headroom for normal operator/report usage.
Risk if left unverified: RC may pass functional checks while still being fragile
under repeated report/prewarm/operator access.
## PEG-006: Capacity Baseline and Headroom Evidence
Why it matters: Operators need a measured baseline for CPU, RAM, disk, queue,
ClickHouse, Grafana, and ActivityWatch behavior before scaling beyond the
current small deployment.
Current state: Missing / P1. The roadmap explicitly places capacity monitoring
after P0 load gate data. Retention policy documents storage areas and cleanup
impact, but not production headroom evidence.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:778-805`
- `docs/RETENTION_POLICY_RU.md:65-102`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md:187-209`
How it could be verified: collect production baseline snapshots for disk usage,
service health, ClickHouse table sizes, queue/backlog state, and operational
smoke output over an agreed observation window.
Expected operational benefit: gives operators thresholds for safe growth and
early warning before disk or load incidents.
Risk if left unverified: capacity problems may be discovered only after
operator-visible degradation.
## PEG-007: Backup Verification and Off-Host Backup Evidence
Why it matters: Retention prevents uncontrolled growth, but backup verification
proves recoverability. A local backup without checksum/off-host evidence can
fail during actual disaster recovery.
Current state: Missing / partial. Gitea backup is documented with checksum and
timer metadata. The recovery audit records missing scheduled full AW DB backup,
off-host backup copy, and restore ownership for several runtime stores.
Repository evidence:
- `RECOVERY_AUDIT.md:128-149`
- `RECOVERY_AUDIT.md:306-321`
- `RECOVERY_AUDIT.md:326-338`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:25-40`
- `docs/RETENTION_POLICY_RU.md:65-102`
How it could be verified: use existing backup locations/runbooks where present,
verify checksums, confirm backup age/retention, confirm off-host copy status if
owned outside the repository, and record components that remain without a repo
backup/restore path.
Expected operational benefit: separates "data is retained" from "data can be
restored".
Risk if left unverified: backups may be absent, stale, local-only, or
unreadable when needed.
## PEG-008: Install Kit Reproducibility and Stale Payload Evidence
Why it matters: Windows installer payload drift can deploy old collectors even
when repository scripts and CI are current.
Current state: Missing / P1. The backlog lists install kit reproducibility and
stale payload gate as open.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:441-475`
- `scripts/rebuild_install_kit.sh`
- `scripts/validate_install_kit.sh`
- `scripts/check_install_kit_vs_repo.sh`
- `adk-rust/crates/rebuild-install-kit/`
How it could be verified: rebuild the install kit on the controlled runner,
generate or verify payload manifest with source commit and SHA256 values, then
compare packaged collector payloads with repository state and release artifacts.
Expected operational benefit: prevents deploying stale Windows collector
payloads during RC rollout.
Risk if left unverified: endpoint deployment can silently diverge from the
reviewed release.
## PEG-009: Reviewed PR / Release Governance Evidence
Why it matters: Production 1.0 closure requires visible evidence that protected
branch, CODEOWNERS, and PR review processes are not only documented but used for
release-quality changes.
Current state: Missing / P1. Branch protection evidence exists, but the backlog
still lists reviewed PR and release governance evidence as open.
Repository evidence:
- `DEVELOPMENT_PLAN_NEXT.md:519-552`
- `docs/BRANCH_PROTECTION_EVIDENCE_RU.md`
- `docs/PR_REVIEW_EVIDENCE_RU.md`
- `scripts/registry_readiness_check.sh`
How it could be verified: record a reviewed PR flow with required checks,
CODEOWNERS/reviewer approval, no admin bypass, and final merge evidence for the
RC branch or release preparation branch.
Expected operational benefit: proves that Production 1.0 changes pass the
intended governance workflow.
Risk if left unverified: release governance remains documented but not proven
on the actual RC workflow.
## Release Candidate Evidence Gate
Release Candidate readiness should not be declared until the P0 evidence gaps
are closed or explicitly accepted by the operator with written risk acceptance:
1. PEG-001 production binary parity evidence.
2. PEG-002 separate-host Gitea restore proof.
3. PEG-003 runtime data restore proof or documented operator acceptance for
stores without repository restore support.
4. PEG-004 controlled release evidence package from `awatch-build-01`.
5. PEG-005 portal/worktime hot-path load evidence.
P1 evidence can remain after RC only if it is explicitly tracked as release
follow-up and does not invalidate the operator's Production 1.0 risk decision.
+332
View File
@@ -0,0 +1,332 @@
# TASK_065: Production readiness report
Audit date: 2026-07-01.
Scope: repository-level engineering and operational audit for AWatch-rus
Production 1.0 readiness.
Production context: the project is already deployed in a small company
environment with approximately 5 RDP users. Stability, rollback, diagnostics
and maintainability have priority over new functionality.
## Method
The audit used repository evidence only. No readiness claim in this report is
based on an unverified assumption.
Reviewed evidence:
- `README.md`, `CONTRIBUTING.md`, `SECURITY.md`, `ROADMAP.md`.
- `docs/PROJECT_STATUS_RU.md`, `docs/RESIDUAL_RISKS_RU.md`,
`docs/QUALITY_STATUS_RU.md`, `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`,
`docs/DLP_OPTIONAL_RUNTIME_RU.md`, `docs/DLP_RESOURCE_PROFILES_RU.md`,
`docs/POWERSHELL_SCRIPT_STATUS_MATRIX_RU.md`.
- `docs/registry/*` including `registry-evidence-manifest.json`.
- `.github/workflows/*`.
- `scripts/*`, `scripts/detmir-full-diagnostics/*`.
- `adk-rust/` workspace manifests and crate tree.
- `ansible/`, `aw-server/`, `windows/`, `proxmox/`, `clickhouse-1c/`,
`grafana/`, `pfsense/`.
- `configs/operational-maturity-contract.json`.
Commands used for evidence gathering:
```bash
git ls-files | wc -l
find adk-rust/crates -maxdepth 2 -name Cargo.toml
cargo metadata --locked --format-version 1
cargo audit --deny warnings
cargo machete --with-metadata
cargo tree --duplicates --locked
cargo deny check --config ../deny.toml --hide-inclusion-graph --show-stats
wc -l adk-rust/crates/detmir-portal/src/main.rs adk-rust/crates/aw-windows-telemetry/src/main.rs proxmox/tsj_guardian_bot.py adk-rust/crates/worktime-api/src/main.rs ansible/deploy_aw_server.yml
sha256sum scripts/aw-contour-diag.sh scripts/detmir-full-diagnostics/aw-contour-diag.sh scripts/check_production_inventory_placeholders.sh scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh
rg -n "CLICKHOUSE_PASSWORD|--password|password" clickhouse-1c/ops clickhouse-1c/ai clickhouse-1c/etl
```
## Implemented Capability Inventory
### Core Runtime
- Rust workspace under `adk-rust/` with 58 crates.
- ActivityWatch checks and wrappers: `check-aw-data`, `check-aw-full`,
`aw-health-check`, `aw-rus-healthd`, `aw-slo-monitor`.
- DetMir portal: Rust HTML/API portal with contracts, metrics, readiness,
reports, operator/manager/security views and role-based access logic.
- Worktime stack: `worktime-api`, `worktime-prewarm`,
`worktime-influx-exporter`, `worktime-autoheal`,
`worktime-ui-bridge`, `rdp-worktime-report`.
- Windows telemetry: `aw-windows-telemetry` for collectors, validation,
file-1C upload, DLP evidence sync and collector guard paths.
- Release/install tooling: `rebuild-install-kit`, `validate-install-kit`,
`verify-innosetup-installer`, `check-install-kit-vs-repo`.
### Workforce / 1C / ClickHouse
- `aw-workforce-ingest` and `aw-1c-ingest` exist for workforce and 1C
ingestion paths.
- `clickhouse-1c/` includes ETL, SQL, Grafana provisioning and operational
wrappers.
- Grafana dashboards are version-controlled under `grafana/` and related
ClickHouse/Grafana directories.
### Security / DLP / Forensics
- DLP server-side helpers exist as Rust crates:
`dlp-policy-engine`, `dlp-case-management`, `dlp-compliance`,
`dlp-aggregator`, `dlp-health-check`, `dlp-content-analyzer`,
exporters/senders.
- DLP production runtime is intentionally conservative:
`core_only/disabled` default with documented `light` profile and load guard.
- Security Finding Inbox and Hayabusa/Velociraptor findings paths are optional
and separated from Workforce hot path.
- Hayabusa tooling exists under `hayabusa-tools` and `aw-server/hayabusa/`.
### Deployment / Operations
- Ansible deployment exists for server and Windows contours.
- Windows deployment and recovery scripts exist under `windows/`.
- Proxmox/pfSense support assets exist under `proxmox/` and `pfsense/`.
- Operational wrappers exist at root and under `scripts/`.
- `scripts/operational-maturity-check.mjs` validates API compatibility,
fixtures, fault injection, bounded load, config, systemd, ClickHouse
migration and observability contracts.
### Governance / Release
- Public CI workflows exist.
- Security workflow includes cargo audit, cargo deny, secret pattern scan and
dependency review.
- Dependency hygiene workflow includes cargo metadata, machete, duplicates,
audit, deny and advisory udeps.
- Registry-readiness documentation exists under `docs/registry/`.
- CODEOWNERS, PR template, review checklist and branch protection evidence docs
exist.
## Confirmed Gap Analysis
### Technical Debt
- Large modules increase review risk:
- `adk-rust/crates/detmir-portal/src/main.rs`: 14200 lines.
- `adk-rust/crates/aw-windows-telemetry/src/main.rs`: 6411 lines.
- `proxmox/tsj_guardian_bot.py`: 4610 lines.
- `adk-rust/crates/worktime-api/src/main.rs`: 3988 lines.
- `ansible/deploy_aw_server.yml`: 3099 lines.
- PowerShell fallback remains necessary and documented; it is not dead code, but
it increases parity and validation burden.
### Duplicated Logic
- Exact duplicate scripts confirmed by SHA256:
- `scripts/aw-contour-diag.sh`
- `scripts/detmir-full-diagnostics/aw-contour-diag.sh`
- `scripts/check_production_inventory_placeholders.sh`
- `scripts/detmir-full-diagnostics/check_production_inventory_placeholders.sh`
### Outdated Or Pending Documentation
- `docs/PROJECT_STATUS_RU.md` still records first reviewed PR evidence as
pending and contains historical required check names.
- `docs/RESIDUAL_RISKS_RU.md` records Gitea restore test, build-runner,
release evidence and legal package as open.
- `ROADMAP.md` records coverage threshold and Russian OS compatibility as
planned, not complete.
### Obsolete Or Deprecated Components
- Some Ansible DLP roles are explicitly marked deprecated because they deployed
old service paths.
- `serde_yaml 0.9.34+deprecated` is documented as a medium third-party risk.
- Legacy scripts remain as fallback/reference; they should not be removed
without parity gates.
### Missing Operational Checks
- No confirmed repository check currently proves production binary SHA parity
across all actually running units/timers/tasks and local release artifacts.
- Retention/cleanup policy for long-lived state/evidence/diagnostic output is
not yet complete.
- Existing operational maturity bounded load is useful, but does not yet cover
Production 1.0 scale scenarios for 5/20/50 users and portal/worktime prewarm.
### Missing Tests
- Load regression tests for portal/worktime full report and prewarm hot paths
are not yet sufficient for Production 1.0 scale confidence.
- Windows Rust validation parity still needs canary evidence against the
PowerShell validation path.
### Security Gaps
- ClickHouse/1C ops wrappers pass `CLICKHOUSE_PASSWORD` via `--password`, which
exposes secrets in process argv.
- `cargo deny` passes but currently permits non-blocking duplicate/wildcard
dependency warnings. This is not an immediate vulnerability, but it needs a
Production 1.0 baseline.
## Readiness Scores
Scoring scale:
- 90-100: production-ready with evidence.
- 75-89: strong, but with bounded gaps.
- 60-74: usable in current production, but not yet 1.0 release-grade.
- below 60: material blocker.
| Category | Score | Justification |
|---|---:|---|
| Architecture status | 82 | Rust-first runtime, documented boundaries and conservative DLP separation exist. Large modules remain maintainability risk. |
| Repository health | 80 | 910 tracked files, clear ownership areas and runbooks. Some duplicate scripts and historical docs remain. |
| Dependency health | 78 | Audit and machete pass; 349 packages; cargo deny exits 0. Remaining deny warnings and duplicate roots need baseline policy. |
| CI health | 82 | CI/security/coverage/dependency/operational workflows exist. Toolchain drift remains between pinned `1.94.0` and floating `stable`. |
| Operational maturity | 84 | Offline operational maturity harness covers compatibility, fixtures, fault injection, bounded load, config and observability. Production-scale load gate still needed. |
| Documentation status | 78 | Extensive docs and runbooks exist. Current-state docs need cleanup around historical statuses and pending evidence. |
| Security status | 76 | Secret scan, audit, deny and conservative claims exist. ClickHouse password-in-argv is a direct hygiene gap. |
| Testing status | 80 | Full Rust pipeline recently passed and operational smokes exist. Missing scale/load and Windows parity evidence remain. |
| Deployment readiness | 76 | Ansible, Windows install kit, runbooks and wrappers exist. Install kit stale-payload gate and production binary parity still missing. |
| Upgrade readiness | 72 | Release scripts and rollback docs exist, but controlled release evidence and binary parity are not proven. |
| Recovery readiness | 70 | Recovery runbooks and backups exist; Gitea restore test is not done. |
| Configuration validation | 82 | Operational maturity validates JSON/YAML/systemd/ClickHouse files; production inventory placeholder checks exist. Coverage must be extended to retention/binary parity. |
| Observability | 84 | Metrics contract and operational maturity observability checks exist; capacity metrics need scale scenarios. |
| Support readiness | 78 | Many runbooks exist and DetMir guardrails are strong; docs need current-state cleanup for 1.0. |
| Maintainability | 72 | Strong tests and Rust-first direction, but large files and fallback parity increase maintenance cost. |
Overall Production Readiness Score: **78 / 100**.
Recommended release decision: **not yet Production 1.0**. The project is fit
for the current small production/pilot environment with conservative runtime
guardrails, but Production 1.0 should wait until P0 blockers in
`DEVELOPMENT_PLAN_NEXT.md` are closed and evidenced.
## Risk Assessment
### High Risk
1. Production binary drift
- Probability: medium.
- Impact: high.
- Description: running binaries may not match reviewed release artifacts.
- Mitigation: implement production binary parity gate.
- Complexity: medium.
2. Missing controlled release evidence
- Probability: high.
- Impact: high.
- Description: GitHub mirror validation is not release evidence.
- Mitigation: controlled runner release evidence build.
- Complexity: medium.
3. Untested repository restore
- Probability: medium.
- Impact: high.
- Description: backup exists, but restore is not proven.
- Mitigation: separate-host restore drill.
- Complexity: medium.
4. Unbounded operational artifacts
- Probability: medium.
- Impact: high.
- Description: state/evidence/diagnostic artifacts can accumulate and fill
disks.
- Mitigation: allowlisted retention and cleanup policy.
- Complexity: medium.
5. ClickHouse password in process argv
- Probability: high on affected scripts.
- Impact: high.
- Description: local process listing can expose credentials.
- Mitigation: remove password from argv and verify with `ps`.
- Complexity: low-medium.
6. Portal/worktime hot-path overload
- Probability: medium.
- Impact: high.
- Description: full report/snapshot prewarm remains documented as CPU/IO
expensive.
- Mitigation: Production 1.0 load gate with synthetic datasets.
- Complexity: medium-high.
### Medium Risk
1. CI toolchain drift
- Probability: medium.
- Impact: medium.
- Mitigation: align workflows to pinned toolchain.
- Complexity: low.
2. Dependency warning drift
- Probability: medium.
- Impact: medium.
- Mitigation: baseline current warnings and block new unapproved warnings.
- Complexity: medium.
3. Windows validation parity gap
- Probability: medium.
- Impact: medium-high.
- Mitigation: Rust/PowerShell canary comparison.
- Complexity: medium.
4. Install kit stale payload
- Probability: medium.
- Impact: medium-high.
- Mitigation: install kit manifest and validation gate.
- Complexity: medium.
5. Documentation drift
- Probability: high.
- Impact: medium.
- Mitigation: current-state cleanup and historical labeling.
- Complexity: low-medium.
6. Large module maintainability
- Probability: high.
- Impact: medium.
- Mitigation: incremental extraction with tests.
- Complexity: medium-high.
### Low Risk
1. Exact duplicate diagnostic scripts
- Probability: medium.
- Impact: low-medium.
- Mitigation: canonical implementation or drift check.
- Complexity: low.
2. Coverage threshold not yet enforced
- Probability: medium.
- Impact: low for current production, medium long-term.
- Mitigation: baseline review, advisory threshold, later blocking policy.
- Complexity: medium.
3. Russian OS compatibility not yet matrixed
- Probability: low for current DetMir, medium for wider distribution.
- Impact: medium for new deployments.
- Mitigation: compatibility matrix with evidence.
- Complexity: medium-high.
## Known Limitations
- The audit did not claim legal readiness, certification or registry
submission completion.
- The audit did not validate live production services during this documentation
update.
- Heavy DLP, Loki and always-on Velociraptor are intentionally not required for
Production 1.0.
- PowerShell fallback remains a supported rollback/support path until parity
evidence allows retirement.
## Release Readiness Summary
Production 1.0 should be blocked on:
1. production binary parity evidence;
2. controlled release evidence build;
3. Gitea restore drill;
4. bounded retention/cleanup;
5. ClickHouse password argv fix;
6. portal/worktime hot-path load gate.
After these are complete, the project can reasonably move from current small
production/pilot readiness to Production 1.0 readiness, assuming validation
passes and no new runtime regressions are introduced.
+560 -9
View File
@@ -1,11 +1,18 @@
# AWatch-rus
[![CI](https://github.com/igor04091968/AWatch-rus/actions/workflows/ci.yml/badge.svg)](https://github.com/igor04091968/AWatch-rus/actions/workflows/ci.yml)
[![Security](https://github.com/igor04091968/AWatch-rus/actions/workflows/security.yml/badge.svg)](https://github.com/igor04091968/AWatch-rus/actions/workflows/security.yml)
[![Coverage](https://github.com/igor04091968/AWatch-rus/actions/workflows/coverage.yml/badge.svg)](https://github.com/igor04091968/AWatch-rus/actions/workflows/coverage.yml)
AWatch-rus - программный комплекс операционного контроля,
технического аудита, оценки трудоотдачи сотрудников и мониторинга
корпоративной ИТ-инфраструктуры на базе ActivityWatch, Rust-сервисов
автоматизации, Grafana/Prometheus-витрин и модулей расследования инцидентов.
Проект не позиционируется как сертифицированная DLP/SIEM/EDR/XDR/СЗИ,хотя DLP,evidence и Hayabusa используются в проекте.
Проект не позиционируется как сертифицированная DLP/SIEM/EDR/XDR/СЗИ,
не заявляет ML/LLM UEBA и не подменяет штатные средства защиты, хотя
DLP-сигналы, evidence и Hayabusa используются как аналитические и
расследовательские слои.
## Назначение
@@ -21,9 +28,13 @@ AWatch-rus - программный комплекс операционного
## Rust-first runtime
Основной серверный runtime AWatch-rus переведен на Rust: status/check/auto-heal,
Основной серверный runtime AWatch-rus переведен на Rust(ранее использовался инструментарий powershell):status/check/auto-heal,
SLO, worktime, DLP server-side helpers, evidence и install-kit tooling.
Оставшиеся PowerShell runtime/fallback/installer/repair scripts сохраняются
как документированный слой отката, установки и поддержки до отдельной задачи
удаления с burn-in периодом, canary test, rollback plan и acceptance gate.
Python, присутствующий в коде репозитория, остается для вспомогательных направлений: Telegram bot
runtime(для оперативного оповещения), OCR/content-analysis, 1C/AI/ETL integration и MCP/dev helpers. Эти части не являются ядром Rust-first runtime.
@@ -49,13 +60,13 @@ Implemented:
- Workforce reports.
- UEBA v1.
- Forensics reporting.
- pfSense contract/readiness layer со статусом `contract_only`, без заявления
production ingestion.
- pfSense contract/readiness layer со статусом `contract_only`.
Planned:
- Provider detail expansion under `/portal/architecture`.
- PowerShell Provider.
- PowerShell Provider как planned/agentless direction, не как возврат новых
runtime-функций на PowerShell.
- SSH Provider.
- Syslog Provider.
- 1C Provider как формализация текущего file-based 1C analytics направления.
@@ -89,6 +100,7 @@ Security Analytics + Forensics для ролей `executive`, `manager`, `securi
Pilot validation:
- [чеклист проверки пилота](docs/PILOT_VALIDATION_CHECKLIST_RU.md);
- [pilot freeze readiness](docs/PILOT_FREEZE_READINESS_RU.md);
- [gap analysis пилота](docs/PILOT_GAP_ANALYSIS_RU.md);
- [вопросы для discovery с заказчиком](docs/CUSTOMER_DISCOVERY_QUESTIONS_RU.md);
- [критерии успеха пилота](docs/PILOT_SUCCESS_CRITERIA_RU.md);
@@ -98,6 +110,8 @@ Pilot validation:
- pfSense показывается только как `contract_only/readiness`, без заявления
production ingestion или SIEM;
- pfSense в текущем пилоте допускается только как contract/readiness/optional
integration layer;
- UEBA Score v1 является прозрачной rule-based моделью, без ML/LLM;
- demo fixtures не содержат реальных IP-адресов, hostname, логинов, ФИО,
подразделений заказчика или событий безопасности;
@@ -111,12 +125,12 @@ Pilot validation:
- Есть ли события, важные для ИБ: копирование, печать, USB, подозрительные сайты.
- Не пропали ли данные с рабочих компьютеров и RDP-сессий.
## Кому это полезно
## Кому это полезно в работе
- Владельцу и руководителю - видеть активность, загрузку команды,
простои, перегрузки и рабочие приложения без просмотра логов.
- ИБ - заметить DLP-сигналы и подозрительную активность.
- Администратору - проверить, что сборщики и сервер работают стабильно.
простои, перегрузки и рабочие приложения.
- ИБ - заметить DLP-сигналы и подозрительную активность, а при отсутствии специалистов по ИБ - дать оповещение бизнесу о проблемах с информационной безопасностью, для привлечения специалистов по ИБ.
- Администратору - проверить, что сервер и все узлы информационных потоков работают стабильно, оценить состояние внутренней сети с точки зрения ИБ.
## Интерфейс
@@ -190,6 +204,12 @@ collectors.
## Поставка и регистрация
- Ежедневная/еженедельная проверка эксплуатационного контура:
[матрица проверки контура](docs/CONTOUR_CHECK_MATRIX_RU.md).
- Проверка после инженерных изменений: cargo/security gates, browser smoke и
production smoke:
[эксплуатационный validation runbook](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md).
- Enterprise deployment documentation:
[deployment guide](docs/ENTERPRISE_DEPLOYMENT_GUIDE_RU.md),
[topologies](docs/DEPLOYMENT_TOPOLOGIES_RU.md),
@@ -208,6 +228,83 @@ collectors.
[commercial positioning](docs/REGISTRY_COMMERCIAL_POSITIONING_RU.md),
[readiness checklist](docs/REGISTRY_READINESS_CHECKLIST_RU.md).
### Подготовка к реестру российского ПО
- Основной российский Git-контур / Gitea-дубликат GitHub-репозитория:
`https://git.iri1968.dpdns.org/awatch-rus/AWatch-rus`.
- GitHub используется как публичное зеркало и public validation surface.
- Gitea operator account: `igor`; пароль/токены не хранятся в репозитории.
- Доказательная документация:
[docs/registry/](docs/registry/REGISTER_RU_SOFTWARE_READINESS_RU.md).
- Gitea Wiki используется только как навигация, не как единственный источник
документов.
- Российский build-runner и release evidence описаны в
[RU_BUILD_RUNNER_READINESS_RU.md](docs/registry/RU_BUILD_RUNNER_READINESS_RU.md).
- Текущий status freeze проекта:
[docs/PROJECT_STATUS_RU.md](docs/PROJECT_STATUS_RU.md).
- Остаточные риски:
[docs/RESIDUAL_RISKS_RU.md](docs/RESIDUAL_RISKS_RU.md).
- План публичных GitHub issues:
[docs/PUBLIC_ISSUES_PLAN_RU.md](docs/PUBLIC_ISSUES_PLAN_RU.md).
- GitHub remains public mirror only.
### Public engineering transparency
- Public CI, coverage baseline and security scanning are enabled on GitHub.
- Issue templates, PR template and public roadmap are maintained for process
visibility.
- Public secret scanning policy:
[docs/SECURITY_SCANNING_POLICY_RU.md](docs/SECURITY_SCANNING_POLICY_RU.md).
- GitHub remains public mirror validation only.
- Primary registry contour remains Gitea plus the Russian build-runner.
- Quality status:
[docs/QUALITY_STATUS_RU.md](docs/QUALITY_STATUS_RU.md).
- Residual risks:
[docs/RESIDUAL_RISKS_RU.md](docs/RESIDUAL_RISKS_RU.md).
- Public issues plan:
[docs/PUBLIC_ISSUES_PLAN_RU.md](docs/PUBLIC_ISSUES_PLAN_RU.md).
- Public issue templates are prepared and real GitHub issue URLs are recorded
in the manifest; this improves roadmap visibility but does not claim
community adoption:
[creation runbook](docs/PUBLIC_ISSUES_CREATION_RUNBOOK_RU.md),
[manifest](docs/public-issues/public-issues-manifest.json).
### Engineering governance and residual risks
- Enterprise quality standard:
[docs/ENTERPRISE_QUALITY_STANDARD_RU.md](docs/ENTERPRISE_QUALITY_STANDARD_RU.md).
- GitHub governance entrypoint:
[.github/GOVERNANCE.md](.github/GOVERNANCE.md).
- Review checklist:
[docs/REVIEW_CHECKLIST_RU.md](docs/REVIEW_CHECKLIST_RU.md).
- Operational validation runbook:
[docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md](docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md).
- Daily maintenance runbook with optional Pollinations AI evidence assistant:
[docs/DAILY_MAINTENANCE_RU.md](docs/DAILY_MAINTENANCE_RU.md).
- Operational maturity harness:
[docs/OPERATIONAL_MATURITY_RU.md](docs/OPERATIONAL_MATURITY_RU.md).
- Residual risks register:
[docs/RESIDUAL_RISKS_RU.md](docs/RESIDUAL_RISKS_RU.md).
- Public issues plan:
[docs/PUBLIC_ISSUES_PLAN_RU.md](docs/PUBLIC_ISSUES_PLAN_RU.md).
- Public issues creation runbook:
[docs/PUBLIC_ISSUES_CREATION_RUNBOOK_RU.md](docs/PUBLIC_ISSUES_CREATION_RUNBOOK_RU.md).
- Public issues manifest:
[docs/public-issues/public-issues-manifest.json](docs/public-issues/public-issues-manifest.json).
- Advisory branch protection policy:
[docs/BRANCH_PROTECTION_POLICY_RU.md](docs/BRANCH_PROTECTION_POLICY_RU.md).
- Branch protection evidence template:
[docs/BRANCH_PROTECTION_EVIDENCE_RU.md](docs/BRANCH_PROTECTION_EVIDENCE_RU.md).
- PR-based review workflow:
[docs/PR_REVIEW_WORKFLOW_RU.md](docs/PR_REVIEW_WORKFLOW_RU.md).
- PR review evidence template:
[docs/PR_REVIEW_EVIDENCE_RU.md](docs/PR_REVIEW_EVIDENCE_RU.md).
- CODEOWNERS and PR template are maintained for review routing and public
change-control visibility.
- Visible external code review is still pending until public reviewed PRs exist.
- Branch protection policy is documented as advisory; it is not claimed as
enabled here.
- [Позиционирование для реестра российского ПО](docs/RUSSIAN_SOFTWARE_REGISTRY_POSITIONING_RU.md)
- [Сведения для подачи в реестр](REGISTER_RU_SOFTWARE.md)
- [Registry product passport](docs/REGISTRY_PRODUCT_PASSPORT_RU.md)
@@ -217,6 +314,8 @@ collectors.
- [Registry deployment model](docs/REGISTRY_DEPLOYMENT_MODEL_RU.md)
- [Registry commercial positioning](docs/REGISTRY_COMMERCIAL_POSITIONING_RU.md)
- [Registry readiness checklist](docs/REGISTRY_READINESS_CHECKLIST_RU.md)
- [Остаточные риски](docs/RESIDUAL_RISKS_RU.md)
- [План публичных issues](docs/PUBLIC_ISSUES_PLAN_RU.md)
- [Описание продукта](PRODUCT_DESCRIPTION_RU.md)
- [Журнал изменений](CHANGELOG_RU.md)
- [Установка для эксперта](INSTALL_FOR_EXPERT_RU.md)
@@ -289,6 +388,8 @@ collectors.
- [Grafana and Prometheus Monitoring Stack](docs/wiki/Grafana-and-Prometheus-Monitoring-Stack.md)
- [Grafana dashboards guide](docs/GRAFANA_DASHBOARDS_RU.md)
- [План внедрения ClickHouse Dictionaries для DetMir](docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md)
- [ClickHouse Workforce scaffold](clickhouse-workforce/README.md)
- [Prometheus Exporter](docs/wiki/Prometheus-Exporter.md)
Для сборщиков и интерфейса:
@@ -296,3 +397,453 @@ collectors.
- [Windows Collector Suite](docs/wiki/Windows-Collector-Suite.md)
- [Worktime API and UI Bridge](docs/wiki/Worktime-API-and-UI-Bridge.md)
- [Russian WebUI Patch and Localization](docs/wiki/Russian-WebUI-Patch-and-Localization.md)
- Актуальные ссылки по этой тематике: https://www.securitylab.ru/analytics/573771.php (Как собрать ролевую модель доступа при хаосе в инфраструктуре)
---
## 📊 **ОЦЕНКА ЗРЕЛОСТИ И КАЧЕСТВА ПРОЕКТА** (обновлено 22 июня 2026)
### **1️⃣ ОБЩИЕ МЕТРИКИ ПРОЕКТА**
| Метрика | Значение | Тренд | Оценка |
|---------|----------|-------|--------|
| **Возраст проекта** | 58 дней | ✅ Active | Молодой, но стабильный |
| **Размер репо** | ~11 MB | ✅ Compact | Хорошо структурирован |
| **Основной язык** | Rust | ✅ Production | Правильный выбор |
| **Лицензия** | Apache 2.0 | ✅ Open-friendly | Коммерчески дружелюбно |
| **Звезды** | 3 ⭐ | ⚠️ Нишевой продукт | Целевая аудитория |
| **Форки** | 2 | ⚠️ Низко | Early-stage / pilot-stage OSS |
| **Open Issues** | 1 | ⚠️ Низкая публичная активность | Issue templates уже есть |
| **Последний коммит** | 22 июня 2026 | ✅ **СЕГОДНЯ** | **АКТИВНО РАЗРАБАТЫВАЕТСЯ** |
| **Проектный статус** | main branch | ✅ Единая стратегия | Production-ready focus |
| **Public CI** | passed | ✅ Visible | GitHub Actions mirror validation |
| **Coverage workflow** | passed | ✅ Visible | Baseline workflow, threshold позже |
| **Security workflow** | passed | ✅ Visible | cargo audit/deny + secret scan |
| **Secret scan** | hardened + passed | ✅ Conservative | Fail-closed public scanner |
---
### **2️⃣ АРХИТЕКТУРНАЯ ЗРЕЛОСТЬ: 9.2/10** 🏗️
#### ✅ **Rust-first Migration (ПОЛНОСТЬЮ ЗАВЕРШЕНА)**
```
Миграция на Rust: 32+ фазы, ВСЕ ЗАВЕРШЕНЫ ✅
Phase 0-7: Foundation & Read-only [DONE ✅]
Phase 8-17: State orchestration & Telegram [DONE ✅]
Phase 18-26: DLP & Hayabusa services [DONE ✅]
Phase 27-32: AW health & maintenance [DONE ✅]
Текущий статус: 30+ Rust crates в production
- detmir-auto ✅
- detmir-status ✅
- detmir-check ✅
- dlp-policy-engine ✅
- dlp-case-management ✅
- dlp-compliance ✅
- aw-db-maintenance ✅ (НОВОЕ: vacuum с integrity check!)
- aw-hayabusa-tools ✅
```
#### 🆕 **НОВОЕ: SQLite VACUUM & MAINTENANCE**
```rust
adk-rust/crates/aw-db-maintenance:
- Trim mode: удаление старых allowlisted rows (по умолчанию dry-run)
- VACUUM mode: компактирование DB с PRAGMA integrity_check
- Lock-based concurrency protection
- Service stop/start guards
- Backup-before-delete policy
- Rollback из /var/lib/activitywatch/backups/db/aw-sqlite-before-db-vacuum-*.db
```
**Это серьёзное, enterprise-grade решение для production DB maintenance.**
---
### **3️⃣ ДОКУМЕНТАЦИЯ: EXCEPTIONAL (10/10)** 📚
#### 🎯 **Полнота документации**
```
КЛАССИФИКАЦИЯ ДОКУМЕНТОВ:
DEPLOYMENT:
✅ ENTERPRISE_DEPLOYMENT_GUIDE_RU.md
✅ DEPLOYMENT_TOPOLOGIES_RU.md
✅ SIZING_GUIDE_RU.md
✅ BACKUP_AND_RECOVERY_RU.md
✅ SECURITY_HARDENING_RU.md
✅ FULL_DEPLOYMENT_MANUAL_RU.md
REGISTRY (для реестра РПО):
✅ REGISTRY_PRODUCT_PASSPORT_RU.md
✅ REGISTRY_ARCHITECTURE_RU.md
✅ REGISTRY_FUNCTIONAL_SCOPE_RU.md
✅ REGISTRY_DEPENDENCY_STATEMENT_RU.md
✅ REGISTRY_DEPLOYMENT_MODEL_RU.md
✅ REGISTRY_COMMERCIAL_POSITIONING_RU.md
PILOT & VALIDATION:
✅ PILOT_V1_RU.md
✅ PILOT_DEMO_SCENARIO_RU.md
✅ PILOT_FREEZE_READINESS_RU.md (НОВОЕ!)
✅ PILOT_VALIDATION_CHECKLIST_RU.md
✅ PILOT_SUCCESS_CRITERIA_RU.md
OPERATIONAL:
✅ OPERATIONS_RUNBOOK_RU.md
✅ OPERATIONS_RUNBOOK_WORKTIME_RU.md
✅ ADMIN_GUIDE_RU.md
✅ OPERATOR_GUIDE_RU.md
✅ ARCHITECTURE_RU.md
RISK & SECURITY:
✅ THREAT_MODEL_RU.md
✅ SECURITY_HARDENING_RU.md
✅ RISK_NARRATIVE_RU.md
✅ PRODUCTION_INCIDENT_REPORT_2026-06-07_RU.md
TECHNICAL:
✅ Wiki (Getting Started, Infrastructure, CI/CD, QA)
✅ Grafana dashboards guide
✅ Windows Collector Suite
✅ adk-rust/RUNBOOK.md (32 фазы миграции!)
SALES & POSITIONING:
✅ COMPETITIVE_POSITIONING_RU.md
✅ SALES_POSITIONING_RU.md
✅ CUSTOMER_PILOT_PACK_RU.md
✅ CUSTOMER_DEMO_SCENARIO_RU.md
TOTAL: 60+ документов НА РУССКОМ ЯЗЫКЕ
```
**Это НЕ типичный уровень документации. Это КОРПОРАТИВНЫЙ СТАНДАРТ.**
---
### **4️⃣ КАЧЕСТВО КОДА: 8.5/10** 💎
#### ✅ Сильные стороны:
```rust
// 1. Правильная обработка ошибок
// Все Rust crates используют Result<T, Error> с context
cargo clippy --workspace --all-targets -- -D warnings
// 2. Structured JSON output для всех операций
detmir-status --json
detmir-check --json
detmir-dlp --json
// Машинечитаемые контракты везде!
// 3. Safety gates и guardrails
// - dry-run по умолчанию для mutation команд
// - allowlist для systemd restart
// - lock files для concurrent protection
// - audit logging для всех действий
// 4. Idempotent Ansible playbooks
// - deploy_aw_server.yml идемпотентен
// - WinRM retry с exponential backoff
// - Syntax checks перед apply
// 5. Production-grade operational patterns
// - systemd drop-ins для переключения binaries
// - Rollback scripts задокументированы
// - Shadow-mode validation перед switch
```
#### ⚠️ Оставшиеся слабые стороны:
```
⚠️ Низкая публичная активность в issue tracker
- issue templates есть
- public roadmap есть
- открытых публичных задач пока мало
⚠️ Низкая community adoption
- мало forks/stars
- проект пока выглядит как early-stage / pilot-stage OSS
- это нормально для нового специализированного продукта
⚠️ Restore test еще не выполнен
- backup Gitea работает
- SHA256 verification работает
- daily timer работает
- restore_tested пока false
⚠️ Российский build-runner пока planned
- release evidence scripts есть
- первый настоящий release build на awatch-build-01 еще не выполнен
⚠️ Юридический пакет правообладателя еще pending
- техническая readiness сильная
- юридическая часть для реестра еще требует отдельной подготовки
```
#### ✅ Уже закрыто после последних коммитов:
```
✅ Public CI/CD visibility
✅ Public coverage workflow
✅ Public security scanning
✅ Secret scan policy
✅ SECURITY.md
✅ CONTRIBUTING.md
✅ ROADMAP.md
✅ Issue templates
✅ PR template
✅ CODEOWNERS
✅ Review checklist
✅ Branch protection policy documented
✅ Registry docs
✅ Russian Gitea contour
✅ GitHub public mirror validation
✅ Gitea backup
✅ Status freeze
```
---
### **5️⃣ PRODUCTION READINESS: 9/10** 🚀
#### ✅ Enterprise Features
```
✅ Multi-role RBAC (executive, manager, security, forensics, admin)
✅ DLP incident management с evidence хранилищем
✅ SLO monitoring и автоматический heal
✅ Ansible-powered deployment с idempotency
✅ Backup/restore procedures
✅ Grafana dashboards version-controlled
✅ Hayabusa forensics integration
✅ Telegram bot уведомления
✅ ClickHouse data warehouse
✅ Prometheus/Influx exporters
✅ SAFETY PATTERNS:
- read-only smoke tests перед production
- --dry-run по умолчанию для risky operations
- Rollback procedures documented
- Production incident report существует (2026-06-07)
- Lock-based concurrency protection
```
#### ⚠️ Production Risks
```
⚠️ Один разработчик (igor04091968) — BUS FACTOR ⚠️
- Все коммиты от одного человека
- Нет code reviews видно
- Нет pull request culture
⚠️ Молодой проект (56 дней)
- Нет долгосрочной production history
- Нет documented post-mortems (кроме одного)
⚠️ Limited public activity / community adoption
- 2 форка, 3 звезды
- Issue templates и roadmap есть, но публичных задач пока мало
- Community adoption низкая, это не технический blocker
⚠️ Registry release evidence еще не завершен
- GitHub Actions зеленые, но это только public mirror validation
- Первый release evidence build должен быть выполнен на awatch-build-01
- Gitea restore_tested пока false
```
---
### **6️⃣ РОССИЙСКИЙ РЫНОК READY: 9.5/10** 🇷🇺
#### ✅ Идеальная позиция для РФ
```
✅ ЛОКАЛИЗАЦИЯ:
- Полностью на русском (все документы)
- Russian UI patch для ActivityWatch
- Поддержка русских Windows локализаций
- Cyrillic-aware logging
✅ РЕЕСТР РПО / REGISTRY-READINESS:
- Подготовлен registry-readiness пакет документов
- Product passport и architecture documents описаны
- Dependency statement зафиксирован
- Российский Gitea-контур поднят
- GitHub Actions используется только как public mirror validation
- Release evidence требует российского build-runner
✅ ТЕХНОЛОГИЧЕСКИЙ STACK:
- Rust (не зависит от США)
- Debian/Ubuntu Linux
- Grafana/Prometheus (open-source)
- ClickHouse (российская компания!)
- Hayabusa (DFIR forensics)
- Ansible (open infrastructure)
✅ NO CLOUD-DEPENDENCY:
- Полностью on-prem
- Нет телеметрии в облако
- Нет SaaS lock-in
- Может быть air-gapped
✅ HONESTY POSITIONING:
- НЕ претендует на ФСТЕК/ФСБ сертификацию
- НЕ использует ML/LLM (transparent rule-based UEBA)
- Явно указывает границы показа (contract_only для pfSense)
- Не маскирует ограничения
```
---
### **7️⃣ PILOT v1 FREEZE READINESS (НОВОЕ!)** 🎯
Заметил в README новый документ:
```
✅ docs/PILOT_FREEZE_READINESS_RU.md (добавлен недавно)
```
Это указывает на:
- **Проект готовится к Pilot freeze** (закрытию features)
- **Feature complete для Pilot v1.0**
- **Production readiness gates активны**
```
PILOT V1 SCOPE (ГОТОВО):
- Workforce Analytics ✅
- Security DLP Analytics ✅
- Forensics Reporting ✅
- Evidence Management ✅
- UEBA v1 (rule-based) ✅
- Telegram notifications ✅
- Grafana dashboards ✅
- Role-based access ✅
```
---
### **8️⃣ ИСТОРИЧЕСКАЯ ВЕХИ РАЗРАБОТКИ** 📅
```
2026-06-01: Миграция на Rust начинается (фазы 0-7)
2026-06-07: Production incident (подробный postmortem)
2026-06-09: Grafana panels development
2026-06-11: Security hardening improvements
2026-06-12: Release candidate preflight
2026-06-12-19: Intensive hardening phase
2026-06-20: Pilot freeze readiness doc добавлен
2026-06-21: Public CI/Coverage/Security workflows добавлены
2026-06-22: GitHub Actions validation прошел после hardening secret scan
ВЫВОД: Проект в PRODUCTION HARDENING фазе перед Pilot release
```
---
### **9️⃣ КОНКУРЕНТНЫЙ АНАЛИЗ** 🏆
Проект позиционирует себя против:
```
КОНКУРЕНТЫ (по docs/COMPETITIVE_POSITIONING_RU.md):
- Splunk (слишком дорого, облако)
- Okta (не для локального ИБ)
- ArcSight (legacy, дорого)
- ELK Stack (требует экспертизы)
- Grafana Loki (только logs, не worktime)
УНИКАЛЬНОСТЬ AWatch-rus:
✅ Workforce + Security + Forensics в одном
✅ Русский язык & локализация
✅ Без облака & без ML-черного ящика
✅ Open-source компоненты (ActivityWatch)
✅ Прозрачность (rule-based UEBA)
✅ РФ registry ready
```
---
### **🔟 FINAL ASSESSMENT: ПЕРЕОЦЕНКА**
| Категория | Была | Сейчас | Изменение | Комментарий |
|-----------|------|--------|-----------|------------|
| **Полнота** | 8.5 | **9.2** | ⬆️ +0.7 | DB maintenance added |
| **Качество** | 8.0 | **8.5** | ⬆️ +0.5 | Production incident handled professionally |
| **Профессионализм** | 9.0 | **9.3** | ⬆️ +0.3 | Pilot freeze readiness shows maturity |
| **Российский рынок** | 9.0 | **9.5** | ⬆️ +0.5 | Registry docs enhanced, freeze ready |
| **Production Ready** | 8.5 | **9.0** | ⬆️ +0.5 | Safety gates, rollback procedures validated |
| **Public Validation** | 6.5 | **8.8** | ⬆️ +2.3 | CI/Coverage/Security workflows green |
| **ИТОГО** | **8.6** | **9.1** | ⬆️ **+0.5** | **PRODUCTION GRADE** |
---
### **🎯 КЛЮЧЕВЫЕ ВЫВОДЫ**
```
1. ✅ ПРОЕКТ ГОТОВ К PRODUCTION PILOTING
- Rust-first migration полностью завершена
- Safety gates реализованы
- Documentation на уровне enterprise
- DB maintenance добавлено (новое)
2. ✅ ИДЕАЛЕН ДЛЯ РОССИЙСКОГО РЫНКА
- Полностью локализован
- Registry documents готовы
- Технологический stack без зависимостей
3. ✅ PUBLIC VALIDATION VISIBILITY УЖЕ ЗАКРЫТА
- Public CI/CD visibility ✅
- Public coverage workflow ✅
- Public security scanning ✅
- Secret scan policy hardened ✅
- GitHub public mirror validation ✅
4. ⚠️ ОСТАВШИЕСЯ РИСКИ
- Один разработчик
- Нет visible code review
- Низкая публичная активность issue tracker
- Низкая community adoption
- Gitea restore test еще не выполнен
- Российский build-runner пока planned
- Branch protection policy documented, but enablement not yet verified
5. 🚀 TIMELINE К PRODUCTION:
- Pilot v1 freeze: готовится (freeze readiness doc)
- Beta release: Q3 2026 (est.)
- GA production: Q4 2026 (est.)
6. 📊 QUALITY METRICS:
- Code: Rust clippy strict mode ✅
- Testing: Cargo test suite ✅
- Public coverage workflow ✅
- Public security workflow ✅
- Deployment: Ansible idempotent ✅
- Operations: Runbook-driven ✅
- Documentation: 60+ doc pages ✅
```
---
## 💡 **РЕКОМЕНДАЦИИ**
### Для потенциального инвестора/партнера:
```
✅ ИНВЕСТИРОВАТЬ: Проект достаточно зрелый для pilot
✅ ТРЕБОВАТЬ: Bus factor mitigation (второй разработчик)
✅ ТРЕБОВАТЬ: Community code review process (GitHub PRs)
✅ ТРЕБОВАТЬ: Первый release evidence build на российском build-runner
✅ ТРЕБОВАТЬ: Restore test Gitea backup на отдельном сервере
⚠️ НАБЛЮДАТЬ: Feedback из first customers на Pilot v1
```
### Для Russian enterprises:
```
✅ ИСПОЛЬЗОВАТЬ: Как operational intelligence platform
✅ НЕ ИСПОЛЬЗОВАТЬ: Как certified DLP/SIEM (не позиционируется)
✅ ТРЕБОВАТЬ: Support contract перед production
✅ ПЛАНИРОВАТЬ: Intern training на Rust maintenance
```
+443
View File
@@ -0,0 +1,443 @@
# Operational Recovery Audit
Date: 2026-07-01
Scope: repository-first audit of the current AWatch-rus / DetMir operational
recovery capability. This document describes only mechanisms already present in
the repository. It does not define new backup or recovery mechanisms.
## Evidence Sources
The audit reviewed these repository sources:
- `docs/BACKUP_AND_RECOVERY_RU.md`
- `docs/RETENTION_POLICY_RU.md`
- `docs/OPERATIONS_RUNBOOK_RU.md`
- `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`
- `docs/DETMIR_RESTORE_BASELINE_2026-06-29_RU.md`
- `scripts/prod_backup_restore.sh`
- `adk-rust/crates/prod-backup-restore/src/main.rs`
- `ansible/deploy_aw_server.yml`
- `ansible/deploy_aw_windows.yml`
- `ansible/post_validate_aw_windows.yml`
- `aw-server/*.service`, `aw-server/*.timer`, `aw-server/*.path`
- `windows/ActivityWatch.Windows.Common.psm1`
- `windows/hardening-recovery.ps1`
- `windows/rebuild-worktime-tasks.ps1`
- `windows/fix-session-watchers.ps1`
- `windows/cleanup-disc-sessions.ps1`
- `clickhouse-1c/docker-compose.yml`
- `clickhouse-workforce/docker-compose.yml`
- `grafana-1c/docker-compose.yml`
- `docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md`
- `docs/registry/registry-evidence-manifest.json`
## Current Recovery Procedure Actually Supported
### Server-side service recovery
The repository supports service restart and health validation through systemd,
Ansible deployment, and smoke checks:
- `activitywatch-server.service` restarts on failure and runs with the configured
AW server env file. Evidence: `aw-server/activitywatch-server.service`.
- `ansible/deploy_aw_server.yml` installs and restarts
`activitywatch-server.service`, then waits for `/api/0/info`. Evidence:
`ansible/deploy_aw_server.yml:2365-2375`.
- The generic operations runbook requires checking `/healthz`, `/readyz`,
`/metrics`, service status, journal, and smoke scripts after recovery.
Evidence: `docs/OPERATIONS_RUNBOOK_RU.md:59-71`.
- The production validation runbook defines the DetMir smoke sequence through
`check-aw-data.sh`, `check-aw-full.sh`, and local contour smoke. Evidence:
`docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md`.
### ActivityWatch SQLite maintenance and recovery support
The repository supports guarded maintenance and limited DB merge/migration
flows, not a complete automated restore:
- Weekly guarded SQLite maintenance is installed as
`aw-db-maintenance.service` / `aw-db-maintenance.timer`. Evidence:
`aw-server/aw-db-maintenance.service`, `aw-server/aw-db-maintenance.timer`.
- Optional SQLite vacuum is defined as `aw-db-vacuum.service` /
`aw-db-vacuum.timer`. It is opt-in in Ansible through
`aw_db_vacuum_timer_enabled`. Evidence:
`aw-server/aw-db-vacuum.service`, `aw-server/aw-db-vacuum.timer`,
`ansible/deploy_aw_server.yml:613-649`.
- Legacy root DB merge is implemented in `ansible/deploy_aw_server.yml` when
`aw_legacy_db_merge_enabled` is true. It stops the service, backs up target
and legacy DB files, merges them with `merge-aw-server-dbs`, installs the
merged DB, restarts the service, and waits for API readiness. Evidence:
`ansible/deploy_aw_server.yml:2248-2375`.
- A separate Rust `prod-backup-restore` binary exists only as a plan/checker.
It explicitly rejects `--apply`. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
### Windows collector recovery
The Windows side has the strongest implemented recovery automation:
- Deployment writes `deployment-config.json`, launcher scripts, and
`recovery-loop.ps1`. Evidence:
`windows/deploy-domain-users.ps1`, `windows/hardening-recovery.ps1`.
- `Write-ActivityWatchRecoveryScript` generates a script that imports
`ActivityWatch.Windows.Common.psm1` and calls
`Invoke-ActivityWatchRecoveryLoop`. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:1576-1597`.
- `Register-ActivityWatchRecoveryTask` creates the scheduled task
`ActivityWatch Recovery`, using an interactive user when possible and SYSTEM
fallback otherwise. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:2447-2488`.
- `Invoke-ActivityWatchRecoveryLoop` is an actual loop: it uses a lock file,
cleans non-live session processes, starts the worktime session collector when
allowed, starts configured live user launch tasks, and uses console fallback.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:2114-2168`.
- `Start-ActivityWatchTasks` starts launch tasks for live users and starts the
recovery task. Evidence:
`windows/ActivityWatch.Windows.Common.psm1:2656-2672`.
- `rebuild-worktime-tasks.ps1`, `fix-session-watchers.ps1`, and
`cleanup-disc-sessions.ps1` provide manual repair paths for task/script
regeneration, stale recovery loop restart, and disconnected-session cleanup.
### Hayabusa intake recovery
The repository supports event-driven reprocessing of uploaded Hayabusa packages:
- `aw-hayabusa-drop.path` watches `/opt/activitywatch/aw-rus-ops/drop` for zip
packages and triggers `aw-hayabusa-drop.service`. Evidence:
`aw-server/aw-hayabusa-drop.path`.
- `aw-hayabusa-drop.service` runs `/usr/local/bin/aw-hayabusa-autoprocess`.
Evidence: `aw-server/aw-hayabusa-drop.service`.
- `ansible/deploy_aw_server.yml` creates Hayabusa directories, installs pinned
Hayabusa, installs Rust helpers, creates the drop zone, enables
`aw-hayabusa-drop.path`, and runs `aw-hayabusa doctor`. Evidence:
`ansible/deploy_aw_server.yml:2780-3070`.
### Readiness evidence recovery support
The repository supports periodic readiness bundle generation:
- `detmir-readiness.service` writes readiness output to
`/var/lib/activitywatch/health/readiness-bundle`. Evidence:
`aw-server/detmir-readiness.service`.
- `detmir-readiness.timer` runs daily with persistent timer behavior. Evidence:
`aw-server/detmir-readiness.timer`.
## Current Backup Procedure Actually Supported
### ActivityWatch server local backup artifacts
Implemented:
- `ansible/deploy_aw_server.yml` creates `/var/lib/activitywatch/backups` and
`/var/lib/activitywatch/backups/db`. Evidence:
`ansible/deploy_aw_server.yml:80-128`, `ansible/deploy_aw_server.yml:2263-2269`.
- Before legacy DB merge, Ansible copies the target DB and legacy root DB into
timestamped files under `{{ aw_server_data_dir }}/backups/db`. Evidence:
`ansible/deploy_aw_server.yml:2274-2294`.
- Before applying server-side settings/views/classes, Ansible copies current
payloads to timestamped JSON backups under `{{ aw_server_data_dir }}/backups`.
Evidence: `ansible/deploy_aw_server.yml:2406-2420`.
- `aw-prune-local-state.timer` prunes old local backups using configured
retention and keep-last values. Evidence:
`ansible/deploy_aw_server.yml:236-281`, `docs/RETENTION_POLICY_RU.md:70-71`.
Not currently implemented:
- A general scheduled full backup of the active ActivityWatch SQLite DB.
- A repository-defined off-host backup copy for AW DB, configs, Grafana,
ClickHouse, DLP evidence, Hayabusa archives, or Windows collector state.
### Windows package rollback backup
Implemented:
- `Install-ActivityWatchPackage` backs up the existing install root into
`install-<timestamp>` before replacing it, keeps only the latest two install
backups, checks free space, and cleans temporary extraction directories.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:80-95`,
`windows/ActivityWatch.Windows.Common.psm1:148-245`.
- `hardening-recovery.ps1` can run `Install-ActivityWatchPackage` when
`-RepairPackage` is supplied. Evidence: `windows/hardening-recovery.ps1`.
Not currently implemented:
- Automatic restore from the saved Windows `install-*` backup directory.
- Backup of `C:\ProgramData\AWatch-rus` as a whole before repair.
### Gitea registry backup
Implemented as registry-readiness support, not DetMir runtime recovery:
- Registry docs define Gitea backup path, script, systemd service/timer,
`gitea dump` ZIP format, SHA256 checksum, daily schedule, and 14-day
retention. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:7-23`,
`docs/registry/registry-evidence-manifest.json:26-39`.
Not currently implemented:
- Tested Gitea restore. The manifest explicitly says `restore_tested=false` and
`production_ready=false`. Evidence:
`docs/registry/registry-evidence-manifest.json:37-39`.
## Current Restore Procedure
### ActivityWatch DB merge/restore-like flow
Supported:
1. Enable `aw_legacy_db_merge_enabled`.
2. Deploy server playbook.
3. Playbook checks legacy root DB and target DB.
4. Playbook stops `activitywatch-server.service`.
5. Playbook backs up target and legacy DB files.
6. Playbook runs `/usr/local/bin/merge-aw-server-dbs`.
7. Playbook installs the merged DB as active target DB.
8. Playbook restarts `activitywatch-server.service`.
9. Playbook waits for `/api/0/info`.
Evidence: `ansible/deploy_aw_server.yml:2248-2375`.
Not currently implemented:
- A generic "restore selected backup file to active AW DB" command.
- A tested end-to-end AW DB restore runbook.
- Automated checksum verification for AW DB backup files.
- Automated rollback from a failed DB merge to the backup file.
### `prod-backup-restore` plan-only flow
Supported:
1. `scripts/prod_backup_restore.sh` locates the Rust planner binary or exits
with build instructions. Evidence: `scripts/prod_backup_restore.sh:14-29`.
2. The Rust planner reads `private-config/runtime.env` if available, checks
required env vars, checks `sshpass`, `ansible-playbook`, inventory, and
`merge-aw-server-dbs`. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:111-131`,
`adk-rust/crates/prod-backup-restore/src/main.rs:175-209`.
3. The planner prints planned commands including remote backup directory
creation, DB copies, service stop, merge, install, and Ansible validation.
Evidence: `adk-rust/crates/prod-backup-restore/src/main.rs:211-280`.
Not currently implemented:
- Execution of the planned restore. `--apply` fails by design. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
### Windows collector recovery
Supported:
1. Run `ActivityWatch Recovery` scheduled task or let it run on its configured
trigger.
2. Recovery loop enforces a single lock.
3. Recovery loop stops collectors/watchers in non-live sessions.
4. Recovery loop starts the global worktime collector when allowed.
5. Recovery loop starts configured live-user launch tasks.
6. Recovery loop attempts console fallback when no configured live task starts.
Evidence: `windows/ActivityWatch.Windows.Common.psm1:2114-2168`,
`windows/ActivityWatch.Windows.Common.psm1:2447-2488`,
`windows/ActivityWatch.Windows.Common.psm1:2656-2672`.
Not currently implemented:
- Automatic reconstruction of lost Windows state from an external backup.
- Automatic restore of Windows collector queues after corruption or deletion.
### Gitea restore
Supported:
- Manual outline exists: prepare isolated test server, install same Gitea
version, stop Gitea, verify checksum, unpack dump, restore app/data/repos/db
according to Gitea official procedure, fix ownership, start Gitea, run
`gitea doctor check`, regenerate hooks if needed, and run post-restore
checks. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-90`.
Not currently implemented:
- Tested Gitea restore. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:92-100`.
## Components That Cannot Yet Be Restored Automatically
The following are confirmed by repository inspection:
| Component | Current state | Evidence |
| --- | --- | --- |
| ActivityWatch active SQLite DB | Backups exist around merge/settings operations, but generic restore is Not currently implemented. | `ansible/deploy_aw_server.yml:2274-2294`, `adk-rust/crates/prod-backup-restore/src/main.rs:100-105` |
| ClickHouse 1C data | Docker volume exists; no backup/restore automation found. Not currently implemented. | `clickhouse-1c/docker-compose.yml:13-19` |
| ClickHouse Workforce data | Docker volume exists; no backup/restore automation found. Not currently implemented. | `clickhouse-workforce/docker-compose.yml:11-17` |
| Grafana data volume | Docker volume exists; no repo cleanup and no restore automation. Not currently implemented. | `grafana-1c/docker-compose.yml:67-79`, `docs/RETENTION_POLICY_RU.md:99` |
| Prometheus TSDB | Retention configured, backup/restore not documented. Not currently implemented. | `grafana-1c/docker-compose.yml:40-49` |
| DLP policy/case/warehouse DBs | Retention doc says no automatic deletion; backup/restore not implemented. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:43-47`, `docs/RETENTION_POLICY_RU.md:87-90` |
| DLP evidence and compliance reports | Cleanup disabled; restore depends on customer backup if manually deleted. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:221-223` |
| Hayabusa reports/archive | Processing is automated; restore of archive/reports is not automated. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:48`, `docs/RETENTION_POLICY_RU.md:91` |
| Windows collector state and queues | Recovery restarts collectors; external backup/restore of state is not implemented. | `docs/RETENTION_POLICY_RU.md:40`, `docs/RETENTION_POLICY_RU.md:85` |
| Diagnostic bundles and release evidence | No automatic backup/restore found. Not currently implemented. | `docs/RETENTION_POLICY_RU.md:100-101` |
## Components Requiring Manual Intervention
- AW DB merge/recovery: operator must enable `aw_legacy_db_merge_enabled`, run
Ansible, review backup files, and verify API. Evidence:
`ansible/deploy_aw_server.yml:2248-2375`.
- `prod-backup-restore`: operator can only review a plan; execution is manual
because `--apply` is disabled. Evidence:
`adk-rust/crates/prod-backup-restore/src/main.rs:100-105`.
- Windows collector recovery after severe state loss: operator must use
`hardening-recovery.ps1`, `rebuild-worktime-tasks.ps1`,
`fix-session-watchers.ps1`, or redeploy. Evidence: `windows/*.ps1`.
- Hayabusa stuck path/service: operator may need to reset failed units, repair
drop-zone permissions, and rerun processing. Evidence:
`aw-server/aw-hayabusa-drop.path`, `aw-server/aw-hayabusa-drop.service`.
- Gitea restore: manual isolated test restore is required; tested restore is
not yet recorded. Evidence:
`docs/registry/GITEA_BACKUP_AND_RESTORE_RUNBOOK_RU.md:42-100`.
- ClickHouse/Grafana/Prometheus/DLP data restore: Not currently implemented.
## Missing Documentation
Critical and high-confidence gaps only:
- Exact AW DB restore runbook from `/var/lib/activitywatch/backups/db` to the
active DB path. Not currently implemented.
- Post-restore verification checklist for AW DB backup restore, including
checksum, ownership, service restart, bucket freshness, and worktime report
checks. Not currently implemented.
- Backup inventory mapping each persistent component to an actual backup owner,
schedule, storage location, retention, and restore command. Partially covered
by `docs/BACKUP_AND_RECOVERY_RU.md` and `docs/RETENTION_POLICY_RU.md`, but
operational restore ownership is Not currently implemented.
- ClickHouse 1C and ClickHouse Workforce backup/restore runbooks. Not currently
implemented.
- Grafana volume restore runbook. Not currently implemented.
- DLP evidence/case/policy restore runbook. Not currently implemented.
- Windows `C:\ProgramData\AWatch-rus` state backup/restore runbook. Not
currently implemented.
- Gitea restore evidence result. The runbook exists, but restore test is marked
false. Evidence: `docs/registry/registry-evidence-manifest.json:37-39`.
## Missing Automation
- Automated AW DB restore from a selected backup file. Not currently
implemented.
- Automated AW DB backup with checksum on a schedule independent of merge
operations. Not currently implemented.
- Off-host/offline copy for AW DB, ClickHouse volumes, Grafana data, DLP
evidence, Hayabusa archives, Windows state, and release evidence. Not
currently implemented.
- ClickHouse backup and restore automation. Not currently implemented.
- Grafana data volume backup and restore automation. Not currently implemented.
- DLP evidence/case/policy backup and restore automation. Not currently
implemented.
- Windows state backup and restore automation. Not currently implemented.
- Automated restore drill evidence generation. Not currently implemented.
## Operational Risks
| Risk | Severity | Evidence | Impact |
| --- | --- | --- | --- |
| Restore is partially plan-only for AW DB | Critical | `prod-backup-restore` rejects `--apply` | Operator can plan but cannot run a deterministic automated restore through this tool |
| AW DB backups are created around specific operations, not as a general scheduled full backup | Critical | `ansible/deploy_aw_server.yml:2274-2294` | A recent recovery point may be unavailable if no merge/settings operation occurred |
| ClickHouse data has no repo-defined restore path | Critical | ClickHouse Docker volumes only | Loss/corruption of 1C or workforce ClickHouse data requires ad hoc operator recovery |
| DLP/Hayabusa evidence has no automated restore | High | `docs/RETENTION_POLICY_RU.md:87-91`, `docs/RETENTION_POLICY_RU.md:221-223` | Forensic/case continuity depends on external/customer backup |
| Grafana data volume has no repo-defined backup/restore | High | `grafana-1c/docker-compose.yml:67-79` | Dashboard DB/users/session state may require manual reconstruction even though provisioned dashboards exist |
| Gitea restore is documented but untested | High | `restore_tested=false` | Source-control recovery confidence remains limited |
| Windows recovery restarts collectors but does not restore deleted state | Medium | Windows recovery loop evidence | Collector state/queues/logs can be lost if state root is deleted |
| Generic backup document is intentionally high-level | Medium | `docs/BACKUP_AND_RECOVERY_RU.md:5-6` | Operators need component-specific procedures during incidents |
| Hayabusa path recovery depends on service/path health and permissions | Medium | `aw-hayabusa-drop.path`, `aw-hayabusa-drop.service` | Drop backlog or permission drift can stall forensic intake |
| Recovery smoke is documented but not tied to a single recovery command | Low | `docs/OPERATIONS_RUNBOOK_RU.md`, `docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md` | Operator can validate, but command sequencing remains manual |
## Recovery Confidence Score
Overall score: **48 / 100**
Justification:
- Windows collector process recovery is mature: scheduled task, lock, live
session handling, cleanup of non-live sessions, and task restart are
implemented.
- Server service recovery and smoke validation are present.
- Local maintenance and retention are present.
- AW DB merge has safety backups and API validation, but generic restore is not
implemented.
- `prod-backup-restore` is explicitly plan-only.
- ClickHouse, Grafana, Prometheus, DLP evidence/cases, Hayabusa archive, Windows
state, diagnostic bundles, and release evidence do not have automated restore
procedures in the repository.
- Gitea backup is documented with checksum and timer metadata, but restore is
marked untested.
## Prioritized Confirmed Gaps
### Critical
1. Generic AW DB restore from backup is Not currently implemented.
Evidence: backup files are created by Ansible, but `prod-backup-restore`
refuses `--apply`.
2. Scheduled full AW DB backup independent of merge/settings changes is Not
currently implemented.
Evidence: backups are tied to merge/settings operations and pruning exists,
but no scheduled full DB backup unit is present.
3. ClickHouse 1C and Workforce backup/restore are Not currently implemented.
Evidence: both stacks persist to Docker volumes; no backup/restore runbook or
automation is present.
### High
4. DLP evidence/case/policy restore is Not currently implemented.
Evidence: retention policy explicitly leaves DLP evidence/cases without
automatic cleanup and says recovery depends on customer backup after manual
deletion.
5. Hayabusa reports/archive restore is Not currently implemented.
Evidence: intake processing is automated, but archive/report restore is not.
6. Grafana data volume backup/restore is Not currently implemented.
Evidence: Grafana uses `grafana-data`; no repo restore procedure exists.
7. Gitea restore test is Not currently implemented.
Evidence: registry manifest has `restore_tested=false`.
### Medium
8. Windows state root backup/restore is Not currently implemented.
Evidence: Windows recovery restarts collectors and repairs tasks, but does
not restore `C:\ProgramData\AWatch-rus` from backup.
9. Component-specific recovery ownership matrix is Not currently implemented.
Evidence: generic backup doc says component list must be refined by release
profile and customer infrastructure.
10. Restore drill evidence generation is Not currently implemented.
Evidence: validation commands exist, but no restore-drill artifact workflow
is present.
### Low
11. Prometheus TSDB restore is Not currently implemented.
Evidence: compose config bounds retention, but no backup/restore procedure
is present.
12. Diagnostic bundle and release evidence restore is Not currently
implemented.
Evidence: retention policy intentionally avoids pruning, but does not define
backup or restore automation.
## Release-Relevant Conclusion
The repository currently supports operational restart, Windows collector
self-healing, local maintenance, retention, readiness evidence, and limited
backup-before-mutation behavior.
It does not yet support deterministic full production recovery for all persisted
state. The most important gap is not service restart; it is data restore:
ActivityWatch DB, ClickHouse volumes, Grafana state, DLP/Hayabusa evidence, and
Windows state are not covered by a tested automated restore process in the
repository.
+67
View File
@@ -0,0 +1,67 @@
# AWatch-rus Roadmap
This roadmap is public planning. It does not claim completion of unverified
work and does not replace `docs/registry/` evidence for registry-readiness.
## Registry-readiness
- Maintain `docs/registry/` as the authoritative registry-readiness
documentation package.
- Keep conservative product claims and explicit remaining gaps.
- Prepare final rightsholder confirmation and legal review separately.
## Russian Git/build contour
- Keep self-hosted Gitea as the target Russian Git contour for
registry-readiness.
- Keep GitHub as public mirror validation only.
- Provision `awatch-build-01` as a separate Russian build-runner.
## Release evidence
- Run release candidate checks on the Russian build-runner.
- Generate source archive, binary archive, SBOM, SHA256SUMS, smoke logs and
release evidence manifest.
- Keep public GitHub Actions separate from registry release evidence.
## Backup/restore test
- Complete a test restore of Gitea backup on a separate server.
- Keep `restore_tested=false` until evidence exists.
- Document offsite backup in RF before registry submission.
## Coverage and CI
- Use public CI for engineering transparency.
- Track coverage baseline without enforcing a threshold at first.
- Add coverage threshold after baseline review.
## Security scanning
- Maintain cargo audit, cargo deny, dependency review and secret-pattern checks.
- Treat public security checks as advisory validation.
- Produce registry release security evidence in the Russian build contour.
## Russian OS compatibility
- Validate deployment and agent behavior on target Russian OS variants.
- Document unsupported combinations explicitly.
## Pilot hardening
- Keep demo data anonymized.
- Improve smoke coverage for install kit and operational reports.
- Preserve clear rollback and backup-first operational procedures.
## Future UI
- Future UI work remains planned unless backed by implemented code and tests.
- Public roadmap entries are not product claims.
## Not claimed / out of scope
- No claim of FSTEC/FSB certification.
- No claim of replacing DLP or SIEM.
- No claim of ML/LLM-based detection.
- No claim of automatic remediation.
- No claim of legal completion of Russian software registry registration.
+41
View File
@@ -0,0 +1,41 @@
# Security Policy
## Reporting a vulnerability
Please report suspected vulnerabilities privately to the project maintainer
before publishing technical details. If a private contact channel is not
available, open a GitHub issue with a minimal description and no exploit,
secret, customer data, employee logs or personal data.
Do not include:
- passwords, tokens, cookies or private keys;
- real employee logs;
- personal data;
- private network details;
- customer evidence that has not been anonymized.
Use demo or anonymized evidence whenever possible.
## Security scope
AWatch-rus is not positioned as a certified security product. It is not a replacement for DLP or SIEM platforms. Public security checks are advisory validation for engineering transparency.
## Public validation
GitHub Actions security checks run in the public mirror:
- cargo audit;
- cargo deny;
- secret-pattern check;
- dependency review for pull requests.
GitHub remains public mirror validation only. Registry release security
evidence must be produced in the Russian build contour on the Russian
build-runner.
## Registry-readiness note
Security checks do not confirm legal completion of Russian software registry
registration. Final submission requires rightsholder confirmation and legal
review.
+27 -72
View File
@@ -122,9 +122,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.102"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
[[package]]
name = "arbitrary"
@@ -169,12 +169,12 @@ name = "aw-1c-ingest"
version = "0.1.0"
dependencies = [
"anyhow",
"calamine",
"chrono",
"clap",
"csv",
"filetime",
"fs2",
"quick-xml",
"regex",
"reqwest",
"serde",
@@ -182,6 +182,7 @@ dependencies = [
"serde_yaml",
"sha1",
"tempfile",
"zip",
]
[[package]]
@@ -198,12 +199,9 @@ name = "aw-contour-smoke"
version = "0.1.0"
dependencies = [
"anyhow",
"chrono",
"clap",
"reqwest",
"serde_json",
"tempfile",
"url",
]
[[package]]
@@ -250,7 +248,6 @@ dependencies = [
"anyhow",
"reqwest",
"serde_json",
"tempfile",
]
[[package]]
@@ -324,6 +321,21 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "aw-workforce-ingest"
version = "0.1.0"
dependencies = [
"anyhow",
"chrono",
"clap",
"detmir-aw-client",
"reqwest",
"serde",
"serde_json",
"tempfile",
"urlencoding",
]
[[package]]
name = "awatch-agent"
version = "0.1.0"
@@ -386,33 +398,12 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "calamine"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a3a315226fdc5b1c3e33521073e1712a05944bc0664d665ff1f6ff0396334da"
dependencies = [
"byteorder",
"codepage",
"encoding_rs",
"log",
"quick-xml",
"serde",
"zip 0.6.6",
]
[[package]]
name = "cc"
version = "1.2.63"
@@ -444,7 +435,6 @@ dependencies = [
"clap",
"reqwest",
"serde_json",
"tempfile",
]
[[package]]
@@ -456,7 +446,6 @@ dependencies = [
"clap",
"reqwest",
"serde_json",
"tempfile",
]
[[package]]
@@ -531,15 +520,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "codepage"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
dependencies = [
"encoding_rs",
]
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -724,6 +704,7 @@ dependencies = [
"sha2",
"tempfile",
"tiny_http",
"url",
]
[[package]]
@@ -761,7 +742,6 @@ dependencies = [
"anyhow",
"clap",
"detmir-state",
"serde",
"serde_json",
]
@@ -772,7 +752,6 @@ dependencies = [
"anyhow",
"chrono",
"clap",
"tempfile",
]
[[package]]
@@ -941,15 +920,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -1241,7 +1211,7 @@ dependencies = [
"serde",
"serde_json",
"tempfile",
"urlencoding",
"zip",
]
[[package]]
@@ -1794,11 +1764,10 @@ dependencies = [
[[package]]
name = "quick-xml"
version = "0.31.0"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"encoding_rs",
"memchr",
]
@@ -1824,9 +1793,9 @@ dependencies = [
[[package]]
name = "quinn-proto"
version = "0.11.14"
version = "0.11.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
dependencies = [
"bytes",
"getrandom 0.3.4",
@@ -1917,7 +1886,6 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
"tempfile",
]
[[package]]
@@ -1930,7 +1898,7 @@ dependencies = [
"sha2",
"tar",
"tempfile",
"zip 2.4.2",
"zip",
]
[[package]]
@@ -2603,7 +2571,7 @@ dependencies = [
"sha2",
"tar",
"tempfile",
"zip 2.4.2",
"zip",
]
[[package]]
@@ -3277,7 +3245,6 @@ dependencies = [
"tempfile",
"tiny_http",
"url",
"urlencoding",
]
[[package]]
@@ -3450,18 +3417,6 @@ dependencies = [
"syn",
]
[[package]]
name = "zip"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
dependencies = [
"byteorder",
"crc32fast",
"crossbeam-utils",
"flate2",
]
[[package]]
name = "zip"
version = "2.4.2"
+3 -1
View File
@@ -32,6 +32,7 @@ members = [
"crates/detmir-readiness",
"crates/detmir-portal",
"crates/aw-slo-monitor",
"crates/aw-workforce-ingest",
"crates/aw-rus-healthd",
"crates/detmir-check",
"crates/detmir-core",
@@ -70,7 +71,6 @@ publish = false
adk-rust = { version = "0.9.1", default-features = false }
anyhow = "1"
base64 = "0.22"
calamine = "=0.24.0"
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] }
clap = { version = "4", features = ["derive", "env"] }
csv = "1"
@@ -80,6 +80,7 @@ detmir-state = { path = "crates/detmir-state" }
filetime = "0.2"
fs2 = "0.4"
notify = "6"
quick-xml = "0.41"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] }
regex = "1"
rusqlite = "0.32"
@@ -92,4 +93,5 @@ tempfile = "3"
tiny_http = "0.12"
url = "2"
urlencoding = "2"
zip = { version = "2", default-features = false, features = ["deflate"] }
windows-sys = "0.59"
+35 -1
View File
@@ -22,7 +22,9 @@ scripts with durable standalone Rust modules.
- `detmir-dlp` - SSH wrapper replacement for remote DLP health JSON collection.
- `dlp-health-check` - AW server DLP health check replacement.
- `aw-db-maintenance` - guarded weekly SQLite maintenance for old allowlisted
process-level session events, with backup-before-delete.
process-level session events, with backup-before-delete; nightly SQLite
compaction is handled by the same binary in `--vacuum` mode and scheduled
separately from the trim job.
- `aw-ensure-reliability` - safe dry-run/apply planner for AW service
reliability repair actions that were previously immediate Bash mutations.
- `aw-linux-install` - safe dry-run/apply planner for Linux ActivityWatch
@@ -58,6 +60,38 @@ scripts with durable standalone Rust modules.
- `detmir-status` - read-only DetMir state normalizer with text, JSON, and ADK
`Content` output. Also builds `detmir-adk-status` as a compatibility binary.
## SQLite Maintenance Safety
`aw-db-maintenance` has two separate modes:
- default trim mode removes only old allowlisted `process_start` /
`process_stop` rows from the configured session bucket and is dry-run unless
`--apply` is passed;
- `--vacuum` compacts the SQLite DB with `VACUUM INTO`, checks
`PRAGMA integrity_check`, preserves owner/mode, and replaces the DB only after
backup and integrity success.
Both apply modes use `AW_DB_MAINTENANCE_LOCK_PATH` /
`--lock-path` to block concurrent trim/VACUUM runs. VACUUM also checks the
configured `activitywatch-server.service` through systemd, refuses unknown or
failed unit states, stops the service before compaction, and starts it again
through a guard on success or error.
Do not run VACUUM during business hours, active incident response, evidence
collection, active backup/restore, or when the ActivityWatch service/unit state
is unclear. Rollback is replacing the SQLite DB from
`/var/lib/activitywatch/backups/db/aw-sqlite-before-db-vacuum-*.db` while
`activitywatch-server.service` is stopped, then starting the service and
checking `aw-db-health`/`detmir-status`.
The Ansible deploy installs the VACUUM unit files but does not enable the
nightly timer unless `aw_db_vacuum_timer_enabled=true` is set explicitly.
Disable it with:
```bash
systemctl disable --now aw-db-vacuum.timer
```
## Migration Runbook
Use `RUNBOOK.md` as the operational plan for replacing Python and shell modules
+39
View File
@@ -1118,6 +1118,45 @@ systemctl is-active tsj-guardian-bot tsj-guardian-watchdog gost-tg
(`sqlite.db=359.6MiB`, WAL `4.0MiB`, session rows `174`, recent process
events `0`, latest `eventType=logon`), DetMir status OK with
`dlp_counts={ok:22,warn:0,fail:0}`, `ok_for_operator=true`.
35.2. `[done]` Закрепить nightly VACUUM для AW SQLite:
- `aw-db-maintenance` получил отдельный `--vacuum` режим; weekly trim
остается отдельной задачей и не смешивается с compaction;
- добавлены `aw-server/aw-db-vacuum.service` и
`aw-server/aw-db-vacuum.timer`;
- timer schedule: `OnCalendar=*-*-* 02:10:00`,
`RandomizedDelaySec=10m`, `Persistent=true`; Ansible не включает timer
без явного opt-in `aw_db_vacuum_timer_enabled=true`;
- если opt-in не задан, `deploy_aw_server.yml` оставляет unit-файлы на
сервере, но держит `aw-db-vacuum.timer` в `disabled/stopped`;
- nightly job перед VACUUM останавливает `activitywatch-server.service`,
делает rollback backup SQLite DB, выполняет `VACUUM INTO`, проверяет
`PRAGMA integrity_check`, сохраняет владельца и режим файла и
поднимает server обратно;
- apply-режимы weekly trim и nightly VACUUM используют общий lock
`/run/aw-db-maintenance.lock`, поэтому одновременный запуск завершается
fail-closed без записи в SQLite;
- VACUUM нельзя запускать в рабочее время, во время активного
расследования/снятия доказательств, backup/restore, неизвестного
состояния `activitywatch-server.service` или если уже есть maintenance
lock;
- dry-run проверки:
`aw-db-maintenance --vacuum --json` и обычный
`aw-db-maintenance --json`; они не создают backup и не пишут в DB;
- включить timer явно:
`ansible-playbook -i inventory.ini deploy_aw_server.yml -e aw_db_vacuum_timer_enabled=true`;
- отключить timer:
`systemctl disable --now aw-db-vacuum.timer`;
- rollback: остановить `activitywatch-server.service`, заменить
`/var/lib/activitywatch/aw-server-rust/sqlite.db` из последнего
`/var/lib/activitywatch/backups/db/aw-sqlite-before-db-vacuum-*.db`,
вернуть владельца/режим, запустить service, проверить `aw-db-health` и
`detmir-status`;
- local gates: `cargo fmt --all -- --check`, `cargo test --workspace`,
`cargo clippy --workspace --all-targets -- -D warnings`,
`cargo build --workspace --release`, systemd unit verify,
`ansible-playbook -i inventory.ini deploy_aw_server.yml --syntax-check`,
`scripts/check_detmir_rust_release_artifacts.sh`,
`scripts/quality-gate.sh`;
36. `[done]` Устранить blocker полного AW server deploy на Influx token:
- проблема: `deploy_aw_server.yml` падал на assert
`aw_worktime_influx_enabled=true`, потому что локальные env
+2 -1
View File
@@ -8,12 +8,12 @@ publish = false
[dependencies]
anyhow.workspace = true
calamine.workspace = true
chrono.workspace = true
clap.workspace = true
csv.workspace = true
filetime.workspace = true
fs2.workspace = true
quick-xml.workspace = true
regex.workspace = true
reqwest.workspace = true
serde.workspace = true
@@ -21,3 +21,4 @@ serde_json.workspace = true
serde_yaml.workspace = true
sha1.workspace = true
tempfile.workspace = true
zip.workspace = true
+367 -22
View File
@@ -6,16 +6,19 @@ use std::process::Command;
use std::time::Duration;
use anyhow::{Context, Result, anyhow, bail};
use calamine::{Data, Reader, open_workbook_auto};
use chrono::{DateTime, NaiveDateTime, Utc};
use clap::Parser;
use filetime::{FileTime, set_file_times};
use fs2::FileExt;
use quick_xml::Reader as XmlReader;
use quick_xml::XmlVersion;
use quick_xml::events::{BytesStart, Event};
use regex::Regex;
use reqwest::blocking::Client;
use serde::Deserialize;
use serde_json::{Map, Value, json};
use sha1::{Digest, Sha1};
use zip::ZipArchive;
const DATASETS: &[&str] = &[
"documents",
@@ -1103,15 +1106,14 @@ fn load_registry(
}
fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
let mut workbook =
open_workbook_auto(path).with_context(|| format!("open xlsx {}", path.display()))?;
let workbook = read_xlsx_workbook(path)?;
let now = Utc::now()
.naive_utc()
.format("%Y-%m-%d %H:%M:%S")
.to_string();
let mut tax_map: HashMap<String, (String, String)> = HashMap::new();
if let Ok(range) = workbook.worksheet_range("Лист2") {
for row in range.rows().skip(2) {
if let Some(rows) = workbook.get("Лист2") {
for row in rows.iter().skip(2) {
let company_name = cell_text(row.get(1));
if company_name.is_empty() {
continue;
@@ -1123,10 +1125,10 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
}
}
let mut out = Vec::new();
let Ok(range) = workbook.worksheet_range("ОСНОВНОЙ") else {
let Some(sheet_rows) = workbook.get("ОСНОВНОЙ") else {
return Ok(out);
};
let rows: Vec<_> = range.rows().collect();
let rows: Vec<_> = sheet_rows.iter().collect();
if rows.len() < 3 {
return Ok(out);
}
@@ -1134,7 +1136,8 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
let manager_headers: Vec<String> = rows[1].iter().map(|v| cell_text(Some(v))).collect();
let mut specs = Vec::<(usize, String, Option<usize>, String, String)>::new();
let mut current: Option<(usize, String, String)> = None;
for idx in 1..manager_headers.len() {
let header_len = top_headers.len().max(manager_headers.len());
for idx in 1..header_len {
let manager = manager_headers.get(idx).cloned().unwrap_or_default();
let top = top_headers.get(idx).cloned().unwrap_or_default();
if !manager.is_empty() {
@@ -1194,26 +1197,291 @@ fn parse_registry_xlsx(path: &Path) -> Result<Vec<Value>> {
Ok(out)
}
fn cell_text(value: Option<&Data>) -> String {
match value {
Some(Data::String(v)) => v.trim().to_string(),
Some(Data::Float(v)) => {
if v.fract() == 0.0 {
format!("{}", *v as i64)
type XlsxWorkbook = HashMap<String, Vec<Vec<String>>>;
fn read_xlsx_workbook(path: &Path) -> Result<XlsxWorkbook> {
let file = File::open(path).with_context(|| format!("open xlsx {}", path.display()))?;
let mut archive =
ZipArchive::new(file).with_context(|| format!("read xlsx {}", path.display()))?;
let shared_strings = read_xlsx_shared_strings(&mut archive)?;
let sheets = read_xlsx_sheet_paths(&mut archive)?;
let mut workbook = XlsxWorkbook::new();
for (name, sheet_path) in sheets {
let xml = read_zip_text(&mut archive, &sheet_path)
.with_context(|| format!("read sheet {name} from {}", path.display()))?;
let rows = parse_xlsx_sheet(&xml, &shared_strings)
.with_context(|| format!("parse sheet {name} from {}", path.display()))?;
workbook.insert(name, rows);
}
Ok(workbook)
}
fn read_xlsx_shared_strings<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
) -> Result<Vec<String>> {
let Some(xml) = read_zip_text_optional(archive, "xl/sharedStrings.xml")? else {
return Ok(Vec::new());
};
let mut reader = XmlReader::from_str(&xml);
reader.config_mut().trim_text(false);
let mut buf = Vec::new();
let mut strings = Vec::new();
let mut in_si = false;
let mut in_t = false;
let mut current = String::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Start(e) => match local_name(e.name().as_ref()) {
b"si" => {
in_si = true;
current.clear();
}
b"t" if in_si => in_t = true,
_ => {}
},
Event::Text(e) if in_si && in_t => current.push_str(e.decode()?.as_ref()),
Event::End(e) => match local_name(e.name().as_ref()) {
b"t" => in_t = false,
b"si" => {
strings.push(current.clone());
in_si = false;
}
_ => {}
},
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(strings)
}
fn read_xlsx_sheet_paths<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
) -> Result<Vec<(String, String)>> {
let workbook_xml = read_zip_text(archive, "xl/workbook.xml")?;
let rels_xml = read_zip_text(archive, "xl/_rels/workbook.xml.rels")?;
let rels = parse_xlsx_relationships(&rels_xml)?;
let mut reader = XmlReader::from_str(&workbook_xml);
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut sheets = Vec::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) | Event::Start(e) if local_name(e.name().as_ref()) == b"sheet" => {
let name = xml_attr(&reader, &e, b"name")?.unwrap_or_default();
let mut rel_id = xml_attr(&reader, &e, b"r:id")?.unwrap_or_default();
if rel_id.is_empty() {
rel_id = xml_attr(&reader, &e, b"id")?.unwrap_or_default();
}
if !name.is_empty() && !rel_id.is_empty() {
let target = rels
.get(&rel_id)
.with_context(|| format!("missing workbook relationship {rel_id}"))?;
sheets.push((name, normalize_xlsx_target(target)?));
}
}
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(sheets)
}
fn parse_xlsx_relationships(xml: &str) -> Result<HashMap<String, String>> {
let mut reader = XmlReader::from_str(xml);
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut rels = HashMap::new();
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) | Event::Start(e)
if local_name(e.name().as_ref()) == b"Relationship" =>
{
let id = xml_attr(&reader, &e, b"Id")?.unwrap_or_default();
let target = xml_attr(&reader, &e, b"Target")?.unwrap_or_default();
if !id.is_empty() && !target.is_empty() {
rels.insert(id, target);
}
}
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(rels)
}
fn parse_xlsx_sheet(xml: &str, shared_strings: &[String]) -> Result<Vec<Vec<String>>> {
let mut reader = XmlReader::from_str(xml);
reader.config_mut().trim_text(false);
let mut buf = Vec::new();
let mut rows = Vec::<Vec<String>>::new();
let mut current_row = Vec::<String>::new();
let mut current_cell: Option<(usize, String, String)> = None;
let mut in_value = false;
let mut in_inline_text = false;
loop {
match reader.read_event_into(&mut buf)? {
Event::Empty(e) => match local_name(e.name().as_ref()) {
b"row" => rows.push(Vec::new()),
b"c" => {
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
let column = cell_ref
.chars()
.take_while(|ch| ch.is_ascii_alphabetic())
.collect::<String>();
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
if current_row.len() <= col {
current_row.resize(col + 1, String::new());
}
}
_ => {}
},
Event::Start(e) => match local_name(e.name().as_ref()) {
b"row" => current_row.clear(),
b"c" => {
let cell_ref = xml_attr(&reader, &e, b"r")?.unwrap_or_default();
let cell_type = xml_attr(&reader, &e, b"t")?.unwrap_or_default();
let column = cell_ref
.chars()
.take_while(|ch| ch.is_ascii_alphabetic())
.collect::<String>();
let col = xlsx_column_index(&column).unwrap_or(current_row.len());
current_cell = Some((col, cell_type, String::new()));
}
b"v" => in_value = true,
b"t" if current_cell
.as_ref()
.is_some_and(|(_, kind, _)| kind == "inlineStr") =>
{
in_inline_text = true;
}
_ => {}
},
Event::Text(e) if in_value || in_inline_text => {
if let Some((_, _, value)) = current_cell.as_mut() {
value.push_str(e.decode()?.as_ref());
}
}
Event::End(e) => match local_name(e.name().as_ref()) {
b"v" => in_value = false,
b"t" => in_inline_text = false,
b"c" => {
if let Some((col, cell_type, raw)) = current_cell.take() {
let value = xlsx_cell_value(&cell_type, &raw, shared_strings);
if current_row.len() <= col {
current_row.resize(col + 1, String::new());
}
current_row[col] = value;
}
}
b"row" => rows.push(current_row.clone()),
_ => {}
},
Event::Eof => break,
_ => {}
}
buf.clear();
}
Ok(rows)
}
fn read_zip_text<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
path: &str,
) -> Result<String> {
read_zip_text_optional(archive, path)?.with_context(|| format!("missing xlsx entry {path}"))
}
fn read_zip_text_optional<R: Read + std::io::Seek>(
archive: &mut ZipArchive<R>,
path: &str,
) -> Result<Option<String>> {
let Ok(mut file) = archive.by_name(path) else {
return Ok(None);
};
let mut text = String::new();
file.read_to_string(&mut text)
.with_context(|| format!("read xlsx entry {path}"))?;
Ok(Some(text))
}
fn normalize_xlsx_target(target: &str) -> Result<String> {
let normalized = target.replace('\\', "/");
if normalized.split('/').any(|part| part == "..") {
bail!("unsafe xlsx relationship target: {target}");
}
let path = normalized.trim_start_matches('/');
if path.starts_with("xl/") {
Ok(path.to_string())
} else {
Ok(format!("xl/{path}"))
}
}
fn xml_attr(reader: &XmlReader<&[u8]>, e: &BytesStart<'_>, key: &[u8]) -> Result<Option<String>> {
for attr in e.attributes() {
let attr = attr?;
if attr.key.as_ref() == key {
return Ok(Some(
attr.decoded_and_normalized_value(XmlVersion::Implicit1_0, reader.decoder())?
.into_owned(),
));
}
}
Ok(None)
}
fn local_name(name: &[u8]) -> &[u8] {
name.rsplit(|b| *b == b':').next().unwrap_or(name)
}
fn xlsx_column_index(column: &str) -> Option<usize> {
let mut value = 0usize;
for byte in column.bytes() {
if !byte.is_ascii_alphabetic() {
return None;
}
value = value * 26 + usize::from(byte.to_ascii_uppercase() - b'A' + 1);
}
value.checked_sub(1)
}
fn xlsx_cell_value(cell_type: &str, raw: &str, shared_strings: &[String]) -> String {
match cell_type {
"s" => raw
.parse::<usize>()
.ok()
.and_then(|idx| shared_strings.get(idx))
.cloned()
.unwrap_or_default(),
"b" => {
if raw.trim() == "1" {
"true".to_string()
} else {
v.to_string()
"false".to_string()
}
}
Some(Data::Int(v)) => v.to_string(),
Some(Data::Bool(v)) => v.to_string(),
Some(Data::DateTime(v)) => v.to_string(),
Some(Data::DateTimeIso(v)) => v.trim().to_string(),
Some(Data::DurationIso(v)) => v.trim().to_string(),
Some(Data::Empty) | None => String::new(),
Some(other) => other.to_string(),
_ => raw.to_string(),
}
}
fn cell_text(value: Option<&String>) -> String {
value
.map(|v| {
let trimmed = v.trim();
if let Ok(number) = trimmed.parse::<f64>() {
if number.fract() == 0.0 {
return format!("{}", number as i64);
}
}
trimmed.to_string()
})
.unwrap_or_default()
}
fn archive_or_delete(
config: &RawConfig,
dataset: &str,
@@ -1507,4 +1775,81 @@ mod tests {
fn split_sql_ignores_empty_chunks() {
assert_eq!(split_sql_statements("SELECT 1; ; SELECT 2;").len(), 2);
}
#[test]
fn parses_registry_xlsx_with_inline_strings() -> Result<()> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("registry.xlsx");
let file = File::create(&path)?;
let mut zip = zip::ZipWriter::new(file);
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Stored);
for (name, xml) in [
(
"xl/workbook.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<sheets>
<sheet name="Лист2" sheetId="1" r:id="rId1"/>
<sheet name="ОСНОВНОЙ" sheetId="2" r:id="rId2"/>
</sheets>
</workbook>"#,
),
(
"xl/_rels/workbook.xml.rels",
r#"<?xml version="1.0" encoding="UTF-8"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Target="worksheets/sheet1.xml"/>
<Relationship Id="rId2" Target="worksheets/sheet2.xml"/>
</Relationships>"#,
),
(
"xl/worksheets/sheet1.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1"/>
<row r="2"/>
<row r="3">
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
<c r="C3" t="inlineStr"><is><t>1101000000</t></is></c>
<c r="D3" t="inlineStr"><is><t>110101001</t></is></c>
</row>
</sheetData>
</worksheet>"#,
),
(
"xl/worksheets/sheet2.xml",
r#"<?xml version="1.0" encoding="UTF-8"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1">
<c r="C1" t="inlineStr"><is><t>Ключевой контур</t></is></c>
</row>
<row r="2">
<c r="B2" t="inlineStr"><is><t>Иванов И.И.</t></is></c>
</row>
<row r="3">
<c r="B3" t="inlineStr"><is><t>ООО Ромашка</t></is></c>
<c r="C3" t="inlineStr"><is><t>ЕСТЬ</t></is></c>
</row>
</sheetData>
</worksheet>"#,
),
] {
zip.start_file(name, options)?;
zip.write_all(xml.as_bytes())?;
}
zip.finish()?;
let rows = parse_registry_xlsx(&path)?;
assert_eq!(rows.len(), 1);
assert_eq!(rows[0]["company_name"], "ООО Ромашка");
assert_eq!(rows[0]["company_key"], "ООО РОМАШКА");
assert_eq!(rows[0]["assignee_name"], "Иванов И.И.");
assert_eq!(rows[0]["key_contour"], 1);
assert_eq!(rows[0]["inn"], "1101000000");
assert_eq!(rows[0]["kpp"], "110101001");
Ok(())
}
}
@@ -8,11 +8,6 @@ publish.workspace = true
[dependencies]
anyhow.workspace = true
chrono.workspace = true
clap.workspace = true
reqwest.workspace = true
serde_json.workspace = true
url.workspace = true
[dev-dependencies]
tempfile.workspace = true
+543 -8
View File
@@ -1,5 +1,7 @@
use std::fs;
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;
use anyhow::{Context, Result, bail};
@@ -12,6 +14,8 @@ use serde_json::Value;
const DEFAULT_DB_PATH: &str = "/var/lib/activitywatch/aw-server-rust/sqlite.db";
const DEFAULT_BACKUP_DIR: &str = "/var/lib/activitywatch/backups/db";
const DEFAULT_HOST: &str = "HOST-EXAMPLE";
const DEFAULT_SERVICE_UNIT: &str = "activitywatch-server.service";
const DEFAULT_LOCK_PATH: &str = "/run/aw-db-maintenance.lock";
const ALLOWED_EVENT_TYPES: &[&str] = &["process_start", "process_stop"];
#[derive(Debug, Parser)]
@@ -38,6 +42,19 @@ struct Cli {
#[arg(long)]
apply: bool,
#[arg(long)]
vacuum: bool,
#[arg(
long,
default_value = DEFAULT_SERVICE_UNIT,
env = "AW_DB_MAINTENANCE_SERVICE_UNIT"
)]
service_unit: String,
#[arg(long, default_value = DEFAULT_LOCK_PATH, env = "AW_DB_MAINTENANCE_LOCK_PATH")]
lock_path: PathBuf,
#[arg(long)]
json: bool,
}
@@ -56,9 +73,52 @@ struct Report {
planned_delete_rows: usize,
deleted_rows: usize,
backup_created: bool,
lock_path: String,
skipped_reason: Option<String>,
}
#[derive(Debug, Serialize)]
struct VacuumReport {
apply: bool,
generated_at_utc: String,
db_path: String,
service_unit: String,
service_was_active: bool,
service_restarted: bool,
backup_path: Option<String>,
backup_created: bool,
lock_path: String,
db_size_before_bytes: Option<u64>,
vacuumed_path: Option<String>,
vacuumed_size_bytes: Option<u64>,
integrity_check: Option<String>,
replaced_db: bool,
skipped_reason: Option<String>,
}
struct VacuumResult {
backup_path: PathBuf,
vacuumed_path: PathBuf,
db_size_before_bytes: u64,
vacuumed_size_bytes: u64,
integrity_check: String,
}
struct ServiceGuard {
unit: String,
was_active: bool,
restored: bool,
}
struct TempFileGuard {
path: PathBuf,
keep: bool,
}
struct LockFileGuard {
path: PathBuf,
}
fn main() {
let code = match run() {
Ok(code) => code,
@@ -72,11 +132,20 @@ fn main() {
fn run() -> Result<i32> {
let cli = Cli::parse();
let report = build_report(&cli)?;
if cli.json {
println!("{}", serde_json::to_string_pretty(&report)?);
if cli.vacuum {
let report = build_vacuum_report(&cli)?;
if cli.json {
println!("{}", serde_json::to_string_pretty(&report)?);
} else {
print_vacuum_text(&report);
}
} else {
print_text(&report);
let report = build_report(&cli)?;
if cli.json {
println!("{}", serde_json::to_string_pretty(&report)?);
} else {
print_text(&report);
}
}
Ok(0)
}
@@ -116,10 +185,15 @@ fn build_report(cli: &Cli) -> Result<Report> {
let mut backup_file = None;
let mut backup_created = false;
let mut deleted = 0;
let _lock_guard = if cli.apply && planned > 0 {
Some(LockFileGuard::acquire(&cli.lock_path)?)
} else {
None
};
if cli.apply && planned > 0 {
fs::create_dir_all(&cli.backup_dir)
.with_context(|| format!("create backup dir {}", cli.backup_dir.display()))?;
let backup = backup_path(&cli.backup_dir);
let backup = backup_path(&cli.backup_dir, "aw-sqlite-before-db-maintenance");
copy_sqlite_via_backup(&cli.db_path, &backup)?;
backup_file = Some(backup);
backup_created = true;
@@ -139,6 +213,60 @@ fn build_report(cli: &Cli) -> Result<Report> {
))
}
fn build_vacuum_report(cli: &Cli) -> Result<VacuumReport> {
if !cli.db_path.exists() {
return Ok(vacuum_report(
cli,
false,
None,
false,
false,
false,
None,
None,
None,
None,
Some("database not found".to_string()),
));
}
if !cli.apply {
return Ok(vacuum_report(
cli,
false,
Some(file_size(&cli.db_path)?),
false,
false,
false,
None,
None,
None,
None,
Some("dry-run".to_string()),
));
}
let _lock_guard = LockFileGuard::acquire(&cli.lock_path)?;
let mut service_guard = ServiceGuard::stop_if_active(&cli.service_unit)?;
let service_was_active = service_guard.was_active;
let result = vacuum_sqlite_db(&cli.db_path, &cli.backup_dir)?;
let service_restarted = service_guard.restore()?;
Ok(vacuum_report(
cli,
true,
Some(result.db_size_before_bytes),
true,
service_was_active,
service_restarted,
Some(result.backup_path),
Some(result.vacuumed_path),
Some(result.vacuumed_size_bytes),
Some(result.integrity_check),
None,
))
}
#[allow(clippy::too_many_arguments)]
fn base_report(
cli: &Cli,
@@ -164,6 +292,40 @@ fn base_report(
planned_delete_rows,
deleted_rows,
backup_created,
lock_path: cli.lock_path.display().to_string(),
skipped_reason,
}
}
#[allow(clippy::too_many_arguments)]
fn vacuum_report(
cli: &Cli,
apply: bool,
db_size_before_bytes: Option<u64>,
backup_created: bool,
service_was_active: bool,
service_restarted: bool,
backup_path: Option<PathBuf>,
vacuumed_path: Option<PathBuf>,
vacuumed_size_bytes: Option<u64>,
integrity_check: Option<String>,
skipped_reason: Option<String>,
) -> VacuumReport {
VacuumReport {
apply,
generated_at_utc: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true),
db_path: cli.db_path.display().to_string(),
service_unit: cli.service_unit.clone(),
service_was_active,
service_restarted,
backup_path: backup_path.map(|path| path.display().to_string()),
backup_created,
lock_path: cli.lock_path.display().to_string(),
db_size_before_bytes,
vacuumed_path: vacuumed_path.map(|path| path.display().to_string()),
vacuumed_size_bytes,
integrity_check,
replaced_db: apply && skipped_reason.is_none(),
skipped_reason,
}
}
@@ -237,13 +399,256 @@ fn delete_events(conn: &Connection, ids: &[i64], chunk_size: usize) -> Result<us
Ok(deleted)
}
fn backup_path(backup_dir: &Path) -> PathBuf {
fn backup_path(backup_dir: &Path, prefix: &str) -> PathBuf {
backup_dir.join(format!(
"aw-sqlite-before-db-maintenance-{}.db",
"{}-{}.db",
prefix,
Utc::now().format("%Y%m%dT%H%M%SZ")
))
}
fn vacuum_sqlite_db(db_path: &Path, backup_dir: &Path) -> Result<VacuumResult> {
fs::create_dir_all(backup_dir)
.with_context(|| format!("create backup dir {}", backup_dir.display()))?;
let db_size_before_bytes = file_size(db_path)?;
let backup_path = backup_path(backup_dir, "aw-sqlite-before-db-vacuum");
copy_sqlite_via_backup(db_path, &backup_path)?;
let vacuumed_path = vacuumed_path(db_path)?;
let mut vacuum_cleanup = TempFileGuard::new(vacuumed_path.clone());
vacuum_into(db_path, &vacuumed_path)?;
preserve_sqlite_metadata(db_path, &vacuumed_path)?;
let vacuumed_size_bytes = file_size(&vacuumed_path)?;
let integrity_check = integrity_check(&vacuumed_path)?;
remove_sqlite_sidecars(db_path)?;
fs::rename(&vacuumed_path, db_path).with_context(|| {
format!(
"replace {} with {}",
db_path.display(),
vacuumed_path.display()
)
})?;
vacuum_cleanup.disarm();
Ok(VacuumResult {
backup_path,
vacuumed_path,
db_size_before_bytes,
vacuumed_size_bytes,
integrity_check,
})
}
fn vacuum_into(src: &Path, dst: &Path) -> Result<()> {
let conn = open_connection(src, true)?;
let sql = format!("VACUUM INTO {}", sqlite_string_literal(dst));
conn.execute_batch(&sql)
.with_context(|| format!("VACUUM INTO {}", dst.display()))
}
fn integrity_check(path: &Path) -> Result<String> {
let conn = open_connection(path, false)?;
let result: String = conn.query_row("PRAGMA integrity_check", [], |row| row.get(0))?;
if result != "ok" {
bail!("integrity_check failed for {}: {result}", path.display());
}
Ok(result)
}
fn remove_sqlite_sidecars(db_path: &Path) -> Result<()> {
for suffix in ["-wal", "-shm", "-journal"] {
let sidecar = sqlite_sidecar_path(db_path, suffix)?;
match fs::remove_file(&sidecar) {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err).with_context(|| format!("remove {}", sidecar.display())),
}
}
Ok(())
}
fn sqlite_sidecar_path(db_path: &Path, suffix: &str) -> Result<PathBuf> {
let file_name = db_path
.file_name()
.and_then(|value| value.to_str())
.context("database path must have a file name")?;
Ok(db_path.with_file_name(format!("{file_name}{suffix}")))
}
fn vacuumed_path(db_path: &Path) -> Result<PathBuf> {
let file_name = db_path
.file_name()
.and_then(|value| value.to_str())
.context("database path must have a file name")?;
Ok(db_path.with_file_name(format!(
"{file_name}.vacuumed-{}",
Utc::now().format("%Y%m%dT%H%M%SZ")
)))
}
fn file_size(path: &Path) -> Result<u64> {
Ok(fs::metadata(path)
.with_context(|| format!("stat {}", path.display()))?
.len())
}
fn sqlite_string_literal(path: &Path) -> String {
format!("'{}'", path.display().to_string().replace('\'', "''"))
}
fn preserve_sqlite_metadata(src: &Path, dst: &Path) -> Result<()> {
let metadata = fs::metadata(src).with_context(|| format!("stat {}", src.display()))?;
let permissions = metadata.permissions();
fs::set_permissions(dst, permissions)
.with_context(|| format!("preserve permissions for {}", dst.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let dst_metadata = fs::metadata(dst).with_context(|| format!("stat {}", dst.display()))?;
if dst_metadata.uid() != metadata.uid() || dst_metadata.gid() != metadata.gid() {
let status = Command::new("chown")
.arg(format!("{}:{}", metadata.uid(), metadata.gid()))
.arg(dst)
.status()
.context("run chown for vacuumed SQLite DB")?;
if !status.success() {
bail!("chown failed for {}", dst.display());
}
}
}
Ok(())
}
fn systemctl_is_active(unit: &str) -> Result<bool> {
let load_state = systemctl_load_state(unit)?;
if load_state != "loaded" {
bail!("refusing SQLite VACUUM because systemd unit {unit} load_state={load_state:?}");
}
let output = Command::new("systemctl")
.args(["is-active", unit])
.output()
.with_context(|| format!("systemctl is-active {unit}"))?;
let state = String::from_utf8_lossy(&output.stdout).trim().to_string();
if output.status.success() && state == "active" {
return Ok(true);
}
if output.status.code() == Some(3) && state == "inactive" {
return Ok(false);
}
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
bail!(
"refusing SQLite VACUUM because systemctl is-active {unit} returned state={state:?}, status={}, stderr={stderr:?}",
output.status
);
}
fn systemctl_load_state(unit: &str) -> Result<String> {
let output = Command::new("systemctl")
.args(["show", "-p", "LoadState", "--value", unit])
.output()
.with_context(|| format!("systemctl show LoadState {unit}"))?;
let state = String::from_utf8_lossy(&output.stdout).trim().to_string();
if output.status.success() && !state.is_empty() {
return Ok(state);
}
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
bail!(
"refusing SQLite VACUUM because systemctl show LoadState {unit} failed with status={}, stderr={stderr:?}",
output.status
);
}
fn systemctl_action(action: &str, unit: &str) -> Result<()> {
let status = Command::new("systemctl")
.args([action, unit])
.status()
.with_context(|| format!("systemctl {action} {unit}"))?;
if status.success() {
Ok(())
} else {
bail!("systemctl {action} {unit} failed with status {status}");
}
}
impl ServiceGuard {
fn stop_if_active(unit: &str) -> Result<Self> {
let was_active = systemctl_is_active(unit)?;
if was_active {
systemctl_action("stop", unit)?;
}
Ok(Self {
unit: unit.to_string(),
was_active,
restored: !was_active,
})
}
fn restore(&mut self) -> Result<bool> {
if self.was_active && !self.restored {
systemctl_action("start", &self.unit)?;
self.restored = true;
}
Ok(self.was_active)
}
}
impl Drop for ServiceGuard {
fn drop(&mut self) {
if self.was_active && !self.restored {
let _ = systemctl_action("start", &self.unit);
}
}
}
impl TempFileGuard {
fn new(path: PathBuf) -> Self {
Self { path, keep: false }
}
fn disarm(&mut self) {
self.keep = true;
}
}
impl Drop for TempFileGuard {
fn drop(&mut self) {
if !self.keep {
let _ = fs::remove_file(&self.path);
}
}
}
impl LockFileGuard {
fn acquire(path: &Path) -> Result<Self> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("create lock parent {}", parent.display()))?;
}
let mut file = match OpenOptions::new().write(true).create_new(true).open(path) {
Ok(file) => file,
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {
bail!("maintenance lock already exists: {}", path.display());
}
Err(err) => return Err(err).with_context(|| format!("create lock {}", path.display())),
};
writeln!(
file,
"pid={} generated_at_utc={}",
std::process::id(),
Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true)
)
.with_context(|| format!("write lock {}", path.display()))?;
Ok(Self {
path: path.to_path_buf(),
})
}
}
impl Drop for LockFileGuard {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
fn print_text(report: &Report) {
println!(
"aw-db-maintenance: {}",
@@ -255,6 +660,7 @@ fn print_text(report: &Report) {
println!("planned_delete_rows: {}", report.planned_delete_rows);
println!("deleted_rows: {}", report.deleted_rows);
println!("backup_created: {}", report.backup_created);
println!("lock_path: {}", report.lock_path);
if let Some(path) = &report.backup_path {
println!("backup_path: {path}");
}
@@ -263,6 +669,38 @@ fn print_text(report: &Report) {
}
}
fn print_vacuum_text(report: &VacuumReport) {
println!(
"aw-db-vacuum: {}",
if report.apply { "apply" } else { "dry-run" }
);
println!("db_path: {}", report.db_path);
println!("service_unit: {}", report.service_unit);
println!("service_was_active: {}", report.service_was_active);
println!("service_restarted: {}", report.service_restarted);
println!("backup_created: {}", report.backup_created);
println!("lock_path: {}", report.lock_path);
if let Some(path) = &report.backup_path {
println!("backup_path: {path}");
}
if let Some(size) = report.db_size_before_bytes {
println!("db_size_before_bytes: {size}");
}
if let Some(path) = &report.vacuumed_path {
println!("vacuumed_path: {path}");
}
if let Some(size) = report.vacuumed_size_bytes {
println!("vacuumed_size_bytes: {size}");
}
if let Some(check) = &report.integrity_check {
println!("integrity_check: {check}");
}
println!("replaced_db: {}", report.replaced_db);
if let Some(reason) = &report.skipped_reason {
println!("skipped_reason: {reason}");
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -290,6 +728,9 @@ mod tests {
retention_days: 7,
chunk_size: 100,
apply: false,
vacuum: false,
service_unit: DEFAULT_SERVICE_UNIT.to_string(),
lock_path: dir.path().join("maintenance.lock"),
json: true,
};
let report = build_report(&cli).unwrap();
@@ -312,6 +753,9 @@ mod tests {
retention_days: 7,
chunk_size: 1,
apply: true,
vacuum: false,
service_unit: DEFAULT_SERVICE_UNIT.to_string(),
lock_path: dir.path().join("maintenance.lock"),
json: true,
};
let report = build_report(&cli).unwrap();
@@ -321,6 +765,73 @@ mod tests {
assert_eq!(count_events(&db), 1);
}
#[test]
fn vacuum_apply_compacts_database_and_preserves_rows() {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("sqlite.db");
create_vacuum_fixture_db(&db);
let before = file_size(&db).unwrap();
let result = vacuum_sqlite_db(&db, dir.path()).unwrap();
let after = file_size(&db).unwrap();
assert!(result.vacuumed_size_bytes < result.db_size_before_bytes);
assert!(after < before);
assert_eq!(result.integrity_check, "ok");
assert!(result.backup_path.exists());
assert_eq!(count_rows(&db), 32);
}
#[test]
fn vacuum_dry_run_skips_mutation() {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("sqlite.db");
create_vacuum_fixture_db(&db);
let cli = Cli {
db_path: db.clone(),
backup_dir: dir.path().join("backups"),
session_bucket: None,
host: None,
retention_days: 7,
chunk_size: 100,
apply: false,
vacuum: true,
service_unit: DEFAULT_SERVICE_UNIT.to_string(),
lock_path: dir.path().join("maintenance.lock"),
json: true,
};
let report = build_vacuum_report(&cli).unwrap();
assert!(!report.backup_created);
assert!(!report.replaced_db);
assert_eq!(report.skipped_reason.as_deref(), Some("dry-run"));
assert_eq!(count_rows(&db), 32);
}
#[test]
fn apply_refuses_when_lock_exists() {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("aw.db");
create_fixture_db(&db);
let lock_path = dir.path().join("maintenance.lock");
fs::write(&lock_path, "busy").unwrap();
let cli = Cli {
db_path: db.clone(),
backup_dir: dir.path().join("backups"),
session_bucket: Some("aw-session-events_TEST".to_string()),
host: None,
retention_days: 7,
chunk_size: 1,
apply: true,
vacuum: false,
service_unit: DEFAULT_SERVICE_UNIT.to_string(),
lock_path,
json: true,
};
let err = build_report(&cli).unwrap_err().to_string();
assert!(err.contains("maintenance lock already exists"));
assert_eq!(count_events(&db), 3);
}
fn create_fixture_db(path: &Path) {
let conn = Connection::open(path).unwrap();
conn.execute_batch(
@@ -350,10 +861,34 @@ mod tests {
.unwrap();
}
fn create_vacuum_fixture_db(path: &Path) {
let conn = Connection::open(path).unwrap();
conn.execute_batch(
"create table items (id integer primary key autoincrement, payload text);",
)
.unwrap();
let payload = "x".repeat(4096);
for _ in 0..64 {
conn.execute("insert into items (payload) values (?1)", [&payload])
.unwrap();
}
for id in 1..=32 {
conn.execute("delete from items where id = ?1", [id])
.unwrap();
}
}
fn count_events(path: &Path) -> i64 {
Connection::open(path)
.unwrap()
.query_row("select count(*) from events", [], |row| row.get(0))
.unwrap()
}
fn count_rows(path: &Path) -> i64 {
Connection::open(path)
.unwrap()
.query_row("select count(*) from items", [], |row| row.get(0))
.unwrap()
}
}
@@ -10,6 +10,3 @@ publish.workspace = true
anyhow.workspace = true
reqwest.workspace = true
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -7,6 +7,7 @@ use clap::Parser;
use serde::Serialize;
const DEFAULT_DATA_DIR: &str = "/var/lib/activitywatch";
const DEFAULT_WORKTIME_REPORT_CACHE_RETENTION_SECONDS: u64 = 86_400;
const TMP_ARCHIVE_PATTERNS: &[NamePattern] = &[
NamePattern::PrefixSuffix("activitywatch-", ".zip"),
NamePattern::PrefixSuffix("hayabusa-", ".zip"),
@@ -30,6 +31,9 @@ struct Cli {
#[arg(long)]
browser_smoke_dir: Option<PathBuf>,
#[arg(long)]
worktime_report_cache_dir: Option<PathBuf>,
#[arg(long)]
tmp_dir: Option<PathBuf>,
@@ -48,6 +52,9 @@ struct Cli {
#[arg(long, default_value_t = 1)]
browser_smoke_retention_days: u64,
#[arg(long, default_value_t = DEFAULT_WORKTIME_REPORT_CACHE_RETENTION_SECONDS)]
worktime_report_cache_retention_seconds: u64,
#[arg(long, default_value_t = 1)]
tmp_archive_retention_days: u64,
@@ -66,12 +73,14 @@ struct Config {
data_dir: PathBuf,
backup_dir: PathBuf,
browser_smoke_dir: PathBuf,
worktime_report_cache_dir: PathBuf,
tmp_dir: PathBuf,
backup_retention_days: u64,
backup_keep_last_db: usize,
backup_keep_last_json: usize,
browser_smoke_keep_runs: usize,
browser_smoke_retention_days: u64,
worktime_report_cache_retention_seconds: u64,
tmp_archive_retention_days: u64,
tmp_webui_retention_days: u64,
apply: bool,
@@ -154,6 +163,7 @@ fn run() -> Result<i32> {
&mut items,
)?;
plan_browser_smoke(&cfg, &mut items)?;
plan_worktime_report_cache(&cfg, &mut items)?;
plan_tmp(
&cfg.tmp_dir,
TMP_ARCHIVE_PATTERNS,
@@ -200,6 +210,10 @@ impl Config {
.browser_smoke_dir
.or_else(|| env_path("AW_BROWSER_SMOKE_OUTPUT_DIR"))
.unwrap_or_else(|| data_dir.join("browser-smoke"));
let worktime_report_cache_dir = cli
.worktime_report_cache_dir
.or_else(|| env_path("AW_WORKTIME_REPORT_DISK_CACHE_DIR"))
.unwrap_or_else(|| data_dir.join("worktime-report-cache"));
let tmp_dir = cli
.tmp_dir
.or_else(|| env_path("AW_TMP_DIR"))
@@ -208,6 +222,7 @@ impl Config {
data_dir,
backup_dir,
browser_smoke_dir,
worktime_report_cache_dir,
tmp_dir,
backup_retention_days: env_u64("AW_BACKUP_RETENTION_DAYS", cli.backup_retention_days),
backup_keep_last_db: env_usize("AW_BACKUP_KEEP_LAST_DB", cli.backup_keep_last_db),
@@ -220,6 +235,10 @@ impl Config {
"AW_BROWSER_SMOKE_RETENTION_DAYS",
cli.browser_smoke_retention_days,
),
worktime_report_cache_retention_seconds: env_u64(
"AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS",
cli.worktime_report_cache_retention_seconds,
),
tmp_archive_retention_days: env_u64(
"AW_TMP_ARCHIVE_RETENTION_DAYS",
cli.tmp_archive_retention_days,
@@ -267,6 +286,41 @@ fn plan_browser_smoke(cfg: &Config, items: &mut Vec<PruneItem>) -> Result<()> {
Ok(())
}
fn plan_worktime_report_cache(cfg: &Config, items: &mut Vec<PruneItem>) -> Result<()> {
if cfg.worktime_report_cache_retention_seconds == 0 || !cfg.worktime_report_cache_dir.exists() {
return Ok(());
}
let cutoff = cutoff_seconds(cfg.worktime_report_cache_retention_seconds);
for entry in fs::read_dir(&cfg.worktime_report_cache_dir)
.with_context(|| format!("read {}", cfg.worktime_report_cache_dir.display()))?
{
let entry = entry?;
if !entry.file_type()?.is_file() {
continue;
}
let name = entry.file_name();
let name = name.to_string_lossy();
if !name.ends_with(".json") {
continue;
}
let meta = entry.metadata()?;
let modified = meta.modified().unwrap_or(SystemTime::UNIX_EPOCH);
if modified >= cutoff {
continue;
}
items.push(
Candidate {
path: entry.path(),
kind: ItemKind::File,
modified,
size_bytes: meta.len(),
}
.into_item("worktime_report_disk_cache"),
);
}
Ok(())
}
fn plan_tmp(
dir: &Path,
patterns: &[NamePattern],
@@ -395,6 +449,9 @@ fn validate_plan(cfg: &Config, items: &[PruneItem]) -> Result<()> {
"backup_db" => is_under_or_equal(&item.path, &cfg.backup_dir.join("db")),
"backup_root" => is_under_or_equal(&item.path, &cfg.backup_dir),
"browser_smoke_run" => is_under_or_equal(&item.path, &cfg.browser_smoke_dir),
"worktime_report_disk_cache" => {
is_under_or_equal(&item.path, &cfg.worktime_report_cache_dir)
}
"tmp_archive" | "tmp_webui" => is_under_or_equal(&item.path, &cfg.tmp_dir),
_ => false,
};
@@ -410,6 +467,7 @@ fn validate_plan(cfg: &Config, items: &[PruneItem]) -> Result<()> {
if item.path == cfg.data_dir
|| item.path == cfg.backup_dir
|| item.path == cfg.browser_smoke_dir
|| item.path == cfg.worktime_report_cache_dir
{
bail!("refusing to delete root directory {}", item.path.display());
}
@@ -486,8 +544,12 @@ fn print_summary(summary: &Summary, json: bool) -> Result<()> {
}
fn cutoff(days: u64) -> SystemTime {
cutoff_seconds(days.saturating_mul(86_400))
}
fn cutoff_seconds(seconds: u64) -> SystemTime {
SystemTime::now()
.checked_sub(Duration::from_secs(days.saturating_mul(86_400)))
.checked_sub(Duration::from_secs(seconds))
.unwrap_or(SystemTime::UNIX_EPOCH)
}
@@ -598,12 +660,14 @@ mod tests {
data_dir: tmp.path().to_path_buf(),
backup_dir: tmp.path().join("backups"),
browser_smoke_dir: tmp.path().join("browser-smoke"),
worktime_report_cache_dir: tmp.path().join("worktime-report-cache"),
tmp_dir: tmp.path().join("tmp"),
backup_retention_days: 1,
backup_keep_last_db: 1,
backup_keep_last_json: 1,
browser_smoke_keep_runs: 1,
browser_smoke_retention_days: 1,
worktime_report_cache_retention_seconds: 1,
tmp_archive_retention_days: 1,
tmp_webui_retention_days: 1,
apply: false,
@@ -618,4 +682,43 @@ mod tests {
};
assert!(validate_plan(&cfg, &[item]).is_err());
}
#[test]
fn worktime_report_cache_prunes_only_json_files_inside_cache_dir() {
let tmp = tempfile::tempdir().expect("tempdir");
let cache_dir = tmp.path().join("worktime-report-cache");
fs::create_dir_all(&cache_dir).expect("cache dir");
fs::write(cache_dir.join("old-cache.json"), b"{}").expect("cache json");
fs::write(cache_dir.join("keep.txt"), b"keep").expect("non-json");
fs::create_dir_all(cache_dir.join("nested")).expect("nested dir");
fs::write(cache_dir.join("nested").join("nested-cache.json"), b"{}")
.expect("nested cache json");
let cfg = Config {
data_dir: tmp.path().to_path_buf(),
backup_dir: tmp.path().join("backups"),
browser_smoke_dir: tmp.path().join("browser-smoke"),
worktime_report_cache_dir: cache_dir.clone(),
tmp_dir: tmp.path().join("tmp"),
backup_retention_days: 1,
backup_keep_last_db: 1,
backup_keep_last_json: 1,
browser_smoke_keep_runs: 1,
browser_smoke_retention_days: 1,
worktime_report_cache_retention_seconds: 1,
tmp_archive_retention_days: 1,
tmp_webui_retention_days: 1,
apply: false,
json: false,
};
std::thread::sleep(Duration::from_secs(2));
let mut items = Vec::new();
plan_worktime_report_cache(&cfg, &mut items).expect("plan cache");
assert_eq!(items.len(), 1);
assert_eq!(items[0].reason, "worktime_report_disk_cache");
assert_eq!(items[0].path, cache_dir.join("old-cache.json"));
validate_plan(&cfg, &items).expect("valid cache plan");
}
}
+78 -8
View File
@@ -6,7 +6,7 @@ use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Duration as ChronoDuration, SecondsFormat, Utc};
use clap::Parser;
use detmir_core::{exit_codes, parse_utc_rfc3339};
@@ -25,10 +25,10 @@ struct Cli {
#[arg(long, default_value = "http://127.0.0.1:5610")]
worktime_api: String,
#[arg(long, default_value = "198.51.100.18")]
#[arg(long, default_value = "")]
rdp_host: String,
#[arg(long, default_value = "HOST-EXAMPLE")]
#[arg(long, default_value = "")]
rdp_hostname: String,
#[arg(long, default_value = "/var/lib/activitywatch/health")]
@@ -61,6 +61,9 @@ struct Cli {
#[arg(long, default_value_t = 3.0)]
tcp_timeout_seconds: f64,
#[arg(long, default_value_t = true)]
rdp_tcp_required: bool,
#[arg(long)]
json: bool,
}
@@ -129,6 +132,10 @@ impl Cli {
self.tcp_timeout_seconds,
);
}
if !cli_arg_present("--rdp-tcp-required") {
self.rdp_tcp_required =
env_bool_default("AW_RUS_HEALTH_RDP_TCP_REQUIRED", self.rdp_tcp_required);
}
self
}
}
@@ -221,9 +228,42 @@ fn env_f64(name: &str, fallback: f64) -> f64 {
}
fn env_bool(name: &str) -> bool {
env_bool_default(name, false)
}
fn env_bool_default(name: &str, fallback: bool) -> bool {
env_string(name)
.map(|value| matches!(value.to_ascii_lowercase().as_str(), "1" | "true" | "yes"))
.unwrap_or(false)
.map(|value| match value.to_ascii_lowercase().as_str() {
"1" | "true" | "yes" | "on" => true,
"0" | "false" | "no" | "off" => false,
_ => fallback,
})
.unwrap_or(fallback)
}
fn validate_cli_config(cli: &Cli) -> Result<()> {
validate_prod_host("rdp_host", &cli.rdp_host)?;
validate_prod_host("rdp_hostname", &cli.rdp_hostname)?;
Ok(())
}
fn validate_prod_host(name: &str, value: &str) -> Result<()> {
let value = value.trim();
if value.is_empty() {
return Err(anyhow!("invalid config {name}: value is empty"));
}
let lowered = value.to_ascii_lowercase();
if lowered == "host-example"
|| lowered.ends_with(".example")
|| lowered.starts_with("192.0.2.")
|| lowered.starts_with("198.51.100.")
|| lowered.starts_with("203.0.113.")
{
return Err(anyhow!(
"invalid config {name}: placeholder/documentation host is not allowed"
));
}
Ok(())
}
fn load_env_file(path: &Path) {
@@ -681,6 +721,16 @@ fn normalize_aw_api_base(aw_server: &str) -> String {
}
}
fn tcp_check_status(ok: bool, required: bool) -> &'static str {
if ok {
"ok"
} else if required {
"fail"
} else {
"warn"
}
}
fn validation_check(report: &mut ReportBuilder, validation_dir: &Path, max_age_seconds: i64) {
let Some(path) = latest_validation_report(validation_dir) else {
report.add(
@@ -812,15 +862,18 @@ fn run(cli: &Cli) -> Result<HealthReport> {
for (port, label) in [(5985_u16, "winrm"), (3389_u16, "rdp")] {
let (ok, message) = tcp_connect(&cli.rdp_host, port, cli.tcp_timeout_seconds);
let status = tcp_check_status(ok, cli.rdp_tcp_required);
report.add(
format!("tcp:{label}"),
if ok { "ok" } else { "fail" },
status,
if ok {
message
} else {
} else if cli.rdp_tcp_required {
format!("unreachable: {message}")
} else {
format!("optional unreachable: {message}")
},
json!({"host": cli.rdp_host, "port": port}),
json!({"host": cli.rdp_host, "port": port, "required": cli.rdp_tcp_required}),
);
}
@@ -949,6 +1002,7 @@ fn run(cli: &Cli) -> Result<HealthReport> {
fn main() -> Result<()> {
let cli = Cli::parse().apply_env();
validate_cli_config(&cli)?;
let report = run(&cli)?;
let json_text = serde_json::to_string_pretty(&report)? + "\n";
let text = render_text(&report) + "\n";
@@ -1029,4 +1083,20 @@ mod tests {
"http://127.0.0.1:5600/api/0"
);
}
#[test]
fn optional_rdp_tcp_downgrades_unreachable_to_warn() {
assert_eq!(tcp_check_status(false, true), "fail");
assert_eq!(tcp_check_status(false, false), "warn");
assert_eq!(tcp_check_status(true, false), "ok");
}
#[test]
fn healthd_rejects_placeholder_hosts() {
assert!(validate_prod_host("rdp_host", "192.168.100.19").is_ok());
assert!(validate_prod_host("rdp_hostname", "SHARKON2025").is_ok());
assert!(validate_prod_host("rdp_host", "198.51.100.18").is_err());
assert!(validate_prod_host("rdp_hostname", "HOST-EXAMPLE").is_err());
assert!(validate_prod_host("rdp_host", "").is_err());
}
}
+411 -30
View File
@@ -29,7 +29,6 @@ const DEFAULT_DLP_STATE: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-sync-st
const DEFAULT_DLP_TOKEN: &str = r"C:\ProgramData\AWatch-rus\dlp-evidence-upload-token.txt";
const DEFAULT_REMOTE_ROOT: &str = "/opt/activitywatch/clickhouse-1c/landing";
const DEFAULT_SSH_KEY: &str = r"C:\ProgramData\AWatch-rus\ssh\awops_ed25519";
const DEFAULT_REGISTRY_WORKBOOK: &str = r"E:\USER1\СПИСОК ПРЕДПРИЯТИЙ И ИХ РАСПРЕДЕЛЕНИЕ.xlsx";
#[derive(Parser)]
#[command(about = "AWatch-rus Windows telemetry uploader without PowerShell runtime wrappers")]
@@ -459,8 +458,7 @@ fn run_file1c_upload_inner(args: &mut File1cUpload, log_path: &Path) -> Result<(
if args.registry_workbook_path.is_none() {
args.registry_workbook_path = json_string(automation, &["registryWorkbookPath"])
.filter(|v| !v.trim().is_empty())
.map(PathBuf::from)
.or_else(|| Some(PathBuf::from(DEFAULT_REGISTRY_WORKBOOK)));
.map(PathBuf::from);
}
let scp = system32_path("OpenSSH\\scp.exe");
@@ -1614,6 +1612,10 @@ struct ForegroundWindowContext {
window_handle: isize,
}
fn has_foreground_context(context: &ForegroundWindowContext) -> bool {
context.process_id != 0 || !context.app.trim().is_empty() || !context.title.trim().is_empty()
}
#[derive(Debug, Clone)]
struct WebCategoryRule {
name: String,
@@ -1741,17 +1743,19 @@ fn run_browser_domains_collector(args: BrowserDomainsCollector) -> Result<()> {
loop {
let context = foreground_window_context();
let mut loop_failed = false;
match send_browser_window_event(&runtime, &context) {
Ok(()) => events_sent = events_sent.saturating_add(1),
Err(err) => {
loop_failed = true;
record_collector_send_failure(
&runtime,
&mut problems,
&mut send_failures,
"browser window heartbeat",
&err,
);
if has_foreground_context(&context) {
match send_browser_window_event(&runtime, &context) {
Ok(()) => events_sent = events_sent.saturating_add(1),
Err(err) => {
loop_failed = true;
record_collector_send_failure(
&runtime,
&mut problems,
&mut send_failures,
"browser window heartbeat",
&err,
);
}
}
}
match send_browser_category_health(&runtime, &context, events_sent) {
@@ -3981,9 +3985,47 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
let server_port = json_i64(&config, &["server", "port"]).unwrap_or(5600);
let api_base = format!("{server_scheme}://{server_host}:{server_port}/api/0");
let mut process_snapshot = collect_process_snapshot();
let session_snapshot = collect_session_snapshot();
let live_session_ids = live_session_ids(&session_snapshot.sessions);
let mut problems = Vec::new();
let mut actions = Vec::new();
let non_live_stop_plan =
non_live_session_collectors(&process_snapshot.processes, &live_session_ids);
if !non_live_stop_plan.is_empty() {
if args.mode == "enforce" {
for process in &non_live_stop_plan {
let ok = process.pid.is_some_and(terminate_process);
actions.push(json!({
"action": "stop-non-live-session-collector",
"kind": session_scoped_collector_kind(process).unwrap_or("unknown"),
"sessionId": process.session_id,
"pid": process.pid,
"applied": true,
"ok": ok
}));
if !ok {
problems.push(format!(
"failed to stop collector pid {:?} in non-live session {:?}",
process.pid, process.session_id
));
}
}
process_snapshot = collect_process_snapshot();
} else {
for process in &non_live_stop_plan {
actions.push(json!({
"action": "stop-non-live-session-collector",
"kind": session_scoped_collector_kind(process).unwrap_or("unknown"),
"sessionId": process.session_id,
"pid": process.pid,
"applied": false,
"mode": "shadow"
}));
}
}
}
let duplicate_plan = duplicate_legacy_collectors(&process_snapshot.processes);
if !duplicate_plan.is_empty() {
if args.mode == "enforce" {
@@ -4033,6 +4075,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
.unwrap_or_else(|| "powershell_primary".to_string());
let worktime_legacy_fallback_enabled =
json_bool(&config, &["collectors", "worktimeLegacyFallbackEnabled"]).unwrap_or(true);
let window_enabled = json_bool(&config, &["collectors", "windowEnabled"]).unwrap_or(true);
let file_ops_enabled = json_bool(&config, &["collectors", "fileOpsEnabled"]).unwrap_or(true);
let file_ops_mode = json_string(&config, &["collectors", "fileOpsMode"])
.unwrap_or_else(|| "powershell_primary".to_string());
@@ -4043,7 +4086,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
args.interactive_max_age_seconds,
15,
);
let bucket_checks = vec![
let mut bucket_checks = vec![
worktime_bucket.clone(),
get_bucket_health(
&api_base,
@@ -4051,19 +4094,21 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
args.interactive_max_age_seconds,
15,
),
get_bucket_health(
];
if window_enabled {
bucket_checks.push(get_bucket_health(
&api_base,
&format!("aw-watcher-window_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
),
get_bucket_health(
&api_base,
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
),
];
));
}
bucket_checks.push(get_bucket_health(
&api_base,
&format!("aw-dlp-endpoint-signals_{aw_hostname}"),
args.interactive_max_age_seconds,
15,
));
let interactive_stale = bucket_checks
.iter()
.skip(1)
@@ -4081,7 +4126,7 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
}
let task_defs = guard_task_definitions(&config);
let missing_fileops_sessions =
let mut missing_fileops_sessions =
if file_ops_enabled && file_ops_mode.eq_ignore_ascii_case("rust_primary") {
missing_rust_collector_sessions(
&process_snapshot.processes,
@@ -4091,10 +4136,13 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
} else {
Vec::new()
};
let launch_needed = interactive_stale || !missing_fileops_sessions.is_empty();
missing_fileops_sessions.retain(|session_id| live_session_ids.contains(session_id));
let has_live_sessions = !live_session_ids.is_empty();
let effective_interactive_stale = interactive_stale && has_live_sessions;
let launch_needed = effective_interactive_stale || !missing_fileops_sessions.is_empty();
if launch_needed {
let active_legacy_collectors = active_legacy_collector_count(&process_snapshot.processes);
if interactive_stale
if effective_interactive_stale
&& missing_fileops_sessions.is_empty()
&& active_legacy_collectors > 0
&& process_snapshot.command_line_query_ok
@@ -4123,6 +4171,16 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
problems.push(format!("refuse non-allowlisted task {}", task.task_name));
continue;
}
if !task_has_live_session(&task.user_id, &session_snapshot.sessions) {
actions.push(json!({
"action": "run-task",
"target": task.task_name,
"applied": false,
"reason": "no-live-session-for-user",
"userId": task.user_id
}));
continue;
}
let key = format!("task:{}", task.task_name);
let allowed = runtime.action_allowed(
&key,
@@ -4161,7 +4219,8 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
}
}
let status = if problems.is_empty() && (args.mode == "enforce" || !interactive_stale) {
let status = if problems.is_empty() && (args.mode == "enforce" || !effective_interactive_stale)
{
"ok"
} else {
"warn"
@@ -4180,12 +4239,25 @@ fn run_collector_guard_cycle(args: &CollectorGuard, runtime: &mut GuardRuntime)
"rustWorktimeAgentRunning": rust_agent_running,
"powerShellRuntimeByKind": power_shell_by_kind
},
"sessions": {
"queryOk": session_snapshot.query_ok,
"source": session_snapshot.source,
"error": session_snapshot.error,
"liveSessionIds": live_session_ids.iter().copied().collect::<Vec<_>>(),
"records": session_snapshot.sessions.iter().map(|session| json!({
"sessionId": session.session_id,
"userName": session.user_name,
"state": session.state,
"isLive": session.is_live
})).collect::<Vec<_>>()
},
"buckets": bucket_checks,
"tasks": task_defs.iter().map(|task| json!({
"taskName": task.task_name,
"userId": task.user_id
})).collect::<Vec<_>>(),
"interactiveStale": interactive_stale,
"effectiveInteractiveStale": effective_interactive_stale,
"fileOperationsPresence": {
"enabled": file_ops_enabled,
"mode": file_ops_mode,
@@ -4278,6 +4350,51 @@ fn guard_task_definitions(config: &Value) -> Vec<GuardTaskDefinition> {
out
}
fn live_session_ids(sessions: &[SessionInfo]) -> HashSet<u32> {
sessions
.iter()
.filter(|session| session.is_live)
.map(|session| session.session_id)
.collect()
}
fn task_has_live_session(user_id: &str, sessions: &[SessionInfo]) -> bool {
let candidates = user_candidates(user_id);
sessions.iter().any(|session| {
session.is_live
&& session
.user_name
.as_deref()
.is_some_and(|user| user_matches_candidates(user, &candidates))
})
}
fn user_candidates(user_id: &str) -> HashSet<String> {
let normalized = user_id.trim().to_ascii_lowercase();
let mut out = HashSet::new();
if normalized.is_empty() {
return out;
}
out.insert(normalized.clone());
if let Some((_, leaf)) = normalized.rsplit_once('\\') {
out.insert(leaf.to_string());
}
out
}
fn user_matches_candidates(user_name: &str, candidates: &HashSet<String>) -> bool {
let normalized = user_name.trim().to_ascii_lowercase();
if normalized.is_empty() {
return false;
}
if candidates.contains(&normalized) {
return true;
}
normalized
.rsplit_once('\\')
.is_some_and(|(_, leaf)| candidates.contains(leaf))
}
fn run_scheduled_task(task_name: &str) -> bool {
if !task_name.starts_with("ActivityWatch Launch ") {
return false;
@@ -4289,6 +4406,54 @@ fn run_scheduled_task(task_name: &str) -> bool {
.unwrap_or(false)
}
fn non_live_session_collectors<'a>(
processes: &'a [ProcessInfo],
live_session_ids: &HashSet<u32>,
) -> Vec<&'a ProcessInfo> {
processes
.iter()
.filter(|process| {
let Some(session_id) = process.session_id else {
return false;
};
session_id > 0
&& !live_session_ids.contains(&session_id)
&& session_scoped_collector_kind(process).is_some()
&& process.pid.is_some()
})
.collect()
}
fn session_scoped_collector_kind(process: &ProcessInfo) -> Option<&'static str> {
if let Some(kind) = legacy_collector_kind(process) {
return Some(kind);
}
let name = process.name.as_deref().unwrap_or_default();
if name.eq_ignore_ascii_case("aw-watcher-afk.exe") {
return Some("afk");
}
if name.eq_ignore_ascii_case("aw-watcher-window.exe") {
return Some("window");
}
if name.eq_ignore_ascii_case("aw-windows-telemetry.exe") {
let command_line = process
.command_line
.as_deref()
.unwrap_or_default()
.to_ascii_lowercase();
if command_line.contains("browser-domains-collector") {
return Some("browser");
}
if command_line.contains("dlp-endpoint-collector") {
return Some("dlp_endpoint");
}
if command_line.contains("file-operations-collector") {
return Some("fileops");
}
}
None
}
#[derive(Debug, Clone, Serialize)]
struct LegacyCollectorDuplicate {
kind: &'static str,
@@ -5007,6 +5172,22 @@ struct ProcessInfo {
command_line: Option<String>,
}
#[derive(Debug, Default)]
struct SessionSnapshot {
query_ok: bool,
source: String,
error: Option<String>,
sessions: Vec<SessionInfo>,
}
#[derive(Debug, Clone)]
struct SessionInfo {
session_id: u32,
user_name: Option<String>,
state: String,
is_live: bool,
}
fn validate_files(paths: &[PathBuf]) -> Value {
let mut list = Vec::new();
let mut missing = Vec::new();
@@ -5198,6 +5379,118 @@ fn collect_process_snapshot() -> ProcessSnapshot {
}
}
fn collect_session_snapshot() -> SessionSnapshot {
if let Some(raw) = command_output_utf16le("cmd", &["/U", "/C", "query user"])
.or_else(|| command_output_utf16le("cmd", &["/U", "/C", "quser"]))
{
let sessions = parse_query_user_sessions(&raw);
if !sessions.is_empty() {
return SessionSnapshot {
query_ok: true,
source: "quser_utf16".to_string(),
error: None,
sessions,
};
}
}
if let Some(raw) = command_output_lossy_combined("cmd", &["/C", "query user"])
.or_else(|| command_output_lossy_combined("cmd", &["/C", "quser"]))
{
let sessions = parse_query_user_sessions(&raw);
if !sessions.is_empty() {
return SessionSnapshot {
query_ok: true,
source: "quser_lossy".to_string(),
error: None,
sessions,
};
}
}
SessionSnapshot {
query_ok: false,
source: "unavailable".to_string(),
error: Some("query user and quser returned no sessions".to_string()),
sessions: Vec::new(),
}
}
fn command_output_utf16le(program: &str, args: &[&str]) -> Option<String> {
let output = Command::new(program).args(args).output().ok()?;
if !output.status.success() {
return None;
}
let mut bytes = output.stdout;
bytes.extend_from_slice(&output.stderr);
if bytes.is_empty() {
return None;
}
let words = bytes
.chunks_exact(2)
.map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
.collect::<Vec<_>>();
String::from_utf16(&words)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn command_output_lossy_combined(program: &str, args: &[&str]) -> Option<String> {
let output = Command::new(program).args(args).output().ok()?;
if !output.status.success() {
return None;
}
let mut bytes = output.stdout;
bytes.extend_from_slice(&output.stderr);
Some(String::from_utf8_lossy(&bytes).trim().to_string()).filter(|value| !value.is_empty())
}
fn parse_query_user_sessions(raw: &str) -> Vec<SessionInfo> {
raw.lines()
.filter_map(parse_query_user_line)
.collect::<Vec<_>>()
}
fn parse_query_user_line(line: &str) -> Option<SessionInfo> {
let cleaned = line.trim().trim_start_matches('>').trim();
if cleaned.is_empty()
|| cleaned.to_ascii_lowercase().starts_with("username")
|| cleaned.starts_with("ПОЛЬЗОВАТЕЛЬ")
{
return None;
}
let parts = cleaned.split_whitespace().collect::<Vec<_>>();
if parts.len() < 3 {
return None;
}
let username = parts.first()?.trim();
let (session_id, state, has_session_name) =
if parts.get(1)?.chars().all(|ch| ch.is_ascii_digit()) {
(*parts.get(1)?, *parts.get(2)?, false)
} else {
(*parts.get(2)?, *parts.get(3).unwrap_or(&"Unknown"), true)
};
let session_id = session_id.parse::<u32>().ok()?;
Some(SessionInfo {
session_id,
user_name: (!username.is_empty()).then(|| username.to_string()),
state: state.to_string(),
is_live: session_state_is_live(state)
|| (has_session_name && !session_state_is_disconnected(state)),
})
}
fn session_state_is_live(state: &str) -> bool {
let lower = state.to_lowercase();
lower.contains("active") || lower.contains("conn") || lower.contains("актив")
}
fn session_state_is_disconnected(state: &str) -> bool {
let lower = state.to_lowercase();
lower.contains("disc") || lower.contains("диск")
}
#[cfg(windows)]
fn collect_native_process_snapshot() -> Option<ProcessSnapshot> {
use std::mem::{MaybeUninit, size_of};
@@ -5977,7 +6270,7 @@ Connect=File="E:\Bases\Org\Base1";
ID=skip
Connect=Srvr="srv";Ref="x";
"#;
let items = parse_v8i_text(text, "user1", Path::new("ibases.v8i"));
let items = parse_v8i_text(text, "fixture-user", Path::new("ibases.v8i"));
assert_eq!(items.len(), 1);
assert_eq!(items[0].infobase, "База 1");
assert_eq!(items[0].base_id.as_deref(), Some("abc-123"));
@@ -6064,6 +6357,19 @@ Connect=Srvr="srv";Ref="x";
assert!(normalize_browser_url("new tab").is_none());
}
#[test]
fn foreground_context_requires_real_window_signal() {
assert!(!has_foreground_context(&ForegroundWindowContext::default()));
assert!(has_foreground_context(&ForegroundWindowContext {
process_id: 1000,
..ForegroundWindowContext::default()
}));
assert!(has_foreground_context(&ForegroundWindowContext {
title: "1C".to_string(),
..ForegroundWindowContext::default()
}));
}
#[test]
fn dlp_block_is_suppressed_without_native_enforce() {
let policy = dlp_policy_from_value(
@@ -6183,7 +6489,7 @@ SERVICE_NAME: AWatchRusCollectorGuard
#[test]
fn file_operations_queue_token_is_filename_safe() {
let token = queue_name_token(r"DOMAIN\Администратор", 3);
let token = queue_name_token(r"DOMAIN\operator", 3);
assert!(token.ends_with("-s3"));
assert!(!token.contains('\\'));
assert!(
@@ -6351,6 +6657,81 @@ SERVICE_NAME: AWatchRusCollectorGuard
);
}
#[test]
fn collector_guard_parses_live_and_disconnected_sessions() {
let sessions = parse_query_user_sessions(
r#"
USERNAME SESSIONNAME ID STATE IDLE TIME LOGON TIME
user1 rdp-tcp#5 5 Active none 09.07.2026 10:00
user2 2 Disc 3:14 07.07.2026 8:34
"#,
);
assert_eq!(sessions.len(), 2);
assert_eq!(live_session_ids(&sessions), HashSet::from([5]));
assert!(task_has_live_session(r"SHARKON2025\user1", &sessions));
assert!(!task_has_live_session(r"SHARKON2025\user2", &sessions));
}
#[test]
fn collector_guard_treats_named_rdp_session_as_live_when_state_is_localized() {
let sessions = parse_query_user_sessions(
r#"
USERNAME SESSIONNAME ID STATE IDLE TIME LOGON TIME
user1 rdp-tcp#12 12 ????? none 09.07.2026 10:00
user2 2 ????? 3:14 07.07.2026 8:34
"#,
);
assert_eq!(sessions.len(), 2);
assert_eq!(live_session_ids(&sessions), HashSet::from([12]));
assert!(task_has_live_session(r"SHARKON2025\user1", &sessions));
assert!(!task_has_live_session(r"SHARKON2025\user2", &sessions));
}
#[test]
fn collector_guard_stops_session_collectors_outside_live_sessions() {
let processes = vec![
ProcessInfo {
name: Some("aw-watcher-afk.exe".to_string()),
pid: Some(100),
session_id: Some(2),
created_unix_seconds: Some(10),
command_line: Some("aw-watcher-afk.exe --host 10.10.10.13".to_string()),
},
ProcessInfo {
name: Some("aw-windows-telemetry.exe".to_string()),
pid: Some(101),
session_id: Some(2),
created_unix_seconds: Some(11),
command_line: Some(
"aw-windows-telemetry.exe browser-domains-collector --mode enforce".to_string(),
),
},
ProcessInfo {
name: Some("aw-watcher-window.exe".to_string()),
pid: Some(200),
session_id: Some(5),
created_unix_seconds: Some(20),
command_line: Some("aw-watcher-window.exe --host 10.10.10.13".to_string()),
},
ProcessInfo {
name: Some("awatch-agent-rs.exe".to_string()),
pid: Some(300),
session_id: Some(0),
created_unix_seconds: Some(30),
command_line: Some("awatch-agent-rs.exe --config x".to_string()),
},
];
let stop_plan = non_live_session_collectors(&processes, &HashSet::from([5]));
let pids = stop_plan
.iter()
.filter_map(|process| process.pid)
.collect::<Vec<_>>();
assert_eq!(pids, vec![100, 101]);
}
#[test]
fn validate_deployment_parses_wmic_process_csv() {
let csv = br#"Node,CommandLine,Name,ProcessId,SessionId
@@ -0,0 +1,18 @@
[package]
name = "aw-workforce-ingest"
version = "0.1.0"
edition.workspace = true
rust-version.workspace = true
license.workspace = true
publish.workspace = true
[dependencies]
anyhow.workspace = true
chrono.workspace = true
clap.workspace = true
detmir-aw-client.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
urlencoding.workspace = true
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,10 @@
fn main() {
let code = match aw_workforce_ingest::run_from_args() {
Ok(()) => 0,
Err(err) => {
eprintln!("{err:#}");
1
}
};
std::process::exit(code);
}
-3
View File
@@ -12,6 +12,3 @@ chrono.workspace = true
clap.workspace = true
reqwest.workspace = true
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true
-3
View File
@@ -12,6 +12,3 @@ chrono.workspace = true
clap.workspace = true
reqwest.workspace = true
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true
+78 -7
View File
@@ -7,9 +7,9 @@ use clap::Parser;
use reqwest::blocking::Client;
use serde_json::Value;
const DEFAULT_SERVER: &str = "http://192.0.2.13:5600";
const DEFAULT_HOST: &str = "HOST-EXAMPLE";
const DEFAULT_RDP_HOST: &str = "198.51.100.18";
const DEFAULT_SERVER: &str = "http://10.10.10.13:5600";
const DEFAULT_HOST: &str = "SHARKON2025";
const DEFAULT_RDP_HOST: &str = "192.168.100.19";
const BUCKETS: &[(&str, &str)] = &[
("aw-watcher-afk", "AFK watcher"),
("aw-watcher-window", "Window watcher"),
@@ -38,6 +38,9 @@ struct Cli {
#[arg(long)]
no_color: bool,
#[arg(long, default_value_t = true)]
dlp_enabled: bool,
}
#[derive(Debug, Clone)]
@@ -119,7 +122,37 @@ fn main() {
}
fn run() -> Result<i32> {
let cli = Cli::parse();
let mut cli = Cli::parse();
if cli.server == DEFAULT_SERVER {
if let Some(value) = env_nonempty("CHECK_AW_FULL_SERVER")
.or_else(|| env_nonempty("AW_SMOKE_AW_SERVER"))
.or_else(|| env_nonempty("AW_SERVER"))
{
cli.server = value;
}
}
if cli.host == DEFAULT_HOST {
if let Some(value) = env_nonempty("CHECK_AW_FULL_HOST")
.or_else(|| env_nonempty("AW_SMOKE_SOURCE_HOSTNAME"))
.or_else(|| env_nonempty("AW_LOGICAL_HOST_ID"))
.or_else(|| env_nonempty("AW_MONITORED_WINDOWS_HOSTNAME"))
{
cli.host = value;
}
}
if cli.rdp_host == DEFAULT_RDP_HOST {
if let Some(value) = env_nonempty("CHECK_AW_FULL_RDP_HOST")
.or_else(|| env_nonempty("AW_SMOKE_WINDOWS_HOST"))
.or_else(|| env_nonempty("AW_WINDOWS_HOST"))
{
cli.rdp_host = value;
}
}
if let Some(value) =
env_nonempty("AW_DLP_ENABLED").or_else(|| env_nonempty("DETMIR_DLP_ENABLED"))
{
cli.dlp_enabled = parse_env_flag(&value);
}
let server = cli.server.trim_end_matches('/').to_string();
let colors = Colors::new(!cli.no_color && std::env::var_os("NO_COLOR").is_none());
let timeout = Duration::from_secs(cli.timeout_seconds.max(1));
@@ -182,7 +215,11 @@ fn run() -> Result<i32> {
);
let mut rows = Vec::new();
for (bucket, label) in BUCKETS {
for (bucket, label) in BUCKETS
.iter()
.copied()
.filter(|(bucket, _)| cli.dlp_enabled || !bucket.starts_with("aw-dlp-"))
{
let row = read_bucket_row(&client, &server, &cli.host, bucket, label, now, &context);
println!(
" {:<42} {:<8} {:<20} {}",
@@ -193,6 +230,15 @@ fn run() -> Result<i32> {
);
rows.push(row);
}
if !cli.dlp_enabled {
println!(
" {:<42} {:<8} {:<20} {}",
"DLP buckets",
"-",
"disabled",
colors.paint(colors.cyan, "SKIPPED")
);
}
println!();
println!(
@@ -330,7 +376,12 @@ fn read_bucket_row(
status: BucketStatus::Unknown,
};
};
let age_sec = (now - ts).num_seconds().max(0);
let effective_ts = if bucket == "aw-watcher-afk" {
bucket_metadata_end(client, server, &bucket_full).unwrap_or(ts)
} else {
ts
};
let age_sec = (now - effective_ts).num_seconds().max(0);
BucketRow {
label,
last_id,
@@ -407,6 +458,15 @@ fn latest_event(client: &Client, server: &str, bucket: &str) -> Result<Option<Va
Ok(value.as_array().and_then(|items| items.first()).cloned())
}
fn bucket_metadata_end(client: &Client, server: &str, bucket: &str) -> Option<DateTime<Utc>> {
let url = format!("{server}/api/0/buckets/{bucket}");
let value = get_json(client, &url).ok()?;
value
.pointer("/metadata/end")
.and_then(Value::as_str)
.and_then(parse_ts)
}
fn get_json(client: &Client, url: &str) -> Result<Value> {
client
.get(url)
@@ -423,7 +483,7 @@ fn check_cors(client: &Client, server: &str) -> u16 {
let url = format!("{server}/api/0/settings/");
client
.get(&url)
.header("Origin", "http://192.0.2.13:5600")
.header("Origin", server)
.send()
.map(|response| response.status().as_u16())
.unwrap_or(0)
@@ -466,6 +526,17 @@ fn json_value_to_string(value: &Value) -> String {
}
}
fn env_nonempty(name: &str) -> Option<String> {
std::env::var(name).ok().filter(|value| !value.is_empty())
}
fn parse_env_flag(value: &str) -> bool {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
}
#[cfg(test)]
mod tests {
use super::*;
+245 -5
View File
@@ -1,6 +1,7 @@
use std::io::Read;
use std::net::{SocketAddr, TcpStream};
use std::process::Command;
use std::time::Duration;
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use chrono::{DateTime, SecondsFormat, Utc};
@@ -8,7 +9,7 @@ use clap::Parser;
use detmir_aw_client::ActivityWatchClient;
use detmir_core::{exit_codes, now_utc_rfc3339};
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue};
use serde::Serialize;
use serde_json::Value;
@@ -40,6 +41,12 @@ struct Cli {
#[arg(long, default_value = DEFAULT_HOSTNAME)]
hostname: String,
#[arg(long, default_value = DEFAULT_GATEWAY_HOST)]
gateway_host: String,
#[arg(long, default_value = "https://127.0.0.1")]
portal_url: String,
#[arg(long, default_value_t = 5)]
service_timeout_seconds: u64,
@@ -75,6 +82,18 @@ struct Cli {
#[arg(long, default_value = "")]
clickhouse_password: String,
#[arg(long, default_value = "detmir-dlp")]
dlp_command: String,
#[arg(long, default_value_t = 45)]
dlp_timeout_seconds: u64,
#[arg(long, default_value_t = false)]
disable_dlp_health_check: bool,
#[arg(long, default_value_t = false)]
disable_portal_check: bool,
}
#[derive(Debug, Clone, Copy)]
@@ -160,6 +179,20 @@ fn env_or_default(name: &str, default: &str) -> String {
.unwrap_or_else(|| default.to_string())
}
fn env_flag_enabled(name: &str) -> bool {
std::env::var(name)
.ok()
.map(|value| parse_env_flag(&value))
.unwrap_or(false)
}
fn parse_env_flag(value: &str) -> bool {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
}
fn bucket_specs(hostname: &str) -> Vec<BucketSpec> {
vec![
BucketSpec {
@@ -224,6 +257,20 @@ fn build_headers(items: &[(&str, &str)]) -> Result<HeaderMap> {
Ok(headers)
}
fn portal_headers(args: &Cli) -> HeaderMap {
let mut headers = build_headers(&[("Host", args.gateway_host.as_str())]).unwrap_or_default();
if let Some(value) = std::env::var("DETMIR_PORTAL_AUTH_HEADER")
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
{
if let Ok(value) = HeaderValue::from_str(&value) {
headers.insert(AUTHORIZATION, value);
}
}
headers
}
fn fetch_text(
url: &str,
timeout: Duration,
@@ -258,6 +305,7 @@ fn fetch_text(
fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
let timeout = Duration::from_secs(args.service_timeout_seconds);
let one_c_url = args.one_c_url.trim_end_matches('/');
let portal_url = args.portal_url.trim_end_matches('/');
let services = [
(
"aw-info",
@@ -285,10 +333,10 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
),
(
"gateway-healthz",
"https://127.0.0.1/healthz".to_string(),
format!("{portal_url}/healthz"),
true,
true,
build_headers(&[("Host", DEFAULT_GATEWAY_HOST)]).unwrap_or_default(),
portal_headers(args),
),
];
@@ -333,12 +381,185 @@ fn service_checks(args: &Cli) -> Vec<ServiceCheck> {
if !args.disable_grafana_check {
checks.push(grafana_data_check(args));
}
if !args.disable_portal_check {
checks.extend(portal_checks(args));
}
if security_events_clickhouse_enabled(args) {
checks.push(clickhouse_security_events_check(args));
}
if !args.disable_dlp_health_check {
checks.push(dlp_health_check(args));
}
checks
}
fn portal_checks(args: &Cli) -> Vec<ServiceCheck> {
let base = args.portal_url.trim_end_matches('/');
[
("portal-healthz", "/healthz", true),
("portal-readyz", "/readyz", true),
("portal-version", "/version", true),
("portal-metrics", "/metrics", true),
]
.into_iter()
.map(|(name, path, required)| {
let url = format!("{base}{path}");
let headers = portal_headers(args);
match fetch_text(
&url,
Duration::from_secs(args.service_timeout_seconds),
true,
headers,
2,
) {
Ok(raw) => {
let payload = serde_json::from_str::<Value>(&raw).unwrap_or_else(|_| {
Value::String(raw.lines().next().unwrap_or("").to_string())
});
ServiceCheck {
name: name.to_string(),
required,
ok: true,
url: Some(url),
payload: Some(payload),
error: None,
}
}
Err(err) => ServiceCheck {
name: name.to_string(),
required,
ok: false,
url: Some(url),
payload: None,
error: Some(err.to_string()),
},
}
})
.collect()
}
fn dlp_health_check(args: &Cli) -> ServiceCheck {
let name = "aw-dlp-health".to_string();
match run_shell_command_timeout(
&args.dlp_command,
Duration::from_secs(args.dlp_timeout_seconds),
) {
Ok(output) if output.timed_out => ServiceCheck {
name,
required: true,
ok: false,
url: None,
payload: None,
error: Some(format!(
"DLP health command timed out after {} seconds",
args.dlp_timeout_seconds
)),
},
Ok(output) => {
let payload = serde_json::from_str::<Value>(&output.stdout).unwrap_or_else(|_| {
Value::String(output.stdout.lines().next().unwrap_or("").to_string())
});
let ok = output.code == Some(0);
ServiceCheck {
name,
required: true,
ok,
url: None,
payload: Some(payload),
error: if ok {
None
} else {
Some(format!(
"DLP health command exited with {:?}: {}",
output.code,
sanitize_command_stderr(&output.stderr)
))
},
}
}
Err(err) => ServiceCheck {
name,
required: true,
ok: false,
url: None,
payload: None,
error: Some(format!("cannot execute DLP health command: {err:#}")),
},
}
}
#[derive(Debug)]
struct CommandOutput {
code: Option<i32>,
stdout: String,
stderr: String,
timed_out: bool,
}
fn run_shell_command_timeout(command: &str, timeout: Duration) -> Result<CommandOutput> {
let mut child = Command::new("/bin/sh")
.arg("-lc")
.arg(command)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.with_context(|| format!("failed to spawn command: {command}"))?;
let started = Instant::now();
loop {
if let Some(status) = child.try_wait().context("command wait failed")? {
return read_command_output(child, status.code(), false);
}
if started.elapsed() >= timeout {
let _ = child.kill();
let _ = child.wait();
return read_command_output(child, None, true);
}
std::thread::sleep(Duration::from_millis(100));
}
}
fn read_command_output(
mut child: std::process::Child,
code: Option<i32>,
timed_out: bool,
) -> Result<CommandOutput> {
let mut stdout = String::new();
let mut stderr = String::new();
if let Some(mut pipe) = child.stdout.take() {
pipe.read_to_string(&mut stdout)
.context("failed to read command stdout")?;
}
if let Some(mut pipe) = child.stderr.take() {
pipe.read_to_string(&mut stderr)
.context("failed to read command stderr")?;
}
Ok(CommandOutput {
code,
stdout,
stderr,
timed_out,
})
}
fn sanitize_command_stderr(stderr: &str) -> String {
let trimmed = stderr.trim();
if trimmed.is_empty() {
return "no stderr".to_string();
}
trimmed
.lines()
.take(3)
.map(|line| {
if line.chars().count() > 240 {
format!("{}...", line.chars().take(240).collect::<String>())
} else {
line.to_string()
}
})
.collect::<Vec<_>>()
.join(" | ")
}
fn security_events_clickhouse_enabled(args: &Cli) -> bool {
args.security_events_backend
.trim()
@@ -777,6 +998,8 @@ fn main() -> Result<()> {
args.one_c_url = env_or_default("DETMIR_ONE_C_URL", &args.one_c_url);
args.rdp_host = env_or_default("DETMIR_RDP_HOST", &args.rdp_host);
args.hostname = env_or_default("DETMIR_HOSTNAME", &args.hostname);
args.gateway_host = env_or_default("DETMIR_GATEWAY_HOST", &args.gateway_host);
args.portal_url = env_or_default("DETMIR_PORTAL_URL", &args.portal_url);
args.grafana_check_json = env_or_default("DETMIR_GRAFANA_CHECK_JSON", &args.grafana_check_json);
args.security_events_backend =
env_or_default("SECURITY_EVENTS_BACKEND", &args.security_events_backend);
@@ -784,6 +1007,13 @@ fn main() -> Result<()> {
args.clickhouse_database = env_or_default("CLICKHOUSE_DATABASE", &args.clickhouse_database);
args.clickhouse_user = env_or_default("CLICKHOUSE_USER", &args.clickhouse_user);
args.clickhouse_password = env_or_default("CLICKHOUSE_PASSWORD", &args.clickhouse_password);
args.dlp_command = env_or_default("DETMIR_DLP_COMMAND", &args.dlp_command);
if env_flag_enabled("DETMIR_DISABLE_DLP_HEALTH_CHECK") {
args.disable_dlp_health_check = true;
}
if env_flag_enabled("DETMIR_DISABLE_PORTAL_CHECK") {
args.disable_portal_check = true;
}
let report = build_report(&args)?;
if args.json {
@@ -855,6 +1085,16 @@ mod tests {
);
}
#[test]
fn env_flag_accepts_true_values_only() {
assert!(parse_env_flag("true"));
assert!(parse_env_flag("1"));
assert!(parse_env_flag("yes"));
assert!(parse_env_flag("on"));
assert!(!parse_env_flag("0"));
assert!(!parse_env_flag("false"));
}
#[test]
fn clickhouse_database_identifier_rejects_injection() {
assert_eq!(
+1
View File
@@ -18,6 +18,7 @@ serde_json.workspace = true
serde_yaml.workspace = true
sha2.workspace = true
tiny_http.workspace = true
url.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -0,0 +1,57 @@
//! Portal API contract summary payload.
//!
//! CONTRACT: this module describes stable public API routes exposed by the
//! current Rust HTML/HTMX portal and future clients. Keep changes additive
//! unless the OpenAPI/TypeScript contracts are updated in the same PR.
use serde_json::{Value, json};
pub(crate) fn api_contract_summary() -> Value {
json!({
"ok": true,
"contract_version": "2026-06-06.pilot-v1",
"generated_by": "detmir-portal",
"api_base": "/api",
"compatibility": {
"policy": "additive",
"main_ui": "rust-server-rendered-html-htmx-compatible",
"unknown_fields": "clients must ignore unknown fields",
"nullable_fields": "clients must tolerate null and missing optional fields",
"forbidden_ui_stacks": ["dioxus", "react", "tauri", "electron"]
},
"targets": ["rust-html", "htmx-compatible"],
"artifacts": {
"openapi": "/api/contracts/openapi.json",
"typescript": "/api/contracts/typescript.d.ts"
},
"stable_endpoints": [
{"method": "GET", "path": "/healthz", "purpose": "process liveness without external dependency checks"},
{"method": "GET", "path": "/readyz", "purpose": "local readiness and contract-only dependency status"},
{"method": "GET", "path": "/version", "purpose": "safe build and schema version metadata"},
{"method": "GET", "path": "/metrics", "purpose": "Prometheus metrics without high-cardinality labels"},
{"method": "GET", "path": "/api/health", "purpose": "light service health"},
{"method": "GET", "path": "/api/contracts", "purpose": "contract index"},
{"method": "GET", "path": "/api/contracts/openapi.json", "purpose": "OpenAPI contract"},
{"method": "GET", "path": "/api/contracts/typescript.d.ts", "purpose": "TypeScript declarations"},
{"method": "GET", "path": "/api/operator", "purpose": "portal overview data"},
{"method": "GET", "path": "/api/reports", "purpose": "management report payload"},
{"method": "GET", "path": "/api/executive", "purpose": "executive role payload"},
{"method": "GET", "path": "/api/workforce", "purpose": "workforce role payload"},
{"method": "GET", "path": "/api/security", "purpose": "security role payload"},
{"method": "GET", "path": "/api/forensics", "purpose": "forensics role payload"},
{"method": "GET", "path": "/api/ueba", "purpose": "rule-based UEBA score v1"},
{"method": "GET", "path": "/api/pfsense", "purpose": "pfSense readiness contracts and demo fixtures"},
{"method": "GET", "path": "/api/incidents", "purpose": "incident and DLP evidence summary"},
{"method": "GET", "path": "/api/cases", "purpose": "case list"},
{"method": "POST", "path": "/api/incident-review", "purpose": "manual candidate review status"},
{"method": "POST", "path": "/api/cases", "purpose": "manual case creation"},
{"method": "GET", "path": "/api/investigation-pack/{candidate_id}", "purpose": "candidate investigation pack"},
{"method": "GET", "path": "/api/dlp/evidence", "purpose": "DLP evidence list"},
{"method": "GET", "path": "/api/readiness/latest", "purpose": "latest readiness status"},
{"method": "GET", "path": "/api/workforce/policy/explain", "purpose": "workforce policy explanation"},
{"method": "GET", "path": "/api/workforce/kpi/explain", "purpose": "rule-based Workforce KPI explanation"},
{"method": "GET", "path": "/api/risk/narrative", "purpose": "rule-based executive risk narrative"},
{"method": "GET", "path": "/api/actions", "purpose": "rule-based executive action center"}
]
})
}
@@ -0,0 +1,26 @@
//! External command execution helpers for the portal.
//!
//! CONTRACT: these helpers are intentionally small and side-effect explicit.
//! They preserve stdout/stderr error text because readiness verification APIs
//! expose command failure diagnostics to operators.
use std::path::Path;
use std::process::Command;
pub(crate) fn run_in_dir(dir: &Path, command: &mut Command) -> std::result::Result<(), String> {
let output = command
.current_dir(dir)
.output()
.map_err(|err| format!("run command in {}: {err}", dir.display()))?;
if output.status.success() {
Ok(())
} else {
Err(format!(
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
)
.trim()
.to_string())
}
}
@@ -0,0 +1,127 @@
//! HTTP response helpers for the portal.
//!
//! CONTRACT: this module owns response serialization, headers, request-id /
//! correlation-id propagation and response metrics logging. It must not change
//! routes, payload schemas, MIME types or UI contents.
use std::fs;
use std::path::Path;
use anyhow::{Context, Result, anyhow};
use serde::Serialize;
use tiny_http::{Header, Request, Response, StatusCode};
use crate::production::{http_request_metadata, log_http_request, record_http_metric};
use crate::screenshot_basename;
pub(crate) fn respond_json<T: Serialize>(request: Request, value: &T) -> Result<()> {
let body = serde_json::to_string_pretty(value)?;
respond_text(
request,
StatusCode(200),
&body,
"application/json; charset=utf-8",
)
}
pub(crate) fn respond_json_status<T: Serialize>(
request: Request,
status: StatusCode,
value: &T,
) -> Result<()> {
let body = serde_json::to_string_pretty(value)?;
respond_text(request, status, &body, "application/json; charset=utf-8")
}
pub(crate) fn respond_text(
request: Request,
status: StatusCode,
body: &str,
content_type: &str,
) -> Result<()> {
let metadata = http_request_metadata(&request);
record_http_metric(&metadata, status);
log_http_request(&metadata, status, body.len());
let response = Response::from_string(body.to_string())
.with_status_code(status)
.with_header(header("Content-Type", content_type)?)
.with_header(header("Cache-Control", "no-store")?)
.with_header(header("X-Request-Id", &metadata.request_id)?)
.with_header(header("X-Correlation-Id", &metadata.correlation_id)?);
request.respond(response).map_err(|err| anyhow!("{err}"))
}
pub(crate) fn respond_text_download(
request: Request,
status: StatusCode,
body: &str,
content_type: &str,
download_name: &str,
) -> Result<()> {
let metadata = http_request_metadata(&request);
record_http_metric(&metadata, status);
log_http_request(&metadata, status, body.len());
let response = Response::from_string(body.to_string())
.with_status_code(status)
.with_header(header("Content-Type", content_type)?)
.with_header(header("Cache-Control", "no-store")?)
.with_header(header("X-Request-Id", &metadata.request_id)?)
.with_header(header("X-Correlation-Id", &metadata.correlation_id)?)
.with_header(header(
"Content-Disposition",
&format!(
"attachment; filename=\"{}\"",
download_name.replace('"', "")
),
)?);
request.respond(response).map_err(|err| anyhow!("{err}"))
}
pub(crate) fn respond_file(
request: Request,
path: &Path,
content_type: &str,
download_name: Option<&str>,
) -> Result<()> {
let data = fs::read(path).with_context(|| format!("read {}", path.display()))?;
let metadata = http_request_metadata(&request);
record_http_metric(&metadata, StatusCode(200));
log_http_request(&metadata, StatusCode(200), data.len());
let mut response = Response::from_data(data)
.with_status_code(StatusCode(200))
.with_header(header("Content-Type", content_type)?)
.with_header(header("Cache-Control", "no-store")?)
.with_header(header("X-Request-Id", &metadata.request_id)?)
.with_header(header("X-Correlation-Id", &metadata.correlation_id)?);
if let Some(name) = download_name.and_then(screenshot_basename) {
response = response.with_header(header(
"Content-Disposition",
&format!("attachment; filename=\"{}\"", name.replace('"', "")),
)?);
}
request.respond(response).map_err(|err| anyhow!("{err}"))
}
pub(crate) fn safe_download_stem(value: &str) -> String {
let stem = value
.chars()
.map(|ch| {
if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') {
ch
} else {
'_'
}
})
.take(96)
.collect::<String>();
if stem.is_empty() {
"candidate".to_string()
} else {
stem
}
}
fn header(name: &str, value: &str) -> Result<Header> {
Header::from_bytes(name.as_bytes(), value.as_bytes())
.map_err(|_| anyhow!("invalid header {name}: {value}"))
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,54 @@
//! URL path and query parsing helpers for the portal.
//!
//! CONTRACT: these helpers are routing glue. Keep accepted URL shapes stable
//! because API handlers and the HTML portal depend on them.
pub(crate) fn normalize_path(url: &str) -> String {
let path = url.split('?').next().unwrap_or("/");
let path = path.strip_prefix("/portal").unwrap_or(path);
if path.is_empty() {
"/".to_string()
} else {
path.to_string()
}
}
pub(crate) fn query_flag(url: &str, key: &str) -> bool {
let Some(query) = url.split_once('?').map(|(_, query)| query) else {
return false;
};
query.split('&').any(|pair| {
let (name, value) = pair.split_once('=').unwrap_or((pair, "1"));
name == key && matches!(value, "1" | "true" | "yes" | "on")
})
}
pub(crate) fn query_param(url: &str, key: &str) -> Option<String> {
let query = url.split_once('?').map(|(_, query)| query)?;
query.split('&').find_map(|pair| {
let (name, value) = pair.split_once('=').unwrap_or((pair, ""));
(name == key && !value.is_empty()).then(|| value.to_string())
})
}
pub(crate) fn parse_investigation_pack_path(path: &str) -> Option<String> {
path.strip_prefix("/api/investigation-pack/")
.map(str::trim)
.filter(|value| !value.is_empty() && !value.contains('/'))
.map(ToString::to_string)
}
pub(crate) fn parse_case_path(path: &str) -> Option<String> {
path.strip_prefix("/api/cases/")
.map(str::trim)
.filter(|value| !value.is_empty() && !value.contains('/'))
.map(ToString::to_string)
}
pub(crate) fn parse_case_status_path(path: &str) -> Option<String> {
path.strip_prefix("/api/cases/")
.and_then(|value| value.strip_suffix("/status"))
.map(str::trim)
.filter(|value| !value.is_empty() && !value.contains('/'))
.map(ToString::to_string)
}
@@ -0,0 +1,77 @@
//! Portal role model and access-scope contract.
//!
//! CONTRACT: role aliases, serialized values and allowed scopes are part of
//! the portal API/security boundary. Keep changes explicit and covered by
//! existing role-gate tests in `main.rs`.
use serde::Serialize;
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PortalRole {
Executive,
Manager,
Security,
Forensics,
Admin,
}
impl PortalRole {
pub(crate) fn parse(value: &str) -> Option<Self> {
match value.trim().to_ascii_lowercase().as_str() {
"executive" | "owner" | "rukovoditel" | "руководитель" => {
Some(Self::Executive)
}
"manager" | "workforce" | "руководитель_подразделения" => {
Some(Self::Manager)
}
"security" | "ib" | "soc" | "безопасность" => Some(Self::Security),
"forensics" | "investigation" | "расследования" => Some(Self::Forensics),
"admin" | "operations" | "operator" | "эксплуатация" => Some(Self::Admin),
_ => None,
}
}
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Executive => "executive",
Self::Manager => "manager",
Self::Security => "security",
Self::Forensics => "forensics",
Self::Admin => "admin",
}
}
pub(crate) fn label_ru(self) -> &'static str {
match self {
Self::Executive => "Руководитель",
Self::Manager => "Руководитель подразделения",
Self::Security => "Безопасность",
Self::Forensics => "Расследования",
Self::Admin => "Администратор",
}
}
pub(crate) fn allowed_scopes(self) -> &'static [&'static str] {
match self {
Self::Executive => &["executive", "workforce"],
Self::Manager => &["executive", "workforce"],
Self::Security => &["security", "incidents", "ueba", "pfsense"],
Self::Forensics => &["forensics", "incidents", "ueba"],
Self::Admin => &[
"executive",
"workforce",
"security",
"forensics",
"incidents",
"ueba",
"pfsense",
"admin",
],
}
}
pub(crate) fn can_access(self, scope: &str) -> bool {
self.allowed_scopes().contains(&scope)
}
}
@@ -11,6 +11,7 @@ use anyhow::{Result, anyhow};
use chrono::NaiveDate;
use serde_json::{Value, json};
use tiny_http::StatusCode;
use url::Url;
use crate::{
Cli, MAX_ALLOWED_PAGE_SIZE, MAX_ALLOWED_REPORT_DATE_RANGE_DAYS, MAX_ALLOWED_REQUEST_BODY_BYTES,
@@ -77,6 +78,11 @@ pub(crate) fn validate_portal_config(args: &Cli) -> Result<()> {
"invalid config max_request_body_bytes: expected 1024..={MAX_ALLOWED_REQUEST_BODY_BYTES}"
));
}
validate_runtime_url("worktime_url", &args.worktime_url)?;
validate_runtime_url("one_c_url", &args.one_c_url)?;
validate_probe_command("status_cmd", &args.status_cmd)?;
validate_probe_command("check_cmd", &args.check_cmd)?;
validate_probe_command("failed_units_cmd", &args.failed_units_cmd)?;
// SECURITY: environment and module names can reach metrics/log labels.
// Restrict them to short ASCII tokens to avoid label injection and runaway
@@ -112,6 +118,46 @@ pub(crate) fn validate_portal_config(args: &Cli) -> Result<()> {
Ok(())
}
fn validate_runtime_url(name: &str, value: &str) -> Result<()> {
let url = Url::parse(value).map_err(|err| anyhow!("invalid config {name}: {err}"))?;
if !matches!(url.scheme(), "http" | "https") {
return Err(anyhow!("invalid config {name}: expected http or https URL"));
}
let Some(host) = url.host_str() else {
return Err(anyhow!("invalid config {name}: missing host"));
};
if is_placeholder_host(host) {
return Err(anyhow!(
"invalid config {name}: placeholder/documentation host is not allowed in production"
));
}
Ok(())
}
fn is_placeholder_host(host: &str) -> bool {
let host = host.trim().to_ascii_lowercase();
host.is_empty()
|| host == "host-example"
|| host.ends_with(".example")
|| host.starts_with("192.0.2.")
|| host.starts_with("198.51.100.")
|| host.starts_with("203.0.113.")
}
fn validate_probe_command(name: &str, command: &str) -> Result<()> {
let command = command.trim();
if command.is_empty() {
return Err(anyhow!("invalid config {name}: command is empty"));
}
let forbidden = ['\n', '\r', '\0', ';', '|', '&', '<', '>', '`'];
if command.contains("$(") || command.chars().any(|ch| forbidden.contains(&ch)) {
return Err(anyhow!(
"invalid config {name}: shell control operators are not allowed"
));
}
Ok(())
}
fn is_safe_environment_name(value: &str) -> bool {
let value = value.trim();
!value.is_empty()
@@ -230,6 +276,7 @@ mod tests {
slow_request_log_ms: DEFAULT_SLOW_REQUEST_LOG_MS,
environment: "test".to_string(),
enabled_modules: "executive,workforce,security,forensics,admin".to_string(),
dlp_module_enabled: true,
state_dir: dir.join("state"),
dlp_db_path: dir.join("dlp.sqlite"),
evidence_root: dir.to_path_buf(),
@@ -239,7 +286,7 @@ mod tests {
json_smoke: false,
evidence_only: false,
evidence_upload_token: None,
telemetry_api_key: "test-key".to_string(),
telemetry_api_key: "dummy".to_string(),
telemetry_store_path: dir.join("telemetry.jsonl"),
expected_nodes_path: dir.join("expected_nodes.json"),
security_events_backend: "disabled".to_string(),
@@ -293,6 +340,39 @@ mod tests {
);
}
#[test]
fn config_validation_rejects_placeholder_endpoints_and_shell_operators() {
let dir = tempfile::tempdir().unwrap();
let args = test_cli(dir.path());
let mut invalid = args.clone();
invalid.worktime_url = "http://192.0.2.13:5610".to_string();
assert!(
validate_portal_config(&invalid)
.unwrap_err()
.to_string()
.contains("placeholder")
);
let mut invalid = args.clone();
invalid.one_c_url = "http://198.51.100.2:8710".to_string();
assert!(
validate_portal_config(&invalid)
.unwrap_err()
.to_string()
.contains("placeholder")
);
let mut invalid = args.clone();
invalid.check_cmd = "detmir-check --json; curl http://127.0.0.1".to_string();
assert!(
validate_portal_config(&invalid)
.unwrap_err()
.to_string()
.contains("shell control")
);
}
#[test]
fn query_limits_reject_page_size_and_report_range() {
let dir = tempfile::tempdir().unwrap();
@@ -34,6 +34,10 @@ struct HttpMetricValue {
#[derive(Clone, Debug, Default)]
struct PortalMetrics {
http: BTreeMap<HttpMetricKey, HttpMetricValue>,
report_requests_total: u64,
report_cache_hits_total: u64,
report_cache_misses_total: u64,
report_cache_stale_hits_total: u64,
reports_generated_total: u64,
ingestion_records_total: u64,
ingestion_rejected_total: u64,
@@ -70,6 +74,32 @@ pub(crate) fn record_report_generated() {
}
}
pub(crate) fn record_report_request() {
if let Ok(mut metrics) = portal_metrics().lock() {
metrics.report_requests_total = metrics.report_requests_total.saturating_add(1);
}
}
pub(crate) fn record_report_cache_hit() {
if let Ok(mut metrics) = portal_metrics().lock() {
metrics.report_cache_hits_total = metrics.report_cache_hits_total.saturating_add(1);
}
}
pub(crate) fn record_report_cache_stale_hit() {
if let Ok(mut metrics) = portal_metrics().lock() {
metrics.report_cache_hits_total = metrics.report_cache_hits_total.saturating_add(1);
metrics.report_cache_stale_hits_total =
metrics.report_cache_stale_hits_total.saturating_add(1);
}
}
pub(crate) fn record_report_cache_miss() {
if let Ok(mut metrics) = portal_metrics().lock() {
metrics.report_cache_misses_total = metrics.report_cache_misses_total.saturating_add(1);
}
}
pub(crate) fn record_ingestion_accepted() {
if let Ok(mut metrics) = portal_metrics().lock() {
metrics.ingestion_records_total = metrics.ingestion_records_total.saturating_add(1);
@@ -149,6 +179,26 @@ pub(crate) fn render_prometheus_metrics(args: &Cli) -> String {
.ok();
}
for (name, help, value) in [
(
"awatch_report_requests_total",
"Report payload requests handled by the portal cache layer",
metrics.report_requests_total,
),
(
"awatch_report_cache_hits_total",
"Report payload requests served from the in-process cache",
metrics.report_cache_hits_total,
),
(
"awatch_report_cache_misses_total",
"Report payload requests that triggered report regeneration",
metrics.report_cache_misses_total,
),
(
"awatch_report_cache_stale_hits_total",
"Report payload requests served from stale cache while refresh runs",
metrics.report_cache_stale_hits_total,
),
(
"awatch_reports_generated_total",
"Reports generated by the portal",
@@ -22,7 +22,8 @@ pub(crate) use limits::{is_limited_api_route, validate_api_query_limits, validat
pub(crate) use logging::log_http_request;
pub(crate) use metrics::{
record_http_metric, record_ingestion_accepted, record_ingestion_rejected,
record_report_generated, render_prometheus_metrics,
record_report_cache_hit, record_report_cache_miss, record_report_cache_stale_hit,
record_report_generated, record_report_request, render_prometheus_metrics,
};
pub(crate) use readiness::build_readyz;
pub(crate) use request_context::{http_request_metadata, mark_request_started};
@@ -0,0 +1,112 @@
//! Readiness API payload helpers for the portal.
//!
//! CONTRACT: these helpers expose existing readiness bundle/status/verify
//! payloads. Keep file names, JSON fields and verification commands stable
//! unless the customer readiness contract is updated in the same PR.
use std::fs;
use std::path::Path;
use std::process::Command;
use anyhow::{Context, Result};
use serde_json::{Value, json};
use crate::command_runner::run_in_dir;
use crate::{Cli, now};
pub(crate) fn readiness_latest(args: &Cli) -> Value {
read_json_file(
&args
.readiness_bundle_dir
.join("detmir-readiness-latest.json"),
)
.unwrap_or_else(|err| {
json!({
"ok": false,
"generated_at_utc": now(),
"error": err.to_string(),
})
})
}
pub(crate) fn readiness_bundle(args: &Cli) -> Value {
let dir = &args.readiness_bundle_dir;
let status = read_json_file(&dir.join("detmir-readiness-status.json")).unwrap_or_else(|err| {
json!({
"ok": false,
"error": err.to_string(),
})
});
let latest_dir = fs::read_to_string(dir.join("latest-dir.txt"))
.unwrap_or_default()
.trim()
.to_string();
let artifacts = [
"detmir-readiness-latest.json",
"detmir-readiness-act.md",
"detmir-readiness-act.html",
"sha256sums.txt",
"sha256sums.txt.sig",
"public-key.pem",
"detmir-readiness-status.json",
"detmir-readiness.prom",
]
.into_iter()
.filter_map(|name| {
let path = dir.join(name);
path.metadata().ok().map(|meta| {
json!({
"name": name,
"bytes": meta.len(),
"available": true,
})
})
})
.collect::<Vec<_>>();
json!({
"ok": status.get("ok").and_then(Value::as_bool).unwrap_or(false),
"generated_at_utc": now(),
"bundle_dir": dir.display().to_string(),
"latest_archive_dir": latest_dir,
"status": status,
"artifacts": artifacts,
})
}
pub(crate) fn readiness_verify(args: &Cli) -> Value {
let dir = &args.readiness_bundle_dir;
let checksum = run_in_dir(
dir,
Command::new("sha256sum").arg("-c").arg("sha256sums.txt"),
);
let sig_path = dir.join("sha256sums.txt.sig");
let pub_path = dir.join("public-key.pem");
let signature = if sig_path.is_file() && pub_path.is_file() {
run_in_dir(
dir,
Command::new("openssl")
.arg("dgst")
.arg("-sha256")
.arg("-verify")
.arg("public-key.pem")
.arg("-signature")
.arg("sha256sums.txt.sig")
.arg("sha256sums.txt"),
)
} else {
Err("signature files are not available".to_string())
};
json!({
"ok": checksum.is_ok() && signature.is_ok(),
"generated_at_utc": now(),
"checksum_verified": checksum.is_ok(),
"signature_verified": signature.is_ok(),
"checksum_error": checksum.err(),
"signature_error": signature.err(),
})
}
fn read_json_file(path: &Path) -> Result<Value> {
let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
serde_json::from_str(&text).with_context(|| format!("parse {}", path.display()))
}
@@ -0,0 +1,52 @@
//! Portal role extraction and access-denial helpers.
//!
//! CONTRACT: role aliases, role envelope fields and forbidden response shape
//! are part of the portal security boundary. Keep changes explicit and covered
//! by role-gate tests.
use anyhow::Result;
use serde_json::{Value, json};
use tiny_http::{Request, StatusCode};
use crate::path_query::query_param;
use crate::portal_roles::PortalRole;
use crate::respond_json_status;
pub(crate) fn portal_role_from_request(request: &Request, url: &str) -> PortalRole {
query_param(url, "role")
.as_deref()
.and_then(PortalRole::parse)
.or_else(|| {
request
.headers()
.iter()
.find(|header| header.field.equiv("X-AWatch-Role"))
.and_then(|header| PortalRole::parse(header.value.as_str()))
})
.unwrap_or(PortalRole::Executive)
}
pub(crate) fn role_envelope(role: PortalRole, scope: &str) -> Value {
json!({
"role": role.as_str(),
"role_label": role.label_ru(),
"scope": scope,
"allowed_scopes": role.allowed_scopes(),
"server_enforced": true,
})
}
pub(crate) fn respond_forbidden(request: Request, role: PortalRole, scope: &str) -> Result<()> {
respond_json_status(
request,
StatusCode(403),
&json!({
"ok": false,
"error": "forbidden",
"message": format!("Роль {} не имеет доступа к контуру {scope}", role.label_ru()),
"role": role.as_str(),
"scope": scope,
"server_enforced": true,
}),
)
}
@@ -0,0 +1,125 @@
//! Snapshot cache helpers for the portal request path.
//!
//! CONTRACT: this module only owns short-lived in-process cache behavior.
//! It must not change snapshot payloads, source collection, API routes or
//! business calculations.
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use crate::{Cli, HealthResponse, Snapshot, build_health, build_snapshot, now};
const SNAPSHOT_CACHE_TTL: Duration = Duration::from_secs(120);
pub(crate) type SnapshotCache = Arc<Mutex<SnapshotCacheState>>;
#[derive(Clone, Debug, Default)]
pub(crate) struct SnapshotCacheState {
pub(crate) entry: Option<CachedSnapshot>,
pub(crate) refresh_in_progress: bool,
}
#[derive(Clone, Debug)]
pub(crate) struct CachedSnapshot {
created: Instant,
snapshot: Snapshot,
}
pub(crate) fn new_snapshot_cache() -> SnapshotCache {
Arc::new(Mutex::new(SnapshotCacheState::default()))
}
pub(crate) fn clone_snapshot_cache(cache: &SnapshotCache) -> SnapshotCache {
Arc::clone(cache)
}
pub(crate) fn cached_snapshot(args: &Cli, cache: &SnapshotCache) -> Snapshot {
{
let guard = cache.lock().expect("snapshot cache mutex poisoned");
if let Some(cached) = guard.entry.as_ref() {
if cached.created.elapsed() <= SNAPSHOT_CACHE_TTL {
return cached.snapshot.clone();
}
}
}
let snapshot = build_snapshot(args);
let mut guard = cache.lock().expect("snapshot cache mutex poisoned");
guard.entry = Some(CachedSnapshot {
created: Instant::now(),
snapshot: snapshot.clone(),
});
guard.refresh_in_progress = false;
snapshot
}
pub(crate) fn cached_snapshot_or_refresh(args: &Cli, cache: &SnapshotCache) -> Option<Snapshot> {
let mut should_spawn = false;
let mut snapshot_to_return = None;
{
let mut guard = cache.lock().expect("snapshot cache mutex poisoned");
if let Some(cached) = guard.entry.as_ref() {
let snapshot = cached.snapshot.clone();
if cached.created.elapsed() <= SNAPSHOT_CACHE_TTL {
return Some(snapshot);
}
if !guard.refresh_in_progress {
guard.refresh_in_progress = true;
should_spawn = true;
}
snapshot_to_return = Some(snapshot);
} else if !guard.refresh_in_progress {
guard.refresh_in_progress = true;
should_spawn = true;
}
}
if should_spawn {
spawn_snapshot_refresh(args.clone(), clone_snapshot_cache(cache));
}
snapshot_to_return
}
fn spawn_snapshot_refresh(args: Cli, cache: SnapshotCache) {
thread::spawn(move || {
let result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| build_snapshot(&args)));
let mut guard = cache.lock().expect("snapshot cache mutex poisoned");
match result {
Ok(snapshot) => {
guard.entry = Some(CachedSnapshot {
created: Instant::now(),
snapshot,
});
}
Err(_) => {
eprintln!("detmir-portal snapshot cache refresh panicked");
}
}
guard.refresh_in_progress = false;
});
}
pub(crate) fn build_fast_health(cache: &SnapshotCache) -> HealthResponse {
match cache.try_lock() {
Ok(guard) => guard
.entry
.as_ref()
.map(|cached| build_health(&cached.snapshot))
.unwrap_or_else(lightweight_health),
Err(_) => lightweight_health(),
}
}
fn lightweight_health() -> HealthResponse {
let mut sources = BTreeMap::new();
sources.insert("portal".to_string(), true);
HealthResponse {
ok: true,
generated_at_utc: now(),
version: env!("CARGO_PKG_VERSION").to_string(),
sources,
}
}
@@ -0,0 +1,11 @@
//! Static portal assets and generated API contract text.
//!
//! CONTRACT: this module only exposes embedded static files. Do not change
//! file contents, MIME handling, routes or API contracts from here.
pub(crate) const INDEX_HTML: &str = include_str!("static/index.html");
pub(crate) const ARCHITECTURE_HTML: &str = include_str!("static/architecture.html");
pub(crate) const APP_CSS: &str = include_str!("static/app.css");
pub(crate) const APP_JS: &str = include_str!("static/app.js");
pub(crate) const API_CONTRACT_OPENAPI: &str = include_str!("contracts/openapi.json");
pub(crate) const API_CONTRACT_TYPESCRIPT: &str = include_str!("contracts/typescript.d.ts");
@@ -0,0 +1,185 @@
//! Telemetry ingest API for Rust endpoint/agent diagnostics.
//!
//! CONTRACT: this module owns `/api/telemetry` authentication, request-body
//! validation and JSONL append semantics. Keep accepted fields, status codes,
//! metrics counters and response shape stable unless telemetry contracts are
//! updated in the same PR.
use std::fs;
use std::fs::OpenOptions;
use std::io::Write;
use anyhow::{Context, Result, anyhow};
use serde_json::{Value, json};
use tiny_http::{Request, StatusCode};
use crate::production::{record_ingestion_accepted, record_ingestion_rejected};
use crate::{
Cli, is_payload_too_large, now, read_limited_body, respond_json, respond_json_status,
respond_payload_too_large,
};
pub(crate) fn telemetry_authorized(request: &Request, args: &Cli) -> bool {
let expected = args.telemetry_api_key.trim();
if expected.is_empty() || expected == "change-me" {
return false;
}
let actual = request
.headers()
.iter()
.find(|header| header.field.equiv("x-api-key"))
.map(|header| header.value.as_str().trim().to_string())
.or_else(|| bearer_token(request));
actual
.as_deref()
.filter(|value| !value.is_empty())
.map(|value| constant_time_eq(value.as_bytes(), expected.as_bytes()))
.unwrap_or(false)
}
pub(crate) fn bearer_token(request: &Request) -> Option<String> {
request
.headers()
.iter()
.find(|header| header.field.equiv("Authorization"))
.map(|header| header.value.as_str().trim())
.and_then(|value| value.strip_prefix("Bearer "))
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToString::to_string)
}
pub(crate) fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
if left.len() != right.len() {
return false;
}
let mut diff = 0_u8;
for (a, b) in left.iter().zip(right.iter()) {
diff |= a ^ b;
}
diff == 0
}
pub(crate) fn handle_telemetry_ingest(mut request: Request, args: &Cli) -> Result<()> {
if !telemetry_authorized(&request, args) {
record_ingestion_rejected();
return respond_json_status(
request,
StatusCode(401),
&json!({
"ok": false,
"error": "telemetry api key is missing or invalid"
}),
);
}
let telemetry_limit = args.max_request_body_bytes.min(1024 * 1024);
let body = match read_limited_body(&mut request, telemetry_limit) {
Ok(body) => body,
Err(err) if is_payload_too_large(&err) => {
record_ingestion_rejected();
return respond_payload_too_large(request);
}
Err(err) => return Err(err),
};
let response = apply_telemetry_ingest(args, &body);
match response {
Ok(response) => {
record_ingestion_accepted();
respond_json(request, &response)
}
Err(err) => {
record_ingestion_rejected();
respond_json_status(
request,
StatusCode(400),
&json!({
"ok": false,
"error": err.to_string()
}),
)
}
}
}
pub(crate) fn apply_telemetry_ingest(args: &Cli, body: &str) -> Result<Value> {
let payload: Value =
serde_json::from_str(body).map_err(|err| anyhow!("invalid telemetry JSON: {err}"))?;
validate_telemetry_payload(&payload)?;
let received_at_utc = now();
let envelope = json!({
"received_at_utc": received_at_utc,
"prototype": true,
"record": payload,
});
if let Some(parent) = args.telemetry_store_path.parent() {
fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(&args.telemetry_store_path)
.with_context(|| format!("open {}", args.telemetry_store_path.display()))?;
writeln!(file, "{}", serde_json::to_string(&envelope)?)
.with_context(|| format!("append {}", args.telemetry_store_path.display()))?;
Ok(json!({
"ok": true,
"prototype": true,
"stored": "file-backed-jsonl",
"received_at_utc": received_at_utc,
}))
}
pub(crate) fn validate_telemetry_payload(payload: &Value) -> Result<()> {
let Some(object) = payload.as_object() else {
return Err(anyhow!("telemetry payload must be a JSON object"));
};
for field in [
"agent_id",
"hostname",
"os_name",
"os_version",
"platform",
"username",
"timestamp",
"uptime_seconds",
"cpu_usage_percent",
"memory_total",
"memory_used",
"active_sessions",
"rdp_sessions",
"ssh_sessions",
"processes",
"network_interfaces",
"network_connections",
"workforce_activity",
"security_events",
"collector_version",
] {
if !object.contains_key(field) {
return Err(anyhow!("telemetry field is missing: {field}"));
}
}
for field in [
"active_sessions",
"rdp_sessions",
"ssh_sessions",
"processes",
"network_interfaces",
"network_connections",
"security_events",
] {
if payload.get(field).and_then(Value::as_array).is_none() {
return Err(anyhow!("telemetry field must be an array: {field}"));
}
}
if payload
.get("workforce_activity")
.and_then(Value::as_object)
.is_none()
{
return Err(anyhow!(
"telemetry field must be an object: workforce_activity"
));
}
Ok(())
}
@@ -549,6 +549,7 @@ mod tests {
};
Snapshot {
generated_at_utc: "2026-06-07T10:00:00Z".to_string(),
dlp_module_enabled: true,
detmir_status: SourceStatus {
ok: true,
status: "OK".to_string(),
+94 -6
View File
@@ -18,7 +18,9 @@ const DEFAULT_AW_ENV_FILE: &str = "/etc/activitywatch/aw-server.env";
const DEFAULT_GRAFANA_ENV_FILE: &str = "/etc/detmir-grafana-check.env";
const DEFAULT_GRAFANA_URL: &str = "http://127.0.0.1:3000";
const DEFAULT_GRAFANA_DATASOURCE_UID: &str = "influxdb_aw";
const DEFAULT_SYSTEMD_SERVICES: &str = "activitywatch-server,aw-worktime-api,aw-worktime-influx-exporter.timer,aw-dlp-influx-exporter.timer";
const DEFAULT_SYSTEMD_SERVICES: &str =
"activitywatch-server,aw-worktime-api,aw-worktime-influx-exporter.timer";
const DEFAULT_DLP_SYSTEMD_SERVICES: &str = "aw-dlp-influx-exporter.timer";
const DEFAULT_RETENTION_DAYS: i64 = 30;
#[derive(Debug, Parser)]
@@ -218,14 +220,28 @@ fn run(cli: &Cli) -> Result<Report> {
let mut checks = Vec::new();
let worktime = influx_config(&aw_env, "AW_WORKTIME_INFLUX");
let dlp = influx_config(&aw_env, "AW_DLP_INFLUX");
let dlp_enabled = env_bool(&aw_env, "AW_DLP_ENABLED", true);
let dlp_profile = env_value(&aw_env, "AW_DLP_PROFILE", "full");
let dlp_influx_required = dlp_influx_required_for_profile(dlp_enabled, &dlp_profile);
checks.push(check_influx_env(&worktime, cli.allow_disabled_influx));
checks.push(check_influx_env(&dlp, cli.allow_disabled_influx));
if dlp_influx_required {
checks.push(check_influx_env(&dlp, cli.allow_disabled_influx));
} else {
checks.push(ok(
"env:AW_DLP_INFLUX",
"DLP Influx runtime is not required by the current DLP profile",
json!({"enabled": dlp_enabled, "profile": dlp_profile.as_str()}),
));
}
if cli.skip_systemd {
checks.push(warn("systemd", "systemd checks skipped", json!({})));
} else {
checks.extend(check_systemd_services(&cli.systemd_services));
checks.extend(check_systemd_services(&systemd_services_for_mode(
&cli.systemd_services,
dlp_influx_required,
)));
}
if cli.skip_influx_write {
@@ -236,7 +252,15 @@ fn run(cli: &Cli) -> Result<Report> {
));
} else {
checks.push(check_influx_write(&client, "worktime", &worktime));
checks.push(check_influx_write(&client, "dlp", &dlp));
if dlp_influx_required {
checks.push(check_influx_write(&client, "dlp", &dlp));
} else {
checks.push(ok(
"influx:write:dlp",
"DLP write probe skipped because DLP Influx is not required by the current profile",
json!({"enabled": dlp_enabled, "profile": dlp_profile.as_str()}),
));
}
}
if cli.skip_grafana {
@@ -270,7 +294,7 @@ fn run(cli: &Cli) -> Result<Report> {
git_commit: cli.git_commit.clone(),
counts,
checks,
limitations: build_limitations(cli),
limitations: build_limitations(cli, dlp_enabled, dlp_influx_required),
})
}
@@ -338,6 +362,30 @@ fn split_csv(value: &str) -> Vec<String> {
.collect()
}
fn dlp_influx_required_for_profile(dlp_enabled: bool, profile: &str) -> bool {
if !dlp_enabled {
return false;
}
!matches!(
profile.trim().to_ascii_lowercase().as_str(),
"light" | "core_only" | "disabled" | "off" | "on_demand"
)
}
fn systemd_services_for_mode(csv: &str, dlp_influx_required: bool) -> String {
let mut services = split_csv(csv);
if dlp_influx_required {
for service in split_csv(DEFAULT_DLP_SYSTEMD_SERVICES) {
if !services.iter().any(|item| item == &service) {
services.push(service);
}
}
} else {
services.retain(|service| !service.contains("dlp"));
}
services.into_iter().collect::<Vec<_>>().join(",")
}
fn hostname() -> String {
Command::new("hostname")
.output()
@@ -348,7 +396,7 @@ fn hostname() -> String {
.unwrap_or_else(|| "unknown".to_string())
}
fn build_limitations(cli: &Cli) -> Vec<String> {
fn build_limitations(cli: &Cli, dlp_enabled: bool, dlp_influx_required: bool) -> Vec<String> {
let mut limitations = Vec::new();
limitations.push(
"Проверка подтверждает состояние runtime на момент формирования акта и не заменяет аудит конфигурации, нагрузочное тестирование или приемочные испытания заказчика.".to_string(),
@@ -377,6 +425,15 @@ fn build_limitations(cli: &Cli) -> Vec<String> {
.to_string(),
);
}
if !dlp_enabled {
limitations.push(
"DLP runtime отключен штатно через AW_DLP_ENABLED=false; readiness не считает DLP services/timers и DLP Influx write обязательными.".to_string(),
);
} else if !dlp_influx_required {
limitations.push(
"DLP runtime включен в лёгком профиле; readiness не считает DLP Influx timer и DLP Influx write обязательными.".to_string(),
);
}
limitations
}
@@ -1307,6 +1364,37 @@ mod tests {
assert_eq!(counts.fail, 1);
}
#[test]
fn dlp_influx_is_required_only_for_full_profile() {
for profile in ["full", "enabled", "on"] {
assert!(
dlp_influx_required_for_profile(true, profile),
"{profile} should require DLP Influx"
);
}
for profile in ["light", "core_only", "disabled", "off", "on_demand"] {
assert!(
!dlp_influx_required_for_profile(true, profile),
"{profile} should not require DLP Influx"
);
}
assert!(!dlp_influx_required_for_profile(false, "full"));
assert!(dlp_influx_required_for_profile(true, "unexpected"));
}
#[test]
fn light_profile_excludes_dlp_influx_timer() {
let services = systemd_services_for_mode(DEFAULT_SYSTEMD_SERVICES, false);
assert!(services.contains("activitywatch-server"));
assert!(services.contains("aw-worktime-influx-exporter.timer"));
assert!(!services.contains("aw-dlp-influx-exporter.timer"));
let services = systemd_services_for_mode(DEFAULT_SYSTEMD_SERVICES, true);
assert!(services.contains("aw-dlp-influx-exporter.timer"));
}
#[test]
fn renders_readiness_act_without_secrets() {
let checks = vec![ok(
-1
View File
@@ -19,5 +19,4 @@ adk-rust.workspace = true
anyhow.workspace = true
clap.workspace = true
detmir-state.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -10,6 +10,3 @@ publish.workspace = true
anyhow.workspace = true
chrono.workspace = true
clap.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -1045,7 +1045,7 @@ mod tests {
assert!(err.contains("AW_DLP_INFLUX_URL"));
config.influx_url = "http://influxdb.internal:8086".to_string();
config.influx_token = "prod-write-token-value".to_string();
config.influx_token = "dummy".to_string();
let err = validate_runtime_config(&config).unwrap_err().to_string();
assert!(err.contains("AW_DLP_INFLUX_HOSTS"));
+1 -1
View File
@@ -31,7 +31,7 @@ regex.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
urlencoding.workspace = true
zip.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -1,12 +1,16 @@
use std::fs::{self, File};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::thread;
use std::time::{Duration, Instant, SystemTime};
use anyhow::{Context, Result, bail};
use chrono::Utc;
use clap::Parser;
use fs2::FileExt;
use hayabusa_tools::{guess_host_from_filename, read_json_file};
use serde_json::{Value, json};
use zip::ZipArchive;
const LOCK_PATH: &str = "/opt/hayabusa/state/aw-hayabusa-autoprocess.lock";
const WRAPPER: &str = "/usr/local/bin/aw-hayabusa";
@@ -20,6 +24,9 @@ struct Cli {
#[arg(long, default_value = "/opt/activitywatch/aw-rus-ops/drop")]
drop_dir: PathBuf,
#[arg(long, default_value = "/opt/hayabusa/quarantine/drop")]
quarantine_dir: PathBuf,
#[arg(long, default_value_t = true)]
once: bool,
}
@@ -66,15 +73,29 @@ fn run() -> Result<i32> {
return Ok(0);
}
for zip_path in zips {
let result = process_one(&zip_path)?;
println!(
"{}",
serde_json::to_string_pretty(&json!({
"processed": zip_path.display().to_string(),
"latest_intake": result.latest_intake,
"case_alert": result.case_alert,
}))?
);
match process_one(&zip_path) {
Ok(result) => {
println!(
"{}",
serde_json::to_string_pretty(&json!({
"processed": zip_path.display().to_string(),
"latest_intake": result.latest_intake,
"case_alert": result.case_alert,
}))?
);
}
Err(err) => {
let quarantined = quarantine_drop_package(&cli.quarantine_dir, &zip_path, &err)?;
println!(
"{}",
serde_json::to_string_pretty(&json!({
"quarantined": zip_path.display().to_string(),
"quarantine_dir": quarantined.display().to_string(),
"reason": err.to_string(),
}))?
);
}
}
}
Ok(0)
}
@@ -97,6 +118,7 @@ fn list_zips(drop_dir: &Path) -> Result<Vec<PathBuf>> {
}
fn process_one(zip_path: &Path) -> Result<ProcessResult> {
wait_for_stable_zip(zip_path)?;
let sidecars = load_sidecars(zip_path)?;
let host = guess_host(zip_path, &sidecars);
let mode = if sidecars.mode.is_empty() {
@@ -174,6 +196,85 @@ fn process_one(zip_path: &Path) -> Result<ProcessResult> {
})
}
fn wait_for_stable_zip(zip_path: &Path) -> Result<()> {
wait_for_stable_zip_with(
zip_path,
Duration::from_secs(60),
Duration::from_secs(1),
Duration::from_secs(2),
2,
)
}
fn wait_for_stable_zip_with(
zip_path: &Path,
max_wait: Duration,
interval: Duration,
min_modified_age: Duration,
required_stable_checks: u32,
) -> Result<()> {
let started = Instant::now();
let mut last_len = None;
let mut stable_checks = 0;
let mut last_zip_error = None;
loop {
let metadata =
fs::metadata(zip_path).with_context(|| format!("stat {}", zip_path.display()))?;
let len = metadata.len();
let modified_age = metadata
.modified()
.ok()
.and_then(|modified| SystemTime::now().duration_since(modified).ok())
.unwrap_or_default();
if len > 0 && last_len == Some(len) && modified_age >= min_modified_age {
stable_checks += 1;
} else {
stable_checks = 0;
}
if stable_checks >= required_stable_checks {
match verify_zip_readable(zip_path) {
Ok(()) => return Ok(()),
Err(err) => {
last_zip_error = Some(err.to_string());
stable_checks = 0;
}
}
}
if started.elapsed() >= max_wait {
if let Some(err) = last_zip_error {
bail!(
"drop zip {} did not become readable: {err}",
zip_path.display()
);
}
bail!(
"drop zip {} did not stabilize within {}s",
zip_path.display(),
max_wait.as_secs()
);
}
last_len = Some(len);
thread::sleep(interval);
}
}
fn verify_zip_readable(zip_path: &Path) -> Result<()> {
let file = File::open(zip_path).with_context(|| format!("open {}", zip_path.display()))?;
let mut archive =
ZipArchive::new(file).with_context(|| format!("read zip {}", zip_path.display()))?;
for idx in 0..archive.len() {
let _entry = archive
.by_index(idx)
.with_context(|| format!("read zip entry {idx}"))?;
}
Ok(())
}
fn load_sidecars(zip_path: &Path) -> Result<Sidecars> {
let base = zip_path.with_extension("");
let caseid_path = base.with_extension("caseid");
@@ -214,6 +315,49 @@ fn load_sidecars(zip_path: &Path) -> Result<Sidecars> {
})
}
fn quarantine_drop_package(
quarantine_dir: &Path,
zip_path: &Path,
err: &anyhow::Error,
) -> Result<PathBuf> {
fs::create_dir_all(quarantine_dir)
.with_context(|| format!("create {}", quarantine_dir.display()))?;
let timestamp = Utc::now().format("%Y%m%dT%H%M%SZ");
let zip_name = zip_path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or("package.zip");
let target_dir = quarantine_dir.join(format!("{timestamp}_{zip_name}"));
fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?;
let base = zip_path.with_extension("");
for path in [
zip_path.to_path_buf(),
base.with_extension("caseid"),
base.with_extension("meta.json"),
zip_path.with_extension("zip.sha256"),
] {
if path.is_file() {
let target = target_dir.join(path.file_name().context("quarantine file name")?);
fs::rename(&path, &target)
.with_context(|| format!("move {} to {}", path.display(), target.display()))?;
}
}
let reason = json!({
"quarantined_at": Utc::now().to_rfc3339(),
"package": zip_path.display().to_string(),
"reason": err.to_string(),
"error_chain": format!("{err:#}"),
});
fs::write(
target_dir.join("reason.json"),
serde_json::to_vec_pretty(&reason).context("serialize quarantine reason")?,
)
.with_context(|| format!("write {}", target_dir.join("reason.json").display()))?;
Ok(target_dir)
}
fn archive_sidecars(report_dir: &Path, sidecars: &Sidecars) -> Result<()> {
let target_dir = report_dir.join("input-sidecars");
fs::create_dir_all(&target_dir).with_context(|| format!("create {}", target_dir.display()))?;
@@ -302,3 +446,54 @@ fn run_capture(program: &Path, args: &[String]) -> Result<Captured> {
stderr: String::from_utf8_lossy(&output.stderr).to_string(),
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use tempfile::tempdir;
use zip::write::SimpleFileOptions;
#[test]
fn wait_for_stable_zip_accepts_complete_archive() {
let dir = tempdir().expect("tempdir");
let zip_path = dir.path().join("HOST-20260709-000001.zip");
write_test_zip(&zip_path);
wait_for_stable_zip_with(
&zip_path,
Duration::from_secs(1),
Duration::from_millis(1),
Duration::from_secs(0),
1,
)
.expect("complete zip should be accepted");
}
#[test]
fn wait_for_stable_zip_rejects_unreadable_archive() {
let dir = tempdir().expect("tempdir");
let zip_path = dir.path().join("HOST-20260709-000001.zip");
fs::write(&zip_path, b"not a zip").expect("write partial zip");
let err = wait_for_stable_zip_with(
&zip_path,
Duration::from_millis(20),
Duration::from_millis(1),
Duration::from_secs(0),
1,
)
.expect_err("invalid zip should be rejected");
assert!(err.to_string().contains("did not become readable"));
}
fn write_test_zip(path: &Path) {
let file = File::create(path).expect("create zip");
let mut zip = zip::ZipWriter::new(file);
zip.start_file("manifest.json", SimpleFileOptions::default())
.expect("start manifest");
zip.write_all(br#"{"host":"SHARKON2025"}"#)
.expect("write manifest");
zip.finish().expect("finish zip");
}
}
+8
View File
@@ -285,6 +285,8 @@ fn is_allowed_python_runtime_path(rel: &str) -> bool {
|| rel.starts_with("pfsense/")
|| rel == "proxmox/tsj_guardian_bot.py"
|| rel == "proxmox/test_tsj_guardian_bot.py"
|| rel == "scripts/package_rust_release_binaries.py"
|| rel == "scripts/public_secret_pattern_check.py"
}
fn is_detmir_retired_runtime_path(rel: &str) -> bool {
@@ -344,6 +346,12 @@ mod tests {
assert!(is_allowed_python_runtime_path(
"pfsense/pfsense-aw-poller.py"
));
assert!(is_allowed_python_runtime_path(
"scripts/package_rust_release_binaries.py"
));
assert!(is_allowed_python_runtime_path(
"scripts/public_secret_pattern_check.py"
));
}
#[test]
@@ -13,6 +13,3 @@ clap.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true
-1
View File
@@ -15,7 +15,6 @@ serde.workspace = true
serde_json.workspace = true
tiny_http.workspace = true
url.workspace = true
urlencoding.workspace = true
regex.workspace = true
[dev-dependencies]
@@ -1031,7 +1031,7 @@ mod tests {
assert!(err.contains("AW_WORKTIME_INFLUX_URL"));
config.influx_url = "http://influxdb.internal:8086".to_string();
config.influx_token = "prod-write-token-value".to_string();
config.influx_token = "dummy".to_string();
let err = validate_runtime_config(&config).unwrap_err().to_string();
assert!(err.contains("AW_WORKTIME_INFLUX_HOSTS"));
+336 -58
View File
@@ -16,8 +16,25 @@
aw_worktime_classes: "{{ lookup('file', aw_repo_root + '/aw-server/settings/classes-worktime.json') | from_json }}"
aw_default_views: "{{ lookup('file', aw_repo_root + '/aw-server/settings/views-default.json') | from_json }}"
aw_rust_release_dir: "{{ (lookup('env', 'CARGO_TARGET_DIR') | default(aw_repo_root + '/adk-rust/target', true)) + '/release' }}"
aw_db_vacuum_timer_enabled: false
tasks:
- name: Refuse inconsistent DLP resource profile
ansible.builtin.assert:
that:
- aw_dlp_profile | default('core_only') in ['core_only', 'light', 'on_demand', 'full']
- (aw_dlp_profile | default('core_only') == 'core_only') or (aw_dlp_enabled | default(false) | bool)
- (aw_dlp_enabled | default(false) | bool) or not (
aw_dlp_influx_enabled | default(false) | bool
or aw_dlp_ioc_enabled | default(false) | bool
or aw_dlp_policy_engine_enabled | default(false) | bool
or aw_dlp_content_analysis_enabled | default(false) | bool
or aw_dlp_integrations_enabled | default(false) | bool
or aw_dlp_case_management_enabled | default(false) | bool
or aw_dlp_compliance_enabled | default(false) | bool
)
fail_msg: "Inconsistent DLP profile: keep aw_dlp_enabled=false with all DLP component flags false, or explicitly choose aw_dlp_enabled=true and aw_dlp_profile=light|on_demand|full."
- name: Установить базовые пакеты
ansible.builtin.apt:
name:
@@ -77,6 +94,7 @@
- "{{ aw_server_data_dir }}/backups"
- "{{ aw_server_data_dir }}/slo"
- "{{ aw_server_data_dir }}/browser-smoke"
- "{{ aw_security_finding_executor_work_dir | default(aw_server_data_dir ~ '/security-finding-executor') }}"
- "{{ aw_rus_health_state_dir }}"
- "{{ aw_rus_health_validation_dir }}"
- "{{ aw_server_log_dir }}"
@@ -107,6 +125,7 @@
- "{{ aw_server_data_dir }}/backups"
- "{{ aw_server_data_dir }}/slo"
- "{{ aw_server_data_dir }}/browser-smoke"
- "{{ aw_security_finding_executor_work_dir | default(aw_server_data_dir ~ '/security-finding-executor') }}"
- "{{ aw_rus_health_state_dir }}"
- "{{ aw_rus_health_validation_dir }}"
- "{{ aw_server_log_dir }}"
@@ -230,6 +249,8 @@
Environment=AW_BACKUP_RETENTION_DAYS={{ aw_server_backup_retention_days }}
Environment=AW_BACKUP_KEEP_LAST_DB={{ aw_server_backup_keep_last_db }}
Environment=AW_BACKUP_KEEP_LAST_JSON={{ aw_server_backup_keep_last_json }}
Environment=AW_WORKTIME_REPORT_DISK_CACHE_DIR={{ aw_worktime_report_disk_cache_dir | default(aw_server_data_dir ~ '/worktime-report-cache') }}
Environment=AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS={{ aw_worktime_report_disk_stale_ttl_seconds | default(86400) }}
ExecStart=/usr/local/bin/aw-prune-local-state.sh
- name: Установить systemd timer prune локального state
@@ -589,6 +610,44 @@
daemon_reload: true
when: aw_db_maintenance_rust_binary_early.stat.exists | default(false)
- name: Установить aw-db-vacuum service до Influx проверок
ansible.builtin.copy:
src: "{{ aw_repo_root }}/aw-server/aw-db-vacuum.service"
dest: /etc/systemd/system/aw-db-vacuum.service
owner: root
group: root
mode: "0644"
when: aw_db_maintenance_rust_binary_early.stat.exists | default(false)
- name: Установить aw-db-vacuum timer до Influx проверок
ansible.builtin.copy:
src: "{{ aw_repo_root }}/aw-server/aw-db-vacuum.timer"
dest: /etc/systemd/system/aw-db-vacuum.timer
owner: root
group: root
mode: "0644"
when: aw_db_maintenance_rust_binary_early.stat.exists | default(false)
- name: Включить nightly aw-db-vacuum timer до Influx проверок
ansible.builtin.systemd:
name: aw-db-vacuum.timer
enabled: true
state: started
daemon_reload: true
when:
- aw_db_maintenance_rust_binary_early.stat.exists | default(false)
- aw_db_vacuum_timer_enabled | bool
- name: Отключить nightly aw-db-vacuum timer если opt-in не задан
ansible.builtin.systemd:
name: aw-db-vacuum.timer
enabled: false
state: stopped
daemon_reload: true
when:
- aw_db_maintenance_rust_binary_early.stat.exists | default(false)
- not (aw_db_vacuum_timer_enabled | bool)
- name: Прочитать текущий aw-server.env для сохранения Influx token
ansible.builtin.slurp:
path: /etc/activitywatch/aw-server.env
@@ -674,7 +733,9 @@
- aw_effective_dlp_influx_token | length > 0
- (aw_effective_dlp_influx_token | string | lower | regex_search('^(change_me|changeme|replace-me|replace_me|token|secret|password|api_key|influx_token|write_token|your_.*|<.*>)$')) is none
fail_msg: "aw_dlp_influx_enabled=true, но token пуст и в локальном env, и в текущем /etc/activitywatch/aw-server.env. Exporter будет падать и Grafana не получит DLP-ряды."
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Проверить destination для AW worktime Influx exporter
ansible.builtin.assert:
@@ -706,7 +767,9 @@
- (aw_dlp_influx_hosts | default('') | string | length) > 0
- "'WINDOWS_USER_EXAMPLE' not in (aw_dlp_influx_hosts | default('') | string)"
fail_msg: "aw_dlp_influx_enabled=true, но URL/org/bucket/hosts похожи на public example/TEST-NET значения. Задайте live значения в private inventory/env, не в public repo."
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Записать /etc/activitywatch/aw-server.env перед хотфиксами
ansible.builtin.copy:
@@ -725,12 +788,13 @@
AW_SERVER_GROUP={{ aw_server_group }}
AW_WORKTIME_REPORT_BASE={{ aw_worktime_report_base }}
AW_WORKTIME_TZ={{ aw_worktime_timezone }}
AW_WORKTIME_HOST={{ aw_effective_worktime_host | default(aw_effective_monitored_windows_hostname | default('SHARKON2025')) }}
AW_WORKTIME_HOST={{ aw_effective_worktime_host | default(aw_effective_monitored_windows_hostname | default('HOST-EXAMPLE')) }}
AW_WORKTIME_EVENTS_LIMIT={{ aw_worktime_events_limit | default(5000) }}
AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS={{ aw_worktime_aw_http_timeout_seconds | default(6) }}
AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS={{ aw_worktime_events_cache_ttl_seconds | default(300) }}
AW_WORKTIME_REPORT_CACHE_TTL_SECONDS={{ aw_worktime_report_cache_ttl_seconds | default(300) }}
AW_WORKTIME_REPORT_STALE_TTL_SECONDS={{ aw_worktime_report_stale_ttl_seconds | default(3600) }}
AW_WORKTIME_REPORT_DISK_CACHE_DIR={{ aw_worktime_report_disk_cache_dir | default(aw_server_data_dir ~ '/worktime-report-cache') }}
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS={{ aw_worktime_report_disk_stale_ttl_seconds | default(86400) }}
AW_DLP_IOC_DIR={{ aw_dlp_ioc_workdir }}/output
AW_DLP_POLICY_ENGINE_BIND_HOST={{ aw_dlp_policy_engine_bind_host }}
@@ -748,7 +812,7 @@
AW_WORKTIME_INFLUX_URL={{ aw_worktime_influx_url | default('') }}
AW_WORKTIME_INFLUX_ORG={{ aw_worktime_influx_org | default('proxmox') }}
AW_WORKTIME_INFLUX_BUCKET={{ aw_worktime_influx_bucket | default('aw_metrics') }}
AW_WORKTIME_INFLUX_HOSTS={{ aw_worktime_influx_hosts | default('SHARKON2025') }}
AW_WORKTIME_INFLUX_HOSTS={{ aw_worktime_influx_hosts | default(aw_effective_monitored_windows_hostname | default('HOST-EXAMPLE')) }}
AW_WORKTIME_INFLUX_DAYS={{ aw_worktime_influx_days | default('today,yesterday') }}
AW_WORKTIME_INFLUX_TOKEN={{ aw_effective_worktime_influx_token | default('') }}
AW_WORKTIME_MANAGEMENT_HISTORY_DIR={{ aw_worktime_management_history_dir | default(aw_server_data_dir ~ '/worktime-management-history') }}
@@ -759,11 +823,28 @@
AW_WORKTIME_MANAGER_TREND_DELTA_PCT={{ aw_worktime_manager_trend_delta_pct | default(10) }}
AW_WORKTIME_MANAGER_OFF_HOURS_THRESHOLD_SECONDS={{ aw_worktime_manager_off_hours_threshold_seconds | default(1800) }}
AW_WORKTIME_MANAGER_INTERPRETATION_POLICY={{ aw_worktime_interpretation_policy_path | default('/etc/activitywatch/worktime-interpretation-policy.json') }}
AW_DLP_INFLUX_ENABLED={{ 'true' if (aw_dlp_influx_enabled | default(false) | bool) else 'false' }}
AW_DLP_ENABLED={{ 'true' if (aw_dlp_enabled | default(false) | bool) else 'false' }}
AW_DLP_PROFILE={{ aw_dlp_profile | default('core_only') }}
AW_DLP_DISABLED_REASON={{ aw_dlp_disabled_reason | default('') }}
AW_DLP_DISABLED_SINCE={{ aw_dlp_disabled_since | default('') }}
AW_DLP_GUARD_ENABLED={{ 'true' if (aw_dlp_light_guard_enabled | default(true) | bool) else 'false' }}
AW_DLP_GUARD_STATE_DIR={{ aw_dlp_light_guard_state_dir | default(aw_server_data_dir ~ '/health') }}
AW_DLP_GUARD_LOAD_RATIO={{ aw_dlp_light_guard_load_ratio | default('1.50') }}
AW_DLP_GUARD_MEM_AVAILABLE_PCT_MIN={{ aw_dlp_light_guard_mem_available_pct_min | default('15') }}
AW_DLP_GUARD_IOWAIT_PCT_MAX={{ aw_dlp_light_guard_iowait_pct_max | default('20') }}
AW_DLP_GUARD_STRIKES_REQUIRED={{ aw_dlp_light_guard_strikes_required | default(3) }}
AW_DLP_CONTROL_BIN=/usr/local/bin/detmir-dlp-runtime-control
AW_CONTAINMENT_ENABLED={{ 'true' if (aw_containment_enabled | default(false) | bool) else 'false' }}
AW_CONTAINMENT_MODE={{ aw_containment_mode | default('shadow') }}
AW_CONTAINMENT_POLICY={{ aw_containment_policy_path | default('/etc/activitywatch/containment-policy.json') }}
AW_CONTAINMENT_DEFAULT_TTL_MINUTES={{ aw_containment_default_ttl_minutes | default(60) }}
AW_CONTAINMENT_REQUIRE_ADMIN_CHANNEL_CHECK={{ 'true' if (aw_containment_require_admin_channel_check | default(true) | bool) else 'false' }}
AW_CONTAINMENT_ALLOW_AUTO_FOR_SERVERS={{ 'true' if (aw_containment_allow_auto_for_servers | default(false) | bool) else 'false' }}
AW_DLP_INFLUX_ENABLED={{ 'true' if ((aw_dlp_enabled | default(false) | bool) and (aw_dlp_influx_enabled | default(false) | bool)) else 'false' }}
AW_DLP_INFLUX_URL={{ aw_dlp_influx_url | default('') }}
AW_DLP_INFLUX_ORG={{ aw_dlp_influx_org | default('proxmox') }}
AW_DLP_INFLUX_BUCKET={{ aw_dlp_influx_bucket | default('aw_metrics') }}
AW_DLP_INFLUX_HOSTS={{ aw_dlp_influx_hosts | default('SHARKON2025') }}
AW_DLP_INFLUX_HOSTS={{ aw_dlp_influx_hosts | default(aw_effective_monitored_windows_hostname | default('HOST-EXAMPLE')) }}
AW_DLP_INFLUX_LOOKBACK_DAYS={{ aw_dlp_influx_lookback_days | default(30) }}
AW_DLP_INFLUX_EVENT_LIMIT={{ aw_dlp_influx_event_limit | default(2000) }}
AW_DLP_INFLUX_TOKEN={{ aw_effective_dlp_influx_token | default('') }}
@@ -781,6 +862,7 @@
AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS={{ aw_rus_health_session_events_max_age_seconds | default(86400) }}
AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS={{ aw_rus_health_guard_max_age_seconds | default(300) }}
AW_RUS_HEALTH_GUARD_REQUIRED={{ 1 if (aw_rus_health_guard_required | default(true) | bool) else 0 }}
AW_RUS_HEALTH_RDP_TCP_REQUIRED={{ 'true' if (aw_rus_health_rdp_tcp_required | default(true) | bool) else 'false' }}
AW_RUS_SLO_STATE_DIR={{ aw_server_data_dir }}/slo
AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600
AW_RUS_SLO_WORKTIME_BASE={{ aw_rus_health_worktime_api_base | default('http://127.0.0.1:5610') }}
@@ -798,6 +880,104 @@
AW_HAYABUSA_TELEGRAM_MIN_SEVERITY={{ aw_hayabusa_telegram_min_severity | default('high') }}
AW_HAYABUSA_TELEGRAM_BOT_TOKEN={{ aw_hayabusa_telegram_bot_token | default('') }}
AW_HAYABUSA_TELEGRAM_CHAT_IDS={{ aw_hayabusa_telegram_chat_ids | default('') }}
AW_SECURITY_FINDING_INBOX_ENABLED={{ 'true' if (aw_security_finding_inbox_enabled | default(false) | bool) else 'false' }}
AW_SECURITY_FINDING_INBOX_REQUIRED={{ 'true' if (aw_security_finding_inbox_required | default(false) | bool) else 'false' }}
AW_SECURITY_FINDING_INBOX_BIN={{ aw_security_finding_inbox_bin | default('/usr/local/bin/security-finding-inbox') }}
AW_SECURITY_FINDING_INBOX_MIN_SEVERITY={{ aw_security_finding_inbox_min_severity | default('medium') }}
AW_SECURITY_FINDING_EXECUTOR_WORK_DIR={{ aw_security_finding_executor_work_dir | default(aw_server_data_dir ~ '/security-finding-executor') }}
AW_SECURITY_FINDING_EXECUTOR_LOCK={{ aw_security_finding_executor_lock | default('/var/lock/aw-security-finding-executor.lock') }}
AW_CONTAINMENT_ENGINE_BIN={{ aw_containment_engine_bin | default('/usr/local/bin/containment-engine') }}
AW_CONTAINMENT_MANAGEMENT_ALLOWLIST={{ aw_containment_management_allowlist | default('') }}
AW_CONTAINMENT_BLOCKED_REMOTE_ADDRESSES={{ aw_containment_blocked_remote_addresses | default('') }}
- name: Установить runtime control для optional DLP контура
ansible.builtin.copy:
src: "{{ aw_repo_root }}/scripts/detmir_dlp_runtime_control.sh"
dest: /usr/local/bin/detmir-dlp-runtime-control
owner: root
group: root
mode: "0755"
- name: Установить load guard для lightweight DLP контура
ansible.builtin.copy:
src: "{{ aw_repo_root }}/scripts/detmir_dlp_load_guard.sh"
dest: /usr/local/bin/detmir-dlp-load-guard
owner: root
group: root
mode: "0755"
- name: Установить systemd unit DLP load guard
ansible.builtin.copy:
dest: /etc/systemd/system/detmir-dlp-load-guard.service
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=DetMir lightweight DLP load guard
After=activitywatch-server.service
[Service]
Type=oneshot
EnvironmentFile=-/etc/activitywatch/aw-server.env
ExecStart=/usr/local/bin/detmir-dlp-load-guard
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
TimeoutStartSec=45
- name: Установить systemd timer DLP load guard
ansible.builtin.copy:
dest: /etc/systemd/system/detmir-dlp-load-guard.timer
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=Run DetMir lightweight DLP load guard
[Timer]
OnBootSec=3min
OnUnitActiveSec=1min
AccuracySec=30s
Persistent=false
[Install]
WantedBy=timers.target
- name: Включить DLP load guard timer
ansible.builtin.systemd:
name: detmir-dlp-load-guard.timer
enabled: true
state: started
daemon_reload: true
when: aw_dlp_light_guard_enabled | default(true) | bool
- name: Отключить DLP load guard timer, если guard явно выключен
ansible.builtin.systemd:
name: detmir-dlp-load-guard.timer
enabled: false
state: stopped
daemon_reload: true
failed_when: false
when: not (aw_dlp_light_guard_enabled | default(true) | bool)
- name: Создать каталог containment policy
ansible.builtin.file:
path: "{{ (aw_containment_policy_path | default('/etc/activitywatch/containment-policy.json')) | dirname }}"
state: directory
owner: root
group: root
mode: "0755"
- name: Установить default containment policy, если live policy отсутствует
ansible.builtin.copy:
src: "{{ aw_repo_root }}/configs/containment-policy.example.json"
dest: "{{ aw_containment_policy_path | default('/etc/activitywatch/containment-policy.json') }}"
owner: root
group: root
mode: "0644"
force: false
- name: Создать каталог DLP policy engine
ansible.builtin.file:
@@ -806,7 +986,9 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
when: aw_dlp_policy_engine_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_policy_engine_enabled | default(false) | bool
- name: Установить systemd unit DLP policy engine
ansible.builtin.copy:
@@ -815,7 +997,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_policy_engine_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_policy_engine_enabled | default(false) | bool
- name: Проверить локальный Rust DLP policy engine
ansible.builtin.stat:
@@ -850,7 +1034,7 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Скопировать файлы DLP content analysis
ansible.builtin.copy:
@@ -859,7 +1043,7 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0644"
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Установить wrapper запуска DLP content analysis через virtualenv
ansible.builtin.copy:
@@ -868,7 +1052,7 @@
owner: root
group: root
mode: "0755"
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Проверить локальный Rust DLP content analyzer
ansible.builtin.stat:
@@ -876,7 +1060,7 @@
delegate_to: localhost
register: dlp_content_analyzer_rust_binary
become: false
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Установить Rust DLP content analyzer
ansible.builtin.copy:
@@ -886,7 +1070,7 @@
group: root
mode: "0755"
when:
- aw_dlp_content_analysis_enabled | default(true) | bool
- aw_dlp_content_analysis_enabled | default(false) | bool
- dlp_content_analyzer_rust_binary.stat.exists | default(false)
- name: Создать virtualenv DLP content analysis
@@ -894,13 +1078,13 @@
cmd: python3 -m venv /opt/activitywatch/dlp-content-analysis/.venv
args:
creates: /opt/activitywatch/dlp-content-analysis/.venv/bin/python
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Установить зависимости DLP content analysis
ansible.builtin.pip:
requirements: /opt/activitywatch/dlp-content-analysis/requirements.txt
virtualenv: /opt/activitywatch/dlp-content-analysis/.venv
when: aw_dlp_content_analysis_enabled | default(true) | bool
when: aw_dlp_content_analysis_enabled | default(false) | bool
- name: Создать каталог DLP integrations
ansible.builtin.file:
@@ -909,7 +1093,9 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Скопировать файлы DLP integrations
ansible.builtin.copy:
@@ -922,7 +1108,9 @@
- cef-config.yaml
- syslog-forwarder-config.yaml
- webhook-config.yaml
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Создать state каталог DLP integrations
ansible.builtin.file:
@@ -931,7 +1119,9 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Установить systemd unit CEF exporter
ansible.builtin.copy:
@@ -940,7 +1130,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Установить systemd timer CEF exporter
ansible.builtin.copy:
@@ -949,7 +1141,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Проверить локальный Rust CEF exporter
ansible.builtin.stat:
@@ -957,14 +1151,16 @@
delegate_to: localhost
register: dlp_cef_exporter_rust_binary
become: false
when: aw_dlp_integrations_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- name: Требовать Rust CEF exporter artifact
ansible.builtin.assert:
that:
- dlp_cef_exporter_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-cef-exporter"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Установить Rust CEF exporter
ansible.builtin.copy:
@@ -974,7 +1170,7 @@
group: root
mode: "0755"
when:
- aw_dlp_integrations_enabled | default(true) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- dlp_cef_exporter_rust_binary.stat.exists | default(false)
- name: Установить systemd unit syslog forwarder
@@ -984,7 +1180,7 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Установить systemd timer syslog forwarder
ansible.builtin.copy:
@@ -993,7 +1189,7 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Проверить локальный Rust syslog forwarder
ansible.builtin.stat:
@@ -1001,14 +1197,14 @@
delegate_to: localhost
register: dlp_syslog_forwarder_rust_binary
become: false
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Требовать Rust syslog forwarder artifact
ansible.builtin.assert:
that:
- dlp_syslog_forwarder_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-syslog-forwarder"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Установить Rust syslog forwarder
ansible.builtin.copy:
@@ -1018,7 +1214,7 @@
group: root
mode: "0755"
when:
- aw_dlp_integrations_enabled | default(true) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- dlp_syslog_forwarder_rust_binary.stat.exists | default(false)
- name: Установить systemd unit webhook sender
@@ -1028,7 +1224,7 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Установить systemd timer webhook sender
ansible.builtin.copy:
@@ -1037,7 +1233,7 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Проверить локальный Rust webhook sender
ansible.builtin.stat:
@@ -1045,14 +1241,14 @@
delegate_to: localhost
register: dlp_webhook_sender_rust_binary
become: false
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Требовать Rust webhook sender artifact
ansible.builtin.assert:
that:
- dlp_webhook_sender_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-webhook-sender"
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Установить Rust webhook sender
ansible.builtin.copy:
@@ -1062,7 +1258,7 @@
group: root
mode: "0755"
when:
- aw_dlp_integrations_enabled | default(true) | bool
- aw_dlp_integrations_enabled | default(false) | bool
- dlp_webhook_sender_rust_binary.stat.exists | default(false)
- name: Создать каталог DLP case management
@@ -1072,7 +1268,9 @@
owner: "{{ aw_server_user }}"
group: "{{ aw_server_group }}"
mode: "0755"
when: aw_dlp_case_management_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_case_management_enabled | default(false) | bool
- name: Установить systemd unit DLP case management
ansible.builtin.copy:
@@ -1081,7 +1279,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_case_management_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_case_management_enabled | default(false) | bool
- name: Проверить локальный Rust DLP case management
ansible.builtin.stat:
@@ -1089,14 +1289,14 @@
delegate_to: localhost
register: aw_dlp_case_management_rust_binary
become: false
when: aw_dlp_case_management_enabled | default(true) | bool
when: aw_dlp_case_management_enabled | default(false) | bool
- name: Требовать Rust DLP case management artifact
ansible.builtin.assert:
that:
- aw_dlp_case_management_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-case-management"
when: aw_dlp_case_management_enabled | default(true) | bool
when: aw_dlp_case_management_enabled | default(false) | bool
- name: Установить Rust DLP case management
ansible.builtin.copy:
@@ -1106,7 +1306,7 @@
group: root
mode: "0755"
when:
- aw_dlp_case_management_enabled | default(true) | bool
- aw_dlp_case_management_enabled | default(false) | bool
- aw_dlp_case_management_rust_binary.stat.exists | default(false)
- name: Создать каталоги DLP compliance
@@ -1120,7 +1320,9 @@
- /opt/activitywatch/dlp-compliance
- /opt/activitywatch/dlp-compliance/templates
- "{{ aw_dlp_compliance_report_dir }}"
when: aw_dlp_compliance_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_compliance_enabled | default(false) | bool
- name: Скопировать файлы DLP compliance
ansible.builtin.copy:
@@ -1134,7 +1336,9 @@
- { src: "templates/pci-dss-report.html", dest: "/opt/activitywatch/dlp-compliance/templates/pci-dss-report.html", mode: "0644" }
- { src: "report-scheduler.service", dest: "/etc/systemd/system/aw-dlp-report-scheduler.service", mode: "0644" }
- { src: "report-scheduler.timer", dest: "/etc/systemd/system/aw-dlp-report-scheduler.timer", mode: "0644" }
when: aw_dlp_compliance_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_compliance_enabled | default(false) | bool
- name: Проверить локальный Rust DLP compliance
ansible.builtin.stat:
@@ -1142,14 +1346,18 @@
delegate_to: localhost
register: aw_dlp_compliance_rust_binary
become: false
when: aw_dlp_compliance_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_compliance_enabled | default(false) | bool
- name: Требовать Rust DLP compliance artifact
ansible.builtin.assert:
that:
- aw_dlp_compliance_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-compliance"
when: aw_dlp_compliance_enabled | default(true) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_compliance_enabled | default(false) | bool
- name: Установить Rust DLP compliance
ansible.builtin.copy:
@@ -1159,7 +1367,7 @@
group: root
mode: "0755"
when:
- aw_dlp_compliance_enabled | default(true) | bool
- aw_dlp_compliance_enabled | default(false) | bool
- aw_dlp_compliance_rust_binary.stat.exists | default(false)
- name: Проверить локальный Rust dlp-admin-cli
@@ -1398,6 +1606,47 @@
mode: "0755"
when: dlp_health_check_rust_binary.stat.exists | default(false)
- name: Проверить локальный Rust containment-engine
ansible.builtin.stat:
path: "{{ aw_rust_release_dir }}/containment-engine"
delegate_to: localhost
register: containment_engine_rust_binary
become: false
- name: Установить Rust containment-engine
ansible.builtin.copy:
src: "{{ aw_rust_release_dir }}/containment-engine"
dest: /usr/local/bin/containment-engine
owner: root
group: root
mode: "0755"
when: containment_engine_rust_binary.stat.exists | default(false)
- name: Проверить локальный Rust security-finding-inbox
ansible.builtin.stat:
path: "{{ aw_rust_release_dir }}/security-finding-inbox"
delegate_to: localhost
register: security_finding_inbox_rust_binary
become: false
- name: Установить Rust security-finding-inbox
ansible.builtin.copy:
src: "{{ aw_rust_release_dir }}/security-finding-inbox"
dest: /usr/local/bin/security-finding-inbox
owner: root
group: root
mode: "0755"
when: security_finding_inbox_rust_binary.stat.exists | default(false)
- name: Установить systemd unit Security Finding Inbox executor
ansible.builtin.copy:
src: "{{ aw_repo_root }}/ops/systemd/aw-security-finding-executor.service"
dest: /etc/systemd/system/aw-security-finding-executor.service
owner: root
group: root
mode: "0644"
notify: Перезагрузить systemd
- name: Проверить локальный Rust AW-RUS healthd
ansible.builtin.stat:
path: "{{ aw_rust_release_dir }}/aw-rus-healthd"
@@ -1605,7 +1854,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Проверить локальный Rust AW DLP Influx exporter
ansible.builtin.stat:
@@ -1613,14 +1864,18 @@
delegate_to: localhost
register: aw_dlp_influx_exporter_rust_binary
become: false
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Требовать Rust AW DLP Influx exporter artifact
ansible.builtin.assert:
that:
- aw_dlp_influx_exporter_rust_binary.stat.exists | default(false)
fail_msg: "Missing Rust artifact: {{ aw_rust_release_dir }}/dlp-influx-exporter"
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Установить Rust AW DLP Influx exporter
ansible.builtin.copy:
@@ -1640,7 +1895,9 @@
owner: root
group: root
mode: "0644"
when: aw_dlp_influx_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_influx_enabled | default(false) | bool
- name: Проверить локальный Rust DetMir readiness checker
ansible.builtin.stat:
@@ -1792,42 +2049,42 @@
name: aw-dlp-cef-exporter.timer
enabled: true
state: restarted
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Включить и перезапустить timer syslog forwarder
ansible.builtin.systemd:
name: aw-dlp-syslog-forwarder.timer
enabled: true
state: restarted
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Включить и перезапустить timer webhook sender
ansible.builtin.systemd:
name: aw-dlp-webhook-sender.timer
enabled: true
state: restarted
when: aw_dlp_integrations_enabled | default(true) | bool
when: aw_dlp_integrations_enabled | default(false) | bool
- name: Включить и перезапустить DLP case management
ansible.builtin.systemd:
name: aw-dlp-case-management.service
enabled: true
state: restarted
when: aw_dlp_case_management_enabled | default(true) | bool
when: aw_dlp_case_management_enabled | default(false) | bool
- name: Включить и перезапустить timer DLP compliance report
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.timer
enabled: true
state: restarted
when: aw_dlp_compliance_enabled | default(true) | bool
when: aw_dlp_compliance_enabled | default(false) | bool
- name: Выполнить разовый прогон DLP compliance report
ansible.builtin.systemd:
name: aw-dlp-report-scheduler.service
state: started
failed_when: false
when: aw_dlp_compliance_enabled | default(true) | bool
when: aw_dlp_compliance_enabled | default(false) | bool
- name: Включить и перезапустить AW worktime API
ansible.builtin.systemd:
@@ -2194,6 +2451,11 @@
mode: "0755"
when: dlp_aggregator_rust_binary.stat.exists | default(false)
- name: Удалить stale drop-in, переопределяющий lightweight DLP aggregator
ansible.builtin.file:
path: /etc/systemd/system/activitywatch-dlp-aggregator.service.d/20-rust-switch.conf
state: absent
- name: Установить systemd unit для агрегатора
ansible.builtin.copy:
dest: /etc/systemd/system/activitywatch-dlp-aggregator.service
@@ -2202,7 +2464,7 @@
mode: "0644"
content: |
[Unit]
Description=ActivityWatch DLP Event Aggregator
Description=ActivityWatch Lightweight DLP Event Aggregator
After=activitywatch-server.service
[Service]
@@ -2212,7 +2474,18 @@
ExecStart=/usr/local/bin/dlp-aggregator-rust \
--aw-url http://127.0.0.1:{{ aw_server_port }}/api/0 \
--sqlite-path {{ aw_server_data_dir }}/dlp_warehouse.sqlite \
--state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json
--state-path {{ aw_server_data_dir }}/dlp-aggregator-state.json \
--bucket-prefixes {{ aw_dlp_aggregator_bucket_prefixes | default('aw-file-operations_,aw-dlp-incidents_') }} \
--lookback-hours {{ aw_dlp_aggregator_lookback_hours | default(2) }} \
--overlap-seconds {{ aw_dlp_aggregator_overlap_seconds | default(60) }} \
--limit {{ aw_dlp_aggregator_limit | default(500) }} \
--timeout {{ aw_dlp_aggregator_timeout_seconds | default(8) }}
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
CPUQuota={{ aw_dlp_aggregator_cpu_quota | default('10%') }}
MemoryMax={{ aw_dlp_aggregator_memory_max | default('256M') }}
TimeoutStartSec={{ (aw_dlp_aggregator_timeout_seconds | default(8) | int) + 15 }}
[Install]
WantedBy=multi-user.target
@@ -2222,10 +2495,10 @@
dest: /etc/systemd/system/activitywatch-dlp-aggregator.timer
content: |
[Unit]
Description=Run ActivityWatch DLP Aggregator every 5 minutes
Description=Run ActivityWatch Lightweight DLP Aggregator
[Timer]
OnCalendar=*:3/10:10
OnCalendar={{ aw_dlp_aggregator_on_calendar | default('*:3/15:10') }}
AccuracySec=30s
RandomizedDelaySec=30s
Persistent=false
@@ -2239,9 +2512,14 @@
enabled: true
state: started
daemon_reload: true
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_light_collector_enabled | default(false) | bool
- name: Настроить IOC enrichment из Hayabusa Sigma
when: aw_dlp_ioc_enabled | default(false) | bool
when:
- aw_dlp_enabled | default(false) | bool
- aw_dlp_ioc_enabled | default(false) | bool
block:
- name: Создать каталог IOC enrichment
ansible.builtin.file:
+76
View File
@@ -15,8 +15,16 @@
detmir_portal_workforce_policy_path: "/etc/detmir-portal-workforce-policy.json"
detmir_portal_ueba_policy_path: "/etc/detmir-portal-ueba-policy.yaml"
detmir_portal_readiness_bundle_dir: "{{ detmir_portal_readiness_bundle_dir_override | default('/var/lib/activitywatch/health/readiness-bundle', true) }}"
detmir_portal_dlp_module_enabled: "{{ detmir_portal_dlp_module_enabled_override | default(false) }}"
tasks:
- name: Refuse inconsistent DetMir portal DLP profile
ansible.builtin.assert:
that:
- detmir_portal_dlp_profile | default('core_only') in ['core_only', 'light', 'on_demand', 'full']
- (detmir_portal_dlp_profile | default('core_only') != 'core_only') or not (detmir_portal_dlp_module_enabled | bool)
fail_msg: "Inconsistent DetMir portal DLP profile: core_only must keep DETMIR_PORTAL_DLP_MODULE_ENABLED=false."
- name: Check local detmir-portal binary
ansible.builtin.stat:
path: "{{ aw_rust_release_dir }}/detmir-portal"
@@ -54,6 +62,8 @@
DETMIR_PORTAL_UEBA_POLICY_PATH={{ detmir_portal_ueba_policy_path }}
DETMIR_PORTAL_TIMEOUT_SECONDS=25
DETMIR_PORTAL_STATE_DIR=/var/lib/detmir-portal
DETMIR_PORTAL_DLP_MODULE_ENABLED={{ detmir_portal_dlp_module_enabled | bool | ternary('true', 'false') }}
DETMIR_PORTAL_DLP_PROFILE={{ detmir_portal_dlp_profile | default('core_only') }}
DETMIR_PORTAL_DLP_DB_PATH=/var/lib/activitywatch/dlp_warehouse.sqlite
DETMIR_PORTAL_EVIDENCE_ROOT=/var/lib/detmir-portal/evidence
DETMIR_PORTAL_READINESS_BUNDLE_DIR={{ detmir_portal_readiness_bundle_dir }}
@@ -65,6 +75,65 @@
CLICKHOUSE_USER={{ detmir_clickhouse_user | default('default') }}
CLICKHOUSE_PASSWORD={{ detmir_clickhouse_password | default('') }}
- name: Install lightweight DLP warehouse sync helper
ansible.builtin.copy:
src: "{{ aw_repo_root }}/scripts/detmir_dlp_warehouse_sync.sh"
dest: /usr/local/bin/detmir-dlp-warehouse-sync
owner: root
group: root
mode: "0755"
- name: Install lightweight DLP warehouse sync service
ansible.builtin.copy:
dest: /etc/systemd/system/detmir-dlp-warehouse-sync.service
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=Sync lightweight DetMir DLP SQLite warehouse for portal
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
Environment=AW_DLP_WAREHOUSE_SOURCE_HOST={{ detmir_portal_dlp_warehouse_source_host | default('igor@10.10.10.13') }}
Environment=AW_DLP_WAREHOUSE_SOURCE_PATH={{ detmir_portal_dlp_warehouse_source_path | default('/var/lib/activitywatch/dlp_warehouse.sqlite') }}
Environment=AW_DLP_WAREHOUSE_DEST_PATH={{ detmir_portal_dlp_warehouse_dest_path | default('/var/lib/activitywatch/dlp_warehouse.sqlite') }}
Environment=AW_DLP_WAREHOUSE_SYNC_STATE_DIR={{ detmir_portal_dlp_warehouse_sync_state_dir | default('/var/lib/activitywatch/health') }}
ExecStart=/usr/local/bin/detmir-dlp-warehouse-sync
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
TimeoutStartSec=60
- name: Install lightweight DLP warehouse sync timer
ansible.builtin.copy:
dest: /etc/systemd/system/detmir-dlp-warehouse-sync.timer
owner: root
group: root
mode: "0644"
content: |
[Unit]
Description=Run lightweight DetMir DLP SQLite warehouse sync
[Timer]
OnBootSec=4min
OnUnitActiveSec={{ detmir_portal_dlp_warehouse_sync_interval | default('2min') }}
AccuracySec=30s
Persistent=false
[Install]
WantedBy=timers.target
- name: Enable lightweight DLP warehouse sync timer
ansible.builtin.systemd:
name: detmir-dlp-warehouse-sync.timer
enabled: true
state: started
daemon_reload: true
when: detmir_portal_dlp_module_enabled | bool
- name: Preserve local ClickHouse security-events settings when available
ansible.builtin.shell: |
set -euo pipefail
@@ -117,7 +186,9 @@
state: absent
loop:
- /etc/systemd/system/detmir-portal.service.d/20-timeouts.conf
- /etc/systemd/system/detmir-portal.service.d/20-prod-timeout.conf
- /etc/systemd/system/detmir-portal.service.d/30-warm-cache.conf
- /etc/systemd/system/detmir-portal.service.d/30-prewarm-after-start.conf
register: detmir_portal_stale_overrides
- name: Install initial workforce policy when absent
@@ -174,6 +245,11 @@
WantedBy=multi-user.target
register: detmir_portal_service_unit
- name: Remove stale detmir-portal timeout override
ansible.builtin.file:
path: /etc/systemd/system/detmir-portal.service.d/10-detmir-check-env.conf
state: absent
- name: Reload systemd
ansible.builtin.systemd:
daemon_reload: true
+1
View File
@@ -17,6 +17,7 @@ aw_worktime_aw_http_timeout_seconds: 6
aw_worktime_events_cache_ttl_seconds: 300
aw_worktime_report_cache_ttl_seconds: 300
aw_worktime_report_stale_ttl_seconds: 3600
aw_worktime_report_disk_cache_dir: "{{ aw_server_data_dir }}/worktime-report-cache"
aw_worktime_report_disk_stale_ttl_seconds: 86400
aw_worktime_influx_enabled: false
aw_worktime_influx_url: "http://<INFLUXDB_HOST>:8086"
+36 -7
View File
@@ -17,6 +17,7 @@ aw_worktime_aw_http_timeout_seconds: 6
aw_worktime_events_cache_ttl_seconds: 300
aw_worktime_report_cache_ttl_seconds: 300
aw_worktime_report_stale_ttl_seconds: 3600
aw_worktime_report_disk_cache_dir: "{{ aw_server_data_dir }}/worktime-report-cache"
aw_worktime_report_disk_stale_ttl_seconds: 86400
aw_worktime_influx_enabled: true
aw_worktime_influx_url: "http://192.0.2.10:8086"
@@ -33,7 +34,10 @@ aw_worktime_manager_trend_min_points: 3
aw_worktime_manager_trend_delta_pct: 10
aw_worktime_manager_off_hours_threshold_seconds: 1800
aw_worktime_interpretation_policy_path: "/etc/activitywatch/worktime-interpretation-policy.json"
aw_dlp_influx_enabled: true
aw_dlp_profile: "core_only"
detmir_portal_dlp_profile: "light"
detmir_portal_dlp_module_enabled_override: true
aw_dlp_influx_enabled: false
aw_dlp_influx_url: "http://192.0.2.10:8086"
aw_dlp_influx_org: "proxmox"
aw_dlp_influx_bucket: "aw_metrics"
@@ -47,6 +51,7 @@ aw_worktime_host: "{{ aw_monitored_windows_hostname }}"
aw_rus_health_worktime_api_base: "http://127.0.0.1:5610"
aw_rus_health_state_dir: "{{ aw_server_data_dir }}/health"
aw_rus_health_validation_dir: "{{ aw_rus_health_state_dir }}/windows-validation"
aw_rus_health_rdp_tcp_required: false
aw_hayabusa_auto_case_enabled: true
aw_hayabusa_auto_case_min_severity: "medium"
aw_hayabusa_telegram_enabled: true
@@ -65,22 +70,46 @@ aw_server_cors_origins:
aw_apply_worktime_settings: true
aw_dlp_ioc_enabled: true
aw_dlp_ioc_enabled: false
aw_dlp_enabled: false
aw_dlp_disabled_reason: ""
aw_dlp_disabled_since: ""
aw_dlp_light_collector_enabled: false
aw_dlp_light_guard_enabled: true
aw_dlp_light_guard_load_ratio: "1.50"
aw_dlp_light_guard_mem_available_pct_min: "15"
aw_dlp_light_guard_iowait_pct_max: "20"
aw_dlp_light_guard_strikes_required: 3
aw_dlp_light_guard_state_dir: "{{ aw_server_data_dir }}/health"
aw_dlp_aggregator_bucket_prefixes: "aw-file-operations_,aw-dlp-incidents_"
aw_dlp_aggregator_limit: 500
aw_dlp_aggregator_lookback_hours: 2
aw_dlp_aggregator_overlap_seconds: 60
aw_dlp_aggregator_timeout_seconds: 8
aw_dlp_aggregator_on_calendar: "*:3/15:10"
aw_dlp_aggregator_cpu_quota: "10%"
aw_dlp_aggregator_memory_max: "256M"
aw_containment_enabled: false
aw_containment_mode: "shadow"
aw_containment_policy_path: "/etc/activitywatch/containment-policy.json"
aw_containment_default_ttl_minutes: 60
aw_containment_require_admin_channel_check: true
aw_containment_allow_auto_for_servers: false
aw_dlp_ioc_workdir: "/opt/activitywatch/dlp-ioc"
aw_dlp_ioc_rules_zip_url: "https://github.com/Yamato-Security/hayabusa-rules/archive/refs/heads/main.zip"
aw_dlp_ioc_refresh_on_boot_sec: "5min"
aw_dlp_ioc_refresh_interval: "6h"
aw_dlp_policy_engine_enabled: true
aw_dlp_policy_engine_enabled: false
aw_dlp_policy_engine_bind_host: "0.0.0.0"
aw_dlp_policy_engine_port: 5601
aw_dlp_policy_engine_db_path: "{{ aw_server_data_dir }}/dlp-policy-engine.sqlite"
aw_dlp_content_analysis_enabled: true
aw_dlp_integrations_enabled: true
aw_dlp_case_management_enabled: true
aw_dlp_content_analysis_enabled: false
aw_dlp_integrations_enabled: false
aw_dlp_case_management_enabled: false
aw_dlp_case_bind_host: "0.0.0.0"
aw_dlp_case_port: 5602
aw_dlp_case_db_path: "/opt/activitywatch/dlp-case-management/cases.db"
aw_dlp_compliance_enabled: true
aw_dlp_compliance_enabled: false
aw_dlp_compliance_report_dir: "/opt/activitywatch/dlp-compliance/reports"
aw_dlp_compliance_template_path: "/opt/activitywatch/dlp-compliance/templates/152-fz-report.html"
aw_server_post_deploy_health_check_enabled: true
@@ -13,8 +13,8 @@ aw_pfsense_poller_config:
verify_tls: false
timeout_seconds: 15
auth:
api_key: "replace-me"
api_secret: "replace-me"
api_key: "<SET_VIA_ENV>"
api_secret: "<SET_VIA_ENV>"
endpoints:
- name: "system-status"
path: "/api/v2/status/system"
+5
View File
@@ -0,0 +1,5 @@
---
# SHARKON2025 uses aw-windows-telemetry browser-domains-collector as the
# per-user currentwindow source. The legacy aw-watcher-window process emits
# no-user duplicate rows in this RDP setup, so keep it disabled for this host.
aw_windows_window_enabled: false
+14 -3
View File
@@ -250,6 +250,10 @@ server {
}
{% for route in proxmox_web_gateway_routes %}
location = /go/{{ route.slug }} {
return 302 {{ route.target_url }};
}
{% if route.proxy_target_url is defined %}
location = {{ route.proxy_path }} {
proxy_set_header Authorization "";
@@ -267,14 +271,21 @@ server {
location ^~ /portal/api/dlp/evidence {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://192.0.2.13:8721/api/dlp/evidence;
proxy_pass http://127.0.0.1:8720/api/dlp/evidence;
proxy_redirect off;
}
location ^~ /portal/api/readiness {
location = /portal/api/readiness {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://192.0.2.13:8721/api/readiness;
proxy_pass http://127.0.0.1:8720/readyz;
proxy_redirect off;
}
location ^~ /portal/api/readiness/ {
proxy_set_header Authorization "";
proxy_set_header X-Remote-User $remote_user;
proxy_pass http://127.0.0.1:8720/api/readiness/;
proxy_redirect off;
}
+19
View File
@@ -0,0 +1,19 @@
[Unit]
Description=ActivityWatch SQLite nightly VACUUM
After=activitywatch-server.service
ConditionPathExists=/usr/local/bin/aw-db-maintenance
ConditionPathExists=/var/lib/activitywatch/aw-server-rust/sqlite.db
RequiresMountsFor=/var/lib/activitywatch
[Service]
Type=oneshot
EnvironmentFile=-/etc/activitywatch/aw-server.env
ExecStart=/usr/local/bin/aw-db-maintenance --vacuum --apply --json
TimeoutStartSec=2h
SyslogIdentifier=aw-db-vacuum
StandardOutput=journal
StandardError=journal
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
UMask=0077
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Nightly ActivityWatch SQLite VACUUM
[Timer]
OnCalendar=*-*-* 02:10:00
RandomizedDelaySec=10m
Persistent=true
[Install]
WantedBy=timers.target
+38 -4
View File
@@ -16,12 +16,13 @@ AW_SERVER_GROUP=activitywatch
AW_SERVER_PUBLIC_HOST=aw-server
AW_WORKTIME_REPORT_BASE=http://aw-server:5610
AW_WORKTIME_TZ=Europe/Moscow
AW_WORKTIME_HOST=SHARKON2025
AW_WORKTIME_HOST=HOST-EXAMPLE
AW_WORKTIME_EVENTS_LIMIT=5000
AW_WORKTIME_AW_HTTP_TIMEOUT_SECONDS=6
AW_WORKTIME_EVENTS_CACHE_TTL_SECONDS=300
AW_WORKTIME_REPORT_CACHE_TTL_SECONDS=300
AW_WORKTIME_REPORT_STALE_TTL_SECONDS=3600
AW_WORKTIME_REPORT_DISK_CACHE_DIR=/var/lib/activitywatch/worktime-report-cache
AW_WORKTIME_REPORT_DISK_STALE_TTL_SECONDS=86400
AW_SERVER_URL=http://127.0.0.1:5600
AW_DLP_AW_API_BASE=http://127.0.0.1:5600/api/0
@@ -33,6 +34,16 @@ AW_WORKTIME_MANAGEMENT_WARM_URL=http://127.0.0.1:5610/reports/worktime/managemen
AW_WORKTIME_MANAGEMENT_WARM_TIMEOUT_SECONDS=70
# DLP IOC Configuration
AW_DLP_ENABLED=false
AW_DLP_PROFILE=core_only
AW_DLP_DISABLED_REASON=detmir_prod_resource_guardrail
AW_DLP_DISABLED_SINCE=
AW_CONTAINMENT_ENABLED=false
AW_CONTAINMENT_MODE=shadow
AW_CONTAINMENT_POLICY=/etc/activitywatch/containment-policy.json
AW_CONTAINMENT_DEFAULT_TTL_MINUTES=60
AW_CONTAINMENT_REQUIRE_ADMIN_CHANNEL_CHECK=true
AW_CONTAINMENT_ALLOW_AUTO_FOR_SERVERS=false
AW_DLP_IOC_DIR=/opt/activitywatch/dlp-ioc/output
# DLP Policy Engine Configuration
@@ -50,28 +61,42 @@ AW_HEALTH_CHECK_ENABLED=true
AW_HEALTH_CHECK_INTERVAL=60
AW_EXPECT_START_OF_DAY=00:00
AW_EXPECT_ALWAYS_ACTIVE_PATTERN=aw-watcher-window
AW_EXPECT_LANDINGPAGE=/#/activity/SHARKON2025/view/
AW_EXPECT_LANDINGPAGE=/#/activity/HOST-EXAMPLE/view/
AW_HEALTH_STRICT_FILEOPS=0
AW_MONITORED_WINDOWS_HOST=<WINDOWS_HOST>
AW_MONITORED_WINDOWS_HOSTNAME=SHARKON2025
AW_MONITORED_WINDOWS_HOSTNAME=HOST-EXAMPLE
AW_RUS_HEALTH_WORKTIME_API=http://127.0.0.1:5610
AW_RUS_HEALTH_STATE_DIR=/var/lib/activitywatch/health
AW_RUS_HEALTH_VALIDATION_DIR=/var/lib/activitywatch/health/windows-validation
AW_RUS_HEALTH_SESSION_EVENTS_MAX_AGE_SECONDS=86400
AW_RUS_HEALTH_GUARD_MAX_AGE_SECONDS=300
AW_RUS_HEALTH_GUARD_REQUIRED=1
AW_RUS_HEALTH_RDP_TCP_REQUIRED=true
AW_RUS_HEALTH_WRAPPER_TIMEOUT_SECONDS=90
AW_RUS_SLO_AW_BASE=http://127.0.0.1:5600
AW_RUS_SLO_WORKTIME_BASE=http://127.0.0.1:5610
AW_RUS_SLO_TARGET_PERCENT=99.97
AW_BROWSER_SMOKE_AW_BASE=http://127.0.0.1:5600
AW_BROWSER_SMOKE_WORKTIME_BASE=http://127.0.0.1:5610
AW_BROWSER_SMOKE_HOST=SHARKON2025
AW_BROWSER_SMOKE_HOST=HOST-EXAMPLE
AW_BROWSER_SMOKE_OUTPUT_DIR=/var/lib/activitywatch/browser-smoke
AW_BROWSER_SMOKE_KEEP_RUNS=24
AW_BROWSER_SMOKE_RETENTION_DAYS=1
AW_BROWSER_SMOKE_ENGINE=chromium-cli
AW_BROWSER_SMOKE_TIMEOUT_MS=20000
AW_BROWSER_SMOKE_RENDER_TIMEOUT_MS=15000
# Retention / cleanup policy. Cleanup must stay allowlisted and dry-run tested
# before changing these values in production.
AW_BACKUP_RETENTION_DAYS=7
AW_BACKUP_KEEP_LAST_DB=2
AW_BACKUP_KEEP_LAST_JSON=2
AW_TMP_ARCHIVE_RETENTION_DAYS=1
AW_TMP_WEBUI_RETENTION_DAYS=2
AW_DB_MAINTENANCE_RETENTION_DAYS=7
AW_DB_MAINTENANCE_CHUNK_SIZE=1000
DETMIR_READINESS_RETENTION_DAYS=30
# Hayabusa auto-case / alerting
AW_HAYABUSA_AUTO_CASE_ENABLED=true
AW_HAYABUSA_AUTO_CASE_MIN_SEVERITY=medium
@@ -79,6 +104,15 @@ AW_HAYABUSA_TELEGRAM_ENABLED=true
AW_HAYABUSA_TELEGRAM_MIN_SEVERITY=high
AW_HAYABUSA_TELEGRAM_BOT_TOKEN=
AW_HAYABUSA_TELEGRAM_CHAT_IDS=
AW_SECURITY_FINDING_INBOX_ENABLED=false
AW_SECURITY_FINDING_INBOX_REQUIRED=false
AW_SECURITY_FINDING_INBOX_BIN=/usr/local/bin/security-finding-inbox
AW_SECURITY_FINDING_INBOX_MIN_SEVERITY=medium
AW_SECURITY_FINDING_EXECUTOR_WORK_DIR=/var/lib/activitywatch/security-finding-executor
AW_SECURITY_FINDING_EXECUTOR_LOCK=/var/lock/aw-security-finding-executor.lock
AW_CONTAINMENT_ENGINE_BIN=/usr/local/bin/containment-engine
AW_CONTAINMENT_MANAGEMENT_ALLOWLIST=
AW_CONTAINMENT_BLOCKED_REMOTE_ADDRESSES=
# Integration Test Configuration
AW_INTEGRATION_TEST_ENABLED=false
+1 -1
View File
@@ -118,4 +118,4 @@ Production scheduled task on `SHARKON2025`:
Do not switch this task back to `SYSTEM` on the current RDP host: Task Scheduler starts `powershell.exe` under `SYSTEM`, but the process exits with `0xC0000142` before the upload script starts.
Server-side processing accepts Windows zip packages with backslash path separators and UTF-8 BOM in sidecar JSON. `aw-hayabusa-autoprocess` processes the full incoming queue after accepting a drop package, so stale incoming files from an earlier failed run are drained before the latest intake is recorded.
Server-side processing accepts Windows zip packages with backslash path separators and UTF-8 BOM in sidecar JSON. `aw-hayabusa-autoprocess` waits for a dropped ZIP to become size-stable and readable before accepting it, then processes the full incoming queue. This prevents `systemd.path` from quarantining an archive while SCP is still writing it. ZIPs that remain unreadable after the bounded wait are moved to `/opt/hayabusa/quarantine/drop` with `reason.json`.
+47 -11
View File
@@ -1,7 +1,29 @@
#!/bin/bash
# check-aw-full.sh - Полная проверка ActivityWatch: сервер + RDP-хост
# Сервер: 192.0.2.13:5600
# RDP-хост: 198.51.100.18 (HOST-EXAMPLE)
# Сервер: http://10.10.10.13:5600
# RDP-хост: 192.168.100.19 (logical host id SHARKON2025)
normalize_http_base() {
local value="${1:-}"
value="${value%/}"
case "$value" in
"") return 1 ;;
http://*|https://*) printf '%s' "$value" ;;
*) printf 'http://%s' "$value" ;;
esac
}
normalize_url_env() {
local name="$1"
local value="${!name:-}"
if [ -n "$value" ]; then
export "$name=$(normalize_http_base "$value")"
fi
}
normalize_url_env CHECK_AW_FULL_SERVER
normalize_url_env AW_SMOKE_AW_SERVER
normalize_url_env AW_SERVER
if [[ "${CHECK_AW_FULL_FORCE_LEGACY:-0}" != "1" ]]; then
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -16,12 +38,17 @@ if [[ "${CHECK_AW_FULL_FORCE_LEGACY:-0}" != "1" ]]; then
done
fi
SERVER="http://192.0.2.13:5600"
HOSTNAME_FILTER="HOST-EXAMPLE"
RDP_HOST="198.51.100.18"
SERVER="${CHECK_AW_FULL_SERVER:-${AW_SMOKE_AW_SERVER:-${AW_SERVER:-http://10.10.10.13:5600}}}"
HOSTNAME_FILTER="${CHECK_AW_FULL_HOST:-${AW_SMOKE_SOURCE_HOSTNAME:-${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-SHARKON2025}}}}"
RDP_HOST="${CHECK_AW_FULL_RDP_HOST:-${AW_SMOKE_WINDOWS_HOST:-${AW_WINDOWS_HOST:-192.168.100.19}}}"
NOW=$(date -u +%s)
HOST_INACTIVE=false
GUARD_HEALTHY=false
DLP_ENABLED="${AW_DLP_ENABLED:-${DETMIR_DLP_ENABLED:-true}}"
case "${DLP_ENABLED,,}" in
0|false|no|off) DLP_ENABLED=false ;;
*) DLP_ENABLED=true ;;
esac
classify_bucket_age() {
local bucket="$1"
@@ -100,7 +127,7 @@ echo ""
# 1. Проверка сервера
echo -e "${CYAN}--- 1. AW Server ($SERVER) ---${NC}"
echo -n " Connectivity... "
RESP=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/info" 2>&1)
RESP=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/info" 2>&1)
if [ $? -eq 0 ] && echo "$RESP" | jq -e '.version' > /dev/null 2>&1; then
VERSION=$(echo "$RESP" | jq -r '.version')
echo -e " ${GREEN}OK${NC} (aw-server $VERSION)"
@@ -110,7 +137,7 @@ else
fi
echo -n " CORS... "
CORS_RESP=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 -o /dev/null -w '%{http_code}' -H "Origin: http://192.0.2.13:5600" "$SERVER/api/0/settings/" 2>&1)
CORS_RESP=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 -o /dev/null -w '%{http_code}' -H "Origin: $SERVER" "$SERVER/api/0/settings/" 2>&1)
if [ "$CORS_RESP" = "200" ]; then
echo -e "${GREEN}OK${NC}"
else
@@ -119,7 +146,7 @@ fi
echo ""
# 1b. Context for inactive/event-driven classification
WORKTIME_EVENT_DATA=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-worktime-sessions_$HOSTNAME_FILTER/events?limit=1" 2>&1)
WORKTIME_EVENT_DATA=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-worktime-sessions_$HOSTNAME_FILTER/events?limit=1" 2>&1)
WORKTIME_TS=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null)
WORKTIME_ACTIVE=$(echo "$WORKTIME_EVENT_DATA" | jq -r '.[0].data.active // false' 2>/dev/null)
if [ -n "$WORKTIME_TS" ]; then
@@ -132,7 +159,7 @@ if [ -n "$WORKTIME_TS" ]; then
fi
fi
GUARD_EVENT_DATA=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-rus-collector-guard_$HOSTNAME_FILTER/events?limit=1" 2>&1)
GUARD_EVENT_DATA=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/aw-rus-collector-guard_$HOSTNAME_FILTER/events?limit=1" 2>&1)
GUARD_TS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].timestamp // ""' 2>/dev/null)
GUARD_STATUS=$(echo "$GUARD_EVENT_DATA" | jq -r '.[0].data.status // ""' 2>/dev/null)
GUARD_PROBLEMS=$(echo "$GUARD_EVENT_DATA" | jq -r '([.[0].data.problems[]?] | length) // 0' 2>/dev/null)
@@ -165,9 +192,12 @@ BUCKETS=(
for entry in "${BUCKETS[@]}"; do
bucket="${entry%%|*}"
label="${entry##*|}"
if [ "$DLP_ENABLED" = "false" ] && [[ "$bucket" == aw-dlp-* ]]; then
continue
fi
bucket_full="${bucket}_${HOSTNAME_FILTER}"
EVENT_DATA=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1)
EVENT_DATA=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1)
LAST_ID=$(echo "$EVENT_DATA" | jq '.[0].id // 0')
LAST_TS=$(echo "$EVENT_DATA" | jq -r '.[0].timestamp // "no events"')
@@ -200,6 +230,9 @@ for entry in "${BUCKETS[@]}"; do
printf " %-42s %-8s %-20s %b\n" "$label" "$LAST_ID" "$AGE" "$STATUS"
done
if [ "$DLP_ENABLED" = "false" ]; then
printf " %-42s %-8s %-20s %b\n" "DLP buckets" "-" "disabled" "${CYAN}SKIPPED${NC}"
fi
echo ""
# 3. Проверка RDP-хоста
@@ -230,8 +263,11 @@ DEAD_COUNT=0
for entry in "${BUCKETS[@]}"; do
bucket="${entry%%|*}"
if [ "$DLP_ENABLED" = "false" ] && [[ "$bucket" == aw-dlp-* ]]; then
continue
fi
bucket_full="${bucket}_${HOSTNAME_FILTER}"
EVENT_DATA=$(no_proxy=192.0.2.13 curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1)
EVENT_DATA=$(no_proxy='*' curl -s --connect-timeout 10 --max-time 15 "$SERVER/api/0/buckets/$bucket_full/events?limit=1" 2>&1)
LAST_TS=$(echo "$EVENT_DATA" | jq -r '.[0].timestamp // "no events"')
if [ "$LAST_TS" != "no events" ] && [ -n "$LAST_TS" ]; then
@@ -0,0 +1,13 @@
<clickhouse>
<users>
<aw_workforce_dict>
<no_password/>
<networks>
<ip>127.0.0.1</ip>
<ip>::1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</aw_workforce_dict>
</users>
</clickhouse>
+1
View File
@@ -13,6 +13,7 @@ services:
volumes:
- clickhouse_1c_data:/var/lib/clickhouse
- ./clickhouse/init:/docker-entrypoint-initdb.d:ro
- ./clickhouse/users.d/99-aw-workforce-dict.xml:/etc/clickhouse-server/users.d/99-aw-workforce-dict.xml:ro
volumes:
clickhouse_1c_data:
+3 -4
View File
@@ -18,13 +18,12 @@ fi
# shellcheck disable=SC1090
. "${ENV_FILE}"
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
query_max_age() {
local table="$1"
docker exec "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
-q "SELECT if(count()=0, -1, dateDiff('hour', max(ts), now())) FROM ${table}"
}
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
aw_1c_xml_escape() {
local value="$1"
value="${value//&/&amp;}"
value="${value//</&lt;}"
value="${value//>/&gt;}"
value="${value//\"/&quot;}"
value="${value//\'/&apos;}"
printf '%s' "${value}"
}
aw_1c_clickhouse_client() {
local container="${CH_CONTAINER:-${AW_1C_CLICKHOUSE_CONTAINER:-aw-rus-1c-clickhouse}}"
local local_cfg remote_cfg status
if [[ -z "${CLICKHOUSE_USER:-}" ]]; then
echo "CLICKHOUSE_USER is required" >&2
return 1
fi
if [[ -z "${CLICKHOUSE_DB:-}" ]]; then
echo "CLICKHOUSE_DB is required" >&2
return 1
fi
if [[ -z "${CLICKHOUSE_PASSWORD+x}" ]]; then
echo "CLICKHOUSE_PASSWORD is required" >&2
return 1
fi
local_cfg="$(mktemp "${TMPDIR:-/tmp}/aw-1c-clickhouse-client.XXXXXX.xml")"
chmod 0600 "${local_cfg}"
remote_cfg="/tmp/aw-1c-clickhouse-client.$(date +%s).$$.xml"
{
printf '<config>\n'
printf ' <user>%s</user>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_USER}")"
printf ' <password>%s</password>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_PASSWORD}")"
printf ' <database>%s</database>\n' "$(aw_1c_xml_escape "${CLICKHOUSE_DB}")"
printf '</config>\n'
} > "${local_cfg}"
if ! docker exec -i "${container}" sh -c 'umask 077 && cat > "$1"' sh "${remote_cfg}" < "${local_cfg}"; then
rm -f "${local_cfg}"
echo "failed to stage ClickHouse client config in container" >&2
return 1
fi
rm -f "${local_cfg}"
status=0
docker exec -i "${container}" clickhouse-client --config-file "${remote_cfg}" "$@" || status=$?
docker exec "${container}" rm -f "${remote_cfg}" >/dev/null 2>&1 || true
return "${status}"
}
+7
View File
@@ -8,4 +8,11 @@
- core tables: `365` дней
- detections/cases/timeline: `365` дней или по регламенту ИБ
Важно: это policy target, а не заявление о текущем автоматическом TTL. В
`clickhouse/init/*.sql` сейчас нет TTL clauses, поэтому production cleanup для
ClickHouse должен внедряться отдельной staged migration после backup, dry-run
оценки объема и operator/customer approval.
Сводная политика хранения всего контура: `../../docs/RETENTION_POLICY_RU.md`.
Если регуляторика требует больше, меняется отдельно от Grafana UI.
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# shellcheck disable=SC2119
set -euo pipefail
ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
@@ -21,18 +22,20 @@ if ! docker ps --format '{{.Names}}' | grep -qx "${CH_CONTAINER}"; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_registry_bindings_refresh.sh"
@@ -41,5 +44,4 @@ docker exec -i "${CH_CONTAINER}" clickhouse-client \
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"
@@ -15,17 +15,19 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
"${VENV}/bin/python" "${ROOT}/ai/refresh_company_registry_bindings.py" \
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"
+7 -16
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# shellcheck disable=SC2119
set -euo pipefail
ROOT="${AW_1C_ROOT:-/opt/activitywatch/clickhouse-1c}"
@@ -40,6 +41,8 @@ fi
# shellcheck disable=SC1090
. "${ENV_FILE}"
# shellcheck source=clickhouse-1c/ops/clickhouse-client-safe.sh
. "${ROOT}/ops/clickhouse-client-safe.sh"
if [[ "${RUN_MCP_TOOLKIT_EXTRACT_BEFORE_INGEST}" == "1" ]]; then
"${VENV}/bin/python" "${ROOT}/etl/extract_1c_mcp_toolkit.py" --config "${CONFIG}"
@@ -49,31 +52,19 @@ fi
"${VENV}/bin/python" "${ROOT}/etl/load_1c_exports.py" --config "${CONFIG}"
"${VENV}/bin/python" "${ROOT}/etl/load_company_registry_xlsx.py" --config "${CONFIG}" --landing "${ROOT}/landing/registry"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/build_entity_timeline.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/clickhouse/init/04_company_intelligence.sql"
"${ROOT}/ops/run_company_registry_bindings_refresh.sh"
"${ROOT}/ops/run_company_intelligence_refresh.sh"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/insert_detections.sql"
docker exec -i "${CH_CONTAINER}" clickhouse-client \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}" \
aw_1c_clickhouse_client \
< "${ROOT}/detections/open_cases_from_detections.sql"
if [[ "${RUN_MANAGER_BRIEF_AFTER_INGEST}" == "1" ]]; then
+2 -2
View File
@@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
@@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
export CLICKHOUSE_HOST="${CH_RUNTIME_HOST}"
@@ -43,7 +44,6 @@ while (( attempt <= RETRIES )); do
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"; then
exit 0
fi
+2 -2
View File
@@ -19,8 +19,8 @@ if [[ ! -x "${VENV}/bin/python" ]]; then
exit 1
fi
# shellcheck disable=SC1090
set -a
# shellcheck source=/dev/null
. "${ENV_FILE}"
set +a
@@ -28,6 +28,7 @@ CH_RUNTIME_HOST="${AW_1C_CLICKHOUSE_RUNTIME_HOST:-${CLICKHOUSE_HOST}}"
if [[ "${CH_RUNTIME_HOST}" == "clickhouse" ]]; then
CH_RUNTIME_HOST="127.0.0.1"
fi
: "${CLICKHOUSE_PORT:?CLICKHOUSE_PORT is required}"
exec 9>"${LOCK_FILE}"
if ! flock -w "${LOCK_WAIT_SEC}" 9; then
@@ -41,7 +42,6 @@ while (( attempt <= RETRIES )); do
--host "${CH_RUNTIME_HOST}" \
--port "${CLICKHOUSE_PORT}" \
--user "${CLICKHOUSE_USER}" \
--password "${CLICKHOUSE_PASSWORD}" \
--database "${CLICKHOUSE_DB}"; then
exit 0
fi
+5
View File
@@ -0,0 +1,5 @@
CLICKHOUSE_DB=aw_workforce
CLICKHOUSE_HTTP_BIND=127.0.0.1
CLICKHOUSE_PORT=8124
CLICKHOUSE_NATIVE_BIND=127.0.0.1
CLICKHOUSE_NATIVE_PORT=9001
+2
View File
@@ -0,0 +1,2 @@
.env
.local/
+155
View File
@@ -0,0 +1,155 @@
# ClickHouse Workforce analytics for AWatch-rus / DetMir
Этот каталог содержит воспроизводимый ClickHouse-слой для привязки событий
AWatch-rus к оргструктуре, классификации приложений и доменов, а также для
быстрых агрегатов Grafana.
Слой не заменяет `clickhouse-1c/`. Это отдельный контур для workforce/web
аналитики ActivityWatch-событий.
## Состав
- `docker-compose.yml` - локальный ClickHouse scaffold.
- `clickhouse/init/00_database.sql` - база `aw_workforce`.
- `clickhouse/init/01_raw_tables.sql` - нормализованные staging tables для
window/browser events.
- `clickhouse/init/02_dimensions_dictionaries.sql` - dimension tables и
ClickHouse Dictionaries.
- `clickhouse/init/03_materialized_views.sql` - агрегированная таблица и
materialized views для Grafana.
- `clickhouse/init/04_quality_views.sql` - views контроля unknown-зон.
- `sample/seed_demo.sql` - минимальные demo-данные для smoke-проверки.
- `sample/seed_sharkon2025_p3.sql` - первая реальная привязка
`SHARKON2025/sharkon2025/user1/tsj`.
- `ops/run_smoke.sh` - локальный smoke для DDL, dictionaries и агрегатов.
- `ops/aw-workforce-ingest.service` / `.timer` - production timer для
инкрементальной загрузки.
- `ops/aw-workforce-ingest.env.example` - переменные окружения loader-а.
- `catalog/*.tsv` - управляемые администратором справочники.
- `ops/apply_catalogs.sh` - полная загрузка справочников, reload dictionaries,
опциональный rebuild агрегатов.
- `ops/report_unknowns.sh` - быстрый отчет top unknown users/processes/domains.
## Быстрый старт
```bash
cd clickhouse-workforce
docker compose up -d
./ops/run_smoke.sh
```
Локальный scaffold не задает `CLICKHOUSE_USER/PASSWORD` через Docker entrypoint:
это оставляет штатный dev-доступ ClickHouse без пароля и не ломает
`SOURCE(CLICKHOUSE(...))` у dictionaries. Файл
`clickhouse/users.d/99-aw-workforce-local.xml` разрешает HTTP-запросы от Docker
host, а HTTP/native порты по умолчанию привязаны только к `127.0.0.1`.
Скрипт применяет SQL в правильном порядке, загружает demo seed и проверяет:
- статус dictionaries;
- наличие hourly aggregate rows;
- daily productivity view;
- unknown quality views.
`sample/seed_demo.sql` добавляет демонстрационные строки. Для чистого повтора
локального smoke пересоздайте volume:
```bash
docker compose down -v
docker compose up -d
./ops/run_smoke.sh
```
## Production порядок
1. Реальные источники `aw_window_events` и `aw_browser_events` для
`SHARKON2025` подтверждены:
`docs/clickhouse/AW_WORKFORCE_SOURCES_SHARKON2025_RU.md`.
2. Настроить ingest из ActivityWatch/exporter в staging tables.
3. Загрузить `dim_workstation_user`, `dim_application_category`,
`dim_domain_category`.
4. Проверить `system.dictionaries`.
5. Включить materialized views.
6. Перевести Grafana на `agg_workforce_productivity_hourly` и
`v_workforce_productivity_daily`.
Исправление справочников не пересчитывает старые агрегаты автоматически.
Для исторических периодов нужен backfill по регламенту из
`docs/clickhouse/DICTIONARIES_IMPLEMENTATION_PLAN_RU.md`.
## Live ingest P2/P3
Rust loader находится в `adk-rust/crates/aw-workforce-ingest`.
Пример загрузки bounded-окна из живого AW API в локальный ClickHouse:
```bash
cargo run --manifest-path ../adk-rust/Cargo.toml -p aw-workforce-ingest -- \
--aw-url http://10.10.10.13:5600/api/0 \
--clickhouse-url http://127.0.0.1:8124 \
--host SHARKON2025 \
--hours 24 \
--json
```
Применение первой привязки P3:
```bash
docker exec -i aw-rus-workforce-clickhouse clickhouse-client --multiquery \
< sample/seed_sharkon2025_p3.sql
```
## Production ingest P4
В штатном режиме loader запускается без `--since/--until`: он читает
`AW_WORKFORCE_STATE_PATH`, берет `last_end - AW_WORKFORCE_OVERLAP_SECONDS`,
загружает bounded range и атомарно сохраняет новый `last_end`. Повторная
загрузка overlap-окна не удваивает данные, потому что loader перед вставкой
проверяет `source_bucket + source_event_id`.
Runtime-файлы:
```bash
cd clickhouse-workforce
sudo bash ./ops/bootstrap_runtime.sh
sudo install -m 0755 ../adk-rust/target/release/aw-workforce-ingest \
/usr/local/bin/aw-workforce-ingest
sudo editor /etc/activitywatch/aw-workforce-ingest.env
sudo systemctl enable --now aw-workforce-ingest.timer
```
Ручная production-проверка одного цикла:
```bash
sudo systemctl start aw-workforce-ingest.service
sudo journalctl -u aw-workforce-ingest.service -n 80 --no-pager
```
## Admin workflow справочников P5
Справочники ведутся через `catalog/*.tsv`. Это полный source of truth:
`ops/apply_catalogs.sh` очищает dimension tables, загружает TSV, reload-ит
dictionaries и, если нужно, пересобирает агрегаты.
Посмотреть слепые зоны:
```bash
./ops/report_unknowns.sh
```
Добавить или изменить категорию:
```bash
editor catalog/application_categories.tsv
REBUILD_AGGREGATES=1 ./ops/apply_catalogs.sh
```
Убрать запись из отчетов без потери аудита: поставить `is_active=0` в TSV и
запустить:
```bash
REBUILD_AGGREGATES=1 ./ops/apply_catalogs.sh
```
Если менялись только future-facing справочники и старые агрегаты пересчитывать
не нужно, можно запустить без `REBUILD_AGGREGATES=1`.
@@ -0,0 +1,118 @@
TRUNCATE TABLE aw_workforce.agg_workforce_productivity_hourly;
INSERT INTO aw_workforce.agg_workforce_productivity_hourly
SELECT
toStartOfHour(event_time) AS bucket_start,
toDate(event_time) AS event_date,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
'unknown'
) AS branch,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
'unknown'
) AS department,
'desktop' AS activity_type,
if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, 'unknown'),
'unknown'
) AS category,
if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, 'unknown'),
'unknown'
) AS productivity_class,
toUInt64(sum(duration_sec)) AS duration_sec,
toUInt64(count()) AS event_count,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
0,
1
))) AS unknown_subject_events,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') NOT IN ('', 'unknown'),
0,
1
))) AS unknown_category_events
FROM aw_workforce.aw_window_events
GROUP BY
bucket_start,
event_date,
branch,
department,
activity_type,
category,
productivity_class;
INSERT INTO aw_workforce.agg_workforce_productivity_hourly
WITH
lowerUTF8(
domain(if(position(url, '://') = 0, concat('http://', url), url))
) AS domain_name
SELECT
toStartOfHour(event_time) AS bucket_start,
toDate(event_time) AS event_date,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
'unknown'
) AS branch,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
'unknown'
) AS department,
'browser' AS activity_type,
if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, 'unknown'),
'unknown'
) AS category,
if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, 'unknown'),
'unknown'
) AS productivity_class,
toUInt64(sum(duration_sec)) AS duration_sec,
toUInt64(count()) AS event_count,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
0,
1
))) AS unknown_subject_events,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') NOT IN ('', 'unknown'),
0,
1
))) AS unknown_category_events
FROM aw_workforce.aw_browser_events
WHERE domain_name != ''
GROUP BY
bucket_start,
event_date,
branch,
department,
activity_type,
category,
productivity_class;
+43
View File
@@ -0,0 +1,43 @@
# Workforce catalogs
Эти TSV-файлы являются source of truth для справочников `aw_workforce`.
## Операции администратора
- Добавить категорию: добавить строку в соответствующий `*.tsv`, поставить
`is_active=1`, запустить `ops/apply_catalogs.sh`.
- Изменить категорию: изменить строку в `*.tsv`, запустить
`REBUILD_AGGREGATES=1 ops/apply_catalogs.sh`.
- Удалить категорию из отчетов: либо удалить строку из `*.tsv`, либо оставить
строку для аудита и поставить `is_active=0`, затем запустить
`REBUILD_AGGREGATES=1 ops/apply_catalogs.sh`.
`is_active=0` трактуется отчетами как `unknown`: запись остается видимой в
каталоге, но не используется для обогащения.
## Файлы
- `workstation_users.tsv` - привязка `host_name + user_login` к оргструктуре.
- `application_categories.tsv` - классификация desktop processes.
- `domain_categories.tsv` - классификация browser domains.
Формат: `TabSeparatedWithNames`, первая строка - имена колонок. Не используйте
tab-символы внутри значений.
## Таксономия РФ baseline
Baseline `catalog-ru-20260623` делит домены и приложения на рабочие для РФ
категории: `1c`, `edo_reporting`, `reporting`, `banking`, `government`,
`procurement`, `business_reference`, `legal_reference`, `mail`, `office`,
`pdf`, `browser`, `cloud_docs`, `communication`, `developer`, `admin_tool`,
`remote_admin`, `security_crypto`, `search`, `maps_reference`, `marketplace`,
`news`, `social`, `media`, `gaming`, `system`.
Правило ведения: справочник хранит только точные ключи. Для приложений это
нормализованный `process_name` в нижнем регистре, например `1cv8c.exe`; для
web - точный host из URL, например `online.sbis.ru`. Wildcard-строки не
используются, потому что ClickHouse Dictionary выполняет точный lookup.
`productivity_class` держите в одном из значений: `productive`, `neutral`,
`non_productive`, `unknown`. Для облаков, мессенджеров, AI и внешней почты
ставьте `risk_level=medium`, если нужна последующая DLP/policy проверка.
@@ -0,0 +1,97 @@
process_name application_name vendor category productivity_class risk_level is_system source comment is_active
1cv8.exe 1C:Enterprise 1C 1c productive low 0 catalog-ru-20260623 1C thick client 1
1cv8c.exe 1C:Enterprise client 1C 1c productive low 0 catalog-ru-20260623 1C current RDP workload 1
1cv8s.exe 1C:Enterprise starter 1C 1c productive low 0 catalog-ru-20260623 1C server or launcher process 1
1cestart.exe 1C:Enterprise launcher 1C 1c productive low 0 catalog-ru-20260623 1C launcher 1
1cv8u.exe 1C:Enterprise updater 1C 1c productive low 0 catalog-ru-20260623 1C updater 1
sbis.exe SBIS SBIS edo_reporting productive low 0 catalog-ru-20260623 EDO and reporting client 1
sbisplugin.exe SBIS Plugin SBIS edo_reporting productive low 0 catalog-ru-20260623 SBIS browser integration 1
saby.exe Saby SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby business client 1
sabyapps.exe Saby Apps SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby desktop apps 1
diadoc.exe Diadoc Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic document exchange client 1
kontur.plugin.exe Kontur Plugin Kontur edo_reporting productive low 0 catalog-ru-20260623 Kontur browser integration 1
extern.exe Kontur Extern Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic reporting client 1
cryptoarm.exe CryptoARM Digital Technologies security_crypto productive low 0 catalog-ru-20260623 Crypto signing tool 1
cryptcp.exe CryptoPro CryptCP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro command line signing tool 1
cprocsp.exe CryptoPro CSP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro CSP utility 1
certmgr.exe Certificate Manager Microsoft security_crypto productive low 0 catalog-ru-20260623 Certificate management 1
cadesplugin.exe CAdES Plugin CryptoPro security_crypto productive low 0 catalog-ru-20260623 Browser crypto plugin 1
chrome.exe Google Chrome Google browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
msedge.exe Microsoft Edge Microsoft browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
browser.exe Yandex Browser Yandex browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
firefox.exe Mozilla Firefox Mozilla browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
opera.exe Opera Opera browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
iexplore.exe Internet Explorer Microsoft browser neutral medium 0 catalog-ru-20260623 Legacy browser; domains classify productivity 1
outlook.exe Microsoft Outlook Microsoft mail productive low 0 catalog-ru-20260623 Business mail client 1
thunderbird.exe Thunderbird Mozilla mail productive low 0 catalog-ru-20260623 Mail client 1
winword.exe Microsoft Word Microsoft office productive low 0 catalog-ru-20260623 Office document editor 1
excel.exe Microsoft Excel Microsoft office productive low 0 catalog-ru-20260623 Spreadsheet editor 1
powerpnt.exe Microsoft PowerPoint Microsoft office productive low 0 catalog-ru-20260623 Presentation editor 1
onenote.exe Microsoft OneNote Microsoft office productive low 0 catalog-ru-20260623 Notes and documents 1
msaccess.exe Microsoft Access Microsoft office productive low 0 catalog-ru-20260623 Database office tool 1
soffice.bin LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite 1
soffice.exe LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite launcher 1
swriter.exe LibreOffice Writer The Document Foundation office productive low 0 catalog-ru-20260623 Office document editor 1
scalc.exe LibreOffice Calc The Document Foundation office productive low 0 catalog-ru-20260623 Spreadsheet editor 1
simpress.exe LibreOffice Impress The Document Foundation office productive low 0 catalog-ru-20260623 Presentation editor 1
notepad.exe Notepad Microsoft office neutral low 0 catalog-ru-20260623 Text editor 1
notepad++.exe Notepad++ Notepad++ office productive low 0 catalog-ru-20260623 Text and config editor 1
acrord32.exe Adobe Acrobat Reader Adobe pdf productive low 0 catalog-ru-20260623 PDF reader 1
acrobat.exe Adobe Acrobat Adobe pdf productive low 0 catalog-ru-20260623 PDF editor 1
foxitpdfreader.exe Foxit PDF Reader Foxit pdf productive low 0 catalog-ru-20260623 PDF reader 1
pdfxedit.exe PDF-XChange Editor Tracker Software pdf productive low 0 catalog-ru-20260623 PDF editor 1
7zfm.exe 7-Zip File Manager 7-Zip archive productive low 0 catalog-ru-20260623 Archive manager 1
7zg.exe 7-Zip GUI 7-Zip archive productive low 0 catalog-ru-20260623 Archive tool 1
7z.exe 7-Zip CLI 7-Zip archive productive low 0 catalog-ru-20260623 Archive command line tool 1
winrar.exe WinRAR RARLAB archive productive low 0 catalog-ru-20260623 Archive manager 1
explorer.exe Windows Explorer Microsoft system neutral low 1 catalog-ru-20260623 Windows shell and file manager 1
dwm.exe Desktop Window Manager Microsoft system neutral low 1 catalog-ru-20260623 Windows desktop compositor 1
taskhostw.exe Task Host Microsoft system neutral low 1 catalog-ru-20260623 Windows task host 1
applicationframehost.exe Application Frame Host Microsoft system neutral low 1 catalog-ru-20260623 Windows app frame host 1
startmenuexperiencehost.exe Start Menu Experience Host Microsoft system neutral low 1 catalog-ru-20260623 Windows Start menu 1
searchapp.exe Windows Search Microsoft system neutral low 1 catalog-ru-20260623 Windows search UI 1
systemsettings.exe Windows Settings Microsoft system neutral low 1 catalog-ru-20260623 Windows settings 1
rdpclip.exe RDP Clipboard Microsoft system neutral low 1 catalog-ru-20260623 RDP clipboard process 1
conhost.exe Console Host Microsoft system neutral low 1 catalog-ru-20260623 Windows console host 1
taskmgr.exe Task Manager Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1
mmc.exe Microsoft Management Console Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console 1
eventvwr.exe Event Viewer Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1
services.exe Windows Services Microsoft admin_tool productive low 1 catalog-ru-20260623 Service control manager 1
compmgmtlauncher.exe Computer Management Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console launcher 1
regedit.exe Registry Editor Microsoft admin_tool productive medium 1 catalog-ru-20260623 Registry editor 1
cmd.exe Command Prompt Microsoft admin_tool productive medium 1 catalog-ru-20260623 Command shell 1
powershell.exe Windows PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1
pwsh.exe PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1
windowsterminal.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal 1
wt.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal launcher 1
mstsc.exe Remote Desktop Connection Microsoft remote_admin productive low 0 catalog-ru-20260623 RDP client 1
putty.exe PuTTY PuTTY remote_admin productive low 0 catalog-ru-20260623 SSH client 1
winscp.exe WinSCP WinSCP remote_admin productive low 0 catalog-ru-20260623 SFTP and SCP client 1
filezilla.exe FileZilla FileZilla remote_admin productive low 0 catalog-ru-20260623 File transfer client 1
procexp.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
procexp64.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
procmon.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
procmon64.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
telegram.exe Telegram Telegram communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1
whatsapp.exe WhatsApp Meta communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1
teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1
ms-teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1
zoom.exe Zoom Zoom communication productive low 0 catalog-ru-20260623 Business communication 1
skype.exe Skype Microsoft communication neutral medium 0 catalog-ru-20260623 Messenger 1
discord.exe Discord Discord social non_productive medium 0 catalog-ru-20260623 Social and gaming communication 1
yandexdisk.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
yandexdisk2.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
onedrive.exe OneDrive Microsoft cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
dropbox.exe Dropbox Dropbox cloud_docs neutral medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
code.exe Visual Studio Code Microsoft developer productive low 0 catalog-ru-20260623 Code editor and admin scripting 1
devenv.exe Visual Studio Microsoft developer productive low 0 catalog-ru-20260623 Development IDE 1
git.exe Git Git developer productive low 0 catalog-ru-20260623 Version control CLI 1
docker desktop.exe Docker Desktop Docker developer productive low 0 catalog-ru-20260623 Container development tool 1
vlc.exe VLC VideoLAN media non_productive medium 0 catalog-ru-20260623 Media player 1
wmplayer.exe Windows Media Player Microsoft media non_productive medium 0 catalog-ru-20260623 Media player 1
spotify.exe Spotify Spotify media non_productive medium 0 catalog-ru-20260623 Music streaming 1
steam.exe Steam Valve gaming non_productive high 0 catalog-ru-20260623 Gaming platform 1
gamebar.exe Xbox Game Bar Microsoft gaming non_productive high 0 catalog-ru-20260623 Gaming overlay 1
securityhealthsystray.exe Windows Security Microsoft security_crypto neutral low 1 catalog-ru-20260623 Windows security UI 1
avgui.exe AVG Antivirus AVG security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1
avpui.exe Kaspersky Kaspersky security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1
1 process_name application_name vendor category productivity_class risk_level is_system source comment is_active
2 1cv8.exe 1C:Enterprise 1C 1c productive low 0 catalog-ru-20260623 1C thick client 1
3 1cv8c.exe 1C:Enterprise client 1C 1c productive low 0 catalog-ru-20260623 1C current RDP workload 1
4 1cv8s.exe 1C:Enterprise starter 1C 1c productive low 0 catalog-ru-20260623 1C server or launcher process 1
5 1cestart.exe 1C:Enterprise launcher 1C 1c productive low 0 catalog-ru-20260623 1C launcher 1
6 1cv8u.exe 1C:Enterprise updater 1C 1c productive low 0 catalog-ru-20260623 1C updater 1
7 sbis.exe SBIS SBIS edo_reporting productive low 0 catalog-ru-20260623 EDO and reporting client 1
8 sbisplugin.exe SBIS Plugin SBIS edo_reporting productive low 0 catalog-ru-20260623 SBIS browser integration 1
9 saby.exe Saby SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby business client 1
10 sabyapps.exe Saby Apps SBIS edo_reporting productive low 0 catalog-ru-20260623 Saby desktop apps 1
11 diadoc.exe Diadoc Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic document exchange client 1
12 kontur.plugin.exe Kontur Plugin Kontur edo_reporting productive low 0 catalog-ru-20260623 Kontur browser integration 1
13 extern.exe Kontur Extern Kontur edo_reporting productive low 0 catalog-ru-20260623 Electronic reporting client 1
14 cryptoarm.exe CryptoARM Digital Technologies security_crypto productive low 0 catalog-ru-20260623 Crypto signing tool 1
15 cryptcp.exe CryptoPro CryptCP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro command line signing tool 1
16 cprocsp.exe CryptoPro CSP CryptoPro security_crypto productive low 0 catalog-ru-20260623 CryptoPro CSP utility 1
17 certmgr.exe Certificate Manager Microsoft security_crypto productive low 0 catalog-ru-20260623 Certificate management 1
18 cadesplugin.exe CAdES Plugin CryptoPro security_crypto productive low 0 catalog-ru-20260623 Browser crypto plugin 1
19 chrome.exe Google Chrome Google browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
20 msedge.exe Microsoft Edge Microsoft browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
21 browser.exe Yandex Browser Yandex browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
22 firefox.exe Mozilla Firefox Mozilla browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
23 opera.exe Opera Opera browser neutral low 0 catalog-ru-20260623 Browser; domains classify productivity 1
24 iexplore.exe Internet Explorer Microsoft browser neutral medium 0 catalog-ru-20260623 Legacy browser; domains classify productivity 1
25 outlook.exe Microsoft Outlook Microsoft mail productive low 0 catalog-ru-20260623 Business mail client 1
26 thunderbird.exe Thunderbird Mozilla mail productive low 0 catalog-ru-20260623 Mail client 1
27 winword.exe Microsoft Word Microsoft office productive low 0 catalog-ru-20260623 Office document editor 1
28 excel.exe Microsoft Excel Microsoft office productive low 0 catalog-ru-20260623 Spreadsheet editor 1
29 powerpnt.exe Microsoft PowerPoint Microsoft office productive low 0 catalog-ru-20260623 Presentation editor 1
30 onenote.exe Microsoft OneNote Microsoft office productive low 0 catalog-ru-20260623 Notes and documents 1
31 msaccess.exe Microsoft Access Microsoft office productive low 0 catalog-ru-20260623 Database office tool 1
32 soffice.bin LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite 1
33 soffice.exe LibreOffice The Document Foundation office productive low 0 catalog-ru-20260623 Office suite launcher 1
34 swriter.exe LibreOffice Writer The Document Foundation office productive low 0 catalog-ru-20260623 Office document editor 1
35 scalc.exe LibreOffice Calc The Document Foundation office productive low 0 catalog-ru-20260623 Spreadsheet editor 1
36 simpress.exe LibreOffice Impress The Document Foundation office productive low 0 catalog-ru-20260623 Presentation editor 1
37 notepad.exe Notepad Microsoft office neutral low 0 catalog-ru-20260623 Text editor 1
38 notepad++.exe Notepad++ Notepad++ office productive low 0 catalog-ru-20260623 Text and config editor 1
39 acrord32.exe Adobe Acrobat Reader Adobe pdf productive low 0 catalog-ru-20260623 PDF reader 1
40 acrobat.exe Adobe Acrobat Adobe pdf productive low 0 catalog-ru-20260623 PDF editor 1
41 foxitpdfreader.exe Foxit PDF Reader Foxit pdf productive low 0 catalog-ru-20260623 PDF reader 1
42 pdfxedit.exe PDF-XChange Editor Tracker Software pdf productive low 0 catalog-ru-20260623 PDF editor 1
43 7zfm.exe 7-Zip File Manager 7-Zip archive productive low 0 catalog-ru-20260623 Archive manager 1
44 7zg.exe 7-Zip GUI 7-Zip archive productive low 0 catalog-ru-20260623 Archive tool 1
45 7z.exe 7-Zip CLI 7-Zip archive productive low 0 catalog-ru-20260623 Archive command line tool 1
46 winrar.exe WinRAR RARLAB archive productive low 0 catalog-ru-20260623 Archive manager 1
47 explorer.exe Windows Explorer Microsoft system neutral low 1 catalog-ru-20260623 Windows shell and file manager 1
48 dwm.exe Desktop Window Manager Microsoft system neutral low 1 catalog-ru-20260623 Windows desktop compositor 1
49 taskhostw.exe Task Host Microsoft system neutral low 1 catalog-ru-20260623 Windows task host 1
50 applicationframehost.exe Application Frame Host Microsoft system neutral low 1 catalog-ru-20260623 Windows app frame host 1
51 startmenuexperiencehost.exe Start Menu Experience Host Microsoft system neutral low 1 catalog-ru-20260623 Windows Start menu 1
52 searchapp.exe Windows Search Microsoft system neutral low 1 catalog-ru-20260623 Windows search UI 1
53 systemsettings.exe Windows Settings Microsoft system neutral low 1 catalog-ru-20260623 Windows settings 1
54 rdpclip.exe RDP Clipboard Microsoft system neutral low 1 catalog-ru-20260623 RDP clipboard process 1
55 conhost.exe Console Host Microsoft system neutral low 1 catalog-ru-20260623 Windows console host 1
56 taskmgr.exe Task Manager Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1
57 mmc.exe Microsoft Management Console Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console 1
58 eventvwr.exe Event Viewer Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative diagnostics 1
59 services.exe Windows Services Microsoft admin_tool productive low 1 catalog-ru-20260623 Service control manager 1
60 compmgmtlauncher.exe Computer Management Microsoft admin_tool productive low 1 catalog-ru-20260623 Administrative console launcher 1
61 regedit.exe Registry Editor Microsoft admin_tool productive medium 1 catalog-ru-20260623 Registry editor 1
62 cmd.exe Command Prompt Microsoft admin_tool productive medium 1 catalog-ru-20260623 Command shell 1
63 powershell.exe Windows PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1
64 pwsh.exe PowerShell Microsoft admin_tool productive medium 1 catalog-ru-20260623 Administrative shell 1
65 windowsterminal.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal 1
66 wt.exe Windows Terminal Microsoft admin_tool productive medium 0 catalog-ru-20260623 Administrative terminal launcher 1
67 mstsc.exe Remote Desktop Connection Microsoft remote_admin productive low 0 catalog-ru-20260623 RDP client 1
68 putty.exe PuTTY PuTTY remote_admin productive low 0 catalog-ru-20260623 SSH client 1
69 winscp.exe WinSCP WinSCP remote_admin productive low 0 catalog-ru-20260623 SFTP and SCP client 1
70 filezilla.exe FileZilla FileZilla remote_admin productive low 0 catalog-ru-20260623 File transfer client 1
71 procexp.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
72 procexp64.exe Process Explorer Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
73 procmon.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
74 procmon64.exe Process Monitor Microsoft admin_tool productive medium 0 catalog-ru-20260623 Advanced diagnostics 1
75 telegram.exe Telegram Telegram communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1
76 whatsapp.exe WhatsApp Meta communication neutral medium 0 catalog-ru-20260623 Messenger; business use depends on policy 1
77 teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1
78 ms-teams.exe Microsoft Teams Microsoft communication productive low 0 catalog-ru-20260623 Business communication 1
79 zoom.exe Zoom Zoom communication productive low 0 catalog-ru-20260623 Business communication 1
80 skype.exe Skype Microsoft communication neutral medium 0 catalog-ru-20260623 Messenger 1
81 discord.exe Discord Discord social non_productive medium 0 catalog-ru-20260623 Social and gaming communication 1
82 yandexdisk.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
83 yandexdisk2.exe Yandex Disk Yandex cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
84 onedrive.exe OneDrive Microsoft cloud_docs productive medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
85 dropbox.exe Dropbox Dropbox cloud_docs neutral medium 0 catalog-ru-20260623 Cloud storage; data handling policy applies 1
86 code.exe Visual Studio Code Microsoft developer productive low 0 catalog-ru-20260623 Code editor and admin scripting 1
87 devenv.exe Visual Studio Microsoft developer productive low 0 catalog-ru-20260623 Development IDE 1
88 git.exe Git Git developer productive low 0 catalog-ru-20260623 Version control CLI 1
89 docker desktop.exe Docker Desktop Docker developer productive low 0 catalog-ru-20260623 Container development tool 1
90 vlc.exe VLC VideoLAN media non_productive medium 0 catalog-ru-20260623 Media player 1
91 wmplayer.exe Windows Media Player Microsoft media non_productive medium 0 catalog-ru-20260623 Media player 1
92 spotify.exe Spotify Spotify media non_productive medium 0 catalog-ru-20260623 Music streaming 1
93 steam.exe Steam Valve gaming non_productive high 0 catalog-ru-20260623 Gaming platform 1
94 gamebar.exe Xbox Game Bar Microsoft gaming non_productive high 0 catalog-ru-20260623 Gaming overlay 1
95 securityhealthsystray.exe Windows Security Microsoft security_crypto neutral low 1 catalog-ru-20260623 Windows security UI 1
96 avgui.exe AVG Antivirus AVG security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1
97 avpui.exe Kaspersky Kaspersky security_crypto neutral low 0 catalog-ru-20260623 Endpoint security UI 1
@@ -0,0 +1,128 @@
domain site_name category productivity_class risk_level business_allowed source comment is_active
intranet.local Internal portal internal_service productive low 1 catalog-ru-20260623 Internal work portal 1
dbo.sevnb.ru SEVNB online banking banking productive low 1 catalog-ru-20260623 Banking workflow seen in RDP title stream 1
sberbank.ru Sberbank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
online.sberbank.ru Sberbank Online banking productive low 1 catalog-ru-20260623 Russian banking web client 1
sbi.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking 1
business-online.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking legacy host 1
vtb.ru VTB banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
dbo.vtb.ru VTB Business Online banking productive low 1 catalog-ru-20260623 VTB business banking 1
alfabank.ru Alfa-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
ibank.alfabank.ru Alfa-Bank Business banking productive low 1 catalog-ru-20260623 Alfa business banking 1
tbank.ru T-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
business.tbank.ru T-Business banking productive low 1 catalog-ru-20260623 T-Bank business banking 1
tinkoff.ru Tinkoff banking productive low 1 catalog-ru-20260623 T-Bank legacy domain 1
business.tinkoff.ru Tinkoff Business banking productive low 1 catalog-ru-20260623 T-Bank business legacy domain 1
tochka.com Tochka Bank banking productive low 1 catalog-ru-20260623 Business banking 1
bspb.ru Bank Saint Petersburg banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
dbo.bspb.ru Bank Saint Petersburg Business banking productive low 1 catalog-ru-20260623 Business banking 1
gazprombank.ru Gazprombank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
bankuralsib.ru Uralsib banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
rshb.ru Russian Agricultural Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
modulbank.ru Modulbank banking productive low 1 catalog-ru-20260623 Business banking 1
gosuslugi.ru Gosuslugi government productive low 1 catalog-ru-20260623 Russian government services 1
esia.gosuslugi.ru ESIA government productive low 1 catalog-ru-20260623 Government authentication 1
lk.gosuslugi.ru Gosuslugi account government productive low 1 catalog-ru-20260623 Government services account 1
nalog.gov.ru FNS government productive low 1 catalog-ru-20260623 Russian tax service 1
service.nalog.ru FNS services government productive low 1 catalog-ru-20260623 Russian tax service portals 1
egrul.nalog.ru FNS EGRUL government productive low 1 catalog-ru-20260623 Company registry lookup 1
pb.nalog.ru FNS transparent business government productive low 1 catalog-ru-20260623 Tax counterparty lookup 1
lkfl2.nalog.ru FNS personal account government productive low 1 catalog-ru-20260623 Tax account 1
lkul.nalog.ru FNS legal entity account government productive low 1 catalog-ru-20260623 Tax account for legal entities 1
fssp.gov.ru FSSP government productive low 1 catalog-ru-20260623 Bailiff service lookup 1
rosreestr.gov.ru Rosreestr government productive low 1 catalog-ru-20260623 Real estate registry 1
kad.arbitr.ru Arbitration cases government productive low 1 catalog-ru-20260623 Arbitration case database 1
my.arbitr.ru Arbitration e-filing government productive low 1 catalog-ru-20260623 Arbitration electronic filing 1
sudrf.ru Courts of Russia government productive low 1 catalog-ru-20260623 Court information 1
zakupki.gov.ru Unified procurement government productive low 1 catalog-ru-20260623 State procurement 1
torgi.gov.ru Torgi.gov government productive low 1 catalog-ru-20260623 State auctions and property tenders 1
mos.ru Mos.ru government productive low 1 catalog-ru-20260623 Moscow city services 1
cbr.ru Bank of Russia government productive low 1 catalog-ru-20260623 Central bank reference data 1
rosstat.gov.ru Rosstat government productive low 1 catalog-ru-20260623 Statistics and reporting 1
sfr.gov.ru Social Fund government productive low 1 catalog-ru-20260623 Social fund services 1
trudvsem.ru Trudvsem government productive low 1 catalog-ru-20260623 Labor and vacancies portal 1
kontur.ru Kontur edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business checks 1
extern.kontur.ru Kontur Extern edo_reporting productive low 1 catalog-ru-20260623 Electronic reporting 1
diadoc.kontur.ru Diadoc edo_reporting productive low 1 catalog-ru-20260623 Electronic document exchange 1
focus.kontur.ru Kontur Focus business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
normativ.kontur.ru Kontur Normativ business_reference productive low 1 catalog-ru-20260623 Legal and accounting reference 1
sbis.ru SBIS edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business services 1
online.sbis.ru SBIS Online edo_reporting productive low 1 catalog-ru-20260623 EDO reporting portal 1
saby.ru Saby edo_reporting productive low 1 catalog-ru-20260623 SBIS/Saby services 1
ofd.ru OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
ofd.yandex.ru Yandex OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
1-ofd.ru 1-OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
sberbank-ast.ru Sberbank AST procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
roseltorg.ru Roseltorg procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
rts-tender.ru RTS Tender procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
fabrikant.ru Fabrikant procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
spark-interfax.ru SPARK Interfax business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
rusprofile.ru Rusprofile business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
checko.ru Checko business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
zachestnyibiznes.ru Za Chestny Biznes business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
list-org.com List-Org business_reference neutral medium 1 catalog-ru-20260623 Counterparty checks with mixed data quality 1
garant.ru Garant legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1
consultant.ru ConsultantPlus legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1
mail.yandex.ru Yandex Mail mail productive low 1 catalog-ru-20260623 Business mail when used with org account 1
360.yandex.ru Yandex 360 mail productive low 1 catalog-ru-20260623 Business mail and documents 1
mail.ru Mail.ru mail neutral medium 1 catalog-ru-20260623 Mail portal; classify account policy separately if needed 1
e.mail.ru Mail.ru webmail mail neutral medium 1 catalog-ru-20260623 Mail.ru webmail 1
biz.mail.ru Mail.ru business mail productive low 1 catalog-ru-20260623 Business mail 1
corp.mail.ru Mail.ru corporate mail productive low 1 catalog-ru-20260623 Corporate services 1
gmail.com Gmail mail neutral medium 1 catalog-ru-20260623 External mail 1
outlook.office.com Outlook Web mail productive low 1 catalog-ru-20260623 Business mail 1
yandex.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1
ya.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1
google.com Google Search search neutral low 1 catalog-ru-20260623 Search portal 1
2gis.ru 2GIS maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1
maps.yandex.ru Yandex Maps maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1
dadata.ru DaData business_reference productive low 1 catalog-ru-20260623 Address and company reference 1
disk.yandex.ru Yandex Disk cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
docs.yandex.ru Yandex Documents cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
cloud.mail.ru Cloud Mail.ru cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1
docs.google.com Google Docs cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
drive.google.com Google Drive cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1
office.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
microsoft365.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
web.telegram.org Telegram Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1
t.me Telegram links communication neutral medium 1 catalog-ru-20260623 Messenger links 1
telegram.org Telegram communication neutral medium 1 catalog-ru-20260623 Messenger 1
web.whatsapp.com WhatsApp Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1
whatsapp.com WhatsApp communication neutral medium 1 catalog-ru-20260623 Messenger 1
teams.microsoft.com Microsoft Teams communication productive low 1 catalog-ru-20260623 Business communication 1
meet.google.com Google Meet communication productive low 1 catalog-ru-20260623 Business communication 1
zoom.us Zoom communication productive low 1 catalog-ru-20260623 Business communication 1
github.com GitHub developer productive low 1 catalog-ru-20260623 Development and documentation workflow 1
gitlab.com GitLab developer productive low 1 catalog-ru-20260623 Development workflow 1
stackoverflow.com Stack Overflow developer productive low 1 catalog-ru-20260623 Technical reference 1
learn.microsoft.com Microsoft Learn developer productive low 1 catalog-ru-20260623 Technical documentation 1
docs.microsoft.com Microsoft Docs developer productive low 1 catalog-ru-20260623 Technical documentation legacy host 1
microsoft.com Microsoft developer neutral low 1 catalog-ru-20260623 Vendor documentation and downloads 1
docker.com Docker developer productive low 1 catalog-ru-20260623 Developer tooling documentation 1
software.qnap.com QNAP Software work_service neutral low 1 catalog-ru-20260623 Vendor software/download portal for admin work 1
cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1
www.cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1
chatgpt.com ChatGPT ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
chat.openai.com ChatGPT legacy ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
giga.chat GigaChat ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
alice.yandex.ru Yandex Alice ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
wildberries.ru Wildberries marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
ozon.ru Ozon marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
market.yandex.ru Yandex Market marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
avito.ru Avito marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business use depends on policy 1
dns-shop.ru DNS Shop marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1
citilink.ru Citilink marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1
vk.com VK social non_productive medium 0 catalog-ru-20260623 Social network; whitelist business communities separately if needed 1
ok.ru Odnoklassniki social non_productive medium 0 catalog-ru-20260623 Social network 1
dzen.ru Dzen news non_productive medium 0 catalog-ru-20260623 News/feed portal 1
rbc.ru RBC news neutral medium 1 catalog-ru-20260623 Business news; review by department policy 1
ria.ru RIA Novosti news neutral medium 1 catalog-ru-20260623 News; review by department policy 1
lenta.ru Lenta news non_productive medium 0 catalog-ru-20260623 News/feed portal 1
youtube.com YouTube media non_productive medium 0 catalog-ru-20260623 Video platform; business use requires URL-level exception 1
rutube.ru Rutube media non_productive medium 0 catalog-ru-20260623 Video platform 1
vkvideo.ru VK Video media non_productive medium 0 catalog-ru-20260623 Video platform 1
music.yandex.ru Yandex Music media non_productive medium 0 catalog-ru-20260623 Music streaming 1
kinopoisk.ru Kinopoisk media non_productive medium 0 catalog-ru-20260623 Entertainment streaming 1
twitch.tv Twitch media non_productive high 0 catalog-ru-20260623 Entertainment streaming 1
example.com Example Domain test_reference neutral low 0 catalog-ru-20260623 Test/reference domain 1
unknown.example Unknown example test_reference neutral low 0 catalog-ru-20260623 Test/reference domain from demo data 1
1 domain site_name category productivity_class risk_level business_allowed source comment is_active
2 intranet.local Internal portal internal_service productive low 1 catalog-ru-20260623 Internal work portal 1
3 dbo.sevnb.ru SEVNB online banking banking productive low 1 catalog-ru-20260623 Banking workflow seen in RDP title stream 1
4 sberbank.ru Sberbank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
5 online.sberbank.ru Sberbank Online banking productive low 1 catalog-ru-20260623 Russian banking web client 1
6 sbi.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking 1
7 business-online.sberbank.ru SberBusiness banking productive low 1 catalog-ru-20260623 Sber business banking legacy host 1
8 vtb.ru VTB banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
9 dbo.vtb.ru VTB Business Online banking productive low 1 catalog-ru-20260623 VTB business banking 1
10 alfabank.ru Alfa-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
11 ibank.alfabank.ru Alfa-Bank Business banking productive low 1 catalog-ru-20260623 Alfa business banking 1
12 tbank.ru T-Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
13 business.tbank.ru T-Business banking productive low 1 catalog-ru-20260623 T-Bank business banking 1
14 tinkoff.ru Tinkoff banking productive low 1 catalog-ru-20260623 T-Bank legacy domain 1
15 business.tinkoff.ru Tinkoff Business banking productive low 1 catalog-ru-20260623 T-Bank business legacy domain 1
16 tochka.com Tochka Bank banking productive low 1 catalog-ru-20260623 Business banking 1
17 bspb.ru Bank Saint Petersburg banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
18 dbo.bspb.ru Bank Saint Petersburg Business banking productive low 1 catalog-ru-20260623 Business banking 1
19 gazprombank.ru Gazprombank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
20 bankuralsib.ru Uralsib banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
21 rshb.ru Russian Agricultural Bank banking productive low 1 catalog-ru-20260623 Russian banking and business services 1
22 modulbank.ru Modulbank banking productive low 1 catalog-ru-20260623 Business banking 1
23 gosuslugi.ru Gosuslugi government productive low 1 catalog-ru-20260623 Russian government services 1
24 esia.gosuslugi.ru ESIA government productive low 1 catalog-ru-20260623 Government authentication 1
25 lk.gosuslugi.ru Gosuslugi account government productive low 1 catalog-ru-20260623 Government services account 1
26 nalog.gov.ru FNS government productive low 1 catalog-ru-20260623 Russian tax service 1
27 service.nalog.ru FNS services government productive low 1 catalog-ru-20260623 Russian tax service portals 1
28 egrul.nalog.ru FNS EGRUL government productive low 1 catalog-ru-20260623 Company registry lookup 1
29 pb.nalog.ru FNS transparent business government productive low 1 catalog-ru-20260623 Tax counterparty lookup 1
30 lkfl2.nalog.ru FNS personal account government productive low 1 catalog-ru-20260623 Tax account 1
31 lkul.nalog.ru FNS legal entity account government productive low 1 catalog-ru-20260623 Tax account for legal entities 1
32 fssp.gov.ru FSSP government productive low 1 catalog-ru-20260623 Bailiff service lookup 1
33 rosreestr.gov.ru Rosreestr government productive low 1 catalog-ru-20260623 Real estate registry 1
34 kad.arbitr.ru Arbitration cases government productive low 1 catalog-ru-20260623 Arbitration case database 1
35 my.arbitr.ru Arbitration e-filing government productive low 1 catalog-ru-20260623 Arbitration electronic filing 1
36 sudrf.ru Courts of Russia government productive low 1 catalog-ru-20260623 Court information 1
37 zakupki.gov.ru Unified procurement government productive low 1 catalog-ru-20260623 State procurement 1
38 torgi.gov.ru Torgi.gov government productive low 1 catalog-ru-20260623 State auctions and property tenders 1
39 mos.ru Mos.ru government productive low 1 catalog-ru-20260623 Moscow city services 1
40 cbr.ru Bank of Russia government productive low 1 catalog-ru-20260623 Central bank reference data 1
41 rosstat.gov.ru Rosstat government productive low 1 catalog-ru-20260623 Statistics and reporting 1
42 sfr.gov.ru Social Fund government productive low 1 catalog-ru-20260623 Social fund services 1
43 trudvsem.ru Trudvsem government productive low 1 catalog-ru-20260623 Labor and vacancies portal 1
44 kontur.ru Kontur edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business checks 1
45 extern.kontur.ru Kontur Extern edo_reporting productive low 1 catalog-ru-20260623 Electronic reporting 1
46 diadoc.kontur.ru Diadoc edo_reporting productive low 1 catalog-ru-20260623 Electronic document exchange 1
47 focus.kontur.ru Kontur Focus business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
48 normativ.kontur.ru Kontur Normativ business_reference productive low 1 catalog-ru-20260623 Legal and accounting reference 1
49 sbis.ru SBIS edo_reporting productive low 1 catalog-ru-20260623 EDO reporting and business services 1
50 online.sbis.ru SBIS Online edo_reporting productive low 1 catalog-ru-20260623 EDO reporting portal 1
51 saby.ru Saby edo_reporting productive low 1 catalog-ru-20260623 SBIS/Saby services 1
52 ofd.ru OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
53 ofd.yandex.ru Yandex OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
54 1-ofd.ru 1-OFD reporting productive low 1 catalog-ru-20260623 Fiscal data operator 1
55 sberbank-ast.ru Sberbank AST procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
56 roseltorg.ru Roseltorg procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
57 rts-tender.ru RTS Tender procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
58 fabrikant.ru Fabrikant procurement productive low 1 catalog-ru-20260623 Electronic trading platform 1
59 spark-interfax.ru SPARK Interfax business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
60 rusprofile.ru Rusprofile business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
61 checko.ru Checko business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
62 zachestnyibiznes.ru Za Chestny Biznes business_reference productive low 1 catalog-ru-20260623 Counterparty checks 1
63 list-org.com List-Org business_reference neutral medium 1 catalog-ru-20260623 Counterparty checks with mixed data quality 1
64 garant.ru Garant legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1
65 consultant.ru ConsultantPlus legal_reference productive low 1 catalog-ru-20260623 Legal reference system 1
66 mail.yandex.ru Yandex Mail mail productive low 1 catalog-ru-20260623 Business mail when used with org account 1
67 360.yandex.ru Yandex 360 mail productive low 1 catalog-ru-20260623 Business mail and documents 1
68 mail.ru Mail.ru mail neutral medium 1 catalog-ru-20260623 Mail portal; classify account policy separately if needed 1
69 e.mail.ru Mail.ru webmail mail neutral medium 1 catalog-ru-20260623 Mail.ru webmail 1
70 biz.mail.ru Mail.ru business mail productive low 1 catalog-ru-20260623 Business mail 1
71 corp.mail.ru Mail.ru corporate mail productive low 1 catalog-ru-20260623 Corporate services 1
72 gmail.com Gmail mail neutral medium 1 catalog-ru-20260623 External mail 1
73 outlook.office.com Outlook Web mail productive low 1 catalog-ru-20260623 Business mail 1
74 yandex.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1
75 ya.ru Yandex search neutral low 1 catalog-ru-20260623 Search portal 1
76 google.com Google Search search neutral low 1 catalog-ru-20260623 Search portal 1
77 2gis.ru 2GIS maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1
78 maps.yandex.ru Yandex Maps maps_reference productive low 1 catalog-ru-20260623 Maps and organization reference 1
79 dadata.ru DaData business_reference productive low 1 catalog-ru-20260623 Address and company reference 1
80 disk.yandex.ru Yandex Disk cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
81 docs.yandex.ru Yandex Documents cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
82 cloud.mail.ru Cloud Mail.ru cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1
83 docs.google.com Google Docs cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
84 drive.google.com Google Drive cloud_docs productive medium 1 catalog-ru-20260623 Cloud storage; data handling policy applies 1
85 office.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
86 microsoft365.com Microsoft 365 cloud_docs productive medium 1 catalog-ru-20260623 Cloud documents; data handling policy applies 1
87 web.telegram.org Telegram Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1
88 t.me Telegram links communication neutral medium 1 catalog-ru-20260623 Messenger links 1
89 telegram.org Telegram communication neutral medium 1 catalog-ru-20260623 Messenger 1
90 web.whatsapp.com WhatsApp Web communication neutral medium 1 catalog-ru-20260623 Messenger; business use depends on policy 1
91 whatsapp.com WhatsApp communication neutral medium 1 catalog-ru-20260623 Messenger 1
92 teams.microsoft.com Microsoft Teams communication productive low 1 catalog-ru-20260623 Business communication 1
93 meet.google.com Google Meet communication productive low 1 catalog-ru-20260623 Business communication 1
94 zoom.us Zoom communication productive low 1 catalog-ru-20260623 Business communication 1
95 github.com GitHub developer productive low 1 catalog-ru-20260623 Development and documentation workflow 1
96 gitlab.com GitLab developer productive low 1 catalog-ru-20260623 Development workflow 1
97 stackoverflow.com Stack Overflow developer productive low 1 catalog-ru-20260623 Technical reference 1
98 learn.microsoft.com Microsoft Learn developer productive low 1 catalog-ru-20260623 Technical documentation 1
99 docs.microsoft.com Microsoft Docs developer productive low 1 catalog-ru-20260623 Technical documentation legacy host 1
100 microsoft.com Microsoft developer neutral low 1 catalog-ru-20260623 Vendor documentation and downloads 1
101 docker.com Docker developer productive low 1 catalog-ru-20260623 Developer tooling documentation 1
102 software.qnap.com QNAP Software work_service neutral low 1 catalog-ru-20260623 Vendor software/download portal for admin work 1
103 cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1
104 www.cryptopro.ru CryptoPro security_crypto productive low 1 catalog-ru-20260623 CryptoPro documentation and downloads 1
105 chatgpt.com ChatGPT ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
106 chat.openai.com ChatGPT legacy ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
107 giga.chat GigaChat ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
108 alice.yandex.ru Yandex Alice ai_assistant productive medium 1 catalog-ru-20260623 AI assistant; data handling policy applies 1
109 wildberries.ru Wildberries marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
110 ozon.ru Ozon marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
111 market.yandex.ru Yandex Market marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business purchases possible 1
112 avito.ru Avito marketplace neutral medium 1 catalog-ru-20260623 Marketplace; business use depends on policy 1
113 dns-shop.ru DNS Shop marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1
114 citilink.ru Citilink marketplace neutral medium 1 catalog-ru-20260623 IT and office equipment purchases 1
115 vk.com VK social non_productive medium 0 catalog-ru-20260623 Social network; whitelist business communities separately if needed 1
116 ok.ru Odnoklassniki social non_productive medium 0 catalog-ru-20260623 Social network 1
117 dzen.ru Dzen news non_productive medium 0 catalog-ru-20260623 News/feed portal 1
118 rbc.ru RBC news neutral medium 1 catalog-ru-20260623 Business news; review by department policy 1
119 ria.ru RIA Novosti news neutral medium 1 catalog-ru-20260623 News; review by department policy 1
120 lenta.ru Lenta news non_productive medium 0 catalog-ru-20260623 News/feed portal 1
121 youtube.com YouTube media non_productive medium 0 catalog-ru-20260623 Video platform; business use requires URL-level exception 1
122 rutube.ru Rutube media non_productive medium 0 catalog-ru-20260623 Video platform 1
123 vkvideo.ru VK Video media non_productive medium 0 catalog-ru-20260623 Video platform 1
124 music.yandex.ru Yandex Music media non_productive medium 0 catalog-ru-20260623 Music streaming 1
125 kinopoisk.ru Kinopoisk media non_productive medium 0 catalog-ru-20260623 Entertainment streaming 1
126 twitch.tv Twitch media non_productive high 0 catalog-ru-20260623 Entertainment streaming 1
127 example.com Example Domain test_reference neutral low 0 catalog-ru-20260623 Test/reference domain 1
128 unknown.example Unknown example test_reference neutral low 0 catalog-ru-20260623 Test/reference domain from demo data 1
@@ -0,0 +1,8 @@
host_name user_login user_domain employee_id employee_name department branch position source is_active
SHARKON2025 user1 sharkon2025 sharkon2025\\user1 user1 tsj tsj RDP user catalog-p5 1
SHARKON2025 user4 sharkon2025 sharkon2025\\user4 user4 tsj tsj RDP user catalog-p5 1
SHARKON2025 user5 sharkon2025 sharkon2025\\user5 user5 tsj tsj RDP user catalog-p5 1
SHARKON2025 Администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator catalog-p5 1
SHARKON2025 администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator legacy lowercase alias catalog-p5 1
ws-001 ivanov corp E001 Иванов И.И. Бухгалтерия Филиал 1 Бухгалтер demo 1
ws-002 petrova corp E002 Петрова П.П. Операционный отдел Филиал 1 Оператор demo 1
1 host_name user_login user_domain employee_id employee_name department branch position source is_active
2 SHARKON2025 user1 sharkon2025 sharkon2025\\user1 user1 tsj tsj RDP user catalog-p5 1
3 SHARKON2025 user4 sharkon2025 sharkon2025\\user4 user4 tsj tsj RDP user catalog-p5 1
4 SHARKON2025 user5 sharkon2025 sharkon2025\\user5 user5 tsj tsj RDP user catalog-p5 1
5 SHARKON2025 Администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator catalog-p5 1
6 SHARKON2025 администратор sharkon2025 sharkon2025\\Администратор Администратор it_admin tsj Windows administrator legacy lowercase alias catalog-p5 1
7 ws-001 ivanov corp E001 Иванов И.И. Бухгалтерия Филиал 1 Бухгалтер demo 1
8 ws-002 petrova corp E002 Петрова П.П. Операционный отдел Филиал 1 Оператор demo 1
@@ -0,0 +1 @@
CREATE DATABASE IF NOT EXISTS aw_workforce;
@@ -0,0 +1,32 @@
CREATE TABLE IF NOT EXISTS aw_workforce.aw_window_events
(
event_time DateTime,
host_name String,
user_login String,
process_name String,
window_title String,
duration_sec UInt32,
source_bucket LowCardinality(String),
source_event_id String,
ingested_at DateTime DEFAULT now()
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(event_time)
ORDER BY (event_time, host_name, user_login, process_name, source_event_id);
CREATE TABLE IF NOT EXISTS aw_workforce.aw_browser_events
(
event_time DateTime,
host_name String,
user_login String,
browser_name String,
url String,
title String,
duration_sec UInt32,
source_bucket LowCardinality(String),
source_event_id String,
ingested_at DateTime DEFAULT now()
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(event_time)
ORDER BY (event_time, host_name, user_login, browser_name, source_event_id);
@@ -0,0 +1,145 @@
CREATE TABLE IF NOT EXISTS aw_workforce.dim_workstation_user
(
host_name String,
user_login String,
user_domain String,
employee_id String,
employee_name String,
department String,
branch String,
position String,
source LowCardinality(String),
is_active UInt8 DEFAULT 1,
updated_at DateTime DEFAULT now()
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY (host_name, user_login);
CREATE TABLE IF NOT EXISTS aw_workforce.dim_application_category
(
process_name String,
application_name String,
vendor String,
category LowCardinality(String),
productivity_class LowCardinality(String),
risk_level LowCardinality(String),
is_system UInt8 DEFAULT 0,
is_active UInt8 DEFAULT 1,
source LowCardinality(String),
comment String,
updated_at DateTime DEFAULT now()
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY process_name;
CREATE TABLE IF NOT EXISTS aw_workforce.dim_domain_category
(
domain String,
site_name String,
category LowCardinality(String),
productivity_class LowCardinality(String),
risk_level LowCardinality(String),
business_allowed UInt8 DEFAULT 0,
source LowCardinality(String),
comment String,
is_active UInt8 DEFAULT 1,
updated_at DateTime DEFAULT now()
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY domain;
CREATE TABLE IF NOT EXISTS aw_workforce.dim_url_rule
(
rule_id String,
domain String,
path_pattern String,
category LowCardinality(String),
productivity_class LowCardinality(String),
risk_level LowCardinality(String),
priority UInt16 DEFAULT 100,
is_active UInt8 DEFAULT 1,
comment String,
updated_at DateTime DEFAULT now()
)
ENGINE = ReplacingMergeTree(updated_at)
ORDER BY (domain, priority, rule_id);
DROP DICTIONARY IF EXISTS aw_workforce.dict_workstation_user;
CREATE DICTIONARY aw_workforce.dict_workstation_user
(
host_name String,
user_login String,
user_domain String,
employee_id String,
employee_name String,
department String,
branch String,
position String,
is_active UInt8
)
PRIMARY KEY host_name, user_login
SOURCE(CLICKHOUSE(
USER 'aw_workforce_dict'
PASSWORD ''
DB 'aw_workforce'
TABLE 'dim_workstation_user'
))
LAYOUT(COMPLEX_KEY_HASHED())
LIFETIME(MIN 3600 MAX 86400);
DROP DICTIONARY IF EXISTS aw_workforce.dict_application_category;
CREATE DICTIONARY aw_workforce.dict_application_category
(
process_name String,
application_name String,
vendor String,
category String,
productivity_class String,
risk_level String,
is_system UInt8,
is_active UInt8
)
PRIMARY KEY process_name
SOURCE(CLICKHOUSE(
USER 'aw_workforce_dict'
PASSWORD ''
DB 'aw_workforce'
TABLE 'dim_application_category'
))
LAYOUT(HASHED())
LIFETIME(MIN 3600 MAX 86400);
DROP DICTIONARY IF EXISTS aw_workforce.dict_domain_category;
CREATE DICTIONARY aw_workforce.dict_domain_category
(
domain String,
site_name String,
category String,
productivity_class String,
risk_level String,
business_allowed UInt8,
is_active UInt8
)
PRIMARY KEY domain
SOURCE(CLICKHOUSE(
USER 'aw_workforce_dict'
PASSWORD ''
DB 'aw_workforce'
TABLE 'dim_domain_category'
))
LAYOUT(HASHED())
LIFETIME(MIN 3600 MAX 86400);
@@ -0,0 +1,158 @@
CREATE TABLE IF NOT EXISTS aw_workforce.agg_workforce_productivity_hourly
(
bucket_start DateTime,
event_date Date,
branch LowCardinality(String),
department LowCardinality(String),
activity_type LowCardinality(String),
category LowCardinality(String),
productivity_class LowCardinality(String),
duration_sec UInt64,
event_count UInt64,
unknown_subject_events UInt64,
unknown_category_events UInt64
)
ENGINE = SummingMergeTree((
duration_sec,
event_count,
unknown_subject_events,
unknown_category_events
))
PARTITION BY toYYYYMM(event_date)
ORDER BY (
event_date,
bucket_start,
branch,
department,
activity_type,
productivity_class,
category
);
DROP VIEW IF EXISTS aw_workforce.mv_desktop_productivity_hourly;
CREATE MATERIALIZED VIEW aw_workforce.mv_desktop_productivity_hourly
TO aw_workforce.agg_workforce_productivity_hourly
AS
SELECT
toStartOfHour(event_time) AS bucket_start,
toDate(event_time) AS event_date,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
'unknown'
) AS branch,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
'unknown'
) AS department,
'desktop' AS activity_type,
if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, 'unknown'),
'unknown'
) AS category,
if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_application_category', 'productivity_class', process_name, 'unknown'),
'unknown'
) AS productivity_class,
toUInt64(sum(duration_sec)) AS duration_sec,
toUInt64(count()) AS event_count,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
0,
1
))) AS unknown_subject_events,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_application_category', 'is_active', process_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_application_category', 'category', process_name, '') NOT IN ('', 'unknown'),
0,
1
))) AS unknown_category_events
FROM aw_workforce.aw_window_events
GROUP BY
bucket_start,
event_date,
branch,
department,
activity_type,
category,
productivity_class;
DROP VIEW IF EXISTS aw_workforce.mv_browser_productivity_hourly;
CREATE MATERIALIZED VIEW aw_workforce.mv_browser_productivity_hourly
TO aw_workforce.agg_workforce_productivity_hourly
AS
WITH
lowerUTF8(
domain(if(position(url, '://') = 0, concat('http://', url), url))
) AS domain_name
SELECT
toStartOfHour(event_time) AS bucket_start,
toDate(event_time) AS event_date,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'branch', (host_name, user_login), 'unknown'),
'unknown'
) AS branch,
if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), '') != '',
dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'department', (host_name, user_login), 'unknown'),
'unknown'
) AS department,
'browser' AS activity_type,
if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, 'unknown'),
'unknown'
) AS category,
if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, '') != '',
dictGetStringOrDefault('aw_workforce.dict_domain_category', 'productivity_class', domain_name, 'unknown'),
'unknown'
) AS productivity_class,
toUInt64(sum(duration_sec)) AS duration_sec,
toUInt64(count()) AS event_count,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_workstation_user', 'is_active', (host_name, user_login), 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_workstation_user', 'employee_name', (host_name, user_login), '') != '',
0,
1
))) AS unknown_subject_events,
toUInt64(sum(if(
dictGetUInt8OrDefault('aw_workforce.dict_domain_category', 'is_active', domain_name, 0) = 1
AND dictGetStringOrDefault('aw_workforce.dict_domain_category', 'category', domain_name, '') NOT IN ('', 'unknown'),
0,
1
))) AS unknown_category_events
FROM aw_workforce.aw_browser_events
WHERE domain_name != ''
GROUP BY
bucket_start,
event_date,
branch,
department,
activity_type,
category,
productivity_class;

Some files were not shown because too many files have changed in this diff Show More