diff --git a/.github/workflows/operational-maturity.yml b/.github/workflows/operational-maturity.yml new file mode 100644 index 0000000..5d93f79 --- /dev/null +++ b/.github/workflows/operational-maturity.yml @@ -0,0 +1,73 @@ +name: Operational maturity + +# GitHub Actions is public mirror validation only. +# Live DetMir production checks must be run explicitly by an operator with +# private network access and must not enable heavy DLP, Loki or always-on +# Velociraptor. + +on: + push: + pull_request: + workflow_dispatch: + inputs: + live: + description: "Run live endpoint contract if AWATCH_OPS_LIVE_URL is configured" + required: false + default: "false" + schedule: + - cron: "41 3 * * 2" + +permissions: + contents: read + +jobs: + offline-operational-maturity: + name: Offline operational maturity + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: false + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Script syntax + run: node --check scripts/operational-maturity-check.mjs + + - name: Operational maturity harness + run: node scripts/operational-maturity-check.mjs --json + + live-operational-contract: + name: Live operational contract + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && inputs.live == 'true' + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: false + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Run live contract when URL is available + env: + AWATCH_OPS_LIVE_URL: ${{ secrets.AWATCH_OPS_LIVE_URL }} + run: | + if [[ -z "${AWATCH_OPS_LIVE_URL}" ]]; then + echo "skipped: AWATCH_OPS_LIVE_URL secret is not configured" + exit 0 + fi + node scripts/operational-maturity-check.mjs --json --live diff --git a/configs/operational-maturity-contract.json b/configs/operational-maturity-contract.json new file mode 100644 index 0000000..e4c74dc --- /dev/null +++ b/configs/operational-maturity-contract.json @@ -0,0 +1,143 @@ +{ + "version": "2026-07-01.operational-maturity-v1", + "apiCompatibility": { + "openapiPath": "adk-rust/crates/detmir-portal/src/contracts/openapi.json", + "requiredOpenApiPaths": { + "/contracts": ["get"], + "/contracts/openapi.json": ["get"], + "/health": ["get"], + "/operator": ["get"], + "/reports": ["get"], + "/workforce/kpi/explain": ["get"], + "/ueba": ["get"], + "/risk/narrative": ["get"], + "/actions": ["get"], + "/readiness/latest": ["get"], + "/telemetry": ["post"] + }, + "requiredSchemas": [ + "ContractIndex", + "JsonObject", + "ActionCenterResponse", + "RiskNarrative", + "UebaPayload", + "WorkforceKpiExplain" + ], + "runtimeEndpoints": [ + "/healthz", + "/readyz", + "/version", + "/metrics" + ], + "runtimeSourcePath": "adk-rust/crates/detmir-portal/src/main.rs" + }, + "integrationFixtures": [ + { + "path": "/healthz", + "fixture": "docs/fixtures/operational-maturity/portal-healthz.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["status", "service", "schema_version"], + "equals": { "status": "ok", "schema_version": "pilot-v1" } + }, + { + "path": "/readyz", + "fixture": "docs/fixtures/operational-maturity/portal-readyz.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["status", "checks.portal", "checks.dlp_runtime"], + "equals": { "status": "ready", "checks.dlp_runtime.status": "skipped" } + }, + { + "path": "/version", + "fixture": "docs/fixtures/operational-maturity/portal-version.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["app_version", "schema_version", "environment"], + "equals": { "schema_version": "pilot-v1" } + }, + { + "path": "/reports/worktime/management", + "fixture": "docs/fixtures/operational-maturity/worktime-management-ok.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["status", "stale", "report_cache_hit", "aw_query_duration_ms"], + "equals": { "status": "OK", "stale": false } + }, + { + "path": "/reports/worktime/management?fault=aw_timeout", + "fixture": "docs/fixtures/operational-maturity/worktime-management-degraded.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["status", "reason", "stale", "report_stale_served", "aw_query_timeout_count"], + "equals": { "status": "DEGRADED", "reason": "report_unavailable", "stale": false } + }, + { + "path": "/security/finding-inbox", + "fixture": "docs/fixtures/operational-maturity/security-finding-inbox-shadow.json", + "status": 200, + "contentType": "application/json", + "requiredFields": ["mode", "auto_apply", "workflow"], + "equals": { "mode": "shadow", "auto_apply": false } + } + ], + "observability": { + "metricsFixture": "docs/fixtures/operational-maturity/portal-metrics.prom", + "requiredMetrics": [ + "awatch_http_requests_total", + "awatch_http_request_duration_seconds_sum", + "awatch_http_request_duration_seconds_count", + "awatch_report_requests_total", + "awatch_report_cache_hits_total", + "awatch_report_cache_misses_total", + "awatch_report_cache_stale_hits_total", + "awatch_reports_generated_total", + "awatch_ingestion_records_total", + "awatch_ingestion_rejected_total", + "awatch_role_denied_total", + "awatch_readyz_status" + ], + "sourcePath": "adk-rust/crates/detmir-portal/src/production/metrics.rs", + "diagnosticHeaders": ["x-request-id", "x-correlation-id"] + }, + "configValidation": { + "jsonFiles": [ + "configs/worktime-interpretation-policy.example.json", + "configs/detmir-workforce-policy.example.json", + "adk-rust/crates/detmir-portal/src/contracts/openapi.json", + "docs/fixtures/operational-maturity/portal-healthz.json", + "docs/fixtures/operational-maturity/portal-readyz.json", + "docs/fixtures/operational-maturity/portal-version.json", + "docs/fixtures/operational-maturity/worktime-management-ok.json", + "docs/fixtures/operational-maturity/worktime-management-degraded.json", + "docs/fixtures/operational-maturity/security-finding-inbox-shadow.json" + ], + "systemdDirs": [ + "ops/systemd", + "clickhouse-1c/ops" + ], + "clickhouseInitDir": "clickhouse-1c/clickhouse/init", + "yamlFiles": [ + "configs/detmir-ueba-risk-policy.example.yaml", + "clickhouse-1c/etl/config.example.yml", + "clickhouse-1c/grafana/provisioning/datasources/clickhouse.yml", + "clickhouse-1c/grafana/provisioning/dashboards/dashboards.yml" + ] + }, + "faultInjection": { + "clientTimeoutMs": 350, + "maxFailureClassificationMs": 1200 + }, + "boundedLoad": { + "requests": 60, + "concurrency": 8, + "p95MaxMs": 750, + "heapGrowthMaxBytes": 20971520, + "paths": [ + "/healthz", + "/readyz", + "/version", + "/reports/worktime/management" + ] + } +} diff --git a/docs/OPERATIONAL_MATURITY_RU.md b/docs/OPERATIONAL_MATURITY_RU.md new file mode 100644 index 0000000..fae4fc7 --- /dev/null +++ b/docs/OPERATIONAL_MATURITY_RU.md @@ -0,0 +1,58 @@ +# Эксплуатационная зрелость DetMir/AWatch-rus + +Дата актуализации: 2026-07-01. + +Этот контур добавляет автоматическую проверку эксплуатационной зрелости без +нагрузки на production. Public CI запускает только offline checks: fixtures, +локальный mock HTTP, статическую проверку конфигураций, ClickHouse DDL и +контракт наблюдаемости. Live checks запускаются только вручную оператором с +private access. + +## Что проверяется + +- Integration harness: локальный mock обслуживает ключевые endpoints + `/healthz`, `/readyz`, `/version`, Worktime management и Security Finding + Inbox shadow payload. +- Fault injection: клиент должен быстро классифицировать `503`, timeout и + connection reset, не зависая сверх заданного бюджета. +- API compatibility: DetMir Portal OpenAPI обязан сохранять ключевые paths, + schemas и runtime endpoints. +- Config/migration validation: JSON/YAML examples, systemd units/timers и + ClickHouse init SQL проверяются на базовую пригодность и idempotency. +- Bounded load: короткий локальный load smoke с concurrency и p95 budget, + без sizing claims. +- Observability: обязательные health/version/readiness fields, diagnostic + headers и Prometheus metric names закреплены manifest/fixtures/source check. + +## Команды + +Offline PR/CI gate: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +node --check scripts/operational-maturity-check.mjs +node scripts/operational-maturity-check.mjs --json +``` + +Live contract, только оператором и только если контур доступен: + +```bash +cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian +AWATCH_OPS_LIVE_URL=http://127.0.0.1:8720 \ +node scripts/operational-maturity-check.mjs --json --live +``` + +Live check не включает DLP, Loki или Velociraptor. Он только читает +health/readiness/version/metrics endpoints и принимает controlled statuses +`200` или `503`. + +## Governance + +Контракт расположен в `configs/operational-maturity-contract.json`. +Fixture payloads лежат в `docs/fixtures/operational-maturity/`. +CI workflow: `.github/workflows/operational-maturity.yml`. + +Правило изменений: если endpoint, metric, schema, systemd unit или ClickHouse +migration меняется, сначала обновляется manifest/fixture, затем код. Удаление +полей или paths считается breaking change, если нет отдельного operator-approved +major contract change. diff --git a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md index dfa69b9..1a7e447 100644 --- a/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md +++ b/docs/OPERATIONS_VALIDATION_RUNBOOK_RU.md @@ -51,12 +51,16 @@ Repository-specific gate: ```bash cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian python3 scripts/public_secret_pattern_check.py +node scripts/operational-maturity-check.mjs --json cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian ``` +Подробности эксплуатационного maturity harness: [эксплуатационная зрелость +DetMir/AWatch-rus](OPERATIONAL_MATURITY_RU.md). + ## Browser smoke Browser smoke не заменяет API/CLI проверки. Он подтверждает, что операторский diff --git a/docs/fixtures/operational-maturity/portal-healthz.json b/docs/fixtures/operational-maturity/portal-healthz.json new file mode 100644 index 0000000..2ede19e --- /dev/null +++ b/docs/fixtures/operational-maturity/portal-healthz.json @@ -0,0 +1,6 @@ +{ + "status": "ok", + "service": "detmir-portal", + "schema_version": "pilot-v1", + "timestamp": "2026-07-01T00:00:00Z" +} diff --git a/docs/fixtures/operational-maturity/portal-metrics.prom b/docs/fixtures/operational-maturity/portal-metrics.prom new file mode 100644 index 0000000..8bb314d --- /dev/null +++ b/docs/fixtures/operational-maturity/portal-metrics.prom @@ -0,0 +1,26 @@ +# HELP awatch_http_requests_total HTTP requests handled by AWatch-rus portal +# TYPE awatch_http_requests_total counter +awatch_http_requests_total{method="GET",route="/healthz",status="200",module="portal"} 1 +# HELP awatch_http_request_duration_seconds HTTP request duration in seconds +# TYPE awatch_http_request_duration_seconds summary +awatch_http_request_duration_seconds_sum{method="GET",route="/healthz",status="200",module="portal"} 0.001 +awatch_http_request_duration_seconds_count{method="GET",route="/healthz",status="200",module="portal"} 1 +# TYPE awatch_report_requests_total counter +awatch_report_requests_total 1 +# TYPE awatch_report_cache_hits_total counter +awatch_report_cache_hits_total 0 +# TYPE awatch_report_cache_misses_total counter +awatch_report_cache_misses_total 1 +# TYPE awatch_report_cache_stale_hits_total counter +awatch_report_cache_stale_hits_total 0 +# TYPE awatch_reports_generated_total counter +awatch_reports_generated_total 1 +# TYPE awatch_ingestion_records_total counter +awatch_ingestion_records_total 0 +# TYPE awatch_ingestion_rejected_total counter +awatch_ingestion_rejected_total 0 +# TYPE awatch_role_denied_total counter +awatch_role_denied_total 0 +# HELP awatch_readyz_status Portal readiness status, 1=ready, 0=not_ready +# TYPE awatch_readyz_status gauge +awatch_readyz_status 1 diff --git a/docs/fixtures/operational-maturity/portal-readyz.json b/docs/fixtures/operational-maturity/portal-readyz.json new file mode 100644 index 0000000..a9293ce --- /dev/null +++ b/docs/fixtures/operational-maturity/portal-readyz.json @@ -0,0 +1,26 @@ +{ + "status": "ready", + "checks": { + "portal": { + "status": "ok", + "latency_ms": 1 + }, + "worktime_api": { + "status": "ok", + "stale": false + }, + "clickhouse": { + "status": "ok", + "mode": "external" + }, + "dlp_runtime": { + "status": "skipped", + "profile": "core_only", + "reason": "disabled_by_production_default" + }, + "security_finding_inbox": { + "status": "ok", + "mode": "shadow" + } + } +} diff --git a/docs/fixtures/operational-maturity/portal-version.json b/docs/fixtures/operational-maturity/portal-version.json new file mode 100644 index 0000000..ed02b32 --- /dev/null +++ b/docs/fixtures/operational-maturity/portal-version.json @@ -0,0 +1,10 @@ +{ + "app_name": "AWatch-rus DetMir Portal", + "app_version": "0.1.0", + "schema_version": "pilot-v1", + "environment": "test", + "build": { + "profile": "ci", + "source": "fixture" + } +} diff --git a/docs/fixtures/operational-maturity/security-finding-inbox-shadow.json b/docs/fixtures/operational-maturity/security-finding-inbox-shadow.json new file mode 100644 index 0000000..ea44410 --- /dev/null +++ b/docs/fixtures/operational-maturity/security-finding-inbox-shadow.json @@ -0,0 +1,18 @@ +{ + "mode": "shadow", + "auto_apply": false, + "workflow": ["decide", "plan", "approve", "apply", "verify"], + "findings": [ + { + "id": "fixture-001", + "source": "hayabusa", + "severity": "medium", + "status": "new", + "containment": { + "recommended": true, + "approved": false, + "applied": false + } + } + ] +} diff --git a/docs/fixtures/operational-maturity/worktime-management-degraded.json b/docs/fixtures/operational-maturity/worktime-management-degraded.json new file mode 100644 index 0000000..e5260bd --- /dev/null +++ b/docs/fixtures/operational-maturity/worktime-management-degraded.json @@ -0,0 +1,11 @@ +{ + "status": "DEGRADED", + "stale": false, + "reason": "report_unavailable", + "report_cache_hit": false, + "report_stale_served": false, + "aw_query_duration_ms": 350, + "aw_query_timeout_count": 1, + "report_build_error_count": 1, + "users": [] +} diff --git a/docs/fixtures/operational-maturity/worktime-management-ok.json b/docs/fixtures/operational-maturity/worktime-management-ok.json new file mode 100644 index 0000000..b9cbc46 --- /dev/null +++ b/docs/fixtures/operational-maturity/worktime-management-ok.json @@ -0,0 +1,17 @@ +{ + "status": "OK", + "stale": false, + "reason": null, + "report_cache_hit": false, + "report_stale_served": false, + "aw_query_duration_ms": 42, + "aw_query_timeout_count": 0, + "report_build_error_count": 0, + "users": [ + { + "user": "demo-user", + "active_seconds": 1800, + "apps_count": 3 + } + ] +} diff --git a/scripts/operational-maturity-check.mjs b/scripts/operational-maturity-check.mjs new file mode 100644 index 0000000..92a7e20 --- /dev/null +++ b/scripts/operational-maturity-check.mjs @@ -0,0 +1,487 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const manifestPath = "configs/operational-maturity-contract.json"; + +function readText(relativePath) { + return fs.readFileSync(path.join(root, relativePath), "utf8"); +} + +function readJson(relativePath) { + return JSON.parse(readText(relativePath)); +} + +function exists(relativePath) { + return fs.existsSync(path.join(root, relativePath)); +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +function getPath(value, dotted) { + return dotted.split(".").reduce((current, key) => { + if (current === null || current === undefined) return undefined; + return current[key]; + }, value); +} + +function result(name, fn) { + const started = Date.now(); + try { + const details = fn() || {}; + return { name, ok: true, duration_ms: Date.now() - started, ...details }; + } catch (error) { + return { name, ok: false, duration_ms: Date.now() - started, error: error.message }; + } +} + +async function asyncResult(name, fn) { + const started = Date.now(); + try { + const details = (await fn()) || {}; + return { name, ok: true, duration_ms: Date.now() - started, ...details }; + } catch (error) { + return { name, ok: false, duration_ms: Date.now() - started, error: error.message }; + } +} + +function walk(dir, predicate = () => true) { + const base = path.join(root, dir); + if (!fs.existsSync(base)) return []; + const files = []; + for (const entry of fs.readdirSync(base, { withFileTypes: true })) { + const absolute = path.join(base, entry.name); + const relative = path.relative(root, absolute).replaceAll(path.sep, "/"); + if (entry.isDirectory()) { + files.push(...walk(relative, predicate)); + } else if (predicate(relative)) { + files.push(relative); + } + } + return files.sort(); +} + +function parseArgs() { + const args = new Set(process.argv.slice(2)); + return { + json: args.has("--json"), + live: args.has("--live"), + failOnDuplicateDebt: args.has("--fail-on-duplicate-debt"), + }; +} + +function randomLocalPort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + server.close(() => resolve(port)); + }); + }); +} + +function fixtureResponse(fixture) { + if (fixture.contentType === "text/plain") { + return readText(fixture.fixture); + } + return JSON.stringify(readJson(fixture.fixture)); +} + +async function startFixtureServer(manifest) { + const fixturesByPath = new Map(); + for (const fixture of manifest.integrationFixtures) { + fixturesByPath.set(fixture.path, fixture); + } + fixturesByPath.set("/metrics", { + fixture: manifest.observability.metricsFixture, + status: 200, + contentType: "text/plain", + }); + + const sockets = new Set(); + const server = http.createServer((request, response) => { + const parsed = new URL(request.url || "/", "http://127.0.0.1"); + const fixture = fixturesByPath.get(`${parsed.pathname}${parsed.search}`) || fixturesByPath.get(parsed.pathname); + if (!fixture) { + response.statusCode = 404; + response.setHeader("Content-Type", "application/json; charset=utf-8"); + response.end(JSON.stringify({ error: "not_found", path: parsed.pathname })); + return; + } + response.statusCode = fixture.status || 200; + response.setHeader("Content-Type", fixture.contentType === "text/plain" ? "text/plain; charset=utf-8" : "application/json; charset=utf-8"); + response.setHeader("X-Request-Id", request.headers["x-request-id"] || "operational-maturity-fixture"); + response.setHeader("X-Correlation-Id", request.headers["x-request-id"] || "operational-maturity-fixture"); + response.end(fixtureResponse(fixture)); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + + const port = await randomLocalPort(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + return { + port, + close: () => + new Promise((resolve) => { + for (const socket of sockets) socket.destroy(); + server.close(resolve); + }), + }; +} + +async function fetchWithTimeout(url, timeoutMs, options = {}) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = Date.now(); + try { + const response = await fetch(url, { ...options, signal: controller.signal }); + const text = await response.text(); + let json = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + return { + status: response.status, + ok: response.ok, + headers: response.headers, + text, + json, + elapsed_ms: Date.now() - started, + }; + } finally { + clearTimeout(timeout); + } +} + +function checkApiCompatibility(manifest) { + const spec = readJson(manifest.apiCompatibility.openapiPath); + const missing = []; + for (const [apiPath, methods] of Object.entries(manifest.apiCompatibility.requiredOpenApiPaths)) { + for (const method of methods) { + if (!spec.paths?.[apiPath]?.[method]) missing.push(`${method.toUpperCase()} ${apiPath}`); + } + } + for (const schema of manifest.apiCompatibility.requiredSchemas) { + if (!spec.components?.schemas?.[schema]) missing.push(`schema ${schema}`); + } + const runtimeSource = readText(manifest.apiCompatibility.runtimeSourcePath); + for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) { + if (!runtimeSource.includes(`"${endpoint}"`)) missing.push(`runtime endpoint ${endpoint}`); + } + assert(missing.length === 0, `missing API compatibility anchors: ${missing.join(", ")}`); + return { + required_paths: Object.keys(manifest.apiCompatibility.requiredOpenApiPaths).length, + required_schemas: manifest.apiCompatibility.requiredSchemas.length, + runtime_endpoints: manifest.apiCompatibility.runtimeEndpoints.length, + }; +} + +function validateFixtureContract(manifest) { + const checked = []; + for (const fixture of manifest.integrationFixtures) { + const payload = readJson(fixture.fixture); + for (const field of fixture.requiredFields || []) { + assert(getPath(payload, field) !== undefined, `${fixture.fixture} missing field ${field}`); + } + for (const [field, expected] of Object.entries(fixture.equals || {})) { + assert(getPath(payload, field) === expected, `${fixture.fixture} expected ${field}=${JSON.stringify(expected)}`); + } + checked.push(fixture.path); + } + return { fixtures: checked.length }; +} + +async function checkIntegrationHarness(manifest) { + const server = await startFixtureServer(manifest); + const checked = []; + try { + for (const fixture of manifest.integrationFixtures) { + const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${fixture.path}`, 1500, { + headers: { "X-Request-Id": "operational-maturity" }, + }); + assert(response.status === fixture.status, `${fixture.path} status ${response.status}, expected ${fixture.status}`); + if (fixture.contentType === "application/json") { + assert(response.json && typeof response.json === "object", `${fixture.path} did not return JSON`); + } + assert(response.headers.get("x-correlation-id") === "operational-maturity", `${fixture.path} did not echo correlation id`); + checked.push({ path: fixture.path, elapsed_ms: response.elapsed_ms }); + } + return { endpoints: checked.length, max_elapsed_ms: Math.max(...checked.map((item) => item.elapsed_ms)) }; + } finally { + await server.close(); + } +} + +async function startFaultServer() { + const sockets = new Set(); + const server = http.createServer((request, response) => { + const parsed = new URL(request.url || "/", "http://127.0.0.1"); + if (parsed.pathname === "/fault/503") { + response.statusCode = 503; + response.end("unavailable"); + return; + } + if (parsed.pathname === "/fault/slow") { + setTimeout(() => { + response.statusCode = 200; + response.end("slow-ok"); + }, 1200); + return; + } + if (parsed.pathname === "/fault/reset") { + request.socket.destroy(); + return; + } + response.statusCode = 200; + response.end("ok"); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + const port = await randomLocalPort(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + return { + port, + close: () => + new Promise((resolve) => { + for (const socket of sockets) socket.destroy(); + server.close(resolve); + }), + }; +} + +async function checkFaultInjection(manifest) { + const server = await startFaultServer(); + const timeoutMs = manifest.faultInjection.clientTimeoutMs; + const maxClassifyMs = manifest.faultInjection.maxFailureClassificationMs; + const classifications = []; + try { + const unavailable = await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/503`, timeoutMs); + assert(unavailable.status === 503, "503 fault was not classified as HTTP 503"); + classifications.push("http_503"); + + const slowStart = Date.now(); + try { + await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/slow`, timeoutMs); + throw new Error("slow fault did not timeout"); + } catch (error) { + const elapsed = Date.now() - slowStart; + assert(elapsed < maxClassifyMs, `timeout classification exceeded ${maxClassifyMs}ms`); + classifications.push("timeout"); + } + + try { + await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/reset`, timeoutMs); + throw new Error("connection reset fault did not fail"); + } catch { + classifications.push("connection_reset"); + } + return { classifications }; + } finally { + await server.close(); + } +} + +function percentile(values, pct) { + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((pct / 100) * sorted.length) - 1); + return sorted[index] || 0; +} + +async function mapLimit(items, limit, fn) { + const results = []; + let next = 0; + async function worker() { + while (next < items.length) { + const index = next; + next += 1; + results[index] = await fn(items[index], index); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return results; +} + +async function checkBoundedLoad(manifest) { + const server = await startFixtureServer(manifest); + const cfg = manifest.boundedLoad; + const heapBefore = process.memoryUsage().heapUsed; + try { + const work = Array.from({ length: cfg.requests }, (_, index) => cfg.paths[index % cfg.paths.length]); + const responses = await mapLimit(work, cfg.concurrency, async (apiPath) => { + const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${apiPath}`, 1500); + assert(response.status >= 200 && response.status < 300, `${apiPath} failed with ${response.status}`); + return response.elapsed_ms; + }); + const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore); + const p95 = percentile(responses, 95); + assert(p95 <= cfg.p95MaxMs, `bounded load p95 ${p95}ms exceeds ${cfg.p95MaxMs}ms`); + assert(heapGrowth <= cfg.heapGrowthMaxBytes, `heap growth ${heapGrowth} exceeds ${cfg.heapGrowthMaxBytes}`); + return { requests: cfg.requests, concurrency: cfg.concurrency, p95_ms: p95, heap_growth_bytes: heapGrowth }; + } finally { + await server.close(); + } +} + +function validateJsonAndYaml(manifest) { + const checked = []; + for (const file of manifest.configValidation.jsonFiles) { + assert(exists(file), `missing JSON config/fixture ${file}`); + readJson(file); + checked.push(file); + } + for (const file of manifest.configValidation.yamlFiles) { + assert(exists(file), `missing YAML config ${file}`); + const text = readText(file); + assert(!text.includes("\t"), `${file} contains tabs`); + if (!/(\.example\.ya?ml|example\.yml)$/i.test(file)) { + assert(!/password:\s*(change-me|changeme|password)\b/i.test(text), `${file} contains unsafe password placeholder`); + } + checked.push(file); + } + return { files: checked.length }; +} + +function validateSystemdUnits(manifest) { + const files = manifest.configValidation.systemdDirs.flatMap((dir) => + walk(dir, (file) => file.endsWith(".service") || file.endsWith(".timer")), + ); + assert(files.length > 0, "no systemd files found"); + const findings = []; + for (const file of files) { + const text = readText(file); + if (!text.includes("[Unit]")) findings.push(`${file}: missing [Unit]`); + if (file.endsWith(".service")) { + if (!text.includes("[Service]")) findings.push(`${file}: missing [Service]`); + if (!/^ExecStart=/m.test(text)) findings.push(`${file}: missing ExecStart`); + if (/^Type=oneshot$/m.test(text) && /^Restart=always$/m.test(text)) { + findings.push(`${file}: oneshot service must not use Restart=always`); + } + } + if (file.endsWith(".timer")) { + if (!text.includes("[Timer]")) findings.push(`${file}: missing [Timer]`); + if (!/^Unit=.*\.service$/m.test(text)) findings.push(`${file}: missing service Unit`); + if (!text.includes("[Install]")) findings.push(`${file}: missing [Install]`); + } + } + assert(findings.length === 0, findings.join("; ")); + return { files: files.length }; +} + +function validateClickHouseMigrations(manifest) { + const dir = manifest.configValidation.clickhouseInitDir; + const files = walk(dir, (file) => file.endsWith(".sql")); + assert(files.length > 0, "no ClickHouse init SQL files found"); + const findings = []; + const order = files.map((file) => path.basename(file)); + const sorted = [...order].sort(); + if (order.join("|") !== sorted.join("|")) findings.push("ClickHouse init filenames are not sorted"); + for (const file of files) { + const text = readText(file); + if (/\b(DROP|TRUNCATE)\s+(DATABASE|TABLE)\b/i.test(text)) { + findings.push(`${file}: destructive DDL is not allowed in init validation`); + } + const createStatements = text.match(/\bCREATE\s+(DATABASE|TABLE|VIEW|OR\s+REPLACE\s+VIEW)\b/gi) || []; + for (const create of createStatements) { + if (/CREATE\s+(DATABASE|TABLE)/i.test(create) && !/IF\s+NOT\s+EXISTS/i.test(text.slice(text.indexOf(create), text.indexOf(create) + 140))) { + findings.push(`${file}: CREATE DATABASE/TABLE must use IF NOT EXISTS`); + } + if (/CREATE\s+VIEW/i.test(create) && !/CREATE\s+(OR\s+REPLACE\s+)?VIEW/i.test(create)) { + findings.push(`${file}: CREATE VIEW must be OR REPLACE or otherwise idempotent`); + } + } + } + assert(findings.length === 0, findings.join("; ")); + return { files: files.length }; +} + +function checkObservability(manifest) { + const metricsText = readText(manifest.observability.metricsFixture); + const sourceText = readText(manifest.observability.sourcePath); + const missing = []; + for (const metric of manifest.observability.requiredMetrics) { + if (!metricsText.includes(metric)) missing.push(`fixture metric ${metric}`); + if (!sourceText.includes(metric)) missing.push(`source metric ${metric}`); + } + const hardeningSmoke = readText("scripts/awatch-production-hardening-smoke.mjs"); + for (const header of manifest.observability.diagnosticHeaders) { + if (!hardeningSmoke.includes(header)) missing.push(`diagnostic header ${header}`); + } + assert(missing.length === 0, `missing observability anchors: ${missing.join(", ")}`); + return { metrics: manifest.observability.requiredMetrics.length }; +} + +async function checkLive(manifest) { + const baseUrl = (process.env.AWATCH_OPS_LIVE_URL || "").replace(/\/+$/, ""); + assert(baseUrl, "AWATCH_OPS_LIVE_URL is required for --live"); + const checked = []; + for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) { + const response = await fetchWithTimeout(`${baseUrl}${endpoint}`, 3000, { + headers: { "X-Request-Id": "operational-maturity-live" }, + }); + assert([200, 503].includes(response.status), `${endpoint} returned uncontrolled status ${response.status}`); + checked.push({ endpoint, status: response.status, elapsed_ms: response.elapsed_ms }); + } + return { baseUrl, checked }; +} + +async function main() { + const args = parseArgs(); + const manifest = readJson(manifestPath); + const checks = [ + result("api_compatibility", () => checkApiCompatibility(manifest)), + result("fixture_contracts", () => validateFixtureContract(manifest)), + await asyncResult("integration_harness", () => checkIntegrationHarness(manifest)), + await asyncResult("fault_injection", () => checkFaultInjection(manifest)), + await asyncResult("bounded_load", () => checkBoundedLoad(manifest)), + result("json_yaml_config_validation", () => validateJsonAndYaml(manifest)), + result("systemd_config_validation", () => validateSystemdUnits(manifest)), + result("clickhouse_migration_validation", () => validateClickHouseMigrations(manifest)), + result("observability_contract", () => checkObservability(manifest)), + ]; + if (args.live) { + checks.push(await asyncResult("live_operational_contract", () => checkLive(manifest))); + } + + const report = { + ok: checks.every((check) => check.ok), + mode: args.live ? "live" : "offline", + manifest: manifest.version, + checks, + }; + + if (args.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + for (const check of checks) { + console.log(`${check.ok ? "ok" : "fail"} ${check.name} ${check.duration_ms}ms`); + if (!check.ok) console.log(` ${check.error}`); + } + } + if (!report.ok) process.exit(1); +} + +main().catch((error) => { + console.error(JSON.stringify({ ok: false, error: error.message }, null, 2)); + process.exit(1); +});