Add DetMir operational maturity harness

2026-07-01 07:34:43 +03:00
12 changed files with 879 additions and 0 deletions
@@ -0,0 +1,73 @@
+name: Operational maturity
+
+# GitHub Actions is public mirror validation only.
+# Live DetMir production checks must be run explicitly by an operator with
+# private network access and must not enable heavy DLP, Loki or always-on
+# Velociraptor.
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      live:
+        description: "Run live endpoint contract if AWATCH_OPS_LIVE_URL is configured"
+        required: false
+        default: "false"
+  schedule:
+    - cron: "41 3 * * 2"
+
+permissions:
+  contents: read
+
+jobs:
+  offline-operational-maturity:
+    name: Offline operational maturity
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Script syntax
+        run: node --check scripts/operational-maturity-check.mjs
+
+      - name: Operational maturity harness
+        run: node scripts/operational-maturity-check.mjs --json
+
+  live-operational-contract:
+    name: Live operational contract
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' && inputs.live == 'true'
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Run live contract when URL is available
+        env:
+          AWATCH_OPS_LIVE_URL: ${{ secrets.AWATCH_OPS_LIVE_URL }}
+        run: |
+          if [[ -z "${AWATCH_OPS_LIVE_URL}" ]]; then
+            echo "skipped: AWATCH_OPS_LIVE_URL secret is not configured"
+            exit 0
+          fi
+          node scripts/operational-maturity-check.mjs --json --live
@@ -0,0 +1,143 @@
+{
+  "version": "2026-07-01.operational-maturity-v1",
+  "apiCompatibility": {
+    "openapiPath": "adk-rust/crates/detmir-portal/src/contracts/openapi.json",
+    "requiredOpenApiPaths": {
+      "/contracts": ["get"],
+      "/contracts/openapi.json": ["get"],
+      "/health": ["get"],
+      "/operator": ["get"],
+      "/reports": ["get"],
+      "/workforce/kpi/explain": ["get"],
+      "/ueba": ["get"],
+      "/risk/narrative": ["get"],
+      "/actions": ["get"],
+      "/readiness/latest": ["get"],
+      "/telemetry": ["post"]
+    },
+    "requiredSchemas": [
+      "ContractIndex",
+      "JsonObject",
+      "ActionCenterResponse",
+      "RiskNarrative",
+      "UebaPayload",
+      "WorkforceKpiExplain"
+    ],
+    "runtimeEndpoints": [
+      "/healthz",
+      "/readyz",
+      "/version",
+      "/metrics"
+    ],
+    "runtimeSourcePath": "adk-rust/crates/detmir-portal/src/main.rs"
+  },
+  "integrationFixtures": [
+    {
+      "path": "/healthz",
+      "fixture": "docs/fixtures/operational-maturity/portal-healthz.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["status", "service", "schema_version"],
+      "equals": { "status": "ok", "schema_version": "pilot-v1" }
+    },
+    {
+      "path": "/readyz",
+      "fixture": "docs/fixtures/operational-maturity/portal-readyz.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["status", "checks.portal", "checks.dlp_runtime"],
+      "equals": { "status": "ready", "checks.dlp_runtime.status": "skipped" }
+    },
+    {
+      "path": "/version",
+      "fixture": "docs/fixtures/operational-maturity/portal-version.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["app_version", "schema_version", "environment"],
+      "equals": { "schema_version": "pilot-v1" }
+    },
+    {
+      "path": "/reports/worktime/management",
+      "fixture": "docs/fixtures/operational-maturity/worktime-management-ok.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["status", "stale", "report_cache_hit", "aw_query_duration_ms"],
+      "equals": { "status": "OK", "stale": false }
+    },
+    {
+      "path": "/reports/worktime/management?fault=aw_timeout",
+      "fixture": "docs/fixtures/operational-maturity/worktime-management-degraded.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["status", "reason", "stale", "report_stale_served", "aw_query_timeout_count"],
+      "equals": { "status": "DEGRADED", "reason": "report_unavailable", "stale": false }
+    },
+    {
+      "path": "/security/finding-inbox",
+      "fixture": "docs/fixtures/operational-maturity/security-finding-inbox-shadow.json",
+      "status": 200,
+      "contentType": "application/json",
+      "requiredFields": ["mode", "auto_apply", "workflow"],
+      "equals": { "mode": "shadow", "auto_apply": false }
+    }
+  ],
+  "observability": {
+    "metricsFixture": "docs/fixtures/operational-maturity/portal-metrics.prom",
+    "requiredMetrics": [
+      "awatch_http_requests_total",
+      "awatch_http_request_duration_seconds_sum",
+      "awatch_http_request_duration_seconds_count",
+      "awatch_report_requests_total",
+      "awatch_report_cache_hits_total",
+      "awatch_report_cache_misses_total",
+      "awatch_report_cache_stale_hits_total",
+      "awatch_reports_generated_total",
+      "awatch_ingestion_records_total",
+      "awatch_ingestion_rejected_total",
+      "awatch_role_denied_total",
+      "awatch_readyz_status"
+    ],
+    "sourcePath": "adk-rust/crates/detmir-portal/src/production/metrics.rs",
+    "diagnosticHeaders": ["x-request-id", "x-correlation-id"]
+  },
+  "configValidation": {
+    "jsonFiles": [
+      "configs/worktime-interpretation-policy.example.json",
+      "configs/detmir-workforce-policy.example.json",
+      "adk-rust/crates/detmir-portal/src/contracts/openapi.json",
+      "docs/fixtures/operational-maturity/portal-healthz.json",
+      "docs/fixtures/operational-maturity/portal-readyz.json",
+      "docs/fixtures/operational-maturity/portal-version.json",
+      "docs/fixtures/operational-maturity/worktime-management-ok.json",
+      "docs/fixtures/operational-maturity/worktime-management-degraded.json",
+      "docs/fixtures/operational-maturity/security-finding-inbox-shadow.json"
+    ],
+    "systemdDirs": [
+      "ops/systemd",
+      "clickhouse-1c/ops"
+    ],
+    "clickhouseInitDir": "clickhouse-1c/clickhouse/init",
+    "yamlFiles": [
+      "configs/detmir-ueba-risk-policy.example.yaml",
+      "clickhouse-1c/etl/config.example.yml",
+      "clickhouse-1c/grafana/provisioning/datasources/clickhouse.yml",
+      "clickhouse-1c/grafana/provisioning/dashboards/dashboards.yml"
+    ]
+  },
+  "faultInjection": {
+    "clientTimeoutMs": 350,
+    "maxFailureClassificationMs": 1200
+  },
+  "boundedLoad": {
+    "requests": 60,
+    "concurrency": 8,
+    "p95MaxMs": 750,
+    "heapGrowthMaxBytes": 20971520,
+    "paths": [
+      "/healthz",
+      "/readyz",
+      "/version",
+      "/reports/worktime/management"
+    ]
+  }
+}
@@ -0,0 +1,58 @@
+# Эксплуатационная зрелость DetMir/AWatch-rus
+
+Дата актуализации: 2026-07-01.
+
+Этот контур добавляет автоматическую проверку эксплуатационной зрелости без
+нагрузки на production. Public CI запускает только offline checks: fixtures,
+локальный mock HTTP, статическую проверку конфигураций, ClickHouse DDL и
+контракт наблюдаемости. Live checks запускаются только вручную оператором с
+private access.
+
+## Что проверяется
+
+- Integration harness: локальный mock обслуживает ключевые endpoints
+  `/healthz`, `/readyz`, `/version`, Worktime management и Security Finding
+  Inbox shadow payload.
+- Fault injection: клиент должен быстро классифицировать `503`, timeout и
+  connection reset, не зависая сверх заданного бюджета.
+- API compatibility: DetMir Portal OpenAPI обязан сохранять ключевые paths,
+  schemas и runtime endpoints.
+- Config/migration validation: JSON/YAML examples, systemd units/timers и
+  ClickHouse init SQL проверяются на базовую пригодность и idempotency.
+- Bounded load: короткий локальный load smoke с concurrency и p95 budget,
+  без sizing claims.
+- Observability: обязательные health/version/readiness fields, diagnostic
+  headers и Prometheus metric names закреплены manifest/fixtures/source check.
+
+## Команды
+
+Offline PR/CI gate:
+
+```bash
+cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
+node --check scripts/operational-maturity-check.mjs
+node scripts/operational-maturity-check.mjs --json
+```
+
+Live contract, только оператором и только если контур доступен:
+
+```bash
+cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
+AWATCH_OPS_LIVE_URL=http://127.0.0.1:8720 \
+node scripts/operational-maturity-check.mjs --json --live
+```
+
+Live check не включает DLP, Loki или Velociraptor. Он только читает
+health/readiness/version/metrics endpoints и принимает controlled statuses
+`200` или `503`.
+
+## Governance
+
+Контракт расположен в `configs/operational-maturity-contract.json`.
+Fixture payloads лежат в `docs/fixtures/operational-maturity/`.
+CI workflow: `.github/workflows/operational-maturity.yml`.
+
+Правило изменений: если endpoint, metric, schema, systemd unit или ClickHouse
+migration меняется, сначала обновляется manifest/fixture, затем код. Удаление
+полей или paths считается breaking change, если нет отдельного operator-approved
+major contract change.
@@ -51,12 +51,16 @@ Repository-specific gate:
 ```bash
 cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian
 python3 scripts/public_secret_pattern_check.py
+node scripts/operational-maturity-check.mjs --json

 cd /mnt/usb_hdd2/Projects/ActivityWatch-Russian/adk-rust
 export CARGO_TARGET_DIR=/home/igor/.cache/detmir-adk-rust-target
 cargo run -p quality-gate -- --root /mnt/usb_hdd2/Projects/ActivityWatch-Russian
 ```

+Подробности эксплуатационного maturity harness: [эксплуатационная зрелость
+DetMir/AWatch-rus](OPERATIONAL_MATURITY_RU.md).
+
 ## Browser smoke

 Browser smoke не заменяет API/CLI проверки. Он подтверждает, что операторский
@@ -0,0 +1,6 @@
+{
+  "status": "ok",
+  "service": "detmir-portal",
+  "schema_version": "pilot-v1",
+  "timestamp": "2026-07-01T00:00:00Z"
+}
@@ -0,0 +1,26 @@
+# HELP awatch_http_requests_total HTTP requests handled by AWatch-rus portal
+# TYPE awatch_http_requests_total counter
+awatch_http_requests_total{method="GET",route="/healthz",status="200",module="portal"} 1
+# HELP awatch_http_request_duration_seconds HTTP request duration in seconds
+# TYPE awatch_http_request_duration_seconds summary
+awatch_http_request_duration_seconds_sum{method="GET",route="/healthz",status="200",module="portal"} 0.001
+awatch_http_request_duration_seconds_count{method="GET",route="/healthz",status="200",module="portal"} 1
+# TYPE awatch_report_requests_total counter
+awatch_report_requests_total 1
+# TYPE awatch_report_cache_hits_total counter
+awatch_report_cache_hits_total 0
+# TYPE awatch_report_cache_misses_total counter
+awatch_report_cache_misses_total 1
+# TYPE awatch_report_cache_stale_hits_total counter
+awatch_report_cache_stale_hits_total 0
+# TYPE awatch_reports_generated_total counter
+awatch_reports_generated_total 1
+# TYPE awatch_ingestion_records_total counter
+awatch_ingestion_records_total 0
+# TYPE awatch_ingestion_rejected_total counter
+awatch_ingestion_rejected_total 0
+# TYPE awatch_role_denied_total counter
+awatch_role_denied_total 0
+# HELP awatch_readyz_status Portal readiness status, 1=ready, 0=not_ready
+# TYPE awatch_readyz_status gauge
+awatch_readyz_status 1
@@ -0,0 +1,26 @@
+{
+  "status": "ready",
+  "checks": {
+    "portal": {
+      "status": "ok",
+      "latency_ms": 1
+    },
+    "worktime_api": {
+      "status": "ok",
+      "stale": false
+    },
+    "clickhouse": {
+      "status": "ok",
+      "mode": "external"
+    },
+    "dlp_runtime": {
+      "status": "skipped",
+      "profile": "core_only",
+      "reason": "disabled_by_production_default"
+    },
+    "security_finding_inbox": {
+      "status": "ok",
+      "mode": "shadow"
+    }
+  }
+}
@@ -0,0 +1,10 @@
+{
+  "app_name": "AWatch-rus DetMir Portal",
+  "app_version": "0.1.0",
+  "schema_version": "pilot-v1",
+  "environment": "test",
+  "build": {
+    "profile": "ci",
+    "source": "fixture"
+  }
+}
@@ -0,0 +1,18 @@
+{
+  "mode": "shadow",
+  "auto_apply": false,
+  "workflow": ["decide", "plan", "approve", "apply", "verify"],
+  "findings": [
+    {
+      "id": "fixture-001",
+      "source": "hayabusa",
+      "severity": "medium",
+      "status": "new",
+      "containment": {
+        "recommended": true,
+        "approved": false,
+        "applied": false
+      }
+    }
+  ]
+}
@@ -0,0 +1,11 @@
+{
+  "status": "DEGRADED",
+  "stale": false,
+  "reason": "report_unavailable",
+  "report_cache_hit": false,
+  "report_stale_served": false,
+  "aw_query_duration_ms": 350,
+  "aw_query_timeout_count": 1,
+  "report_build_error_count": 1,
+  "users": []
+}
@@ -0,0 +1,17 @@
+{
+  "status": "OK",
+  "stale": false,
+  "reason": null,
+  "report_cache_hit": false,
+  "report_stale_served": false,
+  "aw_query_duration_ms": 42,
+  "aw_query_timeout_count": 0,
+  "report_build_error_count": 0,
+  "users": [
+    {
+      "user": "demo-user",
+      "active_seconds": 1800,
+      "apps_count": 3
+    }
+  ]
+}
@@ -0,0 +1,487 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import http from "node:http";
+import net from "node:net";
+import path from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const manifestPath = "configs/operational-maturity-contract.json";
+
+function readText(relativePath) {
+  return fs.readFileSync(path.join(root, relativePath), "utf8");
+}
+
+function readJson(relativePath) {
+  return JSON.parse(readText(relativePath));
+}
+
+function exists(relativePath) {
+  return fs.existsSync(path.join(root, relativePath));
+}
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message);
+}
+
+function getPath(value, dotted) {
+  return dotted.split(".").reduce((current, key) => {
+    if (current === null || current === undefined) return undefined;
+    return current[key];
+  }, value);
+}
+
+function result(name, fn) {
+  const started = Date.now();
+  try {
+    const details = fn() || {};
+    return { name, ok: true, duration_ms: Date.now() - started, ...details };
+  } catch (error) {
+    return { name, ok: false, duration_ms: Date.now() - started, error: error.message };
+  }
+}
+
+async function asyncResult(name, fn) {
+  const started = Date.now();
+  try {
+    const details = (await fn()) || {};
+    return { name, ok: true, duration_ms: Date.now() - started, ...details };
+  } catch (error) {
+    return { name, ok: false, duration_ms: Date.now() - started, error: error.message };
+  }
+}
+
+function walk(dir, predicate = () => true) {
+  const base = path.join(root, dir);
+  if (!fs.existsSync(base)) return [];
+  const files = [];
+  for (const entry of fs.readdirSync(base, { withFileTypes: true })) {
+    const absolute = path.join(base, entry.name);
+    const relative = path.relative(root, absolute).replaceAll(path.sep, "/");
+    if (entry.isDirectory()) {
+      files.push(...walk(relative, predicate));
+    } else if (predicate(relative)) {
+      files.push(relative);
+    }
+  }
+  return files.sort();
+}
+
+function parseArgs() {
+  const args = new Set(process.argv.slice(2));
+  return {
+    json: args.has("--json"),
+    live: args.has("--live"),
+    failOnDuplicateDebt: args.has("--fail-on-duplicate-debt"),
+  };
+}
+
+function randomLocalPort() {
+  return new Promise((resolve, reject) => {
+    const server = net.createServer();
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      const port = typeof address === "object" && address ? address.port : 0;
+      server.close(() => resolve(port));
+    });
+  });
+}
+
+function fixtureResponse(fixture) {
+  if (fixture.contentType === "text/plain") {
+    return readText(fixture.fixture);
+  }
+  return JSON.stringify(readJson(fixture.fixture));
+}
+
+async function startFixtureServer(manifest) {
+  const fixturesByPath = new Map();
+  for (const fixture of manifest.integrationFixtures) {
+    fixturesByPath.set(fixture.path, fixture);
+  }
+  fixturesByPath.set("/metrics", {
+    fixture: manifest.observability.metricsFixture,
+    status: 200,
+    contentType: "text/plain",
+  });
+
+  const sockets = new Set();
+  const server = http.createServer((request, response) => {
+    const parsed = new URL(request.url || "/", "http://127.0.0.1");
+    const fixture = fixturesByPath.get(`${parsed.pathname}${parsed.search}`) || fixturesByPath.get(parsed.pathname);
+    if (!fixture) {
+      response.statusCode = 404;
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+      response.end(JSON.stringify({ error: "not_found", path: parsed.pathname }));
+      return;
+    }
+    response.statusCode = fixture.status || 200;
+    response.setHeader("Content-Type", fixture.contentType === "text/plain" ? "text/plain; charset=utf-8" : "application/json; charset=utf-8");
+    response.setHeader("X-Request-Id", request.headers["x-request-id"] || "operational-maturity-fixture");
+    response.setHeader("X-Correlation-Id", request.headers["x-request-id"] || "operational-maturity-fixture");
+    response.end(fixtureResponse(fixture));
+  });
+  server.on("connection", (socket) => {
+    sockets.add(socket);
+    socket.on("close", () => sockets.delete(socket));
+  });
+
+  const port = await randomLocalPort();
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, "127.0.0.1", resolve);
+  });
+  return {
+    port,
+    close: () =>
+      new Promise((resolve) => {
+        for (const socket of sockets) socket.destroy();
+        server.close(resolve);
+      }),
+  };
+}
+
+async function fetchWithTimeout(url, timeoutMs, options = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  const started = Date.now();
+  try {
+    const response = await fetch(url, { ...options, signal: controller.signal });
+    const text = await response.text();
+    let json = null;
+    try {
+      json = text ? JSON.parse(text) : null;
+    } catch {
+      json = null;
+    }
+    return {
+      status: response.status,
+      ok: response.ok,
+      headers: response.headers,
+      text,
+      json,
+      elapsed_ms: Date.now() - started,
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function checkApiCompatibility(manifest) {
+  const spec = readJson(manifest.apiCompatibility.openapiPath);
+  const missing = [];
+  for (const [apiPath, methods] of Object.entries(manifest.apiCompatibility.requiredOpenApiPaths)) {
+    for (const method of methods) {
+      if (!spec.paths?.[apiPath]?.[method]) missing.push(`${method.toUpperCase()} ${apiPath}`);
+    }
+  }
+  for (const schema of manifest.apiCompatibility.requiredSchemas) {
+    if (!spec.components?.schemas?.[schema]) missing.push(`schema ${schema}`);
+  }
+  const runtimeSource = readText(manifest.apiCompatibility.runtimeSourcePath);
+  for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) {
+    if (!runtimeSource.includes(`"${endpoint}"`)) missing.push(`runtime endpoint ${endpoint}`);
+  }
+  assert(missing.length === 0, `missing API compatibility anchors: ${missing.join(", ")}`);
+  return {
+    required_paths: Object.keys(manifest.apiCompatibility.requiredOpenApiPaths).length,
+    required_schemas: manifest.apiCompatibility.requiredSchemas.length,
+    runtime_endpoints: manifest.apiCompatibility.runtimeEndpoints.length,
+  };
+}
+
+function validateFixtureContract(manifest) {
+  const checked = [];
+  for (const fixture of manifest.integrationFixtures) {
+    const payload = readJson(fixture.fixture);
+    for (const field of fixture.requiredFields || []) {
+      assert(getPath(payload, field) !== undefined, `${fixture.fixture} missing field ${field}`);
+    }
+    for (const [field, expected] of Object.entries(fixture.equals || {})) {
+      assert(getPath(payload, field) === expected, `${fixture.fixture} expected ${field}=${JSON.stringify(expected)}`);
+    }
+    checked.push(fixture.path);
+  }
+  return { fixtures: checked.length };
+}
+
+async function checkIntegrationHarness(manifest) {
+  const server = await startFixtureServer(manifest);
+  const checked = [];
+  try {
+    for (const fixture of manifest.integrationFixtures) {
+      const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${fixture.path}`, 1500, {
+        headers: { "X-Request-Id": "operational-maturity" },
+      });
+      assert(response.status === fixture.status, `${fixture.path} status ${response.status}, expected ${fixture.status}`);
+      if (fixture.contentType === "application/json") {
+        assert(response.json && typeof response.json === "object", `${fixture.path} did not return JSON`);
+      }
+      assert(response.headers.get("x-correlation-id") === "operational-maturity", `${fixture.path} did not echo correlation id`);
+      checked.push({ path: fixture.path, elapsed_ms: response.elapsed_ms });
+    }
+    return { endpoints: checked.length, max_elapsed_ms: Math.max(...checked.map((item) => item.elapsed_ms)) };
+  } finally {
+    await server.close();
+  }
+}
+
+async function startFaultServer() {
+  const sockets = new Set();
+  const server = http.createServer((request, response) => {
+    const parsed = new URL(request.url || "/", "http://127.0.0.1");
+    if (parsed.pathname === "/fault/503") {
+      response.statusCode = 503;
+      response.end("unavailable");
+      return;
+    }
+    if (parsed.pathname === "/fault/slow") {
+      setTimeout(() => {
+        response.statusCode = 200;
+        response.end("slow-ok");
+      }, 1200);
+      return;
+    }
+    if (parsed.pathname === "/fault/reset") {
+      request.socket.destroy();
+      return;
+    }
+    response.statusCode = 200;
+    response.end("ok");
+  });
+  server.on("connection", (socket) => {
+    sockets.add(socket);
+    socket.on("close", () => sockets.delete(socket));
+  });
+  const port = await randomLocalPort();
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, "127.0.0.1", resolve);
+  });
+  return {
+    port,
+    close: () =>
+      new Promise((resolve) => {
+        for (const socket of sockets) socket.destroy();
+        server.close(resolve);
+      }),
+  };
+}
+
+async function checkFaultInjection(manifest) {
+  const server = await startFaultServer();
+  const timeoutMs = manifest.faultInjection.clientTimeoutMs;
+  const maxClassifyMs = manifest.faultInjection.maxFailureClassificationMs;
+  const classifications = [];
+  try {
+    const unavailable = await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/503`, timeoutMs);
+    assert(unavailable.status === 503, "503 fault was not classified as HTTP 503");
+    classifications.push("http_503");
+
+    const slowStart = Date.now();
+    try {
+      await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/slow`, timeoutMs);
+      throw new Error("slow fault did not timeout");
+    } catch (error) {
+      const elapsed = Date.now() - slowStart;
+      assert(elapsed < maxClassifyMs, `timeout classification exceeded ${maxClassifyMs}ms`);
+      classifications.push("timeout");
+    }
+
+    try {
+      await fetchWithTimeout(`http://127.0.0.1:${server.port}/fault/reset`, timeoutMs);
+      throw new Error("connection reset fault did not fail");
+    } catch {
+      classifications.push("connection_reset");
+    }
+    return { classifications };
+  } finally {
+    await server.close();
+  }
+}
+
+function percentile(values, pct) {
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.ceil((pct / 100) * sorted.length) - 1);
+  return sorted[index] || 0;
+}
+
+async function mapLimit(items, limit, fn) {
+  const results = [];
+  let next = 0;
+  async function worker() {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      results[index] = await fn(items[index], index);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return results;
+}
+
+async function checkBoundedLoad(manifest) {
+  const server = await startFixtureServer(manifest);
+  const cfg = manifest.boundedLoad;
+  const heapBefore = process.memoryUsage().heapUsed;
+  try {
+    const work = Array.from({ length: cfg.requests }, (_, index) => cfg.paths[index % cfg.paths.length]);
+    const responses = await mapLimit(work, cfg.concurrency, async (apiPath) => {
+      const response = await fetchWithTimeout(`http://127.0.0.1:${server.port}${apiPath}`, 1500);
+      assert(response.status >= 200 && response.status < 300, `${apiPath} failed with ${response.status}`);
+      return response.elapsed_ms;
+    });
+    const heapGrowth = Math.max(0, process.memoryUsage().heapUsed - heapBefore);
+    const p95 = percentile(responses, 95);
+    assert(p95 <= cfg.p95MaxMs, `bounded load p95 ${p95}ms exceeds ${cfg.p95MaxMs}ms`);
+    assert(heapGrowth <= cfg.heapGrowthMaxBytes, `heap growth ${heapGrowth} exceeds ${cfg.heapGrowthMaxBytes}`);
+    return { requests: cfg.requests, concurrency: cfg.concurrency, p95_ms: p95, heap_growth_bytes: heapGrowth };
+  } finally {
+    await server.close();
+  }
+}
+
+function validateJsonAndYaml(manifest) {
+  const checked = [];
+  for (const file of manifest.configValidation.jsonFiles) {
+    assert(exists(file), `missing JSON config/fixture ${file}`);
+    readJson(file);
+    checked.push(file);
+  }
+  for (const file of manifest.configValidation.yamlFiles) {
+    assert(exists(file), `missing YAML config ${file}`);
+    const text = readText(file);
+    assert(!text.includes("\t"), `${file} contains tabs`);
+    if (!/(\.example\.ya?ml|example\.yml)$/i.test(file)) {
+      assert(!/password:\s*(change-me|changeme|password)\b/i.test(text), `${file} contains unsafe password placeholder`);
+    }
+    checked.push(file);
+  }
+  return { files: checked.length };
+}
+
+function validateSystemdUnits(manifest) {
+  const files = manifest.configValidation.systemdDirs.flatMap((dir) =>
+    walk(dir, (file) => file.endsWith(".service") || file.endsWith(".timer")),
+  );
+  assert(files.length > 0, "no systemd files found");
+  const findings = [];
+  for (const file of files) {
+    const text = readText(file);
+    if (!text.includes("[Unit]")) findings.push(`${file}: missing [Unit]`);
+    if (file.endsWith(".service")) {
+      if (!text.includes("[Service]")) findings.push(`${file}: missing [Service]`);
+      if (!/^ExecStart=/m.test(text)) findings.push(`${file}: missing ExecStart`);
+      if (/^Type=oneshot$/m.test(text) && /^Restart=always$/m.test(text)) {
+        findings.push(`${file}: oneshot service must not use Restart=always`);
+      }
+    }
+    if (file.endsWith(".timer")) {
+      if (!text.includes("[Timer]")) findings.push(`${file}: missing [Timer]`);
+      if (!/^Unit=.*\.service$/m.test(text)) findings.push(`${file}: missing service Unit`);
+      if (!text.includes("[Install]")) findings.push(`${file}: missing [Install]`);
+    }
+  }
+  assert(findings.length === 0, findings.join("; "));
+  return { files: files.length };
+}
+
+function validateClickHouseMigrations(manifest) {
+  const dir = manifest.configValidation.clickhouseInitDir;
+  const files = walk(dir, (file) => file.endsWith(".sql"));
+  assert(files.length > 0, "no ClickHouse init SQL files found");
+  const findings = [];
+  const order = files.map((file) => path.basename(file));
+  const sorted = [...order].sort();
+  if (order.join("|") !== sorted.join("|")) findings.push("ClickHouse init filenames are not sorted");
+  for (const file of files) {
+    const text = readText(file);
+    if (/\b(DROP|TRUNCATE)\s+(DATABASE|TABLE)\b/i.test(text)) {
+      findings.push(`${file}: destructive DDL is not allowed in init validation`);
+    }
+    const createStatements = text.match(/\bCREATE\s+(DATABASE|TABLE|VIEW|OR\s+REPLACE\s+VIEW)\b/gi) || [];
+    for (const create of createStatements) {
+      if (/CREATE\s+(DATABASE|TABLE)/i.test(create) && !/IF\s+NOT\s+EXISTS/i.test(text.slice(text.indexOf(create), text.indexOf(create) + 140))) {
+        findings.push(`${file}: CREATE DATABASE/TABLE must use IF NOT EXISTS`);
+      }
+      if (/CREATE\s+VIEW/i.test(create) && !/CREATE\s+(OR\s+REPLACE\s+)?VIEW/i.test(create)) {
+        findings.push(`${file}: CREATE VIEW must be OR REPLACE or otherwise idempotent`);
+      }
+    }
+  }
+  assert(findings.length === 0, findings.join("; "));
+  return { files: files.length };
+}
+
+function checkObservability(manifest) {
+  const metricsText = readText(manifest.observability.metricsFixture);
+  const sourceText = readText(manifest.observability.sourcePath);
+  const missing = [];
+  for (const metric of manifest.observability.requiredMetrics) {
+    if (!metricsText.includes(metric)) missing.push(`fixture metric ${metric}`);
+    if (!sourceText.includes(metric)) missing.push(`source metric ${metric}`);
+  }
+  const hardeningSmoke = readText("scripts/awatch-production-hardening-smoke.mjs");
+  for (const header of manifest.observability.diagnosticHeaders) {
+    if (!hardeningSmoke.includes(header)) missing.push(`diagnostic header ${header}`);
+  }
+  assert(missing.length === 0, `missing observability anchors: ${missing.join(", ")}`);
+  return { metrics: manifest.observability.requiredMetrics.length };
+}
+
+async function checkLive(manifest) {
+  const baseUrl = (process.env.AWATCH_OPS_LIVE_URL || "").replace(/\/+$/, "");
+  assert(baseUrl, "AWATCH_OPS_LIVE_URL is required for --live");
+  const checked = [];
+  for (const endpoint of manifest.apiCompatibility.runtimeEndpoints) {
+    const response = await fetchWithTimeout(`${baseUrl}${endpoint}`, 3000, {
+      headers: { "X-Request-Id": "operational-maturity-live" },
+    });
+    assert([200, 503].includes(response.status), `${endpoint} returned uncontrolled status ${response.status}`);
+    checked.push({ endpoint, status: response.status, elapsed_ms: response.elapsed_ms });
+  }
+  return { baseUrl, checked };
+}
+
+async function main() {
+  const args = parseArgs();
+  const manifest = readJson(manifestPath);
+  const checks = [
+    result("api_compatibility", () => checkApiCompatibility(manifest)),
+    result("fixture_contracts", () => validateFixtureContract(manifest)),
+    await asyncResult("integration_harness", () => checkIntegrationHarness(manifest)),
+    await asyncResult("fault_injection", () => checkFaultInjection(manifest)),
+    await asyncResult("bounded_load", () => checkBoundedLoad(manifest)),
+    result("json_yaml_config_validation", () => validateJsonAndYaml(manifest)),
+    result("systemd_config_validation", () => validateSystemdUnits(manifest)),
+    result("clickhouse_migration_validation", () => validateClickHouseMigrations(manifest)),
+    result("observability_contract", () => checkObservability(manifest)),
+  ];
+  if (args.live) {
+    checks.push(await asyncResult("live_operational_contract", () => checkLive(manifest)));
+  }
+
+  const report = {
+    ok: checks.every((check) => check.ok),
+    mode: args.live ? "live" : "offline",
+    manifest: manifest.version,
+    checks,
+  };
+
+  if (args.json) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    for (const check of checks) {
+      console.log(`${check.ok ? "ok" : "fail"} ${check.name} ${check.duration_ms}ms`);
+      if (!check.ok) console.log(`  ${check.error}`);
+    }
+  }
+  if (!report.ok) process.exit(1);
+}
+
+main().catch((error) => {
+  console.error(JSON.stringify({ ok: false, error: error.message }, null, 2));
+  process.exit(1);
+});