Harden DetMir DLP production runtime

- default DetMir DLP runtime to core_only/disabled with load-guard protection - add fail-closed placeholder validation and runtime-scoped artifact checks - document operator re-enable flow for light profile and guard rollback - update prod docs, env examples, and Ansible DLP defaults
2026-07-01 00:05:23 +03:00
parent 1149f5dfbd
commit fe87c85a31
26 changed files with 3053 additions and 220 deletions
@@ -0,0 +1,260 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ENABLED="${AW_DLP_GUARD_ENABLED:-true}"
+PROFILE="${AW_DLP_PROFILE:-light}"
+STATE_DIR="${AW_DLP_GUARD_STATE_DIR:-/var/lib/activitywatch/health}"
+STATE_FILE="${AW_DLP_GUARD_STATE_FILE:-${STATE_DIR}/dlp-light-guard-state.json}"
+STATE_HISTORY_DIR="${AW_DLP_GUARD_HISTORY_DIR:-${STATE_DIR}/dlp-light-guard-history}"
+CONTROL_BIN="${AW_DLP_CONTROL_BIN:-/usr/local/bin/detmir-dlp-runtime-control}"
+LOAD_RATIO="${AW_DLP_GUARD_LOAD_RATIO:-1.50}"
+MEM_AVAILABLE_PCT_MIN="${AW_DLP_GUARD_MEM_AVAILABLE_PCT_MIN:-15}"
+IOWAIT_PCT_MAX="${AW_DLP_GUARD_IOWAIT_PCT_MAX:-20}"
+STRIKES_REQUIRED="${AW_DLP_GUARD_STRIKES_REQUIRED:-3}"
+
+DLP_GUARDED_UNITS=(
+  aw-dlp-influx-exporter.timer
+  aw-dlp-influx-exporter.service
+  activitywatch-dlp-aggregator.timer
+  activitywatch-dlp-aggregator.service
+  aw-dlp-report-scheduler.timer
+  aw-dlp-report-scheduler.service
+  aw-dlp-syslog-forwarder.timer
+  aw-dlp-syslog-forwarder.service
+  aw-dlp-webhook-sender.timer
+  aw-dlp-webhook-sender.service
+  aw-dlp-cef-exporter.timer
+  aw-dlp-cef-exporter.service
+  aw-dlp-ioc-refresh.timer
+  aw-dlp-ioc-refresh.service
+  aw-dlp-policy-engine.service
+  aw-dlp-case-management.service
+  detmir-portal-evidence.service
+)
+
+json_string() {
+  python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
+}
+
+number_or_null() {
+  local value="${1:-}"
+  if [[ "$value" =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
+    printf '%s' "$value"
+  else
+    printf 'null'
+  fi
+}
+
+active_dlp_units_json() {
+  local first=1 unit
+  printf '['
+  if command -v systemctl >/dev/null 2>&1; then
+    for unit in "${DLP_GUARDED_UNITS[@]}"; do
+      if systemctl is-active --quiet "$unit" 2>/dev/null; then
+        [[ "$first" -eq 1 ]] || printf ','
+        first=0
+        json_string "$unit"
+      fi
+    done
+  fi
+  printf ']'
+}
+
+active_dlp_unit_count() {
+  local count=0 unit
+  if command -v systemctl >/dev/null 2>&1; then
+    for unit in "${DLP_GUARDED_UNITS[@]}"; do
+      if systemctl is-active --quiet "$unit" 2>/dev/null; then
+        count=$((count + 1))
+      fi
+    done
+  fi
+  printf '%s\n' "$count"
+}
+
+read_load1() {
+  awk '{print $1}' /proc/loadavg 2>/dev/null || printf '0'
+}
+
+read_cpu_count() {
+  local cores
+  cores="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1')"
+  if [[ ! "$cores" =~ ^[0-9]+$ || "$cores" -lt 1 ]]; then
+    cores=1
+  fi
+  printf '%s\n' "$cores"
+}
+
+read_mem_available_pct() {
+  awk '
+    /^MemTotal:/ { total=$2 }
+    /^MemAvailable:/ { available=$2 }
+    END {
+      if (total > 0) {
+        printf "%.2f", (available * 100.0 / total)
+      } else {
+        printf "0"
+      }
+    }
+  ' /proc/meminfo 2>/dev/null || printf '0'
+}
+
+read_cpu_sample() {
+  awk '/^cpu / {
+    idle=$5
+    iowait=$6
+    total=0
+    for (i=2; i<=NF; i++) total += $i
+    printf "%s %s\n", total, iowait
+    exit
+  }' /proc/stat 2>/dev/null || printf '0 0'
+}
+
+read_iowait_pct() {
+  local total1 wait1 total2 wait2 dtotal dwait
+  read -r total1 wait1 < <(read_cpu_sample)
+  sleep 1
+  read -r total2 wait2 < <(read_cpu_sample)
+  dtotal=$((total2 - total1))
+  dwait=$((wait2 - wait1))
+  if [[ "$dtotal" -le 0 || "$dwait" -lt 0 ]]; then
+    printf '0'
+    return
+  fi
+  awk -v wait="$dwait" -v total="$dtotal" 'BEGIN { printf "%.2f", wait * 100.0 / total }'
+}
+
+is_over_threshold() {
+  local value="$1"
+  local threshold="$2"
+  awk -v value="$value" -v threshold="$threshold" 'BEGIN { exit !(value > threshold) }'
+}
+
+is_under_threshold() {
+  local value="$1"
+  local threshold="$2"
+  awk -v value="$value" -v threshold="$threshold" 'BEGIN { exit !(value < threshold) }'
+}
+
+write_state() {
+  local action="$1"
+  local reason="$2"
+  local load1="$3"
+  local cores="$4"
+  local load_threshold="$5"
+  local mem_pct="$6"
+  local iowait_pct="$7"
+  local active_count="$8"
+  local active_units_json="$9"
+  local control_exit="${10}"
+  local strikes="${11:-0}"
+  local now stamp tmp history
+
+  now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+  mkdir -p "$STATE_DIR" "$STATE_HISTORY_DIR"
+  tmp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")"
+  {
+    printf '{'
+    printf '"generated_at_utc":%s,' "$(json_string "$now")"
+    printf '"profile":%s,' "$(json_string "$PROFILE")"
+    printf '"guard_enabled":%s,' "$(json_string "$ENABLED")"
+    printf '"action":%s,' "$(json_string "$action")"
+    printf '"reason":%s,' "$(json_string "$reason")"
+    printf '"consecutive_overload_count":%s,' "$(number_or_null "$strikes")"
+    printf '"consecutive_overload_required":%s,' "$(number_or_null "$STRIKES_REQUIRED")"
+    printf '"control_bin":%s,' "$(json_string "$CONTROL_BIN")"
+    printf '"control_exit":%s,' "$(number_or_null "$control_exit")"
+    printf '"metrics":{'
+    printf '"load1":%s,' "$(number_or_null "$load1")"
+    printf '"cpu_count":%s,' "$(number_or_null "$cores")"
+    printf '"load_threshold":%s,' "$(number_or_null "$load_threshold")"
+    printf '"mem_available_pct":%s,' "$(number_or_null "$mem_pct")"
+    printf '"mem_available_pct_min":%s,' "$(number_or_null "$MEM_AVAILABLE_PCT_MIN")"
+    printf '"iowait_pct":%s,' "$(number_or_null "$iowait_pct")"
+    printf '"iowait_pct_max":%s' "$(number_or_null "$IOWAIT_PCT_MAX")"
+    printf '},'
+    printf '"active_dlp_unit_count":%s,' "$(number_or_null "$active_count")"
+    printf '"active_dlp_units":%s' "$active_units_json"
+    printf '}\n'
+  } >"$tmp"
+  mv "$tmp" "$STATE_FILE"
+  history="${STATE_HISTORY_DIR}/dlp-light-guard-${stamp}.json"
+  cp -a "$STATE_FILE" "$history"
+  printf 'dlp guard action=%s reason=%s state=%s history=%s\n' "$action" "$reason" "$STATE_FILE" "$history"
+}
+
+main() {
+  local load1 cores load_threshold mem_pct iowait_pct active_count active_units_json overloaded reason control_exit strikes prev_strikes
+
+  load1="$(read_load1)"
+  cores="$(read_cpu_count)"
+  load_threshold="$(awk -v cores="$cores" -v ratio="$LOAD_RATIO" 'BEGIN { printf "%.2f", cores * ratio }')"
+  mem_pct="$(read_mem_available_pct)"
+  iowait_pct="$(read_iowait_pct)"
+  active_units_json="$(active_dlp_units_json)"
+  active_count="$(active_dlp_unit_count)"
+  overloaded=0
+  reason="within_thresholds"
+  prev_strikes="$(
+    python3 - "$STATE_FILE" <<'PY' 2>/dev/null || true
+import json, sys
+try:
+    print(int(json.load(open(sys.argv[1])).get("consecutive_overload_count", 0)))
+except Exception:
+    print(0)
+PY
+  )"
+  [[ "$prev_strikes" =~ ^[0-9]+$ ]] || prev_strikes=0
+  strikes=0
+
+  if is_over_threshold "$load1" "$load_threshold"; then
+    overloaded=1
+    reason="load1_above_threshold"
+  elif is_under_threshold "$mem_pct" "$MEM_AVAILABLE_PCT_MIN"; then
+    overloaded=1
+    reason="mem_available_below_threshold"
+  elif is_over_threshold "$iowait_pct" "$IOWAIT_PCT_MAX"; then
+    overloaded=1
+    reason="iowait_above_threshold"
+  fi
+
+  if [[ "$ENABLED" != "true" && "$ENABLED" != "1" && "$ENABLED" != "yes" ]]; then
+    write_state "skipped" "guard_disabled" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "0"
+    return 0
+  fi
+
+  if [[ "$overloaded" -eq 0 ]]; then
+    write_state "none" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "0"
+    return 0
+  fi
+
+  strikes=$((prev_strikes + 1))
+
+  if [[ "$strikes" -lt "$STRIKES_REQUIRED" ]]; then
+    write_state "observe_overload" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "$strikes"
+    return 0
+  fi
+
+  if [[ "$active_count" -eq 0 ]]; then
+    write_state "none" "${reason}_but_no_active_dlp_units" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "$strikes"
+    return 0
+  fi
+
+  if [[ ! -x "$CONTROL_BIN" ]]; then
+    write_state "failed" "${reason}_control_bin_missing" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "127" "$strikes"
+    printf 'DLP guard cannot disable overloaded DLP: executable not found: %s\n' "$CONTROL_BIN" >&2
+    return 127
+  fi
+
+  control_exit=0
+  AW_DLP_DISABLED_REASON="auto_disabled_by_dlp_load_guard:${reason}" "$CONTROL_BIN" set-profile core_only || control_exit=$?
+  if [[ "$control_exit" -eq 0 ]]; then
+    write_state "auto_disabled" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "$control_exit" "$strikes"
+  else
+    write_state "failed" "${reason}_control_exit_${control_exit}" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "$control_exit" "$strikes"
+  fi
+  return "$control_exit"
+}
+
+main "$@"