Harden DetMir DLP production runtime
CI / Rust checks (push) Canceled after 0s
CI / Docs and registry checks (push) Canceled after 0s
CI / Smoke checks (push) Canceled after 0s
Coverage / Coverage baseline (push) Canceled after 0s
Security / Cargo audit (push) Canceled after 0s
Security / Cargo deny (push) Canceled after 0s
Security / Secret pattern check (push) Canceled after 0s
Security / Dependency review (push) Canceled after 0s

- default DetMir DLP runtime to core_only/disabled with load-guard protection

- add fail-closed placeholder validation and runtime-scoped artifact checks

- document operator re-enable flow for light profile and guard rollback

- update prod docs, env examples, and Ansible DLP defaults
This commit is contained in:
igor04091968
2026-07-01 00:05:23 +03:00
parent 1149f5dfbd
commit fe87c85a31
26 changed files with 3053 additions and 220 deletions
+43 -3
View File
@@ -4,8 +4,33 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TARGET_ROOT="${CARGO_TARGET_DIR:-$ROOT_DIR/adk-rust/target}"
RELEASE_DIR="$TARGET_ROOT/release"
SCOPE="${CHECK_DETMIR_RUST_RELEASE_SCOPE:-prod-runtime}"
required_bins=(
prod_runtime_bins=(
aw-1c-ingest
aw-hayabusa-autoprocess-rust
aw-rus-healthd
aw-slo-monitor
aw-workforce-ingest
detmir-auto
detmir-portal
detmir-readiness
dlp-aggregator
dlp-case-management
dlp-cef-exporter
dlp-compliance
dlp-influx-exporter
dlp-policy-engine
dlp-syslog-forwarder
dlp-webhook-sender
worktime-api
worktime-autoheal
worktime-influx-exporter
worktime-prewarm
worktime-ui-bridge
)
workspace_bins=(
detmir-status
detmir-adk-status
detmir-check
@@ -61,8 +86,23 @@ required_bins=(
aw-hayabusa-from-windows-rust
aw-hayabusa-autoprocess-rust
aw-1c-ingest
containment-engine
security-finding-inbox
)
case "$SCOPE" in
prod-runtime)
required_bins=("${prod_runtime_bins[@]}")
;;
workspace)
required_bins=("${workspace_bins[@]}")
;;
*)
echo "Unsupported CHECK_DETMIR_RUST_RELEASE_SCOPE=$SCOPE; expected prod-runtime or workspace" >&2
exit 2
;;
esac
missing=0
for bin in "${required_bins[@]}"; do
if [[ -x "$RELEASE_DIR/$bin" ]]; then
@@ -76,7 +116,7 @@ done
if (( missing != 0 )); then
cat >&2 <<EOF
Missing DetMir Rust release artifacts.
Missing DetMir Rust release artifacts for scope: $SCOPE.
Build them with:
cd "$ROOT_DIR/adk-rust"
CARGO_TARGET_DIR="$TARGET_ROOT" cargo build --release --workspace
@@ -84,4 +124,4 @@ EOF
exit 1
fi
echo "detmir rust release artifacts: OK ($RELEASE_DIR)"
echo "detmir rust release artifacts: OK scope=$SCOPE ($RELEASE_DIR)"
+260
View File
@@ -0,0 +1,260 @@
#!/usr/bin/env bash
set -euo pipefail
ENABLED="${AW_DLP_GUARD_ENABLED:-true}"
PROFILE="${AW_DLP_PROFILE:-light}"
STATE_DIR="${AW_DLP_GUARD_STATE_DIR:-/var/lib/activitywatch/health}"
STATE_FILE="${AW_DLP_GUARD_STATE_FILE:-${STATE_DIR}/dlp-light-guard-state.json}"
STATE_HISTORY_DIR="${AW_DLP_GUARD_HISTORY_DIR:-${STATE_DIR}/dlp-light-guard-history}"
CONTROL_BIN="${AW_DLP_CONTROL_BIN:-/usr/local/bin/detmir-dlp-runtime-control}"
LOAD_RATIO="${AW_DLP_GUARD_LOAD_RATIO:-1.50}"
MEM_AVAILABLE_PCT_MIN="${AW_DLP_GUARD_MEM_AVAILABLE_PCT_MIN:-15}"
IOWAIT_PCT_MAX="${AW_DLP_GUARD_IOWAIT_PCT_MAX:-20}"
STRIKES_REQUIRED="${AW_DLP_GUARD_STRIKES_REQUIRED:-3}"
DLP_GUARDED_UNITS=(
aw-dlp-influx-exporter.timer
aw-dlp-influx-exporter.service
activitywatch-dlp-aggregator.timer
activitywatch-dlp-aggregator.service
aw-dlp-report-scheduler.timer
aw-dlp-report-scheduler.service
aw-dlp-syslog-forwarder.timer
aw-dlp-syslog-forwarder.service
aw-dlp-webhook-sender.timer
aw-dlp-webhook-sender.service
aw-dlp-cef-exporter.timer
aw-dlp-cef-exporter.service
aw-dlp-ioc-refresh.timer
aw-dlp-ioc-refresh.service
aw-dlp-policy-engine.service
aw-dlp-case-management.service
detmir-portal-evidence.service
)
json_string() {
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
}
number_or_null() {
local value="${1:-}"
if [[ "$value" =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
printf '%s' "$value"
else
printf 'null'
fi
}
active_dlp_units_json() {
local first=1 unit
printf '['
if command -v systemctl >/dev/null 2>&1; then
for unit in "${DLP_GUARDED_UNITS[@]}"; do
if systemctl is-active --quiet "$unit" 2>/dev/null; then
[[ "$first" -eq 1 ]] || printf ','
first=0
json_string "$unit"
fi
done
fi
printf ']'
}
active_dlp_unit_count() {
local count=0 unit
if command -v systemctl >/dev/null 2>&1; then
for unit in "${DLP_GUARDED_UNITS[@]}"; do
if systemctl is-active --quiet "$unit" 2>/dev/null; then
count=$((count + 1))
fi
done
fi
printf '%s\n' "$count"
}
read_load1() {
awk '{print $1}' /proc/loadavg 2>/dev/null || printf '0'
}
read_cpu_count() {
local cores
cores="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1')"
if [[ ! "$cores" =~ ^[0-9]+$ || "$cores" -lt 1 ]]; then
cores=1
fi
printf '%s\n' "$cores"
}
read_mem_available_pct() {
awk '
/^MemTotal:/ { total=$2 }
/^MemAvailable:/ { available=$2 }
END {
if (total > 0) {
printf "%.2f", (available * 100.0 / total)
} else {
printf "0"
}
}
' /proc/meminfo 2>/dev/null || printf '0'
}
read_cpu_sample() {
awk '/^cpu / {
idle=$5
iowait=$6
total=0
for (i=2; i<=NF; i++) total += $i
printf "%s %s\n", total, iowait
exit
}' /proc/stat 2>/dev/null || printf '0 0'
}
read_iowait_pct() {
local total1 wait1 total2 wait2 dtotal dwait
read -r total1 wait1 < <(read_cpu_sample)
sleep 1
read -r total2 wait2 < <(read_cpu_sample)
dtotal=$((total2 - total1))
dwait=$((wait2 - wait1))
if [[ "$dtotal" -le 0 || "$dwait" -lt 0 ]]; then
printf '0'
return
fi
awk -v wait="$dwait" -v total="$dtotal" 'BEGIN { printf "%.2f", wait * 100.0 / total }'
}
is_over_threshold() {
local value="$1"
local threshold="$2"
awk -v value="$value" -v threshold="$threshold" 'BEGIN { exit !(value > threshold) }'
}
is_under_threshold() {
local value="$1"
local threshold="$2"
awk -v value="$value" -v threshold="$threshold" 'BEGIN { exit !(value < threshold) }'
}
write_state() {
local action="$1"
local reason="$2"
local load1="$3"
local cores="$4"
local load_threshold="$5"
local mem_pct="$6"
local iowait_pct="$7"
local active_count="$8"
local active_units_json="$9"
local control_exit="${10}"
local strikes="${11:-0}"
local now stamp tmp history
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
mkdir -p "$STATE_DIR" "$STATE_HISTORY_DIR"
tmp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")"
{
printf '{'
printf '"generated_at_utc":%s,' "$(json_string "$now")"
printf '"profile":%s,' "$(json_string "$PROFILE")"
printf '"guard_enabled":%s,' "$(json_string "$ENABLED")"
printf '"action":%s,' "$(json_string "$action")"
printf '"reason":%s,' "$(json_string "$reason")"
printf '"consecutive_overload_count":%s,' "$(number_or_null "$strikes")"
printf '"consecutive_overload_required":%s,' "$(number_or_null "$STRIKES_REQUIRED")"
printf '"control_bin":%s,' "$(json_string "$CONTROL_BIN")"
printf '"control_exit":%s,' "$(number_or_null "$control_exit")"
printf '"metrics":{'
printf '"load1":%s,' "$(number_or_null "$load1")"
printf '"cpu_count":%s,' "$(number_or_null "$cores")"
printf '"load_threshold":%s,' "$(number_or_null "$load_threshold")"
printf '"mem_available_pct":%s,' "$(number_or_null "$mem_pct")"
printf '"mem_available_pct_min":%s,' "$(number_or_null "$MEM_AVAILABLE_PCT_MIN")"
printf '"iowait_pct":%s,' "$(number_or_null "$iowait_pct")"
printf '"iowait_pct_max":%s' "$(number_or_null "$IOWAIT_PCT_MAX")"
printf '},'
printf '"active_dlp_unit_count":%s,' "$(number_or_null "$active_count")"
printf '"active_dlp_units":%s' "$active_units_json"
printf '}\n'
} >"$tmp"
mv "$tmp" "$STATE_FILE"
history="${STATE_HISTORY_DIR}/dlp-light-guard-${stamp}.json"
cp -a "$STATE_FILE" "$history"
printf 'dlp guard action=%s reason=%s state=%s history=%s\n' "$action" "$reason" "$STATE_FILE" "$history"
}
main() {
local load1 cores load_threshold mem_pct iowait_pct active_count active_units_json overloaded reason control_exit strikes prev_strikes
load1="$(read_load1)"
cores="$(read_cpu_count)"
load_threshold="$(awk -v cores="$cores" -v ratio="$LOAD_RATIO" 'BEGIN { printf "%.2f", cores * ratio }')"
mem_pct="$(read_mem_available_pct)"
iowait_pct="$(read_iowait_pct)"
active_units_json="$(active_dlp_units_json)"
active_count="$(active_dlp_unit_count)"
overloaded=0
reason="within_thresholds"
prev_strikes="$(
python3 - "$STATE_FILE" <<'PY' 2>/dev/null || true
import json, sys
try:
print(int(json.load(open(sys.argv[1])).get("consecutive_overload_count", 0)))
except Exception:
print(0)
PY
)"
[[ "$prev_strikes" =~ ^[0-9]+$ ]] || prev_strikes=0
strikes=0
if is_over_threshold "$load1" "$load_threshold"; then
overloaded=1
reason="load1_above_threshold"
elif is_under_threshold "$mem_pct" "$MEM_AVAILABLE_PCT_MIN"; then
overloaded=1
reason="mem_available_below_threshold"
elif is_over_threshold "$iowait_pct" "$IOWAIT_PCT_MAX"; then
overloaded=1
reason="iowait_above_threshold"
fi
if [[ "$ENABLED" != "true" && "$ENABLED" != "1" && "$ENABLED" != "yes" ]]; then
write_state "skipped" "guard_disabled" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "0"
return 0
fi
if [[ "$overloaded" -eq 0 ]]; then
write_state "none" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "0"
return 0
fi
strikes=$((prev_strikes + 1))
if [[ "$strikes" -lt "$STRIKES_REQUIRED" ]]; then
write_state "observe_overload" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "$strikes"
return 0
fi
if [[ "$active_count" -eq 0 ]]; then
write_state "none" "${reason}_but_no_active_dlp_units" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "0" "$strikes"
return 0
fi
if [[ ! -x "$CONTROL_BIN" ]]; then
write_state "failed" "${reason}_control_bin_missing" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "127" "$strikes"
printf 'DLP guard cannot disable overloaded DLP: executable not found: %s\n' "$CONTROL_BIN" >&2
return 127
fi
control_exit=0
AW_DLP_DISABLED_REASON="auto_disabled_by_dlp_load_guard:${reason}" "$CONTROL_BIN" set-profile core_only || control_exit=$?
if [[ "$control_exit" -eq 0 ]]; then
write_state "auto_disabled" "$reason" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "$control_exit" "$strikes"
else
write_state "failed" "${reason}_control_exit_${control_exit}" "$load1" "$cores" "$load_threshold" "$mem_pct" "$iowait_pct" "$active_count" "$active_units_json" "$control_exit" "$strikes"
fi
return "$control_exit"
}
main "$@"
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env bash
set -euo pipefail
ACTION="${1:-status}"
PROFILE="${2:-${AW_DLP_PROFILE:-core_only}}"
AW_BASE="${AW_DLP_CONTROL_AW_BASE:-http://127.0.0.1:5600}"
HOSTNAME_FILTER="${AW_DLP_CONTROL_HOSTNAME:-${AW_LOGICAL_HOST_ID:-${AW_MONITORED_WINDOWS_HOSTNAME:-HOST-EXAMPLE}}}"
STATE_DIR="${AW_DLP_CONTROL_STATE_DIR:-/var/lib/activitywatch/health}"
STATE_FILE="${AW_DLP_CONTROL_STATE_FILE:-${STATE_DIR}/dlp-runtime-state.json}"
STATE_HISTORY_DIR="${AW_DLP_CONTROL_HISTORY_DIR:-${STATE_DIR}/dlp-runtime-history}"
ROLLBACK_FILE="${AW_DLP_CONTROL_ROLLBACK_FILE:-${STATE_DIR}/dlp-runtime-rollback.state}"
REASON="${AW_DLP_DISABLED_REASON:-dlp_runtime_profile_control}"
DLP_UNITS=(
aw-dlp-influx-exporter.timer
aw-dlp-influx-exporter.service
activitywatch-dlp-aggregator.timer
activitywatch-dlp-aggregator.service
aw-dlp-report-scheduler.timer
aw-dlp-report-scheduler.service
aw-dlp-syslog-forwarder.timer
aw-dlp-syslog-forwarder.service
aw-dlp-webhook-sender.timer
aw-dlp-webhook-sender.service
aw-dlp-cef-exporter.timer
aw-dlp-cef-exporter.service
aw-dlp-ioc-refresh.timer
aw-dlp-ioc-refresh.service
aw-dlp-policy-engine.service
aw-dlp-case-management.service
detmir-portal-evidence.service
)
DLP_BUCKET_PREFIXES=(
aw-dlp-endpoint-signals
aw-dlp-incidents
aw-dlp-review
aw-dlp-rules
)
DLP_LIGHT_UNITS=(
activitywatch-dlp-aggregator.timer
aw-dlp-ioc-refresh.timer
)
DLP_ON_DEMAND_UNITS=(
aw-dlp-ioc-refresh.timer
aw-dlp-policy-engine.service
aw-dlp-case-management.service
detmir-portal-evidence.service
)
json_escape() {
local value="$1"
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$value"
}
unit_json() {
local first=1 unit active enabled load
printf '['
for unit in "${DLP_UNITS[@]}"; do
load="$(systemctl show -p LoadState --value "$unit" 2>/dev/null || true)"
if [[ "$load" == "not-found" || -z "$load" ]]; then
active="not-found"
enabled="not-found"
else
active="$(systemctl is-active "$unit" 2>/dev/null || true)"
enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
fi
[[ "$first" -eq 1 ]] || printf ','
first=0
printf '{"unit":%s,"load":%s,"active":%s,"enabled":%s}' \
"$(json_escape "$unit")" \
"$(json_escape "${load:-not-found}")" \
"$(json_escape "${active:-unknown}")" \
"$(json_escape "${enabled:-unknown}")"
done
printf ']'
}
bucket_json() {
local first=1 prefix bucket url payload ts count
printf '['
for prefix in "${DLP_BUCKET_PREFIXES[@]}"; do
bucket="${prefix}_${HOSTNAME_FILTER}"
url="${AW_BASE%/}/api/0/buckets/${bucket}/events?limit=1"
payload="$(curl -sS --connect-timeout 3 --max-time 8 "$url" 2>/dev/null || true)"
ts="$(printf '%s' "$payload" | jq -r '.[0].timestamp // ""' 2>/dev/null || true)"
count="$(printf '%s' "$payload" | jq -r 'if type == "array" then length else 0 end' 2>/dev/null || printf '0')"
[[ "$first" -eq 1 ]] || printf ','
first=0
printf '{"bucket":%s,"sample_count":%s,"latest_timestamp":%s}' \
"$(json_escape "$bucket")" \
"${count:-0}" \
"$(json_escape "$ts")"
done
printf ']'
}
unit_exists() {
local unit="$1"
systemctl list-unit-files "$unit" --no-legend 2>/dev/null | grep -q . || systemctl status "$unit" >/dev/null 2>&1
}
stop_disable_all_dlp() {
local unit
for unit in "${DLP_UNITS[@]}"; do
if unit_exists "$unit"; then
systemctl stop "$unit" >/dev/null 2>&1 || true
systemctl disable "$unit" >/dev/null 2>&1 || true
systemctl reset-failed "$unit" >/dev/null 2>&1 || true
fi
done
}
enable_start_units() {
local unit
for unit in "$@"; do
if unit_exists "$unit"; then
systemctl enable --now "$unit" >/dev/null 2>&1 || true
fi
done
}
capture_rollback_state() {
local tmp unit load active enabled
mkdir -p "$STATE_DIR"
tmp="$(mktemp "${ROLLBACK_FILE}.tmp.XXXXXX")"
{
printf '# generated_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf '# reason=pre_profile_change\n'
for unit in "${DLP_UNITS[@]}"; do
load="$(systemctl show -p LoadState --value "$unit" 2>/dev/null || true)"
if [[ "$load" == "not-found" || -z "$load" ]]; then
active="not-found"
enabled="not-found"
else
active="$(systemctl is-active "$unit" 2>/dev/null || true)"
enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
fi
printf '%s|%s|%s|%s\n' "$unit" "${load:-not-found}" "$active" "$enabled"
done
} >"$tmp"
mv "$tmp" "$ROLLBACK_FILE"
}
write_stats() {
local mode="${1:-current}" now stamp tmp history_file
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
mkdir -p "$STATE_DIR" "$STATE_HISTORY_DIR"
tmp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")"
{
printf '{'
printf '"generated_at_utc":%s,' "$(json_escape "$now")"
printf '"mode":%s,' "$(json_escape "$mode")"
printf '"profile":%s,' "$(json_escape "${AW_DLP_PROFILE:-$PROFILE}")"
printf '"reason":%s,' "$(json_escape "$REASON")"
printf '"aw_base":%s,' "$(json_escape "$AW_BASE")"
printf '"hostname":%s,' "$(json_escape "$HOSTNAME_FILTER")"
printf '"units":'
unit_json
printf ',"buckets":'
bucket_json
printf '}\n'
} >"$tmp"
mv "$tmp" "$STATE_FILE"
history_file="${STATE_HISTORY_DIR}/dlp-runtime-${mode}-${stamp}.json"
cp -a "$STATE_FILE" "$history_file"
printf 'latest=%s\nhistory=%s\n' "$STATE_FILE" "$history_file"
}
apply_profile() {
local target_profile="$1"
capture_rollback_state
case "$target_profile" in
core_only|disabled|off)
PROFILE="core_only"
stop_disable_all_dlp
AW_DLP_PROFILE="core_only" write_stats "disabled"
;;
light)
PROFILE="light"
stop_disable_all_dlp
enable_start_units "${DLP_LIGHT_UNITS[@]}"
AW_DLP_PROFILE="light" write_stats "enabled_light"
;;
on_demand)
PROFILE="on_demand"
stop_disable_all_dlp
enable_start_units "${DLP_ON_DEMAND_UNITS[@]}"
AW_DLP_PROFILE="on_demand" write_stats "enabled_on_demand"
;;
full|enabled|on)
PROFILE="full"
stop_disable_all_dlp
enable_start_units "${DLP_LIGHT_UNITS[@]}"
enable_start_units \
aw-dlp-influx-exporter.timer \
activitywatch-dlp-aggregator.timer \
aw-dlp-report-scheduler.timer \
aw-dlp-syslog-forwarder.timer \
aw-dlp-webhook-sender.timer \
aw-dlp-cef-exporter.timer \
aw-dlp-policy-engine.service \
aw-dlp-case-management.service \
detmir-portal-evidence.service
AW_DLP_PROFILE="full" write_stats "enabled_full"
;;
*)
printf 'unsupported DLP profile: %s\n' "$target_profile" >&2
printf 'supported profiles: core_only, light, on_demand, full\n' >&2
exit 2
;;
esac
}
disable_dlp() {
apply_profile "core_only"
}
enable_dlp() {
apply_profile "full"
}
rollback_dlp() {
local unit load active enabled
if [[ ! -s "$ROLLBACK_FILE" ]]; then
printf 'rollback state not found: %s\n' "$ROLLBACK_FILE" >&2
exit 1
fi
stop_disable_all_dlp
while IFS='|' read -r unit load active enabled; do
[[ -n "${unit:-}" && "${unit:0:1}" != "#" ]] || continue
[[ "$load" != "not-found" ]] || continue
if [[ "$enabled" == "enabled" ]]; then
systemctl enable "$unit" >/dev/null 2>&1 || true
fi
if [[ "$active" == "active" ]]; then
systemctl start "$unit" >/dev/null 2>&1 || true
fi
done <"$ROLLBACK_FILE"
write_stats "rollback"
}
case "$ACTION" in
status|stats)
write_stats "current"
;;
profile)
printf '%s\n' "${AW_DLP_PROFILE:-$PROFILE}"
;;
set-profile)
apply_profile "$PROFILE"
;;
disable)
disable_dlp
;;
enable)
enable_dlp
;;
rollback)
rollback_dlp
;;
*)
printf 'Usage: %s [status|stats|profile|set-profile <core_only|light|on_demand|full>|disable|enable|rollback]\n' "$0" >&2
exit 2
;;
esac
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env bash
set -euo pipefail
SOURCE_HOST="${AW_DLP_WAREHOUSE_SOURCE_HOST:-igor@10.10.10.13}"
SOURCE_PATH="${AW_DLP_WAREHOUSE_SOURCE_PATH:-/var/lib/activitywatch/dlp_warehouse.sqlite}"
DEST_PATH="${AW_DLP_WAREHOUSE_DEST_PATH:-/var/lib/activitywatch/dlp_warehouse.sqlite}"
STATE_DIR="${AW_DLP_WAREHOUSE_SYNC_STATE_DIR:-/var/lib/activitywatch/health}"
STATE_FILE="${AW_DLP_WAREHOUSE_SYNC_STATE_FILE:-${STATE_DIR}/dlp-warehouse-sync-state.json}"
SSH_OPTS="${AW_DLP_WAREHOUSE_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=5}"
REMOTE_TMP="/tmp/dlp_warehouse_sync_$$.sqlite"
LOCAL_TMP=""
json_string() {
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
}
write_state() {
local status="$1"
local message="$2"
local rows="${3:-}"
local bytes="${4:-}"
local now tmp
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
mkdir -p "$STATE_DIR"
tmp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")"
{
printf '{'
printf '"generated_at_utc":%s,' "$(json_string "$now")"
printf '"status":%s,' "$(json_string "$status")"
printf '"message":%s,' "$(json_string "$message")"
printf '"source_host":%s,' "$(json_string "$SOURCE_HOST")"
printf '"source_path":%s,' "$(json_string "$SOURCE_PATH")"
printf '"dest_path":%s,' "$(json_string "$DEST_PATH")"
if [[ "$rows" =~ ^[0-9]+$ ]]; then
printf '"dlp_events":%s,' "$rows"
else
printf '"dlp_events":null,'
fi
if [[ "$bytes" =~ ^[0-9]+$ ]]; then
printf '"bytes":%s' "$bytes"
else
printf '"bytes":null'
fi
printf '}\n'
} >"$tmp"
mv "$tmp" "$STATE_FILE"
}
cleanup_remote() {
ssh $SSH_OPTS "$SOURCE_HOST" "rm -f '$REMOTE_TMP'" >/dev/null 2>&1 || true
}
main() {
local dest_dir rows bytes
dest_dir="$(dirname "$DEST_PATH")"
mkdir -p "$dest_dir" "$STATE_DIR"
LOCAL_TMP="$(mktemp "${DEST_PATH}.tmp.XXXXXX")"
trap 'rm -f "${LOCAL_TMP:-}"; cleanup_remote' EXIT
ssh $SSH_OPTS "$SOURCE_HOST" \
"set -euo pipefail; if command -v sqlite3 >/dev/null 2>&1; then sqlite3 '$SOURCE_PATH' \".backup '$REMOTE_TMP'\" || cp -f '$SOURCE_PATH' '$REMOTE_TMP'; else cp -f '$SOURCE_PATH' '$REMOTE_TMP'; fi; test -s '$REMOTE_TMP'"
scp $SSH_OPTS "$SOURCE_HOST:$REMOTE_TMP" "$LOCAL_TMP"
chmod 0644 "$LOCAL_TMP"
mv "$LOCAL_TMP" "$DEST_PATH"
bytes="$(stat -c %s "$DEST_PATH" 2>/dev/null || printf '')"
rows="$(sqlite3 "$DEST_PATH" 'select count(*) from dlp_events;' 2>/dev/null || printf '')"
write_state "ok" "synced" "$rows" "$bytes"
printf 'dlp warehouse synced: source=%s:%s dest=%s rows=%s bytes=%s\n' \
"$SOURCE_HOST" "$SOURCE_PATH" "$DEST_PATH" "${rows:-unknown}" "${bytes:-unknown}"
}
main "$@"
+52 -14
View File
@@ -53,6 +53,26 @@ done
log() { printf "%s %s\n" "$(date +"%F %T")" "$*" >&2; }
die() { log "ERROR: $*"; exit 1; }
is_truthy() {
case "${1:-}" in
1|true|TRUE|yes|YES|on|ON) return 0 ;;
*) return 1 ;;
esac
}
require_real_value() {
local name="$1"
local value="${!name:-}"
if [[ -z "$value" ]]; then
die "missing required variable: $name"
fi
case "$value" in
*192.0.2.*|*198.51.100.*|*203.0.113.*|*HOST-EXAMPLE*|*.example*)
die "refusing placeholder value for $name: $value"
;;
esac
}
command -v ansible >/dev/null 2>&1 || die "ansible not found"
command -v ansible-playbook >/dev/null 2>&1 || die "ansible-playbook not found"
[[ -f "$INVENTORY" ]] || die "inventory not found: $INVENTORY"
@@ -68,10 +88,14 @@ restart_server_components() {
"activitywatch-server"
"aw-worktime-api"
"aw-worktime-ui-bridge.timer"
"aw-dlp-policy-engine.service"
"aw-dlp-aggregator.timer"
"activitywatch-dlp-aggregator.timer"
)
if is_truthy "${DETMIR_DLP_ENABLED:-${AW_DLP_ENABLED:-false}}"; then
units+=(
"aw-dlp-policy-engine.service"
"aw-dlp-aggregator.timer"
"activitywatch-dlp-aggregator.timer"
)
fi
for unit in "${units[@]}"; do
if ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.command -a "systemctl status ${unit}" >/dev/null 2>&1; then
ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.systemd -a "name=${unit} state=restarted enabled=true" || true
@@ -80,24 +104,32 @@ restart_server_components() {
}
seed_server_dlp_events() {
if ! is_truthy "${ALLOW_DLP_SEED_EVENTS:-0}"; then
log "Skipping DLP freshness seeding; set ALLOW_DLP_SEED_EVENTS=1 with real DETMIR_HOSTNAME/DETMIR_AW_SERVER_HOST to allow it."
return 0
fi
require_real_value DETMIR_HOSTNAME
require_real_value DETMIR_AW_SERVER_HOST
log "Seeding DLP freshness events on aw_server..."
local ts
local ts host server_host
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
host="${DETMIR_HOSTNAME}"
server_host="${DETMIR_AW_SERVER_HOST}"
ansible -i "$INVENTORY" aw_server -b -m ansible.builtin.shell -a "cat >/tmp/aw-endpoint-seed.json <<'JSON'
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"HOST-EXAMPLE\",\"signalType\":\"self_test\",\"source\":\"diag_and_manual_restart\",\"username\":\"system\",\"queueDepth\":0,\"eventsEnqueued\":0,\"eventsFlushed\":0,\"sendFailures\":0}}
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"${host}\",\"signalType\":\"self_test\",\"source\":\"diag_and_manual_restart\",\"username\":\"system\",\"queueDepth\":0,\"eventsEnqueued\":0,\"eventsFlushed\":0,\"sendFailures\":0}}
JSON
cat >/tmp/aw-fileops-seed-host.json <<'JSON'
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"HOST-EXAMPLE\",\"operation\":\"self_test\",\"source\":\"diag_and_manual_restart\"}}
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"${host}\",\"operation\":\"self_test\",\"source\":\"diag_and_manual_restart\"}}
JSON
cat >/tmp/aw-fileops-seed-server.json <<'JSON'
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"192.0.2.13\",\"operation\":\"self_test\",\"source\":\"diag_and_manual_restart\"}}
{\"timestamp\":\"${ts}\",\"duration\":0.0,\"data\":{\"hostname\":\"${server_host}\",\"operation\":\"self_test\",\"source\":\"diag_and_manual_restart\"}}
JSON
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_HOST-EXAMPLE' -H 'Content-Type: application/json' -d '{\"client\":\"aw-dlp-endpoint-signals\",\"type\":\"aw.dlp.endpoint.signal\",\"hostname\":\"HOST-EXAMPLE\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_HOST-EXAMPLE' -H 'Content-Type: application/json' -d '{\"client\":\"aw-file-operations\",\"type\":\"aw.file.operation\",\"hostname\":\"HOST-EXAMPLE\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_192.0.2.13' -H 'Content-Type: application/json' -d '{\"client\":\"aw-file-operations\",\"type\":\"aw.file.operation\",\"hostname\":\"192.0.2.13\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_HOST-EXAMPLE/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-endpoint-seed.json >/dev/null
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_HOST-EXAMPLE/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-host.json >/dev/null
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_192.0.2.13/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-server.json >/dev/null
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_${host}' -H 'Content-Type: application/json' -d '{\"client\":\"aw-dlp-endpoint-signals\",\"type\":\"aw.dlp.endpoint.signal\",\"hostname\":\"${host}\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_${host}' -H 'Content-Type: application/json' -d '{\"client\":\"aw-file-operations\",\"type\":\"aw.file.operation\",\"hostname\":\"${host}\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_${server_host}' -H 'Content-Type: application/json' -d '{\"client\":\"aw-file-operations\",\"type\":\"aw.file.operation\",\"hostname\":\"${server_host}\"}' >/dev/null 2>&1 || true
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-dlp-endpoint-signals_${host}/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-endpoint-seed.json >/dev/null
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_${host}/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-host.json >/dev/null
curl -sS -X POST 'http://127.0.0.1:5600/api/0/buckets/aw-file-operations_${server_host}/heartbeat?pulsetime=30' -H 'Content-Type: application/json' --data-binary @/tmp/aw-fileops-seed-server.json >/dev/null
" >/dev/null
}
@@ -107,8 +139,14 @@ restart_windows_collectors() {
}
seed_windows_dlp_events() {
if ! is_truthy "${ALLOW_DLP_SEED_EVENTS:-0}"; then
log "Skipping Windows DLP freshness seeding; set ALLOW_DLP_SEED_EVENTS=1 with real DETMIR_HOSTNAME/DETMIR_AW_API to allow it."
return 0
fi
require_real_value DETMIR_HOSTNAME
require_real_value DETMIR_AW_API
log "Seeding endpoint/file-ops events from aw_windows..."
ansible -i "$INVENTORY" aw_windows -m ansible.windows.win_shell -a "powershell -NoProfile -ExecutionPolicy Bypass -Command \"\$ErrorActionPreference = 'Stop'; \$ts = (Get-Date).ToUniversalTime().ToString('o'); \$api='http://192.0.2.13:5600/api/0'; \$endpoint=@{timestamp=\$ts;duration=0.0;data=@{hostname='HOST-EXAMPLE';signalType='self_test';source='diag_and_manual_restart';username=\$env:USERNAME;queueDepth=0;eventsEnqueued=0;eventsFlushed=0;sendFailures=0}} | ConvertTo-Json -Depth 8 -Compress; \$fileops=@{timestamp=\$ts;duration=0.0;data=@{hostname='HOST-EXAMPLE';operation='self_test';source='diag_and_manual_restart';username=\$env:USERNAME}} | ConvertTo-Json -Depth 8 -Compress; Invoke-RestMethod -Method Post -Uri \$api'/buckets/aw-dlp-endpoint-signals_HOST-EXAMPLE' -ContentType 'application/json' -Body '{\\\"client\\\":\\\"aw-dlp-endpoint-signals\\\",\\\"type\\\":\\\"aw.dlp.endpoint.signal\\\",\\\"hostname\\\":\\\"HOST-EXAMPLE\\\"}' -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri \$api'/buckets/aw-file-operations_HOST-EXAMPLE' -ContentType 'application/json' -Body '{\\\"client\\\":\\\"aw-file-operations\\\",\\\"type\\\":\\\"aw.file.operation\\\",\\\"hostname\\\":\\\"HOST-EXAMPLE\\\"}' -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri \$api'/buckets/aw-dlp-endpoint-signals_HOST-EXAMPLE/heartbeat?pulsetime=30' -ContentType 'application/json' -Body \$endpoint -TimeoutSec 15 -DisableKeepAlive | Out-Null; Invoke-RestMethod -Method Post -Uri \$api'/buckets/aw-file-operations_HOST-EXAMPLE/heartbeat?pulsetime=30' -ContentType 'application/json' -Body \$fileops -TimeoutSec 15 -DisableKeepAlive | Out-Null; Write-Output 'windows-dlp-seeded'\""
ansible -i "$INVENTORY" aw_windows -m ansible.windows.win_shell -a "powershell -NoProfile -ExecutionPolicy Bypass -Command \"\$ErrorActionPreference = 'Stop'; \$ts = (Get-Date).ToUniversalTime().ToString('o'); \$api='${DETMIR_AW_API}'; \$hostName='${DETMIR_HOSTNAME}'; \$endpointBucket=\$api + '/buckets/aw-dlp-endpoint-signals_' + \$hostName; \$fileopsBucket=\$api + '/buckets/aw-file-operations_' + \$hostName; \$endpoint=@{timestamp=\$ts;duration=0.0;data=@{hostname=\$hostName;signalType='self_test';source='diag_and_manual_restart';username=\$env:USERNAME;queueDepth=0;eventsEnqueued=0;eventsFlushed=0;sendFailures=0}} | ConvertTo-Json -Depth 8 -Compress; \$fileops=@{timestamp=\$ts;duration=0.0;data=@{hostname=\$hostName;operation='self_test';source='diag_and_manual_restart';username=\$env:USERNAME}} | ConvertTo-Json -Depth 8 -Compress; Invoke-RestMethod -Method Post -Uri \$endpointBucket -ContentType 'application/json' -Body (@{client='aw-dlp-endpoint-signals';type='aw.dlp.endpoint.signal';hostname=\$hostName} | ConvertTo-Json -Compress) -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri \$fileopsBucket -ContentType 'application/json' -Body (@{client='aw-file-operations';type='aw.file.operation';hostname=\$hostName} | ConvertTo-Json -Compress) -TimeoutSec 15 -DisableKeepAlive -ErrorAction SilentlyContinue | Out-Null; Invoke-RestMethod -Method Post -Uri (\$endpointBucket + '/heartbeat?pulsetime=30') -ContentType 'application/json' -Body \$endpoint -TimeoutSec 15 -DisableKeepAlive | Out-Null; Invoke-RestMethod -Method Post -Uri (\$fileopsBucket + '/heartbeat?pulsetime=30') -ContentType 'application/json' -Body \$fileops -TimeoutSec 15 -DisableKeepAlive | Out-Null; Write-Output 'windows-dlp-seeded'\""
}
confirm_restart() {
+23 -3
View File
@@ -4,14 +4,29 @@ set -euo pipefail
DAY=""
FROM=""
TO=""
AW_BASE_URL="${AW_BASE_URL:-http://192.0.2.13:5600/api/0}"
AW_WORKTIME_HOST="${AW_WORKTIME_HOST:-HOST-EXAMPLE}"
AW_BASE_URL="${AW_BASE_URL:-}"
AW_WORKTIME_HOST="${AW_WORKTIME_HOST:-}"
AW_WORKTIME_DEFAULT_SAMPLE_SECONDS="${AW_WORKTIME_DEFAULT_SAMPLE_SECONDS:-30}"
AW_WORKTIME_MAX_SAMPLE_SECONDS="${AW_WORKTIME_MAX_SAMPLE_SECONDS:-300}"
OUT_DIR="${OUT_DIR:-reports}"
TARGET_ROOT="${CARGO_TARGET_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/adk-rust/target}"
RUST_BIN="${RDP_WORKTIME_REPORT_RUST:-}"
require_live_value() {
local name="$1"
local value="${!name:-}"
if [[ -z "$value" ]]; then
echo "Missing required variable: $name" >&2
exit 2
fi
case "$value" in
*192.0.2.*|*198.51.100.*|*203.0.113.*|*HOST-EXAMPLE*|*.example*)
echo "Refusing placeholder value for $name: $value" >&2
exit 2
;;
esac
}
usage() {
cat <<EOF
Usage:
@@ -52,6 +67,9 @@ if [[ -z "$FROM" || -z "$TO" ]]; then
exit 2
fi
require_live_value AW_BASE_URL
require_live_value AW_WORKTIME_HOST
mkdir -p "$OUT_DIR"
CSV_OUT="${OUT_DIR}/rdp-worktime-${FROM}_${TO}.csv"
JSON_OUT="${OUT_DIR}/rdp-worktime-${FROM}_${TO}.json"
@@ -80,7 +98,9 @@ import urllib.request
from datetime import datetime, timedelta, timezone
base, host, default_sample, max_sample, from_d, to_d, csv_out, json_out = sys.argv[1:9]
base = (base or "http://192.0.2.13:5600").rstrip("/")
if not base:
raise SystemExit("AW_BASE_URL is required")
base = base.rstrip("/")
if not base.endswith("/api/0"):
base = base + "/api/0"
default_sample = max(1.0, float(default_sample))
+29 -5
View File
@@ -43,11 +43,27 @@ configure_detmir_env() {
fi
fi
export DETMIR_AW_API="${DETMIR_AW_API:-http://192.0.2.13:5600/api/0}"
export DETMIR_WORKTIME_URL="${DETMIR_WORKTIME_URL:-http://192.0.2.13:5610}"
export DETMIR_ONE_C_URL="${DETMIR_ONE_C_URL:-http://192.0.2.2:8710}"
export DETMIR_RDP_HOST="${DETMIR_RDP_HOST:-198.51.100.18}"
export DETMIR_HOSTNAME="${DETMIR_HOSTNAME:-HOST-EXAMPLE}"
export DETMIR_DLP_ENABLED="${DETMIR_DLP_ENABLED:-${AW_DLP_ENABLED:-false}}"
require_live_value DETMIR_AW_API
require_live_value DETMIR_WORKTIME_URL
require_live_value DETMIR_ONE_C_URL
require_live_value DETMIR_RDP_HOST
require_live_value DETMIR_HOSTNAME
}
require_live_value() {
local name="$1"
local value="${!name:-}"
if [[ -z "${value}" ]]; then
printf 'Missing required live contour variable: %s. Set it in %s or the environment.\n' "${name}" "${ENV_FILE}" >&2
exit 2
fi
case "${value}" in
*192.0.2.*|*198.51.100.*|*203.0.113.*|*HOST-EXAMPLE*|*.example*)
printf 'Refusing placeholder value for %s: %s\n' "${name}" "${value}" >&2
exit 2
;;
esac
}
write_summary() {
@@ -64,6 +80,7 @@ write_summary() {
printf 'DETMIR_HOSTNAME=%s\n' "${DETMIR_HOSTNAME}"
printf 'DETMIR_GATEWAY_HOST=%s\n' "${DETMIR_GATEWAY_HOST}"
printf 'DETMIR_PORTAL_URL=%s\n' "${DETMIR_PORTAL_URL}"
printf 'DETMIR_DLP_ENABLED=%s\n' "${DETMIR_DLP_ENABLED}"
printf 'DETMIR_DLP_COMMAND=%s\n' "${DETMIR_DLP_COMMAND}"
printf 'DETMIR_DISABLE_PORTAL_CHECK=%s\n' "${DETMIR_DISABLE_PORTAL_CHECK:-0}"
printf 'DETMIR_DISABLE_DLP_HEALTH_CHECK=%s\n' "${DETMIR_DISABLE_DLP_HEALTH_CHECK:-0}"
@@ -181,5 +198,12 @@ if [[ "${RUN_REGISTRY_CHECK:-0}" == "1" ]] && [[ -x "${REPO_ROOT}/scripts/regist
fi
fi
if [[ "${RUN_RESILIENCE_CHECK:-0}" == "1" ]] && [[ -f "${REPO_ROOT}/scripts/detmir_resilience_check.sh" ]]; then
resilience_mode="${RESILIENCE_CHECK_MODE:-repo}"
if ! run_and_log "detmir-resilience-check" bash "${REPO_ROOT}/scripts/detmir_resilience_check.sh" "--${resilience_mode}"; then
status=1
fi
fi
printf 'final_status: %s\n' "$([[ "${status}" -eq 0 ]] && printf ok || printf fail)" | tee -a "${OUTPUT_DIR}/SUMMARY.md"
exit "${status}"