refactor(detmir): retire python runtime paths

This commit is contained in:
igor04091968
2026-06-03 03:27:52 +03:00
parent 109c31f291
commit dbef90a09e
103 changed files with 914 additions and 19800 deletions
-517
View File
@@ -1,517 +0,0 @@
#!/usr/bin/env python3
import argparse
import json
import os
import sqlite3
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Iterable
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Protocol, TypeAlias
JsonScalar: TypeAlias = str | int | float | bool | None
JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"]
DEFAULT_BUCKET_PREFIXES = ("aw-file-operations_", "aw-dlp-incidents_")
DEFAULT_SQLITE_PATH = "data/dlp-events.sqlite3"
EVENT_COLUMNS = (
"bucket_id",
"event_id",
"stream_type",
"hostname",
"username",
"event_ts",
"duration",
"operation",
"file_path",
"old_file_path",
"extension",
"archive_hint",
"rule_id",
"action",
"severity",
"signal_type",
"message",
"source",
"screenshot_path",
"raw_json",
"ingested_at",
)
@dataclass(frozen=True)
class Bucket:
id: str
type: str
client: str
hostname: str
@dataclass(frozen=True)
class AwEvent:
bucket_id: str
hostname: str
stream_type: str
event_id: str
timestamp: str
duration: float
data: dict[str, JsonValue]
class PsycopgConnection(Protocol):
def cursor(self):
...
def commit(self) -> None:
...
def utc_now() -> datetime:
return datetime.now(tz=UTC)
def parse_timestamp(value: str) -> datetime:
normalized = value.replace("Z", "+00:00")
parsed = datetime.fromisoformat(normalized)
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def format_aw_timestamp(value: datetime) -> str:
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
def load_state(path: Path) -> dict[str, str]:
if not path.exists():
return {}
return json.loads(path.read_text(encoding="utf-8"))
def save_state(path: Path, state: dict[str, str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def normalize_base_url(base_url: str) -> str:
return base_url.rstrip("/")
def aw_get_json(base_url: str, path: str, timeout: int) -> JsonValue:
url = normalize_base_url(base_url) + path
request = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def list_buckets(base_url: str, timeout: int) -> list[Bucket]:
payload = aw_get_json(base_url, "/buckets", timeout)
if not isinstance(payload, dict):
raise ValueError("ActivityWatch /buckets response must be a JSON object")
buckets: list[Bucket] = []
for bucket_id, bucket_data in payload.items():
if not isinstance(bucket_data, dict):
continue
buckets.append(
Bucket(
id=str(bucket_id),
type=str(bucket_data.get("type", "")),
client=str(bucket_data.get("client", "")),
hostname=str(bucket_data.get("hostname", "")),
)
)
return buckets
def bucket_stream_type(bucket: Bucket) -> str | None:
if bucket.id.startswith("aw-file-operations_") or bucket.type == "aw.file.operation":
return "file_operation"
if bucket.id.startswith("aw-dlp-incidents_") or bucket.type == "aw.dlp.incident":
return "dlp_incident"
return None
def select_buckets(buckets: Iterable[Bucket], prefixes: tuple[str, ...]) -> list[tuple[Bucket, str]]:
selected: list[tuple[Bucket, str]] = []
for bucket in buckets:
stream_type = bucket_stream_type(bucket)
if stream_type and any(bucket.id.startswith(prefix) for prefix in prefixes):
selected.append((bucket, stream_type))
return selected
def build_events_path(bucket_id: str, start: datetime, end: datetime, limit: int) -> str:
query = urllib.parse.urlencode(
{
"start": format_aw_timestamp(start),
"end": format_aw_timestamp(end),
"limit": str(limit),
}
)
return f"/buckets/{urllib.parse.quote(bucket_id, safe='')}/events?{query}"
def event_key(bucket_id: str, timestamp: str, duration: float, data: dict[str, JsonValue]) -> str:
payload = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return f"{bucket_id}|{timestamp}|{duration}|{payload}"
def fetch_bucket_events(
base_url: str,
bucket: Bucket,
stream_type: str,
start: datetime,
end: datetime,
limit: int,
timeout: int,
) -> list[AwEvent]:
payload = aw_get_json(base_url, build_events_path(bucket.id, start, end, limit), timeout)
if not isinstance(payload, list):
raise ValueError(f"ActivityWatch events response for {bucket.id} must be a JSON array")
events: list[AwEvent] = []
for item in payload:
if not isinstance(item, dict):
continue
timestamp = str(item["timestamp"])
duration = float(item.get("duration", 0) or 0)
data = item.get("data") or {}
if not isinstance(data, dict):
data = {"raw": data}
item_id = str(item.get("id") or event_key(bucket.id, timestamp, duration, data))
events.append(
AwEvent(
bucket_id=bucket.id,
hostname=bucket.hostname or str(data.get("hostname") or ""),
stream_type=stream_type,
event_id=item_id,
timestamp=timestamp,
duration=duration,
data=data,
)
)
return events
def connect_sqlite(path: Path) -> sqlite3.Connection:
path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(str(path))
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("PRAGMA synchronous=NORMAL")
connection.execute("PRAGMA foreign_keys=ON")
return connection
def ensure_schema(connection: sqlite3.Connection) -> None:
connection.executescript(
"""
create table if not exists dlp_events (
id integer primary key autoincrement,
bucket_id text not null,
event_id text not null,
stream_type text not null,
hostname text not null,
username text,
event_ts text not null,
duration real not null default 0,
operation text,
file_path text,
old_file_path text,
extension text,
archive_hint integer not null default 0,
rule_id text,
action text,
severity text,
signal_type text,
message text,
source text,
screenshot_path text,
raw_json text not null,
ingested_at text not null,
unique (bucket_id, event_id)
);
create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts);
create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts);
create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts);
create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts);
create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts);
create view if not exists dlp_file_operations as
select *
from dlp_events
where stream_type = 'file_operation';
create view if not exists dlp_incidents as
select *
from dlp_events
where stream_type = 'dlp_incident';
"""
)
connection.commit()
def ensure_postgres_schema(connection: PsycopgConnection) -> None:
with connection.cursor() as cursor:
cursor.execute(
"""
create table if not exists dlp_events (
id bigserial primary key,
bucket_id text not null,
event_id text not null,
stream_type text not null,
hostname text not null,
username text,
event_ts timestamptz not null,
duration double precision not null default 0,
operation text,
file_path text,
old_file_path text,
extension text,
archive_hint boolean not null default false,
rule_id text,
action text,
severity text,
signal_type text,
message text,
source text,
screenshot_path text,
raw_json jsonb not null,
ingested_at timestamptz not null,
unique (bucket_id, event_id)
);
create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts);
create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts);
create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts);
create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts);
create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts);
create or replace view dlp_file_operations as
select *
from dlp_events
where stream_type = 'file_operation';
create or replace view dlp_incidents as
select *
from dlp_events
where stream_type = 'dlp_incident';
"""
)
connection.commit()
def first_string(data: dict[str, JsonValue], keys: tuple[str, ...]) -> str | None:
for key in keys:
value = data.get(key)
if value is not None and str(value) != "":
return str(value)
return None
def bool_as_int(value: JsonValue) -> int:
if isinstance(value, bool):
return int(value)
if isinstance(value, str):
return int(value.lower() in {"1", "true", "yes", "y"})
return int(bool(value))
def event_row(event: AwEvent, ingested_at: str) -> tuple[JsonValue, ...]:
data = event.data
event_id = event.event_id or event_key(event.bucket_id, event.timestamp, event.duration, data)
return (
event.bucket_id,
event_id,
event.stream_type,
event.hostname,
first_string(data, ("username", "user")),
event.timestamp,
event.duration,
first_string(data, ("operation",)),
first_string(data, ("path", "filePath")),
first_string(data, ("oldPath", "oldFilePath")),
first_string(data, ("extension",)),
bool_as_int(data.get("archiveHint")),
first_string(data, ("ruleId", "rule")),
first_string(data, ("action",)),
first_string(data, ("severity",)),
first_string(data, ("signalType",)),
first_string(data, ("message",)),
first_string(data, ("source",)),
first_string(data, ("screenshotPath", "capturePath", "artifactPath")),
json.dumps(data, ensure_ascii=False, sort_keys=True),
ingested_at,
)
def insert_events(connection: sqlite3.Connection, events: Iterable[AwEvent]) -> int:
inserted = 0
now = format_aw_timestamp(utc_now())
for event in events:
cursor = connection.execute(
"""
insert or ignore into dlp_events (
bucket_id,
event_id,
stream_type,
hostname,
username,
event_ts,
duration,
operation,
file_path,
old_file_path,
extension,
archive_hint,
rule_id,
action,
severity,
signal_type,
message,
source,
screenshot_path,
raw_json,
ingested_at
)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
event_row(event, now),
)
inserted += int(cursor.rowcount > 0)
connection.commit()
return inserted
def insert_postgres_events(dsn: str, events: Iterable[AwEvent]) -> int:
try:
import psycopg
except ImportError as exc:
raise SystemExit("PostgreSQL mode requires psycopg: python3 -m pip install 'psycopg[binary]'") from exc
inserted = 0
now = format_aw_timestamp(utc_now())
columns = ", ".join(EVENT_COLUMNS)
placeholders = ", ".join(["%s"] * len(EVENT_COLUMNS))
sql = f"""
insert into dlp_events ({columns})
values ({placeholders})
on conflict (bucket_id, event_id) do nothing
"""
with psycopg.connect(dsn) as connection:
ensure_postgres_schema(connection)
with connection.cursor() as cursor:
for event in events:
row = list(event_row(event, now))
row[EVENT_COLUMNS.index("archive_hint")] = bool(row[EVENT_COLUMNS.index("archive_hint")])
cursor.execute(sql, row)
inserted += int(cursor.rowcount > 0)
connection.commit()
return inserted
def get_start_time(args: argparse.Namespace, state: dict[str, str]) -> datetime:
if args.since:
return parse_timestamp(args.since)
if state.get("last_end"):
return parse_timestamp(state["last_end"]) - timedelta(seconds=args.overlap_seconds)
return utc_now() - timedelta(hours=args.lookback_hours)
def parse_prefixes(value: str) -> tuple[str, ...]:
prefixes = tuple(item.strip() for item in value.split(",") if item.strip())
if not prefixes:
raise argparse.ArgumentTypeError("at least one bucket prefix is required")
return prefixes
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Aggregate AWatch-rus DLP buckets into a local warehouse database.")
parser.add_argument("--aw-url", default=os.environ.get("AW_URL", "http://127.0.0.1:5600/api/0"))
parser.add_argument("--postgres-dsn", default=os.environ.get("DLP_AGGREGATOR_POSTGRES_DSN"))
parser.add_argument("--sqlite-path", default=os.environ.get("DLP_AGGREGATOR_SQLITE_PATH", DEFAULT_SQLITE_PATH))
parser.add_argument("--state-path", default=os.environ.get("DLP_AGGREGATOR_STATE_PATH", "data/dlp-aggregator-state.json"))
parser.add_argument("--bucket-prefixes", type=parse_prefixes, default=DEFAULT_BUCKET_PREFIXES)
parser.add_argument("--since", help="UTC ISO timestamp. Overrides saved state, for example 2026-05-02T00:00:00Z.")
parser.add_argument("--lookback-hours", type=int, default=24)
parser.add_argument("--overlap-seconds", type=int, default=60)
parser.add_argument("--limit", type=int, default=10000)
parser.add_argument("--timeout", type=int, default=15)
parser.add_argument("--dry-run", action="store_true")
return parser
def main() -> int:
args = build_parser().parse_args()
state_path = Path(args.state_path)
state = load_state(state_path)
start = get_start_time(args, state)
end = utc_now()
buckets = select_buckets(list_buckets(args.aw_url, args.timeout), args.bucket_prefixes)
all_events: list[AwEvent] = []
for bucket, stream_type in buckets:
all_events.extend(fetch_bucket_events(args.aw_url, bucket, stream_type, start, end, args.limit, args.timeout))
if args.dry_run:
print(
json.dumps(
{
"aw_url": args.aw_url,
"start": format_aw_timestamp(start),
"end": format_aw_timestamp(end),
"selected_buckets": [bucket.id for bucket, _stream_type in buckets],
"fetched_events": len(all_events),
},
ensure_ascii=False,
indent=2,
)
)
return 0
if args.postgres_dsn:
target = "postgres"
target_path = args.postgres_dsn.split("@")[-1]
inserted = insert_postgres_events(args.postgres_dsn, all_events)
else:
target = "sqlite"
sqlite_path = Path(args.sqlite_path)
target_path = str(sqlite_path)
connection = connect_sqlite(sqlite_path)
try:
ensure_schema(connection)
inserted = insert_events(connection, all_events)
finally:
connection.close()
state["last_end"] = format_aw_timestamp(end)
save_state(state_path, state)
print(
json.dumps(
{
"aw_url": args.aw_url,
"target": target,
"target_path": target_path,
"state_path": str(state_path),
"start": format_aw_timestamp(start),
"end": format_aw_timestamp(end),
"selected_buckets": len(buckets),
"fetched_events": len(all_events),
"inserted_events": inserted,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except urllib.error.URLError as exc:
print(f"ActivityWatch API request failed: {exc}", file=sys.stderr)
raise SystemExit(2)
+2 -5
View File
@@ -43,8 +43,5 @@ for candidate in "${rust_candidates[@]}"; do
fi
done
python3 "$REPO_ROOT/scripts/extract_ioc_from_sigma.py" \
--rules-root "$RULES_ROOT" \
--out-dir "$OUT_DIR"
echo "IOC artifacts generated in: $OUT_DIR"
echo "ERROR: Rust extractor not found. Build it with: cd '$REPO_ROOT/adk-rust' && cargo build --release -p extract-ioc-from-sigma" >&2
exit 127
+2 -56
View File
@@ -24,59 +24,5 @@ for candidate in "${rust_candidates[@]}"; do
fi
done
python3 - <<'PY'
from pathlib import Path
import hashlib
import sys
root=Path('.')
kit=Path('install-kit-awindows-20260427-211240')
if not kit.exists():
raise SystemExit('Install kit directory not found')
def sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
all_compared=[]
mismatches=[]
missing_in_repo=[]
allowed_kit_only_prefixes=('server-configs-' ,)
allowed_kit_only_files={'README-INSTALL-KIT.txt'}
for kp in sorted(p for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'):
rel=kp.relative_to(kit)
rp=root/rel
if not rp.exists():
rel_str=str(rel)
if rel_str in allowed_kit_only_files or rel_str.startswith(allowed_kit_only_prefixes):
continue
missing_in_repo.append(str(rel))
continue
all_compared.append(str(rel))
if sha(kp)!=sha(rp):
mismatches.append(str(rel))
ps_mismatches=[
p for p in mismatches
if p.startswith('windows/') and p.endswith(('.ps1', '.psm1', '.psd1'))
]
print(f'Compared files: {len(all_compared)}')
print(f'Missing in repo: {len(missing_in_repo)}')
print(f'Mismatched content: {len(mismatches)}')
if missing_in_repo:
print('--- Missing in repo ---')
for p in missing_in_repo:
print(p)
if mismatches:
print('--- Mismatches ---')
for p in mismatches:
print(p)
print(f'PowerShell mismatches: {len(ps_mismatches)}')
if ps_mismatches:
print('--- PowerShell mismatches ---')
for p in ps_mismatches:
print(p)
if missing_in_repo or mismatches:
sys.exit(1)
PY
echo "ERROR: Rust checker not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p check-install-kit-vs-repo" >&2
exit 127
-158
View File
@@ -1,158 +0,0 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from datetime import UTC, datetime, timedelta
from urllib import request
from urllib.parse import quote
def get_json(url: str):
with request.urlopen(url, timeout=30) as r:
return json.loads(r.read().decode("utf-8"))
def send_json(url: str, method: str, payload: dict | None = None):
body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8")
req = request.Request(url, data=body, method=method, headers={"Content-Type": "application/json"})
with request.urlopen(req, timeout=30) as r:
raw = r.read().decode("utf-8")
return json.loads(raw) if raw else {}
def parse_iso(ts: str | None) -> datetime | None:
if not ts:
return None
try:
return datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone(UTC)
except ValueError:
return None
def main() -> None:
p = argparse.ArgumentParser(description="AWatch DLP admin CLI")
p.add_argument("--policy-server", default="http://127.0.0.1:5601")
p.add_argument("--case-server", default="http://127.0.0.1:5602")
p.add_argument("--aw-server", default="http://127.0.0.1:5600")
sub = p.add_subparsers(dest="cmd", required=True)
policies = sub.add_parser("policies")
policies_sub = policies.add_subparsers(dest="policies_cmd", required=True)
policies_sub.add_parser("list")
policies_sub.add_parser("active")
incidents = sub.add_parser("incidents")
incidents_sub = incidents.add_subparsers(dest="incidents_cmd", required=True)
incidents_list = incidents_sub.add_parser("list")
incidents_list.add_argument("--host")
incidents_list.add_argument("--severity")
incidents_list.add_argument("--limit", type=int, default=100)
incidents_list.add_argument("--since-hours", type=int, default=24)
cases = sub.add_parser("cases")
cases_sub = cases.add_subparsers(dest="cases_cmd", required=True)
cases_list = cases_sub.add_parser("list")
cases_list.add_argument("--host")
cases_list.add_argument("--status")
cases_list.add_argument("--limit", type=int, default=100)
cases_create = cases_sub.add_parser("create")
cases_create.add_argument("--incident-id", required=True)
cases_create.add_argument("--title", required=True)
cases_create.add_argument("--host")
cases_create.add_argument("--severity", default="medium")
health = sub.add_parser("health")
health_sub = health.add_subparsers(dest="health_cmd", required=True)
health_sub.add_parser("check")
args = p.parse_args()
if args.cmd == "policies" and args.policies_cmd == "list":
data = get_json(f"{args.policy_server}/api/0/dlp/policies")
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "policies" and args.policies_cmd == "active":
data = get_json(f"{args.policy_server}/api/0/dlp/policies/active")
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "incidents" and args.incidents_cmd == "list":
bucket_map = get_json(f"{args.aw_server}/api/0/buckets")
if not isinstance(bucket_map, dict):
print("[]")
return
bucket_ids = [x for x in bucket_map.keys() if str(x).startswith("aw-dlp-incidents_")]
if args.host:
bucket_ids = [x for x in bucket_ids if str(x).endswith("_" + args.host)]
after = datetime.now(UTC) - timedelta(hours=max(1, args.since_hours))
rows = []
for bucket_id in sorted(bucket_ids):
encoded = quote(str(bucket_id), safe="")
events = get_json(f"{args.aw_server}/api/0/buckets/{encoded}/events?limit={max(1, args.limit)}")
if not isinstance(events, list):
continue
for ev in events:
if not isinstance(ev, dict):
continue
ts = parse_iso(ev.get("timestamp"))
if ts is None or ts < after:
continue
data = ev.get("data") or {}
if args.severity and str((data or {}).get("severity", "")).lower() != args.severity.lower():
continue
rows.append(ev)
print(json.dumps(rows, ensure_ascii=False, indent=2))
return
if args.cmd == "cases" and args.cases_cmd == "list":
query = []
if args.host:
query.append(f"host={quote(args.host, safe='')}")
if args.status:
query.append(f"status={quote(args.status, safe='')}")
query.append(f"limit={max(1, args.limit)}")
data = get_json(f"{args.case_server}/api/0/dlp/cases?{'&'.join(query)}")
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "cases" and args.cases_cmd == "create":
payload = {
"incident_id": args.incident_id,
"title": args.title,
"host": args.host,
"severity": args.severity,
"evidence": {"source": "dlp-admin-cli"},
}
data = send_json(f"{args.case_server}/api/0/dlp/cases", "POST", payload)
print(json.dumps(data, ensure_ascii=False, indent=2))
return
if args.cmd == "health" and args.health_cmd == "check":
out = {}
try:
out["policy"] = get_json(f"{args.policy_server}/healthz")
except Exception as exc:
out["policy"] = {"status": "error", "error": str(exc)}
try:
out["cases"] = get_json(f"{args.case_server}/health")
except Exception as exc:
out["cases"] = {"status": "error", "error": str(exc)}
try:
out["aw"] = get_json(f"{args.aw_server}/api/0/info")
except Exception as exc:
out["aw"] = {"status": "error", "error": str(exc)}
print(json.dumps(out, ensure_ascii=False, indent=2))
return
raise SystemExit("unsupported command")
if __name__ == "__main__":
main()
File diff suppressed because it is too large Load Diff
-235
View File
@@ -1,235 +0,0 @@
#!/usr/bin/env python3
"""
Extract IOC-like indicators from Sigma YAML rules for DLP preload.
Extracted fields:
- Image|endswith
- CommandLine|contains
- OriginalFileName
- Hashes|SHA256 (plus SHA256 values embedded in Hashes strings)
"""
from __future__ import annotations
import argparse
import csv
import json
import re
from pathlib import Path
from typing import Any
import yaml
BASE_FIELDS = {"image", "commandline", "originalfilename", "hashes"}
SHA256_RE = re.compile(r"\b[a-fA-F0-9]{64}\b")
def split_key(key: str) -> tuple[str, list[str]]:
parts = [p.strip() for p in str(key).split("|") if p.strip()]
if not parts:
return "", []
return parts[0].lower(), [p.lower() for p in parts[1:]]
def flatten_values(value: Any) -> list[str]:
if value is None:
return []
if isinstance(value, str):
v = value.strip()
return [v] if v else []
if isinstance(value, (int, float, bool)):
return [str(value)]
if isinstance(value, list):
out: list[str] = []
for item in value:
out.extend(flatten_values(item))
return out
if isinstance(value, dict):
out: list[str] = []
for k, v in value.items():
vals = flatten_values(v)
for vv in vals:
out.append(f"{k}:{vv}")
return out
return []
def detect_ioc_type(base: str, ops: list[str], raw: str) -> str | None:
if base == "image" and "endswith" in ops:
return "process_image_endswith"
if base == "commandline" and "contains" in ops:
return "commandline_contains"
if base == "originalfilename":
return "original_filename"
if base == "hashes" and ("sha256" in ops or SHA256_RE.search(raw)):
return "sha256"
return None
def parse_sha256(raw: str) -> list[str]:
vals = SHA256_RE.findall(raw)
seen = set()
out = []
for v in vals:
lv = v.lower()
if lv in seen:
continue
seen.add(lv)
out.append(lv)
return out
def walk(node: Any, *, rule_id: str, rule_title: str, source_file: str, out: list[dict[str, str]]) -> None:
if isinstance(node, dict):
for k, v in node.items():
base, ops = split_key(str(k))
if base in BASE_FIELDS:
for raw in flatten_values(v):
ioc_type = detect_ioc_type(base, ops, raw)
if not ioc_type:
continue
if ioc_type == "sha256":
for h in parse_sha256(raw):
out.append(
{
"ioc_type": "sha256",
"ioc_value": h,
"field": str(k),
"rule_id": rule_id,
"rule_title": rule_title,
"source_file": source_file,
}
)
else:
out.append(
{
"ioc_type": ioc_type,
"ioc_value": raw,
"field": str(k),
"rule_id": rule_id,
"rule_title": rule_title,
"source_file": source_file,
}
)
walk(v, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out)
elif isinstance(node, list):
for item in node:
walk(item, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out)
def extract_from_yaml(path: Path) -> list[dict[str, str]]:
try:
doc = yaml.safe_load(path.read_text(encoding="utf-8", errors="ignore"))
except Exception:
return []
if not isinstance(doc, dict):
return []
detection = doc.get("detection")
if detection is None:
return []
rid = str(doc.get("id") or "")
title = str(doc.get("title") or "")
rows: list[dict[str, str]] = []
walk(detection, rule_id=rid, rule_title=title, source_file=str(path), out=rows)
return rows
def dedupe(rows: list[dict[str, str]]) -> list[dict[str, str]]:
seen = set()
out = []
for r in rows:
key = (r["ioc_type"], r["ioc_value"].lower(), r["field"])
if key in seen:
continue
seen.add(key)
out.append(r)
return out
def write_json(path: Path, rows: list[dict[str, str]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
def write_csv(path: Path, rows: list[dict[str, str]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fields = ["ioc_type", "ioc_value", "field", "rule_id", "rule_title", "source_file"]
with path.open("w", encoding="utf-8", newline="") as f:
w = csv.DictWriter(f, fieldnames=fields)
w.writeheader()
for row in rows:
w.writerow(row)
def sql_escape(s: str) -> str:
return s.replace("'", "''")
def write_sql(path: Path, rows: list[dict[str, str]], table_name: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
f.write(
f"CREATE TABLE IF NOT EXISTS {table_name} (\n"
" id INTEGER PRIMARY KEY AUTOINCREMENT,\n"
" ioc_type TEXT NOT NULL,\n"
" ioc_value TEXT NOT NULL,\n"
" field TEXT,\n"
" rule_id TEXT,\n"
" rule_title TEXT,\n"
" source_file TEXT\n"
");\n\n"
)
for r in rows:
f.write(
f"INSERT INTO {table_name} (ioc_type, ioc_value, field, rule_id, rule_title, source_file) VALUES "
f"('{sql_escape(r['ioc_type'])}',"
f"'{sql_escape(r['ioc_value'])}',"
f"'{sql_escape(r['field'])}',"
f"'{sql_escape(r['rule_id'])}',"
f"'{sql_escape(r['rule_title'])}',"
f"'{sql_escape(r['source_file'])}');\n"
)
def main() -> int:
ap = argparse.ArgumentParser(description="Extract IOC-like Sigma values for DLP preload.")
ap.add_argument("--rules-root", default="rules", help="Path to hayabusa-rules root")
ap.add_argument("--out-dir", default="ioc_export", help="Output directory")
ap.add_argument("--table-name", default="dlp_blacklist_ioc", help="SQL table name")
args = ap.parse_args()
rules_root = Path(args.rules_root)
if not rules_root.exists():
raise SystemExit(f"rules root not found: {rules_root}")
yaml_files = [p for p in rules_root.rglob("*") if p.is_file() and p.suffix.lower() in {".yml", ".yaml"}]
all_rows: list[dict[str, str]] = []
for yp in yaml_files:
all_rows.extend(extract_from_yaml(yp))
rows = dedupe(all_rows)
rows.sort(key=lambda r: (r["ioc_type"], r["ioc_value"].lower()))
out_dir = Path(args.out_dir)
write_json(out_dir / "ioc_blacklist.json", rows)
write_csv(out_dir / "ioc_blacklist.csv", rows)
write_sql(out_dir / "ioc_blacklist.sql", rows, args.table_name)
counts: dict[str, int] = {}
for r in rows:
counts[r["ioc_type"]] = counts.get(r["ioc_type"], 0) + 1
print(f"rules_scanned={len(yaml_files)}")
print(f"iocs_extracted={len(rows)}")
for k in sorted(counts):
print(f"{k}={counts[k]}")
print(f"json={out_dir / 'ioc_blacklist.json'}")
print(f"csv={out_dir / 'ioc_blacklist.csv'}")
print(f"sql={out_dir / 'ioc_blacklist.sql'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
-236
View File
@@ -1,236 +0,0 @@
#!/usr/bin/env python3
import argparse
import json
import os
import sqlite3
import sys
from pathlib import Path
def maybe_exec_rust() -> None:
if os.environ.get("MERGE_AW_SERVER_DBS_FORCE_LEGACY") == "1":
return
script_path = Path(__file__).resolve()
repo_root = script_path.parent.parent if script_path.parent.name == "scripts" else None
candidates = [
os.environ.get("MERGE_AW_SERVER_DBS_RUST"),
str(Path(os.environ.get("CARGO_TARGET_DIR", "")) / "release" / "merge-aw-server-dbs")
if os.environ.get("CARGO_TARGET_DIR")
else None,
str(repo_root / "adk-rust" / "target" / "release" / "merge-aw-server-dbs")
if repo_root
else None,
"/usr/local/bin/merge-aw-server-dbs",
]
for candidate in candidates:
if candidate and os.path.isfile(candidate) and os.access(candidate, os.X_OK):
os.execv(candidate, [candidate, *sys.argv[1:]])
maybe_exec_rust()
def connect(path: Path) -> sqlite3.Connection:
connection = sqlite3.connect(str(path))
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("PRAGMA synchronous=NORMAL")
return connection
def bucket_key(row: sqlite3.Row) -> tuple[str, str, str, str]:
return (
str(row["name"]),
str(row["type"]),
str(row["client"]),
str(row["hostname"]),
)
def find_bucket_by_name(connection: sqlite3.Connection, name: str) -> int | None:
row = connection.execute(
"select rowid as bucketrow from buckets where name = ? order by rowid limit 1",
(name,),
).fetchone()
return int(row["bucketrow"]) if row else None
def find_bucket_by_key(connection: sqlite3.Connection, name: str, type_: str, client: str, hostname: str) -> int | None:
row = connection.execute(
"select rowid as bucketrow from buckets where name = ? and type = ? and client = ? and hostname = ? order by rowid limit 1",
(name, type_, client, hostname),
).fetchone()
return int(row["bucketrow"]) if row else None
def copy_sqlite_via_backup(src: Path, dst: Path) -> None:
"""Create a consistent copy of an sqlite DB using the sqlite backup API.
This avoids corrupt/inconsistent files if the source DB is live.
"""
# ensure parent exists
dst.parent.mkdir(parents=True, exist_ok=True)
# remove any existing tmp file
if dst.exists():
dst.unlink()
src_conn = sqlite3.connect(str(src))
dst_conn = sqlite3.connect(str(dst))
try:
# perform online backup
src_conn.backup(dst_conn)
dst_conn.commit()
finally:
try:
src_conn.close()
finally:
dst_conn.close()
def ensure_parent(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
def load_existing_events(connection: sqlite3.Connection, bucketrow: int) -> set[tuple[int, int, str]]:
cursor = connection.execute(
"select starttime, endtime, data from events where bucketrow = ?",
(bucketrow,),
)
return {(int(start), int(end), str(data)) for start, end, data in cursor.fetchall()}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--base", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--overlay")
args = parser.parse_args()
base = Path(args.base)
output = Path(args.output)
overlay = Path(args.overlay) if args.overlay else None
if not base.exists():
raise SystemExit(f"Base DB not found: {base}")
ensure_parent(output)
tmp_output = output.with_suffix(output.suffix + ".tmp")
# make a consistent copy of the base DB into tmp_output
copy_sqlite_via_backup(base, tmp_output)
dest = connect(tmp_output)
dest.row_factory = sqlite3.Row
inserted_buckets = 0
inserted_events = 0
if overlay and overlay.exists():
source = connect(overlay)
source.row_factory = sqlite3.Row
try:
source_buckets = source.execute(
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
).fetchall()
dest_bucket_map = {
bucket_key(row): row["bucketrow"]
for row in dest.execute(
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
).fetchall()
}
dest_id_map = {
row["id"]: row["bucketrow"]
for row in dest.execute(
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
).fetchall()
if row["id"]
}
for src_bucket in source_buckets:
key = bucket_key(src_bucket)
src_id = src_bucket["id"] if "id" in src_bucket.keys() else None
dest_rowid = None
# Prefer exact id match if available
if src_id:
dest_rowid = dest_id_map.get(src_id)
if dest_rowid is None:
dest_rowid = dest_bucket_map.get(key)
if dest_rowid is None:
dest_rowid = find_bucket_by_name(dest, str(src_bucket["name"]))
if dest_rowid is not None:
dest.execute(
"""
UPDATE buckets
SET type = ?, client = ?, hostname = ?, created = ?,
data_deprecated = ?, data = ?
WHERE rowid = ?
""",
(
src_bucket["type"],
src_bucket["client"],
src_bucket["hostname"],
src_bucket["created"],
src_bucket["data_deprecated"],
src_bucket["data"],
dest_rowid,
),
)
else:
cursor = dest.execute(
"""
INSERT INTO buckets (id, name, type, client, hostname, created, data_deprecated, data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
src_bucket["id"],
src_bucket["name"],
src_bucket["type"],
src_bucket["client"],
src_bucket["hostname"],
src_bucket["created"],
src_bucket["data_deprecated"],
src_bucket["data"],
),
)
dest_rowid = int(cursor.lastrowid)
inserted_buckets += 1
dest_bucket_map[key] = dest_rowid
if src_id:
dest_id_map[str(src_id)] = dest_rowid
existing_events = load_existing_events(dest, dest_rowid)
for starttime, endtime, data in source.execute(
"select starttime, endtime, data from events where bucketrow = ? order by id",
(src_bucket["bucketrow"],),
).fetchall():
event_key = (int(starttime), int(endtime), str(data))
if event_key in existing_events:
continue
dest.execute(
"insert into events (bucketrow, starttime, endtime, data) values (?, ?, ?, ?)",
(dest_rowid, int(starttime), int(endtime), str(data)),
)
existing_events.add(event_key)
inserted_events += 1
dest.commit()
finally:
source.close()
dest.close()
os.replace(tmp_output, output)
print(
json.dumps(
{
"base": str(base),
"overlay": str(overlay) if overlay else None,
"output": str(output),
"inserted_buckets": inserted_buckets,
"inserted_events": inserted_events,
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+14 -79
View File
@@ -11,84 +11,19 @@ if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then
set +a
fi
if [[ "${1:-}" == "--apply-legacy" ]]; then
shift
else
for candidate in \
"${PROD_BACKUP_RESTORE_RUST:-}" \
"${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \
"$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \
"/usr/local/bin/prod-backup-restore"; do
if [[ -n "$candidate" && -x "$candidate" ]]; then
exec "$candidate" --root "$ROOT_DIR" "$@"
fi
done
cat >&2 <<'EOF'
prod_backup_restore.sh is destructive and now requires the Rust planner.
for candidate in \
"${PROD_BACKUP_RESTORE_RUST:-}" \
"${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \
"$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \
"/usr/local/bin/prod-backup-restore"; do
if [[ -n "$candidate" && -x "$candidate" ]]; then
exec "$candidate" --root "$ROOT_DIR" "$@"
fi
done
cat >&2 <<EOF
prod_backup_restore.sh now requires the Rust planner/checker.
Build it first:
cd adk-rust && cargo build --release -p prod-backup-restore
To run the old destructive restore explicitly:
scripts/prod_backup_restore.sh --apply-legacy
cd "$ROOT_DIR/adk-rust" && cargo build --release -p prod-backup-restore
EOF
exit 2
fi
: "${AW_SSH_PASSWORD:?AW_SSH_PASSWORD is required}"
: "${AW_WINRM_PASSWORD:?AW_WINRM_PASSWORD is required}"
command -v sshpass >/dev/null 2>&1 || { echo "missing sshpass" >&2; exit 127; }
command -v ansible-playbook >/dev/null 2>&1 || { echo "missing ansible-playbook" >&2; exit 127; }
SERVER_HOST="${AW_SERVER_HOST:-10.10.10.13}"
SERVER_USER="${AW_SERVER_USER:-igor}"
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
REMOTE_BACKUP_DIR="/var/lib/activitywatch/backups/prod-restore-${TIMESTAMP}"
LEGACY_DB="/root/.local/share/activitywatch/aw-server-rust/sqlite.db"
TARGET_DB="/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db"
REMOTE_MERGE_SCRIPT="/tmp/merge_aw_server_dbs.py"
ssh_remote() {
sshpass -p "$AW_SSH_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${SERVER_USER}@${SERVER_HOST}" "$@"
}
scp_remote() {
sshpass -p "$AW_SSH_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
}
scp_remote "${ROOT_DIR}/scripts/merge_aw_server_dbs.py" "${SERVER_USER}@${SERVER_HOST}:${REMOTE_MERGE_SCRIPT}"
ssh_remote "sudo mkdir -p '${REMOTE_BACKUP_DIR}' && sudo chown root:root '${REMOTE_BACKUP_DIR}'"
ssh_remote "sudo test -f '${LEGACY_DB}'"
ssh_remote "sudo test -f '${TARGET_DB}'"
ssh_remote "sudo cp -a '${LEGACY_DB}' '${REMOTE_BACKUP_DIR}/legacy-root-sqlite.db' && sudo cp -a '${TARGET_DB}' '${REMOTE_BACKUP_DIR}/target-before-merge-sqlite.db'"
ssh_remote "sudo systemctl stop activitywatch-server.service || true"
ssh_remote "sudo python3 '${REMOTE_MERGE_SCRIPT}' --base '${LEGACY_DB}' --overlay '${TARGET_DB}' --output '${REMOTE_BACKUP_DIR}/sqlite.merged.db'"
ssh_remote "sudo install -o activitywatch -g activitywatch -m 0644 '${REMOTE_BACKUP_DIR}/sqlite.merged.db' '${TARGET_DB}'"
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_windows.yml
ansible-playbook -i ansible/inventory.ini ansible/post_validate_aw_windows.yml
python3 - <<'PY'
import json, urllib.request
base = 'http://10.10.10.13:5600'
window_payload = {
'timeperiods': ['2026-04-29T00:00:00+03:00/2026-04-29T23:59:59+03:00'],
'query': [
'window_events = query_bucket(find_bucket("aw-watcher-window_SHARKON2025"));',
'RETURN = window_events;'
]
}
req = urllib.request.Request(base + '/api/0/query/', data=json.dumps(window_payload).encode(), method='POST', headers={'Content-Type': 'application/json', 'Origin': 'http://10.10.10.13:5600'})
with urllib.request.urlopen(req) as response:
data = json.loads(response.read().decode())
window_count = len(data[0]) if isinstance(data, list) and data else 0
if window_count <= 0:
raise SystemExit('no historical window data restored for 2026-04-29')
with urllib.request.urlopen(base + '/api/0/settings/') as response:
settings = json.loads(response.read().decode())
if settings.get('always_active_pattern') != 'aw-watcher-window':
raise SystemExit('always_active_pattern is not configured')
print(json.dumps({'restored_window_events_2026_04_29': window_count, 'always_active_pattern': settings.get('always_active_pattern')}, ensure_ascii=False))
PY
exit 127
+2 -147
View File
@@ -23,150 +23,5 @@ for candidate in "${rust_candidates[@]}"; do
fi
done
KIT_DIR="install-kit-awindows-20260427-211240"
ZIP_ARCHIVE="${KIT_DIR}.zip"
TAR_ARCHIVE="${KIT_DIR}.tar.gz"
SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.18"
OLD_SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.21"
TMP_SERVER_CONFIG_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_SERVER_CONFIG_DIR"' EXIT
copy_file() {
local src="$1"
local dest="$2"
mkdir -p "$(dirname "$dest")"
cp "$src" "$dest"
}
sync_tree() {
local base="$1"
shift
for rel in "$@"; do
copy_file "$rel" "${KIT_DIR}/${rel}"
done
}
if [[ -d "$OLD_SERVER_CONFIG_DIR" ]]; then
cp "$OLD_SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/
fi
if [[ -d "$SERVER_CONFIG_DIR" ]]; then
cp "$SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/
fi
rm -rf "${KIT_DIR}/ansible" "${KIT_DIR}/aw-server" "${KIT_DIR}/windows" "${KIT_DIR}/scripts" "${KIT_DIR}/server-configs-"*
ansible_files=(
ansible/README.md
ansible/deploy_aw_pfsense_poller.yml
ansible/deploy_aw_server.yml
ansible/deploy_aw_windows.yml
ansible/group_vars/all.example.yml
ansible/group_vars/pfsense-poller.example.yml
ansible/group_vars/proxmox-matrix.example.yml
ansible/group_vars/proxmox.example.yml
ansible/group_vars/windows.example.yml
ansible/install_full_stack.yml
ansible/inventory.example.ini
ansible/provision_proxmox_ct_and_deploy_aw.yml
ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml
ansible/tasks/provision_ct_and_deploy_aw.yml
)
aw_server_files=(
aw-server/activitywatch-server.service
aw-server/apply_webui_ru_patch.sh
aw-server/aw-host-groups.json
aw-server/aw-ru-patch.js
aw-server/aw-rus-healthd.py
aw-server/aw-rus-healthd.service
aw-server/aw-rus-healthd.timer
aw-server/aw-browser-smoke.service
aw-server/aw-browser-smoke.timer
aw-server/aw-slo-monitor.py
aw-server/aw-slo-monitor.service
aw-server/aw-slo-monitor.timer
aw-server/aw-server.env.example
aw-server/aw-sw-cleanup.js
aw-server/aw-worktime-api.py
aw-server/aw-worktime-api.service
aw-server/aw-worktime-prewarm.sh
aw-server/aw-worktime-prewarm.service
aw-server/aw-worktime-prewarm.timer
aw-server/aw-worktime-panel.js
aw-server/health-check.sh
aw-server/install_aw_server.sh
aw-server/settings/classes-worktime.json
aw-server/settings/views-default.json
)
windows_files=(
windows/ActivityWatch.Windows.Common.psd1
windows/ActivityWatch.Windows.Common.psm1
windows/browser-domains-native-collector.ps1
windows/deploy-domain-users.ps1
windows/deploy-ensemble.ps1
windows/deploy-single-user.ps1
windows/AWatchRusCollectorGuardService.cs
windows/aw-collector-guard.ps1
windows/install-collector-guard-service.ps1
windows/dlp-endpoint-signals-collector.ps1
windows/dlp-policy.example.json
windows/dlp-policy.native-cross-os.example.json
windows/email-outbound-collector.ps1
windows/hardening-recovery.ps1
windows/migrate-awatch-rus-paths.ps1
windows/validate-deployment.ps1
windows/web-category-rules.example.json
windows/worktime-session-collector.ps1
)
scripts_files=(
scripts/aw-webui-browser-smoke.mjs
scripts/aw-webui-browser-smoke.sh
scripts/check_install_kit_vs_repo.sh
scripts/quality-gate.sh
scripts/rebuild_install_kit.sh
scripts/validate_install_kit.sh
scripts/verify_innosetup_installer.sh
)
sync_tree ansible "${ansible_files[@]}"
sync_tree aw-server "${aw_server_files[@]}"
sync_tree windows "${windows_files[@]}"
sync_tree scripts "${scripts_files[@]}"
mkdir -p "$SERVER_CONFIG_DIR"
if compgen -G "$TMP_SERVER_CONFIG_DIR/*.deployment-config.json" >/dev/null; then
cp "$TMP_SERVER_CONFIG_DIR"/*.deployment-config.json "$SERVER_CONFIG_DIR"/
fi
cat > "${KIT_DIR}/README-INSTALL-KIT.txt" <<'EOF'
ActivityWatch DetMir Windows Install Kit
Includes:
- windows/* (deploy scripts, collectors, common module, configs/examples)
- ansible/* (Windows and AW server playbooks, examples, inventory, tasks)
- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings)
- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks)
- server-configs-192.168.100.18/* (working Windows/RDP config snapshots)
Source:
- Local project snapshot at build time.
EOF
python3 - <<'PY'
from pathlib import Path
import hashlib
root = Path('install-kit-awindows-20260427-211240')
manifest = root / 'MANIFEST.txt'
files = sorted(p for p in root.rglob('*') if p.is_file() and p.name != 'MANIFEST.txt')
with manifest.open('w', encoding='utf-8') as handle:
for path in files:
digest = hashlib.sha256(path.read_bytes()).hexdigest()
handle.write(f"{digest} {path.as_posix()}\n")
PY
rm -f "$ZIP_ARCHIVE" "$TAR_ARCHIVE"
zip -rq "$ZIP_ARCHIVE" "$KIT_DIR"
tar -czf "$TAR_ARCHIVE" "$KIT_DIR"
echo "ERROR: Rust install-kit builder not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p rebuild-install-kit" >&2
exit 127
-421
View File
@@ -1,421 +0,0 @@
#!/usr/bin/env python3
import importlib.util
import sys
from pathlib import Path
MODULE_PATH = Path(__file__).with_name("dlp-health-check.py")
SPEC = importlib.util.spec_from_file_location("dlp_health_check", MODULE_PATH)
MODULE = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = MODULE
SPEC.loader.exec_module(MODULE)
def test_endpoint_signals_ok_when_no_active_managed_hosts(monkeypatch):
buckets = {
"aw-worktime-sessions_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}},
"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T07:00:00Z"}},
}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:00:10Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url: [{"timestamp": "2026-05-30T12:00:00Z", "data": {"active": False}}]
if "aw-worktime-sessions_SHARKON2025/events" in url
else [],
)
report = MODULE.HealthReport()
MODULE.check_endpoint_signal_buckets(report, "http://127.0.0.1:5600/api/0", buckets, 900)
result = report.results[0]
assert result.name == "buckets:endpoint-signals"
assert result.status == "ok"
assert result.summary == "no active managed hosts require endpoint-signals freshness"
assert result.details["ignored_inactive"] == ["aw-dlp-endpoint-signals_SHARKON2025"]
def test_endpoint_self_test_metrics_reports_transport_counters(monkeypatch):
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url, **kwargs: [
{
"timestamp": "2026-05-30T11:59:00Z",
"data": {
"signalType": "self_test",
"queueDepth": 8,
"eventsEnqueued": 100,
"eventsFlushed": 92,
"sendFailures": 0,
},
},
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "self_test",
"queueDepth": "3",
"eventsEnqueued": "120",
"eventsFlushed": "117",
"sendFailures": "0",
},
},
],
)
report = MODULE.HealthReport()
MODULE.check_endpoint_self_test_metrics(report, "http://127.0.0.1:5600/api/0", buckets)
result = report.results[0]
assert result.name == "endpoint-self-test-metrics"
assert result.status == "ok"
latest = result.details["latest_self_tests"][0]
assert latest["bucket"] == "aw-dlp-endpoint-signals_SHARKON2025"
assert latest["timestamp"] == "2026-05-30T12:00:30Z"
assert latest["age_seconds"] == 30
assert latest["queueDepth"] == 3
assert latest["eventsEnqueued"] == 120
assert latest["eventsFlushed"] == 117
assert latest["sendFailures"] == 0
assert latest["sendFailuresDelta"] == 0
def test_endpoint_self_test_metrics_warns_on_transport_counters(monkeypatch):
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url, **kwargs: [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "self_test",
"queueDepth": 101,
"eventsEnqueued": 120,
"eventsFlushed": 19,
"sendFailures": 1,
},
}
],
)
report = MODULE.HealthReport()
MODULE.check_endpoint_self_test_metrics(
report,
"http://127.0.0.1:5600/api/0",
buckets,
queue_warn_depth=100,
send_failure_warn_count=1,
)
result = report.results[0]
assert result.status == "warn"
assert result.summary == "endpoint transport counters outside thresholds"
assert result.details["warnings"] == [
{"bucket": "aw-dlp-endpoint-signals_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100},
{
"bucket": "aw-dlp-endpoint-signals_SHARKON2025",
"metric": "sendFailuresDelta",
"value": 1,
"current": 1,
"previous": None,
"threshold": 1,
},
]
def test_file_operations_runtime_reports_health_and_latest_operations(monkeypatch):
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url: [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "collector_health",
"queueDepth": "0",
"eventsEnqueued": "5",
"eventsFlushed": "5",
"sendFailures": "0",
"username": "USER1",
"hostname": "SHARKON2025",
"sessionId": "3",
},
},
{
"timestamp": "2026-05-30T12:00:20Z",
"data": {
"operation": "Created",
"path": "C:\\Users\\USER1\\Downloads\\report.zip",
"extension": ".zip",
"archiveHint": True,
"username": "USER1",
"hostname": "SHARKON2025",
"size": "42",
},
},
],
)
report = MODULE.HealthReport()
MODULE.check_file_operations_runtime(report, "http://127.0.0.1:5600/api/0", buckets)
result = report.results[0]
assert result.name == "file-operations-runtime"
assert result.status == "ok"
assert result.details["latest_health"][0]["queueDepth"] == 0
assert result.details["latest_health"][0]["eventsEnqueued"] == 5
assert result.details["latest_health"][0]["sendFailuresDelta"] == 0
latest_op = result.details["latest_operations"][0]
assert latest_op["operation"] == "Created"
assert latest_op["extension"] == ".zip"
assert latest_op["archiveHint"] is True
assert latest_op["path_tail"] == "Downloads/report.zip"
assert latest_op["size"] == 42
assert result.details["sampled"][0]["operation_counts"] == {"Created": 1}
def test_file_operations_runtime_warns_on_transport_counters(monkeypatch):
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url: [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "collector_health",
"queueDepth": 101,
"eventsEnqueued": 5,
"eventsFlushed": 2,
"sendFailures": 1,
"username": "USER1",
"hostname": "SHARKON2025",
"sessionId": 3,
},
}
],
)
report = MODULE.HealthReport()
MODULE.check_file_operations_runtime(
report,
"http://127.0.0.1:5600/api/0",
buckets,
queue_warn_depth=100,
send_failure_warn_count=1,
)
result = report.results[0]
assert result.status == "warn"
assert result.summary == "file-operations runtime counters outside expectations"
assert result.details["warnings"] == [
{"bucket": "aw-file-operations_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100},
{
"bucket": "aw-file-operations_SHARKON2025",
"metric": "sendFailuresDelta",
"value": 1,
"current": 1,
"previous": None,
"threshold": 1,
},
]
def test_endpoint_send_failures_uses_delta_baseline(monkeypatch):
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
state = {"counters": {}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
values = [12, 12, 13]
def fake_http(url, **kwargs):
value = values.pop(0)
return [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "self_test",
"queueDepth": 0,
"eventsEnqueued": 120,
"eventsFlushed": 117,
"sendFailures": value,
},
}
]
monkeypatch.setattr(MODULE, "_http_json", fake_http)
first = MODULE.HealthReport()
MODULE.check_endpoint_self_test_metrics(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert first.results[0].status == "ok"
assert first.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] is None
assert first.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0
second = MODULE.HealthReport()
MODULE.check_endpoint_self_test_metrics(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert second.results[0].status == "ok"
assert second.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] == 12
assert second.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0
third = MODULE.HealthReport()
MODULE.check_endpoint_self_test_metrics(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert third.results[0].status == "warn"
assert third.results[0].details["warnings"] == [
{
"bucket": "aw-dlp-endpoint-signals_SHARKON2025",
"metric": "sendFailuresDelta",
"value": 1,
"current": 13,
"previous": 12,
"threshold": 1,
}
]
def test_file_operations_send_failures_uses_delta_baseline(monkeypatch):
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
state = {"counters": {}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
values = [28, 28, 29]
def fake_http(url, **kwargs):
value = values.pop(0)
return [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"signalType": "collector_health",
"queueDepth": 0,
"eventsEnqueued": 120,
"eventsFlushed": 117,
"sendFailures": value,
},
}
]
monkeypatch.setattr(MODULE, "_http_json", fake_http)
first = MODULE.HealthReport()
MODULE.check_file_operations_runtime(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert first.results[0].status == "ok"
assert first.results[0].details["latest_health"][0]["sendFailuresPrevious"] is None
assert first.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0
second = MODULE.HealthReport()
MODULE.check_file_operations_runtime(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert second.results[0].status == "ok"
assert second.results[0].details["latest_health"][0]["sendFailuresPrevious"] == 28
assert second.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0
third = MODULE.HealthReport()
MODULE.check_file_operations_runtime(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
assert third.results[0].status == "warn"
assert third.results[0].details["warnings"] == [
{
"bucket": "aw-file-operations_SHARKON2025",
"metric": "sendFailuresDelta",
"value": 1,
"current": 29,
"previous": 28,
"threshold": 1,
}
]
def test_incident_runtime_reports_counts_and_latest_real_incidents(monkeypatch):
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
monkeypatch.setattr(
MODULE,
"_http_json",
lambda url, **kwargs: [
{
"timestamp": "2026-05-30T12:00:30Z",
"data": {
"ruleId": "usb-archive-copy",
"severity": "high",
"action": "alert",
"message": "Archive copied to removable device with a long message that should not leak full raw content.",
"username": "USER1",
"hostname": "SHARKON2025",
"source": "endpoint",
},
},
{
"timestamp": "2026-05-30T12:00:00Z",
"data": {
"ruleId": "selftest-dlp-incident",
"severity": "low",
"action": "alert",
"message": "Self-test DLP incident from validation",
"signalType": "self_test",
"source": "self-test",
},
},
],
)
report = MODULE.HealthReport()
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20)
result = report.results[0]
assert result.name == "incident-runtime"
assert result.status == "ok"
assert result.summary == "1 real incidents in sampled events"
assert result.details["totals"] == {"sampled_events": 2, "real_incidents": 1, "self_tests": 1}
assert result.details["severity_counts"] == {"high": 1}
assert result.details["action_counts"] == {"alert": 1}
assert result.details["rule_counts"] == {"usb-archive-copy": 1}
latest = result.details["latest_incidents"][0]
assert latest["age_seconds"] == 30
assert latest["ruleId"] == "usb-archive-copy"
assert latest["message_excerpt"].endswith(".")
def test_incident_runtime_defaults_to_metadata_without_event_sampling(monkeypatch):
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
def fail_if_called(url, **kwargs):
raise AssertionError("event sampling should stay disabled")
monkeypatch.setattr(MODULE, "_http_json", fail_if_called)
report = MODULE.HealthReport()
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=0)
result = report.results[0]
assert result.name == "incident-runtime"
assert result.status == "ok"
assert result.summary == "incident event sampling disabled"
assert result.details["metadata"] == [
{
"bucket": "aw-dlp-incidents_SHARKON2025",
"end": "2026-05-30T12:00:00Z",
"age_seconds": 60,
}
]
def test_incident_runtime_warns_on_read_failure(monkeypatch):
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
def fail_http(url, **kwargs):
raise RuntimeError("timeout")
monkeypatch.setattr(MODULE, "_http_json", fail_http)
report = MODULE.HealthReport()
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20)
result = report.results[0]
assert result.name == "incident-runtime"
assert result.status == "warn"
assert result.summary == "1 incident buckets failed to sample"
assert result.details["read_failed"] == [{"bucket": "aw-dlp-incidents_SHARKON2025", "error": "timeout"}]
+2 -72
View File
@@ -32,75 +32,5 @@ for candidate in "${rust_candidates[@]}"; do
fi
done
required_files=(
"$MANIFEST"
"$KIT_DIR/README-INSTALL-KIT.txt"
"$KIT_DIR/windows/deploy-ensemble.ps1"
"$KIT_DIR/windows/validate-deployment.ps1"
"$KIT_DIR/ansible/deploy_aw_windows.yml"
"$KIT_DIR/aw-server/install_aw_server.sh"
"$KIT_DIR/scripts/rebuild_install_kit.sh"
"$KIT_DIR/scripts/validate_install_kit.sh"
"$KIT_DIR/scripts/check_install_kit_vs_repo.sh"
"$KIT_DIR/scripts/quality-gate.sh"
)
echo "[1/4] Required files presence"
for file in "${required_files[@]}"; do
[[ -f "$file" ]] || { echo "Missing required file: $file"; exit 1; }
done
echo "[2/4] Manifest checksum verification"
sha256sum -c "$MANIFEST" >/dev/null
echo "[3/4] Manifest completeness"
python3 - <<'PY'
from pathlib import Path
import sys
kit=Path('install-kit-awindows-20260427-211240')
manifest=kit/'MANIFEST.txt'
listed=[]
for line in manifest.read_text().splitlines():
line=line.strip()
if not line:
continue
parts=line.split(' ',1)
if len(parts)!=2:
print(f'Invalid MANIFEST line: {line}')
sys.exit(1)
listed.append(parts[1])
listed_set=set(listed)
actual_set={str(p) for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'}
missing=sorted(listed_set-actual_set)
extra=sorted(actual_set-listed_set)
if missing or extra:
print('Missing files listed in MANIFEST:', missing)
print('Files not listed in MANIFEST:', extra)
sys.exit(1)
print(f'MANIFEST complete: {len(actual_set)} files tracked')
PY
echo "[4/4] Archive composition check"
python3 - <<'PY'
from pathlib import Path
import tarfile, zipfile, sys
kit_prefix='install-kit-awindows-20260427-211240/'
zip_path=Path('install-kit-awindows-20260427-211240.zip')
tar_path=Path('install-kit-awindows-20260427-211240.tar.gz')
if not zip_path.exists() or not tar_path.exists():
print('Archives not found')
sys.exit(1)
with zipfile.ZipFile(zip_path) as z:
zip_files=sorted(i for i in z.namelist() if not i.endswith('/'))
with tarfile.open(tar_path, 'r:gz') as t:
tar_files=sorted(m.name for m in t.getmembers() if m.isfile())
if zip_files != tar_files:
print('ZIP and TAR contents differ')
sys.exit(1)
if not all(f.startswith(kit_prefix) for f in zip_files):
print('Unexpected archive prefix layout')
sys.exit(1)
print(f'Archives match: {len(zip_files)} files')
PY
echo "validate_install_kit: OK"
echo "ERROR: Rust validator not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p validate-install-kit" >&2
exit 127