refactor(detmir): retire python runtime paths
This commit is contained in:
@@ -1,517 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Protocol, TypeAlias
|
||||
|
||||
|
||||
JsonScalar: TypeAlias = str | int | float | bool | None
|
||||
JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"]
|
||||
|
||||
|
||||
DEFAULT_BUCKET_PREFIXES = ("aw-file-operations_", "aw-dlp-incidents_")
|
||||
DEFAULT_SQLITE_PATH = "data/dlp-events.sqlite3"
|
||||
EVENT_COLUMNS = (
|
||||
"bucket_id",
|
||||
"event_id",
|
||||
"stream_type",
|
||||
"hostname",
|
||||
"username",
|
||||
"event_ts",
|
||||
"duration",
|
||||
"operation",
|
||||
"file_path",
|
||||
"old_file_path",
|
||||
"extension",
|
||||
"archive_hint",
|
||||
"rule_id",
|
||||
"action",
|
||||
"severity",
|
||||
"signal_type",
|
||||
"message",
|
||||
"source",
|
||||
"screenshot_path",
|
||||
"raw_json",
|
||||
"ingested_at",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Bucket:
|
||||
id: str
|
||||
type: str
|
||||
client: str
|
||||
hostname: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AwEvent:
|
||||
bucket_id: str
|
||||
hostname: str
|
||||
stream_type: str
|
||||
event_id: str
|
||||
timestamp: str
|
||||
duration: float
|
||||
data: dict[str, JsonValue]
|
||||
|
||||
|
||||
class PsycopgConnection(Protocol):
|
||||
def cursor(self):
|
||||
...
|
||||
|
||||
def commit(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(tz=UTC)
|
||||
|
||||
|
||||
def parse_timestamp(value: str) -> datetime:
|
||||
normalized = value.replace("Z", "+00:00")
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def format_aw_timestamp(value: datetime) -> str:
|
||||
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def load_state(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def save_state(path: Path, state: dict[str, str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(state, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def normalize_base_url(base_url: str) -> str:
|
||||
return base_url.rstrip("/")
|
||||
|
||||
|
||||
def aw_get_json(base_url: str, path: str, timeout: int) -> JsonValue:
|
||||
url = normalize_base_url(base_url) + path
|
||||
request = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def list_buckets(base_url: str, timeout: int) -> list[Bucket]:
|
||||
payload = aw_get_json(base_url, "/buckets", timeout)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("ActivityWatch /buckets response must be a JSON object")
|
||||
buckets: list[Bucket] = []
|
||||
for bucket_id, bucket_data in payload.items():
|
||||
if not isinstance(bucket_data, dict):
|
||||
continue
|
||||
buckets.append(
|
||||
Bucket(
|
||||
id=str(bucket_id),
|
||||
type=str(bucket_data.get("type", "")),
|
||||
client=str(bucket_data.get("client", "")),
|
||||
hostname=str(bucket_data.get("hostname", "")),
|
||||
)
|
||||
)
|
||||
return buckets
|
||||
|
||||
|
||||
def bucket_stream_type(bucket: Bucket) -> str | None:
|
||||
if bucket.id.startswith("aw-file-operations_") or bucket.type == "aw.file.operation":
|
||||
return "file_operation"
|
||||
if bucket.id.startswith("aw-dlp-incidents_") or bucket.type == "aw.dlp.incident":
|
||||
return "dlp_incident"
|
||||
return None
|
||||
|
||||
|
||||
def select_buckets(buckets: Iterable[Bucket], prefixes: tuple[str, ...]) -> list[tuple[Bucket, str]]:
|
||||
selected: list[tuple[Bucket, str]] = []
|
||||
for bucket in buckets:
|
||||
stream_type = bucket_stream_type(bucket)
|
||||
if stream_type and any(bucket.id.startswith(prefix) for prefix in prefixes):
|
||||
selected.append((bucket, stream_type))
|
||||
return selected
|
||||
|
||||
|
||||
def build_events_path(bucket_id: str, start: datetime, end: datetime, limit: int) -> str:
|
||||
query = urllib.parse.urlencode(
|
||||
{
|
||||
"start": format_aw_timestamp(start),
|
||||
"end": format_aw_timestamp(end),
|
||||
"limit": str(limit),
|
||||
}
|
||||
)
|
||||
return f"/buckets/{urllib.parse.quote(bucket_id, safe='')}/events?{query}"
|
||||
|
||||
|
||||
def event_key(bucket_id: str, timestamp: str, duration: float, data: dict[str, JsonValue]) -> str:
|
||||
payload = json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return f"{bucket_id}|{timestamp}|{duration}|{payload}"
|
||||
|
||||
|
||||
def fetch_bucket_events(
|
||||
base_url: str,
|
||||
bucket: Bucket,
|
||||
stream_type: str,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
limit: int,
|
||||
timeout: int,
|
||||
) -> list[AwEvent]:
|
||||
payload = aw_get_json(base_url, build_events_path(bucket.id, start, end, limit), timeout)
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError(f"ActivityWatch events response for {bucket.id} must be a JSON array")
|
||||
events: list[AwEvent] = []
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
timestamp = str(item["timestamp"])
|
||||
duration = float(item.get("duration", 0) or 0)
|
||||
data = item.get("data") or {}
|
||||
if not isinstance(data, dict):
|
||||
data = {"raw": data}
|
||||
item_id = str(item.get("id") or event_key(bucket.id, timestamp, duration, data))
|
||||
events.append(
|
||||
AwEvent(
|
||||
bucket_id=bucket.id,
|
||||
hostname=bucket.hostname or str(data.get("hostname") or ""),
|
||||
stream_type=stream_type,
|
||||
event_id=item_id,
|
||||
timestamp=timestamp,
|
||||
duration=duration,
|
||||
data=data,
|
||||
)
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
def connect_sqlite(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
connection = sqlite3.connect(str(path))
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("PRAGMA synchronous=NORMAL")
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def ensure_schema(connection: sqlite3.Connection) -> None:
|
||||
connection.executescript(
|
||||
"""
|
||||
create table if not exists dlp_events (
|
||||
id integer primary key autoincrement,
|
||||
bucket_id text not null,
|
||||
event_id text not null,
|
||||
stream_type text not null,
|
||||
hostname text not null,
|
||||
username text,
|
||||
event_ts text not null,
|
||||
duration real not null default 0,
|
||||
operation text,
|
||||
file_path text,
|
||||
old_file_path text,
|
||||
extension text,
|
||||
archive_hint integer not null default 0,
|
||||
rule_id text,
|
||||
action text,
|
||||
severity text,
|
||||
signal_type text,
|
||||
message text,
|
||||
source text,
|
||||
screenshot_path text,
|
||||
raw_json text not null,
|
||||
ingested_at text not null,
|
||||
unique (bucket_id, event_id)
|
||||
);
|
||||
create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts);
|
||||
create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts);
|
||||
create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts);
|
||||
create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts);
|
||||
create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts);
|
||||
|
||||
create view if not exists dlp_file_operations as
|
||||
select *
|
||||
from dlp_events
|
||||
where stream_type = 'file_operation';
|
||||
|
||||
create view if not exists dlp_incidents as
|
||||
select *
|
||||
from dlp_events
|
||||
where stream_type = 'dlp_incident';
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def ensure_postgres_schema(connection: PsycopgConnection) -> None:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
create table if not exists dlp_events (
|
||||
id bigserial primary key,
|
||||
bucket_id text not null,
|
||||
event_id text not null,
|
||||
stream_type text not null,
|
||||
hostname text not null,
|
||||
username text,
|
||||
event_ts timestamptz not null,
|
||||
duration double precision not null default 0,
|
||||
operation text,
|
||||
file_path text,
|
||||
old_file_path text,
|
||||
extension text,
|
||||
archive_hint boolean not null default false,
|
||||
rule_id text,
|
||||
action text,
|
||||
severity text,
|
||||
signal_type text,
|
||||
message text,
|
||||
source text,
|
||||
screenshot_path text,
|
||||
raw_json jsonb not null,
|
||||
ingested_at timestamptz not null,
|
||||
unique (bucket_id, event_id)
|
||||
);
|
||||
create index if not exists idx_dlp_events_event_ts on dlp_events(event_ts);
|
||||
create index if not exists idx_dlp_events_host_ts on dlp_events(hostname, event_ts);
|
||||
create index if not exists idx_dlp_events_stream_ts on dlp_events(stream_type, event_ts);
|
||||
create index if not exists idx_dlp_events_archive on dlp_events(archive_hint, event_ts);
|
||||
create index if not exists idx_dlp_events_rule on dlp_events(rule_id, event_ts);
|
||||
|
||||
create or replace view dlp_file_operations as
|
||||
select *
|
||||
from dlp_events
|
||||
where stream_type = 'file_operation';
|
||||
|
||||
create or replace view dlp_incidents as
|
||||
select *
|
||||
from dlp_events
|
||||
where stream_type = 'dlp_incident';
|
||||
"""
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def first_string(data: dict[str, JsonValue], keys: tuple[str, ...]) -> str | None:
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value is not None and str(value) != "":
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def bool_as_int(value: JsonValue) -> int:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
return int(value.lower() in {"1", "true", "yes", "y"})
|
||||
return int(bool(value))
|
||||
|
||||
|
||||
def event_row(event: AwEvent, ingested_at: str) -> tuple[JsonValue, ...]:
|
||||
data = event.data
|
||||
event_id = event.event_id or event_key(event.bucket_id, event.timestamp, event.duration, data)
|
||||
return (
|
||||
event.bucket_id,
|
||||
event_id,
|
||||
event.stream_type,
|
||||
event.hostname,
|
||||
first_string(data, ("username", "user")),
|
||||
event.timestamp,
|
||||
event.duration,
|
||||
first_string(data, ("operation",)),
|
||||
first_string(data, ("path", "filePath")),
|
||||
first_string(data, ("oldPath", "oldFilePath")),
|
||||
first_string(data, ("extension",)),
|
||||
bool_as_int(data.get("archiveHint")),
|
||||
first_string(data, ("ruleId", "rule")),
|
||||
first_string(data, ("action",)),
|
||||
first_string(data, ("severity",)),
|
||||
first_string(data, ("signalType",)),
|
||||
first_string(data, ("message",)),
|
||||
first_string(data, ("source",)),
|
||||
first_string(data, ("screenshotPath", "capturePath", "artifactPath")),
|
||||
json.dumps(data, ensure_ascii=False, sort_keys=True),
|
||||
ingested_at,
|
||||
)
|
||||
|
||||
|
||||
def insert_events(connection: sqlite3.Connection, events: Iterable[AwEvent]) -> int:
|
||||
inserted = 0
|
||||
now = format_aw_timestamp(utc_now())
|
||||
for event in events:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
insert or ignore into dlp_events (
|
||||
bucket_id,
|
||||
event_id,
|
||||
stream_type,
|
||||
hostname,
|
||||
username,
|
||||
event_ts,
|
||||
duration,
|
||||
operation,
|
||||
file_path,
|
||||
old_file_path,
|
||||
extension,
|
||||
archive_hint,
|
||||
rule_id,
|
||||
action,
|
||||
severity,
|
||||
signal_type,
|
||||
message,
|
||||
source,
|
||||
screenshot_path,
|
||||
raw_json,
|
||||
ingested_at
|
||||
)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
event_row(event, now),
|
||||
)
|
||||
inserted += int(cursor.rowcount > 0)
|
||||
connection.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
def insert_postgres_events(dsn: str, events: Iterable[AwEvent]) -> int:
|
||||
try:
|
||||
import psycopg
|
||||
except ImportError as exc:
|
||||
raise SystemExit("PostgreSQL mode requires psycopg: python3 -m pip install 'psycopg[binary]'") from exc
|
||||
|
||||
inserted = 0
|
||||
now = format_aw_timestamp(utc_now())
|
||||
columns = ", ".join(EVENT_COLUMNS)
|
||||
placeholders = ", ".join(["%s"] * len(EVENT_COLUMNS))
|
||||
sql = f"""
|
||||
insert into dlp_events ({columns})
|
||||
values ({placeholders})
|
||||
on conflict (bucket_id, event_id) do nothing
|
||||
"""
|
||||
with psycopg.connect(dsn) as connection:
|
||||
ensure_postgres_schema(connection)
|
||||
with connection.cursor() as cursor:
|
||||
for event in events:
|
||||
row = list(event_row(event, now))
|
||||
row[EVENT_COLUMNS.index("archive_hint")] = bool(row[EVENT_COLUMNS.index("archive_hint")])
|
||||
cursor.execute(sql, row)
|
||||
inserted += int(cursor.rowcount > 0)
|
||||
connection.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
def get_start_time(args: argparse.Namespace, state: dict[str, str]) -> datetime:
|
||||
if args.since:
|
||||
return parse_timestamp(args.since)
|
||||
if state.get("last_end"):
|
||||
return parse_timestamp(state["last_end"]) - timedelta(seconds=args.overlap_seconds)
|
||||
return utc_now() - timedelta(hours=args.lookback_hours)
|
||||
|
||||
|
||||
def parse_prefixes(value: str) -> tuple[str, ...]:
|
||||
prefixes = tuple(item.strip() for item in value.split(",") if item.strip())
|
||||
if not prefixes:
|
||||
raise argparse.ArgumentTypeError("at least one bucket prefix is required")
|
||||
return prefixes
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Aggregate AWatch-rus DLP buckets into a local warehouse database.")
|
||||
parser.add_argument("--aw-url", default=os.environ.get("AW_URL", "http://127.0.0.1:5600/api/0"))
|
||||
parser.add_argument("--postgres-dsn", default=os.environ.get("DLP_AGGREGATOR_POSTGRES_DSN"))
|
||||
parser.add_argument("--sqlite-path", default=os.environ.get("DLP_AGGREGATOR_SQLITE_PATH", DEFAULT_SQLITE_PATH))
|
||||
parser.add_argument("--state-path", default=os.environ.get("DLP_AGGREGATOR_STATE_PATH", "data/dlp-aggregator-state.json"))
|
||||
parser.add_argument("--bucket-prefixes", type=parse_prefixes, default=DEFAULT_BUCKET_PREFIXES)
|
||||
parser.add_argument("--since", help="UTC ISO timestamp. Overrides saved state, for example 2026-05-02T00:00:00Z.")
|
||||
parser.add_argument("--lookback-hours", type=int, default=24)
|
||||
parser.add_argument("--overlap-seconds", type=int, default=60)
|
||||
parser.add_argument("--limit", type=int, default=10000)
|
||||
parser.add_argument("--timeout", type=int, default=15)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
state_path = Path(args.state_path)
|
||||
state = load_state(state_path)
|
||||
start = get_start_time(args, state)
|
||||
end = utc_now()
|
||||
|
||||
buckets = select_buckets(list_buckets(args.aw_url, args.timeout), args.bucket_prefixes)
|
||||
all_events: list[AwEvent] = []
|
||||
for bucket, stream_type in buckets:
|
||||
all_events.extend(fetch_bucket_events(args.aw_url, bucket, stream_type, start, end, args.limit, args.timeout))
|
||||
|
||||
if args.dry_run:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"aw_url": args.aw_url,
|
||||
"start": format_aw_timestamp(start),
|
||||
"end": format_aw_timestamp(end),
|
||||
"selected_buckets": [bucket.id for bucket, _stream_type in buckets],
|
||||
"fetched_events": len(all_events),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.postgres_dsn:
|
||||
target = "postgres"
|
||||
target_path = args.postgres_dsn.split("@")[-1]
|
||||
inserted = insert_postgres_events(args.postgres_dsn, all_events)
|
||||
else:
|
||||
target = "sqlite"
|
||||
sqlite_path = Path(args.sqlite_path)
|
||||
target_path = str(sqlite_path)
|
||||
connection = connect_sqlite(sqlite_path)
|
||||
try:
|
||||
ensure_schema(connection)
|
||||
inserted = insert_events(connection, all_events)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
state["last_end"] = format_aw_timestamp(end)
|
||||
save_state(state_path, state)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"aw_url": args.aw_url,
|
||||
"target": target,
|
||||
"target_path": target_path,
|
||||
"state_path": str(state_path),
|
||||
"start": format_aw_timestamp(start),
|
||||
"end": format_aw_timestamp(end),
|
||||
"selected_buckets": len(buckets),
|
||||
"fetched_events": len(all_events),
|
||||
"inserted_events": inserted,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except urllib.error.URLError as exc:
|
||||
print(f"ActivityWatch API request failed: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
@@ -43,8 +43,5 @@ for candidate in "${rust_candidates[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
python3 "$REPO_ROOT/scripts/extract_ioc_from_sigma.py" \
|
||||
--rules-root "$RULES_ROOT" \
|
||||
--out-dir "$OUT_DIR"
|
||||
|
||||
echo "IOC artifacts generated in: $OUT_DIR"
|
||||
echo "ERROR: Rust extractor not found. Build it with: cd '$REPO_ROOT/adk-rust' && cargo build --release -p extract-ioc-from-sigma" >&2
|
||||
exit 127
|
||||
|
||||
@@ -24,59 +24,5 @@ for candidate in "${rust_candidates[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
root=Path('.')
|
||||
kit=Path('install-kit-awindows-20260427-211240')
|
||||
if not kit.exists():
|
||||
raise SystemExit('Install kit directory not found')
|
||||
|
||||
def sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
all_compared=[]
|
||||
mismatches=[]
|
||||
missing_in_repo=[]
|
||||
allowed_kit_only_prefixes=('server-configs-' ,)
|
||||
allowed_kit_only_files={'README-INSTALL-KIT.txt'}
|
||||
|
||||
for kp in sorted(p for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'):
|
||||
rel=kp.relative_to(kit)
|
||||
rp=root/rel
|
||||
if not rp.exists():
|
||||
rel_str=str(rel)
|
||||
if rel_str in allowed_kit_only_files or rel_str.startswith(allowed_kit_only_prefixes):
|
||||
continue
|
||||
missing_in_repo.append(str(rel))
|
||||
continue
|
||||
all_compared.append(str(rel))
|
||||
if sha(kp)!=sha(rp):
|
||||
mismatches.append(str(rel))
|
||||
|
||||
ps_mismatches=[
|
||||
p for p in mismatches
|
||||
if p.startswith('windows/') and p.endswith(('.ps1', '.psm1', '.psd1'))
|
||||
]
|
||||
|
||||
print(f'Compared files: {len(all_compared)}')
|
||||
print(f'Missing in repo: {len(missing_in_repo)}')
|
||||
print(f'Mismatched content: {len(mismatches)}')
|
||||
if missing_in_repo:
|
||||
print('--- Missing in repo ---')
|
||||
for p in missing_in_repo:
|
||||
print(p)
|
||||
if mismatches:
|
||||
print('--- Mismatches ---')
|
||||
for p in mismatches:
|
||||
print(p)
|
||||
print(f'PowerShell mismatches: {len(ps_mismatches)}')
|
||||
if ps_mismatches:
|
||||
print('--- PowerShell mismatches ---')
|
||||
for p in ps_mismatches:
|
||||
print(p)
|
||||
if missing_in_repo or mismatches:
|
||||
sys.exit(1)
|
||||
PY
|
||||
echo "ERROR: Rust checker not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p check-install-kit-vs-repo" >&2
|
||||
exit 127
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from urllib import request
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
def get_json(url: str):
|
||||
with request.urlopen(url, timeout=30) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
|
||||
def send_json(url: str, method: str, payload: dict | None = None):
|
||||
body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
req = request.Request(url, data=body, method=method, headers={"Content-Type": "application/json"})
|
||||
with request.urlopen(req, timeout=30) as r:
|
||||
raw = r.read().decode("utf-8")
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def parse_iso(ts: str | None) -> datetime | None:
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone(UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description="AWatch DLP admin CLI")
|
||||
p.add_argument("--policy-server", default="http://127.0.0.1:5601")
|
||||
p.add_argument("--case-server", default="http://127.0.0.1:5602")
|
||||
p.add_argument("--aw-server", default="http://127.0.0.1:5600")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
policies = sub.add_parser("policies")
|
||||
policies_sub = policies.add_subparsers(dest="policies_cmd", required=True)
|
||||
policies_sub.add_parser("list")
|
||||
policies_sub.add_parser("active")
|
||||
|
||||
incidents = sub.add_parser("incidents")
|
||||
incidents_sub = incidents.add_subparsers(dest="incidents_cmd", required=True)
|
||||
incidents_list = incidents_sub.add_parser("list")
|
||||
incidents_list.add_argument("--host")
|
||||
incidents_list.add_argument("--severity")
|
||||
incidents_list.add_argument("--limit", type=int, default=100)
|
||||
incidents_list.add_argument("--since-hours", type=int, default=24)
|
||||
|
||||
cases = sub.add_parser("cases")
|
||||
cases_sub = cases.add_subparsers(dest="cases_cmd", required=True)
|
||||
cases_list = cases_sub.add_parser("list")
|
||||
cases_list.add_argument("--host")
|
||||
cases_list.add_argument("--status")
|
||||
cases_list.add_argument("--limit", type=int, default=100)
|
||||
cases_create = cases_sub.add_parser("create")
|
||||
cases_create.add_argument("--incident-id", required=True)
|
||||
cases_create.add_argument("--title", required=True)
|
||||
cases_create.add_argument("--host")
|
||||
cases_create.add_argument("--severity", default="medium")
|
||||
|
||||
health = sub.add_parser("health")
|
||||
health_sub = health.add_subparsers(dest="health_cmd", required=True)
|
||||
health_sub.add_parser("check")
|
||||
|
||||
args = p.parse_args()
|
||||
|
||||
if args.cmd == "policies" and args.policies_cmd == "list":
|
||||
data = get_json(f"{args.policy_server}/api/0/dlp/policies")
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.cmd == "policies" and args.policies_cmd == "active":
|
||||
data = get_json(f"{args.policy_server}/api/0/dlp/policies/active")
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.cmd == "incidents" and args.incidents_cmd == "list":
|
||||
bucket_map = get_json(f"{args.aw_server}/api/0/buckets")
|
||||
if not isinstance(bucket_map, dict):
|
||||
print("[]")
|
||||
return
|
||||
|
||||
bucket_ids = [x for x in bucket_map.keys() if str(x).startswith("aw-dlp-incidents_")]
|
||||
if args.host:
|
||||
bucket_ids = [x for x in bucket_ids if str(x).endswith("_" + args.host)]
|
||||
|
||||
after = datetime.now(UTC) - timedelta(hours=max(1, args.since_hours))
|
||||
rows = []
|
||||
for bucket_id in sorted(bucket_ids):
|
||||
encoded = quote(str(bucket_id), safe="")
|
||||
events = get_json(f"{args.aw_server}/api/0/buckets/{encoded}/events?limit={max(1, args.limit)}")
|
||||
if not isinstance(events, list):
|
||||
continue
|
||||
for ev in events:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
ts = parse_iso(ev.get("timestamp"))
|
||||
if ts is None or ts < after:
|
||||
continue
|
||||
data = ev.get("data") or {}
|
||||
if args.severity and str((data or {}).get("severity", "")).lower() != args.severity.lower():
|
||||
continue
|
||||
rows.append(ev)
|
||||
print(json.dumps(rows, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.cmd == "cases" and args.cases_cmd == "list":
|
||||
query = []
|
||||
if args.host:
|
||||
query.append(f"host={quote(args.host, safe='')}")
|
||||
if args.status:
|
||||
query.append(f"status={quote(args.status, safe='')}")
|
||||
query.append(f"limit={max(1, args.limit)}")
|
||||
data = get_json(f"{args.case_server}/api/0/dlp/cases?{'&'.join(query)}")
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.cmd == "cases" and args.cases_cmd == "create":
|
||||
payload = {
|
||||
"incident_id": args.incident_id,
|
||||
"title": args.title,
|
||||
"host": args.host,
|
||||
"severity": args.severity,
|
||||
"evidence": {"source": "dlp-admin-cli"},
|
||||
}
|
||||
data = send_json(f"{args.case_server}/api/0/dlp/cases", "POST", payload)
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
if args.cmd == "health" and args.health_cmd == "check":
|
||||
out = {}
|
||||
try:
|
||||
out["policy"] = get_json(f"{args.policy_server}/healthz")
|
||||
except Exception as exc:
|
||||
out["policy"] = {"status": "error", "error": str(exc)}
|
||||
|
||||
try:
|
||||
out["cases"] = get_json(f"{args.case_server}/health")
|
||||
except Exception as exc:
|
||||
out["cases"] = {"status": "error", "error": str(exc)}
|
||||
|
||||
try:
|
||||
out["aw"] = get_json(f"{args.aw_server}/api/0/info")
|
||||
except Exception as exc:
|
||||
out["aw"] = {"status": "error", "error": str(exc)}
|
||||
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
raise SystemExit("unsupported command")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,235 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract IOC-like indicators from Sigma YAML rules for DLP preload.
|
||||
|
||||
Extracted fields:
|
||||
- Image|endswith
|
||||
- CommandLine|contains
|
||||
- OriginalFileName
|
||||
- Hashes|SHA256 (plus SHA256 values embedded in Hashes strings)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
BASE_FIELDS = {"image", "commandline", "originalfilename", "hashes"}
|
||||
SHA256_RE = re.compile(r"\b[a-fA-F0-9]{64}\b")
|
||||
|
||||
|
||||
def split_key(key: str) -> tuple[str, list[str]]:
|
||||
parts = [p.strip() for p in str(key).split("|") if p.strip()]
|
||||
if not parts:
|
||||
return "", []
|
||||
return parts[0].lower(), [p.lower() for p in parts[1:]]
|
||||
|
||||
|
||||
def flatten_values(value: Any) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
v = value.strip()
|
||||
return [v] if v else []
|
||||
if isinstance(value, (int, float, bool)):
|
||||
return [str(value)]
|
||||
if isinstance(value, list):
|
||||
out: list[str] = []
|
||||
for item in value:
|
||||
out.extend(flatten_values(item))
|
||||
return out
|
||||
if isinstance(value, dict):
|
||||
out: list[str] = []
|
||||
for k, v in value.items():
|
||||
vals = flatten_values(v)
|
||||
for vv in vals:
|
||||
out.append(f"{k}:{vv}")
|
||||
return out
|
||||
return []
|
||||
|
||||
|
||||
def detect_ioc_type(base: str, ops: list[str], raw: str) -> str | None:
|
||||
if base == "image" and "endswith" in ops:
|
||||
return "process_image_endswith"
|
||||
if base == "commandline" and "contains" in ops:
|
||||
return "commandline_contains"
|
||||
if base == "originalfilename":
|
||||
return "original_filename"
|
||||
if base == "hashes" and ("sha256" in ops or SHA256_RE.search(raw)):
|
||||
return "sha256"
|
||||
return None
|
||||
|
||||
|
||||
def parse_sha256(raw: str) -> list[str]:
|
||||
vals = SHA256_RE.findall(raw)
|
||||
seen = set()
|
||||
out = []
|
||||
for v in vals:
|
||||
lv = v.lower()
|
||||
if lv in seen:
|
||||
continue
|
||||
seen.add(lv)
|
||||
out.append(lv)
|
||||
return out
|
||||
|
||||
|
||||
def walk(node: Any, *, rule_id: str, rule_title: str, source_file: str, out: list[dict[str, str]]) -> None:
|
||||
if isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
base, ops = split_key(str(k))
|
||||
if base in BASE_FIELDS:
|
||||
for raw in flatten_values(v):
|
||||
ioc_type = detect_ioc_type(base, ops, raw)
|
||||
if not ioc_type:
|
||||
continue
|
||||
if ioc_type == "sha256":
|
||||
for h in parse_sha256(raw):
|
||||
out.append(
|
||||
{
|
||||
"ioc_type": "sha256",
|
||||
"ioc_value": h,
|
||||
"field": str(k),
|
||||
"rule_id": rule_id,
|
||||
"rule_title": rule_title,
|
||||
"source_file": source_file,
|
||||
}
|
||||
)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"ioc_type": ioc_type,
|
||||
"ioc_value": raw,
|
||||
"field": str(k),
|
||||
"rule_id": rule_id,
|
||||
"rule_title": rule_title,
|
||||
"source_file": source_file,
|
||||
}
|
||||
)
|
||||
walk(v, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item, rule_id=rule_id, rule_title=rule_title, source_file=source_file, out=out)
|
||||
|
||||
|
||||
def extract_from_yaml(path: Path) -> list[dict[str, str]]:
|
||||
try:
|
||||
doc = yaml.safe_load(path.read_text(encoding="utf-8", errors="ignore"))
|
||||
except Exception:
|
||||
return []
|
||||
if not isinstance(doc, dict):
|
||||
return []
|
||||
detection = doc.get("detection")
|
||||
if detection is None:
|
||||
return []
|
||||
rid = str(doc.get("id") or "")
|
||||
title = str(doc.get("title") or "")
|
||||
rows: list[dict[str, str]] = []
|
||||
walk(detection, rule_id=rid, rule_title=title, source_file=str(path), out=rows)
|
||||
return rows
|
||||
|
||||
|
||||
def dedupe(rows: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
seen = set()
|
||||
out = []
|
||||
for r in rows:
|
||||
key = (r["ioc_type"], r["ioc_value"].lower(), r["field"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def write_json(path: Path, rows: list[dict[str, str]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(rows, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, str]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fields = ["ioc_type", "ioc_value", "field", "rule_id", "rule_title", "source_file"]
|
||||
with path.open("w", encoding="utf-8", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fields)
|
||||
w.writeheader()
|
||||
for row in rows:
|
||||
w.writerow(row)
|
||||
|
||||
|
||||
def sql_escape(s: str) -> str:
|
||||
return s.replace("'", "''")
|
||||
|
||||
|
||||
def write_sql(path: Path, rows: list[dict[str, str]], table_name: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
f.write(
|
||||
f"CREATE TABLE IF NOT EXISTS {table_name} (\n"
|
||||
" id INTEGER PRIMARY KEY AUTOINCREMENT,\n"
|
||||
" ioc_type TEXT NOT NULL,\n"
|
||||
" ioc_value TEXT NOT NULL,\n"
|
||||
" field TEXT,\n"
|
||||
" rule_id TEXT,\n"
|
||||
" rule_title TEXT,\n"
|
||||
" source_file TEXT\n"
|
||||
");\n\n"
|
||||
)
|
||||
for r in rows:
|
||||
f.write(
|
||||
f"INSERT INTO {table_name} (ioc_type, ioc_value, field, rule_id, rule_title, source_file) VALUES "
|
||||
f"('{sql_escape(r['ioc_type'])}',"
|
||||
f"'{sql_escape(r['ioc_value'])}',"
|
||||
f"'{sql_escape(r['field'])}',"
|
||||
f"'{sql_escape(r['rule_id'])}',"
|
||||
f"'{sql_escape(r['rule_title'])}',"
|
||||
f"'{sql_escape(r['source_file'])}');\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Extract IOC-like Sigma values for DLP preload.")
|
||||
ap.add_argument("--rules-root", default="rules", help="Path to hayabusa-rules root")
|
||||
ap.add_argument("--out-dir", default="ioc_export", help="Output directory")
|
||||
ap.add_argument("--table-name", default="dlp_blacklist_ioc", help="SQL table name")
|
||||
args = ap.parse_args()
|
||||
|
||||
rules_root = Path(args.rules_root)
|
||||
if not rules_root.exists():
|
||||
raise SystemExit(f"rules root not found: {rules_root}")
|
||||
|
||||
yaml_files = [p for p in rules_root.rglob("*") if p.is_file() and p.suffix.lower() in {".yml", ".yaml"}]
|
||||
all_rows: list[dict[str, str]] = []
|
||||
for yp in yaml_files:
|
||||
all_rows.extend(extract_from_yaml(yp))
|
||||
|
||||
rows = dedupe(all_rows)
|
||||
rows.sort(key=lambda r: (r["ioc_type"], r["ioc_value"].lower()))
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
write_json(out_dir / "ioc_blacklist.json", rows)
|
||||
write_csv(out_dir / "ioc_blacklist.csv", rows)
|
||||
write_sql(out_dir / "ioc_blacklist.sql", rows, args.table_name)
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
for r in rows:
|
||||
counts[r["ioc_type"]] = counts.get(r["ioc_type"], 0) + 1
|
||||
|
||||
print(f"rules_scanned={len(yaml_files)}")
|
||||
print(f"iocs_extracted={len(rows)}")
|
||||
for k in sorted(counts):
|
||||
print(f"{k}={counts[k]}")
|
||||
print(f"json={out_dir / 'ioc_blacklist.json'}")
|
||||
print(f"csv={out_dir / 'ioc_blacklist.csv'}")
|
||||
print(f"sql={out_dir / 'ioc_blacklist.sql'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def maybe_exec_rust() -> None:
|
||||
if os.environ.get("MERGE_AW_SERVER_DBS_FORCE_LEGACY") == "1":
|
||||
return
|
||||
script_path = Path(__file__).resolve()
|
||||
repo_root = script_path.parent.parent if script_path.parent.name == "scripts" else None
|
||||
candidates = [
|
||||
os.environ.get("MERGE_AW_SERVER_DBS_RUST"),
|
||||
str(Path(os.environ.get("CARGO_TARGET_DIR", "")) / "release" / "merge-aw-server-dbs")
|
||||
if os.environ.get("CARGO_TARGET_DIR")
|
||||
else None,
|
||||
str(repo_root / "adk-rust" / "target" / "release" / "merge-aw-server-dbs")
|
||||
if repo_root
|
||||
else None,
|
||||
"/usr/local/bin/merge-aw-server-dbs",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate and os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
os.execv(candidate, [candidate, *sys.argv[1:]])
|
||||
|
||||
|
||||
maybe_exec_rust()
|
||||
|
||||
|
||||
def connect(path: Path) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(str(path))
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("PRAGMA synchronous=NORMAL")
|
||||
return connection
|
||||
|
||||
|
||||
def bucket_key(row: sqlite3.Row) -> tuple[str, str, str, str]:
|
||||
return (
|
||||
str(row["name"]),
|
||||
str(row["type"]),
|
||||
str(row["client"]),
|
||||
str(row["hostname"]),
|
||||
)
|
||||
|
||||
def find_bucket_by_name(connection: sqlite3.Connection, name: str) -> int | None:
|
||||
row = connection.execute(
|
||||
"select rowid as bucketrow from buckets where name = ? order by rowid limit 1",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return int(row["bucketrow"]) if row else None
|
||||
|
||||
|
||||
def find_bucket_by_key(connection: sqlite3.Connection, name: str, type_: str, client: str, hostname: str) -> int | None:
|
||||
row = connection.execute(
|
||||
"select rowid as bucketrow from buckets where name = ? and type = ? and client = ? and hostname = ? order by rowid limit 1",
|
||||
(name, type_, client, hostname),
|
||||
).fetchone()
|
||||
return int(row["bucketrow"]) if row else None
|
||||
|
||||
|
||||
def copy_sqlite_via_backup(src: Path, dst: Path) -> None:
|
||||
"""Create a consistent copy of an sqlite DB using the sqlite backup API.
|
||||
|
||||
This avoids corrupt/inconsistent files if the source DB is live.
|
||||
"""
|
||||
# ensure parent exists
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
# remove any existing tmp file
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
src_conn = sqlite3.connect(str(src))
|
||||
dst_conn = sqlite3.connect(str(dst))
|
||||
try:
|
||||
# perform online backup
|
||||
src_conn.backup(dst_conn)
|
||||
dst_conn.commit()
|
||||
finally:
|
||||
try:
|
||||
src_conn.close()
|
||||
finally:
|
||||
dst_conn.close()
|
||||
|
||||
|
||||
def ensure_parent(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def load_existing_events(connection: sqlite3.Connection, bucketrow: int) -> set[tuple[int, int, str]]:
|
||||
cursor = connection.execute(
|
||||
"select starttime, endtime, data from events where bucketrow = ?",
|
||||
(bucketrow,),
|
||||
)
|
||||
return {(int(start), int(end), str(data)) for start, end, data in cursor.fetchall()}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--overlay")
|
||||
args = parser.parse_args()
|
||||
|
||||
base = Path(args.base)
|
||||
output = Path(args.output)
|
||||
overlay = Path(args.overlay) if args.overlay else None
|
||||
|
||||
if not base.exists():
|
||||
raise SystemExit(f"Base DB not found: {base}")
|
||||
|
||||
ensure_parent(output)
|
||||
tmp_output = output.with_suffix(output.suffix + ".tmp")
|
||||
# make a consistent copy of the base DB into tmp_output
|
||||
copy_sqlite_via_backup(base, tmp_output)
|
||||
|
||||
dest = connect(tmp_output)
|
||||
dest.row_factory = sqlite3.Row
|
||||
|
||||
inserted_buckets = 0
|
||||
inserted_events = 0
|
||||
|
||||
if overlay and overlay.exists():
|
||||
source = connect(overlay)
|
||||
source.row_factory = sqlite3.Row
|
||||
try:
|
||||
source_buckets = source.execute(
|
||||
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
|
||||
).fetchall()
|
||||
|
||||
dest_bucket_map = {
|
||||
bucket_key(row): row["bucketrow"]
|
||||
for row in dest.execute(
|
||||
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
|
||||
).fetchall()
|
||||
}
|
||||
dest_id_map = {
|
||||
row["id"]: row["bucketrow"]
|
||||
for row in dest.execute(
|
||||
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
|
||||
).fetchall()
|
||||
if row["id"]
|
||||
}
|
||||
|
||||
for src_bucket in source_buckets:
|
||||
key = bucket_key(src_bucket)
|
||||
src_id = src_bucket["id"] if "id" in src_bucket.keys() else None
|
||||
dest_rowid = None
|
||||
# Prefer exact id match if available
|
||||
if src_id:
|
||||
dest_rowid = dest_id_map.get(src_id)
|
||||
if dest_rowid is None:
|
||||
dest_rowid = dest_bucket_map.get(key)
|
||||
if dest_rowid is None:
|
||||
dest_rowid = find_bucket_by_name(dest, str(src_bucket["name"]))
|
||||
if dest_rowid is not None:
|
||||
dest.execute(
|
||||
"""
|
||||
UPDATE buckets
|
||||
SET type = ?, client = ?, hostname = ?, created = ?,
|
||||
data_deprecated = ?, data = ?
|
||||
WHERE rowid = ?
|
||||
""",
|
||||
(
|
||||
src_bucket["type"],
|
||||
src_bucket["client"],
|
||||
src_bucket["hostname"],
|
||||
src_bucket["created"],
|
||||
src_bucket["data_deprecated"],
|
||||
src_bucket["data"],
|
||||
dest_rowid,
|
||||
),
|
||||
)
|
||||
else:
|
||||
cursor = dest.execute(
|
||||
"""
|
||||
INSERT INTO buckets (id, name, type, client, hostname, created, data_deprecated, data)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
src_bucket["id"],
|
||||
src_bucket["name"],
|
||||
src_bucket["type"],
|
||||
src_bucket["client"],
|
||||
src_bucket["hostname"],
|
||||
src_bucket["created"],
|
||||
src_bucket["data_deprecated"],
|
||||
src_bucket["data"],
|
||||
),
|
||||
)
|
||||
dest_rowid = int(cursor.lastrowid)
|
||||
inserted_buckets += 1
|
||||
|
||||
dest_bucket_map[key] = dest_rowid
|
||||
if src_id:
|
||||
dest_id_map[str(src_id)] = dest_rowid
|
||||
|
||||
existing_events = load_existing_events(dest, dest_rowid)
|
||||
for starttime, endtime, data in source.execute(
|
||||
"select starttime, endtime, data from events where bucketrow = ? order by id",
|
||||
(src_bucket["bucketrow"],),
|
||||
).fetchall():
|
||||
event_key = (int(starttime), int(endtime), str(data))
|
||||
if event_key in existing_events:
|
||||
continue
|
||||
dest.execute(
|
||||
"insert into events (bucketrow, starttime, endtime, data) values (?, ?, ?, ?)",
|
||||
(dest_rowid, int(starttime), int(endtime), str(data)),
|
||||
)
|
||||
existing_events.add(event_key)
|
||||
inserted_events += 1
|
||||
|
||||
dest.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
dest.close()
|
||||
os.replace(tmp_output, output)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"base": str(base),
|
||||
"overlay": str(overlay) if overlay else None,
|
||||
"output": str(output),
|
||||
"inserted_buckets": inserted_buckets,
|
||||
"inserted_events": inserted_events,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -11,84 +11,19 @@ if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "--apply-legacy" ]]; then
|
||||
shift
|
||||
else
|
||||
for candidate in \
|
||||
"${PROD_BACKUP_RESTORE_RUST:-}" \
|
||||
"${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \
|
||||
"$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \
|
||||
"/usr/local/bin/prod-backup-restore"; do
|
||||
if [[ -n "$candidate" && -x "$candidate" ]]; then
|
||||
exec "$candidate" --root "$ROOT_DIR" "$@"
|
||||
fi
|
||||
done
|
||||
cat >&2 <<'EOF'
|
||||
prod_backup_restore.sh is destructive and now requires the Rust planner.
|
||||
for candidate in \
|
||||
"${PROD_BACKUP_RESTORE_RUST:-}" \
|
||||
"${CARGO_TARGET_DIR:-}/release/prod-backup-restore" \
|
||||
"$ROOT_DIR/adk-rust/target/release/prod-backup-restore" \
|
||||
"/usr/local/bin/prod-backup-restore"; do
|
||||
if [[ -n "$candidate" && -x "$candidate" ]]; then
|
||||
exec "$candidate" --root "$ROOT_DIR" "$@"
|
||||
fi
|
||||
done
|
||||
|
||||
cat >&2 <<EOF
|
||||
prod_backup_restore.sh now requires the Rust planner/checker.
|
||||
Build it first:
|
||||
cd adk-rust && cargo build --release -p prod-backup-restore
|
||||
|
||||
To run the old destructive restore explicitly:
|
||||
scripts/prod_backup_restore.sh --apply-legacy
|
||||
cd "$ROOT_DIR/adk-rust" && cargo build --release -p prod-backup-restore
|
||||
EOF
|
||||
exit 2
|
||||
fi
|
||||
|
||||
: "${AW_SSH_PASSWORD:?AW_SSH_PASSWORD is required}"
|
||||
: "${AW_WINRM_PASSWORD:?AW_WINRM_PASSWORD is required}"
|
||||
|
||||
command -v sshpass >/dev/null 2>&1 || { echo "missing sshpass" >&2; exit 127; }
|
||||
command -v ansible-playbook >/dev/null 2>&1 || { echo "missing ansible-playbook" >&2; exit 127; }
|
||||
|
||||
SERVER_HOST="${AW_SERVER_HOST:-10.10.10.13}"
|
||||
SERVER_USER="${AW_SERVER_USER:-igor}"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
REMOTE_BACKUP_DIR="/var/lib/activitywatch/backups/prod-restore-${TIMESTAMP}"
|
||||
LEGACY_DB="/root/.local/share/activitywatch/aw-server-rust/sqlite.db"
|
||||
TARGET_DB="/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db"
|
||||
REMOTE_MERGE_SCRIPT="/tmp/merge_aw_server_dbs.py"
|
||||
|
||||
ssh_remote() {
|
||||
sshpass -p "$AW_SSH_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${SERVER_USER}@${SERVER_HOST}" "$@"
|
||||
}
|
||||
|
||||
scp_remote() {
|
||||
sshpass -p "$AW_SSH_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
|
||||
}
|
||||
|
||||
scp_remote "${ROOT_DIR}/scripts/merge_aw_server_dbs.py" "${SERVER_USER}@${SERVER_HOST}:${REMOTE_MERGE_SCRIPT}"
|
||||
|
||||
ssh_remote "sudo mkdir -p '${REMOTE_BACKUP_DIR}' && sudo chown root:root '${REMOTE_BACKUP_DIR}'"
|
||||
ssh_remote "sudo test -f '${LEGACY_DB}'"
|
||||
ssh_remote "sudo test -f '${TARGET_DB}'"
|
||||
ssh_remote "sudo cp -a '${LEGACY_DB}' '${REMOTE_BACKUP_DIR}/legacy-root-sqlite.db' && sudo cp -a '${TARGET_DB}' '${REMOTE_BACKUP_DIR}/target-before-merge-sqlite.db'"
|
||||
ssh_remote "sudo systemctl stop activitywatch-server.service || true"
|
||||
ssh_remote "sudo python3 '${REMOTE_MERGE_SCRIPT}' --base '${LEGACY_DB}' --overlay '${TARGET_DB}' --output '${REMOTE_BACKUP_DIR}/sqlite.merged.db'"
|
||||
ssh_remote "sudo install -o activitywatch -g activitywatch -m 0644 '${REMOTE_BACKUP_DIR}/sqlite.merged.db' '${TARGET_DB}'"
|
||||
|
||||
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml
|
||||
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_windows.yml
|
||||
ansible-playbook -i ansible/inventory.ini ansible/post_validate_aw_windows.yml
|
||||
|
||||
python3 - <<'PY'
|
||||
import json, urllib.request
|
||||
base = 'http://10.10.10.13:5600'
|
||||
window_payload = {
|
||||
'timeperiods': ['2026-04-29T00:00:00+03:00/2026-04-29T23:59:59+03:00'],
|
||||
'query': [
|
||||
'window_events = query_bucket(find_bucket("aw-watcher-window_SHARKON2025"));',
|
||||
'RETURN = window_events;'
|
||||
]
|
||||
}
|
||||
req = urllib.request.Request(base + '/api/0/query/', data=json.dumps(window_payload).encode(), method='POST', headers={'Content-Type': 'application/json', 'Origin': 'http://10.10.10.13:5600'})
|
||||
with urllib.request.urlopen(req) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
window_count = len(data[0]) if isinstance(data, list) and data else 0
|
||||
if window_count <= 0:
|
||||
raise SystemExit('no historical window data restored for 2026-04-29')
|
||||
with urllib.request.urlopen(base + '/api/0/settings/') as response:
|
||||
settings = json.loads(response.read().decode())
|
||||
if settings.get('always_active_pattern') != 'aw-watcher-window':
|
||||
raise SystemExit('always_active_pattern is not configured')
|
||||
print(json.dumps({'restored_window_events_2026_04_29': window_count, 'always_active_pattern': settings.get('always_active_pattern')}, ensure_ascii=False))
|
||||
PY
|
||||
exit 127
|
||||
|
||||
@@ -23,150 +23,5 @@ for candidate in "${rust_candidates[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
KIT_DIR="install-kit-awindows-20260427-211240"
|
||||
ZIP_ARCHIVE="${KIT_DIR}.zip"
|
||||
TAR_ARCHIVE="${KIT_DIR}.tar.gz"
|
||||
SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.18"
|
||||
OLD_SERVER_CONFIG_DIR="${KIT_DIR}/server-configs-192.168.100.21"
|
||||
TMP_SERVER_CONFIG_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_SERVER_CONFIG_DIR"' EXIT
|
||||
|
||||
copy_file() {
|
||||
local src="$1"
|
||||
local dest="$2"
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
cp "$src" "$dest"
|
||||
}
|
||||
|
||||
sync_tree() {
|
||||
local base="$1"
|
||||
shift
|
||||
for rel in "$@"; do
|
||||
copy_file "$rel" "${KIT_DIR}/${rel}"
|
||||
done
|
||||
}
|
||||
|
||||
if [[ -d "$OLD_SERVER_CONFIG_DIR" ]]; then
|
||||
cp "$OLD_SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/
|
||||
fi
|
||||
if [[ -d "$SERVER_CONFIG_DIR" ]]; then
|
||||
cp "$SERVER_CONFIG_DIR"/*.deployment-config.json "$TMP_SERVER_CONFIG_DIR"/
|
||||
fi
|
||||
|
||||
rm -rf "${KIT_DIR}/ansible" "${KIT_DIR}/aw-server" "${KIT_DIR}/windows" "${KIT_DIR}/scripts" "${KIT_DIR}/server-configs-"*
|
||||
|
||||
ansible_files=(
|
||||
ansible/README.md
|
||||
ansible/deploy_aw_pfsense_poller.yml
|
||||
ansible/deploy_aw_server.yml
|
||||
ansible/deploy_aw_windows.yml
|
||||
ansible/group_vars/all.example.yml
|
||||
ansible/group_vars/pfsense-poller.example.yml
|
||||
ansible/group_vars/proxmox-matrix.example.yml
|
||||
ansible/group_vars/proxmox.example.yml
|
||||
ansible/group_vars/windows.example.yml
|
||||
ansible/install_full_stack.yml
|
||||
ansible/inventory.example.ini
|
||||
ansible/provision_proxmox_ct_and_deploy_aw.yml
|
||||
ansible/provision_proxmox_ct_matrix_and_deploy_aw.yml
|
||||
ansible/tasks/provision_ct_and_deploy_aw.yml
|
||||
)
|
||||
|
||||
aw_server_files=(
|
||||
aw-server/activitywatch-server.service
|
||||
aw-server/apply_webui_ru_patch.sh
|
||||
aw-server/aw-host-groups.json
|
||||
aw-server/aw-ru-patch.js
|
||||
aw-server/aw-rus-healthd.py
|
||||
aw-server/aw-rus-healthd.service
|
||||
aw-server/aw-rus-healthd.timer
|
||||
aw-server/aw-browser-smoke.service
|
||||
aw-server/aw-browser-smoke.timer
|
||||
aw-server/aw-slo-monitor.py
|
||||
aw-server/aw-slo-monitor.service
|
||||
aw-server/aw-slo-monitor.timer
|
||||
aw-server/aw-server.env.example
|
||||
aw-server/aw-sw-cleanup.js
|
||||
aw-server/aw-worktime-api.py
|
||||
aw-server/aw-worktime-api.service
|
||||
aw-server/aw-worktime-prewarm.sh
|
||||
aw-server/aw-worktime-prewarm.service
|
||||
aw-server/aw-worktime-prewarm.timer
|
||||
aw-server/aw-worktime-panel.js
|
||||
aw-server/health-check.sh
|
||||
aw-server/install_aw_server.sh
|
||||
aw-server/settings/classes-worktime.json
|
||||
aw-server/settings/views-default.json
|
||||
)
|
||||
|
||||
windows_files=(
|
||||
windows/ActivityWatch.Windows.Common.psd1
|
||||
windows/ActivityWatch.Windows.Common.psm1
|
||||
windows/browser-domains-native-collector.ps1
|
||||
windows/deploy-domain-users.ps1
|
||||
windows/deploy-ensemble.ps1
|
||||
windows/deploy-single-user.ps1
|
||||
windows/AWatchRusCollectorGuardService.cs
|
||||
windows/aw-collector-guard.ps1
|
||||
windows/install-collector-guard-service.ps1
|
||||
windows/dlp-endpoint-signals-collector.ps1
|
||||
windows/dlp-policy.example.json
|
||||
windows/dlp-policy.native-cross-os.example.json
|
||||
windows/email-outbound-collector.ps1
|
||||
windows/hardening-recovery.ps1
|
||||
windows/migrate-awatch-rus-paths.ps1
|
||||
windows/validate-deployment.ps1
|
||||
windows/web-category-rules.example.json
|
||||
windows/worktime-session-collector.ps1
|
||||
)
|
||||
|
||||
scripts_files=(
|
||||
scripts/aw-webui-browser-smoke.mjs
|
||||
scripts/aw-webui-browser-smoke.sh
|
||||
scripts/check_install_kit_vs_repo.sh
|
||||
scripts/quality-gate.sh
|
||||
scripts/rebuild_install_kit.sh
|
||||
scripts/validate_install_kit.sh
|
||||
scripts/verify_innosetup_installer.sh
|
||||
)
|
||||
|
||||
sync_tree ansible "${ansible_files[@]}"
|
||||
sync_tree aw-server "${aw_server_files[@]}"
|
||||
sync_tree windows "${windows_files[@]}"
|
||||
sync_tree scripts "${scripts_files[@]}"
|
||||
|
||||
mkdir -p "$SERVER_CONFIG_DIR"
|
||||
if compgen -G "$TMP_SERVER_CONFIG_DIR/*.deployment-config.json" >/dev/null; then
|
||||
cp "$TMP_SERVER_CONFIG_DIR"/*.deployment-config.json "$SERVER_CONFIG_DIR"/
|
||||
fi
|
||||
|
||||
cat > "${KIT_DIR}/README-INSTALL-KIT.txt" <<'EOF'
|
||||
ActivityWatch DetMir Windows Install Kit
|
||||
|
||||
Includes:
|
||||
- windows/* (deploy scripts, collectors, common module, configs/examples)
|
||||
- ansible/* (Windows and AW server playbooks, examples, inventory, tasks)
|
||||
- aw-server/* (server installer, health orchestrator, RU patch loader, host groups, default settings)
|
||||
- scripts/* (install-kit rebuild/validation, quality gates and browser/web smoke checks)
|
||||
- server-configs-192.168.100.18/* (working Windows/RDP config snapshots)
|
||||
|
||||
Source:
|
||||
- Local project snapshot at build time.
|
||||
EOF
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
|
||||
root = Path('install-kit-awindows-20260427-211240')
|
||||
manifest = root / 'MANIFEST.txt'
|
||||
files = sorted(p for p in root.rglob('*') if p.is_file() and p.name != 'MANIFEST.txt')
|
||||
with manifest.open('w', encoding='utf-8') as handle:
|
||||
for path in files:
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
handle.write(f"{digest} {path.as_posix()}\n")
|
||||
PY
|
||||
|
||||
rm -f "$ZIP_ARCHIVE" "$TAR_ARCHIVE"
|
||||
zip -rq "$ZIP_ARCHIVE" "$KIT_DIR"
|
||||
tar -czf "$TAR_ARCHIVE" "$KIT_DIR"
|
||||
echo "ERROR: Rust install-kit builder not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p rebuild-install-kit" >&2
|
||||
exit 127
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_PATH = Path(__file__).with_name("dlp-health-check.py")
|
||||
SPEC = importlib.util.spec_from_file_location("dlp_health_check", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = MODULE
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_endpoint_signals_ok_when_no_active_managed_hosts(monkeypatch):
|
||||
buckets = {
|
||||
"aw-worktime-sessions_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}},
|
||||
"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T07:00:00Z"}},
|
||||
}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:00:10Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url: [{"timestamp": "2026-05-30T12:00:00Z", "data": {"active": False}}]
|
||||
if "aw-worktime-sessions_SHARKON2025/events" in url
|
||||
else [],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_endpoint_signal_buckets(report, "http://127.0.0.1:5600/api/0", buckets, 900)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "buckets:endpoint-signals"
|
||||
assert result.status == "ok"
|
||||
assert result.summary == "no active managed hosts require endpoint-signals freshness"
|
||||
assert result.details["ignored_inactive"] == ["aw-dlp-endpoint-signals_SHARKON2025"]
|
||||
|
||||
|
||||
def test_endpoint_self_test_metrics_reports_transport_counters(monkeypatch):
|
||||
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url, **kwargs: [
|
||||
{
|
||||
"timestamp": "2026-05-30T11:59:00Z",
|
||||
"data": {
|
||||
"signalType": "self_test",
|
||||
"queueDepth": 8,
|
||||
"eventsEnqueued": 100,
|
||||
"eventsFlushed": 92,
|
||||
"sendFailures": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "self_test",
|
||||
"queueDepth": "3",
|
||||
"eventsEnqueued": "120",
|
||||
"eventsFlushed": "117",
|
||||
"sendFailures": "0",
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_endpoint_self_test_metrics(report, "http://127.0.0.1:5600/api/0", buckets)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "endpoint-self-test-metrics"
|
||||
assert result.status == "ok"
|
||||
latest = result.details["latest_self_tests"][0]
|
||||
assert latest["bucket"] == "aw-dlp-endpoint-signals_SHARKON2025"
|
||||
assert latest["timestamp"] == "2026-05-30T12:00:30Z"
|
||||
assert latest["age_seconds"] == 30
|
||||
assert latest["queueDepth"] == 3
|
||||
assert latest["eventsEnqueued"] == 120
|
||||
assert latest["eventsFlushed"] == 117
|
||||
assert latest["sendFailures"] == 0
|
||||
assert latest["sendFailuresDelta"] == 0
|
||||
|
||||
|
||||
def test_endpoint_self_test_metrics_warns_on_transport_counters(monkeypatch):
|
||||
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url, **kwargs: [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "self_test",
|
||||
"queueDepth": 101,
|
||||
"eventsEnqueued": 120,
|
||||
"eventsFlushed": 19,
|
||||
"sendFailures": 1,
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_endpoint_self_test_metrics(
|
||||
report,
|
||||
"http://127.0.0.1:5600/api/0",
|
||||
buckets,
|
||||
queue_warn_depth=100,
|
||||
send_failure_warn_count=1,
|
||||
)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.status == "warn"
|
||||
assert result.summary == "endpoint transport counters outside thresholds"
|
||||
assert result.details["warnings"] == [
|
||||
{"bucket": "aw-dlp-endpoint-signals_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100},
|
||||
{
|
||||
"bucket": "aw-dlp-endpoint-signals_SHARKON2025",
|
||||
"metric": "sendFailuresDelta",
|
||||
"value": 1,
|
||||
"current": 1,
|
||||
"previous": None,
|
||||
"threshold": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_file_operations_runtime_reports_health_and_latest_operations(monkeypatch):
|
||||
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url: [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "collector_health",
|
||||
"queueDepth": "0",
|
||||
"eventsEnqueued": "5",
|
||||
"eventsFlushed": "5",
|
||||
"sendFailures": "0",
|
||||
"username": "USER1",
|
||||
"hostname": "SHARKON2025",
|
||||
"sessionId": "3",
|
||||
},
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:20Z",
|
||||
"data": {
|
||||
"operation": "Created",
|
||||
"path": "C:\\Users\\USER1\\Downloads\\report.zip",
|
||||
"extension": ".zip",
|
||||
"archiveHint": True,
|
||||
"username": "USER1",
|
||||
"hostname": "SHARKON2025",
|
||||
"size": "42",
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_file_operations_runtime(report, "http://127.0.0.1:5600/api/0", buckets)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "file-operations-runtime"
|
||||
assert result.status == "ok"
|
||||
assert result.details["latest_health"][0]["queueDepth"] == 0
|
||||
assert result.details["latest_health"][0]["eventsEnqueued"] == 5
|
||||
assert result.details["latest_health"][0]["sendFailuresDelta"] == 0
|
||||
latest_op = result.details["latest_operations"][0]
|
||||
assert latest_op["operation"] == "Created"
|
||||
assert latest_op["extension"] == ".zip"
|
||||
assert latest_op["archiveHint"] is True
|
||||
assert latest_op["path_tail"] == "Downloads/report.zip"
|
||||
assert latest_op["size"] == 42
|
||||
assert result.details["sampled"][0]["operation_counts"] == {"Created": 1}
|
||||
|
||||
|
||||
def test_file_operations_runtime_warns_on_transport_counters(monkeypatch):
|
||||
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url: [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "collector_health",
|
||||
"queueDepth": 101,
|
||||
"eventsEnqueued": 5,
|
||||
"eventsFlushed": 2,
|
||||
"sendFailures": 1,
|
||||
"username": "USER1",
|
||||
"hostname": "SHARKON2025",
|
||||
"sessionId": 3,
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_file_operations_runtime(
|
||||
report,
|
||||
"http://127.0.0.1:5600/api/0",
|
||||
buckets,
|
||||
queue_warn_depth=100,
|
||||
send_failure_warn_count=1,
|
||||
)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.status == "warn"
|
||||
assert result.summary == "file-operations runtime counters outside expectations"
|
||||
assert result.details["warnings"] == [
|
||||
{"bucket": "aw-file-operations_SHARKON2025", "metric": "queueDepth", "value": 101, "threshold": 100},
|
||||
{
|
||||
"bucket": "aw-file-operations_SHARKON2025",
|
||||
"metric": "sendFailuresDelta",
|
||||
"value": 1,
|
||||
"current": 1,
|
||||
"previous": None,
|
||||
"threshold": 1,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_endpoint_send_failures_uses_delta_baseline(monkeypatch):
|
||||
buckets = {"aw-dlp-endpoint-signals_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
state = {"counters": {}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
values = [12, 12, 13]
|
||||
|
||||
def fake_http(url, **kwargs):
|
||||
value = values.pop(0)
|
||||
return [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "self_test",
|
||||
"queueDepth": 0,
|
||||
"eventsEnqueued": 120,
|
||||
"eventsFlushed": 117,
|
||||
"sendFailures": value,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(MODULE, "_http_json", fake_http)
|
||||
|
||||
first = MODULE.HealthReport()
|
||||
MODULE.check_endpoint_self_test_metrics(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert first.results[0].status == "ok"
|
||||
assert first.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] is None
|
||||
assert first.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0
|
||||
|
||||
second = MODULE.HealthReport()
|
||||
MODULE.check_endpoint_self_test_metrics(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert second.results[0].status == "ok"
|
||||
assert second.results[0].details["latest_self_tests"][0]["sendFailuresPrevious"] == 12
|
||||
assert second.results[0].details["latest_self_tests"][0]["sendFailuresDelta"] == 0
|
||||
|
||||
third = MODULE.HealthReport()
|
||||
MODULE.check_endpoint_self_test_metrics(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert third.results[0].status == "warn"
|
||||
assert third.results[0].details["warnings"] == [
|
||||
{
|
||||
"bucket": "aw-dlp-endpoint-signals_SHARKON2025",
|
||||
"metric": "sendFailuresDelta",
|
||||
"value": 1,
|
||||
"current": 13,
|
||||
"previous": 12,
|
||||
"threshold": 1,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_file_operations_send_failures_uses_delta_baseline(monkeypatch):
|
||||
buckets = {"aw-file-operations_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
state = {"counters": {}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
values = [28, 28, 29]
|
||||
|
||||
def fake_http(url, **kwargs):
|
||||
value = values.pop(0)
|
||||
return [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"signalType": "collector_health",
|
||||
"queueDepth": 0,
|
||||
"eventsEnqueued": 120,
|
||||
"eventsFlushed": 117,
|
||||
"sendFailures": value,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(MODULE, "_http_json", fake_http)
|
||||
|
||||
first = MODULE.HealthReport()
|
||||
MODULE.check_file_operations_runtime(first, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert first.results[0].status == "ok"
|
||||
assert first.results[0].details["latest_health"][0]["sendFailuresPrevious"] is None
|
||||
assert first.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0
|
||||
|
||||
second = MODULE.HealthReport()
|
||||
MODULE.check_file_operations_runtime(second, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert second.results[0].status == "ok"
|
||||
assert second.results[0].details["latest_health"][0]["sendFailuresPrevious"] == 28
|
||||
assert second.results[0].details["latest_health"][0]["sendFailuresDelta"] == 0
|
||||
|
||||
third = MODULE.HealthReport()
|
||||
MODULE.check_file_operations_runtime(third, "http://127.0.0.1:5600/api/0", buckets, counter_state=state)
|
||||
assert third.results[0].status == "warn"
|
||||
assert third.results[0].details["warnings"] == [
|
||||
{
|
||||
"bucket": "aw-file-operations_SHARKON2025",
|
||||
"metric": "sendFailuresDelta",
|
||||
"value": 1,
|
||||
"current": 29,
|
||||
"previous": 28,
|
||||
"threshold": 1,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_incident_runtime_reports_counts_and_latest_real_incidents(monkeypatch):
|
||||
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
monkeypatch.setattr(
|
||||
MODULE,
|
||||
"_http_json",
|
||||
lambda url, **kwargs: [
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:30Z",
|
||||
"data": {
|
||||
"ruleId": "usb-archive-copy",
|
||||
"severity": "high",
|
||||
"action": "alert",
|
||||
"message": "Archive copied to removable device with a long message that should not leak full raw content.",
|
||||
"username": "USER1",
|
||||
"hostname": "SHARKON2025",
|
||||
"source": "endpoint",
|
||||
},
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-05-30T12:00:00Z",
|
||||
"data": {
|
||||
"ruleId": "selftest-dlp-incident",
|
||||
"severity": "low",
|
||||
"action": "alert",
|
||||
"message": "Self-test DLP incident from validation",
|
||||
"signalType": "self_test",
|
||||
"source": "self-test",
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "incident-runtime"
|
||||
assert result.status == "ok"
|
||||
assert result.summary == "1 real incidents in sampled events"
|
||||
assert result.details["totals"] == {"sampled_events": 2, "real_incidents": 1, "self_tests": 1}
|
||||
assert result.details["severity_counts"] == {"high": 1}
|
||||
assert result.details["action_counts"] == {"alert": 1}
|
||||
assert result.details["rule_counts"] == {"usb-archive-copy": 1}
|
||||
latest = result.details["latest_incidents"][0]
|
||||
assert latest["age_seconds"] == 30
|
||||
assert latest["ruleId"] == "usb-archive-copy"
|
||||
assert latest["message_excerpt"].endswith(".")
|
||||
|
||||
|
||||
def test_incident_runtime_defaults_to_metadata_without_event_sampling(monkeypatch):
|
||||
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
monkeypatch.setattr(MODULE, "_now_utc", lambda: MODULE._parse_ts("2026-05-30T12:01:00Z"))
|
||||
|
||||
def fail_if_called(url, **kwargs):
|
||||
raise AssertionError("event sampling should stay disabled")
|
||||
|
||||
monkeypatch.setattr(MODULE, "_http_json", fail_if_called)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=0)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "incident-runtime"
|
||||
assert result.status == "ok"
|
||||
assert result.summary == "incident event sampling disabled"
|
||||
assert result.details["metadata"] == [
|
||||
{
|
||||
"bucket": "aw-dlp-incidents_SHARKON2025",
|
||||
"end": "2026-05-30T12:00:00Z",
|
||||
"age_seconds": 60,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_incident_runtime_warns_on_read_failure(monkeypatch):
|
||||
buckets = {"aw-dlp-incidents_SHARKON2025": {"metadata": {"end": "2026-05-30T12:00:00Z"}}}
|
||||
|
||||
def fail_http(url, **kwargs):
|
||||
raise RuntimeError("timeout")
|
||||
|
||||
monkeypatch.setattr(MODULE, "_http_json", fail_http)
|
||||
report = MODULE.HealthReport()
|
||||
|
||||
MODULE.check_incident_runtime(report, "http://127.0.0.1:5600/api/0", buckets, sample_limit=20)
|
||||
|
||||
result = report.results[0]
|
||||
assert result.name == "incident-runtime"
|
||||
assert result.status == "warn"
|
||||
assert result.summary == "1 incident buckets failed to sample"
|
||||
assert result.details["read_failed"] == [{"bucket": "aw-dlp-incidents_SHARKON2025", "error": "timeout"}]
|
||||
@@ -32,75 +32,5 @@ for candidate in "${rust_candidates[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
required_files=(
|
||||
"$MANIFEST"
|
||||
"$KIT_DIR/README-INSTALL-KIT.txt"
|
||||
"$KIT_DIR/windows/deploy-ensemble.ps1"
|
||||
"$KIT_DIR/windows/validate-deployment.ps1"
|
||||
"$KIT_DIR/ansible/deploy_aw_windows.yml"
|
||||
"$KIT_DIR/aw-server/install_aw_server.sh"
|
||||
"$KIT_DIR/scripts/rebuild_install_kit.sh"
|
||||
"$KIT_DIR/scripts/validate_install_kit.sh"
|
||||
"$KIT_DIR/scripts/check_install_kit_vs_repo.sh"
|
||||
"$KIT_DIR/scripts/quality-gate.sh"
|
||||
)
|
||||
|
||||
echo "[1/4] Required files presence"
|
||||
for file in "${required_files[@]}"; do
|
||||
[[ -f "$file" ]] || { echo "Missing required file: $file"; exit 1; }
|
||||
done
|
||||
|
||||
echo "[2/4] Manifest checksum verification"
|
||||
sha256sum -c "$MANIFEST" >/dev/null
|
||||
|
||||
echo "[3/4] Manifest completeness"
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
kit=Path('install-kit-awindows-20260427-211240')
|
||||
manifest=kit/'MANIFEST.txt'
|
||||
listed=[]
|
||||
for line in manifest.read_text().splitlines():
|
||||
line=line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts=line.split(' ',1)
|
||||
if len(parts)!=2:
|
||||
print(f'Invalid MANIFEST line: {line}')
|
||||
sys.exit(1)
|
||||
listed.append(parts[1])
|
||||
listed_set=set(listed)
|
||||
actual_set={str(p) for p in kit.rglob('*') if p.is_file() and p.name!='MANIFEST.txt'}
|
||||
missing=sorted(listed_set-actual_set)
|
||||
extra=sorted(actual_set-listed_set)
|
||||
if missing or extra:
|
||||
print('Missing files listed in MANIFEST:', missing)
|
||||
print('Files not listed in MANIFEST:', extra)
|
||||
sys.exit(1)
|
||||
print(f'MANIFEST complete: {len(actual_set)} files tracked')
|
||||
PY
|
||||
|
||||
echo "[4/4] Archive composition check"
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import tarfile, zipfile, sys
|
||||
kit_prefix='install-kit-awindows-20260427-211240/'
|
||||
zip_path=Path('install-kit-awindows-20260427-211240.zip')
|
||||
tar_path=Path('install-kit-awindows-20260427-211240.tar.gz')
|
||||
if not zip_path.exists() or not tar_path.exists():
|
||||
print('Archives not found')
|
||||
sys.exit(1)
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
zip_files=sorted(i for i in z.namelist() if not i.endswith('/'))
|
||||
with tarfile.open(tar_path, 'r:gz') as t:
|
||||
tar_files=sorted(m.name for m in t.getmembers() if m.isfile())
|
||||
if zip_files != tar_files:
|
||||
print('ZIP and TAR contents differ')
|
||||
sys.exit(1)
|
||||
if not all(f.startswith(kit_prefix) for f in zip_files):
|
||||
print('Unexpected archive prefix layout')
|
||||
sys.exit(1)
|
||||
print(f'Archives match: {len(zip_files)} files')
|
||||
PY
|
||||
|
||||
echo "validate_install_kit: OK"
|
||||
echo "ERROR: Rust validator not found. Build it with: cd '$ROOT_DIR/adk-rust' && cargo build --release -p validate-install-kit" >&2
|
||||
exit 127
|
||||
|
||||
Reference in New Issue
Block a user