Feat & Fix: implement File Telemetry, restore DB history, and stabilize production
- Added File Operations Collector (Plan A) for Windows endpoints - Restored historical server DB via merging and moved to durable /var/lib/activitywatch path - Forced XDG_DATA_HOME and XDG_CONFIG_HOME for aw-server-rust in environment and systemd - Updated Ansible playbooks to handle new file collector and durable server paths - Added DB merge and backup-restore automation scripts - Fixed CORS and RU WebUI persistence in production deployment Generated with [Devin](https://cli.devin.ai/docs) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
igor04091968
co-authored by
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
parent
f436950bda
commit
fae2e2ca14
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def connect(path: Path) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(str(path))
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("PRAGMA synchronous=NORMAL")
|
||||
return connection
|
||||
|
||||
|
||||
def bucket_key(row: sqlite3.Row) -> tuple[str, str, str, str]:
|
||||
return (
|
||||
str(row["name"]),
|
||||
str(row["type"]),
|
||||
str(row["client"]),
|
||||
str(row["hostname"]),
|
||||
)
|
||||
|
||||
|
||||
def ensure_parent(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def load_existing_events(connection: sqlite3.Connection, bucketrow: int) -> set[tuple[int, int, str]]:
|
||||
cursor = connection.execute(
|
||||
"select starttime, endtime, data from events where bucketrow = ?",
|
||||
(bucketrow,),
|
||||
)
|
||||
return {(int(start), int(end), str(data)) for start, end, data in cursor.fetchall()}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base", required=True)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--overlay")
|
||||
args = parser.parse_args()
|
||||
|
||||
base = Path(args.base)
|
||||
output = Path(args.output)
|
||||
overlay = Path(args.overlay) if args.overlay else None
|
||||
|
||||
if not base.exists():
|
||||
raise SystemExit(f"Base DB not found: {base}")
|
||||
|
||||
ensure_parent(output)
|
||||
tmp_output = output.with_suffix(output.suffix + ".tmp")
|
||||
if tmp_output.exists():
|
||||
tmp_output.unlink()
|
||||
shutil.copy2(base, tmp_output)
|
||||
|
||||
dest = connect(tmp_output)
|
||||
dest.row_factory = sqlite3.Row
|
||||
|
||||
inserted_buckets = 0
|
||||
inserted_events = 0
|
||||
|
||||
if overlay and overlay.exists():
|
||||
source = connect(overlay)
|
||||
source.row_factory = sqlite3.Row
|
||||
try:
|
||||
source_buckets = source.execute(
|
||||
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
|
||||
).fetchall()
|
||||
|
||||
dest_bucket_map = {
|
||||
bucket_key(row): row["bucketrow"]
|
||||
for row in dest.execute(
|
||||
"select rowid as bucketrow, id, name, type, client, hostname, created, data_deprecated, data from buckets order by rowid"
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
for src_bucket in source_buckets:
|
||||
key = bucket_key(src_bucket)
|
||||
dest_rowid = dest_bucket_map.get(key)
|
||||
if dest_rowid is None:
|
||||
cursor = dest.execute(
|
||||
"""
|
||||
insert into buckets (name, type, client, hostname, created, data_deprecated, data)
|
||||
values (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
src_bucket["name"],
|
||||
src_bucket["type"],
|
||||
src_bucket["client"],
|
||||
src_bucket["hostname"],
|
||||
src_bucket["created"],
|
||||
src_bucket["data_deprecated"],
|
||||
src_bucket["data"],
|
||||
),
|
||||
)
|
||||
dest_rowid = int(cursor.lastrowid)
|
||||
dest_bucket_map[key] = dest_rowid
|
||||
inserted_buckets += 1
|
||||
|
||||
existing_events = load_existing_events(dest, dest_rowid)
|
||||
for starttime, endtime, data in source.execute(
|
||||
"select starttime, endtime, data from events where bucketrow = ? order by id",
|
||||
(src_bucket["bucketrow"],),
|
||||
).fetchall():
|
||||
event_key = (int(starttime), int(endtime), str(data))
|
||||
if event_key in existing_events:
|
||||
continue
|
||||
dest.execute(
|
||||
"insert into events (bucketrow, starttime, endtime, data) values (?, ?, ?, ?)",
|
||||
(dest_rowid, int(starttime), int(endtime), str(data)),
|
||||
)
|
||||
existing_events.add(event_key)
|
||||
inserted_events += 1
|
||||
|
||||
dest.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
dest.close()
|
||||
os.replace(tmp_output, output)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"base": str(base),
|
||||
"overlay": str(overlay) if overlay else None,
|
||||
"output": str(output),
|
||||
"inserted_buckets": inserted_buckets,
|
||||
"inserted_events": inserted_events,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
if [[ -f "${ROOT_DIR}/secrets/runtime.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source "${ROOT_DIR}/secrets/runtime.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
: "${AW_SSH_PASSWORD:?AW_SSH_PASSWORD is required}"
|
||||
: "${AW_WINRM_PASSWORD:?AW_WINRM_PASSWORD is required}"
|
||||
|
||||
command -v sshpass >/dev/null 2>&1 || { echo "missing sshpass" >&2; exit 127; }
|
||||
command -v ansible-playbook >/dev/null 2>&1 || { echo "missing ansible-playbook" >&2; exit 127; }
|
||||
|
||||
SERVER_HOST="${AW_SERVER_HOST:-10.10.10.13}"
|
||||
SERVER_USER="${AW_SERVER_USER:-igor}"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
REMOTE_BACKUP_DIR="/var/lib/activitywatch/backups/prod-restore-${TIMESTAMP}"
|
||||
LEGACY_DB="/root/.local/share/activitywatch/aw-server-rust/sqlite.db"
|
||||
TARGET_DB="/var/lib/activitywatch/.local/share/activitywatch/aw-server-rust/sqlite.db"
|
||||
REMOTE_MERGE_SCRIPT="/tmp/merge_aw_server_dbs.py"
|
||||
|
||||
ssh_remote() {
|
||||
sshpass -p "$AW_SSH_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${SERVER_USER}@${SERVER_HOST}" "$@"
|
||||
}
|
||||
|
||||
scp_remote() {
|
||||
sshpass -p "$AW_SSH_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
|
||||
}
|
||||
|
||||
scp_remote "${ROOT_DIR}/scripts/merge_aw_server_dbs.py" "${SERVER_USER}@${SERVER_HOST}:${REMOTE_MERGE_SCRIPT}"
|
||||
|
||||
ssh_remote "sudo mkdir -p '${REMOTE_BACKUP_DIR}' && sudo chown root:root '${REMOTE_BACKUP_DIR}'"
|
||||
ssh_remote "sudo test -f '${LEGACY_DB}'"
|
||||
ssh_remote "sudo test -f '${TARGET_DB}'"
|
||||
ssh_remote "sudo cp -a '${LEGACY_DB}' '${REMOTE_BACKUP_DIR}/legacy-root-sqlite.db' && sudo cp -a '${TARGET_DB}' '${REMOTE_BACKUP_DIR}/target-before-merge-sqlite.db'"
|
||||
ssh_remote "sudo systemctl stop activitywatch-server.service || true"
|
||||
ssh_remote "sudo python3 '${REMOTE_MERGE_SCRIPT}' --base '${LEGACY_DB}' --overlay '${TARGET_DB}' --output '${REMOTE_BACKUP_DIR}/sqlite.merged.db'"
|
||||
ssh_remote "sudo install -o activitywatch -g activitywatch -m 0644 '${REMOTE_BACKUP_DIR}/sqlite.merged.db' '${TARGET_DB}'"
|
||||
|
||||
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_server.yml
|
||||
ansible-playbook -i ansible/inventory.ini ansible/deploy_aw_windows.yml
|
||||
ansible-playbook -i ansible/inventory.ini ansible/post_validate_aw_windows.yml
|
||||
|
||||
python3 - <<'PY'
|
||||
import json, urllib.request
|
||||
base = 'http://10.10.10.13:5600'
|
||||
window_payload = {
|
||||
'timeperiods': ['2026-04-29T00:00:00+03:00/2026-04-29T23:59:59+03:00'],
|
||||
'query': [
|
||||
'window_events = query_bucket(find_bucket("aw-watcher-window_SHARKON2025"));',
|
||||
'RETURN = window_events;'
|
||||
]
|
||||
}
|
||||
req = urllib.request.Request(base + '/api/0/query/', data=json.dumps(window_payload).encode(), method='POST', headers={'Content-Type': 'application/json', 'Origin': 'http://10.10.10.13:5600'})
|
||||
with urllib.request.urlopen(req) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
window_count = len(data[0]) if isinstance(data, list) and data else 0
|
||||
if window_count <= 0:
|
||||
raise SystemExit('no historical window data restored for 2026-04-29')
|
||||
with urllib.request.urlopen(base + '/api/0/settings/') as response:
|
||||
settings = json.loads(response.read().decode())
|
||||
if settings.get('always_active_pattern') != 'aw-watcher-window':
|
||||
raise SystemExit('always_active_pattern is not configured')
|
||||
print(json.dumps({'restored_window_events_2026_04_29': window_count, 'always_active_pattern': settings.get('always_active_pattern')}, ensure_ascii=False))
|
||||
PY
|
||||
Reference in New Issue
Block a user