feat(detmir): ship ops, dlp, 1c and mcp updates

clickhouse-1c/etl/test_extract_1c_mcp_toolkit.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import sys
+import tempfile
+import unittest
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+sys.modules.setdefault("clickhouse_connect", SimpleNamespace(get_client=lambda **_: None))
+
+import extract_1c_mcp_toolkit as extractor
+from load_1c_exports import Config as LoaderConfig
+
+
+class Extract1CMcpToolkitTests(unittest.TestCase):
+    def test_validate_runtime_config_flags_invalid_enabled_query_and_limit(self) -> None:
+        loader_conf = LoaderConfig(
+            clickhouse={},
+            landing={
+                "documents": "/tmp/documents",
+                "postings": "/tmp/postings",
+                "business_events": "/tmp/business_events",
+                "document_changes": "/tmp/document_changes",
+                "companies": "/tmp/companies",
+                "reglog": "/tmp/reglog",
+                "audit": "/tmp/audit",
+                "host": "/tmp/host",
+            },
+            formats={"default": "jsonl"},
+            archive_dir=None,
+            delete_after_load=False,
+            min_file_age_seconds=180,
+        )
+        toolkit_conf = extractor.ToolkitConfig(
+            base_url="http://127.0.0.1:6003",
+            channel="prod",
+            timeout_seconds=30,
+            verify_tls=True,
+            state_dir="./state",
+            datasets={
+                "documents": extractor.QueryDatasetConfig(enabled=True, query="", limit=1001),
+                "postings": extractor.QueryDatasetConfig(enabled=False, query="SELECT"),
+                "business_events": extractor.QueryDatasetConfig(enabled=False, query="SELECT"),
+                "document_changes": extractor.QueryDatasetConfig(enabled=False, query="SELECT"),
+                "companies": extractor.QueryDatasetConfig(enabled=False, query="SELECT"),
+            },
+            event_log=extractor.EventLogConfig(enabled=False, limit=500),
+        )
+        issues = extractor.validate_runtime_config(loader_conf, toolkit_conf)
+        self.assertIn("mcp_toolkit.datasets.documents.query is required when dataset is enabled", issues)
+        self.assertIn("mcp_toolkit.datasets.documents.limit should not exceed 1000 for 1c-mcp-toolkit execute_query", issues)
+
+    def test_normalize_query_row_derives_entity_key_and_evidence(self) -> None:
+        spec = extractor.QueryDatasetConfig(
+            enabled=True,
+            query="SELECT",
+            field_map={"Номер": "document_number", "База": "infobase", "Путь": "base_path"},
+            static_fields={"organization": "ООО Тест"},
+        )
+        row = {
+            "Номер": "0001",
+            "База": "ФЕЛИЦТ ГРУПП 2026",
+            "Путь": r"c:\1c\felict",
+        }
+        normalized = extractor.normalize_query_row("document_changes", row, spec)
+        self.assertEqual(normalized["document_number"], "0001")
+        self.assertEqual(normalized["organization"], "ООО Тест")
+        self.assertTrue(normalized["company_entity_key"].startswith("basepath:"))
+        self.assertEqual(normalized["evidence_ref"], "mcp:document_change:0001")
+
+    def test_resolve_incremental_bounds_applies_lookback(self) -> None:
+        spec = extractor.QueryDatasetConfig(
+            enabled=True,
+            query="SELECT",
+            incremental=extractor.IncrementalConfig(
+                since_param="since_ts",
+                until_param="until_ts",
+                initial_since="2026-05-01T00:00:00Z",
+                lookback_seconds=120,
+            ),
+        )
+        now = datetime(2026, 5, 23, 10, 30, tzinfo=UTC)
+        params, completion_cursor = extractor.resolve_incremental_bounds(
+            spec,
+            {"last_success_ts": "2026-05-23T10:00:00Z"},
+            now,
+        )
+        self.assertEqual(params["since_ts"], "2026-05-23T09:58:00Z")
+        self.assertEqual(params["until_ts"], "2026-05-23T10:30:00Z")
+        self.assertEqual(completion_cursor, "2026-05-23T10:30:00Z")
+
+    def test_normalize_reglog_row_renders_message(self) -> None:
+        spec = extractor.EventLogConfig(
+            enabled=True,
+            static_fields={"infobase": "ИБ-1"},
+        )
+        row = {
+            "date": "2026-05-23T11:00:00",
+            "user": "Иванов И.И.",
+            "computer": "WS-01",
+            "application": "ThinClient",
+            "event": "_$Data$_.Post",
+            "level": "Error",
+            "comment": "Ошибка проведения",
+            "metadata": "Документ.РеализацияТоваровУслуг",
+            "transaction_status": "RolledBack",
+            "session": 42,
+        }
+        normalized = extractor.normalize_reglog_row(row, spec)
+        self.assertEqual(normalized["infobase"], "ИБ-1")
+        self.assertEqual(normalized["host"], "WS-01")
+        self.assertEqual(normalized["app"], "ThinClient")
+        self.assertEqual(normalized["event_name"], "_$Data$_.Post")
+        self.assertIn("metadata=Документ.РеализацияТоваровУслуг", normalized["message"])
+        self.assertIn("txn=RolledBack", normalized["message"])
+
+    def test_extract_reglog_dataset_updates_cursor_and_writes_jsonl(self) -> None:
+        class FakeClient:
+            def __init__(self) -> None:
+                self.calls: list[dict[str, object]] = []
+                self.responses = [
+                    {
+                        "success": True,
+                        "data": [
+                            {
+                                "date": "2026-05-23T11:00:00",
+                                "user": "USER1",
+                                "computer": "WS-01",
+                                "application": "ThinClient",
+                                "event": "_$Data$_.Post",
+                                "level": "Error",
+                                "comment": "Ошибка 1",
+                            }
+                        ],
+                        "last_date": "2026-05-23T11:00:00",
+                        "next_same_second_offset": 1,
+                        "has_more": True,
+                    },
+                    {
+                        "success": True,
+                        "data": [
+                            {
+                                "date": "2026-05-23T11:00:01",
+                                "user": "USER2",
+                                "computer": "WS-02",
+                                "application": "ThinClient",
+                                "event": "_$Data$_.Update",
+                                "level": "Warning",
+                                "comment": "Изменение",
+                            }
+                        ],
+                        "last_date": "2026-05-23T11:00:01",
+                        "next_same_second_offset": 0,
+                        "has_more": False,
+                    },
+                ]
+
+            def post_json(self, endpoint: str, payload: dict[str, object]) -> dict[str, object]:
+                self.calls.append({"endpoint": endpoint, "payload": payload})
+                return self.responses.pop(0)
+
+        now = datetime(2026, 5, 23, 12, 0, tzinfo=UTC)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            loader_conf = LoaderConfig(
+                clickhouse={},
+                landing={
+                    "documents": f"{tmpdir}/documents",
+                    "postings": f"{tmpdir}/postings",
+                    "business_events": f"{tmpdir}/business_events",
+                    "document_changes": f"{tmpdir}/document_changes",
+                    "companies": f"{tmpdir}/companies",
+                    "reglog": f"{tmpdir}/reglog",
+                    "audit": f"{tmpdir}/audit",
+                    "host": f"{tmpdir}/host",
+                },
+                formats={"default": "jsonl"},
+                archive_dir=None,
+                delete_after_load=False,
+                min_file_age_seconds=180,
+            )
+            toolkit_conf = extractor.ToolkitConfig(
+                base_url="http://127.0.0.1:6003",
+                channel="prod",
+                timeout_seconds=30,
+                verify_tls=True,
+                state_dir=f"{tmpdir}/state",
+                datasets={dataset: extractor.QueryDatasetConfig(enabled=False, query="") for dataset in extractor.QUERY_DATASETS},
+                event_log=extractor.EventLogConfig(enabled=True, initial_start_date="2026-05-23T10:00:00Z", static_fields={"infobase": "ИБ-1"}),
+            )
+            state = {"datasets": {}, "reglog": {}}
+            client = FakeClient()
+            options = extractor.RunOptions(dry_run=False, sample_size=3, max_pages=1, validate_config=False)
+
+            rows = extractor.extract_reglog_dataset(client, loader_conf, toolkit_conf, state, now, options)
+
+            self.assertEqual(rows, 2)
+            self.assertEqual(client.calls[0]["endpoint"], "get_event_log")
+            self.assertEqual(client.calls[1]["payload"]["start_date"], "2026-05-23T11:00:00")
+            self.assertEqual(client.calls[1]["payload"]["same_second_offset"], 1)
+            self.assertEqual(state["reglog"]["last_date"], "2026-05-23T11:00:01")
+            self.assertEqual(state["reglog"]["same_second_offset"], 0)
+            reglog_files = list(Path(loader_conf.landing["reglog"]).glob("*.jsonl"))
+            self.assertEqual(len(reglog_files), 1)
+            payload = reglog_files[0].read_text(encoding="utf-8")
+            self.assertIn('"infobase": "ИБ-1"', payload)
+            self.assertIn('"event_name": "_$Data$_.Update"', payload)
+
+    def test_extract_query_dataset_dry_run_does_not_write_or_update_state(self) -> None:
+        class FakeClient:
+            def __init__(self) -> None:
+                self.calls: list[dict[str, object]] = []
+
+            def post_json(self, endpoint: str, payload: dict[str, object]) -> dict[str, object]:
+                self.calls.append({"endpoint": endpoint, "payload": payload})
+                return {
+                    "success": True,
+                    "data": [
+                        {
+                            "infobase": "ИБ-1",
+                            "doc_number": "0001",
+                            "doc_id": "DOC-1",
+                            "amount": 10,
+                        }
+                    ],
+                    "schema": {"columns": [{"name": "doc_number", "types": ["Строка"]}]},
+                }
+
+        now = datetime(2026, 5, 23, 12, 0, tzinfo=UTC)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            loader_conf = LoaderConfig(
+                clickhouse={},
+                landing={
+                    "documents": f"{tmpdir}/documents",
+                    "postings": f"{tmpdir}/postings",
+                    "business_events": f"{tmpdir}/business_events",
+                    "document_changes": f"{tmpdir}/document_changes",
+                    "companies": f"{tmpdir}/companies",
+                    "reglog": f"{tmpdir}/reglog",
+                    "audit": f"{tmpdir}/audit",
+                    "host": f"{tmpdir}/host",
+                },
+                formats={"default": "jsonl"},
+                archive_dir=None,
+                delete_after_load=False,
+                min_file_age_seconds=180,
+            )
+            spec = extractor.QueryDatasetConfig(
+                enabled=True,
+                query="SELECT *",
+                limit=500,
+                incremental=extractor.IncrementalConfig(since_param="since_ts", until_param="until_ts", initial_since="2026-05-01T00:00:00Z"),
+            )
+            state = {"datasets": {"documents": {"last_success_ts": "2026-05-23T10:00:00Z"}}, "reglog": {}}
+            client = FakeClient()
+            options = extractor.RunOptions(dry_run=True, sample_size=2, max_pages=1, validate_config=False)
+
+            rows = extractor.extract_query_dataset("documents", spec, client, loader_conf, state, now, options)
+
+            self.assertEqual(rows, 1)
+            self.assertEqual(client.calls[0]["endpoint"], "execute_query")
+            self.assertEqual(client.calls[0]["payload"]["limit"], 2)
+            self.assertTrue(client.calls[0]["payload"]["include_schema"])
+            self.assertEqual(state["datasets"]["documents"]["last_success_ts"], "2026-05-23T10:00:00Z")
+            self.assertFalse(Path(loader_conf.landing["documents"]).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()