From eb1b5a04c696dcca4add486164f7e65a4007d46a Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 09:37:14 -0400 Subject: [PATCH 1/2] Fix full reset retaining default database --- src/minilake/admin.py | 11 ++++----- src/minilake/duckdb_pool.py | 20 +++++++++++++++-- tests/test_admin.py | 19 +++++++++++++++- tests/test_persistence.py | 45 +++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/src/minilake/admin.py b/src/minilake/admin.py index 94f5418..78d3fcd 100644 --- a/src/minilake/admin.py +++ b/src/minilake/admin.py @@ -62,7 +62,7 @@ async def ready() -> ReadyResponse: return ReadyResponse(ready=True, message="minilake is ready") -# Directories under data_dir that `full=True` wipes. +# Data directories under data_dir that `full=True` wipes. # # `catalogs`, `volumes`, `warehouses` and `scratch` are the ones that make `full` # mean anything: no service `reset()` touches them, so a plain reset clears the @@ -80,8 +80,8 @@ def _wipe_data_dirs() -> list[str]: """Delete the on-disk trees holding user content. Returns what was removed. Only safe once every service `reset()` has detached its catalogs and the pool - has closed its warehouse connections — deleting a database file out from under - an open DuckDB handle is what this ordering avoids. + has closed its DuckDB connections — deleting database files out from under an + open DuckDB handle is what this ordering avoids. """ removed = [] for name in _WIPED_ON_FULL_RESET: @@ -122,9 +122,10 @@ async def reset_state(full: bool = False) -> dict[str, str]: except Exception as e: logger.warning(f"Failed to reset {service_name}: {e}") - # Reset warehouse connections + # Reset warehouse connections and, for a full reset, remove the shared + # Unity Catalog database after service reset hooks have detached catalogs. if _duckdb_pool: - await _duckdb_pool.reset_all() + await _duckdb_pool.reset_all(full=full) if full: removed = _wipe_data_dirs() diff --git a/src/minilake/duckdb_pool.py b/src/minilake/duckdb_pool.py index e46ae00..d760d15 100644 --- a/src/minilake/duckdb_pool.py +++ b/src/minilake/duckdb_pool.py @@ -188,11 +188,27 @@ async def close_all(self) -> None: logger.info("Closed all DuckDB connections") - async def reset_all(self) -> None: - """Reset all warehouse connections (clear in-memory data, keep UC catalog).""" + async def reset_all(self, *, full: bool = False) -> None: + """Reset warehouse connections and optionally clear the shared UC database. + + A plain state reset preserves Unity Catalog's default database. A full reset + closes its connection and removes both the database and its WAL so the next + query starts with a fresh database. + """ async with self._warehouse_dict_lock: for conn in self._warehouse_connections.values(): conn.close() self._warehouse_connections.clear() self._warehouse_locks.clear() + + if full: + async with self._uc_lock: + if self._uc_connection is not None: + self._uc_connection.close() + self._uc_connection = None + + uc_path = self.data_dir / "unity_catalog.duckdb" + uc_path.unlink(missing_ok=True) + Path(str(uc_path) + ".wal").unlink(missing_ok=True) + logger.info("Reset all warehouse connections") diff --git a/tests/test_admin.py b/tests/test_admin.py index 28bd8e8..9b6c25a 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -1,6 +1,7 @@ """Admin endpoints tests (/health, /ready, /reset, /services).""" import urllib.request +from uuid import uuid4 import pytest from databricks.sdk import WorkspaceClient @@ -73,6 +74,14 @@ def test_reset_without_full_keeps_files_on_disk(workspace_client: WorkspaceClien without `full`, a volume's directory survives and can be listed again after the volume is recreated. """ + warehouse = workspace_client.warehouses.create(name="keep_wh") + table_name = f"survives_plain_reset_{uuid4().hex}" + created = workspace_client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"CREATE TABLE {table_name} AS SELECT 777 AS value", + ) + assert created.status.state.value == "SUCCEEDED" + workspace_client.catalogs.create(name="keep_cat") workspace_client.schemas.create(name="keep_schema", catalog_name="keep_cat") workspace_client.volumes.create( @@ -97,7 +106,15 @@ def test_reset_without_full_keeps_files_on_disk(workspace_client: WorkspaceClien volume_type=catalog.VolumeType.MANAGED, ) - print("✓ Plain reset left the volume directory on disk") + warehouse = workspace_client.warehouses.create(name="keep_wh_after_reset") + result = workspace_client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"SELECT value FROM {table_name}", + ) + assert result.status.state.value == "SUCCEEDED" + assert result.result.data_array == [[777]] + + print("✓ Plain reset left volume files and the shared SQL database intact") @pytest.mark.crud diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 31a40ed..5c82211 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -17,9 +17,11 @@ import tempfile import time import urllib.request +import uuid import pytest from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError from databricks.sdk.service.catalog import DataSourceFormat, TableType from databricks.sdk.service.sql import ColumnInfo @@ -128,3 +130,46 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_full_reset_removes_default_database_across_restart(): + """A full reset must remove unqualified SQL tables from the shared database, + including after a process restart against the same data directory.""" + data_dir = tempfile.mkdtemp(prefix="minilake_full_reset_") + port = _free_port() + proc = None + table_name = f"reset_survivor_{uuid.uuid4().hex}" + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + warehouse = client.warehouses.create(name="full_reset_before") + created = client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"CREATE TABLE {table_name} AS SELECT 777 AS value", + ) + assert created.status.state.value == "SUCCEEDED" + + request = urllib.request.Request(f"http://127.0.0.1:{port}/_minilake/reset?full=true", method="POST") + assert urllib.request.urlopen(request, timeout=10).status == 200 + + def assert_table_is_gone(client: WorkspaceClient, warehouse_name: str) -> None: + warehouse = client.warehouses.create(name=warehouse_name) + with pytest.raises(DatabricksError) as exc: + client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"SELECT value FROM {table_name}", + ) + assert "does not exist" in str(exc.value).lower() or "not found" in str(exc.value).lower() + + assert_table_is_gone(client, "full_reset_immediate") + + _stop_minilake(proc) + proc = None + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + assert_table_is_gone(client, "full_reset_after_restart") + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) From 3a5863d61a2eb4ac1e242c6798453540f5c28dae Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 16:01:25 -0400 Subject: [PATCH 2/2] refactor: keep this fix independently mergeable with other open PRs --- tests/persistence_helpers.py | 71 ++++++++++++++++++++++++++++ tests/test_full_reset_persistence.py | 64 +++++++++++++++++++++++++ tests/test_persistence.py | 45 ------------------ 3 files changed, 135 insertions(+), 45 deletions(-) create mode 100644 tests/persistence_helpers.py create mode 100644 tests/test_full_reset_persistence.py diff --git a/tests/persistence_helpers.py b/tests/persistence_helpers.py new file mode 100644 index 0000000..da13d01 --- /dev/null +++ b/tests/persistence_helpers.py @@ -0,0 +1,71 @@ +"""Process helpers for isolated persistence regressions.""" + +import os +import socket +import subprocess +import time +import urllib.request + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _drain(proc: subprocess.Popen) -> str: + """Whatever the subprocess printed, without blocking on a live process.""" + if proc.poll() is None: + return "(process still running)" + out, err = proc.communicate(timeout=5) + return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace") + + +def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None: + """Poll until minilake answers, failing early and loudly if it died instead. + + Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a + slow one, and the test reports a timeout while the actual traceback sits unread in a + pipe. CI is also several times slower to start than a laptop, hence the generous + timeout — this waits on a real process spawn, not a request. + """ + deadline = time.time() + timeout + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}") + try: + if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200: + return + except Exception as e: + last_error = e + time.sleep(0.3) + raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") + + +def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen: + env = os.environ.copy() + env["MINILAKE_DATA_DIR"] = data_dir + env["MINILAKE_PERSIST"] = "1" + proc = subprocess.Popen( + ["minilake", "--port", str(port), "--host", "127.0.0.1"], + env=env, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + _wait_healthy(proc, f"http://127.0.0.1:{port}") + except Exception: + _stop_minilake(proc) + raise + return proc + + +def _stop_minilake(proc: subprocess.Popen) -> None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/tests/test_full_reset_persistence.py b/tests/test_full_reset_persistence.py new file mode 100644 index 0000000..3b60ef4 --- /dev/null +++ b/tests/test_full_reset_persistence.py @@ -0,0 +1,64 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import shutil +import tempfile +import urllib.request +import uuid + +import pytest +from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + + +@pytest.mark.serial +def test_full_reset_removes_default_database_across_restart(): + """A full reset must remove unqualified SQL tables from the shared database, + including after a process restart against the same data directory.""" + data_dir = tempfile.mkdtemp(prefix="minilake_full_reset_") + port = _free_port() + proc = None + table_name = f"reset_survivor_{uuid.uuid4().hex}" + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + warehouse = client.warehouses.create(name="full_reset_before") + created = client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"CREATE TABLE {table_name} AS SELECT 777 AS value", + ) + assert created.status.state.value == "SUCCEEDED" + + request = urllib.request.Request(f"http://127.0.0.1:{port}/_minilake/reset?full=true", method="POST") + assert urllib.request.urlopen(request, timeout=10).status == 200 + + def assert_table_is_gone(client: WorkspaceClient, warehouse_name: str) -> None: + warehouse = client.warehouses.create(name=warehouse_name) + with pytest.raises(DatabricksError) as exc: + client.statement_execution.execute_statement( + warehouse_id=warehouse.id, + statement=f"SELECT value FROM {table_name}", + ) + assert "does not exist" in str(exc.value).lower() or "not found" in str(exc.value).lower() + + assert_table_is_gone(client, "full_reset_immediate") + + _stop_minilake(proc) + proc = None + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + assert_table_is_gone(client, "full_reset_after_restart") + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 5c82211..31a40ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -17,11 +17,9 @@ import tempfile import time import urllib.request -import uuid import pytest from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import DatabricksError from databricks.sdk.service.catalog import DataSourceFormat, TableType from databricks.sdk.service.sql import ColumnInfo @@ -130,46 +128,3 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) - - -@pytest.mark.serial -def test_full_reset_removes_default_database_across_restart(): - """A full reset must remove unqualified SQL tables from the shared database, - including after a process restart against the same data directory.""" - data_dir = tempfile.mkdtemp(prefix="minilake_full_reset_") - port = _free_port() - proc = None - table_name = f"reset_survivor_{uuid.uuid4().hex}" - try: - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - warehouse = client.warehouses.create(name="full_reset_before") - created = client.statement_execution.execute_statement( - warehouse_id=warehouse.id, - statement=f"CREATE TABLE {table_name} AS SELECT 777 AS value", - ) - assert created.status.state.value == "SUCCEEDED" - - request = urllib.request.Request(f"http://127.0.0.1:{port}/_minilake/reset?full=true", method="POST") - assert urllib.request.urlopen(request, timeout=10).status == 200 - - def assert_table_is_gone(client: WorkspaceClient, warehouse_name: str) -> None: - warehouse = client.warehouses.create(name=warehouse_name) - with pytest.raises(DatabricksError) as exc: - client.statement_execution.execute_statement( - warehouse_id=warehouse.id, - statement=f"SELECT value FROM {table_name}", - ) - assert "does not exist" in str(exc.value).lower() or "not found" in str(exc.value).lower() - - assert_table_is_gone(client, "full_reset_immediate") - - _stop_minilake(proc) - proc = None - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - assert_table_is_gone(client, "full_reset_after_restart") - finally: - if proc is not None: - _stop_minilake(proc) - shutil.rmtree(data_dir, ignore_errors=True)