From 40eaf65a71a51033d28037040e55fc03b3088c4d Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Sun, 25 Oct 2026 12:17:02 -0400 Subject: [PATCH 1/3] fix(persistence): preserve valid snapshots on save failure --- src/minilake/persistence.py | 33 ++++++++++++++++++++++++++++----- src/minilake/services/dbfs.py | 5 ++++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/minilake/persistence.py b/src/minilake/persistence.py index 9b829bc..ab8e670 100644 --- a/src/minilake/persistence.py +++ b/src/minilake/persistence.py @@ -7,7 +7,9 @@ import json import logging +import os from pathlib import Path +import tempfile from typing import Any, Callable logger = logging.getLogger(__name__) @@ -55,14 +57,35 @@ async def save_state( logger.warning(f"Failed to get state from {name}: {e}") envelope = StateEnvelope(modules_state) - snapshot_path.parent.mkdir(parents=True, exist_ok=True) - + temporary_path = None try: - with open(snapshot_path, "w") as f: - json.dump(envelope.to_dict(), f, indent=2) + # Serialize before touching the current snapshot. A non-serializable + # module state must not truncate the last good snapshot. + snapshot_json = json.dumps(envelope.to_dict(), indent=2) + snapshot_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=snapshot_path.parent, + prefix=f".{snapshot_path.name}.", + suffix=".tmp", + delete=False, + ) as f: + temporary_path = Path(f.name) + f.write(snapshot_json) + f.flush() + os.fsync(f.fileno()) + os.replace(temporary_path, snapshot_path) + temporary_path = None logger.info(f"Saved state snapshot to {snapshot_path}") except Exception as e: - logger.error(f"Failed to save state snapshot: {e}") + logger.error(f"Failed to save state snapshot to {snapshot_path}: {e}") + finally: + if temporary_path is not None: + try: + temporary_path.unlink(missing_ok=True) + except OSError as e: + logger.warning(f"Failed to remove temporary state snapshot {temporary_path}: {e}") async def load_state( diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index ff9443d..ceb1433 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -226,7 +226,10 @@ async def move_path(req: MoveDbfsRequest) -> dict: def get_state() -> Dict[str, Any]: - return _state.copy() + # Upload handles are transient and their bytearray buffers cannot be + # serialized to the JSON persistence snapshot. Only persist the handle + # counter; unfinished uploads are intentionally discarded on restart. + return {"handles": {}, "next_handle": _state["next_handle"]} def restore_state(data: Dict[str, Any]) -> None: From 852b96829e126d3c9af19a2b422c87d4c54507c9 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Sun, 25 Oct 2026 12:24:15 -0400 Subject: [PATCH 2/3] test(persistence): cover unfinished DBFS snapshot saves --- src/minilake/persistence.py | 2 +- tests/test_persistence.py | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/minilake/persistence.py b/src/minilake/persistence.py index ab8e670..941d72d 100644 --- a/src/minilake/persistence.py +++ b/src/minilake/persistence.py @@ -8,8 +8,8 @@ import json import logging import os -from pathlib import Path import tempfile +from pathlib import Path from typing import Any, Callable logger = logging.getLogger(__name__) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 31a40ed..f28daa2 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,6 +10,9 @@ state survived — using the real databricks-sdk client throughout. """ +import asyncio +import base64 +import json import os import shutil import socket @@ -23,6 +26,9 @@ from databricks.sdk.service.catalog import DataSourceFormat, TableType from databricks.sdk.service.sql import ColumnInfo +from minilake import persistence +from minilake.persistence import save_state + def _free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -87,6 +93,32 @@ def _stop_minilake(proc: subprocess.Popen) -> None: proc.wait() +def test_save_state_serialization_failure_preserves_previous_snapshot(tmp_path): + snapshot_path = tmp_path / "snapshot.json" + previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} + snapshot_path.write_text(json.dumps(previous_snapshot)) + + asyncio.run(save_state(snapshot_path, {"dbfs": lambda: {"buffer": bytearray(b"pending")}})) + + assert json.loads(snapshot_path.read_text()) == previous_snapshot + assert not list(tmp_path.glob(".snapshot.json.*.tmp")) + + +def test_save_state_replace_failure_preserves_previous_snapshot(tmp_path, monkeypatch): + snapshot_path = tmp_path / "snapshot.json" + previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} + snapshot_path.write_text(json.dumps(previous_snapshot)) + + def fail_replace(source, destination): + raise OSError("simulated replace failure") + + monkeypatch.setattr(persistence.os, "replace", fail_replace) + asyncio.run(save_state(snapshot_path, {"catalogs": lambda: {"current": True}})) + + assert json.loads(snapshot_path.read_text()) == previous_snapshot + assert not list(tmp_path.glob(".snapshot.json.*.tmp")) + + @pytest.mark.serial def test_persist_state_survives_restart(): """Test: MINILAKE_PERSIST=1 really saves state on shutdown and restores @@ -109,6 +141,27 @@ def test_persist_state_survives_restart(): restored = client.catalogs.get(name=catalog_name) assert restored.name == catalog_name + # An open upload is transient. Shutting down with a bytearray buffer + # must not make the snapshot unreadable or discard other saved state. + empty_handle = client.dbfs.create("/unfinished_empty.txt", overwrite=True).handle + upload_handle = client.dbfs.create("/unfinished.txt", overwrite=True).handle + client.dbfs.add_block(upload_handle, base64.b64encode(b"pending").decode("ascii")) + _stop_minilake(proc) + proc = None + + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + restored = client.catalogs.get(name=catalog_name) + assert restored.name == catalog_name + for path, handle in ( + ("/unfinished_empty.txt", empty_handle), + ("/unfinished.txt", upload_handle), + ): + with pytest.raises(Exception): + client.dbfs.get_status(path) + with pytest.raises(Exception): + client.dbfs.add_block(handle, base64.b64encode(b"stale").decode("ascii")) + # And it must be a REAL, still-queryable catalog after restart, not # just restored metadata - proves the catalog's DuckDB database was # really re-ATTACHed, not just its name remembered. From 2b57f05def856194bbe399a8353ca51d0ce2a9fa Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 16:01:24 -0400 Subject: [PATCH 3/3] refactor: keep this fix independently mergeable with other open PRs --- tests/persistence_helpers.py | 71 ++++++++++++++++ tests/test_dbfs_upload_persistence.py | 116 ++++++++++++++++++++++++++ tests/test_persistence.py | 53 ------------ 3 files changed, 187 insertions(+), 53 deletions(-) create mode 100644 tests/persistence_helpers.py create mode 100644 tests/test_dbfs_upload_persistence.py diff --git a/tests/persistence_helpers.py b/tests/persistence_helpers.py new file mode 100644 index 0000000..da13d01 --- /dev/null +++ b/tests/persistence_helpers.py @@ -0,0 +1,71 @@ +"""Process helpers for isolated persistence regressions.""" + +import os +import socket +import subprocess +import time +import urllib.request + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _drain(proc: subprocess.Popen) -> str: + """Whatever the subprocess printed, without blocking on a live process.""" + if proc.poll() is None: + return "(process still running)" + out, err = proc.communicate(timeout=5) + return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace") + + +def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None: + """Poll until minilake answers, failing early and loudly if it died instead. + + Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a + slow one, and the test reports a timeout while the actual traceback sits unread in a + pipe. CI is also several times slower to start than a laptop, hence the generous + timeout — this waits on a real process spawn, not a request. + """ + deadline = time.time() + timeout + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}") + try: + if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200: + return + except Exception as e: + last_error = e + time.sleep(0.3) + raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") + + +def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen: + env = os.environ.copy() + env["MINILAKE_DATA_DIR"] = data_dir + env["MINILAKE_PERSIST"] = "1" + proc = subprocess.Popen( + ["minilake", "--port", str(port), "--host", "127.0.0.1"], + env=env, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + _wait_healthy(proc, f"http://127.0.0.1:{port}") + except Exception: + _stop_minilake(proc) + raise + return proc + + +def _stop_minilake(proc: subprocess.Popen) -> None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/tests/test_dbfs_upload_persistence.py b/tests/test_dbfs_upload_persistence.py new file mode 100644 index 0000000..63fa20c --- /dev/null +++ b/tests/test_dbfs_upload_persistence.py @@ -0,0 +1,116 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import asyncio +import base64 +import json +import shutil +import tempfile + +import pytest +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.catalog import DataSourceFormat, TableType +from databricks.sdk.service.sql import ColumnInfo +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + +from minilake import persistence +from minilake.persistence import save_state + + +def test_save_state_serialization_failure_preserves_previous_snapshot(tmp_path): + snapshot_path = tmp_path / "snapshot.json" + previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} + snapshot_path.write_text(json.dumps(previous_snapshot)) + + asyncio.run(save_state(snapshot_path, {"dbfs": lambda: {"buffer": bytearray(b"pending")}})) + + assert json.loads(snapshot_path.read_text()) == previous_snapshot + assert not list(tmp_path.glob(".snapshot.json.*.tmp")) + + +def test_save_state_replace_failure_preserves_previous_snapshot(tmp_path, monkeypatch): + snapshot_path = tmp_path / "snapshot.json" + previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} + snapshot_path.write_text(json.dumps(previous_snapshot)) + + def fail_replace(source, destination): + raise OSError("simulated replace failure") + + monkeypatch.setattr(persistence.os, "replace", fail_replace) + asyncio.run(save_state(snapshot_path, {"catalogs": lambda: {"current": True}})) + + assert json.loads(snapshot_path.read_text()) == previous_snapshot + assert not list(tmp_path.glob(".snapshot.json.*.tmp")) + + +@pytest.mark.serial +def test_unfinished_upload_preserves_snapshot_after_restart(): + """Test: MINILAKE_PERSIST=1 really saves state on shutdown and restores + it (including re-attaching the catalog's own DuckDB database) on the + next startup against the same data dir.""" + data_dir = tempfile.mkdtemp(prefix="minilake_persist_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = "persisted_catalog" + client.catalogs.create(name=catalog_name) + _stop_minilake(proc) + proc = None + + # Restart against the SAME data dir - metadata must survive. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + restored = client.catalogs.get(name=catalog_name) + assert restored.name == catalog_name + + # An open upload is transient. Shutting down with a bytearray buffer + # must not make the snapshot unreadable or discard other saved state. + empty_handle = client.dbfs.create("/unfinished_empty.txt", overwrite=True).handle + upload_handle = client.dbfs.create("/unfinished.txt", overwrite=True).handle + client.dbfs.add_block(upload_handle, base64.b64encode(b"pending").decode("ascii")) + _stop_minilake(proc) + proc = None + + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + restored = client.catalogs.get(name=catalog_name) + assert restored.name == catalog_name + for path, handle in ( + ("/unfinished_empty.txt", empty_handle), + ("/unfinished.txt", upload_handle), + ): + with pytest.raises(Exception): + client.dbfs.get_status(path) + with pytest.raises(Exception): + client.dbfs.add_block(handle, base64.b64encode(b"stale").decode("ascii")) + + # And it must be a REAL, still-queryable catalog after restart, not + # just restored metadata - proves the catalog's DuckDB database was + # really re-ATTACHed, not just its name remembered. + client.schemas.create(name="s", catalog_name=catalog_name) + client.tables.create( + name="t", + catalog_name=catalog_name, + schema_name="s", + storage_location=f"/data/{catalog_name}/s/t", + columns=[ColumnInfo(name="id", type_text="INTEGER")], + table_type=TableType.MANAGED, + data_source_format=DataSourceFormat.DELTA, + ) + + print("✓ MINILAKE_PERSIST=1 really survives a process restart") + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f28daa2..31a40ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,9 +10,6 @@ state survived — using the real databricks-sdk client throughout. """ -import asyncio -import base64 -import json import os import shutil import socket @@ -26,9 +23,6 @@ from databricks.sdk.service.catalog import DataSourceFormat, TableType from databricks.sdk.service.sql import ColumnInfo -from minilake import persistence -from minilake.persistence import save_state - def _free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -93,32 +87,6 @@ def _stop_minilake(proc: subprocess.Popen) -> None: proc.wait() -def test_save_state_serialization_failure_preserves_previous_snapshot(tmp_path): - snapshot_path = tmp_path / "snapshot.json" - previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} - snapshot_path.write_text(json.dumps(previous_snapshot)) - - asyncio.run(save_state(snapshot_path, {"dbfs": lambda: {"buffer": bytearray(b"pending")}})) - - assert json.loads(snapshot_path.read_text()) == previous_snapshot - assert not list(tmp_path.glob(".snapshot.json.*.tmp")) - - -def test_save_state_replace_failure_preserves_previous_snapshot(tmp_path, monkeypatch): - snapshot_path = tmp_path / "snapshot.json" - previous_snapshot = {"format_version": 1, "modules": {"catalogs": {"existing": True}}} - snapshot_path.write_text(json.dumps(previous_snapshot)) - - def fail_replace(source, destination): - raise OSError("simulated replace failure") - - monkeypatch.setattr(persistence.os, "replace", fail_replace) - asyncio.run(save_state(snapshot_path, {"catalogs": lambda: {"current": True}})) - - assert json.loads(snapshot_path.read_text()) == previous_snapshot - assert not list(tmp_path.glob(".snapshot.json.*.tmp")) - - @pytest.mark.serial def test_persist_state_survives_restart(): """Test: MINILAKE_PERSIST=1 really saves state on shutdown and restores @@ -141,27 +109,6 @@ def test_persist_state_survives_restart(): restored = client.catalogs.get(name=catalog_name) assert restored.name == catalog_name - # An open upload is transient. Shutting down with a bytearray buffer - # must not make the snapshot unreadable or discard other saved state. - empty_handle = client.dbfs.create("/unfinished_empty.txt", overwrite=True).handle - upload_handle = client.dbfs.create("/unfinished.txt", overwrite=True).handle - client.dbfs.add_block(upload_handle, base64.b64encode(b"pending").decode("ascii")) - _stop_minilake(proc) - proc = None - - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - restored = client.catalogs.get(name=catalog_name) - assert restored.name == catalog_name - for path, handle in ( - ("/unfinished_empty.txt", empty_handle), - ("/unfinished.txt", upload_handle), - ): - with pytest.raises(Exception): - client.dbfs.get_status(path) - with pytest.raises(Exception): - client.dbfs.add_block(handle, base64.b64encode(b"stale").decode("ascii")) - # And it must be a REAL, still-queryable catalog after restart, not # just restored metadata - proves the catalog's DuckDB database was # really re-ATTACHed, not just its name remembered.