From 638f77d314fe2795473c61707e655ba4d1041162 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 13:57:59 -0400 Subject: [PATCH 1/3] Fix relative data roots for DBFS and Files --- src/minilake/services/dbfs.py | 2 +- src/minilake/services/files.py | 2 +- tests/test_dbfs_files_path_resolution.py | 40 +++++++++++++++++++++++ tests/test_persistence.py | 41 +++++++++++++++++++++++- 4 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 tests/test_dbfs_files_path_resolution.py diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index ff9443d..a4625c4 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -56,7 +56,7 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) - root = _dbfs_root() + root = _dbfs_root().resolve() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) diff --git a/src/minilake/services/files.py b/src/minilake/services/files.py index de8c450..653c4d2 100644 --- a/src/minilake/services/files.py +++ b/src/minilake/services/files.py @@ -36,7 +36,7 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) - root = _files_root() + root = _files_root().resolve() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) diff --git a/tests/test_dbfs_files_path_resolution.py b/tests/test_dbfs_files_path_resolution.py new file mode 100644 index 0000000..bfca47b --- /dev/null +++ b/tests/test_dbfs_files_path_resolution.py @@ -0,0 +1,40 @@ +"""Path resolution regressions shared by DBFS and Files.""" + +from pathlib import Path + +import pytest + +from minilake.errors import DatabricksError +from minilake.services import dbfs, files + + +@pytest.mark.parametrize("service", [dbfs, files], ids=["dbfs", "files"]) +@pytest.mark.parametrize("root_kind", ["relative", "absolute", "symlink"], ids=["relative", "absolute", "symlink"]) +def test_file_apis_resolve_paths_under_data_root(monkeypatch, tmp_path, service, root_kind): + if root_kind == "relative": + data_dir = Path("relative-data") + expected_data_dir = tmp_path / data_dir + elif root_kind == "absolute": + data_dir = tmp_path / "absolute-data" + expected_data_dir = data_dir + else: + expected_data_dir = tmp_path / "real-data" + expected_data_dir.mkdir() + data_dir = tmp_path / "linked-data" + data_dir.symlink_to(expected_data_dir, target_is_directory=True) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service.settings, "data_dir", data_dir) + + root = expected_data_dir / "dbfs" + resolved = service._resolve("/hello") + assert resolved == root.resolve() / "hello" + + with pytest.raises(DatabricksError, match="Invalid path"): + service._resolve("../../outside") + + outside = tmp_path / "outside" + outside.mkdir(exist_ok=True) + (root / "escape").symlink_to(outside, target_is_directory=True) + with pytest.raises(DatabricksError, match="Invalid path"): + service._resolve("/escape/secret") diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 31a40ed..dccbc86 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,6 +10,8 @@ state survived — using the real databricks-sdk client throughout. """ +import base64 +import json import os import shutil import socket @@ -60,7 +62,7 @@ def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> No raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") -def _start_minilake(port: int, data_dir: str) -> subprocess.Popen: +def _start_minilake(port: int, data_dir: str, cwd: str | None = None) -> subprocess.Popen: env = os.environ.copy() env["MINILAKE_DATA_DIR"] = data_dir env["MINILAKE_PERSIST"] = "1" @@ -69,6 +71,7 @@ def _start_minilake(port: int, data_dir: str) -> subprocess.Popen: env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, ) try: _wait_healthy(proc, f"http://127.0.0.1:{port}") @@ -128,3 +131,39 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_relative_data_dir_supports_dbfs_and_files(tmp_path): + """Relative data roots should work through both real HTTP file APIs.""" + working_dir = tmp_path / "working_dir" + working_dir.mkdir() + port = _free_port() + proc = _start_minilake(port, "data", cwd=str(working_dir)) + base_url = f"http://127.0.0.1:{port}" + dbfs_path = working_dir / "data" / "dbfs" / "hello" + try: + dbfs_body = json.dumps( + {"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")} + ).encode() + request = urllib.request.Request( + f"{base_url}/api/2.0/dbfs/put", + data=dbfs_body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 200 + assert dbfs_path.read_bytes() == b"dbfs" + + request = urllib.request.Request( + f"{base_url}/api/2.0/fs/files/hello", + data=b"files", + headers={"Content-Type": "application/octet-stream"}, + method="PUT", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 204 + assert dbfs_path.read_bytes() == b"files" + finally: + _stop_minilake(proc) From 5c271fc08ce3eb50445c999c6a05f6cee6731d52 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 14:05:57 -0400 Subject: [PATCH 2/3] Format relative data dir regression test --- tests/test_persistence.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index dccbc86..9f4eb37 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -143,9 +143,7 @@ def test_relative_data_dir_supports_dbfs_and_files(tmp_path): base_url = f"http://127.0.0.1:{port}" dbfs_path = working_dir / "data" / "dbfs" / "hello" try: - dbfs_body = json.dumps( - {"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")} - ).encode() + dbfs_body = json.dumps({"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")}).encode() request = urllib.request.Request( f"{base_url}/api/2.0/dbfs/put", data=dbfs_body, From 97bc8f1f0d029a6148be644138e113615242ef1d Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 16:01:27 -0400 Subject: [PATCH 3/3] refactor: keep this fix independently mergeable with other open PRs --- src/minilake/services/dbfs.py | 4 +- src/minilake/services/files.py | 4 +- tests/persistence_helpers.py | 71 ++++++++++++++++++++++ tests/test_persistence.py | 39 +----------- tests/test_relative_storage_persistence.py | 52 ++++++++++++++++ 5 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 tests/persistence_helpers.py create mode 100644 tests/test_relative_storage_persistence.py diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index a4625c4..b43b6be 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -43,7 +43,7 @@ def _dbfs_root() -> Path: root = settings.data_dir / "dbfs" root.mkdir(parents=True, exist_ok=True) - return root + return root.resolve() def _normalize(path: str) -> str: @@ -56,7 +56,7 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) - root = _dbfs_root().resolve() + root = _dbfs_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) diff --git a/src/minilake/services/files.py b/src/minilake/services/files.py index 653c4d2..3369988 100644 --- a/src/minilake/services/files.py +++ b/src/minilake/services/files.py @@ -23,7 +23,7 @@ def _files_root() -> Path: root = settings.data_dir / "dbfs" root.mkdir(parents=True, exist_ok=True) - return root + return root.resolve() def _normalize(path: str) -> str: @@ -36,7 +36,7 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) - root = _files_root().resolve() + root = _files_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) diff --git a/tests/persistence_helpers.py b/tests/persistence_helpers.py new file mode 100644 index 0000000..da13d01 --- /dev/null +++ b/tests/persistence_helpers.py @@ -0,0 +1,71 @@ +"""Process helpers for isolated persistence regressions.""" + +import os +import socket +import subprocess +import time +import urllib.request + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _drain(proc: subprocess.Popen) -> str: + """Whatever the subprocess printed, without blocking on a live process.""" + if proc.poll() is None: + return "(process still running)" + out, err = proc.communicate(timeout=5) + return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace") + + +def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None: + """Poll until minilake answers, failing early and loudly if it died instead. + + Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a + slow one, and the test reports a timeout while the actual traceback sits unread in a + pipe. CI is also several times slower to start than a laptop, hence the generous + timeout — this waits on a real process spawn, not a request. + """ + deadline = time.time() + timeout + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}") + try: + if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200: + return + except Exception as e: + last_error = e + time.sleep(0.3) + raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") + + +def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen: + env = os.environ.copy() + env["MINILAKE_DATA_DIR"] = data_dir + env["MINILAKE_PERSIST"] = "1" + proc = subprocess.Popen( + ["minilake", "--port", str(port), "--host", "127.0.0.1"], + env=env, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + _wait_healthy(proc, f"http://127.0.0.1:{port}") + except Exception: + _stop_minilake(proc) + raise + return proc + + +def _stop_minilake(proc: subprocess.Popen) -> None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 9f4eb37..31a40ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,8 +10,6 @@ state survived — using the real databricks-sdk client throughout. """ -import base64 -import json import os import shutil import socket @@ -62,7 +60,7 @@ def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> No raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") -def _start_minilake(port: int, data_dir: str, cwd: str | None = None) -> subprocess.Popen: +def _start_minilake(port: int, data_dir: str) -> subprocess.Popen: env = os.environ.copy() env["MINILAKE_DATA_DIR"] = data_dir env["MINILAKE_PERSIST"] = "1" @@ -71,7 +69,6 @@ def _start_minilake(port: int, data_dir: str, cwd: str | None = None) -> subproc env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=cwd, ) try: _wait_healthy(proc, f"http://127.0.0.1:{port}") @@ -131,37 +128,3 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) - - -@pytest.mark.serial -def test_relative_data_dir_supports_dbfs_and_files(tmp_path): - """Relative data roots should work through both real HTTP file APIs.""" - working_dir = tmp_path / "working_dir" - working_dir.mkdir() - port = _free_port() - proc = _start_minilake(port, "data", cwd=str(working_dir)) - base_url = f"http://127.0.0.1:{port}" - dbfs_path = working_dir / "data" / "dbfs" / "hello" - try: - dbfs_body = json.dumps({"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")}).encode() - request = urllib.request.Request( - f"{base_url}/api/2.0/dbfs/put", - data=dbfs_body, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(request, timeout=5) as response: - assert response.status == 200 - assert dbfs_path.read_bytes() == b"dbfs" - - request = urllib.request.Request( - f"{base_url}/api/2.0/fs/files/hello", - data=b"files", - headers={"Content-Type": "application/octet-stream"}, - method="PUT", - ) - with urllib.request.urlopen(request, timeout=5) as response: - assert response.status == 204 - assert dbfs_path.read_bytes() == b"files" - finally: - _stop_minilake(proc) diff --git a/tests/test_relative_storage_persistence.py b/tests/test_relative_storage_persistence.py new file mode 100644 index 0000000..9177974 --- /dev/null +++ b/tests/test_relative_storage_persistence.py @@ -0,0 +1,52 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import base64 +import json +import urllib.request + +import pytest +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + + +@pytest.mark.serial +def test_relative_data_dir_supports_dbfs_and_files(tmp_path): + """Relative data roots should work through both real HTTP file APIs.""" + working_dir = tmp_path / "working_dir" + working_dir.mkdir() + port = _free_port() + proc = _start_minilake(port, "data", cwd=str(working_dir)) + base_url = f"http://127.0.0.1:{port}" + dbfs_path = working_dir / "data" / "dbfs" / "hello" + try: + dbfs_body = json.dumps({"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")}).encode() + request = urllib.request.Request( + f"{base_url}/api/2.0/dbfs/put", + data=dbfs_body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 200 + assert dbfs_path.read_bytes() == b"dbfs" + + request = urllib.request.Request( + f"{base_url}/api/2.0/fs/files/hello", + data=b"files", + headers={"Content-Type": "application/octet-stream"}, + method="PUT", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 204 + assert dbfs_path.read_bytes() == b"files" + finally: + _stop_minilake(proc)