diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index ff9443d..b43b6be 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -43,7 +43,7 @@ def _dbfs_root() -> Path: root = settings.data_dir / "dbfs" root.mkdir(parents=True, exist_ok=True) - return root + return root.resolve() def _normalize(path: str) -> str: diff --git a/src/minilake/services/files.py b/src/minilake/services/files.py index de8c450..3369988 100644 --- a/src/minilake/services/files.py +++ b/src/minilake/services/files.py @@ -23,7 +23,7 @@ def _files_root() -> Path: root = settings.data_dir / "dbfs" root.mkdir(parents=True, exist_ok=True) - return root + return root.resolve() def _normalize(path: str) -> str: diff --git a/tests/persistence_helpers.py b/tests/persistence_helpers.py new file mode 100644 index 0000000..da13d01 --- /dev/null +++ b/tests/persistence_helpers.py @@ -0,0 +1,71 @@ +"""Process helpers for isolated persistence regressions.""" + +import os +import socket +import subprocess +import time +import urllib.request + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _drain(proc: subprocess.Popen) -> str: + """Whatever the subprocess printed, without blocking on a live process.""" + if proc.poll() is None: + return "(process still running)" + out, err = proc.communicate(timeout=5) + return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace") + + +def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None: + """Poll until minilake answers, failing early and loudly if it died instead. + + Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a + slow one, and the test reports a timeout while the actual traceback sits unread in a + pipe. CI is also several times slower to start than a laptop, hence the generous + timeout — this waits on a real process spawn, not a request. + """ + deadline = time.time() + timeout + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}") + try: + if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200: + return + except Exception as e: + last_error = e + time.sleep(0.3) + raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") + + +def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen: + env = os.environ.copy() + env["MINILAKE_DATA_DIR"] = data_dir + env["MINILAKE_PERSIST"] = "1" + proc = subprocess.Popen( + ["minilake", "--port", str(port), "--host", "127.0.0.1"], + env=env, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + _wait_healthy(proc, f"http://127.0.0.1:{port}") + except Exception: + _stop_minilake(proc) + raise + return proc + + +def _stop_minilake(proc: subprocess.Popen) -> None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/tests/test_dbfs_files_path_resolution.py b/tests/test_dbfs_files_path_resolution.py new file mode 100644 index 0000000..bfca47b --- /dev/null +++ b/tests/test_dbfs_files_path_resolution.py @@ -0,0 +1,40 @@ +"""Path resolution regressions shared by DBFS and Files.""" + +from pathlib import Path + +import pytest + +from minilake.errors import DatabricksError +from minilake.services import dbfs, files + + +@pytest.mark.parametrize("service", [dbfs, files], ids=["dbfs", "files"]) +@pytest.mark.parametrize("root_kind", ["relative", "absolute", "symlink"], ids=["relative", "absolute", "symlink"]) +def test_file_apis_resolve_paths_under_data_root(monkeypatch, tmp_path, service, root_kind): + if root_kind == "relative": + data_dir = Path("relative-data") + expected_data_dir = tmp_path / data_dir + elif root_kind == "absolute": + data_dir = tmp_path / "absolute-data" + expected_data_dir = data_dir + else: + expected_data_dir = tmp_path / "real-data" + expected_data_dir.mkdir() + data_dir = tmp_path / "linked-data" + data_dir.symlink_to(expected_data_dir, target_is_directory=True) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service.settings, "data_dir", data_dir) + + root = expected_data_dir / "dbfs" + resolved = service._resolve("/hello") + assert resolved == root.resolve() / "hello" + + with pytest.raises(DatabricksError, match="Invalid path"): + service._resolve("../../outside") + + outside = tmp_path / "outside" + outside.mkdir(exist_ok=True) + (root / "escape").symlink_to(outside, target_is_directory=True) + with pytest.raises(DatabricksError, match="Invalid path"): + service._resolve("/escape/secret") diff --git a/tests/test_relative_storage_persistence.py b/tests/test_relative_storage_persistence.py new file mode 100644 index 0000000..9177974 --- /dev/null +++ b/tests/test_relative_storage_persistence.py @@ -0,0 +1,52 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import base64 +import json +import urllib.request + +import pytest +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + + +@pytest.mark.serial +def test_relative_data_dir_supports_dbfs_and_files(tmp_path): + """Relative data roots should work through both real HTTP file APIs.""" + working_dir = tmp_path / "working_dir" + working_dir.mkdir() + port = _free_port() + proc = _start_minilake(port, "data", cwd=str(working_dir)) + base_url = f"http://127.0.0.1:{port}" + dbfs_path = working_dir / "data" / "dbfs" / "hello" + try: + dbfs_body = json.dumps({"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")}).encode() + request = urllib.request.Request( + f"{base_url}/api/2.0/dbfs/put", + data=dbfs_body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 200 + assert dbfs_path.read_bytes() == b"dbfs" + + request = urllib.request.Request( + f"{base_url}/api/2.0/fs/files/hello", + data=b"files", + headers={"Content-Type": "application/octet-stream"}, + method="PUT", + ) + with urllib.request.urlopen(request, timeout=5) as response: + assert response.status == 204 + assert dbfs_path.read_bytes() == b"files" + finally: + _stop_minilake(proc)