Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/minilake/services/dbfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
def _dbfs_root() -> Path:
root = settings.data_dir / "dbfs"
root.mkdir(parents=True, exist_ok=True)
return root
return root.resolve()


def _normalize(path: str) -> str:
Expand Down
2 changes: 1 addition & 1 deletion src/minilake/services/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
def _files_root() -> Path:
root = settings.data_dir / "dbfs"
root.mkdir(parents=True, exist_ok=True)
return root
return root.resolve()


def _normalize(path: str) -> str:
Expand Down
71 changes: 71 additions & 0 deletions tests/persistence_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""Process helpers for isolated persistence regressions."""

import os
import socket
import subprocess
import time
import urllib.request


def _free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return s.getsockname()[1]


def _drain(proc: subprocess.Popen) -> str:
"""Whatever the subprocess printed, without blocking on a live process."""
if proc.poll() is None:
return "(process still running)"
out, err = proc.communicate(timeout=5)
return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace")


def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None:
"""Poll until minilake answers, failing early and loudly if it died instead.

Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a
slow one, and the test reports a timeout while the actual traceback sits unread in a
pipe. CI is also several times slower to start than a laptop, hence the generous
timeout — this waits on a real process spawn, not a request.
"""
deadline = time.time() + timeout
last_error = None
while time.time() < deadline:
if proc.poll() is not None:
raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}")
try:
if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200:
return
except Exception as e:
last_error = e
time.sleep(0.3)
raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}")


def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen:
env = os.environ.copy()
env["MINILAKE_DATA_DIR"] = data_dir
env["MINILAKE_PERSIST"] = "1"
proc = subprocess.Popen(
["minilake", "--port", str(port), "--host", "127.0.0.1"],
env=env,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
_wait_healthy(proc, f"http://127.0.0.1:{port}")
except Exception:
_stop_minilake(proc)
raise
return proc


def _stop_minilake(proc: subprocess.Popen) -> None:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
40 changes: 40 additions & 0 deletions tests/test_dbfs_files_path_resolution.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""Path resolution regressions shared by DBFS and Files."""

from pathlib import Path

import pytest

from minilake.errors import DatabricksError
from minilake.services import dbfs, files


@pytest.mark.parametrize("service", [dbfs, files], ids=["dbfs", "files"])
@pytest.mark.parametrize("root_kind", ["relative", "absolute", "symlink"], ids=["relative", "absolute", "symlink"])
def test_file_apis_resolve_paths_under_data_root(monkeypatch, tmp_path, service, root_kind):
if root_kind == "relative":
data_dir = Path("relative-data")
expected_data_dir = tmp_path / data_dir
elif root_kind == "absolute":
data_dir = tmp_path / "absolute-data"
expected_data_dir = data_dir
else:
expected_data_dir = tmp_path / "real-data"
expected_data_dir.mkdir()
data_dir = tmp_path / "linked-data"
data_dir.symlink_to(expected_data_dir, target_is_directory=True)

monkeypatch.chdir(tmp_path)
monkeypatch.setattr(service.settings, "data_dir", data_dir)

root = expected_data_dir / "dbfs"
resolved = service._resolve("/hello")
assert resolved == root.resolve() / "hello"

with pytest.raises(DatabricksError, match="Invalid path"):
service._resolve("../../outside")

outside = tmp_path / "outside"
outside.mkdir(exist_ok=True)
(root / "escape").symlink_to(outside, target_is_directory=True)
with pytest.raises(DatabricksError, match="Invalid path"):
service._resolve("/escape/secret")
52 changes: 52 additions & 0 deletions tests/test_relative_storage_persistence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
"""Persistence round-trip test.

Regression test for a real bug found while auditing the project: the
MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed,
but were never actually invoked from app.py's lifespan — so setting
MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated
minilake subprocess (not the shared session-scoped server used by other
tests, since this specifically needs a real process restart), creates real
state, restarts the process against the same data dir, and verifies the
state survived — using the real databricks-sdk client throughout.
"""

import base64
import json
import urllib.request

import pytest
from persistence_helpers import _free_port, _start_minilake, _stop_minilake


@pytest.mark.serial
def test_relative_data_dir_supports_dbfs_and_files(tmp_path):
"""Relative data roots should work through both real HTTP file APIs."""
working_dir = tmp_path / "working_dir"
working_dir.mkdir()
port = _free_port()
proc = _start_minilake(port, "data", cwd=str(working_dir))
base_url = f"http://127.0.0.1:{port}"
dbfs_path = working_dir / "data" / "dbfs" / "hello"
try:
dbfs_body = json.dumps({"path": "/hello", "contents": base64.b64encode(b"dbfs").decode("ascii")}).encode()
request = urllib.request.Request(
f"{base_url}/api/2.0/dbfs/put",
data=dbfs_body,
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=5) as response:
assert response.status == 200
assert dbfs_path.read_bytes() == b"dbfs"

request = urllib.request.Request(
f"{base_url}/api/2.0/fs/files/hello",
data=b"files",
headers={"Content-Type": "application/octet-stream"},
method="PUT",
)
with urllib.request.urlopen(request, timeout=5) as response:
assert response.status == 204
assert dbfs_path.read_bytes() == b"files"
finally:
_stop_minilake(proc)
Loading