Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/minilake/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,15 @@ async def lifespan(app: FastAPI):
except Exception as e:
logger.warning(f"Failed to re-attach catalog {catalog_name} on restore: {e}")

# Older releases stored `/Volumes/...` uploads under dbfs/Volumes,
# separate from the directories Unity Catalog creates in volumes/.
# Migrate those files before API routes or reset hooks can remove them.
from minilake.services.volume_paths import migrate_legacy_volume_data

migrated_volumes = migrate_legacy_volume_data()
if migrated_volumes:
logger.info(f"Migrated legacy Files/DBFS data for {migrated_volumes} Unity Catalog volumes")

# Best-effort, non-blocking: pre-pull the Spark image used for real Job
# execution so the first real job run doesn't pay the cold-start cost.
asyncio.create_task(asyncio.to_thread(prewarm_spark_image))
Expand Down
13 changes: 9 additions & 4 deletions src/minilake/services/dbfs.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
"""DBFS (Databricks File System) API endpoints — real file-backed storage.

Files live under `settings.data_dir / "dbfs"`. Chunked upload (create/add-block/
close) is tracked with a small in-memory handle table, matching the real API's
session-based upload model — the handle itself is ephemeral, only the final
file on disk persists.
Ordinary files live under `settings.data_dir / "dbfs"`; `/Volumes/...` URIs
resolve to their Unity Catalog volume directory. Chunked upload (create/add-
block/close) uses an in-memory handle table, matching the real API's session-
based upload model — only the final file persists.
"""

import base64
Expand All @@ -29,6 +29,7 @@
PutDbfsRequest,
ReadFileResponse,
)
from minilake.services.volume_paths import resolve_volume_path

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -56,6 +57,10 @@ def _normalize(path: str) -> str:

def _resolve(path: str) -> Path:
normalized = _normalize(path)
volume_path = resolve_volume_path(normalized)
if volume_path is not None:
return volume_path

root = _dbfs_root()
resolved = (root / normalized.lstrip("/")).resolve()
if resolved != root and root not in resolved.parents:
Expand Down
11 changes: 8 additions & 3 deletions src/minilake/services/files.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
"""Files API endpoints — real file-backed storage (modern DBFS alternative).

Shares the same on-disk root as DBFS (`settings.data_dir / "dbfs"`), since both
APIs address the same conceptual filesystem in real Databricks. Unlike DBFS's
JSON+base64 shape, this API moves raw bytes directly in the HTTP body.
Ordinary paths share DBFS's on-disk root; `/Volumes/...` paths resolve to their
Unity Catalog volume directory. Unlike DBFS's JSON+base64 shape, this API moves
raw bytes directly in the HTTP body.
"""

import logging
Expand All @@ -14,6 +14,7 @@

from minilake.config import settings
from minilake.errors import DatabricksError
from minilake.services.volume_paths import resolve_volume_path

logger = logging.getLogger(__name__)

Expand All @@ -36,6 +37,10 @@ def _normalize(path: str) -> str:

def _resolve(path: str) -> Path:
normalized = _normalize(path)
volume_path = resolve_volume_path(normalized)
if volume_path is not None:
return volume_path

root = _files_root()
resolved = (root / normalized.lstrip("/")).resolve()
if resolved != root and root not in resolved.parents:
Expand Down
137 changes: 137 additions & 0 deletions src/minilake/services/volume_paths.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
"""Resolve Unity Catalog volume URIs to their on-disk storage."""

import shutil
from pathlib import Path

from minilake.config import settings
from minilake.errors import DatabricksError

_VOLUMES_PREFIX = "/Volumes"


def _merge_legacy_volume(legacy_path: Path, volume_root: Path) -> bool:
"""Move legacy Files/DBFS volume data into its UC volume directory.

Preflight the entire tree before moving anything. A conflicting path leaves
both copies untouched and stops startup with a message for the operator.
"""
legacy_dbfs_root = settings.data_dir / "dbfs"
current = legacy_dbfs_root
if current.is_symlink():
raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}")
for part in legacy_path.relative_to(legacy_dbfs_root).parts:
current = current / part
if current.is_symlink():
raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}")

if not legacy_path.exists():
return False
if not legacy_path.is_dir():
raise RuntimeError(f"Cannot migrate legacy volume data at {legacy_path}: expected a directory")

def check(source: Path, destination: Path) -> None:
for child in source.iterdir():
target = destination / child.name
if child.is_symlink():
raise RuntimeError(f"Cannot migrate legacy volume data: symlink at {child}")
if target.exists() or target.is_symlink():
if not child.is_dir() or not target.is_dir() or target.is_symlink():
raise RuntimeError(
f"Cannot migrate legacy volume data: both legacy and current paths contain {target}"
)
if child.is_dir():
check(child, target)

check(legacy_path, volume_root)

def move_contents(source: Path, destination: Path) -> None:
for child in source.iterdir():
target = destination / child.name
if child.is_dir():
target.mkdir(exist_ok=True)
move_contents(child, target)
child.rmdir()
else:
shutil.move(str(child), str(target))

move_contents(legacy_path, volume_root)
legacy_path.rmdir()
return True


def migrate_legacy_volume_data() -> int:
"""Run the one-time migration of old `/Volumes` uploads into UC directories."""
volumes_root = (settings.data_dir / "volumes").resolve()
migration_marker = volumes_root / ".legacy_volume_migration_complete"
if migration_marker.exists():
return 0

volumes_root.mkdir(parents=True, exist_ok=True)
migrated = 0
for catalog_path in volumes_root.iterdir():
if catalog_path.is_symlink() or not catalog_path.is_dir():
continue
for schema_path in catalog_path.iterdir():
if schema_path.is_symlink() or not schema_path.is_dir():
continue
for volume_root in schema_path.iterdir():
if volume_root.is_symlink() or not volume_root.is_dir():
continue
legacy_path = (
settings.data_dir / "dbfs" / "Volumes" / catalog_path.name / schema_path.name / volume_root.name
)
if _merge_legacy_volume(legacy_path, volume_root):
migrated += 1

# Keep later volume creations from adopting files left by volumes deleted
# before this upgrade. The marker lives inside volumes/ so a full reset
# removes it along with the migrated storage.
migration_marker.touch()
return migrated


def resolve_volume_path(path: str) -> Path | None:
"""Resolve a `/Volumes/...` URI to its Unity Catalog volume directory.

Returns ``None`` for paths outside the volume namespace so Files and DBFS can
use their ordinary shared root. Volume URIs must name an existing volume;
this prevents deleted or unknown volumes from becoming generic DBFS paths.
"""
normalized = "/" + path.lstrip("/")
if normalized != _VOLUMES_PREFIX and not normalized.startswith(_VOLUMES_PREFIX + "/"):
return None

suffix = normalized[len(_VOLUMES_PREFIX) :]
if suffix.startswith("/"):
suffix = suffix[1:]
if suffix.endswith("/"):
suffix = suffix[:-1]
parts = suffix.split("/")
if len(parts) < 3 or any(part in {"", ".", ".."} for part in parts[:3]):
raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400)

catalog_name, schema_name, volume_name = parts[:3]
volumes_root = (settings.data_dir / "volumes").resolve()
volume_root = volumes_root.joinpath(catalog_name, schema_name, volume_name)

# The catalog/schema/volume components identify the volume itself; do not
# follow a symlink there to a different volume or outside the volume tree.
current = volumes_root
for part in (catalog_name, schema_name, volume_name):
current = current / part
if current.is_symlink():
raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400)

volume_root = volume_root.resolve()
if volumes_root not in volume_root.parents or not volume_root.is_dir():
full_name = f"{catalog_name}.{schema_name}.{volume_name}"
raise DatabricksError(
error_code="RESOURCE_DOES_NOT_EXIST",
message=f"Volume '{full_name}' not found",
status_code=404,
)

target = volume_root.joinpath(*parts[3:]).resolve()
if target != volume_root and volume_root not in target.parents:
raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400)
return target
137 changes: 137 additions & 0 deletions tests/test_volume_migration_persistence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
"""Persistence round-trip test.

Regression test for a real bug found while auditing the project: the
MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed,
but were never actually invoked from app.py's lifespan — so setting
MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated
minilake subprocess (not the shared session-scoped server used by other
tests, since this specifically needs a real process restart), creates real
state, restarts the process against the same data dir, and verifies the
state survived — using the real databricks-sdk client throughout.
"""

import base64
import shutil
import tempfile
import uuid
from pathlib import Path

import pytest
from databricks.sdk import WorkspaceClient
from databricks.sdk.errors import DatabricksError
from databricks.sdk.service.catalog import VolumeType
from persistence_helpers import _free_port, _start_minilake, _stop_minilake


@pytest.mark.serial
def test_legacy_volume_files_migrate_on_restart():
"""Files created under the pre-UC volume path remain readable after upgrade."""
data_dir = tempfile.mkdtemp(prefix="minilake_volume_migration_")
port = _free_port()
proc = None
try:
proc = _start_minilake(port, data_dir)
client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat")
catalog_name = f"migration_{uuid.uuid4().hex[:8]}"
schema_name = "migration_schema"
volume_name = "migration_volume"
client.catalogs.create(name=catalog_name)
client.schemas.create(name=schema_name, catalog_name=catalog_name)
client.volumes.create(
name=volume_name,
catalog_name=catalog_name,
schema_name=schema_name,
volume_type=VolumeType.MANAGED,
)

payload = b"legacy volume data survives upgrade\n"
legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt"
legacy_file.parent.mkdir(parents=True)
legacy_file.write_bytes(payload)
# The process running before an upgrade cannot have written the new marker.
(Path(data_dir) / "volumes" / ".legacy_volume_migration_complete").unlink()
volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt"
full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}"

_stop_minilake(proc)
proc = None
proc = _start_minilake(port, data_dir)
client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat")

assert client.files.download(volume_path).contents.read() == payload
dbfs_data = client.dbfs.read(path=volume_path).data
assert base64.b64decode(dbfs_data) == payload
assert (
Path(data_dir) / "volumes" / catalog_name / schema_name / volume_name / "legacy.txt"
).read_bytes() == payload
assert not legacy_file.exists()

client.volumes.delete(name=full_volume_name)
with pytest.raises(DatabricksError):
client.files.download(volume_path)
with pytest.raises(DatabricksError):
client.dbfs.read(path=volume_path)
finally:
if proc is not None:
_stop_minilake(proc)
shutil.rmtree(data_dir, ignore_errors=True)


@pytest.mark.serial
def test_deleted_legacy_volume_files_stay_deleted_after_restart():
"""An upgrade must not import orphaned files when a volume name is reused."""
data_dir = tempfile.mkdtemp(prefix="minilake_orphaned_volume_")
port = _free_port()
proc = None
try:
proc = _start_minilake(port, data_dir)
client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat")
catalog_name = f"orphan_{uuid.uuid4().hex[:8]}"
schema_name = "orphan_schema"
volume_name = "reused_volume"
full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}"
volume_path = f"/Volumes/{full_volume_name.replace('.', '/')}/old.txt"
client.catalogs.create(name=catalog_name)
client.schemas.create(name=schema_name, catalog_name=catalog_name)
client.volumes.create(
name=volume_name,
catalog_name=catalog_name,
schema_name=schema_name,
volume_type=VolumeType.MANAGED,
)

# Simulate an old release: volume deletion removed its UC directory but
# left Files/DBFS content under the legacy dbfs/Volumes path.
legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "old.txt"
legacy_file.parent.mkdir(parents=True)
legacy_file.write_bytes(b"content from deleted volume")
client.volumes.delete(name=full_volume_name)
migration_marker = Path(data_dir) / "volumes" / ".legacy_volume_migration_complete"
migration_marker.unlink()

_stop_minilake(proc)
proc = None
# First upgraded startup must not migrate a volume that had already been deleted.
proc = _start_minilake(port, data_dir)
client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat")
client.volumes.create(
name=volume_name,
catalog_name=catalog_name,
schema_name=schema_name,
volume_type=VolumeType.MANAGED,
)

_stop_minilake(proc)
proc = None
# A later restart must not reinterpret the new volume as a pre-upgrade one.
proc = _start_minilake(port, data_dir)
client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat")
with pytest.raises(DatabricksError):
client.files.download(volume_path)
with pytest.raises(DatabricksError):
client.dbfs.read(path=volume_path)
assert legacy_file.read_bytes() == b"content from deleted volume"
finally:
if proc is not None:
_stop_minilake(proc)
shutil.rmtree(data_dir, ignore_errors=True)
Loading