diff --git a/src/minilake/app.py b/src/minilake/app.py index 68b8471..9601c15 100644 --- a/src/minilake/app.py +++ b/src/minilake/app.py @@ -151,6 +151,15 @@ async def lifespan(app: FastAPI): except Exception as e: logger.warning(f"Failed to re-attach catalog {catalog_name} on restore: {e}") + # Older releases stored `/Volumes/...` uploads under dbfs/Volumes, + # separate from the directories Unity Catalog creates in volumes/. + # Migrate those files before API routes or reset hooks can remove them. + from minilake.services.volume_paths import migrate_legacy_volume_data + + migrated_volumes = migrate_legacy_volume_data() + if migrated_volumes: + logger.info(f"Migrated legacy Files/DBFS data for {migrated_volumes} Unity Catalog volumes") + # Best-effort, non-blocking: pre-pull the Spark image used for real Job # execution so the first real job run doesn't pay the cold-start cost. asyncio.create_task(asyncio.to_thread(prewarm_spark_image)) diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index 3b61d48..6629e97 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -1,9 +1,9 @@ """DBFS (Databricks File System) API endpoints — real file-backed storage. -Files live under `settings.data_dir / "dbfs"`. Chunked upload (create/add-block/ -close) is tracked with a small in-memory handle table, matching the real API's -session-based upload model — the handle itself is ephemeral, only the final -file on disk persists. +Ordinary files live under `settings.data_dir / "dbfs"`; `/Volumes/...` URIs +resolve to their Unity Catalog volume directory. Chunked upload (create/add- +block/close) uses an in-memory handle table, matching the real API's session- +based upload model — only the final file persists. """ import base64 @@ -29,6 +29,7 @@ PutDbfsRequest, ReadFileResponse, ) +from minilake.services.volume_paths import resolve_volume_path logger = logging.getLogger(__name__) @@ -56,6 +57,10 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) + volume_path = resolve_volume_path(normalized) + if volume_path is not None: + return volume_path + root = _dbfs_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: diff --git a/src/minilake/services/files.py b/src/minilake/services/files.py index 3369988..9d6ba42 100644 --- a/src/minilake/services/files.py +++ b/src/minilake/services/files.py @@ -1,8 +1,8 @@ """Files API endpoints — real file-backed storage (modern DBFS alternative). -Shares the same on-disk root as DBFS (`settings.data_dir / "dbfs"`), since both -APIs address the same conceptual filesystem in real Databricks. Unlike DBFS's -JSON+base64 shape, this API moves raw bytes directly in the HTTP body. +Ordinary paths share DBFS's on-disk root; `/Volumes/...` paths resolve to their +Unity Catalog volume directory. Unlike DBFS's JSON+base64 shape, this API moves +raw bytes directly in the HTTP body. """ import logging @@ -14,6 +14,7 @@ from minilake.config import settings from minilake.errors import DatabricksError +from minilake.services.volume_paths import resolve_volume_path logger = logging.getLogger(__name__) @@ -36,6 +37,10 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) + volume_path = resolve_volume_path(normalized) + if volume_path is not None: + return volume_path + root = _files_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: diff --git a/src/minilake/services/volume_paths.py b/src/minilake/services/volume_paths.py new file mode 100644 index 0000000..be24eb7 --- /dev/null +++ b/src/minilake/services/volume_paths.py @@ -0,0 +1,137 @@ +"""Resolve Unity Catalog volume URIs to their on-disk storage.""" + +import shutil +from pathlib import Path + +from minilake.config import settings +from minilake.errors import DatabricksError + +_VOLUMES_PREFIX = "/Volumes" + + +def _merge_legacy_volume(legacy_path: Path, volume_root: Path) -> bool: + """Move legacy Files/DBFS volume data into its UC volume directory. + + Preflight the entire tree before moving anything. A conflicting path leaves + both copies untouched and stops startup with a message for the operator. + """ + legacy_dbfs_root = settings.data_dir / "dbfs" + current = legacy_dbfs_root + if current.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}") + for part in legacy_path.relative_to(legacy_dbfs_root).parts: + current = current / part + if current.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}") + + if not legacy_path.exists(): + return False + if not legacy_path.is_dir(): + raise RuntimeError(f"Cannot migrate legacy volume data at {legacy_path}: expected a directory") + + def check(source: Path, destination: Path) -> None: + for child in source.iterdir(): + target = destination / child.name + if child.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data: symlink at {child}") + if target.exists() or target.is_symlink(): + if not child.is_dir() or not target.is_dir() or target.is_symlink(): + raise RuntimeError( + f"Cannot migrate legacy volume data: both legacy and current paths contain {target}" + ) + if child.is_dir(): + check(child, target) + + check(legacy_path, volume_root) + + def move_contents(source: Path, destination: Path) -> None: + for child in source.iterdir(): + target = destination / child.name + if child.is_dir(): + target.mkdir(exist_ok=True) + move_contents(child, target) + child.rmdir() + else: + shutil.move(str(child), str(target)) + + move_contents(legacy_path, volume_root) + legacy_path.rmdir() + return True + + +def migrate_legacy_volume_data() -> int: + """Run the one-time migration of old `/Volumes` uploads into UC directories.""" + volumes_root = (settings.data_dir / "volumes").resolve() + migration_marker = volumes_root / ".legacy_volume_migration_complete" + if migration_marker.exists(): + return 0 + + volumes_root.mkdir(parents=True, exist_ok=True) + migrated = 0 + for catalog_path in volumes_root.iterdir(): + if catalog_path.is_symlink() or not catalog_path.is_dir(): + continue + for schema_path in catalog_path.iterdir(): + if schema_path.is_symlink() or not schema_path.is_dir(): + continue + for volume_root in schema_path.iterdir(): + if volume_root.is_symlink() or not volume_root.is_dir(): + continue + legacy_path = ( + settings.data_dir / "dbfs" / "Volumes" / catalog_path.name / schema_path.name / volume_root.name + ) + if _merge_legacy_volume(legacy_path, volume_root): + migrated += 1 + + # Keep later volume creations from adopting files left by volumes deleted + # before this upgrade. The marker lives inside volumes/ so a full reset + # removes it along with the migrated storage. + migration_marker.touch() + return migrated + + +def resolve_volume_path(path: str) -> Path | None: + """Resolve a `/Volumes/...` URI to its Unity Catalog volume directory. + + Returns ``None`` for paths outside the volume namespace so Files and DBFS can + use their ordinary shared root. Volume URIs must name an existing volume; + this prevents deleted or unknown volumes from becoming generic DBFS paths. + """ + normalized = "/" + path.lstrip("/") + if normalized != _VOLUMES_PREFIX and not normalized.startswith(_VOLUMES_PREFIX + "/"): + return None + + suffix = normalized[len(_VOLUMES_PREFIX) :] + if suffix.startswith("/"): + suffix = suffix[1:] + if suffix.endswith("/"): + suffix = suffix[:-1] + parts = suffix.split("/") + if len(parts) < 3 or any(part in {"", ".", ".."} for part in parts[:3]): + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400) + + catalog_name, schema_name, volume_name = parts[:3] + volumes_root = (settings.data_dir / "volumes").resolve() + volume_root = volumes_root.joinpath(catalog_name, schema_name, volume_name) + + # The catalog/schema/volume components identify the volume itself; do not + # follow a symlink there to a different volume or outside the volume tree. + current = volumes_root + for part in (catalog_name, schema_name, volume_name): + current = current / part + if current.is_symlink(): + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400) + + volume_root = volume_root.resolve() + if volumes_root not in volume_root.parents or not volume_root.is_dir(): + full_name = f"{catalog_name}.{schema_name}.{volume_name}" + raise DatabricksError( + error_code="RESOURCE_DOES_NOT_EXIST", + message=f"Volume '{full_name}' not found", + status_code=404, + ) + + target = volume_root.joinpath(*parts[3:]).resolve() + if target != volume_root and volume_root not in target.parents: + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) + return target diff --git a/tests/test_volume_migration_persistence.py b/tests/test_volume_migration_persistence.py new file mode 100644 index 0000000..a45edc1 --- /dev/null +++ b/tests/test_volume_migration_persistence.py @@ -0,0 +1,137 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import base64 +import shutil +import tempfile +import uuid +from pathlib import Path + +import pytest +from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError +from databricks.sdk.service.catalog import VolumeType +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + + +@pytest.mark.serial +def test_legacy_volume_files_migrate_on_restart(): + """Files created under the pre-UC volume path remain readable after upgrade.""" + data_dir = tempfile.mkdtemp(prefix="minilake_volume_migration_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"migration_{uuid.uuid4().hex[:8]}" + schema_name = "migration_schema" + volume_name = "migration_volume" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + payload = b"legacy volume data survives upgrade\n" + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(payload) + # The process running before an upgrade cannot have written the new marker. + (Path(data_dir) / "volumes" / ".legacy_volume_migration_complete").unlink() + volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + + _stop_minilake(proc) + proc = None + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + + assert client.files.download(volume_path).contents.read() == payload + dbfs_data = client.dbfs.read(path=volume_path).data + assert base64.b64decode(dbfs_data) == payload + assert ( + Path(data_dir) / "volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + ).read_bytes() == payload + assert not legacy_file.exists() + + client.volumes.delete(name=full_volume_name) + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_deleted_legacy_volume_files_stay_deleted_after_restart(): + """An upgrade must not import orphaned files when a volume name is reused.""" + data_dir = tempfile.mkdtemp(prefix="minilake_orphaned_volume_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"orphan_{uuid.uuid4().hex[:8]}" + schema_name = "orphan_schema" + volume_name = "reused_volume" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + volume_path = f"/Volumes/{full_volume_name.replace('.', '/')}/old.txt" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + # Simulate an old release: volume deletion removed its UC directory but + # left Files/DBFS content under the legacy dbfs/Volumes path. + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "old.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(b"content from deleted volume") + client.volumes.delete(name=full_volume_name) + migration_marker = Path(data_dir) / "volumes" / ".legacy_volume_migration_complete" + migration_marker.unlink() + + _stop_minilake(proc) + proc = None + # First upgraded startup must not migrate a volume that had already been deleted. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + _stop_minilake(proc) + proc = None + # A later restart must not reinterpret the new volume as a pre-upgrade one. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + assert legacy_file.read_bytes() == b"content from deleted volume" + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True)