From 66ca867b6830cd944568561613e8356016f10267 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 09:59:33 -0400 Subject: [PATCH 1/6] Fix Files and DBFS paths for UC volumes --- src/minilake/services/dbfs.py | 13 +++++-- src/minilake/services/files.py | 11 ++++-- src/minilake/services/volume_paths.py | 55 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 src/minilake/services/volume_paths.py diff --git a/src/minilake/services/dbfs.py b/src/minilake/services/dbfs.py index ff9443d..a173a73 100644 --- a/src/minilake/services/dbfs.py +++ b/src/minilake/services/dbfs.py @@ -1,9 +1,9 @@ """DBFS (Databricks File System) API endpoints — real file-backed storage. -Files live under `settings.data_dir / "dbfs"`. Chunked upload (create/add-block/ -close) is tracked with a small in-memory handle table, matching the real API's -session-based upload model — the handle itself is ephemeral, only the final -file on disk persists. +Ordinary files live under `settings.data_dir / "dbfs"`; `/Volumes/...` URIs +resolve to their Unity Catalog volume directory. Chunked upload (create/add- +block/close) uses an in-memory handle table, matching the real API's session- +based upload model — only the final file persists. """ import base64 @@ -29,6 +29,7 @@ PutDbfsRequest, ReadFileResponse, ) +from minilake.services.volume_paths import resolve_volume_path logger = logging.getLogger(__name__) @@ -56,6 +57,10 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) + volume_path = resolve_volume_path(normalized) + if volume_path is not None: + return volume_path + root = _dbfs_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: diff --git a/src/minilake/services/files.py b/src/minilake/services/files.py index de8c450..3a46fcf 100644 --- a/src/minilake/services/files.py +++ b/src/minilake/services/files.py @@ -1,8 +1,8 @@ """Files API endpoints — real file-backed storage (modern DBFS alternative). -Shares the same on-disk root as DBFS (`settings.data_dir / "dbfs"`), since both -APIs address the same conceptual filesystem in real Databricks. Unlike DBFS's -JSON+base64 shape, this API moves raw bytes directly in the HTTP body. +Ordinary paths share DBFS's on-disk root; `/Volumes/...` paths resolve to their +Unity Catalog volume directory. Unlike DBFS's JSON+base64 shape, this API moves +raw bytes directly in the HTTP body. """ import logging @@ -14,6 +14,7 @@ from minilake.config import settings from minilake.errors import DatabricksError +from minilake.services.volume_paths import resolve_volume_path logger = logging.getLogger(__name__) @@ -36,6 +37,10 @@ def _normalize(path: str) -> str: def _resolve(path: str) -> Path: normalized = _normalize(path) + volume_path = resolve_volume_path(normalized) + if volume_path is not None: + return volume_path + root = _files_root() resolved = (root / normalized.lstrip("/")).resolve() if resolved != root and root not in resolved.parents: diff --git a/src/minilake/services/volume_paths.py b/src/minilake/services/volume_paths.py new file mode 100644 index 0000000..673bbb7 --- /dev/null +++ b/src/minilake/services/volume_paths.py @@ -0,0 +1,55 @@ +"""Resolve Unity Catalog volume URIs to their on-disk storage.""" + +from pathlib import Path + +from minilake.config import settings +from minilake.errors import DatabricksError + +_VOLUMES_PREFIX = "/Volumes" + + +def resolve_volume_path(path: str) -> Path | None: + """Resolve a `/Volumes/...` URI to its Unity Catalog volume directory. + + Returns ``None`` for paths outside the volume namespace so Files and DBFS can + use their ordinary shared root. Volume URIs must name an existing volume; + this prevents deleted or unknown volumes from becoming generic DBFS paths. + """ + normalized = "/" + path.lstrip("/") + if normalized != _VOLUMES_PREFIX and not normalized.startswith(_VOLUMES_PREFIX + "/"): + return None + + suffix = normalized[len(_VOLUMES_PREFIX) :] + if suffix.startswith("/"): + suffix = suffix[1:] + if suffix.endswith("/"): + suffix = suffix[:-1] + parts = suffix.split("/") + if len(parts) < 3 or any(part in {"", ".", ".."} for part in parts[:3]): + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400) + + catalog_name, schema_name, volume_name = parts[:3] + volumes_root = (settings.data_dir / "volumes").resolve() + volume_root = volumes_root.joinpath(catalog_name, schema_name, volume_name) + + # The catalog/schema/volume components identify the volume itself; do not + # follow a symlink there to a different volume or outside the volume tree. + current = volumes_root + for part in (catalog_name, schema_name, volume_name): + current = current / part + if current.is_symlink(): + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid volume path", status_code=400) + + volume_root = volume_root.resolve() + if volumes_root not in volume_root.parents or not volume_root.is_dir(): + full_name = f"{catalog_name}.{schema_name}.{volume_name}" + raise DatabricksError( + error_code="RESOURCE_DOES_NOT_EXIST", + message=f"Volume '{full_name}' not found", + status_code=404, + ) + + target = volume_root.joinpath(*parts[3:]).resolve() + if target != volume_root and volume_root not in target.parents: + raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) + return target From b87c66826c5168d879f34cba309a15250984d1c2 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 10:07:30 -0400 Subject: [PATCH 2/6] Migrate existing Files data into UC volumes --- src/minilake/app.py | 9 +++ src/minilake/services/volume_paths.py | 79 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/src/minilake/app.py b/src/minilake/app.py index 68b8471..9601c15 100644 --- a/src/minilake/app.py +++ b/src/minilake/app.py @@ -151,6 +151,15 @@ async def lifespan(app: FastAPI): except Exception as e: logger.warning(f"Failed to re-attach catalog {catalog_name} on restore: {e}") + # Older releases stored `/Volumes/...` uploads under dbfs/Volumes, + # separate from the directories Unity Catalog creates in volumes/. + # Migrate those files before API routes or reset hooks can remove them. + from minilake.services.volume_paths import migrate_legacy_volume_data + + migrated_volumes = migrate_legacy_volume_data() + if migrated_volumes: + logger.info(f"Migrated legacy Files/DBFS data for {migrated_volumes} Unity Catalog volumes") + # Best-effort, non-blocking: pre-pull the Spark image used for real Job # execution so the first real job run doesn't pay the cold-start cost. asyncio.create_task(asyncio.to_thread(prewarm_spark_image)) diff --git a/src/minilake/services/volume_paths.py b/src/minilake/services/volume_paths.py index 673bbb7..a764a23 100644 --- a/src/minilake/services/volume_paths.py +++ b/src/minilake/services/volume_paths.py @@ -1,5 +1,6 @@ """Resolve Unity Catalog volume URIs to their on-disk storage.""" +import shutil from pathlib import Path from minilake.config import settings @@ -8,6 +9,80 @@ _VOLUMES_PREFIX = "/Volumes" +def _merge_legacy_volume(legacy_path: Path, volume_root: Path) -> bool: + """Move legacy Files/DBFS volume data into its UC volume directory. + + Preflight the entire tree before moving anything. A conflicting path leaves + both copies untouched and stops startup with a message for the operator. + """ + legacy_dbfs_root = settings.data_dir / "dbfs" + current = legacy_dbfs_root + if current.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}") + for part in legacy_path.relative_to(legacy_dbfs_root).parts: + current = current / part + if current.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data through symlink {current}") + + if not legacy_path.exists(): + return False + if not legacy_path.is_dir(): + raise RuntimeError(f"Cannot migrate legacy volume data at {legacy_path}: expected a directory") + + def check(source: Path, destination: Path) -> None: + for child in source.iterdir(): + target = destination / child.name + if child.is_symlink(): + raise RuntimeError(f"Cannot migrate legacy volume data: symlink at {child}") + if target.exists() or target.is_symlink(): + if not child.is_dir() or not target.is_dir() or target.is_symlink(): + raise RuntimeError( + f"Cannot migrate legacy volume data: both legacy and current paths contain {target}" + ) + if child.is_dir(): + check(child, target) + + check(legacy_path, volume_root) + + def move_contents(source: Path, destination: Path) -> None: + for child in source.iterdir(): + target = destination / child.name + if child.is_dir(): + target.mkdir(exist_ok=True) + move_contents(child, target) + child.rmdir() + else: + shutil.move(str(child), str(target)) + + move_contents(legacy_path, volume_root) + legacy_path.rmdir() + return True + + +def migrate_legacy_volume_data() -> int: + """Migrate old `/Volumes` uploads for volumes with existing UC directories.""" + volumes_root = (settings.data_dir / "volumes").resolve() + if not volumes_root.is_dir(): + return 0 + + migrated = 0 + for catalog_path in volumes_root.iterdir(): + if catalog_path.is_symlink() or not catalog_path.is_dir(): + continue + for schema_path in catalog_path.iterdir(): + if schema_path.is_symlink() or not schema_path.is_dir(): + continue + for volume_root in schema_path.iterdir(): + if volume_root.is_symlink() or not volume_root.is_dir(): + continue + legacy_path = ( + settings.data_dir / "dbfs" / "Volumes" / catalog_path.name / schema_path.name / volume_root.name + ) + if _merge_legacy_volume(legacy_path, volume_root): + migrated += 1 + return migrated + + def resolve_volume_path(path: str) -> Path | None: """Resolve a `/Volumes/...` URI to its Unity Catalog volume directory. @@ -52,4 +127,8 @@ def resolve_volume_path(path: str) -> Path | None: target = volume_root.joinpath(*parts[3:]).resolve() if target != volume_root and volume_root not in target.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) + _merge_legacy_volume( + settings.data_dir / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name, + volume_root, + ) return target From 499ebc274bcde8ac75e7da0b777cbb29101ba0bc Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 10:14:44 -0400 Subject: [PATCH 3/6] test(volumes): cover legacy data migration on restart --- tests/test_persistence.py | 58 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 31a40ed..51722b5 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,6 +10,7 @@ state survived — using the real databricks-sdk client throughout. """ +import base64 import os import shutil import socket @@ -17,10 +18,13 @@ import tempfile import time import urllib.request +import uuid +from pathlib import Path import pytest from databricks.sdk import WorkspaceClient -from databricks.sdk.service.catalog import DataSourceFormat, TableType +from databricks.sdk.errors import DatabricksError +from databricks.sdk.service.catalog import DataSourceFormat, TableType, VolumeType from databricks.sdk.service.sql import ColumnInfo @@ -128,3 +132,55 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_legacy_volume_files_migrate_on_restart(): + """Files created under the pre-UC volume path remain readable after upgrade.""" + data_dir = tempfile.mkdtemp(prefix="minilake_volume_migration_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"migration_{uuid.uuid4().hex[:8]}" + schema_name = "migration_schema" + volume_name = "migration_volume" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + payload = b"legacy volume data survives upgrade\n" + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(payload) + volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + + _stop_minilake(proc) + proc = None + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + + assert client.files.download(volume_path).contents.read() == payload + dbfs_data = client.dbfs.read(path=volume_path).data + assert base64.b64decode(dbfs_data) == payload + assert ( + Path(data_dir) / "volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + ).read_bytes() == payload + assert not legacy_file.exists() + + client.volumes.delete(name=full_volume_name) + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) From 579c21839b84c8896880c27dee28c7d89c32aece Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 10:31:00 -0400 Subject: [PATCH 4/6] fix(volumes): avoid reviving deleted volume files --- src/minilake/services/volume_paths.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/minilake/services/volume_paths.py b/src/minilake/services/volume_paths.py index a764a23..7c1f8ff 100644 --- a/src/minilake/services/volume_paths.py +++ b/src/minilake/services/volume_paths.py @@ -127,8 +127,4 @@ def resolve_volume_path(path: str) -> Path | None: target = volume_root.joinpath(*parts[3:]).resolve() if target != volume_root and volume_root not in target.parents: raise DatabricksError(error_code="INVALID_REQUEST", message="Invalid path", status_code=400) - _merge_legacy_volume( - settings.data_dir / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name, - volume_root, - ) return target From 8561588e23adb97eed5a6bb00fdf04dd6413ede4 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 10:45:54 -0400 Subject: [PATCH 5/6] fix(volumes): make legacy migration one-time --- src/minilake/services/volume_paths.py | 11 ++++- tests/test_persistence.py | 62 +++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/minilake/services/volume_paths.py b/src/minilake/services/volume_paths.py index 7c1f8ff..be24eb7 100644 --- a/src/minilake/services/volume_paths.py +++ b/src/minilake/services/volume_paths.py @@ -60,11 +60,13 @@ def move_contents(source: Path, destination: Path) -> None: def migrate_legacy_volume_data() -> int: - """Migrate old `/Volumes` uploads for volumes with existing UC directories.""" + """Run the one-time migration of old `/Volumes` uploads into UC directories.""" volumes_root = (settings.data_dir / "volumes").resolve() - if not volumes_root.is_dir(): + migration_marker = volumes_root / ".legacy_volume_migration_complete" + if migration_marker.exists(): return 0 + volumes_root.mkdir(parents=True, exist_ok=True) migrated = 0 for catalog_path in volumes_root.iterdir(): if catalog_path.is_symlink() or not catalog_path.is_dir(): @@ -80,6 +82,11 @@ def migrate_legacy_volume_data() -> int: ) if _merge_legacy_volume(legacy_path, volume_root): migrated += 1 + + # Keep later volume creations from adopting files left by volumes deleted + # before this upgrade. The marker lives inside volumes/ so a full reset + # removes it along with the migrated storage. + migration_marker.touch() return migrated diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 51722b5..d02bf3e 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -159,6 +159,8 @@ def test_legacy_volume_files_migrate_on_restart(): legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt" legacy_file.parent.mkdir(parents=True) legacy_file.write_bytes(payload) + # The process running before an upgrade cannot have written the new marker. + (Path(data_dir) / "volumes" / ".legacy_volume_migration_complete").unlink() volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt" full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" @@ -184,3 +186,63 @@ def test_legacy_volume_files_migrate_on_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_deleted_legacy_volume_files_stay_deleted_after_restart(): + """An upgrade must not import orphaned files when a volume name is reused.""" + data_dir = tempfile.mkdtemp(prefix="minilake_orphaned_volume_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"orphan_{uuid.uuid4().hex[:8]}" + schema_name = "orphan_schema" + volume_name = "reused_volume" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + volume_path = f"/Volumes/{full_volume_name.replace('.', '/')}/old.txt" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + # Simulate an old release: volume deletion removed its UC directory but + # left Files/DBFS content under the legacy dbfs/Volumes path. + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "old.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(b"content from deleted volume") + client.volumes.delete(name=full_volume_name) + migration_marker = Path(data_dir) / "volumes" / ".legacy_volume_migration_complete" + migration_marker.unlink() + + _stop_minilake(proc) + proc = None + # First upgraded startup must not migrate a volume that had already been deleted. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + _stop_minilake(proc) + proc = None + # A later restart must not reinterpret the new volume as a pre-upgrade one. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + assert legacy_file.read_bytes() == b"content from deleted volume" + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) From b09a3823bd92cb8d7ca1eea835cce4c13cbe0fb6 Mon Sep 17 00:00:00 2001 From: Odos Matthews Date: Mon, 26 Oct 2026 16:01:25 -0400 Subject: [PATCH 6/6] refactor: keep this fix independently mergeable with other open PRs --- tests/persistence_helpers.py | 71 +++++++++++ tests/test_persistence.py | 120 +----------------- tests/test_volume_migration_persistence.py | 137 +++++++++++++++++++++ 3 files changed, 209 insertions(+), 119 deletions(-) create mode 100644 tests/persistence_helpers.py create mode 100644 tests/test_volume_migration_persistence.py diff --git a/tests/persistence_helpers.py b/tests/persistence_helpers.py new file mode 100644 index 0000000..da13d01 --- /dev/null +++ b/tests/persistence_helpers.py @@ -0,0 +1,71 @@ +"""Process helpers for isolated persistence regressions.""" + +import os +import socket +import subprocess +import time +import urllib.request + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _drain(proc: subprocess.Popen) -> str: + """Whatever the subprocess printed, without blocking on a live process.""" + if proc.poll() is None: + return "(process still running)" + out, err = proc.communicate(timeout=5) + return (out or b"").decode(errors="replace") + (err or b"").decode(errors="replace") + + +def _wait_healthy(proc: subprocess.Popen, url: str, timeout: float = 60.0) -> None: + """Poll until minilake answers, failing early and loudly if it died instead. + + Checking `proc.poll()` matters: without it a crashed server is indistinguishable from a + slow one, and the test reports a timeout while the actual traceback sits unread in a + pipe. CI is also several times slower to start than a laptop, hence the generous + timeout — this waits on a real process spawn, not a request. + """ + deadline = time.time() + timeout + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"minilake exited with code {proc.returncode} before serving:\n{_drain(proc)}") + try: + if urllib.request.urlopen(f"{url}/_minilake/health", timeout=1).status == 200: + return + except Exception as e: + last_error = e + time.sleep(0.3) + raise RuntimeError(f"minilake at {url} did not become healthy in time: {last_error}\n{_drain(proc)}") + + +def _start_minilake(port: int, data_dir: str, *, cwd: str | None = None) -> subprocess.Popen: + env = os.environ.copy() + env["MINILAKE_DATA_DIR"] = data_dir + env["MINILAKE_PERSIST"] = "1" + proc = subprocess.Popen( + ["minilake", "--port", str(port), "--host", "127.0.0.1"], + env=env, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + _wait_healthy(proc, f"http://127.0.0.1:{port}") + except Exception: + _stop_minilake(proc) + raise + return proc + + +def _stop_minilake(proc: subprocess.Popen) -> None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index d02bf3e..31a40ed 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -10,7 +10,6 @@ state survived — using the real databricks-sdk client throughout. """ -import base64 import os import shutil import socket @@ -18,13 +17,10 @@ import tempfile import time import urllib.request -import uuid -from pathlib import Path import pytest from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import DatabricksError -from databricks.sdk.service.catalog import DataSourceFormat, TableType, VolumeType +from databricks.sdk.service.catalog import DataSourceFormat, TableType from databricks.sdk.service.sql import ColumnInfo @@ -132,117 +128,3 @@ def test_persist_state_survives_restart(): if proc is not None: _stop_minilake(proc) shutil.rmtree(data_dir, ignore_errors=True) - - -@pytest.mark.serial -def test_legacy_volume_files_migrate_on_restart(): - """Files created under the pre-UC volume path remain readable after upgrade.""" - data_dir = tempfile.mkdtemp(prefix="minilake_volume_migration_") - port = _free_port() - proc = None - try: - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - catalog_name = f"migration_{uuid.uuid4().hex[:8]}" - schema_name = "migration_schema" - volume_name = "migration_volume" - client.catalogs.create(name=catalog_name) - client.schemas.create(name=schema_name, catalog_name=catalog_name) - client.volumes.create( - name=volume_name, - catalog_name=catalog_name, - schema_name=schema_name, - volume_type=VolumeType.MANAGED, - ) - - payload = b"legacy volume data survives upgrade\n" - legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt" - legacy_file.parent.mkdir(parents=True) - legacy_file.write_bytes(payload) - # The process running before an upgrade cannot have written the new marker. - (Path(data_dir) / "volumes" / ".legacy_volume_migration_complete").unlink() - volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt" - full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" - - _stop_minilake(proc) - proc = None - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - - assert client.files.download(volume_path).contents.read() == payload - dbfs_data = client.dbfs.read(path=volume_path).data - assert base64.b64decode(dbfs_data) == payload - assert ( - Path(data_dir) / "volumes" / catalog_name / schema_name / volume_name / "legacy.txt" - ).read_bytes() == payload - assert not legacy_file.exists() - - client.volumes.delete(name=full_volume_name) - with pytest.raises(DatabricksError): - client.files.download(volume_path) - with pytest.raises(DatabricksError): - client.dbfs.read(path=volume_path) - finally: - if proc is not None: - _stop_minilake(proc) - shutil.rmtree(data_dir, ignore_errors=True) - - -@pytest.mark.serial -def test_deleted_legacy_volume_files_stay_deleted_after_restart(): - """An upgrade must not import orphaned files when a volume name is reused.""" - data_dir = tempfile.mkdtemp(prefix="minilake_orphaned_volume_") - port = _free_port() - proc = None - try: - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - catalog_name = f"orphan_{uuid.uuid4().hex[:8]}" - schema_name = "orphan_schema" - volume_name = "reused_volume" - full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" - volume_path = f"/Volumes/{full_volume_name.replace('.', '/')}/old.txt" - client.catalogs.create(name=catalog_name) - client.schemas.create(name=schema_name, catalog_name=catalog_name) - client.volumes.create( - name=volume_name, - catalog_name=catalog_name, - schema_name=schema_name, - volume_type=VolumeType.MANAGED, - ) - - # Simulate an old release: volume deletion removed its UC directory but - # left Files/DBFS content under the legacy dbfs/Volumes path. - legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "old.txt" - legacy_file.parent.mkdir(parents=True) - legacy_file.write_bytes(b"content from deleted volume") - client.volumes.delete(name=full_volume_name) - migration_marker = Path(data_dir) / "volumes" / ".legacy_volume_migration_complete" - migration_marker.unlink() - - _stop_minilake(proc) - proc = None - # First upgraded startup must not migrate a volume that had already been deleted. - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - client.volumes.create( - name=volume_name, - catalog_name=catalog_name, - schema_name=schema_name, - volume_type=VolumeType.MANAGED, - ) - - _stop_minilake(proc) - proc = None - # A later restart must not reinterpret the new volume as a pre-upgrade one. - proc = _start_minilake(port, data_dir) - client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") - with pytest.raises(DatabricksError): - client.files.download(volume_path) - with pytest.raises(DatabricksError): - client.dbfs.read(path=volume_path) - assert legacy_file.read_bytes() == b"content from deleted volume" - finally: - if proc is not None: - _stop_minilake(proc) - shutil.rmtree(data_dir, ignore_errors=True) diff --git a/tests/test_volume_migration_persistence.py b/tests/test_volume_migration_persistence.py new file mode 100644 index 0000000..a45edc1 --- /dev/null +++ b/tests/test_volume_migration_persistence.py @@ -0,0 +1,137 @@ +"""Persistence round-trip test. + +Regression test for a real bug found while auditing the project: the +MINILAKE_PERSIST env var and persistence.py's save_state/load_state existed, +but were never actually invoked from app.py's lifespan — so setting +MINILAKE_PERSIST=1 silently did nothing. This test starts an isolated +minilake subprocess (not the shared session-scoped server used by other +tests, since this specifically needs a real process restart), creates real +state, restarts the process against the same data dir, and verifies the +state survived — using the real databricks-sdk client throughout. +""" + +import base64 +import shutil +import tempfile +import uuid +from pathlib import Path + +import pytest +from databricks.sdk import WorkspaceClient +from databricks.sdk.errors import DatabricksError +from databricks.sdk.service.catalog import VolumeType +from persistence_helpers import _free_port, _start_minilake, _stop_minilake + + +@pytest.mark.serial +def test_legacy_volume_files_migrate_on_restart(): + """Files created under the pre-UC volume path remain readable after upgrade.""" + data_dir = tempfile.mkdtemp(prefix="minilake_volume_migration_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"migration_{uuid.uuid4().hex[:8]}" + schema_name = "migration_schema" + volume_name = "migration_volume" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + payload = b"legacy volume data survives upgrade\n" + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(payload) + # The process running before an upgrade cannot have written the new marker. + (Path(data_dir) / "volumes" / ".legacy_volume_migration_complete").unlink() + volume_path = f"/Volumes/{catalog_name}/{schema_name}/{volume_name}/legacy.txt" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + + _stop_minilake(proc) + proc = None + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + + assert client.files.download(volume_path).contents.read() == payload + dbfs_data = client.dbfs.read(path=volume_path).data + assert base64.b64decode(dbfs_data) == payload + assert ( + Path(data_dir) / "volumes" / catalog_name / schema_name / volume_name / "legacy.txt" + ).read_bytes() == payload + assert not legacy_file.exists() + + client.volumes.delete(name=full_volume_name) + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True) + + +@pytest.mark.serial +def test_deleted_legacy_volume_files_stay_deleted_after_restart(): + """An upgrade must not import orphaned files when a volume name is reused.""" + data_dir = tempfile.mkdtemp(prefix="minilake_orphaned_volume_") + port = _free_port() + proc = None + try: + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + catalog_name = f"orphan_{uuid.uuid4().hex[:8]}" + schema_name = "orphan_schema" + volume_name = "reused_volume" + full_volume_name = f"{catalog_name}.{schema_name}.{volume_name}" + volume_path = f"/Volumes/{full_volume_name.replace('.', '/')}/old.txt" + client.catalogs.create(name=catalog_name) + client.schemas.create(name=schema_name, catalog_name=catalog_name) + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + # Simulate an old release: volume deletion removed its UC directory but + # left Files/DBFS content under the legacy dbfs/Volumes path. + legacy_file = Path(data_dir) / "dbfs" / "Volumes" / catalog_name / schema_name / volume_name / "old.txt" + legacy_file.parent.mkdir(parents=True) + legacy_file.write_bytes(b"content from deleted volume") + client.volumes.delete(name=full_volume_name) + migration_marker = Path(data_dir) / "volumes" / ".legacy_volume_migration_complete" + migration_marker.unlink() + + _stop_minilake(proc) + proc = None + # First upgraded startup must not migrate a volume that had already been deleted. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + client.volumes.create( + name=volume_name, + catalog_name=catalog_name, + schema_name=schema_name, + volume_type=VolumeType.MANAGED, + ) + + _stop_minilake(proc) + proc = None + # A later restart must not reinterpret the new volume as a pre-upgrade one. + proc = _start_minilake(port, data_dir) + client = WorkspaceClient(host=f"http://127.0.0.1:{port}", token="dev", auth_type="pat") + with pytest.raises(DatabricksError): + client.files.download(volume_path) + with pytest.raises(DatabricksError): + client.dbfs.read(path=volume_path) + assert legacy_file.read_bytes() == b"content from deleted volume" + finally: + if proc is not None: + _stop_minilake(proc) + shutil.rmtree(data_dir, ignore_errors=True)