ekkolon · ekkolon · Mar 4, 2026 · Mar 3, 2026 · Mar 3, 2026 · Mar 3, 2026
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,35 @@
+# Rust build artifacts — never send to Docker daemon
+target/
+**/*.rs.bk
+
+# Git internals
+.git/
+.gitignore
+.gitattributes
+
+# Development tooling
+.vscode/
+.idea/
+*.code-workspace
+
+# CI/CD artifacts
+.github/
+.gitlab-ci.yml
+
+# Documentation (not needed in image)
+docs/
+*.md
+!README.md
+
+# Test fixtures (not needed in production image)
+**/tests/fixtures/
+**/tests/snapshots/
+
+# Local dev overrides
+docker-compose.override.yml
+.env
+.env.*
+
+# Temporary files
+*.tmp
+*.log
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,105 @@
+# =============================================================================
+# minikv - Multi-Stage Dockerfile
+#
+# Stages:
+#   1. chef      - installs cargo-chef for dependency caching
+#   2. planner   - computes the dependency recipe
+#   3. builder   - compiles dependencies (cached), then the binary
+#   4. runtime   - minimal distroless image with only the binary
+#
+# BLAKE3 hashing requires no external C libs - fully pure Rust.
+# rusty-leveldb is pure Rust - no libleveldb.so dependency.
+# Final image has zero shell, zero package manager, zero attack surface.
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# Stage 1: chef
+# Installs cargo-chef for layer-cached dependency compilation.
+# -----------------------------------------------------------------------------
+FROM rust:1.88-slim-bookworm AS chef
+
+# Install cargo-chef for reproducible dependency caching
+RUN cargo install cargo-chef --locked
+
+WORKDIR /build
+
+# -----------------------------------------------------------------------------
+# Stage 2: planner
+# Computes the dependency recipe from Cargo.toml + Cargo.lock.
+# This layer only re-runs when dependencies change.
+# -----------------------------------------------------------------------------
+FROM chef AS planner
+
+COPY Cargo.toml Cargo.lock ./
+COPY minikv ./minikv
+COPY minikv-core ./minikv-core
+
+RUN cargo chef prepare --recipe-path recipe.json
+
+# -----------------------------------------------------------------------------
+# Stage 3: builder
+# Compiles dependencies first (cached layer), then the application.
+# -----------------------------------------------------------------------------
+FROM chef AS builder
+
+# Build-time dependencies only - no runtime C libs needed.
+# rusty-leveldb and blake3 are both pure Rust.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=planner /build/recipe.json recipe.json
+
+# Compile dependencies - this layer is cached unless Cargo.toml/lock changes
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Copy full source and compile the application binary
+COPY Cargo.toml Cargo.lock ./
+COPY minikv ./minikv
+COPY minikv-core ./minikv-core
+COPY config ./config
+
+# Build release binary
+# RUSTFLAGS for correctness: deny unused, warn on unsafe
+ENV RUSTFLAGS="-D warnings -D unsafe_code"
+
+RUN cargo build --release --locked \
+    && strip target/release/minikv
+
+# -----------------------------------------------------------------------------
+# Stage 4: Distroless image runtime
+# -----------------------------------------------------------------------------
+FROM gcr.io/distroless/cc-debian12:nonroot AS runtime
+
+# Metadata
+LABEL org.opencontainers.image.title="minikv"
+LABEL org.opencontainers.image.description="Tiny distributed key value store in pure Rust"
+LABEL org.opencontainers.image.source="https://github.com/ekkolon/minikv"
+LABEL org.opencontainers.image.licenses="MIT"
+
+# Copy the stripped binary from builder
+COPY --from=builder /build/target/release/minikv /usr/local/bin/minikv
+
+# Copy nginx config (used by operators, not the binary itself)
+COPY --from=builder /build/config/nginx-volume.conf /etc/minikv/nginx-volume.conf
+
+# Data directory for LevelDB - must be mounted as a volume in production
+# The nonroot user (uid=65532) must own this path
+WORKDIR /data
+
+# Expose the default server port
+# Override with: minikv server --port <port>
+EXPOSE 3000
+
+# Run as nonroot (distroless nonroot image sets this by default)
+# UID 65532 - no shell, no sudo, no privilege escalation possible
+USER nonroot
+
+# Default entrypoint - subcommand must be passed at runtime:
+#   docker run minikv server --port 3000 --db /data --volumes ...
+#   docker run minikv rebuild ...
+#   docker run minikv rebalance ...
+ENTRYPOINT ["/usr/local/bin/minikv"]
+
+# No default CMD - operator must provide subcommand explicitly.
+# This prevents accidental runs with wrong configuration.
diff --git a/Dockerfile.volume b/Dockerfile.volume
@@ -0,0 +1,18 @@
+# =============================================================================
+# nginx volume node for minikv
+#
+# Uses alpine base so nginx and nginx-mod-http-dav-ext are installed from
+# the same apk repo and are guaranteed version-matched.
+#
+# Package name on Alpine 3.19: nginx-mod-http-dav-ext (NOT nginx-mod-dav-ext)
+# =============================================================================
+FROM alpine:3.19
+
+RUN apk add --no-cache nginx nginx-mod-http-dav-ext
+
+RUN mkdir -p /data /tmp/nginx-client-body /tmp/nginx-volume \
+    && chown -R nginx:nginx /data /tmp/nginx-client-body /tmp/nginx-volume
+
+EXPOSE 8080
+
+CMD ["nginx", "-c", "/etc/nginx/nginx.conf", "-g", "daemon off;"]
diff --git a/README.md b/README.md
@@ -1,8 +1,167 @@
 # minikv
 
-Minimal, S3-compatible distributed key-value store.
+Minimal, S3-compatible distributed key-value store. Keys and metadata are stored in LevelDB; values (object bytes) live on nginx WebDAV volume servers.
+
+## Architecture
+
+```txt
+Client
+  │
+  ▼
+frontend nginx (port 8080)          <== X-Accel-Redirect proxy
+  │  proxy_pass =>
+  ▼
+minikv coordinator (port 3000)    <== metadata, routing, replication
+  │  replicates to =>
+  ├── volume1 nginx (port 8080)   <== nginx DAV object storage
+  ├── volume2 nginx (port 8080)
+  └── volume3 nginx (port 8080)
+```
+
+**GET/HEAD flow:** The coordinator looks up the key in LevelDB, probes volume servers to find a live replica, then returns `X-Accel-Redirect` to the frontend nginx. nginx fetches the object body directly from the volume server and streams it to the client - the coordinator is never in the data path. Response headers (`Content-Type`, `Content-Blake3`, `Key-Balance`) come from coordinator metadata.
+
+**PUT flow:** The coordinator writes a soft-delete sentinel to LevelDB, replicates the object body to all replica volumes, optionally computes a BLAKE3 checksum, then marks the key as fully present.
+
+## Hashing
+
+This project uses **BLAKE3** for all content-addressing and volume selection.
+
+> ⚠️ The hash function used for `key_to_path` and `key_to_volume` determines the physical layout of all stored data. Changing it after data is written is a **breaking change** requiring a full rebalance.
+
+## Record Wire Format
+
+Each LevelDB value encodes object metadata as a compact byte string:
+
+```txt
+[DELETED][HASH<64hex>][TYPE<mimetype>|]<vol1>,<vol2>,...
+```
+
+- `DELETED` - present if soft-deleted (UNLINK has been called)
+- `HASH<64hex>` - BLAKE3-256 hex digest, present when `--checksum` is enabled
+- `TYPE<mimetype>|` - MIME type terminated by `|`, present when `Content-Type` was supplied on PUT
+- Remaining bytes - comma-separated volume addresses (`host:port/svXX`)
+
+## HTTP API
+
+| Method      | Path                 | Description                                                              |
+| ----------- | -------------------- | ------------------------------------------------------------------------ |
+| `PUT`       | `/<key>`             | Store an object. Supply `Content-Type` header for correct MIME metadata. |
+| `GET`       | `/<key>`             | Retrieve an object (via X-Accel-Redirect or 302).                        |
+| `HEAD`      | `/<key>`             | Returns metadata headers without body.                                   |
+| `DELETE`    | `/<key>`             | Hard delete. Requires prior `UNLINK` when `--protect` is set.            |
+| `UNLINK`    | `/<key>`             | Soft delete.                                                             |
+| `REBALANCE` | `/<key>`             | Move a single key to its ideal volume set.                               |
+| `GET`       | `/<prefix>?list`     | List active keys under prefix.                                           |
+| `GET`       | `/<prefix>?unlinked` | List soft-deleted keys under prefix.                                     |
+| `POST`      | `/<key>?uploads`     | Initiate S3-style multipart upload.                                      |
+| `POST`      | `/<key>?uploadId=X`  | Complete multipart upload.                                               |
+| `POST`      | `/<key>?delete`      | Batch delete.                                                            |
+
+### Response Headers
+
+| Header           | Present on | Description                                               |
+| ---------------- | ---------- | --------------------------------------------------------- |
+| `Content-Type`   | GET, HEAD  | MIME type from stored metadata                            |
+| `Content-Blake3` | GET, HEAD  | BLAKE3-256 hex digest of object body                      |
+| `Key-Balance`    | GET, HEAD  | `balanced` or `unbalanced`                                |
+| `Key-Volumes`    | GET, HEAD  | Comma-separated list of volume addresses holding replicas |
+
+## Running with Docker Compose
+
+```bash
+docker compose up --build
+```
+
+All services start in dependency order: volume nodes => coordinator => frontend nginx.
+
+The only externally exposed port is **8080** (frontend nginx). Volume nodes and the coordinator are internal to the Docker network.
+
+### PUT an object
+
+```bash
+curl -X PUT -H "Content-Type: image/png" \
+     --data-binary @photo \
+     http://localhost:8080/mybucket/photo
+```
+
+> Always supply `Content-Type` on PUT. It is stored in LevelDB and returned on all subsequent GET/HEAD requests. Objects stored without `Content-Type` will be served as `application/octet-stream`.
+
+### GET an object
+
+```bash
+curl http://localhost:8080/mybucket/photo -o photo
+```
+
+### Inspect metadata
+
+```bash
+curl -I http://localhost:8080/mybucket/photo
+```
+
+### Soft delete then hard delete
+
+```bash
+curl -X UNLINK http://localhost:8080/mybucket/photo
+curl -X DELETE http://localhost:8080/mybucket/photo
+```
+
+## CLI Reference
+
+```txt
+minikv <COMMAND>
+
+Commands:
+  server              Run the HTTP metadata coordinator
+  rebuild             Reconstruct LevelDB from volume server autoindex
+  rebalance           Move all keys to their ideal volume set
+```
+
+### server
+
+```txt
+--db <PATH>                    LevelDB directory            [env: MINIKV_DB]
+--volumes <host:port,...>      Volume server addresses      [env: MINIKV_VOLUMES]
+--replicas <N>                 Replica count (default: 3)   [env: MINIKV_REPLICAS]
+--subvolumes <N>               Shard count (default: 10)    [env: MINIKV_SUBVOLUMES]
+--voltimeout <duration>        Volume probe timeout         [env: MINIKV_VOLTIMEOUT]
+--port <N>                     Listen port (default: 3000)  [env: MINIKV_PORT]
+--public-volumes <host:port,…> External volume addresses    [env: MINIKV_PUBLIC_VOLUMES]
+--fallback <host:port>         Fallback for missing keys    [env: MINIKV_FALLBACK]
+--protect                      Require UNLINK before DELETE [env: MINIKV_PROTECT]
+--checksum                     Store BLAKE3 digest on PUT   [env: MINIKV_CHECKSUM]
+--accel-redirect               Use X-Accel-Redirect mode    [env: MINIKV_ACCEL_REDIRECT]
+-v, --verbose                  Structured debug logging     [env: MINIKV_VERBOSE]
+```
+
+All flags can be set via environment variables. Duration values accept `1s`, `500ms`.
+
+### rebuild
+
+Reconstructs LevelDB by scanning nginx autoindex JSON listings on all volume servers. This is a **destructive** operation. It clears the existing DB before scanning. Use when LevelDB is lost but volume data is intact.
+
+> `Content-Type` metadata cannot be recovered during rebuild. It exists only in LevelDB, never on volume servers. Objects will be served as `application/octet-stream` until re-PUT.
+
+### rebalance
+
+Moves all keys to their ideal volume set as computed by the current `--volumes` list. Run after adding or removing volume servers.
+
+## Consistency Model
+
+- **PUT** is atomic at the record level. The key is marked soft-deleted (in-progress sentinel) before any volume write and marked fully present only after all replicas succeed. A crash mid-write leaves a soft-deleted key that can be cleaned up manually.
+- **No read-after-write guarantee across replicas.** GET probes volumes in random order and returns the first live replica.
+- **Rebalance** clears the stored hash for the moved object. The body is not re-verified during rebalance.
+- **Soft delete (UNLINK)** removes the key from client visibility immediately. The object bytes remain on volume servers until a hard DELETE is issued.
+
+## Content-Type and X-Accel-Redirect
+
+When `--accel-redirect` is enabled the coordinator returns `X-Accel-Redirect` instead of `302`. The frontend nginx intercepts this, fetches the object body from the volume server internally, and sends it to the client. Because the body comes from nginx's internal subrequest (not the coordinator response), headers are injected via nginx variable persistence:
+
+1. Coordinator sets `X-Content-Type: image/png` on its response.
+2. nginx captures this as `$upstream_http_x_content_type` - a variable that persists across the internal redirect.
+3. The `/accel/` location suppresses the volume's `Content-Type` and replaces it with `$upstream_http_x_content_type`.
+
+In plain `302` mode, the coordinator sets `Content-Type` directly and the client receives it on the HEAD response. The GET redirect goes to the volume server which returns `application/octet-stream` - this is a known limitation of redirect mode.
 
 ## License
 
-This project is licensed under the **GNU General Public License v2 (GPLv2)**.  
-See the full license text in [`LICENSE`](./LICENSE).
+This project is licensed under the **GNU General Public License v2 (GPLv2)**. See [`LICENSE`](./LICENSE).