diff --git a/.dockerignore b/.dockerignore index 8532421..a8d9c6b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,4 @@ -# Rust build artifacts — never send to Docker daemon +# Rust build artifacts target/ **/*.rs.bk diff --git a/Dockerfile b/Dockerfile index 6340013..7b7a9aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,29 @@ -# ============================================================================= -# minikv - Multi-Stage Dockerfile +# minikv multi-stage Dockerfile # -# Stages: -# 1. chef - installs cargo-chef for dependency caching -# 2. planner - computes the dependency recipe -# 3. builder - compiles dependencies (cached), then the binary -# 4. runtime - minimal distroless image with only the binary +# Build stages: +# 1. chef - installs cargo-chef for dependency layer caching +# 2. planner - computes the dependency recipe +# 3. builder - compiles dependencies (cached) and the binary +# 4. runtime - minimal distroless image containing only the binary # -# BLAKE3 hashing requires no external C libs - fully pure Rust. -# rusty-leveldb is pure Rust - no libleveldb.so dependency. -# Final image has zero shell, zero package manager, zero attack surface. -# ============================================================================= +# BLAKE3 and rusty-leveldb are pure Rust implementations. +# No external C libraries are required at runtime. +# The final image contains no shell and no package manager. # ----------------------------------------------------------------------------- # Stage 1: chef -# Installs cargo-chef for layer-cached dependency compilation. +# Installs cargo-chef for reproducible dependency caching. # ----------------------------------------------------------------------------- FROM rust:1.88-slim-bookworm AS chef -# Install cargo-chef for reproducible dependency caching RUN cargo install cargo-chef --locked WORKDIR /build # ----------------------------------------------------------------------------- # Stage 2: planner -# Computes the dependency recipe from Cargo.toml + Cargo.lock. -# This layer only re-runs when dependencies change. +# Generates the dependency recipe from Cargo manifests. +# This layer changes only when dependency definitions change. # ----------------------------------------------------------------------------- FROM chef AS planner @@ -38,68 +35,62 @@ RUN cargo chef prepare --recipe-path recipe.json # ----------------------------------------------------------------------------- # Stage 3: builder -# Compiles dependencies first (cached layer), then the application. +# Compiles dependencies first (cached), then the application. # ----------------------------------------------------------------------------- FROM chef AS builder -# Build-time dependencies only - no runtime C libs needed. -# rusty-leveldb and blake3 are both pure Rust. +# Build-time dependencies only. RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json -# Compile dependencies - this layer is cached unless Cargo.toml/lock changes RUN cargo chef cook --release --recipe-path recipe.json -# Copy full source and compile the application binary COPY Cargo.toml Cargo.lock ./ COPY minikv ./minikv COPY minikv-core ./minikv-core COPY config ./config -# Build release binary -# RUSTFLAGS for correctness: deny unused, warn on unsafe +# Enforce strict compilation rules. ENV RUSTFLAGS="-D warnings -D unsafe_code" +# Build release binary and strip symbols. RUN cargo build --release --locked \ && strip target/release/minikv # ----------------------------------------------------------------------------- -# Stage 4: Distroless image runtime +# Stage 4: runtime +# Distroless base image containing only required runtime components. # ----------------------------------------------------------------------------- FROM gcr.io/distroless/cc-debian12:nonroot AS runtime -# Metadata LABEL org.opencontainers.image.title="minikv" -LABEL org.opencontainers.image.description="Tiny distributed key value store in pure Rust" +LABEL org.opencontainers.image.description="Distributed key value store in Rust" LABEL org.opencontainers.image.source="https://github.com/ekkolon/minikv" LABEL org.opencontainers.image.licenses="MIT" -# Copy the stripped binary from builder +# Copy compiled binary. COPY --from=builder /build/target/release/minikv /usr/local/bin/minikv -# Copy nginx config (used by operators, not the binary itself) +# Copy nginx reference configuration (for operators). COPY --from=builder /build/config/nginx-volume.conf /etc/minikv/nginx-volume.conf -# Data directory for LevelDB - must be mounted as a volume in production -# The nonroot user (uid=65532) must own this path +# Working directory for the metadata database. +# This path must be mounted as a volume in production. WORKDIR /data -# Expose the default server port -# Override with: minikv server --port +# Default server port. Can be overridden via CLI flag. EXPOSE 3000 -# Run as nonroot (distroless nonroot image sets this by default) -# UID 65532 - no shell, no sudo, no privilege escalation possible +# Run as non-root user (UID 65532). USER nonroot -# Default entrypoint - subcommand must be passed at runtime: +# Entry point. Subcommand must be provided at runtime, for example: # docker run minikv server --port 3000 --db /data --volumes ... # docker run minikv rebuild ... # docker run minikv rebalance ... ENTRYPOINT ["/usr/local/bin/minikv"] -# No default CMD - operator must provide subcommand explicitly. -# This prevents accidental runs with wrong configuration. \ No newline at end of file +# No default CMD. A subcommand must be specified explicitly. \ No newline at end of file diff --git a/config/nginx-frontend.conf b/config/nginx-frontend.conf index dc3ca75..b829c00 100644 --- a/config/nginx-frontend.conf +++ b/config/nginx-frontend.conf @@ -1,16 +1,17 @@ -# ============================================================================= # X-Accel-Redirect reverse proxy for minikv # -# DNS RESOLUTION NOTE: -# nginx resolves upstream hostnames at *startup* by default. If the upstream -# (coordinator, volume servers) is not yet in DNS, nginx refuses to start. +# DNS resolution # -# To overcome this, we use `resolver` + a variable for every upstream. -# When the upstream is stored in a variable, nginx defers DNS resolution -# to *request time*, so startup succeeds even if backends aren't running yet. +# By default, nginx resolves upstream hostnames at startup. If an upstream +# service (coordinator or volume server) is not yet resolvable, nginx fails +# to start. # -# Docker's internal DNS resolver is always at 127.0.0.11. -# ============================================================================= +# To avoid this, upstreams are stored in variables and a `resolver` is +# configured. When `proxy_pass` references a variable, DNS resolution +# happens at request time instead of startup time. +# +# In Docker environments, the internal DNS resolver is available at +# 127.0.0.11. worker_processes auto; error_log /dev/stderr warn; @@ -28,34 +29,30 @@ http { server_tokens off; default_type application/octet-stream; - # Docker's internal DNS — required for runtime upstream resolution. - # `valid=5s` re-resolves every 5 seconds so container restarts are - # picked up quickly without reloading nginx. + # Docker internal DNS for runtime upstream resolution. + # `valid=5s` forces periodic re-resolution so container restarts + # are detected without reloading nginx. resolver 127.0.0.11 valid=5s ipv6=off; server { listen 8080 default_server; server_name _; - # ------------------------------------------------------------------ - # Coordinator upstream as a variable — defers DNS to request time. - # Service name matches docker-compose: "minikv" - # ------------------------------------------------------------------ + # Coordinator upstream stored in a variable to defer DNS resolution + # to request time. Service name matches docker-compose ("minikv"). set $coordinator_upstream "minikv:3000"; - # ------------------------------------------------------------------ - # Main proxy: all client requests go to the coordinator. + # Main proxy. All client requests are forwarded to the coordinator. # - # On GET/HEAD the coordinator returns: - # X-Accel-Redirect: /accel/volume1:8080/sv09/a2/38/... - # Content-Type: image/jpeg - # Content-Blake3: - # Key-Balance: balanced + # For GET and HEAD, the coordinator responds with: + # X-Accel-Redirect: /accel// + # Content-Type + # Content-Blake3 + # Key-Balance # - # nginx intercepts X-Accel-Redirect and performs an internal - # subrequest, streaming the object body to the client with the - # coordinator's headers intact. - # ------------------------------------------------------------------ + # nginx intercepts X-Accel-Redirect and performs an internal subrequest. + # The object body is streamed from the volume server while preserving + # coordinator-provided metadata headers. location / { proxy_pass http://$coordinator_upstream; @@ -64,10 +61,10 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # Disable request buffering — required for streaming PUT uploads. + # Disable request buffering to allow streaming PUT uploads. proxy_request_buffering off; - # Disable response buffering — stream GET bodies directly. + # Disable response buffering to stream GET responses directly. proxy_buffering off; # Pass all coordinator metadata headers through to client. @@ -77,23 +74,21 @@ http { proxy_pass_header Key-Volumes; } - # ------------------------------------------------------------------ # Internal X-Accel-Redirect handler. # # URI format: /accel// - # Example: /accel/volume1:8080/sv09/a2/38/bXlib... + # Example: /accel/volume1:8080/sv09/a2/38/... # - # `internal` makes this location unreachable by direct client - # requests — only X-Accel-Redirect from the coordinator can - # trigger it. Direct requests return 404. + # The `internal` directive prevents direct client access. Only + # X-Accel-Redirect responses from the coordinator can trigger this + # location. Direct requests return 404. # - # The upstream is captured into a variable ($vol_upstream) so - # DNS resolution is deferred to request time (same pattern as above). - # ------------------------------------------------------------------ + # The captured upstream is stored in a variable to defer DNS + # resolution to request time. location ~ ^/accel/([^/]+)/(.*)$ { internal; - # Capture volume host:port and path into variables for runtime DNS. + # Capture volume host:port and object path into variables. set $vol_upstream $1; set $vol_path $2; @@ -102,26 +97,22 @@ http { # Do not forward client request headers to volume servers. proxy_pass_request_headers off; - # --------------------------------------------------------------- - # Content-Type injection via variable persistence. + # Content-Type handling. # - # The coordinator sets X-Content-Type on its response. - # nginx stores this as $upstream_http_x_content_type — a variable - # that persists across the X-Accel-Redirect internal redirect - # (same ngx_http_request_t context). + # The coordinator provides X-Content-Type in its response. nginx + # exposes this as $upstream_http_x_content_type. This variable + # persists across the internal X-Accel-Redirect. # - # If the coordinator has no stored Content-Type for this object - # (object was PUT without a Content-Type header, or rebuilt from - # volume data), $upstream_http_x_content_type will be empty. - # In that case we fall back to application/octet-stream rather - # than emitting an empty Content-Type header. + # If no Content-Type metadata exists (for example, the object was + # uploaded without one or reconstructed from volume data), the + # variable is empty. In that case, application/octet-stream is used. # - # Objects can be re-PUT with Content-Type to populate the field. - # --------------------------------------------------------------- + # Objects may be re-uploaded with a Content-Type header to set + # the stored metadata. proxy_hide_header Content-Type; - # Resolve effective Content-Type: coordinator metadata wins, - # fall back to octet-stream when metadata is absent. + # Coordinator metadata takes precedence. Fall back to + # application/octet-stream when absent. set $effective_ct $upstream_http_x_content_type; if ($effective_ct = "") { set $effective_ct "application/octet-stream"; diff --git a/config/nginx-volume.conf b/config/nginx-volume.conf index 8f8eae2..5b62d4f 100644 --- a/config/nginx-volume.conf +++ b/config/nginx-volume.conf @@ -1,13 +1,19 @@ # Volume server configuration for minikv # -# Requires: nginx-mod-http-dav-ext (installed via apk in Dockerfile.volume) -# Module path on Alpine 3.19: /usr/lib/nginx/modules/ngx_http_dav_ext_module.so +# Requires the nginx DAV extension module: +# nginx-mod-http-dav-ext # -# All volume containers listen on 8080 internally. -# docker-compose maps volume1 => 8001, volume2 => 8002, volume3 => 8003 on the host. +# Alpine 3.19 module path: +# /usr/lib/nginx/modules/ngx_http_dav_ext_module.so # -# daemon off is passed via CMD in Dockerfile.volume, not here, to avoid -# the duplicate-directive fatal error from some nginx base images. +# Each volume container listens on port 8080 internally. +# docker-compose maps: +# volume1 -> 8001 +# volume2 -> 8002 +# volume3 -> 8003 +# +# `daemon off` is set via CMD in Dockerfile.volume to avoid duplicate +# directive errors in certain nginx base images. load_module /usr/lib/nginx/modules/ngx_http_dav_ext_module.so; @@ -43,13 +49,14 @@ http { location / { disable_symlinks off; + # Enable object writes and deletions via WebDAV. dav_methods PUT DELETE; dav_access group:rw all:r; - # Auto-creates parent shard directories on first PUT. + # Automatically create shard directory hierarchy on write. create_full_put_path on; - # JSON directory listing — required by the rebuild subcommand. + # Expose JSON directory listings for rebuild operations. autoindex on; autoindex_format json; } diff --git a/docker-compose.yml b/docker-compose.yml index 3dcf47e..28741f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,16 @@ -# ============================================================================= -# minikv-rs — Docker Compose +# minikv-rs Docker Compose configuration # -# All three nginx volume containers listen on port 8080 internally. -# docker-compose maps them to distinct host ports (8001/8002/8003). -# The minikv coordinator addresses them by their container hostnames on 8080. +# Each volume container listens on port 8080 internally. +# docker-compose maps them to distinct host ports (8001, 8002, 8003). +# The coordinator connects to volumes via container DNS names on port 8080. # # Usage: -# docker compose up --build # build + start everything +# docker compose up --build # docker compose run minikv rebuild # docker compose run minikv rebalance -# ============================================================================= services: - # --------------------------------------------------------------------------- - # Frontend nginx — public entry point, handles X-Accel-Redirect - # --------------------------------------------------------------------------- + # Frontend nginx. Public entry point. Handles X-Accel-Redirect. frontend: image: nginx:1.25-alpine ports: @@ -30,7 +26,7 @@ services: minikv-init: image: busybox:1.36 - command: [ "sh", "-c", "chown -R 65532:65532 /data" ] + command: ["sh", "-c", "chown -R 65532:65532 /data"] volumes: - minikv-db:/data restart: "no" @@ -45,19 +41,20 @@ services: - server - --port=3000 - --db=/data/db - # Internal addresses — used by coordinator to replicate object writes + # Internal volume addresses used for replication. - --volumes=volume1:8080,volume2:8080,volume3:8080 - # Public addresses — returned in Location headers to clients - # Must map 1:1 to --volumes in the same order + # Public volume addresses returned to clients. + # Must match --volumes positionally. - --public-volumes=localhost:8001,localhost:8002,localhost:8003 - --replicas=3 - --subvolumes=10 - --voltimeout=1s - --protect - --checksum - # Enable X-Accel-Redirect: GET/HEAD returns X-Accel-Redirect header - # instead of 302. The frontend nginx intercepts it and streams the - # object body with the correct Content-Type from stored metadata. + # Enable X-Accel-Redirect mode. + # GET and HEAD return X-Accel-Redirect instead of 302. + # The frontend nginx performs the internal redirect and streams + # the object with coordinator-provided metadata headers. - --accel-redirect ports: - "3000:3000" @@ -92,8 +89,11 @@ services: networks: - minikv-net healthcheck: - # Lightweight TCP check — confirms the coordinator is accepting connections. - test: [ "CMD-SHELL", "wget -qO- http://localhost:3000/ 2>&1 | grep -qv 'Connection refused'" ] + test: + [ + "CMD-SHELL", + "wget -qO- http://localhost:3000/ 2>&1 | grep -qv 'Connection refused'", + ] interval: 5s timeout: 2s retries: 5 @@ -113,7 +113,7 @@ services: networks: - minikv-net healthcheck: - test: [ "CMD", "wget", "-qO-", "http://localhost:8080/" ] + test: ["CMD", "wget", "-qO-", "http://localhost:8080/"] interval: 5s timeout: 2s retries: 5 @@ -133,7 +133,7 @@ services: networks: - minikv-net healthcheck: - test: [ "CMD", "wget", "-qO-", "http://localhost:8080/" ] + test: ["CMD", "wget", "-qO-", "http://localhost:8080/"] interval: 5s timeout: 2s retries: 5 @@ -151,4 +151,4 @@ volumes: networks: minikv-net: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/minikv-core/src/lib.rs b/minikv-core/src/lib.rs index bab190f..465ab3b 100644 --- a/minikv-core/src/lib.rs +++ b/minikv-core/src/lib.rs @@ -26,14 +26,14 @@ //! //! # High-Level Architecture //! -//! - `hashing` — Key-to-path and volume scoring primitives. -//! - `volumes` — Replica selection and rebalance detection. -//! - `record` — On-disk metadata encoding format. -//! - `storage` — Metadata storage abstraction. -//! - `replication` — HTTP primitives for volume interaction. -//! - `rebuild` — Metadata reconstruction from volumes. -//! - `rebalance` — Migration to ideal replica sets. -//! - `locking` — Per-key concurrency control. +//! - `hashing` : Key-to-path and volume scoring primitives. +//! - `volumes` : Replica selection and rebalance detection. +//! - `record` : On-disk metadata encoding format. +//! - `storage` : Metadata storage abstraction. +//! - `replication` : HTTP primitives for volume interaction. +//! - `rebuild` : Metadata reconstruction from volumes. +//! - `rebalance` : Migration to ideal replica sets. +//! - `locking` : Per-key concurrency control. //! //! This crate is intended to be embedded in a higher-level service. diff --git a/minikv-core/src/record.rs b/minikv-core/src/record.rs index 7d01770..53df113 100644 --- a/minikv-core/src/record.rs +++ b/minikv-core/src/record.rs @@ -304,7 +304,7 @@ mod tests { #[test] fn decode_rejects_short_hash() { - // HASH prefix followed by only 10 hex chars — must fail + // HASH prefix followed by only 10 hex chars let bad = b"HASH0123456789hello"; assert!(Record::decode(bad).is_err()); } diff --git a/minikv-core/src/state.rs b/minikv-core/src/state.rs index bc3a322..0cd84ac 100644 --- a/minikv-core/src/state.rs +++ b/minikv-core/src/state.rs @@ -62,7 +62,7 @@ pub struct AppState { /// public-facing address (e.g. `"localhost:8001"`). /// /// Built once at startup from `--volumes` + `--public-volumes`. - /// Empty when `--public-volumes` is not set — Location headers will + /// Empty when `--public-volumes` is not set. Location headers will /// then use internal addresses unchanged (correct for bare-metal). pub vol_rewrite: HashMap, diff --git a/minikv/src/cli.rs b/minikv/src/cli.rs index f8c3b19..f2093ea 100644 --- a/minikv/src/cli.rs +++ b/minikv/src/cli.rs @@ -1,10 +1,10 @@ -/// CLI definition for `minikv`. -/// -/// # Subcommands -/// - `server` run the HTTP metadata coordinator -/// - `rebuild` reconstruct LevelDB from volume server autoindex -/// - `rebalance` move all keys to their ideal volume set -/// - `print-nginx-config` emit the nginx volume server config to stdout +//! CLI definition for `minikv`. +//! +//! # Subcommands +//! - `server` run the HTTP metadata coordinator +//! - `rebuild` reconstruct LevelDB from volume server autoindex +//! - `rebalance` move all keys to their ideal volume set + use std::path::PathBuf; use std::time::Duration; diff --git a/minikv/src/http/handlers.rs b/minikv/src/http/handlers.rs index 17f8dde..de656b3 100644 --- a/minikv/src/http/handlers.rs +++ b/minikv/src/http/handlers.rs @@ -1,39 +1,78 @@ -/// HTTP request handlers -/// -/// All standard and custom methods (UNLINK, REBALANCE) are handled here. -/// -/// # Method dispatch -/// ```text -/// GET / HEAD → 302 redirect to volume server (after probe) -/// PUT → write to replicas -/// POST ?uploads → initiate multipart -/// POST ?uploadId=X → complete multipart -/// POST ?delete → batch delete -/// DELETE → hard delete (requires prior UNLINK if protect=true) -/// UNLINK → soft delete -/// REBALANCE → move key to ideal volumes -/// GET ?list → list active keys -/// GET ?unlinked→ list soft-deleted keys -/// ``` -use std::sync::Arc; +//! HTTP request handlers. +//! +//! This module exposes the single entry point for all object routes +//! (`/*key`) and dispatches based on HTTP method and query parameters. +//! +//! The handler coordinates: +//! - per-key locking for mutating operations, +//! - metadata reads/writes via `AppState`, +//! - replica selection and rebalance, +//! - redirect construction (302 or X-Accel-Redirect), +//! - multipart upload lifecycle, +//! - soft and hard deletion. +//! +//! ## Method semantics +//! +//! GET / HEAD +//! Look up metadata, probe volumes, and redirect to a reachable +//! replica. Returns 404 if the key is soft-deleted, hard-deleted, +//! or unreachable on all volumes (unless a fallback is configured). +//! +//! PUT +//! Writes a new object to the ideal replica set. Overwrites are +//! rejected. Also handles multipart part uploads. +//! +//! POST ?uploads +//! Initiates a multipart upload and returns an upload ID. +//! +//! POST ?uploadId=X +//! Completes a multipart upload by concatenating parts and writing +//! the final object to replicas. +//! +//! POST ?delete +//! Batch delete under a prefix. +//! +//! DELETE +//! Hard delete. If `protect=true`, requires a prior UNLINK. +//! +//! UNLINK +//! Soft delete. Metadata remains but the key is treated as absent. +//! +//! REBALANCE +//! Moves the object to its ideal replica set if needed. +//! +//! GET ?list / ?unlinked +//! Query endpoints for active and soft-deleted keys. +//! +//! ## Concurrency model +//! +//! Mutating operations acquire a per-key lock using `KeyLock`. +//! Multipart part uploads are locked per `(key, partNumber)`. +//! +//! ## Redirect modes +//! +//! - In 302 mode, clients are redirected directly to a volume server. +//! - In X-Accel-Redirect mode, nginx performs an internal redirect and +//! serves the object while preserving coordinator-provided headers. + +use crate::http::query::handle_query; +use crate::http::s3::{CompleteMultipartUpload, Delete}; use axum::body::Body; use axum::extract::{Path, RawQuery, State}; use axum::http::{HeaderMap, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use bytes::Bytes; -use rand::seq::SliceRandom; -use tracing::{debug, info, instrument, warn}; -use uuid::Uuid; - -use crate::http::query::handle_query; -use crate::http::s3::{CompleteMultipartUpload, Delete}; use minikv_core::hashing::key_to_path; use minikv_core::rebalance::rebalance_key; use minikv_core::record::Deleted; use minikv_core::replication::remote_head; use minikv_core::state::AppState; use minikv_core::volumes::key_to_volume; +use rand::seq::SliceRandom; +use std::sync::Arc; +use tracing::{debug, info, instrument, warn}; +use uuid::Uuid; /// Top-level axum handler for all routes (`/*key`). /// @@ -187,7 +226,7 @@ async fn handle_get_head(state: &AppState, key: &[u8], _method: &Method) -> Resp // are discarded by nginx unless we use the variable persistence trick: // // 1. We set X-Content-Type on the coordinator response. - // 2. nginx captures it as $upstream_http_x_content_type — this variable + // 2. nginx captures it as $upstream_http_x_content_type. This variable // persists across the internal redirect (same ngx_http_request_t). // 3. The internal /accel/ location uses: // proxy_hide_header Content-Type; @@ -203,7 +242,7 @@ async fn handle_get_head(state: &AppState, key: &[u8], _method: &Method) -> Resp resp_builder = resp_builder .header(axum::http::header::CONTENT_TYPE, ct.as_str()) // X-Content-Type persists as $upstream_http_x_content_type - // across nginx's internal redirect — see nginx-frontend.conf. + // across nginx's internal redirect. See nginx-frontend.conf. .header("X-Content-Type", ct.as_str()); } diff --git a/minikv/src/http/mod.rs b/minikv/src/http/mod.rs index cd736e1..0a77a9d 100644 --- a/minikv/src/http/mod.rs +++ b/minikv/src/http/mod.rs @@ -1,3 +1,17 @@ +//! HTTP API wiring for the minikv coordinator. +//! +//! This crate provides the Axum router and request entry points that +//! expose the minikv HTTP interface. +//! +//! All object paths are routed through a single catch-all handler +//! (`/*key`). The handler internally dispatches based on HTTP method +//! and query parameters, including support for non-standard methods +//! such as `UNLINK` and `REBALANCE`. +//! +//! `build_router` constructs the `Router` and attaches shared +//! `AppState`, which contains configuration, metadata store access, +//! locking, and volume topology. + pub mod handlers; pub mod query; pub mod s3; diff --git a/minikv/src/http/query.rs b/minikv/src/http/query.rs index d889aae..2788c11 100644 --- a/minikv/src/http/query.rs +++ b/minikv/src/http/query.rs @@ -1,12 +1,27 @@ -/// Query parameter parsing and list/unlinked operations. -/// -/// Handles GET requests with a query string, which are routing differently -/// from plain GET (redirect) requests. -/// -/// Supported operations: -/// - `?list[&start=X][&limit=N]` list active keys under prefix -/// - `?unlinked[&start=X][&limit=N]` list soft-deleted keys -/// - `?list-type=2&prefix=X` S3-style listing +//! Query handling for list-style operations. +//! +//! This module processes GET requests that include a query string. +//! Plain GET requests (without a query) are handled elsewhere and +//! result in object redirects. +//! +//! Supported query forms: +//! +//! - `?list[&start=X][&limit=N]` +//! Lists active (non-deleted) keys under the provided prefix. +//! +//! - `?unlinked[&start=X][&limit=N]` +//! Lists soft-deleted keys under the provided prefix. +//! +//! - `?list-type=2&prefix=X` +//! Provides an S3-compatible XML listing. +//! +//! Listing is prefix-based and backed by a metadata scan (`scan_prefix`). +//! Results are filtered by deletion state at decode time. +//! +//! `start` acts as a cursor (lexicographic lower bound). +//! `limit` bounds the number of returned keys. A hard cap of +//! `MAX_KEYS` prevents unbounded responses. + use std::sync::Arc; use axum::http::StatusCode; @@ -17,14 +32,25 @@ use tracing::debug; use minikv_core::record::Deleted; use minikv_core::state::AppState; -/// JSON response for list operations. +/// JSON response body returned by `?list` and `?unlinked`. +/// +/// `next` is a continuation cursor. It is empty if no further +/// results are available. +/// +/// `keys` contains UTF-8 representations of matching object keys. #[derive(Debug, Serialize)] pub struct ListResponse { pub next: String, pub keys: Vec, } -/// Query parameters common to list and unlinked operations. +/// Optional query parameters used by `?list` and `?unlinked`. +/// +/// `start` is a lexicographic cursor. Keys strictly smaller than +/// this value are skipped. +/// +/// `limit` bounds the number of keys returned. If omitted, +/// all matching keys up to `MAX_KEYS` may be returned. #[allow(unused)] #[derive(Debug, Deserialize)] pub struct ListParams { @@ -32,15 +58,20 @@ pub struct ListParams { pub limit: Option, } -/// Maximum number of keys returned in a single list response before a -/// 413 (Payload Too Large) is returned. Matches Go's hard limit of 1,000,000. +/// Absolute upper bound on keys returned in a single response. +/// +/// If this limit is exceeded during iteration, the request +/// fails with `413 Payload Too Large`. const MAX_KEYS: usize = 1_000_000; -/// Handle a GET request that has a non-empty query string. +/// Entry point for GET requests with a non-empty query string. /// -/// Dispatches to: -/// - S3-style listing (`?list-type=2`) -/// - Our own `?list` / `?unlinked` operations +/// Dispatches based on query parameters: +/// +/// - `?list-type=2` → S3-compatible XML listing +/// - `?list` / `?unlinked` → JSON listing +/// +/// Any other query results in `403 Forbidden`. pub async fn handle_query(state: Arc, key_prefix: &[u8], raw_query: &str) -> Response { // S3-style listing: ?list-type=2&prefix=... if raw_query.contains("list-type=2") { @@ -56,7 +87,15 @@ pub async fn handle_query(state: Arc, key_prefix: &[u8], raw_query: &s } } -/// Handle `?list` and `?unlinked` queries. +/// Handles `?list` and `?unlinked` queries. +/// +/// Performs a prefix scan over metadata, applies cursor and limit, +/// filters by deletion state, and returns a JSON response. +/// +/// Returns: +/// - `200 OK` with JSON body on success +/// - `413 Payload Too Large` if `MAX_KEYS` is exceeded +/// - `500 Internal Server Error` on metadata or encoding failure async fn handle_list( state: Arc, key_prefix: &[u8], @@ -131,7 +170,13 @@ async fn handle_list( .into_response() } -/// Handle `?list-type=2` S3-style listing. +/// Handles `?list-type=2` S3-style listing. +/// +/// Extends the provided prefix with the S3 `prefix` parameter, +/// scans metadata, filters out deleted records, and returns +/// a minimal XML response compatible with S3 clients. +/// +/// Only active (non-deleted) keys are included. async fn handle_s3_list(state: Arc, key_prefix: &[u8], raw_query: &str) -> Response { // Append the S3 `prefix` parameter to our key prefix. let s3_prefix = extract_param(raw_query, "prefix").unwrap_or_default(); @@ -170,7 +215,12 @@ async fn handle_s3_list(state: Arc, key_prefix: &[u8], raw_query: &str .into_response() } -/// Extract a named query parameter from a raw query string. +/// Extracts a query parameter from a raw query string. +/// +/// The query string is not URL-decoded. This function performs +/// a simple `key=value` match split by `&`. +/// +/// Returns `None` if the parameter is not present. fn extract_param(raw_query: &str, name: &str) -> Option { for part in raw_query.split('&') { if let Some((k, v)) = part.split_once('=') diff --git a/minikv/src/http/s3.rs b/minikv/src/http/s3.rs index de59966..b0bb155 100644 --- a/minikv/src/http/s3.rs +++ b/minikv/src/http/s3.rs @@ -1,10 +1,11 @@ -/// S3-compatible XML request body parsing. -/// -/// Handles two S3 API shapes: -/// - `CompleteMultipartUpload` finalize a multipart upload. -/// - `Delete` batch-delete multiple objects. -/// -/// Uses `quick-xml` with serde for zero-copy XML deserialization. +//! S3-compatible XML request body parsing. +//! +//! Handles two S3 API shapes: +//! - `CompleteMultipartUpload` finalize a multipart upload. +//! - `Delete` batch-delete multiple objects. +//! +//! Uses `quick-xml` with serde for zero-copy XML deserialization. + use serde::Deserialize; use crate::error::Error; diff --git a/minikv/tests/integration.rs b/minikv/tests/integration.rs index bbe6b55..52366e9 100644 --- a/minikv/tests/integration.rs +++ b/minikv/tests/integration.rs @@ -1,13 +1,9 @@ +//! Integration tests +//! +//! Each test spins up a real axum server with an in-memory metadata store +//! and wiremock volume servers, then exercises the full HTTP flow. + use std::collections::{BTreeMap, HashMap}; -/// Integration tests -/// -/// Each test spins up a real axum server with an in-memory metadata store -/// and wiremock volume servers, then exercises the full HTTP flow. -/// -/// # Running -/// ```sh -/// cargo test --test integration -/// ``` use std::sync::Arc; use std::time::Duration;