aiven · ivanyu · Jun 3, 2026 · Apr 21, 2026 · Apr 29, 2026 · Apr 29, 2026
diff --git a/docs/inkless/configs.rst b/docs/inkless/configs.rst
@@ -40,7 +40,7 @@ Under ``inkless.``
   * Importance: high
 
 ``fetch.lagging.consumer.request.rate.limit``
-  Maximum requests per second for lagging consumer data fetches. Set to 0 to disable rate limiting. The upper bound of 10000 req/s is a safety limit to prevent misconfiguration. For high-throughput systems, consider the relationship between this rate limit, thread pool size, and storage backend capacity. At the default rate of 200 req/s with ~50ms per request latency, this allows ~10 concurrent requests.
+  Maximum requests per second for lagging consumer data fetches. Set to 0 to disable rate limiting. The upper bound of 10000 req/s is a safety limit to prevent misconfiguration. For high-throughput systems, consider the relationship between this rate limit, thread pool size, and storage backend capacity. At the default rate of 200 req/s with ~50ms per request latency, this allows ~10 concurrent requests. Note: hedge requests triggered by slow fetches are exempt from this limit. In the worst case, effective storage GET rate can reach up to 2x this value.
 
   * Type: int
   * Default: 200
@@ -157,6 +157,22 @@ Under ``inkless.``
   * Valid Values: [0,...]
   * Importance: low
 
+``fetch.hedge.total.time.threshold.ms``
+  Total time threshold in milliseconds to trigger a hedge request. When a storage fetch has not completed within this threshold, a competing hedge request is submitted. The first request to complete wins; the other continues in the background and its result is ignored. Set to 0 to disable total-time-based hedging. When both hedging thresholds are enabled, this value must be strictly greater than fetch.hedge.ttfb.threshold.ms. Capacity impact: hedges submit to the same executor as primaries (fetch.data.thread.pool.size for hot path, fetch.lagging.consumer.thread.pool.size for cold path). Normal case: only tail-latency requests (exceeding threshold) trigger hedges — typically <5% of traffic. Worst case: if all in-flight requests exceed the threshold, effective storage GET rate doubles (one primary + one hedge per request), bounded by executor thread pool + queue capacity. Monitor HedgeRequestRate to detect excessive hedging. If hedge rate is too high, increase this threshold.
+
+  * Type: long
+  * Default: 0
+  * Valid Values: [0,...]
+  * Importance: low
+
+``fetch.hedge.ttfb.threshold.ms``
+  Time-to-first-byte threshold in milliseconds to trigger a hedge request. When a storage fetch has not received its first byte within this threshold, a competing hedge request is submitted. This catches stuck connections early, before the total-time threshold. Set to 0 to disable TTFB-based hedging. When both hedging thresholds are enabled, fetch.hedge.total.time.threshold.ms must be strictly greater than this value.
+
+  * Type: long
+  * Default: 0
+  * Valid Values: [0,...]
+  * Importance: low
+
 ``fetch.lagging.consumer.thread.pool.size``
   Thread pool size for lagging consumer fetch requests (consumers reading old data). Set to 0 to disable the lagging consumer feature (all requests will use the recent data path). The default value of 16 is designed as approximately half of the default fetch.data.thread.pool.size (32), providing sufficient capacity for typical cold storage access patterns while leaving headroom for the hot path. The queue capacity is automatically set to thread.pool.size * 100, providing burst buffering (e.g., 16 threads = 1600 queue capacity ≈ 8 seconds buffer at 200 req/s). Tune based on lagging consumer SLA and expected load patterns.
 

diff --git a/docs/inkless/metrics.rst b/docs/inkless/metrics.rst
@@ -130,6 +130,10 @@ FetchTotalTime                       Total time spent processing a fetch request
 FileFetchErrorRate                   Rate of errors when fetching files from storage per second                          
 FindBatchesErrorRate                 Rate of errors when finding batches in the control plane per second                 
 FindBatchesTime                      Time spent finding batch coordinates in the control plane in milliseconds           
+HedgeRequestRate                     Rate of hedged fetch requests issued per second                                     
+HedgeTotalTimeTriggeredRate          Rate of hedge requests triggered by total time timeout per second                   
+HedgeTtfbTriggeredRate               Rate of hedge requests triggered by TTFB timeout per second                         
+HedgeWonRate                         Rate of hedge requests that completed before the original request per second        
 LaggingConsumerRateLimitWaitTime     Wait time for rate-limited lagging consumer requests in milliseconds                
 LaggingConsumerRequestRate           Rate of requests from lagging consumers (cold path, bypasses cache) per second      
 LaggingConsumerRequestRejectedRate   Rate of lagging consumer requests rejected due to executor unavailability per second

diff --git a/storage/inkless/src/main/java/io/aiven/inkless/config/InklessConfig.java b/storage/inkless/src/main/java/io/aiven/inkless/config/InklessConfig.java
@@ -179,12 +179,37 @@ public class InklessConfig extends AbstractConfig {
         + "Set to 0 to disable rate limiting. "
         + "The upper bound of 10000 req/s is a safety limit to prevent misconfiguration. For high-throughput systems, "
         + "consider the relationship between this rate limit, thread pool size, and storage backend capacity. "
-        + "At the default rate of 200 req/s with ~50ms per request latency, this allows ~10 concurrent requests.";
+        + "At the default rate of 200 req/s with ~50ms per request latency, this allows ~10 concurrent requests. "
+        + "Note: hedge requests triggered by slow fetches are exempt from this limit. In the worst case, "
+        + "effective storage GET rate can reach up to 2x this value.";
     // Default 200 req/s: Conservative limit based on typical object storage GET request costs and latency.
     // At ~50ms per request, 200 req/s = ~10 concurrent requests, balancing throughput with cost control.
     // Tune based on storage backend capacity and budget constraints.
     private static final int FETCH_LAGGING_CONSUMER_REQUEST_RATE_LIMIT_DEFAULT = 200;
 
+    public static final String FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG = "fetch.hedge.ttfb.threshold.ms";
+    public static final String FETCH_HEDGE_TTFB_THRESHOLD_MS_DOC = "Time-to-first-byte threshold in milliseconds to trigger a hedge request. "
+        + "When a storage fetch has not received its first byte within this threshold, a competing hedge request is submitted. "
+        + "This catches stuck connections early, before the total-time threshold. "
+        + "Set to 0 to disable TTFB-based hedging. "
+        + "When both hedging thresholds are enabled, fetch.hedge.total.time.threshold.ms must be strictly greater than this value.";
+    private static final long FETCH_HEDGE_TTFB_THRESHOLD_MS_DEFAULT = 0;
+
+    public static final String FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG = "fetch.hedge.total.time.threshold.ms";
+    public static final String FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_DOC = "Total time threshold in milliseconds to trigger a hedge request. "
+        + "When a storage fetch has not completed within this threshold, a competing hedge request is submitted. "
+        + "The first request to complete wins; the other continues in the background and its result is ignored. "
+        + "Set to 0 to disable total-time-based hedging. "
+        + "When both hedging thresholds are enabled, this value must be strictly greater than "
+        + FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG + ". "
+        + "Capacity impact: hedges submit to the same executor as primaries (fetch.data.thread.pool.size for hot path, "
+        + "fetch.lagging.consumer.thread.pool.size for cold path). "
+        + "Normal case: only tail-latency requests (exceeding threshold) trigger hedges — typically <5% of traffic. "
+        + "Worst case: if all in-flight requests exceed the threshold, effective storage GET rate doubles "
+        + "(one primary + one hedge per request), bounded by executor thread pool + queue capacity. "
+        + "Monitor HedgeRequestRate to detect excessive hedging. If hedge rate is too high, increase this threshold.";
+    private static final long FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_DEFAULT = 0;
+
     public static final String FETCH_FIND_BATCHES_MAX_BATCHES_PER_PARTITION_CONFIG = "fetch.find.batches.max.per.partition";
     public static final String FETCH_FIND_BATCHES_MAX_BATCHES_PER_PARTITION_DOC = "The maximum number of batches to find per partition when processing a fetch request. "
         + "A value of 0 means all available batches are fetched. "
@@ -395,6 +420,22 @@ public static ConfigDef configDef() {
             ConfigDef.Importance.MEDIUM,
             FETCH_LAGGING_CONSUMER_REQUEST_RATE_LIMIT_DOC
         );
+        configDef.define(
+            FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG,
+            ConfigDef.Type.LONG,
+            FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_DEFAULT,
+            ConfigDef.Range.atLeast(0),
+            ConfigDef.Importance.LOW,
+            FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_DOC
+        );
+        configDef.define(
+            FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG,
+            ConfigDef.Type.LONG,
+            FETCH_HEDGE_TTFB_THRESHOLD_MS_DEFAULT,
+            ConfigDef.Range.atLeast(0),
+            ConfigDef.Importance.LOW,
+            FETCH_HEDGE_TTFB_THRESHOLD_MS_DOC
+        );
         configDef.define(
             FETCH_FIND_BATCHES_MAX_BATCHES_PER_PARTITION_CONFIG,
             ConfigDef.Type.INT,
@@ -497,6 +538,24 @@ private static ConfigDef validate(final Map<String, ?> props) {
             );
         }
 
+        final long hedgeTtfbMs =
+            ((Number) parsedProps.get(FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG)).longValue();
+        final long hedgeTotalTimeMs =
+            ((Number) parsedProps.get(FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG)).longValue();
+
+        // When both hedging triggers are enabled, total-time threshold must be > TTFB threshold.
+        // TTFB fires early to catch stuck connections; total-time is a broader safety net.
+        // If total-time <= TTFB, the total-time timer would fire first (or simultaneously),
+        // making TTFB redundant and defeating the two-tier design.
+        if (hedgeTtfbMs > 0 && hedgeTotalTimeMs > 0 && hedgeTotalTimeMs <= hedgeTtfbMs) {
+            throw new ConfigException(
+                FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG,
+                hedgeTotalTimeMs,
+                FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG + " (" + hedgeTotalTimeMs + "ms) must be greater than "
+                    + FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG + " (" + hedgeTtfbMs + "ms) when both are enabled."
+            );
+        }
+
         return configDef;
     }
 
@@ -618,6 +677,14 @@ public int fetchLaggingConsumerRequestRateLimit() {
         return getInt(FETCH_LAGGING_CONSUMER_REQUEST_RATE_LIMIT_CONFIG);
     }
 
+    public long fetchHedgeTtfbThresholdMs() {
+        return getLong(FETCH_HEDGE_TTFB_THRESHOLD_MS_CONFIG);
+    }
+
+    public long fetchHedgeTotalTimeThresholdMs() {
+        return getLong(FETCH_HEDGE_TOTAL_TIME_THRESHOLD_MS_CONFIG);
+    }
+
     public int maxBatchesPerPartitionToFind() {
         return getInt(FETCH_FIND_BATCHES_MAX_BATCHES_PER_PARTITION_CONFIG);
     }

diff --git a/storage/inkless/src/main/java/io/aiven/inkless/consume/FetchHandler.java b/storage/inkless/src/main/java/io/aiven/inkless/consume/FetchHandler.java
@@ -59,6 +59,8 @@ public FetchHandler(final SharedState state) {
                 state.config().fetchLaggingConsumerThresholdMs(),
                 state.config().fetchLaggingConsumerRequestRateLimit(),
                 state.config().fetchLaggingConsumerThreadPoolSize(),
+                state.config().fetchHedgeTtfbThresholdMs(),
+                state.config().fetchHedgeTotalTimeThresholdMs(),
                 state.config().maxBatchesPerPartitionToFind()
             )
         );