Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
6298c6c
docs(inkless:release): add 0.51 changelog and cherry-pick sessions (#…
gqmelo Nov 6, 2026
33d9725
fix(inkless:ci): continue when the Env step cannot dump the github co…
jeqo Nov 9, 2026
32b9307
fix(inkless:switch): route pending-upload offsets to the classic log …
giuseppelillo Nov 9, 2026
bebe3df
fix(inkless:consolidation): keep diskless leader epoch across leader …
viktorsomogyi Nov 11, 2026
0331c77
fix(inkless:switch): recreate phase 2 replicas that don't hold the co…
giuseppelillo Nov 11, 2026
7c01033
fix(inkless:switch): advance a follower's high watermark to the seal …
giuseppelillo Nov 12, 2026
cae5e69
fix(inkless:switch): cap tiered reads at the seal [KC-739] (#867)
giuseppelillo Nov 12, 2026
2dc712a
fix(inkless:release): add an upstream sync to a published release (#865)
jeqo Nov 12, 2026
8707a8b
fix(inkless:switch): defer timestamp ListOffsets below the diskless l…
giuseppelillo Nov 13, 2026
88e4903
docs(inkless:switch): describe the last-known-leader election in phas…
giuseppelillo Nov 13, 2026
0e3b157
test(inkless:switch): run switch system tests with consolidation and …
giuseppelillo Nov 16, 2026
364c5bf
fix(inkless:build): omit commitId in make build_release outside git (…
jeqo Nov 16, 2026
974d7ab
fix(inkless:delete): skip file cleaner ticks during backoff instead o…
viktorsomogyi Nov 16, 2026
5578863
fix(inkless:switch): serve pending ListOffsets from the classic log a…
giuseppelillo Nov 16, 2026
cedc550
perf(inkless:metadata): read topic type flags without copying the top…
jeqo Nov 17, 2026
d4ecd58
fix(inkless:switch): abort open transactions only on the leader at th…
giuseppelillo Nov 17, 2026
95aa6bd
refactor(inkless:switch): replace the seal and repair commands with r…
giuseppelillo Nov 17, 2026
fbf02ad
fix(inkless:switch): skip ISR expansion while a phase 1 leader's log …
giuseppelillo Nov 18, 2026
2454d36
fix(inkless:switch): keep a failing partition from stalling the diskl…
giuseppelillo Nov 19, 2026
beacaba
fix(inkless:switch): print partitions whose ListOffsets fails in topi…
giuseppelillo Nov 19, 2026
82c9813
feat(inkless:storage): add per-role storage config overrides (#879)
jeqo Nov 19, 2026
f8ca859
fix(inkless:switch): seal and register a pending partition in the fir…
giuseppelillo Nov 19, 2026
af9faed
feat(inkless:metrics): publish partition rate distributions per broke…
jeqo Nov 19, 2026
dae5fcf
sync(compile): adapt cherry-picked changes to the 4.1 APIs
jeqo Nov 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ jobs:
sha: ${{ steps.configure-outputs.outputs.sha }}
steps:
- name: Env
# Inkless: diverges from upstream. The push event of an upstream sync merge commit lists
# every upstream commit, so toJson(github) exceeds the 128 KB limit of one environment
# string ("Argument list too long"). The dump is for debugging only, so the job continues.
# Keep this change when resolving sync conflicts.
continue-on-error: true
run: printenv
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
Expand Down Expand Up @@ -114,6 +119,8 @@ jobs:
name: Compile and Check (Merge Ref)
steps:
- name: Env
# Inkless: see the Env step of the configure job.
continue-on-error: true
run: printenv
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
Expand Down Expand Up @@ -383,6 +390,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Env
# Inkless: see the Env step of the configure job.
continue-on-error: true
run: printenv
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
Expand Down
7 changes: 6 additions & 1 deletion .github/workflows/inkless-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -420,10 +420,15 @@ jobs:
printf '```\n\n'
printf '## Kafka Versions\n'

if ! TAGS=$(echo "$KAFKA_BASE_MATRIX" | jq -r '.include[].tag // empty' | sort -V | uniq); then
if ! MATRIX_TAGS=$(echo "$KAFKA_BASE_MATRIX" | jq -r '.include[].tag // empty'); then
echo "::error::Could not parse kafka_base_matrix as JSON: $KAFKA_BASE_MATRIX" >&2
exit 1
fi
# List every Kafka-base tag of this version, not only the built ones, so a
# resumed release keeps the earlier Kafka patch versions in the notes.
EXISTING_TAGS=$(git tag -l "inkless-*-${INKLESS_VERSION}")
TAGS=$(printf '%s\n%s\n' "$MATRIX_TAGS" "$EXISTING_TAGS" \
| grep -E "^inkless-[0-9]+\.[0-9]+\.[0-9]+-${INKLESS_VERSION//./\\.}\$" | sort -Vu)
while IFS= read -r tag; do
if [[ "$tag" =~ ^inkless-([0-9]+\.[0-9]+\.[0-9]+)-(.+)$ ]]; then
KAFKA_VER="${BASH_REMATCH[1]}"
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/inkless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ jobs:
is-draft: ${{ steps.check-draft-pr.outputs.is-draft }}
steps:
- name: Env
# Inkless: diverges from upstream. The push event of an upstream sync merge commit lists
# every upstream commit, so toJson(github) exceeds the 128 KB limit of one environment
# string ("Argument list too long"). The dump is for debugging only, so the job continues.
# Keep this change when resolving sync conflicts.
continue-on-error: true
run: printenv
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
Expand Down
12 changes: 9 additions & 3 deletions INKLESS_OWNERSHIP
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ config/inkless/** @aiven/inkless
docker/inkless/** @aiven/inkless
docker/examples/docker-compose-files/inkless/** @aiven/inkless
inkless-benchmarks/** @aiven/inkless
jmh-benchmarks/src/main/java/org/apache/kafka/jmh/*/Inkless*.java @aiven/inkless # Inkless JMH benchmarks
inkless-sync/** @aiven/inkless
tests/kafkatest/tests/inkless/** @aiven/inkless
tests/kafkatest/services/inkless/** @aiven/inkless
Expand Down Expand Up @@ -65,9 +66,14 @@ core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @aiven/inkless @apach
core/src/main/scala/kafka/server/ConfigHandler.scala @aiven/inkless @apache/kafka # topic-config propagation
core/src/main/scala/kafka/server/ControllerServer.scala @aiven/inkless @apache/kafka # controller-side diskless wiring
core/src/main/scala/kafka/server/AbstractFetcherThread.scala @aiven/inkless @apache/kafka # consolidation fetcher skips ConsumerLag JMX registration
core/src/main/java/kafka/server/TierStateMachine.java @aiven/inkless @apache/kafka # phase 2 remote rebuild starts at the classic-to-diskless seal
metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @aiven/inkless @apache/kafka # diskless placement, election, and classic-to-diskless switch commits
metadata/src/main/java/org/apache/kafka/controller/PartitionChangeBuilder.java @aiven/inkless @apache/kafka # classic-to-diskless switch flag commit
storage/src/main/java/org/apache/kafka/storage/internals/log/UnifiedLog.java @aiven/inkless @apache/kafka # forced deletion of the local classic prefix
metadata/src/main/java/org/apache/kafka/image/writer/RaftSnapshotWriter.java @aiven/inkless @apache/kafka # snapshot batches split by size, for the producer states of switched partitions
storage/src/main/java/org/apache/kafka/storage/internals/log/UnifiedLog.java @aiven/inkless @apache/kafka # forced deletion of the local classic prefix; truncation count that fences remote offset updates
storage/src/main/java/org/apache/kafka/storage/internals/epoch/LeaderEpochFileCache.java @aiven/inkless @apache/kafka # a restored list of epoch entries becomes visible to readers all at once
core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala @aiven/inkless @apache/kafka # future log rolls at the seal; promotion waits for classic prefix deletion
storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManager.java @aiven/inkless @apache/kafka # remote read throws LeaderEpochNotAvailableException when the epoch cache has no epoch
core/src/main/scala/kafka/cluster/Partition.scala @aiven/inkless @apache/kafka # classic-to-diskless seal; ISR checks capped at the seal in phase 1
storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManager.java @aiven/inkless @apache/kafka # remote read throws LeaderEpochNotAvailableException when the epoch cache has no epoch; remote offset updates skip a log that truncated during the task; epoch cleanup keeps consolidating segments from the cross-tier log start; remote read ends at the upper bound offset
storage/src/main/java/org/apache/kafka/storage/internals/log/RemoteStorageFetchInfo.java @aiven/inkless @apache/kafka # upper bound offset that ends a tiered read at the classic-to-diskless seal
core/src/main/scala/kafka/cluster/Partition.scala @aiven/inkless @apache/kafka # classic-to-diskless seal; ISR checks capped at the seal in phase 1; no new epoch entry for a consolidating leader
checkstyle/import-control-core.xml @aiven/inkless @apache/kafka # kafka.server allows io.aiven.inkless, Testcontainers, and the AWS SDK
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ build_all:

core/build/distributions/kafka_2.13-$(VERSION).tgz:
echo "Building Kafka distribution with version $(VERSION)"
./gradlew releaseTarGz -PcommitId=$$(git rev-parse HEAD)
./gradlew releaseTarGz $$(commit=$$(git rev-parse HEAD 2>/dev/null) && echo "-PcommitId=$$commit")

.PHONY: build_release
build_release: core/build/distributions/kafka_2.13-$(VERSION).tgz
Expand Down
2 changes: 2 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2523,6 +2523,7 @@ project(':storage:inkless') {
api libs.pitestAnnotations

implementation libs.slf4jApi
implementation libs.hdrHistogram
implementation libs.postgresql
implementation libs.hikariCP
implementation libs.flyway
Expand Down Expand Up @@ -3758,6 +3759,7 @@ project(':jmh-benchmarks') {
implementation project(':group-coordinator:group-coordinator-api')
implementation project(':metadata')
implementation project(':storage')
implementation project(':storage:inkless')
implementation project(':streams')
implementation project(':transaction-coordinator')
implementation project(':core')
Expand Down
6 changes: 6 additions & 0 deletions checkstyle/import-control-jmh-benchmarks.xml
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,10 @@

<subpackage name="cache">
</subpackage>
<subpackage name="metadata">
<allow class="org.apache.kafka.raft.KRaftConfigs"/>
</subpackage>
<subpackage name="metrics">
<allow pkg="io.aiven.inkless.common.metrics"/>
</subpackage>
</import-control>
1 change: 1 addition & 0 deletions checkstyle/import-control-storage-inkless.xml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
<allow pkg="java" />
<allow pkg="javax.management" />
<allow pkg="org.slf4j" />
<allow pkg="org.HdrHistogram" />
<allow pkg="org.junit" />
<allow pkg="org.hamcrest" />
<allow pkg="org.mockito" />
Expand Down
3 changes: 2 additions & 1 deletion checkstyle/suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
<suppress checks="ClassDataAbstractionCoupling|ClassFanOutComplexity" files="SharePartitionManagerTest"/>
<suppress checks="CyclomaticComplexity" files="SharePartition.java"/>
<suppress checks="NPathComplexity|ClassFanOutComplexity" files="InklessConsolidatedDisklessTopicsTest"/>
<suppress checks="ClassFanOutComplexity" files="InklessTopicTypeSwitcherClusterTest"/>

<!-- server tests -->
<suppress checks="MethodLength|JavaNCSS|NPath" files="DescribeTopicPartitionsRequestHandlerTest.java"/>
Expand Down Expand Up @@ -375,7 +376,7 @@
<suppress checks="(ClassDataAbstractionCoupling|ClassFanOutComplexity)"
files="(UnifiedLog|RemoteLogManager|RemoteLogManagerTest).java"/>
<suppress checks="MethodLength" files="(RemoteLogManager|RemoteLogManagerConfig).java"/>
<suppress checks="JavaNCSS" files="RemoteLogManagerTest.java"/>
<suppress checks="JavaNCSS" files="(RemoteLogManager|RemoteLogManagerTest).java"/>

<!-- benchmarks -->
<suppress checks="(ClassDataAbstractionCoupling|ClassFanOutComplexity)"
Expand Down
61 changes: 14 additions & 47 deletions clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
Original file line number Diff line number Diff line change
Expand Up @@ -355,74 +355,41 @@ default DescribeTopicPartitionsResult describeTopicPartitions(Collection<String>
DescribeTopicPartitionsResult describeTopicPartitions(Collection<String> topics, DescribeTopicsOptions options);

/**
* Override the classic-to-diskless switch state of a single partition. This is an operator
* tool that writes the {@code classicToDisklessStartOffset} directly on the controller without
* requiring the caller to be the partition leader.
* Alter the classic-to-diskless switch of a single partition. This is an operator tool that accepts two
* seal offsets: {@code -2} re-arms a pending switch, so the leader seals again and re-runs the switch, and
* a seal of at least {@code 0} lowers a committed seal in phase 1. Lowering the seal loses the records
* between the new seal and the old one.
*
* This is a convenience method for {@link #alterDisklessSwitch(String, int, long, AlterDisklessSwitchOptions)}
* with default options. The same rules apply: once the partition's classic prefix is in remote
* storage, the controller rejects every seal offset.
* with default options.
*
* @param topic The topic name.
* @param partition The partition index.
* @param sealOffset The seal offset to commit: {@code >= 0} forces (re-)sealing at that offset,
* {@code -1} aborts the switch and reverts the partition to classic, and
* {@code -2} re-arms the switch as pending.
* @param sealOffset {@code -2} to re-arm, or the new, lower seal.
* @return The AlterDisklessSwitchResult.
*/
default AlterDisklessSwitchResult alterDisklessSwitch(String topic, int partition, long sealOffset) {
return alterDisklessSwitch(topic, partition, sealOffset, new AlterDisklessSwitchOptions());
}

/**
* Override the classic-to-diskless switch state of a single partition. This is an operator
* tool that writes the {@code classicToDisklessStartOffset} directly on the controller without
* requiring the caller to be the partition leader.
* Alter the classic-to-diskless switch of a single partition. This is an operator tool that accepts two
* seal offsets: {@code -2} re-arms a pending switch, so the leader seals again and re-runs the switch, and
* a seal of at least {@code 0} lowers a committed seal in phase 1. Lowering the seal loses the records
* between the new seal and the old one.
*
* Once the partition's classic prefix is in remote storage ({@code classicPrefixInRemoteStorage} is
* set), the controller rejects every seal offset and the result fails with
* {@code InvalidRequestException}.
* The result fails with {@code InvalidRequestException} for {@code -2} if the switch isn't pending, and for
* a seal of at least {@code 0} if the partition has no committed seal, is in phase 2, or the seal isn't
* lower than the committed seal. It also fails for every other negative seal offset.
*
* @param topic The topic name.
* @param partition The partition index.
* @param sealOffset The seal offset to commit: {@code >= 0} forces (re-)sealing at that offset,
* {@code -1} aborts the switch and reverts the partition to classic, and
* {@code -2} re-arms the switch as pending.
* @param sealOffset {@code -2} to re-arm, or the new, lower seal.
* @param options The options to use.
* @return The AlterDisklessSwitchResult.
*/
AlterDisklessSwitchResult alterDisklessSwitch(String topic, int partition, long sealOffset, AlterDisklessSwitchOptions options);

/**
* Reconcile the control-plane diskless log entry of a single partition with the seal committed in
* KRaft. This is an operator tool; the request must be sent to the partition leader, which owns the
* control-plane connection, so the caller supplies its broker id.
*
* This is a convenience method for {@link #repairDisklessLog(String, int, int, RepairDisklessLogOptions)}
* with default options.
*
* @param topic The topic name.
* @param partition The partition index.
* @param brokerId The id of the partition leader broker to send the request to.
* @return The RepairDisklessLogResult.
*/
default RepairDisklessLogResult repairDisklessLog(String topic, int partition, int brokerId) {
return repairDisklessLog(topic, partition, brokerId, new RepairDisklessLogOptions());
}

/**
* Reconcile the control-plane diskless log entry of a single partition with the seal committed in
* KRaft. This is an operator tool; the request must be sent to the partition leader, which owns the
* control-plane connection, so the caller supplies its broker id.
*
* @param topic The topic name.
* @param partition The partition index.
* @param brokerId The id of the partition leader broker to send the request to.
* @param options The options to use.
* @return The RepairDisklessLogResult.
*/
RepairDisklessLogResult repairDisklessLog(String topic, int partition, int brokerId, RepairDisklessLogOptions options);

/**
* Get information about the nodes in the cluster, using the default options.
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,4 @@
*/
@InterfaceStability.Unstable
public class AlterDisklessSwitchOptions extends AbstractOptions<AlterDisklessSwitchOptions> {
private boolean clearProducerStates = false;

/**
* When forcing a seal (offset {@code >= 0}), whether to clear the committed producer states.
* Ignored for negative seal offsets, which always clear them.
*/
public AlterDisklessSwitchOptions clearProducerStates(boolean clearProducerStates) {
this.clearProducerStates = clearProducerStates;
return this;
}

public boolean clearProducerStates() {
return clearProducerStates;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,6 @@ public AlterDisklessSwitchResult alterDisklessSwitch(String topic, int partition
return delegate.alterDisklessSwitch(topic, partition, sealOffset, options);
}

@Override
public RepairDisklessLogResult repairDisklessLog(String topic, int partition, int brokerId, RepairDisklessLogOptions options) {
return delegate.repairDisklessLog(topic, partition, brokerId, options);
}

@Override
public DescribeClusterResult describeCluster(DescribeClusterOptions options) {
return delegate.describeCluster(options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,6 @@
import org.apache.kafka.common.message.MetadataRequestData;
import org.apache.kafka.common.message.RemoveRaftVoterRequestData;
import org.apache.kafka.common.message.RenewDelegationTokenRequestData;
import org.apache.kafka.common.message.RepairDisklessLogRequestData;
import org.apache.kafka.common.message.UnregisterBrokerRequestData;
import org.apache.kafka.common.message.UpdateFeaturesRequestData;
import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult;
Expand Down Expand Up @@ -251,8 +250,6 @@
import org.apache.kafka.common.requests.RemoveRaftVoterResponse;
import org.apache.kafka.common.requests.RenewDelegationTokenRequest;
import org.apache.kafka.common.requests.RenewDelegationTokenResponse;
import org.apache.kafka.common.requests.RepairDisklessLogRequest;
import org.apache.kafka.common.requests.RepairDisklessLogResponse;
import org.apache.kafka.common.requests.UnregisterBrokerRequest;
import org.apache.kafka.common.requests.UnregisterBrokerResponse;
import org.apache.kafka.common.requests.UpdateFeaturesRequest;
Expand Down Expand Up @@ -4913,8 +4910,7 @@ AlterDisklessSwitchRequest.Builder createRequest(int timeoutMs) {
AlterDisklessSwitchRequestData data = new AlterDisklessSwitchRequestData()
.setTopicName(topic)
.setPartitionIndex(partition)
.setSealOffset(sealOffset)
.setClearProducerStates(options.clearProducerStates());
.setSealOffset(sealOffset);
return new AlterDisklessSwitchRequest.Builder(data);
}

Expand All @@ -4938,43 +4934,6 @@ void handleFailure(Throwable throwable) {
return new AlterDisklessSwitchResult(future);
}

@Override
public RepairDisklessLogResult repairDisklessLog(String topic, int partition, int brokerId,
RepairDisklessLogOptions options) {
final KafkaFutureImpl<Void> future = new KafkaFutureImpl<>();
final long now = time.milliseconds();
// The control-plane write happens on the partition leader, so send directly to that broker.
final Call call = new Call("repairDisklessLog", calcDeadlineMs(now, options.timeoutMs()),
new ConstantNodeIdProvider(brokerId)) {

@Override
RepairDisklessLogRequest.Builder createRequest(int timeoutMs) {
RepairDisklessLogRequestData data = new RepairDisklessLogRequestData()
.setTopicName(topic)
.setPartitionIndex(partition);
return new RepairDisklessLogRequest.Builder(data);
}

@Override
void handleResponse(AbstractResponse abstractResponse) {
final RepairDisklessLogResponse response = (RepairDisklessLogResponse) abstractResponse;
Errors error = Errors.forCode(response.data().errorCode());
if (error == Errors.NONE) {
future.complete(null);
} else {
future.completeExceptionally(error.exception(response.data().errorMessage()));
}
}

@Override
void handleFailure(Throwable throwable) {
future.completeExceptionally(throwable);
}
};
runnable.call(call, now);
return new RepairDisklessLogResult(future);
}

@Override
public DescribeProducersResult describeProducers(Collection<TopicPartition> topicPartitions, DescribeProducersOptions options) {
PartitionLeaderStrategy.PartitionLeaderFuture<DescribeProducersResult.PartitionProducerState> future =
Expand Down
Loading
Loading