Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions checkstyle/suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
<suppress checks="ClassDataAbstractionCoupling|ClassFanOutComplexity" files="SharePartitionManagerTest"/>
<suppress checks="CyclomaticComplexity" files="SharePartition.java"/>
<suppress checks="NPathComplexity|ClassFanOutComplexity" files="InklessConsolidatedDisklessTopicsTest"/>
<suppress checks="ClassFanOutComplexity" files="InklessTopicTypeSwitcherClusterTest"/>

<!-- server tests -->
<suppress checks="MethodLength|JavaNCSS|NPathComplexity" files="DescribeTopicPartitionsRequestHandlerTest.java"/>
Expand Down
10 changes: 9 additions & 1 deletion clients/src/main/java/org/apache/kafka/clients/admin/Admin.java
Original file line number Diff line number Diff line change
Expand Up @@ -383,9 +383,17 @@ default AlterDisklessSwitchResult alterDisklessSwitch(String topic, int partitio
* a seal of at least {@code 0} if the partition has no committed seal, is in phase 2, or the seal isn't
* lower than the committed seal. It also fails for every other negative seal offset.
*
* If the options name a leader with {@link AlterDisklessSwitchOptions#leaderId(int)}, the request elects
* that replica as the leader in phase 1 instead. The seal offset must then be {@code -1}, which keeps the
* seal. If the replica is in the ISR, the election keeps the ISR. If the replica is outside the ISR, the
* election resets the ISR to it, and the partition loses the records that the replica doesn't hold. The
* result fails with {@code InvalidRequestException} if the partition isn't in phase 1, or if the broker
* isn't a replica of the partition, isn't active, has an offline log directory, or already leads the
* partition.
*
* @param topic The topic name.
* @param partition The partition index.
* @param sealOffset {@code -2} to re-arm, or the new, lower seal.
* @param sealOffset {@code -2} to re-arm, the new, lower seal, or {@code -1} with a leader in the options.
* @param options The options to use.
* @return The AlterDisklessSwitchResult.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,33 @@

import org.apache.kafka.common.annotation.InterfaceStability;

import java.util.OptionalInt;

/**
* Options for {@link Admin#alterDisklessSwitch(String, int, long, AlterDisklessSwitchOptions)}.
*
* The API of this class is evolving, see {@link Admin} for details.
*/
@InterfaceStability.Unstable
public class AlterDisklessSwitchOptions extends AbstractOptions<AlterDisklessSwitchOptions> {
private OptionalInt leaderId = OptionalInt.empty();

/**
* Sets the replica to elect as the leader in phase 1, even if it's outside the ISR. The request must
* carry a seal offset of {@code -1}, which keeps the seal.
*
* @param leaderId The broker ID of the replica to elect.
* @return This options object.
*/
public AlterDisklessSwitchOptions leaderId(int leaderId) {
this.leaderId = OptionalInt.of(leaderId);
return this;
}

/**
* Returns the replica to elect as the leader, or empty to keep the leader.
*/
public OptionalInt leaderId() {
return leaderId;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4917,6 +4917,7 @@ AlterDisklessSwitchRequest.Builder createRequest(int timeoutMs) {
.setTopicName(topic)
.setPartitionIndex(partition)
.setSealOffset(sealOffset);
options.leaderId().ifPresent(data::setLeaderId);
return new AlterDisklessSwitchRequest.Builder(data);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
{ "name": "PartitionIndex", "type": "int32", "versions": "0+",
"about": "The partition index." },
{ "name": "SealOffset", "type": "int64", "versions": "0+",
"about": "-2 re-arms a pending switch. A seal of at least 0 lowers a committed seal in phase 1, and must be lower than the committed seal." }
"about": "-2 re-arms a pending switch. A seal of at least 0 lowers a committed seal in phase 1, and must be lower than the committed seal. -1 keeps the seal, and is valid only with a LeaderId." },
{ "name": "LeaderId", "type": "int32", "versions": "0+", "default": "-1", "entityType": "brokerId",
"about": "The replica to elect as the leader in phase 1, even if it's outside the ISR, or -1 to keep the leader." }
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.clients.admin.AdminClient;
import org.apache.kafka.clients.admin.AlterConfigOp;
import org.apache.kafka.clients.admin.AlterDisklessSwitchOptions;
import org.apache.kafka.clients.admin.ConfigEntry;
import org.apache.kafka.clients.admin.CreateTopicsResult;
import org.apache.kafka.clients.admin.NewTopic;
Expand Down Expand Up @@ -63,6 +64,7 @@

import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.io.TempDir;
Expand Down Expand Up @@ -100,6 +102,7 @@
import io.aiven.inkless.test_utils.S3TestContainer;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down Expand Up @@ -129,6 +132,15 @@ private enum AlterConfigsMode {
LEGACY
}

/**
* Tags a test that keeps a switched partition in phase 1. For such a test, the setup configures the
* tiered storage plugin with a bucket that doesn't exist. Segment uploads fail, and the remote copy
* task retries them, until the test creates the bucket.
*/
private static final String BLOCKS_REMOTE_UPLOADS = "blocks-remote-uploads";

private String tieredStorageBucket;

@BeforeEach
public void setup(final TestInfo testInfo) throws Exception {
log.warn("[stage=setup] Initializing containers and 3-broker cluster");
Expand All @@ -149,6 +161,10 @@ public void setup(final TestInfo testInfo) throws Exception {
));
}

tieredStorageBucket = testInfo.getTags().contains(BLOCKS_REMOTE_UPLOADS)
? "tiered-" + UUID.randomUUID().toString().substring(0, 8)
: s3Container.getBucketName();

final TestKitNodes nodes = new TestKitNodes.Builder()
.setCombined(true)
.setNumBrokerNodes(3)
Expand Down Expand Up @@ -178,7 +194,7 @@ public void setup(final TestInfo testInfo) throws Exception {
.setConfigProp(RSM_CONFIG_PREFIX + "custom.metadata.fields.include", "REMOTE_SIZE")
.setConfigProp(RSM_CONFIG_PREFIX + "key.prefix", "switch-int-test/")
.setConfigProp(RSM_CONFIG_PREFIX + "storage.backend.class", "io.aiven.kafka.tieredstorage.storage.s3.S3Storage")
.setConfigProp(RSM_CONFIG_PREFIX + "storage.s3.bucket.name", s3Container.getBucketName())
.setConfigProp(RSM_CONFIG_PREFIX + "storage.s3.bucket.name", tieredStorageBucket)
.setConfigProp(RSM_CONFIG_PREFIX + "storage.s3.region", s3Container.getRegion())
.setConfigProp(RSM_CONFIG_PREFIX + "storage.s3.endpoint.url", s3Container.getEndpoint())
.setConfigProp(RSM_CONFIG_PREFIX + "storage.s3.path.style.access.enabled", "true")
Expand Down Expand Up @@ -570,6 +586,125 @@ public void testAlterDisklessSwitchRejectsReArmAndNonLowerSealOnCommittedSwitch(
}
}

@Test
@Tag(BLOCKS_REMOTE_UPLOADS)
public void testElectedReplicaOutsideTheIsrRecoversAPhaseOnePartition() throws Exception {
final String topic = "switch-elect-" + UUID.randomUUID().toString().substring(0, 8);
final TopicPartition tp = new TopicPartition(topic, 0);
final int lostLeader = 1;
final int electedReplica = 2;
final int classicRecords = 20;
final int disklessRecords = 10;

// The bucket of the tiered storage plugin doesn't exist yet. Uploads fail, and the classic prefix
// stays on the local disks.
try (Admin admin = AdminClient.create(baseClientConfigs())) {
// Broker 0 isn't a replica and stays up for the test's clients.
admin.createTopics(List.of(
new NewTopic(topic, Map.of(0, List.of(lostLeader, electedReplica)))
.configs(Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "false"))
)).all().get(30, TimeUnit.SECONDS);
TopicMetadataProbe.awaitVisible(admin, topic);
produceRecords(topic, classicRecords);

alterTopicConfigWithIncrementalAlterConfigs(
admin, topic, Map.of(TopicConfig.DISKLESS_ENABLE_CONFIG, "true"));
final long sealOffset = waitForSealOffset(admin, topic, 0, offset -> offset >= 0);
assertEquals(classicRecords, sealOffset);

// The follower leaves the ISR with the whole classic prefix. The leader is then the only ISR
// member, and its shutdown stands for the loss of its disk.
cluster.brokers().get(electedReplica).shutdown();
waitForIsrShrink(admin, topic, 0, 2);
assertEquals(List.of(lostLeader), waitForLeader(admin, topic, lostLeader));
assertFalse(readClassicPrefixInRemoteStorage(admin, topic, 0),
"The partition must still be in phase 1 when its last ISR member goes away");
cluster.brokers().get(lostLeader).shutdown();
cluster.brokers().get(electedReplica).startup();

// The replica is back, but it's outside the ISR, and phase 1 blocks unclean leader election.
assertFalse(waitForLeader(admin, topic, -1).contains(electedReplica));
assertFalse(readClassicPrefixInRemoteStorage(admin, topic, 0));

admin.alterDisklessSwitch(topic, 0, PartitionRegistration.NO_CLASSIC_TO_DISKLESS_START_OFFSET,
new AlterDisklessSwitchOptions().leaderId(electedReplica)).all().get(20, TimeUnit.SECONDS);
assertEquals(List.of(electedReplica), waitForLeader(admin, topic, electedReplica));
assertEquals(sealOffset, readSealOffset(admin, topic, 0));

// The elected replica's log ends at the seal, so it serves the whole classic prefix.
assertEquals(classicRecords, countCommittedRecordsUpTo(tp, sealOffset));

// The partition also takes diskless records again.
produceRecords(topic, disklessRecords);
assertEquals(classicRecords + disklessRecords,
countCommittedRecordsUpTo(tp, sealOffset + disklessRecords));

// The old leader follows the elected replica and rejoins the ISR. It's the preferred replica,
// so the controller can move leadership back to it afterward.
cluster.brokers().get(lostLeader).startup();
waitForIsrRecovery(admin, topic, 0, 2);

// With the bucket in place, the leader uploads the classic prefix, and phase 2 starts.
try (var s3Client = s3Container.getS3Client()) {
s3Client.createBucket(request -> request.bucket(tieredStorageBucket));
}
waitForClassicPrefixInRemoteStorage(admin, topic);
}
}

private void waitForClassicPrefixInRemoteStorage(final Admin admin, final String topic) throws Exception {
final long deadline = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis();
while (System.currentTimeMillis() < deadline) {
if (readClassicPrefixInRemoteStorage(admin, topic, 0)) {
return;
}
Thread.sleep(500);
}
throw new AssertionError("Partition " + topic + "-0 did not enter phase 2 within timeout");
}

private void produceRecords(final String topic, final int count) throws Exception {
try (Producer<byte[], byte[]> producer = new KafkaProducer<>(producerConfigs())) {
for (int i = 0; i < count; i++) {
producer.send(new ProducerRecord<>(topic, 0, null,
("record-" + i).getBytes(StandardCharsets.UTF_8))).get(30, TimeUnit.SECONDS);
}
}
}

/**
* Waits until partition 0 of the topic has the leader, and returns its ISR at that time. A leader of
* `-1` means no leader.
*/
private List<Integer> waitForLeader(final Admin admin,
final String topic,
final int expectedLeader) throws Exception {
final long deadline = System.currentTimeMillis() + Duration.ofSeconds(60).toMillis();
int leader = -1;
while (System.currentTimeMillis() < deadline) {
final var partition = admin.describeTopics(List.of(topic)).allTopicNames()
.get(10, TimeUnit.SECONDS).get(topic).partitions().get(0);
leader = partition.leader() == null ? -1 : partition.leader().id();
if (leader == expectedLeader) {
return partition.isr().stream().map(node -> node.id()).sorted().toList();
}
Thread.sleep(200);
}
throw new AssertionError("Partition " + topic + "-0 did not reach leader=" + expectedLeader
+ " within timeout; last observed leader=" + leader);
}

private boolean readClassicPrefixInRemoteStorage(final Admin admin,
final String topic,
final int partition) throws Exception {
return admin.describeTopicPartitions(List.of(topic)).rawResponse().get(10, TimeUnit.SECONDS)
.topics().find(topic).partitions().stream()
.filter(p -> p.partitionIndex() == partition)
.findFirst()
.map(p -> InitDisklessLogFields.decodeClassicPrefixInRemoteStorage(p.unknownTaggedFields()))
.orElseThrow();
}

@Test
public void testTransactionOpenAtSwitchIsAbortedAndCompletes() throws Exception {
final String suffix = UUID.randomUUID().toString().substring(0, 8);
Expand Down
Loading
Loading