Skip to content

AKI-353: Combine the batch and direct write functionality #1147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AlexandraRoatis merged 8 commits into from
Apr 20, 2020
Merged

AKI-353: Combine the batch and direct write functionality #1147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cdd40ee
Implement batch functionality in MockDB
AlexandraRoatis Apr 17, 2020
e3bfdda
Implement batch functionality in H2MVMap
AlexandraRoatis Apr 17, 2020
c4e8b92
AKI-700: Refactor test code to use direct operation only for MockDB
AlexandraRoatis Apr 16, 2020
6803a92
Delete SpecialLockedDatabase
AlexandraRoatis Apr 17, 2020
f99d0cd
Test batch operations in ConcurrencyTest
AlexandraRoatis Apr 17, 2020
e6f9cd2
Remove irrelevant database settings
AlexandraRoatis Apr 17, 2020
8c34bfa
AKI-701: Remove direct write functionality from KeyValueStore
AlexandraRoatis Apr 17, 2020
43ba157
Rename batch methods to complete AKI-353
AlexandraRoatis Apr 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1273,7 +1273,7 @@ public TrieNodeResult importTrieNode(byte[] key, byte[] value, DatabaseType dbTy
}
}

db.putToBatch(key, value);
db.put(key, value);
db.commit();
return TrieNodeResult.IMPORTED;
}
Expand Down
6 changes: 2 additions & 4 deletions modAionImpl/src/org/aion/zero/impl/db/AvmContractDetails.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
Expand All @@ -23,7 +22,6 @@
import org.aion.util.conversions.Hex;
import org.aion.util.types.ByteArrayWrapper;
import org.aion.zero.impl.db.DetailsDataStore.RLPContractDetails;
import org.aion.zero.impl.trie.Node;
import org.aion.zero.impl.trie.SecureTrie;

/**
Expand Down Expand Up @@ -356,9 +354,9 @@ public byte[] getEncoded() {
public void syncStorage() {
byte[] graph = getObjectGraph();
if (!Arrays.equals(graph, EMPTY_BYTE_ARRAY)) {
objectGraphSource.putToBatch(objectGraphHash, graph);
objectGraphSource.put(objectGraphHash, graph);
}
objectGraphSource.putToBatch(computeAvmStorageHash(), RLP.encodeList(RLP.encodeElement(storageTrie.getRootHash()), RLP.encodeElement(objectGraphHash)));
objectGraphSource.put(computeAvmStorageHash(), RLP.encodeList(RLP.encodeElement(storageTrie.getRootHash()), RLP.encodeElement(objectGraphHash)));
// commit both updates together
objectGraphSource.commit();
storageTrie.sync();
Expand Down
2 changes: 1 addition & 1 deletion modAionImpl/src/org/aion/zero/impl/db/DetailsDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ public StoredContractDetails newContractDetails(AionAddress address, InternalVmT
public synchronized void update(AionAddress key, StoredContractDetails contractDetails) {
// Put into cache.
byte[] rawDetails = contractDetails.getEncoded();
detailsSrc.putToBatch(key.toByteArray(), rawDetails);
detailsSrc.put(key.toByteArray(), rawDetails);
detailsSrc.commit(); // TODO AKI-309: flush in bulk by the repository
contractDetails.syncStorage();
}
Expand Down
6 changes: 3 additions & 3 deletions modAionImpl/src/org/aion/zero/impl/db/PendingBlockStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ private int addBlockRange(Block first, List<Block> blockRange) {
levelSource.put(levelKey, levelData);

// index block with queue hash
indexSource.putToBatch(first.getHash(), currentQueueHash);
indexSource.put(first.getHash(), currentQueueHash);
int stored = 1;

// add element to queue
Expand All @@ -356,7 +356,7 @@ private int addBlockRange(Block first, List<Block> blockRange) {
}

// index block to current queue
indexSource.putToBatch(current.getHash(), currentQueueHash);
indexSource.put(current.getHash(), currentQueueHash);

// append block to queue
currentQueue.add(current);
Expand Down Expand Up @@ -509,7 +509,7 @@ public void dropPendingQueues(long level, Collection<ByteArrayWrapper> queues, M
// delete imported blocks
for (Block b : blocks.get(q)) {
// delete index
indexSource.deleteInBatch(b.getHash());
indexSource.delete(b.getHash());
}

// delete queue
Expand Down
2 changes: 1 addition & 1 deletion modAionImpl/src/org/aion/zero/impl/trie/scan/ExtractToDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public ExtractToDatabase(ByteArrayKeyValueDatabase _db) {

@Override
public void doOnNode(byte[] hash, Value node) {
db.putToBatch(hash, dummy_value);
db.put(hash, dummy_value);
count++;
}
}
12 changes: 7 additions & 5 deletions modAionImpl/test/org/aion/zero/impl/blockchain/AionHubTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import java.util.Map;
import org.aion.base.AionTransaction;
import org.aion.db.impl.ByteArrayKeyValueDatabase;
import org.aion.db.impl.mockdb.MockDB;
import org.aion.log.AionLoggerFactory;
import org.aion.log.LogEnum;
import org.aion.log.LogLevel;
Expand Down Expand Up @@ -170,11 +171,11 @@ public void MockHubInst_wStartRecovery() {

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
MockDB database = (MockDB) repo.getStateDatabase();

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand Down Expand Up @@ -249,11 +250,11 @@ public void MockHubInst_wStartRollback() {

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
MockDB database = (MockDB) repo.getStateDatabase();

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand All @@ -263,8 +264,9 @@ public void MockHubInst_wStartRollback() {

// Also, missing the block DB
blocksToImport.remove(0);
database = (MockDB) repo.getBlockDatabase();
for (AionBlock b : blocksToImport) {
repo.getBlockDatabase().delete(b.getHash());
database.deleteAndCommit(b.getHash());
}
repo.flush();

Expand Down
61 changes: 32 additions & 29 deletions modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainDataRecoveryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.aion.base.AionTransaction;
import org.aion.crypto.ECKey;
import org.aion.db.impl.ByteArrayKeyValueDatabase;
import org.aion.db.impl.mockdb.MockDB;
import org.aion.log.AionLoggerFactory;
import org.aion.log.LogEnum;
import org.aion.log.LogLevel;
Expand Down Expand Up @@ -83,13 +84,13 @@ public void testRecoverWorldStateWithPartialWorldState() {

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
MockDB database = (MockDB) repo.getStateDatabase();

// 1: direct recovery call

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand All @@ -109,7 +110,7 @@ public void testRecoverWorldStateWithPartialWorldState() {

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand Down Expand Up @@ -161,13 +162,13 @@ public void testRecoverWorldStateWithStartFromGenesis() {

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
MockDB database = (MockDB) repo.getStateDatabase();

// 1: direct recovery call

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand All @@ -187,7 +188,7 @@ public void testRecoverWorldStateWithStartFromGenesis() {

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand Down Expand Up @@ -241,7 +242,7 @@ public void testRecoverWorldStateWithoutGenesis() {

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
MockDB database = (MockDB) repo.getStateDatabase();

// 1: direct recovery call

Expand All @@ -253,7 +254,7 @@ public void testRecoverWorldStateWithoutGenesis() {
}

for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand All @@ -273,7 +274,7 @@ public void testRecoverWorldStateWithoutGenesis() {

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand Down Expand Up @@ -366,17 +367,18 @@ public void testRecoverWorldState_wDeletedBlock() {

// delete middle block from db
Block middle = chain.getBlockByNumber(NUMBER_OF_BLOCKS / 2);
repo.getBlockDatabase().delete(middle.getHash());
MockDB database = (MockDB) repo.getBlockDatabase();
database.deleteAndCommit(middle.getHash());

// delete some world state root entries from the database
TrieImpl trie = (TrieImpl) repo.getWorldState();
ByteArrayKeyValueDatabase database = repo.getStateDatabase();
database = (MockDB) repo.getStateDatabase();

// 1: direct recovery call

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand All @@ -395,7 +397,7 @@ public void testRecoverWorldState_wDeletedBlock() {

repo.flush();
for (byte[] key : statesToDelete) {
database.delete(key);
database.deleteAndCommit(key);
assertThat(trie.isValidRoot(key)).isFalse();
}

Expand Down Expand Up @@ -478,7 +480,7 @@ public void testRecoverIndexWithPartialIndex_MainChain() {
assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);

// delete index entries from the database
ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
MockDB indexDatabase = (MockDB) repo.getIndexDatabase();

// 1: direct recovery call

Expand All @@ -489,7 +491,7 @@ public void testRecoverIndexWithPartialIndex_MainChain() {
// saving the data for checking recovery
deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -523,7 +525,7 @@ public void testRecoverIndexWithPartialIndex_MainChain() {
for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -632,7 +634,7 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() {
assertThat(bestBlock.getHash()).isEqualTo(mainChainBlock.getHash());

// delete index entries from the database
ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
MockDB indexDatabase = (MockDB) repo.getIndexDatabase();

// 1: direct recovery call

Expand All @@ -643,7 +645,7 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() {
// saving the data for checking recovery
deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -684,7 +686,7 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() {
for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -756,7 +758,7 @@ public void testRecoverIndexWithStartFromGenesis() {
assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);

// delete index entries from the database
ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
MockDB indexDatabase = (MockDB) repo.getIndexDatabase();

// 1: direct recovery call

Expand All @@ -767,7 +769,7 @@ public void testRecoverIndexWithStartFromGenesis() {
// saving the data for checking recovery
deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -801,7 +803,7 @@ public void testRecoverIndexWithStartFromGenesis() {
for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -936,7 +938,7 @@ public void testRecoverIndexWithStartFromGenesisWithoutSize() {
assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);

// delete index entries from the database
ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
MockDB indexDatabase = (MockDB) repo.getIndexDatabase();

// 1: direct recovery call

Expand All @@ -948,7 +950,7 @@ public void testRecoverIndexWithStartFromGenesisWithoutSize() {
// saving the data for checking recovery
deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -983,7 +985,7 @@ public void testRecoverIndexWithStartFromGenesisWithoutSize() {
for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down Expand Up @@ -1081,10 +1083,11 @@ public void testRecoverIndex_wDeletedBlock() {

// delete middle block from db
Block middle = chain.getBlockByNumber(NUMBER_OF_BLOCKS / 2);
repo.getBlockDatabase().delete(middle.getHash());
MockDB database = (MockDB) repo.getBlockDatabase();
database.deleteAndCommit(middle.getHash());

// delete index entries from the database
ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
MockDB indexDatabase = (MockDB) repo.getIndexDatabase();

// 1: direct recovery call

Expand All @@ -1095,7 +1098,7 @@ public void testRecoverIndex_wDeletedBlock() {
// saving the data for checking recovery
deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand All @@ -1117,7 +1120,7 @@ public void testRecoverIndex_wDeletedBlock() {
for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
// deleting the block info
indexDatabase.delete(indexKey);
indexDatabase.deleteAndCommit(indexKey);
// ensure that the index was corrupted
assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
}
Expand Down
Loading