From b87d86512bee15f124f2afa73db73d20fb446db3 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Sun, 8 Dec 2019 20:26:47 +0200 Subject: [PATCH 1/4] Flush column family --- .../rocksDB/RocksDBPersistenceProvider.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index d5ea9bea23..82bae5ce8b 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -408,23 +408,8 @@ public List loadAllKeysFromTable(Class column) { } private void flushHandle(ColumnFamilyHandle handle) throws RocksDBException { - List itemsToDelete = new ArrayList<>(); - try (RocksIterator iterator = db.newIterator(handle)) { - - for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) { - itemsToDelete.add(iterator.key()); - } - } - if (!itemsToDelete.isEmpty()) { - log.info("Amount to delete: " + itemsToDelete.size()); - } - int counter = 0; - for (byte[] itemToDelete : itemsToDelete) { - if (++counter % 10000 == 0) { - log.info("Deleted: {}", counter); - } - db.delete(handle, itemToDelete); - } + log.info("Dropping {} from DB", new String(handle.getName())); + db.dropColumnFamily(handle); } @Override From 0ff7e698e7a1bdeba1ece5ef6f9b99681292649d Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 9 Dec 2019 11:42:11 +0200 Subject: [PATCH 2/4] Add test --- .../rocksDB/RocksDBPersistenceProviderTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index b0f9b49b32..74c58ba5f5 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,5 +1,8 @@ package com.iota.iri.storage.rocksDB; +import com.iota.iri.TransactionTestUtils; +import com.iota.iri.controllers.TransactionViewModel; +import com.iota.iri.model.Hash; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Transaction; import com.iota.iri.storage.Indexable; @@ -76,4 +79,17 @@ public void testDeleteBatch() throws Exception { rocksDBPersistenceProvider.get(Transaction.class, index).bytes()); } } + + @Test + public void testClear() throws Exception { + Transaction tx = TransactionTestUtils.buildTransaction(TransactionTestUtils.getTransactionTrits()); + Hash hash = TransactionTestUtils.getTransactionHash(); + rocksDBPersistenceProvider.save(tx, hash); + Assert.assertTrue("Tx isn't found in DB after it was saved", + rocksDBPersistenceProvider.exists(Transaction.class, hash)); + rocksDBPersistenceProvider.clear(Transaction.class); + rocksDBPersistenceProvider.clearMetadata(Transaction.class); + Assert.assertFalse("Tx is found in DB after it was cleared", + rocksDBPersistenceProvider.exists(Transaction.class, hash)); + } } \ No newline at end of file From c720f5d7d5f2865933cb6a2bb1a29254e7df760f Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 9 Dec 2019 12:15:03 +0200 Subject: [PATCH 3/4] Batch delete --- .../rocksDB/RocksDBPersistenceProvider.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java index 82bae5ce8b..5f8e3a885c 100644 --- a/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java +++ b/src/main/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProvider.java @@ -408,8 +408,28 @@ public List loadAllKeysFromTable(Class column) { } private void flushHandle(ColumnFamilyHandle handle) throws RocksDBException { - log.info("Dropping {} from DB", new String(handle.getName())); - db.dropColumnFamily(handle); + try (WriteBatch delBatch = new WriteBatch(); + WriteOptions writeOptions = new WriteOptions() + //We are explicit about what happens if the node reboots before a flush to the db + .setDisableWAL(false) + //We want to make sure deleted data was indeed deleted + .setSync(true)) { + int counter = 0; + try (RocksIterator iterator = db.newIterator(handle)) { + for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) { + delBatch.delete(handle, iterator.key()); + if (++counter % 10000 == 0) { + db.write(writeOptions, delBatch); + log.info("Deleted: {}", counter); + delBatch.clear(); + } + } + } + + //delete remaining + db.write(writeOptions, delBatch); + log.info("Deleted {} entries in total", counter); + } } @Override From 43ca43606970a2195793d3cb35a7fed3a2ee50c4 Mon Sep 17 00:00:00 2001 From: Gal Rogozinski Date: Mon, 9 Dec 2019 12:38:42 +0200 Subject: [PATCH 4/4] organize imports --- .../storage/rocksDB/RocksDBPersistenceProviderTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java index 74c58ba5f5..56eef2e5cb 100644 --- a/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java +++ b/src/test/java/com/iota/iri/storage/rocksDB/RocksDBPersistenceProviderTest.java @@ -1,7 +1,6 @@ package com.iota.iri.storage.rocksDB; import com.iota.iri.TransactionTestUtils; -import com.iota.iri.controllers.TransactionViewModel; import com.iota.iri.model.Hash; import com.iota.iri.model.IntegerIndex; import com.iota.iri.model.persistables.Transaction; @@ -9,9 +8,6 @@ import com.iota.iri.storage.Persistable; import com.iota.iri.storage.Tangle; import com.iota.iri.utils.Pair; -import org.apache.commons.io.FileUtils; -import org.junit.*; -import org.junit.runners.MethodSorters; import java.io.File; import java.util.Arrays; @@ -19,6 +15,10 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.commons.io.FileUtils; +import org.junit.*; +import org.junit.runners.MethodSorters; + @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RocksDBPersistenceProviderTest {