content: package org.aion.zero.impl.blockchain;
  
  import static com.google.common.truth.Truth.assertThat;
  
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import org.aion.base.AionTransaction;
  import org.aion.crypto.ECKey;
  import org.aion.db.impl.ByteArrayKeyValueDatabase;
  import org.aion.db.impl.mockdb.MockDB;
  import org.aion.log.AionLoggerFactory;
  import org.aion.log.LogEnum;
  import org.aion.log.LogLevel;
  import org.aion.zero.impl.types.Block;
  import org.aion.zero.impl.core.ImportResult;
  import org.aion.zero.impl.trie.TrieImpl;
  import org.aion.util.bytes.ByteUtil;
  import org.aion.zero.impl.types.BlockContext;
  import org.aion.zero.impl.db.AionRepositoryImpl;
  import org.aion.zero.impl.types.MiningBlock;
  import org.aion.zero.impl.types.MiningBlockHeader;
  import org.aion.zero.impl.vm.AvmTestConfig;
  import org.junit.AfterClass;
  import org.junit.BeforeClass;
  import org.junit.Test;
  
  /** @author Alexandra Roatis */
  public class BlockchainDataRecoveryTest {
  
      private static final List<ECKey> accounts = BlockchainTestUtils.generateAccounts(10);
      private static final int NUMBER_OF_BLOCKS = 20;
      private static final int MAX_TX_PER_BLOCK = 60;
  
      @BeforeClass
      public static void setup() {
          // logging to see errors
          AionLoggerFactory.initAll(Map.of(LogEnum.CONS, LogLevel.INFO));
  
          AvmTestConfig.supportOnlyAvmVersion1();
      }
  
      @AfterClass
      public static void tearDown() {
          AvmTestConfig.clearConfigurations();
      }
  
      /** Test the recovery of the world state with start from the state of an ancestor block. */
      @Test
      public void testRecoverWorldStateWithPartialWorldState() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // first half of blocks will be correct
          long time = System.currentTimeMillis();
          for (int i = 0; i < NUMBER_OF_BLOCKS / 2; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          }
  
          // second half of blocks will miss the state root
          List<byte[]> statesToDelete = new ArrayList<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS / 2; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              statesToDelete.add(context.block.getStateRoot());
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete some world state root entries from the database
          TrieImpl trie = (TrieImpl) repo.getWorldState();
          MockDB database = (MockDB) repo.getStateDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverWorldState(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(worked).isTrue();
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isTrue();
  
          // 2: recovery at import
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // state missing before import
              assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // state present after import
              assertThat(trie.isValidRoot(block.getStateRoot())).isTrue();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isTrue();
      }
  
      /** Test the recovery of the world state with start from the state of the genesis block. */
      @Test
      public void testRecoverWorldStateWithStartFromGenesis() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          List<byte[]> statesToDelete = new ArrayList<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              statesToDelete.add(context.block.getStateRoot());
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete some world state root entries from the database
          TrieImpl trie = (TrieImpl) repo.getWorldState();
          MockDB database = (MockDB) repo.getStateDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverWorldState(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(worked).isTrue();
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isTrue();
  
          // 2: recovery at import
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // state missing before import
              assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // state present after import
              assertThat(trie.isValidRoot(block.getStateRoot())).isTrue();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isTrue();
      }
  
      /**
       * Test the recovery of the world state when missing the genesis block state.
       *
       * <p>Under these circumstances the recovery will fail.
       */
      @Test
      public void testRecoverWorldStateWithoutGenesis() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete some world state root entries from the database
          TrieImpl trie = (TrieImpl) repo.getWorldState();
          MockDB database = (MockDB) repo.getStateDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          List<byte[]> statesToDelete = new ArrayList<>();
          Iterator<byte[]> iterator = database.keys();
          while (iterator.hasNext()) {
              statesToDelete.add(iterator.next());
          }
  
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverWorldState(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state was not recovered
          assertThat(worked).isFalse();
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // 2: recovery at import
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // state missing before import
              assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // state missing after import (because there is not genesis to recover from)
              assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
      }
  
      @Test
      public void testRecoverWorldState_wNullBlock() {
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle = builder.withValidatorConfiguration("simple").build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          repo.flush();
  
          assertThat(chain.recoverWorldState(repo, null)).isFalse();
      }
  
      @Test
      public void testRecoverWorldState_wRepositorySnapshot() {
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
  
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          }
          repo.flush();
  
          assertThat(
                          chain.recoverWorldState(
                                  repo.getSnapshotTo(repo.getRoot()), chain.getBestBlock()))
                  .isFalse();
      }
  
      @Test
      public void testRecoverWorldState_wDeletedBlock() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          List<byte[]> statesToDelete = new ArrayList<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              statesToDelete.add(context.block.getStateRoot());
              // skipping middle block
              if (context.block.getNumber() != NUMBER_OF_BLOCKS / 2) {
                  blocksToImport.add(context.block);
              }
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete middle block from db
          Block middle = chain.getBlockByNumber(NUMBER_OF_BLOCKS / 2);
          MockDB database = (MockDB) repo.getBlockDatabase();
          database.deleteAndCommit(middle.getHash());
  
          // delete some world state root entries from the database
          TrieImpl trie = (TrieImpl) repo.getWorldState();
          database = (MockDB) repo.getStateDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality
          assertThat(chain.recoverWorldState(repo, bestBlock)).isFalse();
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state could not be recovered
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // 2: recovery at import
  
          repo.flush();
          for (byte[] key : statesToDelete) {
              database.deleteAndCommit(key);
              assertThat(trie.isValidRoot(key)).isFalse();
          }
  
          // ensure that the world state was corrupted
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // state missing before import
              assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              if (block.getNumber() < middle.getNumber()) {
                  // state present after import
                  assertThat(trie.isValidRoot(block.getStateRoot())).isTrue();
              } else {
                  // block after missing one cannot be recovered
                  assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
              }
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse();
  
          // importing middle block
          assertThat(chain.tryToConnect(middle)).isEqualTo(ImportResult.IMPORTED_NOT_BEST);
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // checking only failed blocks from before
              if (block.getNumber() > middle.getNumber()) {
                  // state missing before import
                  assertThat(trie.isValidRoot(block.getStateRoot())).isFalse();
                  assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
                  // state present after import
                  assertThat(trie.isValidRoot(block.getStateRoot())).isTrue();
              }
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the world state is ok
          assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isTrue();
      }
  
      /** Test the recovery of the index with start from the index of an ancestor block. */
      @Test
      public void testRecoverIndexWithPartialIndex_MainChain() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // first half of blocks will be correct
          long time = System.currentTimeMillis();
          for (int i = 0; i < NUMBER_OF_BLOCKS / 2; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          }
  
          // second half of blocks will miss the index
          Map<Long, byte[]> blocksToDelete = new HashMap<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS / 2; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToImport.add(context.block);
              blocksToDelete.put(context.block.getNumber(), context.block.getHash());
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete index entries from the database
          MockDB indexDatabase = (MockDB) repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          Map<Long, byte[]> deletedInfo = new HashMap<>();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // saving the data for checking recovery
              deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverIndexEntry(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(worked).isTrue();
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              long level = entry.getKey();
              byte[] hash = entry.getValue();
              // checking at block store level
              assertThat(repo.isIndexed(hash, level)).isTrue();
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              assertThat(deletedInfo.get(level)).isEqualTo(indexDatabase.get(indexKey).get());
          }
  
          // 2: recovery at import
  
          repo.flush();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index present after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct at database level
          for (Long key : blocksToDelete.keySet()) {
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(key.intValue());
              assertThat(deletedInfo.get(key)).isEqualTo(indexDatabase.get(indexKey).get());
          }
      }
  
      /** Test the recovery of the index with start from the index of an ancestor block. */
      @Test
      public void testRecoverIndexWithPartialIndex_ShorterSideChain() {
          final int EVEN_NUMBER_OF_BLOCKS =
                  NUMBER_OF_BLOCKS % 2 == 0 ? NUMBER_OF_BLOCKS : NUMBER_OF_BLOCKS + 1;
  
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // adding common blocks
          long time = System.currentTimeMillis();
          for (int i = 0; i < EVEN_NUMBER_OF_BLOCKS / 2 - 1; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          }
  
          // splitting chains
          Block mainChainBlock, sideChainBlock;
          mainChainBlock = chain.getBestBlock();
          sideChainBlock = mainChainBlock;
  
          txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
          context = chain.createNewMiningBlockInternal(mainChainBlock, txs, true, time / 10000L);
          assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          mainChainBlock = context.block;
  
          // starting side chain
          txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
          context = chain.createNewMiningBlockInternal(sideChainBlock, txs, true, time / 10000L);
  
          MiningBlockHeader newHeader =
                  MiningBlockHeader.Builder.newInstance()
                          .withHeader(context.block.getHeader())
                          .withExtraData("other".getBytes()).build();
  
          context.block.updateHeader(newHeader);
  
          assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_NOT_BEST);
          sideChainBlock = context.block;
  
          // building chains; sidechain will have missing index
          Map<Long, byte[]> blocksToDelete = new HashMap<>();
          List<MiningBlock> blocksToImportMainChain = new ArrayList<>();
          List<MiningBlock> blocksToImportSideChain = new ArrayList<>();
          for (int i = 0; i < EVEN_NUMBER_OF_BLOCKS / 2 - 1; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(mainChainBlock, txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              mainChainBlock = context.block;
              blocksToImportMainChain.add(context.block);
  
              // adding side chain
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(sideChainBlock, txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_NOT_BEST);
              sideChainBlock = context.block;
              blocksToDelete.put(sideChainBlock.getNumber(), sideChainBlock.getHash());
              blocksToImportSideChain.add(context.block);
          }
  
          // making the main chain longer
          txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
          context = chain.createNewMiningBlockInternal(mainChainBlock, txs, true, time / 10000L);
          assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          mainChainBlock = context.block;
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(EVEN_NUMBER_OF_BLOCKS);
          assertThat(bestBlock.getHash()).isEqualTo(mainChainBlock.getHash());
  
          // delete index entries from the database
          MockDB indexDatabase = (MockDB) repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          Map<Long, byte[]> deletedInfo = new HashMap<>();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // saving the data for checking recovery
              deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // call the recovery functionality for the main chain subsection
          boolean worked =
                  chain.recoverIndexEntry(repo, chain.getBlockByHash(mainChainBlock.getParentHash()));
  
          // ensure that the index was corrupted only for the side chain
          assertThat(repo.isIndexed(sideChainBlock.getHash(), sideChainBlock.getNumber())).isFalse();
          assertThat(repo.isIndexed(mainChainBlock.getHash(), mainChainBlock.getNumber())).isTrue();
          assertThat(worked).isTrue();
  
          // call the recovery functionality
          worked = chain.recoverIndexEntry(repo, sideChainBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(worked).isTrue();
          assertThat(repo.isIndexed(sideChainBlock.getHash(), sideChainBlock.getNumber())).isTrue();
  
          // check that the index information is correct
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              long level = entry.getKey();
              byte[] hash = entry.getValue();
              // checking at block store level
              assertThat(repo.isIndexed(hash, level)).isTrue();
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // NOTE: this checks the correction of both main chain and side chain recovery
              assertThat(deletedInfo.get(level)).isEqualTo(indexDatabase.get(indexKey).get());
          }
  
          // 2: recovery at import
  
          repo.flush();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // call the recovery functionality indirectly for main chain
          for (MiningBlock block : blocksToImportMainChain) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index present after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
          }
  
          // ensure that the index was corrupted only for the side chain
          assertThat(repo.isIndexed(sideChainBlock.getHash(), sideChainBlock.getNumber())).isFalse();
          assertThat(repo.isIndexed(mainChainBlock.getHash(), mainChainBlock.getNumber())).isTrue();
  
          // call the recovery functionality indirectly for side chain
          for (MiningBlock block : blocksToImportSideChain) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index present after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
          }
  
          // ensure that the index was corrupted only for the side chain
          assertThat(repo.isIndexed(sideChainBlock.getHash(), sideChainBlock.getNumber())).isTrue();
          assertThat(repo.isIndexed(mainChainBlock.getHash(), mainChainBlock.getNumber())).isTrue();
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
  
          // check that the index information is correct at database level
          for (Long key : blocksToDelete.keySet()) {
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(key.intValue());
              // NOTE: this checks the correction of both main chain and side chain recovery
              assertThat(deletedInfo.get(key)).isEqualTo(indexDatabase.get(indexKey).get());
          }
      }
  
      /** Test the index recovery when the index database contains only the size and genesis index. */
      @Test
      public void testRecoverIndexWithStartFromGenesis() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          Map<Long, byte[]> blocksToDelete = new HashMap<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToDelete.put(context.block.getNumber(), context.block.getHash());
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete index entries from the database
          MockDB indexDatabase = (MockDB) repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          Map<Long, byte[]> deletedInfo = new HashMap<>();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // saving the data for checking recovery
              deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverIndexEntry(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(worked).isTrue();
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              long level = entry.getKey();
              byte[] hash = entry.getValue();
              // checking at block store level
              assertThat(repo.isIndexed(hash, level)).isTrue();
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              assertThat(deletedInfo.get(level)).isEqualTo(indexDatabase.get(indexKey).get());
          }
  
          // 2: recovery at import
  
          repo.flush();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index present after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct at database level
          for (Long key : blocksToDelete.keySet()) {
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(key.intValue());
              assertThat(deletedInfo.get(key)).isEqualTo(indexDatabase.get(indexKey).get());
          }
      }
  
      /**
       * Test the index recovery when the index database is empty.
       *
       * <p>Under these circumstances the recovery process will fail.
       */
      @Test
      public void testRecoverIndexWithoutGenesis() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          long time = System.currentTimeMillis();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          ByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          // deleting the entire index database
          indexDatabase.drop();
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverIndexEntry(repo, bestBlock);
  
          // ensure that the best block is unchanged
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // check that the index recovery failed
          assertThat(worked).isFalse();
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // 2: recovery at import
  
          repo.flush();
          // deleting the entire index database
          indexDatabase.drop();
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index missing after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
          }
  
          // ensure that the best block is unchanged
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // check that the index recovery failed
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
      }
  
      /**
       * Test the index recovery when the index database contains only the genesis index and is
       * missing the size key.
       */
      @Test
      public void testRecoverIndexWithStartFromGenesisWithoutSize() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          Map<Long, byte[]> blocksToDelete = new HashMap<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToDelete.put(context.block.getNumber(), context.block.getHash());
              blocksToImport.add(context.block);
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete index entries from the database
          MockDB indexDatabase = (MockDB) repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          Map<Long, byte[]> deletedInfo = new HashMap<>();
  
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // saving the data for checking recovery
              deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality
          boolean worked = chain.recoverIndexEntry(repo, bestBlock);
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(worked).isTrue();
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              long level = entry.getKey();
              byte[] hash = entry.getValue();
              // checking at block store level
              assertThat(repo.isIndexed(hash, level)).isTrue();
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              assertThat(deletedInfo.get(level)).isEqualTo(indexDatabase.get(indexKey).get());
          }
  
          // 2: recovery at import
  
          repo.flush();
  
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              // index present after import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct at database level
          for (Long key : blocksToDelete.keySet()) {
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(key.intValue());
              assertThat(deletedInfo.get(key)).isEqualTo(indexDatabase.get(indexKey).get());
          }
      }
  
      @Test
      public void testRecoverIndex_wNullBlock() {
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle = builder.withValidatorConfiguration("simple").build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          repo.flush();
  
          assertThat(chain.recoverIndexEntry(repo, null)).isFalse();
      }
  
      @Test
      public void testRecoverIndex_wRepositorySnapshot() {
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
  
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
          }
          repo.flush();
  
          assertThat(chain.recoverIndexEntry((AionRepositoryImpl) repo.getSnapshotTo(repo.getRoot()), chain.getBestBlock())).isFalse();
      }
  
      @Test
      public void testRecoverIndex_wDeletedBlock() {
          // build a blockchain with a few blocks
          StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder();
          StandaloneBlockchain.Bundle bundle =
                  builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).build();
  
          StandaloneBlockchain chain = bundle.bc;
          AionRepositoryImpl repo = chain.getRepository();
          BlockContext context;
          List<AionTransaction> txs;
  
          // all blocks will be incorrect
          long time = System.currentTimeMillis();
          Map<Long, byte[]> blocksToDelete = new HashMap<>();
          List<MiningBlock> blocksToImport = new ArrayList<>();
          for (int i = 0; i < NUMBER_OF_BLOCKS; i++) {
              txs = BlockchainTestUtils.generateTransactions(MAX_TX_PER_BLOCK, accounts, repo);
              context = chain.createNewMiningBlockInternal(chain.getBestBlock(), txs, true, time / 10000L);
              assertThat(chain.tryToConnect(context.block)).isEqualTo(ImportResult.IMPORTED_BEST);
              blocksToDelete.put(context.block.getNumber(), context.block.getHash());
              // skipping middle block
              if (context.block.getNumber() != NUMBER_OF_BLOCKS / 2) {
                  blocksToImport.add(context.block);
              }
          }
  
          Block bestBlock = chain.getBestBlock();
          assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS);
  
          // delete middle block from db
          Block middle = chain.getBlockByNumber(NUMBER_OF_BLOCKS / 2);
          MockDB database = (MockDB) repo.getBlockDatabase();
          database.deleteAndCommit(middle.getHash());
  
          // delete index entries from the database
          MockDB indexDatabase = (MockDB) repo.getIndexDatabase();
  
          // 1: direct recovery call
  
          repo.flush();
          Map<Long, byte[]> deletedInfo = new HashMap<>();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // saving the data for checking recovery
              deletedInfo.put(entry.getKey(), indexDatabase.get(indexKey).get());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality
          assertThat(chain.recoverIndexEntry(repo, bestBlock)).isFalse();
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // 2: recovery at import
  
          repo.flush();
          for (Map.Entry<Long, byte[]> entry : blocksToDelete.entrySet()) {
              byte[] indexKey = ByteUtil.intToBytes(entry.getKey().intValue());
              // deleting the block info
              indexDatabase.deleteAndCommit(indexKey);
              // ensure that the index was corrupted
              assertThat(repo.isIndexed(entry.getValue(), entry.getKey())).isFalse();
          }
  
          // ensure that the index was corrupted
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // index missing before import
              assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
              if (block.getNumber() < middle.getNumber()) {
                  // index present after import
                  assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
              } else {
                  // blocks after missing one cannot be recovered
                  assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
              }
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse();
  
          // importing middle block
          assertThat(chain.tryToConnect(middle)).isEqualTo(ImportResult.IMPORTED_NOT_BEST);
  
          // call the recovery functionality indirectly
          for (MiningBlock block : blocksToImport) {
              // checking only failed blocks from before
              if (block.getNumber() > middle.getNumber()) {
                  // index missing before import
                  assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isFalse();
                  assertThat(chain.tryToConnect(block)).isEqualTo(ImportResult.EXIST);
                  // index present after import
                  assertThat(repo.isIndexed(block.getHash(), block.getNumber())).isTrue();
              }
          }
  
          // ensure that the blockchain is ok
          assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash());
          // ensure that the index was recovered
          assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isTrue();
  
          // check that the index information is correct at database level
          for (Long key : blocksToDelete.keySet()) {
              // checking at database level
              byte[] indexKey = ByteUtil.intToBytes(key.intValue());
              assertThat(deletedInfo.get(key)).isEqualTo(indexDatabase.get(indexKey).get());
          }
      }
  }
  