content: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  package org.apache.cassandra.fuzz.ring;
  
  import java.util.concurrent.Callable;
  import java.util.concurrent.atomic.AtomicReference;
  import java.util.function.Consumer;
  
  import org.junit.Assert;
  import org.junit.Test;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import org.apache.cassandra.distributed.Cluster;
  import org.apache.cassandra.distributed.Constants;
  import org.apache.cassandra.distributed.api.ConsistencyLevel;
  import org.apache.cassandra.distributed.api.Feature;
  import org.apache.cassandra.distributed.api.IInstanceConfig;
  import org.apache.cassandra.distributed.api.IInvokableInstance;
  import org.apache.cassandra.distributed.api.TokenSupplier;
  import org.apache.cassandra.distributed.shared.ClusterUtils;
  import org.apache.cassandra.distributed.shared.NetworkTopology;
  import org.apache.cassandra.distributed.test.log.FuzzTestBase;
  import org.apache.cassandra.harry.SchemaSpec;
  import org.apache.cassandra.harry.dsl.HistoryBuilder;
  import org.apache.cassandra.harry.dsl.HistoryBuilderHelper;
  import org.apache.cassandra.harry.dsl.ReplayingHistoryBuilder;
  import org.apache.cassandra.harry.execution.DataTracker;
  import org.apache.cassandra.harry.execution.InJvmDTestVisitExecutor;
  import org.apache.cassandra.harry.execution.RingAwareInJvmDTestVisitExecutor;
  import org.apache.cassandra.harry.gen.Generator;
  import org.apache.cassandra.harry.gen.Generators;
  import org.apache.cassandra.harry.gen.SchemaGenerators;
  import org.apache.cassandra.harry.model.QuiescentChecker;
  import org.apache.cassandra.harry.model.TokenPlacementModel;
  import org.apache.cassandra.metrics.TCMMetrics;
  import org.apache.cassandra.net.Verb;
  import org.apache.cassandra.tcm.Epoch;
  import org.apache.cassandra.tcm.transformations.PrepareJoin;
  
  import static org.apache.cassandra.distributed.shared.ClusterUtils.getSequenceAfterCommit;
  import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit;
  import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits;
  import static org.apache.cassandra.distributed.shared.ClusterUtils.waitForCMSToQuiesce;
  import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
  
  public class ConsistentBootstrapTest extends FuzzTestBase
  {
      private static final Logger logger = LoggerFactory.getLogger(ConsistentBootstrapTest.class);
      private static int WRITES = 500;
  
      @Test
      public void bootstrapFuzzTest() throws Throwable
      {
          Generator<SchemaSpec> schemaGen = SchemaGenerators.schemaSpecGen(KEYSPACE, "bootstrap_fuzz", 1000);
          try (Cluster cluster = builder().withNodes(3)
                                          .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(4))
                                          .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(4, "dc0", "rack0"))
                                          .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)
                                                                        .set("write_request_timeout", "10s")
                                                                        .set("metadata_snapshot_frequency", 5))
                                          .start();
               CloseableRef<IInvokableInstance> forShutdown = new CloseableRef<>(ClusterUtils::unpauseCommits))
          {
              IInvokableInstance cmsInstance = cluster.get(1);
              waitForCMSToQuiesce(cluster, cmsInstance);
  
              withRandom(rng -> {
                  SchemaSpec schema = schemaGen.generate(rng);
                  Generators.TrackingGenerator<Integer> pkGen = Generators.tracking(Generators.int32(0, Math.min(schema.valueGenerators.pkPopulation(), 1000)));
                  Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
  
                  HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
                  Runnable writeAndValidate = () -> {
                      for (int i = 0; i < WRITES; i++)
                          HistoryBuilderHelper.insertRandomData(schema, pkGen, ckGen, rng, history);
  
                      for (int pk : pkGen.generated())
                          history.selectPartition(pk);
                  };
  
                  history.customThrowing(() -> {
                      cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", KEYSPACE));
                      cluster.schemaChange(schema.compile());
                      waitForCMSToQuiesce(cluster, cmsInstance);
                  }, "Setup");
  
                  writeAndValidate.run();
  
                  history.customThrowing(() -> {
                      IInstanceConfig config = cluster.newInstanceConfig()
                                                      .set("auto_bootstrap", true)
                                                      .set(Constants.KEY_DTEST_FULL_STARTUP, true);
                      IInvokableInstance newInstance = cluster.bootstrap(config);
  
                      forShutdown.set(cmsInstance);
                      // Prime the CMS node to pause before the finish join event is committed
                      Callable<?> pending = pauseBeforeCommit(cmsInstance, (e) -> e instanceof PrepareJoin.FinishJoin);
                      new Thread(() -> newInstance.startup()).start();
                      pending.call();
                  }, "Start boostrap");
  
                  writeAndValidate.run();
  
                  history.customThrowing(() -> {
                      // Make sure there can be only one FinishJoin in flight
                      waitForCMSToQuiesce(cluster, cmsInstance);
                      // set expectation of finish join & retrieve the sequence when it gets committed
                      Callable<Epoch> bootstrapVisible = getSequenceAfterCommit(cmsInstance, (e, r) -> e instanceof PrepareJoin.FinishJoin && r.isSuccess());
  
                      // wait for the cluster to all witness the finish join event
                      unpauseCommits(cmsInstance);
                      forShutdown.set(null);
                      waitForCMSToQuiesce(cluster, bootstrapVisible.call());
                  }, "Finish bootstrap");
                  writeAndValidate.run();
  
                  RingAwareInJvmDTestVisitExecutor.replay(RingAwareInJvmDTestVisitExecutor.builder()
                                                                                          .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                          .consistencyLevel(ConsistencyLevel.ALL)
                                                                                          .build(schema, history, cluster),
                                                          history);
              });
          }
      }
  
      public static class CloseableRef<T> extends AtomicReference<T> implements AutoCloseable
      {
          private final Consumer<T> onClose;
  
          public CloseableRef(Consumer<T> onClose)
          {
              this.onClose = onClose;
          }
  
          @Override
          public void close() throws Exception
          {
              T v = getAndSet(null);
              if (v != null)
                  onClose.accept(v);
          }
      }
      @Test
      public void coordinatorIsBehindTest() throws Throwable
      {
          Generator<SchemaSpec> schemaGen = SchemaGenerators.schemaSpecGen(KEYSPACE, "coordinator_is_behind", 1000);
  
          try (Cluster cluster = builder().withNodes(3)
                                          .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(4))
                                          .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(4, "dc0", "rack0"))
                                          .withConfig((config) -> config.with(Feature.NETWORK, Feature.GOSSIP)
                                                                        .set("write_request_timeout", "10s")
                                                                        .set("accord.enabled", false)
                                                                        .set("cms_await_timeout", "60s")
                                                                        .set("metadata_snapshot_frequency", 5))
                                          .start();
               CloseableRef<IInvokableInstance> forShutdown = new CloseableRef<>(ClusterUtils::unpauseCommits))
          {
              IInvokableInstance cmsInstance = cluster.get(1);
              waitForCMSToQuiesce(cluster, cmsInstance);
  
              withRandom(rng -> {
                  SchemaSpec schema = schemaGen.generate(rng);
                  Generator<Integer> ckGen = Generators.int32(0, Math.min(schema.valueGenerators.ckPopulation(), 1000));
  
                  HistoryBuilder history = new ReplayingHistoryBuilder(schema.valueGenerators,
                                                                       hb -> RingAwareInJvmDTestVisitExecutor.builder()
                                                                                                             .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(3))
                                                                                                             .consistencyLevel(ConsistencyLevel.ALL)
                                                                                                             .build(schema, hb, cluster));
  
                  cluster.schemaChange(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 3};", KEYSPACE));
                  cluster.schemaChange(schema.compile());
                  waitForCMSToQuiesce(cluster, cluster.get(1));
  
                  cluster.filters().verbs(Verb.TCM_REPLICATION.id,
                                          Verb.TCM_FETCH_CMS_LOG_RSP.id,
                                          Verb.TCM_FETCH_PEER_LOG_RSP.id,
                                          Verb.TCM_CURRENT_EPOCH_REQ.id)
                         .to(2)
                         .drop()
                         .on();
  
                  DataTracker tracker = new DataTracker.SequentialDataTracker();
                  RingAwareInJvmDTestVisitExecutor executor = RingAwareInJvmDTestVisitExecutor.builder()
                                                                                              .replicationFactor(new TokenPlacementModel.SimpleReplicationFactor(2))
                                                                                              .nodeSelector(i -> 2)
                                                                                              .consistencyLevel(ConsistencyLevel.ALL)
                                                                                              .retryPolicy(InJvmDTestVisitExecutor.RetryPolicy.NO_RETRY)
                                                                                              .build(schema,
                                                                                                     tracker,
                                                                                                     new QuiescentChecker(schema.valueGenerators, tracker, history),
                                                                                                     cluster);
  
                  // Prime the CMS node to pause before the finish join event is committed
                  Callable<?> pending = pauseBeforeCommit(cmsInstance, (e) -> e instanceof PrepareJoin.MidJoin);
                  forShutdown.set(cmsInstance);
                  IInstanceConfig config = cluster.newInstanceConfig()
                                                  .set("auto_bootstrap", true)
                                                  .set(Constants.KEY_DTEST_FULL_STARTUP, true)
                                                  .set("progress_barrier_default_consistency_level", "NODE_LOCAL");
                  IInvokableInstance newInstance = cluster.bootstrap(config);
                  long[] metricCounts = new long[4];
                  for (int i = 1; i <= 4; i++)
                      metricCounts[i - 1] = cluster.get(i).callOnInstance(() -> TCMMetrics.instance.coordinatorBehindPlacements.getCount());
                  Thread startup = new Thread(() -> newInstance.startup());
                  startup.start();
                  pending.call();
  
                  long[] markers = new long[4];
                  for (int n = 0; n < 4; n++)
                      markers[n] = cluster.get(n + 1).logs().mark();
  
                  boolean triggered = false;
  
                  outer:
                  for (int i = 0; i < history.valueGenerators().pkPopulation(); i++)
                  {
                      long pd = history.valueGenerators().pkGen().descriptorAt(i);
                      for (TokenPlacementModel.Replica replica : executor.getReplicasFor(pd))
                      {
                          if (cluster.get(1).config().broadcastAddress().toString().contains(replica.node().id()))
                          {
                              try
                              {
                                  HistoryBuilderHelper.insertRandomData(schema, i, ckGen.generate(rng), rng, history);
                              }
                              catch (Throwable t)
                              {
                                  // ignore; expected
                              }
                          }
                      }
  
                      for (int n = 0; n < markers.length; n++)
                      {
                          if ((n + 1) == 2) // skip 2nd node
                              continue;
  
                          if (!cluster.get(n + 1)
                                      .logs()
                                      .grep(markers[n], "Routing is correct, but coordinator needs to catch-up")
                                      .getResult()
                                      .isEmpty())
                          {
                              triggered = true;
                              break outer;
                          }
                      }
                  }
  
                  Assert.assertTrue("Should have triggered routing exception on the replica", triggered);
                  boolean metricTriggered = false;
                  for (int i = 1; i <= 4; i++)
                  {
                      long prevMetric = metricCounts[i - 1];
                      long newMetric = cluster.get(i).callOnInstance(() -> TCMMetrics.instance.coordinatorBehindPlacements.getCount());
                      if (newMetric - prevMetric > 0)
                      {
                          metricTriggered = true;
                          break;
                      }
                  }
                  Assert.assertTrue("Metric CoordinatorBehindRing should have been bumped by at least one replica", metricTriggered);
  
                  cluster.filters().reset();
                  unpauseCommits(cmsInstance);
                  forShutdown.set(null);
                  startup.join();
              });
          }
      }
  }