**Build scan:**
https://gradle-enterprise.elastic.co/s/uhf5aydjvtmzw/tests/:server:internalClusterTest/org.elasticsearch.gateway.ReplicaShardAllocatorIT/testFullClusterRestartPerformNoopRecovery

**Reproduction line:**
```
./gradlew ':server:internalClusterTest' --tests "org.elasticsearch.gateway.ReplicaShardAllocatorIT.testFullClusterRestartPerformNoopRecovery" -Dtests.seed=B7ECF6653A89D3B7 -Dtests.locale=he -Dtests.timezone=Canada/Newfoundland -Druntime.java=17
```

**Applicable branches:**
main

**Reproduces locally?:**
Didn't try

**Failure history:**
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.gateway.ReplicaShardAllocatorIT&tests.test=testFullClusterRestartPerformNoopRecovery

**Failure excerpt:**
```
java.lang.AssertionError: 
Expected: an empty collection
     but: <[file (name [_c.cfs], reused [false], length [3461], recovered [3461], recovered from snapshot [0]) , file (name [_a_Lucene90_0.doc], reused [false], length [442], recovered [442], recovered from snapshot [0]) , file (name [_a_1_Lucene90_0.dvd], reused [false], length [355], recovered [355], recovered from snapshot [0]) , file (name [_a.si], reused [false], length [650], recovered [650], recovered from snapshot [0]) , file (name [_c.si], reused [false], length [343], recovered [343], recovered from snapshot [0]) , file (name [_f.cfe], reused [false], length [672], recovered [672], recovered from snapshot [0]) , file (name [_a.nvm], reused [false], length [103], recovered [103], recovered from snapshot [0]) , file (name [_a.fnm], reused [false], length [1153], recovered [1153], recovered from snapshot [0]) , file (name [_a_ES85BloomFilter_0.bfi], reused [false], length [733], recovered [733], recovered from snapshot [0]) , file (name [_a_0.tmd], reused [false], length [298], recovered [298], recovered from snapshot [0]) , file (name [_d.cfs], reused [false], length [4831], recovered [4831], recovered from snapshot [0]) , file (name [_f.si], reused [false], length [343], recovered [343], recovered from snapshot [0]) , file (name [_a_ES85BloomFilter_0.bfm], reused [false], length [93], recovered [93], recovered from snapshot [0]) , file (name [_a.kdd], reused [false], length [1663], recovered [1663], recovered from snapshot [0]) , file (name [_d.si], reused [false], length [343], recovered [343], recovered from snapshot [0]) , file (name [_e.cfe], reused [false], length [672], recovered [672], recovered from snapshot [0]) , file (name [_a.nvd], reused [false], length [853], recovered [853], recovered from snapshot [0]) , file (name [_a_0.pos], reused [false], length [68], recovered [68], recovered from snapshot [0]) , file (name [_a_0.tim], reused [false], length [10494], recovered [10494], recovered from snapshot [0]) , file (name [_a_Lucene90_0.dvm], reused [false], length [667], recovered [667], recovered from snapshot [0]) , file (name [_a.kdm], reused [false], length [145], recovered [145], recovered from snapshot [0]) , file (name [_a_Lucene90_0.tip], reused [false], length [112], recovered [112], recovered from snapshot [0]) , file (name [_a.fdt], reused [false], length [8616], recovered [8616], recovered from snapshot [0]) , file (name [_a_0.tip], reused [false], length [102], recovered [102], recovered from snapshot [0]) , file (name [_a.fdx], reused [false], length [64], recovered [64], recovered from snapshot [0]) , file (name [_a_Lucene90_0.tim], reused [false], length [3836], recovered [3836], recovered from snapshot [0]) , file (name [_a.fdm], reused [false], length [158], recovered [158], recovered from snapshot [0]) , file (name [_d.cfe], reused [false], length [672], recovered [672], recovered from snapshot [0]) , file (name [_e.cfs], reused [false], length [3461], recovered [3461], recovered from snapshot [0]) , file (name [_a.kdi], reused [false], length [79], recovered [79], recovered from snapshot [0]) , file (name [_a_Lucene90_0.tmd], reused [false], length [353], recovered [353], recovered from snapshot [0]) , file (name [_e.si], reused [false], length [343], recovered [343], recovered from snapshot [0]) , file (name [_f.cfs], reused [false], length [3464], recovered [3464], recovered from snapshot [0]) , file (name [segments_a], reused [false], length [718], recovered [718], recovered from snapshot [0]) , file (name [_a_1.fnm], reused [false], length [1154], recovered [1154], recovered from snapshot [0]) , file (name [_a_Lucene90_0.pos], reused [false], length [91], recovered [91], recovered from snapshot [0]) , file (name [_c.cfe], reused [false], length [672], recovered [672], recovered from snapshot [0]) , file (name [_a_0.doc], reused [false], length [68], recovered [68], recovered from snapshot [0]) , file (name [_a_Lucene90_0.dvd], reused [false], length [1661], recovered [1661], recovered from snapshot [0]) , file (name [_a_1_Lucene90_0.dvm], reused [false], length [160], recovered [160], recovered from snapshot [0]) ]>

  at __randomizedtesting.SeedInfo.seed([B7ECF6653A89D3B7:5E4F6B9C3F07E714]:0)
  at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:18)
  at org.junit.Assert.assertThat(Assert.java:956)
  at org.junit.Assert.assertThat(Assert.java:923)
  at org.elasticsearch.gateway.ReplicaShardAllocatorIT.assertNoOpRecoveries(ReplicaShardAllocatorIT.java:532)
  at org.elasticsearch.gateway.ReplicaShardAllocatorIT.testFullClusterRestartPerformNoopRecovery(ReplicaShardAllocatorIT.java:301)
  at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-2)
  at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
  at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:568)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
  at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
  at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
  at java.lang.Thread.run(Thread.java:833)

```
server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
.build();
        logger.info("--> adding 2 nodes on same rack and do rerouting");
        clusterState = ClusterState.builder(clusterState)
            .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
            .build();
        clusterState = allocation.reroute(clusterState, "reroute", ActionListener.noop());
        
        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
        
        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
        logger.info("--> verifying all is allocated");
        assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
        logger.info("--> adding additional node");
        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
        clusterState = allocation.reroute(clusterState, "reroute", ActionListener.noop());
        assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
        assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
        String origPrimaryNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
        String origReplicaNodeId = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
        logger.info("--> moving primary shard to node3");
        AllocationService.CommandsResult commandsResult = allocation.reroute(
            clusterState,
            new AllocationCommands(
                new MoveAllocationCommand(