### CI Link

https://gradle-enterprise.elastic.co/s/ihqhydeyivro4/console-log/task/:qa:full-cluster-restart:v8.12.0%23bwcTest?anchor=119&page=1

### Repro line

./gradlew ':qa:full-cluster-restart:v8.12.0#bwcTest' -Dtests.class="org.elasticsearch.upgrades.FullClusterRestartIT" -Dtests.method="testPeerRecoveryRetentionLeases {cluster=UPGRADED}" -Dtests.seed=DB593AC604795E85 -Dtests.bwc=true -Dbuild.snapshot=false -Dtests.jvm.argline="-Dbuild.snapshot=false" -Dtests.locale=es-ES -Dtests.timezone=Africa/Mogadishu -Druntime.java=21 -Dlicense.key=${PWD}/x-pack/license-tools/src/test/resources/public.key

### Does it reproduce?

Yes

### Applicable branches

main

### Failure history

_No response_

### Failure excerpt


REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v8.12.0#bwcTest' -Dtests.class="org.elasticsearch.upgrades.FullClusterRestartIT" -Dtests.method="testPeerRecoveryRetentionLeases {cluster=UPGRADED}" -Dtests.seed=DB593AC604795E85 -Dtests.bwc=true -Dbuild.snapshot=false -Dtests.jvm.argline="-Dbuild.snapshot=false" -Dtests.locale=es-ES -Dtests.timezone=Africa/Mogadishu -Druntime.java=21 |  
-- | --
  |   |  
  | org.elasticsearch.upgrades.FullClusterRestartIT > testPeerRecoveryRetentionLeases {cluster=UPGRADED} FAILED |  
  | java.lang.RuntimeException: An error occurred orchestrating test cluster. |  
  | at __randomizedtesting.SeedInfo.seed([DB593AC604795E85:7CAD3B3E4818E181]:0) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.execute(DefaultLocalClusterHandle.java:259) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.writeUnicastHostsFile(DefaultLocalClusterHandle.java:240) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.waitUntilReady(DefaultLocalClusterHandle.java:183) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.upgradeToVersion(DefaultLocalClusterHandle.java:161) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalElasticsearchCluster.upgradeToVersion(DefaultLocalElasticsearchCluster.java:150) |  
  | at org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase.maybeUpgrade(ParameterizedFullClusterRestartTestCase.java:95) |  
  | at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) |  
  | at java.base/java.lang.reflect.Method.invoke(Method.java:580) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) |  
  | at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalElasticsearchCluster$1.evaluate(DefaultLocalElasticsearchCluster.java:47) |  
  | at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) |  
  | at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) |  
  | at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850) |  
  | at java.base/java.lang.Thread.run(Thread.java:1583) |  
  |   |  
  | Caused by: |  
  | java.lang.RuntimeException: Elasticsearch process died while waiting for ports file. See console output for details. |  
  | at org.elasticsearch.test.cluster.local.AbstractLocalClusterFactory$Node.lambda$waitUntilReady$0(AbstractLocalClusterFactory.java:274) |  
  | at org.elasticsearch.test.cluster.util.Retry.lambda$retryUntilTrue$0(Retry.java:28) |  
  | ... 1 more |  
  |   |  
  | REPRODUCE WITH: ./gradlew ':qa:full-cluster-restart:v8.12.0#bwcTest' -Dtests.class="org.elasticsearch.upgrades.QueryBuilderBWCIT" -Dtests.method="testQueryBuilderBWC {cluster=UPGRADED}" -Dtests.seed=DB593AC604795E85 -Dtests.bwc=true -Dbuild.snapshot=false -Dtests.jvm.argline="-Dbuild.snapshot=false" -Dtests.locale=sl -Dtests.timezone=Asia/Thimphu -Druntime.java=21 |  
  |   |  
  | org.elasticsearch.upgrades.QueryBuilderBWCIT > testQueryBuilderBWC {cluster=UPGRADED} FAILED |  
  | java.lang.RuntimeException: An error occurred orchestrating test cluster. |  
  | at __randomizedtesting.SeedInfo.seed([DB593AC604795E85:D8C5D605EDD63D95]:0) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.execute(DefaultLocalClusterHandle.java:259) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.writeUnicastHostsFile(DefaultLocalClusterHandle.java:240) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.waitUntilReady(DefaultLocalClusterHandle.java:183) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalClusterHandle.upgradeToVersion(DefaultLocalClusterHandle.java:161) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalElasticsearchCluster.upgradeToVersion(DefaultLocalElasticsearchCluster.java:150) |  
  | at org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase.maybeUpgrade(ParameterizedFullClusterRestartTestCase.java:95) |  
  | at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) |  
  | at java.base/java.lang.reflect.Method.invoke(Method.java:580) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) |  
  | at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891) |  
  | at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902) |  
  | at org.elasticsearch.test.cluster.local.DefaultLocalElasticsearchCluster$1.evaluate(DefaultLocalElasticsearchCluster.java:47) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) |  
  | at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) |  
  | at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) |  
  | at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) |  
  | at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) |  
  | at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) |  
  | at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) |  
  | at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850) |  
  | at java.base/java.lang.Thread.run(Thread.java:1583) |  
  |   |  
  | Caused by: |  
  | java.lang.RuntimeException: Elasticsearch process died while waiting for ports file. See console output for details. |  
  | at org.elasticsearch.test.cluster.local.AbstractLocalClusterFactory$Node.lambda$waitUntilReady$0(AbstractLocalClusterFactory.java:274) |  
  | at org.elasticsearch.test.cluster.util.Retry.lambda$retryUntilTrue$0(Retry.java:28) |  
  | ... 1 more |  
  |   |  
  | Tests with failures: |  
  | - org.elasticsearch.upgrades.FullClusterRestartIT.testPeerRecoveryRetentionLeases {cluster=UPGRADED} |  
  | - org.elasticsearch.upgrades.QueryBuilderBWCIT.testQueryBuilderBWC {cluster=UPGRADED}


qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
bulkRequest.setJsonEntity(bulk);
        bulkRequest.addParameter("refresh", "");
        assertThat(EntityUtils.toString(client().performRequest(bulkRequest).getEntity()), containsString("\"errors\":false"));
        if (isRunningAgainstOldCluster()) {
            Request rolloverRequest = new Request("POST", "/" + index + "_write/_rollover");
            rolloverRequest.setJsonEntity("""
                {  "conditions": {    "max_docs": 5  }}""");
            client().performRequest(rolloverRequest);
            assertThat(
                EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v")).getEntity()),
                containsString("testrollover-000002")
            );
        }
        Request countRequest = new Request("POST", "/" + index + "-*/_search");
        countRequest.addParameter("size", "0");
        Map<String, Object> count = entityAsMap(client().performRequest(countRequest));
        assertNoFailures(count);
        int expectedCount = bulkCount + (isRunningAgainstOldCluster() ? 0 : bulkCount);
        assertEquals(expectedCount, extractTotalHits(count));
    }
    void assertCountAll(int count) throws IOException {
        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
        assertNoFailures(response);
        int numDocs = extractTotalHits(response);
        logger.info("Found {} in old index", numDocs);
        assertEquals(count, numDocs);
    }
    void assertBasicSearchWorks(int count) throws IOException {
        logger.info("--> testing basic search");
        {