Seems like a long-standing super-rare race condition at shutdown time. The following delays make it rather easy to reproduce:

```diff
diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index c21f2c17279..a2ac7df90c1 100644
--- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -1264,6 +1264,11 @@ public final class NodeEnvironment implements Closeable {

     @Override
     public void close() {
+        try {
+            Thread.sleep(1500);
+        } catch (InterruptedException e) {
+            throw new AssertionError(e);
+        }
         if (closed.compareAndSet(false, true) && locks != null) {
             for (Lock lock : locks) {
                 try {
@@ -1278,6 +1283,11 @@ public final class NodeEnvironment implements Closeable {

     private void assertEnvIsLocked() {
         if (closed.get() == false && locks != null) {
+            try {
+                Thread.sleep(200);
+            } catch (InterruptedException e) {
+                throw new AssertionError(e);
+            }
             for (Lock lock : locks) {
                 try {
                     lock.ensureValid();
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 812088100d5..f7d6fcd4229 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1632,6 +1632,23 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }

     public void close(String reason, boolean flushEngine) throws IOException {
+        store.incRef();
+        threadPool.generic().execute(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                Thread.sleep(1500);
+            }
+
+            @Override
+            public void onAfter() {
+                store.decRef();
+            }
+        });
         synchronized (engineMutex) {
             try {
                 synchronized (mutex) {
```

**Build scan:**
https://gradle-enterprise.elastic.co/s/4t6cp2obsipzy/tests/:server:internalClusterTest/org.elasticsearch.indices.IndexingMemoryControllerIT

**Reproduction line:**
```
null
```

**Applicable branches:**
main

**Reproduces locally?:**
Yes

**Failure history:**
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.indices.IndexingMemoryControllerIT&tests.test=classMethod

**Failure excerpt:**
```
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1292, name=elasticsearch[node_s_0][refresh][T#1], state=RUNNABLE, group=TGRP-IndexingMemoryControllerIT]


  Caused by: java.lang.AssertionError: java.lang.IllegalStateException: environment is not locked

    at __randomizedtesting.SeedInfo.seed([86C46FAD6D930468]:0)
    at org.elasticsearch.core.AbstractRefCounted.decRef(AbstractRefCounted.java:56)
    at org.elasticsearch.index.store.Store.decRef(Store.java:408)
    at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1878)
    at org.elasticsearch.index.engine.InternalEngine.writeIndexingBuffer(InternalEngine.java:1915)
    at org.elasticsearch.index.shard.IndexShard.writeIndexingBuffer(IndexShard.java:2463)
    at org.elasticsearch.indices.IndexingMemoryController$1.doRun(IndexingMemoryController.java:191)
    at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:958)
    at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
    at java.lang.Thread.run(Thread.java:1623)

    Caused by: java.lang.IllegalStateException: environment is not locked

      at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1286)
      at org.elasticsearch.env.NodeEnvironment.availableShardPaths(NodeEnvironment.java:1111)
      at org.elasticsearch.env.NodeEnvironment.deleteShardDirectoryUnderLock(NodeEnvironment.java:720)
      at org.elasticsearch.indices.IndicesService.deleteShardStore(IndicesService.java:1054)
      at org.elasticsearch.index.IndexService.onShardClose(IndexService.java:603)
      at org.elasticsearch.index.IndexService$StoreCloseListener.accept(IndexService.java:713)
      at org.elasticsearch.index.IndexService$StoreCloseListener.accept(IndexService.java:700)
      at org.elasticsearch.index.store.Store.closeInternal(Store.java:440)
      at org.elasticsearch.core.AbstractRefCounted$1.closeInternal(AbstractRefCounted.java:102)
      at org.elasticsearch.core.AbstractRefCounted.decRef(AbstractRefCounted.java:54)
      at org.elasticsearch.index.store.Store.decRef(Store.java:408)
      at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1878)
      at org.elasticsearch.index.engine.InternalEngine.writeIndexingBuffer(InternalEngine.java:1915)
      at org.elasticsearch.index.shard.IndexShard.writeIndexingBuffer(IndexShard.java:2463)
      at org.elasticsearch.indices.IndexingMemoryController$1.doRun(IndexingMemoryController.java:191)
      at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:958)
      at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
      at java.lang.Thread.run(Thread.java:1623)

      Caused by: java.nio.channels.AsynchronousCloseException: (No message provided)

        at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
        at sun.nio.ch.FileChannelImpl.endBlocking(FileChannelImpl.java:171)
        at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:430)
        at org.apache.lucene.tests.mockfile.FilterFileChannel.size(FilterFileChannel.java:82)
        at org.apache.lucene.tests.mockfile.FilterFileChannel.size(FilterFileChannel.java:82)
        at org.apache.lucene.tests.mockfile.FilterFileChannel.size(FilterFileChannel.java:82)
        at org.apache.lucene.tests.mockfile.FilterFileChannel.size(FilterFileChannel.java:82)
        at org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:169)
        at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:1283)
        at org.elasticsearch.env.NodeEnvironment.availableShardPaths(NodeEnvironment.java:1111)
        at org.elasticsearch.env.NodeEnvironment.deleteShardDirectoryUnderLock(NodeEnvironment.java:720)
        at org.elasticsearch.indices.IndicesService.deleteShardStore(IndicesService.java:1054)
        at org.elasticsearch.index.IndexService.onShardClose(IndexService.java:603)
        at org.elasticsearch.index.IndexService$StoreCloseListener.accept(IndexService.java:713)
        at org.elasticsearch.index.IndexService$StoreCloseListener.accept(IndexService.java:700)
        at org.elasticsearch.index.store.Store.closeInternal(Store.java:440)
        at org.elasticsearch.core.AbstractRefCounted$1.closeInternal(AbstractRefCounted.java:102)
        at org.elasticsearch.core.AbstractRefCounted.decRef(AbstractRefCounted.java:54)
        at org.elasticsearch.index.store.Store.decRef(Store.java:408)
        at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1878)
        at org.elasticsearch.index.engine.InternalEngine.writeIndexingBuffer(InternalEngine.java:1915)
        at org.elasticsearch.index.shard.IndexShard.writeIndexingBuffer(IndexShard.java:2463)
        at org.elasticsearch.indices.IndexingMemoryController$1.doRun(IndexingMemoryController.java:191)
        at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:958)
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.lang.Thread.run(Thread.java:1623)

```
server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
* scans the node paths and loads existing metadata file. If not found a new meta data will be generated
     */
    private static NodeMetadata loadNodeMetadata(Settings settings, Logger logger, DataPath... dataPaths) throws IOException {
        final Path[] paths = Arrays.stream(dataPaths).map(np -> np.path).toArray(Path[]::new);
        NodeMetadata metadata = PersistedClusterStateService.nodeMetadata(paths);
        if (metadata == null) {
            
            final Set<String> nodeIds = new HashSet<>();
            for (final Path path : paths) {
                final NodeMetadata oldStyleMetadata = NodeMetadata.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
                if (oldStyleMetadata != null) {
                    nodeIds.add(oldStyleMetadata.nodeId());
                }
            }
            if (nodeIds.size() > 1) {
                throw new IllegalStateException("data paths " + Arrays.toString(paths) + " belong to multiple nodes with IDs " + nodeIds);
            }
            
            final NodeMetadata legacyMetadata = NodeMetadata.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths);
            if (legacyMetadata == null) {
                assert nodeIds.isEmpty() : nodeIds;
                
                
                metadata = new NodeMetadata(generateNodeId(settings), Version.CURRENT, Version.CURRENT);
            } else {
                assert nodeIds.equals(Collections.singleton(legacyMetadata.nodeId())) : nodeIds + " doesn't match " + legacyMetadata;
                metadata = legacyMetadata;
            }
        }
        metadata = metadata.upgradeToCurrentVersion();