Ideally, we would be able to run each yaml test with a different cluster setup, but that's currently limited by only supporting a single yaml test suite client. @mark-vieira suggested we may able to support multiple clients that have a method to filter which yaml tests are run. This would also be great for running the same tests under different cluster conditions (single node vs multi node, etc).
test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
});
                } else {
                    path = root.resolve(strPath + ".yml");
                    assert Files.exists(path);
                    addSuite(root, path, files);
                }
            }
        }
        return files;
    }
    
    private static void addSuite(Path root, Path file, Map<String, Set<Path>> files) {
        String groupName = root.relativize(file.getParent()).toString();
        Set<Path> filesSet = files.get(groupName);
        if (filesSet == null) {
            filesSet = new HashSet<>();
            files.put(groupName, filesSet);
        }
        filesSet.add(file);
        List<String> fileNames = filesSet.stream().map(p -> p.getFileName().toString()).collect(Collectors.toList());
        if (Collections.frequency(fileNames, file.getFileName().toString()) > 1) {
            Logger logger = LogManager.getLogger(ESClientYamlSuiteTestCase.class);
            logger.warn(
                "Found duplicate test name ["
                    + groupName
                    + "/"
                    + file.getFileName()
                    + "] on the class path. "
                    + "This can result in class loader dependent execution commands and reproduction commands "
                    + "(will add #2 to one of the test names dependent on the classloading order)"