content: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.cassandra.config;
  
  import java.io.ByteArrayInputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.net.URL;
  import java.util.ArrayList;
  import java.util.Collections;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Objects;
  import java.util.Set;
  import java.util.TreeMap;
  import javax.annotation.Nullable;
  
  import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Lists;
  import com.google.common.collect.Maps;
  import com.google.common.collect.Sets;
  import com.google.common.io.ByteStreams;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
  import org.apache.cassandra.exceptions.ConfigurationException;
  import org.apache.cassandra.io.util.File;
  import org.apache.cassandra.utils.JsonUtils;
  import org.apache.cassandra.utils.LocalizeString;
  import org.yaml.snakeyaml.DumperOptions;
  import org.yaml.snakeyaml.LoaderOptions;
  import org.yaml.snakeyaml.TypeDescription;
  import org.yaml.snakeyaml.Yaml;
  import org.yaml.snakeyaml.composer.Composer;
  import org.yaml.snakeyaml.constructor.CustomClassLoaderConstructor;
  import org.yaml.snakeyaml.constructor.SafeConstructor;
  import org.yaml.snakeyaml.error.YAMLException;
  import org.yaml.snakeyaml.introspector.MissingProperty;
  import org.yaml.snakeyaml.introspector.Property;
  import org.yaml.snakeyaml.introspector.PropertyUtils;
  import org.yaml.snakeyaml.nodes.Node;
  import org.yaml.snakeyaml.nodes.Tag;
  import org.yaml.snakeyaml.parser.ParserImpl;
  import org.yaml.snakeyaml.representer.Representer;
  import org.yaml.snakeyaml.resolver.Resolver;
  
  import static org.apache.cassandra.config.CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES;
  import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_DUPLICATE_CONFIG_KEYS;
  import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_NEW_OLD_CONFIG_KEYS;
  import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
  import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_ENVIRONMENT_VARIABLES;
  import static org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES;
  import static org.apache.cassandra.config.Replacements.getNameReplacements;
  
  public class YamlConfigurationLoader implements ConfigurationLoader
  {
      final static Set<String> OVERRIDABLE_CONFIG_NAMES;
  
      static
      {
          // Configs can be overriden via system properties and environment variables entirely or partially
          // For example: sai_options.prioritize_over_legacy_index=true or
          //              sai_options: {prioritize_over_legacy_index=true, segment_write_buffer_size=100MiB}
          Loader loader = Properties.defaultLoader();
          Map<String, Property> topLevelConfigs = loader.getProperties(Config.class);
          Map<String, Property> flattenedConfigs = Properties.flatten(loader, loader.getProperties(Config.class), Properties.DELIMITER, true);
          OVERRIDABLE_CONFIG_NAMES = Collections.unmodifiableSet(Sets.union(topLevelConfigs.keySet(), flattenedConfigs.keySet()));
      }
  
      private static final Logger logger = LoggerFactory.getLogger(YamlConfigurationLoader.class);
  
      /**
       * This is related to {@link Config#PROPERTY_PREFIX} but is different to make sure Config properties updated via
       * system properties do not conflict with other system properties; the name "settings" matches system_views.settings.
       */
      static final String SYSTEM_PROPERTY_PREFIX = "cassandra.settings.";
      static final String ENVIRONMENT_VARIABLE_PREFIX = "CASSANDRA_SETTINGS_";
      public static final String NESTED_CONFIG_SEPARATOR = ".";
      public static final String NESTED_CONFIG_SEPARATOR_ENVIRONMENT = "__";
  
      /**
       * Inspect the classpath to find storage configuration file
       */
      private static URL getStorageConfigURL() throws ConfigurationException
      {
          String configUrl = CASSANDRA_CONFIG.getString();
  
          URL url;
          try
          {
              url = new URL(configUrl);
              url.openStream().close(); // catches well-formed but bogus URLs
          }
          catch (Exception e)
          {
              ClassLoader loader = DatabaseDescriptor.class.getClassLoader();
              url = loader.getResource(configUrl);
              if (url == null)
              {
                  String required = "file:" + File.pathSeparator() + File.pathSeparator();
                  if (!configUrl.startsWith(required))
                      throw new ConfigurationException(String.format(
                          "Expecting URI in variable: [cassandra.config]. Found[%s]. Please prefix the file with [%s%s] for local " +
                          "files and [%s<server>%s] for remote files. If you are executing this from an external tool, it needs " +
                          "to set Config.setClientMode(true) to avoid loading configuration.",
                          configUrl, required, File.pathSeparator(), required, File.pathSeparator()));
                  throw new ConfigurationException("Cannot locate " + configUrl + ".  If this is a local file, please confirm you've provided " + required + File.pathSeparator() + " as a URI prefix.");
              }
          }
  
          logger.info("Configuration location: {}", url);
  
          return url;
      }
  
      private static URL storageConfigURL;
  
      @Override
      public Config loadConfig() throws ConfigurationException
      {
          if (storageConfigURL == null)
              storageConfigURL = getStorageConfigURL();
  
          return loadConfig(storageConfigURL);
      }
  
      public Config loadConfig(URL url) throws ConfigurationException
      {
          try
          {
              logger.debug("Loading settings from {}", url);
              byte[] configBytes;
              try (InputStream is = url.openStream())
              {
                  configBytes = ByteStreams.toByteArray(is);
              }
              catch (IOException e)
              {
                  // getStorageConfigURL should have ruled this out
                  throw new AssertionError(e);
              }
  
              return loadConfig(configBytes);
          }
          catch (YAMLException e)
          {
              throw new ConfigurationException("Invalid yaml: " + url, e);
          }
      }
  
      @VisibleForTesting
      static Config loadConfig(byte[] configBytes)
      {
          SafeConstructor constructor = new CustomConstructor(Config.class, Yaml.class.getClassLoader());
          Map<Class<?>, Map<String, Replacement>> replacements = getNameReplacements(Config.class);
          verifyReplacements(replacements, configBytes);
          PropertiesChecker propertiesChecker = new PropertiesChecker(replacements);
          constructor.setPropertyUtils(propertiesChecker);
          Yaml yaml = new Yaml(constructor);
          Config result = loadConfig(yaml, configBytes);
          propertiesChecker.check();
          maybeAddEnvironmentVariables(result);
          maybeAddSystemProperties(result);
          return result;
      }
  
      private static void maybeAddSystemProperties(Object obj)
      {
          if (CONFIG_ALLOW_SYSTEM_PROPERTIES.getBoolean())
          {
              Map<String, String> orderedPropertiesMap = new TreeMap<>();
              java.util.Properties props = System.getProperties();
              props.stringPropertyNames().forEach(key -> orderedPropertiesMap.put(key, props.getProperty(key)));
  
              Map<String, Object> overridingProperties = new HashMap<>();
              for (String originalKey : orderedPropertiesMap.keySet())
              {
                  if (originalKey.startsWith(SYSTEM_PROPERTY_PREFIX))
                  {
                      String value = props.getProperty(originalKey);
                      String configKey = originalKey.replace(SYSTEM_PROPERTY_PREFIX, "");
                      if (OVERRIDABLE_CONFIG_NAMES.contains(configKey))
                      {
                          if (value != null && !overridingProperties.containsKey(configKey))
                          {
                              if (!DatabaseDescriptor.hasLoggedConfig()) // CASSANDRA-9909: Avoid flooding config during initialization
                                  logger.warn("Detected JVM property {}={} override for Cassandra configuration '{}'.", originalKey, value, configKey);
                              overridingProperties.put(configKey, getScalarOrJsonTree(value));
                          }
                      }
                      else
                      {
                          logger.warn("Used sytem property variable {} to override Cassandra configuration but there is no such system property counter-part to override.", originalKey);
                      }
                  }
              }
              if (!overridingProperties.isEmpty())
                  updateFromMap(maybeFlattenNestedProperties(overridingProperties), false, obj);
          }
      }
  
      private static void maybeAddEnvironmentVariables(Object obj)
      {
          if (CONFIG_ALLOW_ENVIRONMENT_VARIABLES.getBoolean(CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getBooleanOrDefault(false)))
          {
              Map<String, String> orderedEnvironmentMap = new TreeMap<>(System.getenv()); // checkstyle: suppress nearby 'blockSystemPropertyUsage'
              Map<String, Object> overridingProperties = new HashMap<>();
              for (Map.Entry<String, String> env : orderedEnvironmentMap.entrySet())
              {
                  String originalKey = env.getKey();
                  if (env.getKey().startsWith(ENVIRONMENT_VARIABLE_PREFIX))
                  {
                      String configKey = LocalizeString.toLowerCaseLocalized(originalKey.replace(ENVIRONMENT_VARIABLE_PREFIX, "")
                                                                                        .replace(NESTED_CONFIG_SEPARATOR_ENVIRONMENT, NESTED_CONFIG_SEPARATOR));
                      String configValue = env.getValue();
                      if (OVERRIDABLE_CONFIG_NAMES.contains(configKey))
                      {
                          if (configValue != null && !overridingProperties.containsKey(configKey))
                          {
                              if (!DatabaseDescriptor.hasLoggedConfig()) // CASSANDRA-9909: Avoid flooding config during initialization
                                  logger.warn("Detected environment variable {}={} override for Cassandra configuration '{}'.", originalKey, configValue, configKey);
                              overridingProperties.put(configKey, getScalarOrJsonTree(configValue));
                          }
                      }
                      else
                      {
                          logger.warn("Used environment property variable {} to override Cassandra configuration but there is no such environment property counter-part to override.", originalKey);
                      }
                  }
              }
              if (!overridingProperties.isEmpty())
                  updateFromMap(maybeFlattenNestedProperties(overridingProperties), false, obj);
          }
      }
  
      private static Map<String, Object> maybeFlattenNestedProperties(Map<String, Object> overridingProperties)
      {
          Map<String, Object> copyOfProperties = new HashMap<>(overridingProperties);
          for (Map.Entry<String, Object> entry : overridingProperties.entrySet())
          {
              String[] parts = entry.getKey().split("\\.");
              if (parts.length > 1 && !parts[parts.length - 1].equals("parameters") && !parts[parts.length - 1].equals("configurations"))
              {
                  if (entry.getValue() instanceof Map)
                  {
                      copyOfProperties.remove(entry.getKey());
                      for (Map.Entry<String, Object> mapEntry : ((Map<String, Object>) entry.getValue()).entrySet())
                      {
                          String newKey = entry.getKey() + '.' + mapEntry.getKey();
                          Object newValue = mapEntry.getValue();
                          copyOfProperties.put(newKey, newValue);
                      }
                  }
              }
          }
          return copyOfProperties;
      }
  
      private static Object getScalarOrJsonTree(String value)
      {
          try
          {
              return JsonUtils.decodeJson(value);
          }
          catch (Exception e)
          {
              return value;
          }
      }
  
      private static void verifyReplacements(Map<Class<?>, Map<String, Replacement>> replacements, Map<String, ?> rawConfig)
      {
          List<String> duplicates = new ArrayList<>();
          for (Map.Entry<Class<?>, Map<String, Replacement>> outerEntry : replacements.entrySet())
          {
              for (Map.Entry<String, Replacement> entry : outerEntry.getValue().entrySet())
              {
                  Replacement r = entry.getValue();
                  if (!r.isValueFormatReplacement() && rawConfig.containsKey(r.oldName) && rawConfig.containsKey(r.newName))
                  {
                      String msg = String.format("[%s -> %s]", r.oldName, r.newName);
                      duplicates.add(msg);
                  }
              }
          }
  
          if (!duplicates.isEmpty())
          {
              String msg = String.format("Config contains both old and new keys for the same configuration parameters, migrate old -> new: %s", String.join(", ", duplicates));
              if (!ALLOW_NEW_OLD_CONFIG_KEYS.getBoolean())
                  throw new ConfigurationException(msg);
              else
                  logger.warn(msg);
          }
      }
  
      private static void verifyReplacements(Map<Class<?>, Map<String, Replacement>> replacements, byte[] configBytes)
      {
          LoaderOptions loaderOptions = getDefaultLoaderOptions();
          loaderOptions.setAllowDuplicateKeys(ALLOW_DUPLICATE_CONFIG_KEYS.getBoolean());
          Yaml rawYaml = new Yaml(loaderOptions);
  
          Map<String, Object> rawConfig = rawYaml.load(new ByteArrayInputStream(configBytes));
          if (rawConfig == null)
              rawConfig = new HashMap<>();
          verifyReplacements(replacements, rawConfig);
  
      }
  
      @VisibleForTesting
      public static <T> T fromMap(Map<String,Object> map, Class<T> klass)
      {
          return fromMap(map, true, klass);
      }
  
      @SuppressWarnings("unchecked") //getSingleData returns Object, not T
      public static <T> T fromMap(Map<String,Object> map, boolean shouldCheck, Class<T> klass)
      {
          SafeConstructor constructor = new YamlConfigurationLoader.CustomConstructor(klass, klass.getClassLoader());
          Map<Class<?>, Map<String, Replacement>> replacements = getNameReplacements(Config.class);
          verifyReplacements(replacements, map);
          YamlConfigurationLoader.PropertiesChecker propertiesChecker = new YamlConfigurationLoader.PropertiesChecker(replacements);
          constructor.setPropertyUtils(propertiesChecker);
          Yaml yaml = new Yaml(constructor);
          Node node = yaml.represent(map);
          constructor.setComposer(getDefaultComposer(node));
          T value = (T) constructor.getSingleData(klass);
          if (shouldCheck)
              propertiesChecker.check();
          maybeAddEnvironmentVariables(value);
          maybeAddSystemProperties(value);
          return value;
      }
  
      public static <T> T updateFromMap(Map<String, ?> map, boolean shouldCheck, T obj)
      {
          Class<T> klass = (Class<T>) obj.getClass();
          SafeConstructor constructor = new YamlConfigurationLoader.CustomConstructor(klass, klass.getClassLoader())
          {
              @Override
              protected Object newInstance(Node node)
              {
                  if (node.getType() == obj.getClass())
                      return obj;
                  return super.newInstance(node);
              }
          };
          Map<Class<?>, Map<String, Replacement>> replacements = getNameReplacements(Config.class);
          verifyReplacements(replacements, map);
          YamlConfigurationLoader.PropertiesChecker propertiesChecker = new YamlConfigurationLoader.PropertiesChecker(replacements);
          constructor.setPropertyUtils(propertiesChecker);
          Yaml yaml = new Yaml(constructor);
          Node node = yaml.represent(map);
          constructor.setComposer(getDefaultComposer(node));
          T value = (T) constructor.getSingleData(klass);
          if (shouldCheck)
              propertiesChecker.check();
          return value;
      }
  
      public static String toYaml(Object map)
      {
          DumperOptions options = new DumperOptions();
          options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
          options.setExplicitStart(true);
          class NoTags extends Representer
          {
              public NoTags() {
                  super(options);
                  this.multiRepresenters.put(Enum.class, data -> representScalar(Tag.STR, ((Enum<?>) data).name()));
              }
          }
  
          Yaml yaml = new Yaml(new NoTags(), options);
  
          return yaml.dump(map);
      }
  
      private static Composer getDefaultComposer(Node node)
      {
          return new Composer(new ParserImpl(null), new Resolver(), getDefaultLoaderOptions())
          {
              @Override
              public Node getSingleNode()
              {
                  return node;
              }
          };
      }
  
      @VisibleForTesting
      static class CustomConstructor extends CustomClassLoaderConstructor
      {
          CustomConstructor(Class<?> theRoot, ClassLoader classLoader)
          {
              super(theRoot, classLoader, getDefaultLoaderOptions());
  
              TypeDescription seedDesc = new TypeDescription(ParameterizedClass.class);
              seedDesc.putMapPropertyType("parameters", String.class, String.class);
              addTypeDescription(seedDesc);
  
              TypeDescription memtableDesc = new TypeDescription(Config.MemtableOptions.class);
              memtableDesc.addPropertyParameters("configurations", String.class, InheritingClass.class);
              addTypeDescription(memtableDesc);
          }
  
          @Override
          protected List<Object> createDefaultList(int initSize)
          {
              return Lists.newCopyOnWriteArrayList();
          }
  
          @Override
          protected Map<Object, Object> createDefaultMap(int initSize)
          {
              return Maps.newConcurrentMap();
          }
  
          @Override
          protected Set<Object> createDefaultSet(int initSize)
          {
              return Sets.newConcurrentHashSet();
          }
      }
  
      private static Config loadConfig(Yaml yaml, byte[] configBytes)
      {
          Config config = yaml.loadAs(new ByteArrayInputStream(configBytes), Config.class);
          // If the configuration file is empty yaml will return null. In this case we should use the default
          // configuration to avoid hitting a NPE at a later stage.
          return config == null ? new Config() : config;
      }
  
      /**
       * Utility class to check that there are no extra properties and that properties that are not null by default
       * are not set to null.
       */
      @VisibleForTesting
      private static class PropertiesChecker extends PropertyUtils
      {
          private final Loader loader = Properties.defaultLoader();
          private final Set<String> missingProperties = new HashSet<>();
  
          private final Set<String> nullProperties = new HashSet<>();
  
          private final Set<String> deprecationWarnings = new HashSet<>();
  
          private final Map<Class<?>, Map<String, Replacement>> replacements;
  
          PropertiesChecker(Map<Class<?>, Map<String, Replacement>> replacements)
          {
              this.replacements = Objects.requireNonNull(replacements, "Replacements should not be null");
              setSkipMissingProperties(true);
          }
  
          @Override
          public Property getProperty(Class<?> type, String name)
          {
              final Property result;
              Map<String, Replacement> typeReplacements = replacements.getOrDefault(type, Collections.emptyMap());
              if (typeReplacements.containsKey(name))
              {
                  Replacement replacement = typeReplacements.get(name);
                  result = replacement.toProperty(getProperty0(type, replacement.newName));
  
                  if (replacement.deprecated)
                      deprecationWarnings.add(replacement.oldName);
              }
              else
              {
                  result = getProperty0(type, name);
              }
  
              if (result instanceof MissingProperty)
              {
                  missingProperties.add(result.getName());
              }
              else if (result.getAnnotation(Deprecated.class) != null)
              {
                  deprecationWarnings.add(result.getName());
              }
  
              return new ForwardingProperty(result.getName(), result)
              {
                  boolean allowsNull = result.getAnnotation(Nullable.class) != null;
  
                  @Override
                  public void set(Object object, Object value) throws Exception
                  {
                      // TODO: CASSANDRA-17785, add @Nullable to all nullable Config properties and remove value == null
                      if (value == null && get(object) != null && !allowsNull)
                          nullProperties.add(getName());
  
                      result.set(object, value);
                  }
  
                  @Override
                  public Object get(Object object)
                  {
                      return result.get(object);
                  }
              };
          }
  
          private Property getProperty0(Class<? extends Object> type, String name)
          {
              if (name.contains(NESTED_CONFIG_SEPARATOR))
                  return getNestedProperty(type, name);
              return getFlatProperty(type, name);
          }
  
          private Property getFlatProperty(Class<?> type, String name)
          {
              Property prop = loader.getProperties(type).get(name);
              return prop == null ? new MissingProperty(name) : prop;
          }
  
          private Property getNestedProperty(Class<?> type, String name)
          {
              Property root = null;
              for (String s : name.split("\\."))
              {
                  Property prop = getFlatProperty(type, s);
                  if (prop instanceof MissingProperty)
                  {
                      root = null;
                      break;
                  }
                  root = root == null ? prop : Properties.andThen(root, prop);
                  type = root.getType();
              }
              return root != null ? root : new MissingProperty(name);
          }
  
          public void check() throws ConfigurationException
          {
              if (!nullProperties.isEmpty())
                  throw new ConfigurationException("Invalid yaml. Those properties " + nullProperties + " are not valid", false);
  
              if (!missingProperties.isEmpty())
                  throw new ConfigurationException("Invalid yaml. Please remove properties " + missingProperties + " from your cassandra.yaml", false);
  
              if (!deprecationWarnings.isEmpty())
                  logger.warn("{} parameters have been deprecated. They have new names and/or value format; For more information, please refer to NEWS.txt", deprecationWarnings);
          }
      }
  
      public static LoaderOptions getDefaultLoaderOptions()
      {
          LoaderOptions loaderOptions = new LoaderOptions();
          loaderOptions.setCodePointLimit(64 * 1024 * 1024); // 64 MiB
          loaderOptions.setWarnOnDuplicateKeys(false);
          return loaderOptions;
      }
  }
  