<commit_before>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio;

import alluxio.exception.ExceptionMessage;
import alluxio.exception.PreconditionMessage;
import alluxio.network.ChannelType;
import alluxio.util.ConfigurationUtils;
import alluxio.util.FormatUtils;
import alluxio.util.OSUtils;

import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import io.netty.util.internal.chmv8.ConcurrentHashMapV8;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.management.ManagementFactory;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.concurrent.NotThreadSafe;

/**
 * <p>
 * All the runtime configuration properties of Alluxio. This class works like a dictionary and
 * serves each Alluxio configuration property as a key-value pair.
 *
 * <p>
 * Alluxio configuration properties are loaded into this class in the following order with
 * decreasing priority:
 * <ol>
 * <li>Java system properties;</li>
 * <li>Environment variables via {@code alluxio-env.sh} or from OS settings;</li>
 * <li>Site specific properties via {@code alluxio-site.properties} file;</li>
 * <li>Default properties via {@code alluxio-default.properties} file.</li>
 * </ol>
 *
 * <p>
 * The default properties are defined in a property file {@code alluxio-default.properties}
 * distributed with Alluxio jar. Alluxio users can override values of these default properties by
 * creating {@code alluxio-site.properties} and putting it under java {@code CLASSPATH} when running
 * Alluxio (e.g., ${ALLUXIO_HOME}/conf/)
 */
@NotThreadSafe
public final class Configuration {
  private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);

  /** Regex string to find "${key}" for variable substitution. */
  private static final String REGEX_STRING = "(\\$\\{([^{}]*)\\})";
  /** Regex to find ${key} for variable substitution. */
  private static final Pattern CONF_REGEX = Pattern.compile(REGEX_STRING);
  /** Map of properties. */
  private static final ConcurrentHashMapV8<String, String> PROPERTIES = new ConcurrentHashMapV8<>();

  /** File to set customized properties for Alluxio server (both master and worker) and client. */
  public static final String SITE_PROPERTIES = "alluxio-site.properties";

  static {
    defaultInit();
  }

  /**
   * Initializes the default {@link Configuration}.
   *
   * The order of preference is (1) system properties, (2) properties in the specified file, (3)
   * default property values.
   */
  public static void defaultInit() {
    // Load default
    Properties defaultProps = new Properties();
    for (PropertyKey key : PropertyKey.values()) {
      String value = key.getDefaultValue();
      if (value != null) {
        defaultProps.setProperty(key.toString(), value);
      }
    }
    // Override runtime default
    defaultProps.setProperty(PropertyKey.WORKER_NETWORK_NETTY_CHANNEL.toString(),
        String.valueOf(ChannelType.defaultType()));
    defaultProps.setProperty(PropertyKey.USER_NETWORK_NETTY_CHANNEL.toString(),
        String.valueOf(ChannelType.defaultType()));
    // Set ramdisk volume according to OS type
    if (OSUtils.isLinux()) {
      defaultProps
          .setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(), "/mnt/ramdisk");
    } else if (OSUtils.isMacOS()) {
      defaultProps.setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(),
          "/Volumes/ramdisk");
    }
    // Set a reasonable default for worker memory
    try {
      com.sun.management.OperatingSystemMXBean operatingSystemMXBean =
          (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
      long memSize = operatingSystemMXBean.getTotalPhysicalMemorySize();
      defaultProps
          .setProperty(PropertyKey.WORKER_MEMORY_SIZE.toString(), String.valueOf(memSize * 2 / 3));
    } catch (Exception e) {
      // The package com.sun.management may not be available on every platform.
      // fallback to use the default value
    }

    // Load system properties
    Properties systemProps = new Properties();
    systemProps.putAll(System.getProperties());

    // Now lets combine, order matters here
    PROPERTIES.clear();
    merge(defaultProps);
    merge(systemProps);

    // Load site specific properties file if not in test mode. Note that we decide whether in test
    // mode by default properties and system properties (via getBoolean). If it is not in test mode
    // the PROPERTIES will be updated again.
    if (!getBoolean(PropertyKey.TEST_MODE)) {
      String confPaths = get(PropertyKey.SITE_CONF_DIR);
      String[] confPathList = confPaths.split(",");
      Properties siteProps = ConfigurationUtils.searchPropertiesFile(SITE_PROPERTIES, confPathList);
      // Update site properties and system properties in order
      if (siteProps != null) {
        merge(siteProps);
        merge(systemProps);
      }
    }

    // TODO(andrew): get rid of the MASTER_ADDRESS property key
    if (containsKey(PropertyKey.MASTER_HOSTNAME)) {
      String masterHostname = get(PropertyKey.MASTER_HOSTNAME);
      String masterPort = get(PropertyKey.MASTER_RPC_PORT);
      boolean useZk = Boolean.parseBoolean(get(PropertyKey.ZOOKEEPER_ENABLED));
      String masterAddress =
          (useZk ? Constants.HEADER_FT : Constants.HEADER) + masterHostname + ":" + masterPort;
      set(PropertyKey.MASTER_ADDRESS, masterAddress);
    }

    Preconditions.checkState(validate());
    checkConfigurationValues();
  }

  /**
   * Merges the current configuration properties with alternate properties. A property from the new
   * configuration wins if it also appears in the current configuration.
   *
   * @param properties The source {@link Properties} to be merged
   */
  public static void merge(Map<?, ?> properties) {
    if (properties != null) {
      // merge the system properties
      for (Map.Entry<?, ?> entry : properties.entrySet()) {
        String key = entry.getKey().toString();
        String value = entry.getValue().toString();
        if (PropertyKey.isValid(key)) {
          PROPERTIES.put(key, value);
        }
      }
    }
    checkUserFileBufferBytes();
  }

  // Public accessor methods

  // TODO(binfan): this method should be hidden and only used during initialization and tests.

  /**<commit_msg>[ALLUXIO-2542] Address comments<commit_after>/*
 * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
 * (the "License"). You may not use this work except in compliance with the License, which is
 * available at www.apache.org/licenses/LICENSE-2.0
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied, as more fully set forth in the License.
 *
 * See the NOTICE file distributed with this work for information regarding copyright ownership.
 */

package alluxio;

import alluxio.exception.ExceptionMessage;
import alluxio.exception.PreconditionMessage;
import alluxio.network.ChannelType;
import alluxio.util.ConfigurationUtils;
import alluxio.util.FormatUtils;
import alluxio.util.OSUtils;

import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.sun.management.OperatingSystemMXBean;
import io.netty.util.internal.chmv8.ConcurrentHashMapV8;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.management.ManagementFactory;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.concurrent.NotThreadSafe;

/**
 * <p>
 * All the runtime configuration properties of Alluxio. This class works like a dictionary and
 * serves each Alluxio configuration property as a key-value pair.
 *
 * <p>
 * Alluxio configuration properties are loaded into this class in the following order with
 * decreasing priority:
 * <ol>
 * <li>Java system properties;</li>
 * <li>Environment variables via {@code alluxio-env.sh} or from OS settings;</li>
 * <li>Site specific properties via {@code alluxio-site.properties} file;</li>
 * <li>Default properties via {@code alluxio-default.properties} file.</li>
 * </ol>
 *
 * <p>
 * The default properties are defined in a property file {@code alluxio-default.properties}
 * distributed with Alluxio jar. Alluxio users can override values of these default properties by
 * creating {@code alluxio-site.properties} and putting it under java {@code CLASSPATH} when running
 * Alluxio (e.g., ${ALLUXIO_HOME}/conf/)
 */
@NotThreadSafe
public final class Configuration {
  private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);

  /** Regex string to find "${key}" for variable substitution. */
  private static final String REGEX_STRING = "(\\$\\{([^{}]*)\\})";
  /** Regex to find ${key} for variable substitution. */
  private static final Pattern CONF_REGEX = Pattern.compile(REGEX_STRING);
  /** Map of properties. */
  private static final ConcurrentHashMapV8<String, String> PROPERTIES = new ConcurrentHashMapV8<>();

  /** File to set customized properties for Alluxio server (both master and worker) and client. */
  public static final String SITE_PROPERTIES = "alluxio-site.properties";

  static {
    defaultInit();
  }

  /**
   * Initializes the default {@link Configuration}.
   *
   * The order of preference is (1) system properties, (2) properties in the specified file, (3)
   * default property values.
   */
  public static void defaultInit() {
    // Load default
    Properties defaultProps = createDefaultProps();

    // Load system properties
    Properties systemProps = new Properties();
    systemProps.putAll(System.getProperties());

    // Now lets combine, order matters here
    PROPERTIES.clear();
    merge(defaultProps);
    merge(systemProps);

    // Load site specific properties file if not in test mode. Note that we decide whether in test
    // mode by default properties and system properties (via getBoolean). If it is not in test mode
    // the PROPERTIES will be updated again.
    if (!getBoolean(PropertyKey.TEST_MODE)) {
      String confPaths = get(PropertyKey.SITE_CONF_DIR);
      String[] confPathList = confPaths.split(",");
      Properties siteProps = ConfigurationUtils.searchPropertiesFile(SITE_PROPERTIES, confPathList);
      // Update site properties and system properties in order
      if (siteProps != null) {
        merge(siteProps);
        merge(systemProps);
      }
    }

    // TODO(andrew): get rid of the MASTER_ADDRESS property key
    if (containsKey(PropertyKey.MASTER_HOSTNAME)) {
      String masterHostname = get(PropertyKey.MASTER_HOSTNAME);
      String masterPort = get(PropertyKey.MASTER_RPC_PORT);
      boolean useZk = Boolean.parseBoolean(get(PropertyKey.ZOOKEEPER_ENABLED));
      String masterAddress =
          (useZk ? Constants.HEADER_FT : Constants.HEADER) + masterHostname + ":" + masterPort;
      set(PropertyKey.MASTER_ADDRESS, masterAddress);
    }

    Preconditions.checkState(validate());
    checkConfigurationValues();
  }

  /**
   * @return default properties
   */
  private static Properties createDefaultProps() {
    Properties defaultProps = new Properties();
    // Load compile-time default
    for (PropertyKey key : PropertyKey.values()) {
      String value = key.getDefaultValue();
      if (value != null) {
        defaultProps.setProperty(key.toString(), value);
      }
    }

    // Load run-time default
    defaultProps.setProperty(PropertyKey.WORKER_NETWORK_NETTY_CHANNEL.toString(),
        String.valueOf(ChannelType.defaultType()));
    defaultProps.setProperty(PropertyKey.USER_NETWORK_NETTY_CHANNEL.toString(),
        String.valueOf(ChannelType.defaultType()));
    // Set ramdisk volume according to OS type
    if (OSUtils.isLinux()) {
      defaultProps
          .setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(), "/mnt/ramdisk");
    } else if (OSUtils.isMacOS()) {
      defaultProps.setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(),
          "/Volumes/ramdisk");
    }
    // Set a reasonable default size for worker memory
    try {
      OperatingSystemMXBean operatingSystemMXBean =
          (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
      long memSize = operatingSystemMXBean.getTotalPhysicalMemorySize();
      defaultProps
          .setProperty(PropertyKey.WORKER_MEMORY_SIZE.toString(), String.valueOf(memSize * 2 / 3));
    } catch (Exception e) {
      // The package com.sun.management may not be available on every platform.
      // fallback to the compile-time default value
    }
    return defaultProps;
  }

  /**
   * Merges the current configuration properties with alternate properties. A property from the new
   * configuration wins if it also appears in the current configuration.
   *
   * @param properties The source {@link Properties} to be merged
   */
  public static void merge(Map<?, ?> properties) {
    if (properties != null) {
      // merge the system properties
      for (Map.Entry<?, ?> entry : properties.entrySet()) {
        String key = entry.getKey().toString();
        String value = entry.getValue().toString();
        if (PropertyKey.isValid(key)) {
          PROPERTIES.put(key, value);
        }
      }
    }
    checkUserFileBufferBytes();
  }

  // Public accessor methods

  // TODO(binfan): this method should be hidden and only used during initialization and tests.

  /**
{'fraction_non_alphanumeric': 0.10110562547806133, 'fraction_numerical': 0.002642375356372992, 'mean_word_length': 3.723152709359606, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1211617', 'n_tokens_mistral': 3942, 'n_tokens_neox': 3914, 'n_words': 1204}