<commit_before>#HADOOP SPECIFIC FUNCTIONS
source_file "$ALOJA_REPO_PATH/shell/common/common_java.sh"
set_java_requires

get_hadoop_config_folder() {
  local config_folder_name

  if [ "$HADOOP_CUSTOM_CONFIG" ] ; then
    config_folder_name="$HADOOP_CUSTOM_CONFIG"
  elif [ "$HADOOP_EXTRA_JARS" == "AOP4Hadoop" ] ; then
    config_folder_name="hadoop1_AOP_conf_template"
  elif [ "$(get_hadoop_major_version)" == "2" ]; then
    config_folder_name="hadoop2_conf_template"
  else
    config_folder_name="hadoop1_conf_template"
  fi

  echo -e "$config_folder_name"
}

set_hadoop_config_folder() {
  BENCH_CONFIG_FOLDERS="$BENCH_CONFIG_FOLDERS
$(get_hadoop_config_folder)"
}

# Sets the required files to download/copy
set_hadoop_requires() {
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    BENCH_REQUIRED_FILES["$HADOOP_VERSION"]="http://archive.apache.org/dist/hadoop/core/$HADOOP_VERSION/$HADOOP_VERSION.tar.gz"
  else
    BENCH_REQUIRED_FILES["$HADOOP_VERSION"]="http://archive.apache.org/dist/hadoop/core/$HADOOP_VERSION/$HADOOP_VERSION-bin.tar.gz"
  fi

  if [ "$HADOOP_EXTRA_JARS" ] ; then
    BENCH_REQUIRED_FILES["HADOOP_EXTRA_JARS"]="$ALOJA_PUBLIC_HTTP/aplic2/tarballs/$HADOOP_EXTRA_JARS.tar.gz"
  fi

  #also set the config here
  set_hadoop_config_folder
}

# Helper to print a line with Hadoop requiered exports
get_hadoop_exports() {
  local to_export

  # For both versions
  to_export="$(get_java_exports)
export HADOOP_CONF_DIR='$HDD/conf';
export HADOOP_LOG_DIR='$HDD/logs';
export HADOOP_HOME='$(get_local_apps_path)/${HADOOP_VERSION}';
export HADOOP_OPTS='$HADOOP_OPTS';"

  # For v2 only
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    to_export="$to_export
export HADOOP_YARN_HOME='$(get_local_apps_path)/${HADOOP_VERSION}';
YARN_LOG_DIR='$HDD/logs';"
  fi

  if [ "$HADOOP_EXTRA_JARS" ] ; then
    # Right now jar files are hard-coded
    to_export="$to_export
export HADOOP_USER_CLASSPATH_FIRST=true;
export HADOOP_CLASSPATH=$(get_local_apps_path)/$HADOOP_EXTRA_JARS/aspectjrt-1.6.5.jar:$(get_local_apps_path)/$HADOOP_EXTRA_JARS/AOP4Hadoop-hadoop-core-1.0.3.jar:\$HADOOP_CLASSPATH;"
  fi

  echo -e "$to_export\n"
}

# Function to return job specific config
# rr in the case of PaaS where we cannot change the server config
get_hadoop_job_config() {
  local job_config="$BENCH_EXTRA_CONFIG"

  # For v2 only
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    job_config+=" -D mapreduce.job.maps $MAX_MAPS"
    job_config+=" -D mapreduce.job.reduces $MAX_MAPS"
  else
    job_config+=" -D mapred.map.tasks $MAX_MAPS"
    job_config+=" -D mapred.reduce.tasks $MAX_MAPS"
  fi

  echo -e "$job_config"
}


# Get the list of slaves
# TODO should be improved to include master node as worker node if necessary
# $1 list of nodes
# $2 master name
get_hadoop_slaves() {
  local all_nodes="$1"
  local master_name="$2"
  local only_slaves

  if [ "$all_nodes" ] && [ "$master_name" ] ; then
    only_slaves="$(echo -e "$all_nodes"|grep -v "$master_name")"<commit_msg>DEPLOY: Fixed custom configs for Hadoop function<commit_after>#HADOOP SPECIFIC FUNCTIONS
source_file "$ALOJA_REPO_PATH/shell/common/common_java.sh"
set_java_requires

get_hadoop_config_folder() {
  local config_folder_name

  if [ "$HADOOP_CUSTOM_CONFIG" ] ; then
    config_folder_name="$HADOOP_CUSTOM_CONFIG"
  elif [ "$HADOOP_EXTRA_JARS" == "AOP4Hadoop" ] ; then
    config_folder_name="hadoop1_AOP_conf_template"
  elif [ "$(get_hadoop_major_version)" == "2" ]; then
    config_folder_name="hadoop2_conf_template"
  else
    config_folder_name="hadoop1_conf_template"
  fi

  echo -e "$config_folder_name"
}

set_hadoop_config_folder() {
  BENCH_CONFIG_FOLDERS="$BENCH_CONFIG_FOLDERS
$(get_hadoop_config_folder)"
}

# Sets the required files to download/copy
set_hadoop_requires() {
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    BENCH_REQUIRED_FILES["$HADOOP_VERSION"]="http://archive.apache.org/dist/hadoop/core/$HADOOP_VERSION/$HADOOP_VERSION.tar.gz"
  else
    BENCH_REQUIRED_FILES["$HADOOP_VERSION"]="http://archive.apache.org/dist/hadoop/core/$HADOOP_VERSION/$HADOOP_VERSION-bin.tar.gz"
  fi

  if [ "$HADOOP_EXTRA_JARS" ] ; then
    BENCH_REQUIRED_FILES["HADOOP_EXTRA_JARS"]="$ALOJA_PUBLIC_HTTP/aplic2/tarballs/$HADOOP_EXTRA_JARS.tar.gz"
  fi

  #also set the config here
  set_hadoop_config_folder
}

# Helper to print a line with Hadoop requiered exports
get_hadoop_exports() {
  local to_export

  # For both versions
  to_export="$(get_java_exports)
export HADOOP_CONF_DIR='$HDD/conf';
export HADOOP_LOG_DIR='$HDD/logs';
export HADOOP_HOME='$(get_local_apps_path)/${HADOOP_VERSION}';
export HADOOP_OPTS='$HADOOP_OPTS';"

  # For v2 only
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    to_export="$to_export
export HADOOP_YARN_HOME='$(get_local_apps_path)/${HADOOP_VERSION}';
YARN_LOG_DIR='$HDD/logs';"
  fi

  if [ "$HADOOP_EXTRA_JARS" ] ; then
    # Right now jar files are hard-coded
    to_export="$to_export
export HADOOP_USER_CLASSPATH_FIRST=true;
export HADOOP_CLASSPATH=$(get_local_apps_path)/$HADOOP_EXTRA_JARS/aspectjrt-1.6.5.jar:$(get_local_apps_path)/$HADOOP_EXTRA_JARS/AOP4Hadoop-hadoop-core-1.0.3.jar:\$HADOOP_CLASSPATH;"
  fi

  echo -e "$to_export\n"
}

# Function to return job specific config
# rr in the case of PaaS where we cannot change the server config
get_hadoop_job_config() {
  local job_config="$BENCH_EXTRA_CONFIG"

  # For v2 only
  if [ "$(get_hadoop_major_version)" == "2" ]; then
    job_config+=" -D mapreduce.job.maps='$MAX_MAPS'"
    job_config+=" -D mapreduce.job.reduces='$MAX_MAPS'"
  else
    job_config+=" -D mapred.map.tasks='$MAX_MAPS'"
    job_config+=" -D mapred.reduce.tasks='$MAX_MAPS'"
  fi

  echo -e "$job_config"
}


# Get the list of slaves
# TODO should be improved to include master node as worker node if necessary
# $1 list of nodes
# $2 master name
get_hadoop_slaves() {
  local all_nodes="$1"
  local master_name="$2"
  local only_slaves

  if [ "$all_nodes" ] && [ "$master_name" ] ; then
    only_slaves="$(echo -e "$all_nodes"|grep -v "$master_name")"
{'fraction_non_alphanumeric': 0.12791659771636604, 'fraction_numerical': 0.0072811517458216115, 'mean_word_length': 5.355415352260778, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7032761', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2381, 'n_words': 409}