<commit_before>#!/usr/bin/env sh
#
# Cookbook Name:: hadoop_cluster
# Script Name::   bootstrap_hadoop_namenode
#
# Copyright 2011, Infochimps, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Make the standard HDFS directories:
#
#   /tmp
#   /user
#   /user/hive/warehouse
#
# and
#
#   /user/USERNAME
#
# for each user in the 'supergroup' group. Quoting Tom White:
#   "The [chmod +w] is questionable, as it allows a user to delete another
#    user. It's needed to allow users to create their own user directories"
#

#
# If you need to blow away an existing HDFS, see /etc/hadoop/conf/nuke_hdfs_from_orbit_its_the_only_way_to_be_sure.rb
#

set -e # die if any command fails

hdfs_user=hdfs
dfs_data_dirs="<%=     @hadoop[:datanode   ][:data_dirs].join(" ") %>"
dfs_name_dirs="<%=     @hadoop[:namenode   ][:data_dirs].join(" ") %>"
dfs_2nn_dirs="<%=      @hadoop[:secondarynn][:data_dirs].join(" ") %>"
dfs_name_dir_root="<%= @hadoop[:namenode   ][:data_dirs].last      %>"
hadoop_tmp_dir="<%=    @hadoop[:tmp_dir] %>"
hadoop_log_dir="<%=    @hadoop[:log_dir] %>"
namenode_daemon="<%= "#{@hadoop[:handle]}-namenode" %>"
sentinel="$hadoop_tmp_dir/made_initial_dirs.log"

make_hadoop_dir_if_not_exists() {
  dir="$1"
  owner="${hdfs_user}:hadoop"
  mode=${2-700}
  if [ ! -d "$dir" ] ; then
    echo     "  creating '$dir' mode $mode owner $owner"
    set -v
    mkdir -p "$dir"
    chmod    "$mode"  "$dir"
    chown    "$owner" "$dir"
    set +v
  fi      
}

#
echo "Ensuring hadoop directories exist"
#
for dir in $dfs_data_dirs $dfs_name_dirs $dfs_2nn_dirs ; do
  make_hadoop_dir_if_not_exists "$dir"
done
make_hadoop_dir_if_not_exists "$hadoop_tmp_dir" 777
make_hadoop_dir_if_not_exists "$hadoop_log_dir" 775

#
echo "Stopping the $namenode_daemon daemon" ; echo -n "  "
#
sudo service $namenode_daemon stop || sleep 3 ; true

#
echo "Formatting namenode in '$dfs_name_dirs'"
echo
#
if [ -f "$dfs_name_dir_root/current/VERSION" ] && [ -f "$dfs_name_dir_root/current/fsimage" ] ; then
  echo "Hadoop namenode appears to be formatted: skipping"
else
  echo "Executing namenode format!"
  sudo -u "$hdfs_user" hadoop namenode -format
fi
echo

#
echo "Restarting the namenode" ; echo -n "  "
#
sudo service $namenode_daemon start

#
echo "Waiting for the namenode to come out of safemode" ; echo -n "  "
#
hadoop dfsadmin -safemode wait

#
echo "Preparing filesystem"
#
if [ -f "$sentinel" ] ; then
  echo "Looks like we already made the hdfs dirs -- try 'hadoop fs -lsr /' to see. if not, remove '$sentinel' and re-run."
else
  set +e
  hadoop_users=/user/"`grep supergroup /etc/group | cut -d: -f4 | sed -e 's|,| /user/|g'`"
  hadoop_users="/user/ubuntu $hadoop_users"
  echo "   /tmp /user /user/hive/warehouse $hadoop_users"
  sudo -u "$hdfs_user" hadoop fs -mkdir           /tmp /user /user/hive/warehouse $hadoop_users
  sudo -u "$hdfs_user" hadoop fs -chmod a+w       /tmp /user /user/hive/warehouse
  echo "   /hadoop/mapred/system /hadoop/mapred/staging"
  sudo -u "$hdfs_user" hadoop fs -mkdir           <%= @hadoop[:jobtracker][:system_hdfsdir] %> <%= @hadoop[:jobtracker][:staging_hdfsdir] %>
  sudo -u "$hdfs_user" hadoop fs -chmod 700       <%= @hadoop[:jobtracker][:system_hdfsdir] %> <%= @hadoop[:jobtracker][:staging_hdfsdir] %>
  sudo -u "$hdfs_user" hadoop fs -chown -R mapred <%= ::File.dirname(@hadoop[:jobtracker][:system_hdfsdir]) %>
  echo "   /hadoop/hbase"
  sudo -u "$hdfs_user" hadoop fs -mkdir           /hadoop/hbase
  sudo -u "$hdfs_user" hadoop fs -chown -R hbase  /hadoop/hbase
  for user in $hadoop_users ; do
    sudo -u "$hdfs_user" hadoop fs -chown ${user#/user/} $user
  done 
  touch "$sentinel"
  set -e
fi

echo
echo "Success! you can start the rest of the daemons now by running:"
echo
echo '  for foo in datanode secondarynamenode tasktracker jobtracker ; do echo $foo ; sudo service hadoop-0.20-$foo start ; done'
echo<commit_msg>Cleaning up hadoop bootstrap script, per Flip<commit_after>#!/usr/bin/env sh
#
# Cookbook Name:: hadoop_cluster
# Script Name::   bootstrap_hadoop_namenode
#
# Copyright 2011, Infochimps, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Make the standard HDFS directories:
#
#   /tmp
#   /user
#   /user/hive/warehouse
#
# and
#
#   /user/USERNAME
#
# for each user in the 'supergroup' group. Quoting <NAME>:
#   "The [chmod +w] is questionable, as it allows a user to delete another
#    user. It's needed to allow users to create their own user directories"
#

#
# If you need to blow away an existing HDFS, see /etc/hadoop/conf/nuke_hdfs_from_orbit_its_the_only_way_to_be_sure.rb
#

set -e # die if any command fails

hdfs_user=hdfs
dfs_data_dirs="<%=     @hadoop[:datanode   ][:data_dirs].join(" ") %>"
dfs_name_dirs="<%=     @hadoop[:namenode   ][:data_dirs].join(" ") %>"
dfs_2nn_dirs="<%=      @hadoop[:secondarynn][:data_dirs].join(" ") %>"
dfs_name_dir_root="<%= @hadoop[:namenode   ][:data_dirs].last      %>"
hadoop_tmp_dir="<%=    @hadoop[:tmp_dir] %>"
hadoop_log_dir="<%=    @hadoop[:log_dir] %>"
namenode_daemon="hadoop_namenode"
sentinel="$hadoop_tmp_dir/made_initial_dirs.log"

make_hadoop_dir_if_not_exists() {
  dir="$1"
  owner="${hdfs_user}:hadoop"
  mode=${2-700}
  if [ ! -d "$dir" ] ; then
    echo     "  creating '$dir' mode $mode owner $owner"
    set -v
    mkdir -p "$dir"
    chmod    "$mode"  "$dir"
    chown    "$owner" "$dir"
    set +v
  fi      
}

#
echo "Ensuring hadoop directories exist"
#
for dir in $dfs_data_dirs $dfs_name_dirs $dfs_2nn_dirs ; do
  make_hadoop_dir_if_not_exists "$dir"
done
make_hadoop_dir_if_not_exists "$hadoop_tmp_dir" 777
make_hadoop_dir_if_not_exists "$hadoop_log_dir" 777

#
echo "Stopping the $namenode_daemon daemon" ; echo -n "  "
#
sudo service $namenode_daemon stop || sleep 3 ; true

#
echo "Formatting namenode in '$dfs_name_dirs'"
echo
#
if [ -f "$dfs_name_dir_root/current/VERSION" ] && [ -f "$dfs_name_dir_root/current/fsimage" ] ; then
  echo "Hadoop namenode appears to be formatted: skipping"
else
  echo "Executing namenode format!"
  sudo -u "$hdfs_user" hadoop namenode -format
fi
echo

#
echo "Restarting the namenode" ; echo -n "  "
#
sudo service $namenode_daemon start

#
echo "Waiting for the namenode to come out of safemode" ; echo -n "  "
#
hadoop dfsadmin -safemode wait

#
echo "Preparing filesystem"
#
if [ -f "$sentinel" ] ; then
  echo "Looks like we already made the hdfs dirs -- try 'hadoop fs -lsr /' to see. if not, remove '$sentinel' and re-run."
else
  set +e
  hadoop_users=/user/"`grep supergroup /etc/group | cut -d: -f4 | sed -e 's|,| /user/|g'`"
  hadoop_users="/user/ubuntu $hadoop_users"
  echo "   /tmp /user /user/hive/warehouse $hadoop_users"
  sudo -u "$hdfs_user" hadoop fs -mkdir           /tmp /user /user/hive/warehouse $hadoop_users
  sudo -u "$hdfs_user" hadoop fs -chmod a+w       /tmp /user /user/hive/warehouse
  echo "   /hadoop/mapred/system /hadoop/mapred/staging"
  sudo -u "$hdfs_user" hadoop fs -mkdir           <%= @hadoop[:jobtracker][:system_hdfsdir] %> <%= @hadoop[:jobtracker][:staging_hdfsdir] %>
  sudo -u "$hdfs_user" hadoop fs -chmod 700       <%= @hadoop[:jobtracker][:system_hdfsdir] %> <%= @hadoop[:jobtracker][:staging_hdfsdir] %>
  sudo -u "$hdfs_user" hadoop fs -chown -R mapred <%= ::File.dirname(@hadoop[:jobtracker][:system_hdfsdir]) %>
  echo "   /hadoop/hbase"
  sudo -u "$hdfs_user" hadoop fs -mkdir           /hadoop/hbase
  sudo -u "$hdfs_user" hadoop fs -chown -R hbase  /hadoop/hbase
  for user in $hadoop_users ; do
    sudo -u "$hdfs_user" hadoop fs -chown ${user#/user/} $user
  done 
  touch "$sentinel"
  set -e
fi

echo
echo "Success! you can start the rest of the daemons now by running:"
echo
echo '  for foo in datanode secondarynamenode tasktracker jobtracker ; do echo $foo ; sudo service hadoop_$foo start ; done'
echo
{'fraction_non_alphanumeric': 0.13297629899726526, 'fraction_numerical': 0.006267092069279855, 'mean_word_length': 3.833149779735683, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5622849', 'n_tokens_mistral': 3345, 'n_tokens_neox': 3194, 'n_words': 925}