<commit_before>#!/bin/bash

# Establish a Local Repository with the appropriate mirrors and templates
#  that can be used by the cluster.  This script will use `hostname` to replace
#  all references in templates to the server this script is being run on. make
#  sure the hostname is correct and resolvable from ALL cluster hosts that will
#  need this repository.

# Ensure script being run as root.
if [ `whoami` != "root" ]; then
echo "Must be root to run this script."
exit -1
fi

HOSTNAME=${1:-`hostname`}

echo "Using $HOSTNAME as the hostname for the repo server."

# Determine if httpd has been installed
HTTPD=`yum list installed | grep httpd`
if [ "$HTTPD" == "" ]; then
# Install httpd
yum -y install httpd
service httpd start
fi

# install the HDP repos
wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/1.x/GA/1.3.0.0/hdp.repo -O /etc/yum.repos.d/hdp.repo
wget -nv http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.2.4.9/ambari.repo -O /etc/yum.repos.d/ambari.repo

yum repolist

# Install the epel repo
yum -y install epel-release
yum -y install yum-utils
yum -y install createrepo

# Copy the repo templates to the httpd server


# Create directories for repos
# Attempting to mimic the published repo structure as much as possible
BASE_REPO_DIR="/var/www/html/repos"

# Copy the repo templates to the httpd server
if [ ! -d  $BASE_REPO_DIR/local.yum.repos.d ]; then
mkdir -p $BASE_REPO_DIR/local.yum.repos.d
fi

if [ ! -d $BASE_REPO_DIR/jdk ]; then
mkdir -p $BASE_REPO_DIR/jdk
fi

if [ ! -f $BASE_REPO_DIR/jdk/jdk-6u31-linux-x64.bin ]; then
wget -nv http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin -O $BASE_REPO_DIR/jdk/jdk-6u31-linux-x64.bin
fi

if [ -d $BASE_REPO_DIR/local.yum.repos.d ]; then
rm $BASE_REPO_DIR/local.yum.repos.d
fi

mkdir -p $BASE_REPO_DIR/local.yum.repos.d

wget https://raw.github.com/dstreev/HWX-PS-Utils/master/scripts/local.repo.n.ambari/templates/ambari.repo -O $BASE_REPO_DIR/local.yum.repos.d/ambari.repo

wget https://raw.github.com/dstreev/HWX-PS-Utils/master/scripts/local.repo.n.ambari/templates/CentOS-Base.repo -O $BASE_REPO_DIR/local.yum.repos.d/CentOS-Base.repo

sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/local.yum.repos.d/ambari.repo
sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/local.yum.repos.d/CentOS-Base.repo

# ambari-1.x
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/GA
if [ ! -d  $BASE_REPO_DIR/ambari/centos6/1.x/GA ]; then
mkdir -p $BASE_REPO_DIR/ambari/centos6/1.x/GA
fi

# HDP-UTILS-1.1.0.15
# http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.15/repos/centos6
if [ ! -d  $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6 ]; then
mkdir -p $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6
fi

# Updates-ambari-1.2.4.9
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.2.4.9
if [ ! -d  $BASE_REPO_DIR/ambari/centos6/1.x/updates/1.2.4.9 ]; then
mkdir -p $BASE_REPO_DIR/ambari/centos6/1.x/updates/1.2.4.9
fi

# epel
# baseurl=http://download.fedoraproject.org/pub/epel/6/$basearch (x86_64)
if [ ! -d  $BASE_REPO_DIR/pub/epel/6/x86_64 ]; then
mkdir -p $BASE_REPO_DIR/pub/epel/6/x86_64
fi

# HDP-1.3.0.0
# http://public-repo-1.hortonworks.com/HDP/centos6/1.x/GA/1.3.0.0
if [ ! -d  $BASE_REPO_DIR/HDP/centos6/1.x/GA/1.3.0.0 ]; then
mkdir -p $BASE_REPO_DIR/HDP/centos6/1.x/GA/1.3.0.0
fi

# Need to get the Ambari templates
if [ -d $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos ]; then
rm -rf $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos
fi

# Get fresh copies
mkdir -p $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos
if [ -f $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml ];
then
# Cleanup old copy.
rm $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml
fi
wget https://raw.github.com/dstreev/HWX-PS-Utils/master/scripts/local.repo.n.ambari/templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml -O $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml
sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml



# Install the repo for Ambari and HDP
echo ""
echo "================================="
echo "Syncing ambari-1.x repo..."
echo "---------------------------------"
reposync -r ambari-1.x -p $BASE_REPO_DIR/ambari/centos6/1.x/GA --norepopath
echo ""
echo "================================="
echo "Syncing HDP-UTILS-1.1.0.15 repo..."
echo "---------------------------------"
reposync -r HDP-UTILS-1.1.0.15 -p $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6 --norepopath
echo ""
echo "================================="<commit_msg>Adjust wgets to pull from my 'dstreev' github, which will be sync'd with this repo (hwx).  Need to do this because the hwx repo won't be visible during installation.<commit_after>#!/bin/bash

# Establish a Local Repository with the appropriate mirrors and templates
#  that can be used by the cluster.  This script will use `hostname` to replace
#  all references in templates to the server this script is being run on. make
#  sure the hostname is correct and resolvable from ALL cluster hosts that will
#  need this repository.

# Ensure script being run as root.
if [ `whoami` != "root" ]; then
echo "Must be root to run this script."
exit -1
fi

HOSTNAME=${1:-`hostname`}

echo "Using $HOSTNAME as the hostname for the repo server."

# Determine if httpd has been installed
HTTPD=`yum list installed | grep httpd`
if [ "$HTTPD" == "" ]; then
# Install httpd
yum -y install httpd
service httpd start
fi

# install the HDP repos
wget -nv http://public-repo-1.hortonworks.com/HDP/centos6/1.x/GA/1.3.0.0/hdp.repo -O /etc/yum.repos.d/hdp.repo
wget -nv http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.2.4.9/ambari.repo -O /etc/yum.repos.d/ambari.repo

yum repolist

# Install the epel repo
yum -y install epel-release
yum -y install yum-utils
yum -y install createrepo

# Copy the repo templates to the httpd server


# Create directories for repos
# Attempting to mimic the published repo structure as much as possible
BASE_REPO_DIR="/var/www/html/repos"

# Copy the repo templates to the httpd server
if [ ! -d  $BASE_REPO_DIR/local.yum.repos.d ]; then
mkdir -p $BASE_REPO_DIR/local.yum.repos.d
fi

if [ ! -d $BASE_REPO_DIR/jdk ]; then
mkdir -p $BASE_REPO_DIR/jdk
fi

if [ ! -f $BASE_REPO_DIR/jdk/jdk-6u31-linux-x64.bin ]; then
wget -nv http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin -O $BASE_REPO_DIR/jdk/jdk-6u31-linux-x64.bin
fi

if [ -d $BASE_REPO_DIR/local.yum.repos.d ]; then
rm $BASE_REPO_DIR/local.yum.repos.d
fi

mkdir -p $BASE_REPO_DIR/local.yum.repos.d

wget https://raw.github.com/dstreev/HWX-PS-Utils/master/templates/ambari.repo -O $BASE_REPO_DIR/local.yum.repos.d/ambari.repo

wget https://raw.github.com/dstreev/HWX-PS-Utils/master/templates/CentOS-Base.repo -O $BASE_REPO_DIR/local.yum.repos.d/CentOS-Base.repo

sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/local.yum.repos.d/ambari.repo
sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/local.yum.repos.d/CentOS-Base.repo

# ambari-1.x
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/GA
if [ ! -d  $BASE_REPO_DIR/ambari/centos6/1.x/GA ]; then
mkdir -p $BASE_REPO_DIR/ambari/centos6/1.x/GA
fi

# HDP-UTILS-1.1.0.15
# http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.15/repos/centos6
if [ ! -d  $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6 ]; then
mkdir -p $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6
fi

# Updates-ambari-1.2.4.9
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos6/1.x/updates/1.2.4.9
if [ ! -d  $BASE_REPO_DIR/ambari/centos6/1.x/updates/1.2.4.9 ]; then
mkdir -p $BASE_REPO_DIR/ambari/centos6/1.x/updates/1.2.4.9
fi

# epel
# baseurl=http://download.fedoraproject.org/pub/epel/6/$basearch (x86_64)
if [ ! -d  $BASE_REPO_DIR/pub/epel/6/x86_64 ]; then
mkdir -p $BASE_REPO_DIR/pub/epel/6/x86_64
fi

# HDP-1.3.0.0
# http://public-repo-1.hortonworks.com/HDP/centos6/1.x/GA/1.3.0.0
if [ ! -d  $BASE_REPO_DIR/HDP/centos6/1.x/GA/1.3.0.0 ]; then
mkdir -p $BASE_REPO_DIR/HDP/centos6/1.x/GA/1.3.0.0
fi

# Need to get the Ambari templates
if [ -d $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos ]; then
rm -rf $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos
fi

# Get fresh copies
mkdir -p $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos
if [ -f $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml ];
then
# Cleanup old copy.
rm $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml
fi
wget https://raw.github.com/dstreev/HWX-PS-Utils/master/templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml -O $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml
sed -i bak -e "s:!local.repo.host!:$HOSTNAME:g" $BASE_REPO_DIR/../templates/ambari-server/resources/stacks/HDPLocal/1.3.0/repos/repoinfo.xml



# Install the repo for Ambari and HDP
echo ""
echo "================================="
echo "Syncing ambari-1.x repo..."
echo "---------------------------------"
reposync -r ambari-1.x -p $BASE_REPO_DIR/ambari/centos6/1.x/GA --norepopath
echo ""
echo "================================="
echo "Syncing HDP-UTILS-1.1.0.15 repo..."
echo "---------------------------------"
reposync -r HDP-UTILS-1.1.0.15 -p $BASE_REPO_DIR/HDP-UTILS-1.1.0.15/repos/centos6 --norepopath
echo ""
echo "================================="
{'fraction_non_alphanumeric': 0.18393623543838136, 'fraction_numerical': 0.03453913754342939, 'mean_word_length': 3.5499767549976755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 34, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3839396', 'n_tokens_mistral': 4250, 'n_tokens_neox': 4072, 'n_words': 742}