<commit_before># Run hive and hadoop locally
- name: Install required packages
  apt: name={{ item }} state=present
  with_items:
    - htop
    - openjdk-7-jre
    - openjdk-7-jdk

- name: Configure hadoop user
  user: name=hadoop shell=/bin/bash home=/home/hadoop uid=711
- name: Set permissions for hadoop user
  file: path=/home/hadoop state=directory owner=hadoop mode=755
- name: Setup bashrc
  template: src=bashrc dest=/home/hadoop/.bashrc owner=hadoop
- name: Setup shared env file
  template: src=hadoop-env.sh dest=/usr/local/etc/hadoop-env.sh
- name: SSH folder for hadoop user
  file: path=/home/hadoop/.ssh state=directory owner=hadoop mode=700
- name: SSH key for hadoop user
  command: "ssh-keygen -t rsa -P '' -f /home/hadoop/.ssh/id_rsa"
  args:
    creates: /home/hadoop/.ssh/id_rsa
  sudo_user: hadoop
  register: setup_hadoop_private_key
- name: Install SSH key for hadoop user
  shell: "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"
  sudo_user: hadoop
  when: setup_hadoop_private_key.changed
- name: Setup known_hosts
  command: "ssh -o StrictHostKeyChecking=no localhost echo"
  sudo_user: hadoop
- name: Setup known_hosts (2)
  command: "ssh -o StrictHostKeyChecking=no 0.0.0.0 echo"
  sudo_user: hadoop


# Install Hadoop
- name: Downloading Hadoop binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/hadoop/common/hadoop-{{hadoop_version}}/hadoop-{{hadoop_version}}.tar.gz"
    dest="/var/cache/ansible-hadoop-{{hadoop_version}}.tar.gz"
  register: hadoop_download
- name: Deleting any existing Hadoop installation
  file: path=/usr/local/hadoop state=absent
  when: hadoop_download.changed
- name: Prep for Hadoop installation
  file: path=/usr/local/hadoop state=directory
- name: Extracting Hadoop
  command: "tar -xzf /var/cache/ansible-hadoop-{{hadoop_version}}.tar.gz --directory /usr/local/hadoop --strip-components=1"
  args:
    creates: /usr/local/hadoop/bin/hadoop

# Configure Hadoop
- name: Configure Hadoop (JAVA_HOME)
  lineinfile:
    dest="/usr/local/hadoop/etc/hadoop/hadoop-env.sh"
    line='export JAVA_HOME="{{java_home}}"'
    regexp='export JAVA_HOME=.*'
- name: Configure Hadoop (HADOOP_PREFIX)
  lineinfile:
    dest="/usr/local/hadoop/etc/hadoop/hadoop-env.sh"
    line='export HADOOP_PREFIX="/usr/local/hadoop"'
    regexp='export HADOOP_PREFIX=.*'
    insertafter='export JAVA_HOME=.*'
- name: Configure Hadoop (single-node pseudo-distributed mode)
  template: src=hadoop-site.xml dest=/usr/local/hadoop/etc/hadoop/core-site.xml
- name: Configure HDFS (single-node pseudo-distributed mode)
  template: src=hdfs-site.xml dest=/usr/local/hadoop/etc/hadoop/hdfs-site.xml
- name: Create Hadoop logs folder
  file: path=/home/hadoop/logs state=directory owner=hadoop
- name: Initialize HDFS
  command: /usr/local/hadoop/bin/hdfs namenode -format
  args:
    creates: /home/hadoop/hdfs/name/current/VERSION
  sudo_user: hadoop
  register: initialize_hdfs


# Install Hive
- name: Downloading Hive binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/hive/hive-{{hive_version}}/apache-hive-{{hive_version}}-bin.tar.gz"
    dest="/var/cache/ansible-hive-{{hive_version}}.tar.gz"
  register: hive_download
- name: Deleting any existing Hive installation
  file: path=/usr/local/hive state=absent
  when: hive_download.changed
- name: Prep for Hive installation
  file: path=/usr/local/hive state=directory
- name: Extracting Hive
  command: "tar -xzf /var/cache/ansible-hive-{{hive_version}}.tar.gz --directory /usr/local/hive --strip-components=1"
  args:
    creates: /usr/local/hive/bin/hive

# Configure Hive
- name: Configure Hive
  template: src=hive-site.xml dest=/usr/local/hive/conf/hive-site.xml

# Install Sqoop
- name: Downloading Sqoop binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/sqoop/{{sqoop_version}}/sqoop-{{sqoop_version}}.bin__hadoop-2.0.4-alpha.tar.gz"
    dest="/var/cache/ansible-sqoop-{{sqoop_version}}.tar.gz"
  register: sqoop_download
- name: Deleting any existing Sqoop installation
  file: path=/usr/local/sqoop state=absent
  when: sqoop_download.changed
- name: Prep for Sqoop installation
  file: path=/usr/local/sqoop state=directory
- name: Extracting Sqoop
  command: "tar -xzf /var/cache/ansible-sqoop-{{sqoop_version}}.tar.gz --directory /usr/local/sqoop --strip-components=1"
  args:
    creates: /usr/local/sqoop/bin/sqoop

# Upgrade Sqoop commons-io to avoid isSymlink symbol not found error
- name: Remove outdated commons-io jar
  file: path=/usr/local/sqoop/lib/commons-io-1.4.jar state=absent
- name: Installing new commons-io into Sqoop lib dir
  command: cp /usr/local/hadoop/share/hadoop/mapreduce/lib/commons-io-2.4.jar /usr/local/sqoop/lib/commons-io-2.4.jar
  args:
    creates: /usr/local/sqoop/lib/commons-io-2.4.jar

# Install MySQL JDBC driver for Sqoop
- name: Downloading MySQL JDBC Driver
  get_url:
    url="http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-{{mysql_connector_java_version}}.tar.gz"
    dest="/var/cache/ansible-mysql-connector-java-{{mysql_connector_java_version}}.tar.gz"
  register: sqoop_mysql_driver_download
- name: Delete any existing MySQL JDBC jar
  shell: "rm -f /usr/local/sqoop/lib/mysql-connector-java-*.jar"
  when: sqoop_mysql_driver_download.changed
- name: Extracting MySQL JDBC tarball
  command: "tar --directory /usr/local/sqoop/lib/ -xzf /var/cache/ansible-mysql-connector-java-{{mysql_connector_java_version}}.tar.gz mysql-connector-java-{{mysql_connector_java_version}}/mysql-connector-java-{{mysql_connector_java_version}}-bin.jar --strip-components=1"
  args:
    creates: "/usr/local/sqoop/lib/mysql-connector-java-{{mysql_connector_java_version}}-bin.jar"

# Load into HDFS some jars that Sqoop requires
- name: Ensure HDFS is running
  command: "bash -c 'source ~/.bashrc && ps x|grep java|grep proc_namenode || start-dfs.sh'"
  sudo_user: "hadoop"
- name: Prepare to load Sqoop Jars into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -mkdir -p /usr/local/sqoop/lib/
  sudo_user: "hadoop"
- name: Load Sqoop lib Jars into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -put -f /usr/local/sqoop/lib/ /usr/local/sqoop/
  sudo_user: "hadoop"
- name: Load Sqoop into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -put -f /usr/local/sqoop/sqoop-1.4.6.jar /usr/local/sqoop/
  sudo_user: "hadoop"<commit_msg>Fix bugs with HDFS startup<commit_after># Run hive and hadoop locally
- name: Install required packages
  apt: name={{ item }} state=present
  with_items:
    - htop
    - openjdk-7-jre
    - openjdk-7-jdk

- name: Configure hadoop user
  user: name=hadoop shell=/bin/bash home=/home/hadoop uid=711
- name: Set permissions for hadoop user
  file: path=/home/hadoop state=directory owner=hadoop mode=755
- name: Setup bashrc
  template: src=bashrc dest=/home/hadoop/.bashrc owner=hadoop
- name: Setup shared env file
  template: src=hadoop-env.sh dest=/usr/local/etc/hadoop-env.sh
- name: SSH folder for hadoop user
  file: path=/home/hadoop/.ssh state=directory owner=hadoop mode=700
- name: SSH key for hadoop user
  command: "ssh-keygen -t rsa -P '' -f /home/hadoop/.ssh/id_rsa"
  args:
    creates: /home/hadoop/.ssh/id_rsa
  sudo_user: hadoop
  register: setup_hadoop_private_key
- name: Install SSH key for hadoop user
  shell: "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"
  sudo_user: hadoop
  when: setup_hadoop_private_key.changed
- name: Setup known_hosts
  command: "ssh -o StrictHostKeyChecking=no localhost echo"
  sudo_user: hadoop
- name: Setup known_hosts (2)
  command: "ssh -o StrictHostKeyChecking=no 0.0.0.0 echo"
  sudo_user: hadoop


# Install Hadoop
- name: Downloading Hadoop binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/hadoop/common/hadoop-{{hadoop_version}}/hadoop-{{hadoop_version}}.tar.gz"
    dest="/var/cache/ansible-hadoop-{{hadoop_version}}.tar.gz"
  register: hadoop_download
- name: Deleting any existing Hadoop installation
  file: path=/usr/local/hadoop state=absent
  when: hadoop_download.changed
- name: Prep for Hadoop installation
  file: path=/usr/local/hadoop state=directory
- name: Extracting Hadoop
  command: "tar -xzf /var/cache/ansible-hadoop-{{hadoop_version}}.tar.gz --directory /usr/local/hadoop --strip-components=1"
  args:
    creates: /usr/local/hadoop/bin/hadoop

# Configure Hadoop
- name: Configure Hadoop (JAVA_HOME)
  lineinfile:
    dest="/usr/local/hadoop/etc/hadoop/hadoop-env.sh"
    line='export JAVA_HOME="{{java_home}}"'
    regexp='export JAVA_HOME=.*'
- name: Configure Hadoop (HADOOP_PREFIX)
  lineinfile:
    dest="/usr/local/hadoop/etc/hadoop/hadoop-env.sh"
    line='export HADOOP_PREFIX="/usr/local/hadoop"'
    regexp='export HADOOP_PREFIX=.*'
    insertafter='export JAVA_HOME=.*'
- name: Configure Hadoop (single-node pseudo-distributed mode)
  template: src=hadoop-site.xml dest=/usr/local/hadoop/etc/hadoop/core-site.xml
- name: Configure HDFS (single-node pseudo-distributed mode)
  template: src=hdfs-site.xml dest=/usr/local/hadoop/etc/hadoop/hdfs-site.xml
- name: Create Hadoop logs folder
  file: path=/home/hadoop/logs state=directory owner=hadoop
- name: Initialize HDFS
  command: /usr/local/hadoop/bin/hdfs namenode -format
  args:
    creates: /home/hadoop/hdfs/name/current/VERSION
  sudo_user: hadoop
  register: initialize_hdfs


# Install Hive
- name: Downloading Hive binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/hive/hive-{{hive_version}}/apache-hive-{{hive_version}}-bin.tar.gz"
    dest="/var/cache/ansible-hive-{{hive_version}}.tar.gz"
  register: hive_download
- name: Deleting any existing Hive installation
  file: path=/usr/local/hive state=absent
  when: hive_download.changed
- name: Prep for Hive installation
  file: path=/usr/local/hive state=directory
- name: Extracting Hive
  command: "tar -xzf /var/cache/ansible-hive-{{hive_version}}.tar.gz --directory /usr/local/hive --strip-components=1"
  args:
    creates: /usr/local/hive/bin/hive

# Configure Hive
- name: Configure Hive
  template: src=hive-site.xml dest=/usr/local/hive/conf/hive-site.xml

# Install Sqoop
- name: Downloading Sqoop binary package
  get_url:
    url="http://apache.sunsite.ualberta.ca/sqoop/{{sqoop_version}}/sqoop-{{sqoop_version}}.bin__hadoop-2.0.4-alpha.tar.gz"
    dest="/var/cache/ansible-sqoop-{{sqoop_version}}.tar.gz"
  register: sqoop_download
- name: Deleting any existing Sqoop installation
  file: path=/usr/local/sqoop state=absent
  when: sqoop_download.changed
- name: Prep for Sqoop installation
  file: path=/usr/local/sqoop state=directory
- name: Extracting Sqoop
  command: "tar -xzf /var/cache/ansible-sqoop-{{sqoop_version}}.tar.gz --directory /usr/local/sqoop --strip-components=1"
  args:
    creates: /usr/local/sqoop/bin/sqoop

# Upgrade Sqoop commons-io to avoid isSymlink symbol not found error
- name: Remove outdated commons-io jar
  file: path=/usr/local/sqoop/lib/commons-io-1.4.jar state=absent
- name: Installing new commons-io into Sqoop lib dir
  command: cp /usr/local/hadoop/share/hadoop/mapreduce/lib/commons-io-2.4.jar /usr/local/sqoop/lib/commons-io-2.4.jar
  args:
    creates: /usr/local/sqoop/lib/commons-io-2.4.jar

# Install MySQL JDBC driver for Sqoop
- name: Downloading MySQL JDBC Driver
  get_url:
    url="http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-{{mysql_connector_java_version}}.tar.gz"
    dest="/var/cache/ansible-mysql-connector-java-{{mysql_connector_java_version}}.tar.gz"
  register: sqoop_mysql_driver_download
- name: Delete any existing MySQL JDBC jar
  shell: "rm -f /usr/local/sqoop/lib/mysql-connector-java-*.jar"
  when: sqoop_mysql_driver_download.changed
- name: Extracting MySQL JDBC tarball
  command: "tar --directory /usr/local/sqoop/lib/ -xzf /var/cache/ansible-mysql-connector-java-{{mysql_connector_java_version}}.tar.gz mysql-connector-java-{{mysql_connector_java_version}}/mysql-connector-java-{{mysql_connector_java_version}}-bin.jar --strip-components=1"
  args:
    creates: "/usr/local/sqoop/lib/mysql-connector-java-{{mysql_connector_java_version}}-bin.jar"

# Load into HDFS some jars that Sqoop requires
- name: Ensure HDFS is running
  command: "bash -c 'source ~/.bashrc && ps h -C java|grep proc_namenode || (start-dfs.sh && hdfs dfsadmin -safemode wait)'"
  sudo_user: "hadoop"
- name: Prepare to load Sqoop Jars into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -mkdir -p /usr/local/sqoop/lib/
  sudo_user: "hadoop"
- name: Load Sqoop lib Jars into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -put -f /usr/local/sqoop/lib/ /usr/local/sqoop/
  sudo_user: "hadoop"
- name: Load Sqoop into HDFS
  command: /usr/local/hadoop/bin/hdfs dfs -put -f /usr/local/sqoop/sqoop-1.4.6.jar /usr/local/sqoop/
  sudo_user: "hadoop"
{'fraction_non_alphanumeric': 0.13372687918142465, 'fraction_numerical': 0.005352223534041716, 'mean_word_length': 4.052087475149105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '402266', 'n_tokens_mistral': 4703, 'n_tokens_neox': 4479, 'n_words': 960}