<commit_before>---
#Setting up Hadoop
- name: Create service account for Hadoop
  user: name={{ hadoop_user }}
        system=yes
        state=present
        groups="{{ hadoop_user_groups | join(',') }}"
  tags: ["hadoop-user"]

- name: Create a .ssh Directory.
  file: path=/data/local/home/{{ hadoop_user }}/.ssh owner="{{ hadoop_user }}" group="{{ hadoop_user_groups[0] }}" state=directory mode=0700

- name: Lets copy the template id_rsa to auth_keys location.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa.pub" dest="/data/local/home/{{ hadoop_user }}/.ssh/authorized_keys" mode=644

- name: Lets copy id_rsa to location .ssh.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa" dest="/data/local/home/{{ hadoop_user }}/.ssh/id_rsa" mode=600

- name: Lets copy id_rsa.pub to location .ssh.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa.pub" dest="/data/local/home/{{ hadoop_user }}/.ssh/id_rsa.pub" mode=644

- name: Ensure Hadoop directory exists
  file: path="{{ hadoop_dir }}" state=directory owner={{ hadoop_user }} group={{ hadoop_user_groups[0] }} mode=g+rw

- name: Ensure Hadoop configuration directory exists
  file: path="{{ hadoop_parent_conf_dir }}" state=directory
  tags: ["config"]

- name: Ensure Hadoop log and run directories exist
  file: path="{{ item }}"
        owner={{ hadoop_user }}
        group={{ hadoop_user_groups[0] }}
        mode=0755
        state=directory
  with_items:
    - "{{ hadoop_log_dir }}"
    - "{{ hadoop_run_dir }}"

- stat: path="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"
  register: hadoop_dirs

- name: Remove Hadoop directories if we are installing a new version
  shell: "{{ item }}"
  with_items:
    - sudo rm -fr {{ hadoop_src_dir }}/hadoop*
    - sudo rm -fr {{ hadoop_usr_parent_dir }}/hadoop*
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Remove symlinks
  file:
    path: "{{ item }}"
    state: absent
  with_items:
    - "{{ hadoop_usr_dir }}"
    - "{{ hadoop_conf_dir }}"
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Remove the shortcuts
  shell: "{{ item }}"
  with_items:
    - sudo rm -f /usr/bin/hadoop
    - sudo rm -f /usr/bin/hdfs
    - sudo rm -f /usr/bin/yarn
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Download Hadoop distribution
  get_url: url="{{ hadoop_mirror }}/hadoop-{{ hadoop_version }}//hadoop-{{ hadoop_version }}.tar.gz"
           dest="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"

- name: Extract Hadoop distribution
  unarchive: src="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"
             dest="{{ hadoop_usr_parent_dir }}"
             copy=no
             creates="{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}"

- name: remove the Hadoop doc dir
  shell: "sudo rm -fr {{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}/share/doc"

- name: Setup Hadoop distribution symlinks
  file: src="{{ item.src }}"
        dest="{{ item.dest }}"
        state=link
  with_items:
    - { src: "{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}", dest: "{{ hadoop_usr_dir }}" }
    - { src: "{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}/etc/hadoop", dest: "{{ hadoop_conf_dir }}" }
  tags: ["symlinks"]

- name: Create shims for Hadoop binaries
  template: src=hadoop-shim.j2
            dest="/usr/bin/{{ item }}"
            mode=0755
  with_items:
    - hadoop
    - hdfs
    - yarn
  tags: ["shims"]

- name: Configure hadoop core-site
  template: src=core-site.xml.j2
            dest="{{ hadoop_conf_dir }}/core-site.xml"
  tags: ["core-site"]

- name: Configure hadoop hdfs-site
  template: src=hdfs-site.xml.j2<commit_msg>Avoid the use of shell and use Ansbile module file.<commit_after>---
#Setting up Hadoop
- name: Create service account for Hadoop
  user: name={{ hadoop_user }}
        system=yes
        state=present
        groups="{{ hadoop_user_groups | join(',') }}"
  tags: ["hadoop-user"]

- name: Create a .ssh Directory.
  file: path=/data/local/home/{{ hadoop_user }}/.ssh owner="{{ hadoop_user }}" group="{{ hadoop_user_groups[0] }}" state=directory mode=0700

- name: Lets copy the template id_rsa to auth_keys location.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa.pub" dest="/data/local/home/{{ hadoop_user }}/.ssh/authorized_keys" mode=644

- name: Lets copy id_rsa to location .ssh.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa" dest="/data/local/home/{{ hadoop_user }}/.ssh/id_rsa" mode=600

- name: Lets copy id_rsa.pub to location .ssh.
  copy: src="{{ role_path }}/../../files/hadoop_id_rsa.pub" dest="/data/local/home/{{ hadoop_user }}/.ssh/id_rsa.pub" mode=644

- name: Ensure Hadoop directory exists
  file: path="{{ hadoop_dir }}" state=directory owner={{ hadoop_user }} group={{ hadoop_user_groups[0] }} mode=g+rw

- name: Ensure Hadoop configuration directory exists
  file: path="{{ hadoop_parent_conf_dir }}" state=directory
  tags: ["config"]

- name: Ensure Hadoop log and run directories exist
  file: path="{{ item }}"
        owner={{ hadoop_user }}
        group={{ hadoop_user_groups[0] }}
        mode=0755
        state=directory
  with_items:
    - "{{ hadoop_log_dir }}"
    - "{{ hadoop_run_dir }}"

- stat: path="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"
  register: hadoop_dirs

- name: Remove Hadoop directories if we are installing a new version
  shell: "{{ item }}"
  with_items:
    - sudo rm -fr {{ hadoop_src_dir }}/hadoop*
    - sudo rm -fr {{ hadoop_usr_parent_dir }}/hadoop*
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Remove symlinks
  file:
    path: "{{ item }}"
    state: absent
  with_items:
    - "{{ hadoop_usr_dir }}"
    - "{{ hadoop_conf_dir }}"
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Remove the shortcuts
  shell: "{{ item }}"
  with_items:
    - sudo rm -f /usr/bin/hadoop
    - sudo rm -f /usr/bin/hdfs
    - sudo rm -f /usr/bin/yarn
  when: hadoop_dirs.stat.exists is defined and not hadoop_dirs.stat.exists

- name: Download Hadoop distribution
  get_url: url="{{ hadoop_mirror }}/hadoop-{{ hadoop_version }}//hadoop-{{ hadoop_version }}.tar.gz"
           dest="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"

- name: Extract Hadoop distribution
  unarchive: src="{{ hadoop_src_dir }}/hadoop-{{ hadoop_version }}.tar.gz"
             dest="{{ hadoop_usr_parent_dir }}"
             copy=no
             creates="{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}"

- name: Remove the Hadoop doc dir
  file:
    state: absent
    path: "{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}/share/doc/"

- name: Setup Hadoop distribution symlinks
  file: src="{{ item.src }}"
        dest="{{ item.dest }}"
        state=link
  with_items:
    - { src: "{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}", dest: "{{ hadoop_usr_dir }}" }
    - { src: "{{ hadoop_usr_parent_dir }}/hadoop-{{ hadoop_version }}/etc/hadoop", dest: "{{ hadoop_conf_dir }}" }
  tags: ["symlinks"]

- name: Create shims for Hadoop binaries
  template: src=hadoop-shim.j2
            dest="/usr/bin/{{ item }}"
            mode=0755
  with_items:
    - hadoop
    - hdfs
    - yarn
  tags: ["shims"]

- name: Configure hadoop core-site
  template: src=core-site.xml.j2
            dest="{{ hadoop_conf_dir }}/core-site.xml"
  tags: ["core-site"]

- name: Configure hadoop hdfs-site
  template: src=hdfs-site.xml.j2
{'fraction_non_alphanumeric': 0.15987752928647497, 'fraction_numerical': 0.00718849840255591, 'mean_word_length': 3.218416619876474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1976636', 'n_tokens_mistral': 2701, 'n_tokens_neox': 2625, 'n_words': 648}