<commit_before>---
- name: get cpu
  tags: [configure_map_reduce, upgrade, scale]
  command: "nproc --all"
  register: node_cpu
  changed_when: false
- name: get host available memory
  tags: [upgrade, configure_yarn, scale]
  shell: "cat /proc/meminfo | grep MemTotal | awk '{print int($2/1024)}'"
  register: out
  changed_when: false
- name: get available memory
  set_fact:
    available_memory_mb: "{{ out.stdout }}"
  tags: [upgrade, configure_yarn, scale]
- name: get available memory for yarn node manager
  set_fact:
    yarn_nodemanager_memory_mb: "{{ (available_memory_mb | int - 800 )| int }}"
  tags: [upgrade, configure_yarn, scale]
- name: crypted disk is running
  tags: [check_hosts, upgrade]
  shell: "if mountpoint -q /{{ crypted_disk }}; then echo 'MOUNTED'; else echo 'NOT-MOUNTED'; fi"
  register: mount_output
  changed_when: false
- name: start crypted disk
  tags: [check_hosts, upgrade]
  shell: |
    echo -n "{{ decrypt_disk_pass }}" | cryptsetup luksOpen --key-file=- /dev/{{ inventory_hostname.split('.')[0] }}-crypt-1/{{ crypted_disk  }} {{ crypted_disk  }}
  args:
    executable: /bin/bash
  register: out
  changed_when: out.stdout=='' and out.stderr==''
  failed_when: out.stderr != '' and not out.stderr.endswith('already exists.')
  when: mount_output.stdout == 'NOT-MOUNTED'
- name: mount crypted disk 
  tags: [check_hosts, upgrade]
  shell: "mount /{{ crypted_disk  }}"
  register: out
  changed_when: true 
  when: mount_output.stdout == 'NOT-MOUNTED'
- name: installation folder
  file:
    path: "{{ hadoop_install }}"
    state: directory
    mode: 0751
    owner: hadoop
    group: hadoop
  tags: upgrade
- name: custom tmp folder present
  file:
    path: "{{ tmp_dir }}"
    state: directory
    mode: u=rwx,g=rwx,o=rwx,g+s
    owner: root
    group: root
  tags: [upgrade, tmp]
- name: custom libs folder exists
  file:
    path: "{{ custom_libs }}"
    state: directory
    mode: 0771
    owner: hadoop
    group: hadoop
  tags: [upgrade, packages, python]
- name: backports are activated
  apt_repository:
    repo: "deb http://ftp.fr.debian.org/debian {{ ansible_distribution_release }}-backports main"
  tags: upgrade
- name: install backports packages
  tags: [packages, upgrade]
  apt:
    name: "{{ item }}"
    default_release: "{{ ansible_distribution_release }}-backports"
    update_cache: true
    state: latest
  with_items: ["{{ java_package }}", "{{ java_package_spark }}", maven, r-base  ]
  notify: [set_java_default, set_java_for_R]
- name: set java home
  tags: [java, upgrade]
  lineinfile:
    path: "/etc/environment" 
    regexp: '^JAVA_HOME='
    line: "JAVA_HOME={{ java_home }}"
  notify: set_java_default
- name: Getting key for installing llvm
  tags: [packages, upgrade, python]
  apt_key:
    data: "{{ lookup('url', 'https://apt.llvm.org/llvm-snapshot.gpg.key', split_lines=False) }}" 
    state: present
  environment:
    HTTPS_PROXY: "http://{{ proxy_host }}:{{ proxy_port }}"
    HTTP_PROXY: "http://{{ proxy_host }}:{{ proxy_port }}"
- name: Ensuring a fresh version llvm is present
  tags: [packages, upgrade, python]
  apt_repository:
    repo: "deb http://apt.llvm.org/{{ ansible_distribution_release }}/ llvm-toolchain-{{ ansible_distribution_release }}-6.0 main"
    state: present
- name: get node version
  tags: [packages, upgrade, test]
  shell: "if [ -f /usr/bin/nodejs  ]; then if [ `/usr/bin/nodejs --version` == 'v10.12.0' ]; then echo 'OK'; else echo 'UPDATE'; fi; else echo 'UPDATE'; fi"
  register: updatenode
  changed_when: false
- name: install node if necessary
  tags: [packages, upgrade, test]
  shell: | 
    export http_proxy=http://{{ proxy_host }}:{{ proxy_port }}
    export https_proxy=http://{{ proxy_host }}:{{ proxy_port }}
    curl -sL https://deb.nodesource.com/setup_10.x | bash -
    apt-get install -y nodejs
  register: updatenode
  changed_when: true
  when: updatenode.stdout == 'UPDATE'
  notify: [ set_npm_proxy ]
- name: installing packages
  tags: [packages, upgrade]
  apt:
    name: "{{ item }}"
    update_cache: true
    state: latest
  with_items: [build-essential, g++, autoconf, automake, libtool, zlib1g-dev, pkg-config, libssl-dev, xmlstarlet, git, libfontconfig ,pandoc, proj-data, locales, curl, jq ]
  notify: [ set_cran_mirror ]
- name: installing backportes packages
  tags: [packages, upgrade, python]
  apt:
    name: "{{ item }}"
    default_release: "{{ ansible_distribution_release }}-backports"
    update_cache: true
    state: latest
  with_items: [python-setuptools, libgdal-dev, libgeos-dev, libproj-dev, cmake, llvm-6.0, libbz2-dev, libreadline-gplv2-dev, libsqlite3-dev, libncurses5-dev, libncursesw5-dev, xz-utils,tk-dev, libgdbm-dev, libc6-dev, libssl-dev ] #, python3, python3-pip, python3-numpy, python3-scipy]
  notify: [ set_npm_proxy, set_cran_mirror ]
- name: download python
  tags: [python, upgrade]
  command: "wget https://www.python.org/ftp/python/{{ python_ver  }}/Python-{{ python_ver  }}.tgz"
  args:
    chdir: "{{ custom_libs }}"
    creates: "{{ custom_libs }}/Python-{{ python_ver }}.tgz"
  become_user: spark
- name: uncompress python
  tags: [python, upgrade]
  unarchive:
    src: "{{ custom_libs }}/Python-{{ python_ver }}.tgz"
    dest: "{{ custom_libs }}"<commit_msg>Adding prerequisites for java netlib (missing fortrand upgrade to stretch)<commit_after>---
- name: get cpu
  tags: [configure_map_reduce, upgrade, scale]
  command: "nproc --all"
  register: node_cpu
  changed_when: false
- name: get host available memory
  tags: [upgrade, configure_yarn, scale]
  shell: "cat /proc/meminfo | grep MemTotal | awk '{print int($2/1024)}'"
  register: out
  changed_when: false
- name: get available memory
  set_fact:
    available_memory_mb: "{{ out.stdout }}"
  tags: [upgrade, configure_yarn, scale]
- name: get available memory for yarn node manager
  set_fact:
    yarn_nodemanager_memory_mb: "{{ (available_memory_mb | int - 800 )| int }}"
  tags: [upgrade, configure_yarn, scale]
- name: crypted disk is running
  tags: [check_hosts, upgrade]
  shell: "if mountpoint -q /{{ crypted_disk }}; then echo 'MOUNTED'; else echo 'NOT-MOUNTED'; fi"
  register: mount_output
  changed_when: false
- name: start crypted disk
  tags: [check_hosts, upgrade]
  shell: |
    echo -n "{{ decrypt_disk_pass }}" | cryptsetup luksOpen --key-file=- /dev/{{ inventory_hostname.split('.')[0] }}-crypt-1/{{ crypted_disk  }} {{ crypted_disk  }}
  args:
    executable: /bin/bash
  register: out
  changed_when: out.stdout=='' and out.stderr==''
  failed_when: out.stderr != '' and not out.stderr.endswith('already exists.')
  when: mount_output.stdout == 'NOT-MOUNTED'
- name: mount crypted disk 
  tags: [check_hosts, upgrade]
  shell: "mount /{{ crypted_disk  }}"
  register: out
  changed_when: true 
  when: mount_output.stdout == 'NOT-MOUNTED'
- name: installation folder
  file:
    path: "{{ hadoop_install }}"
    state: directory
    mode: 0751
    owner: hadoop
    group: hadoop
  tags: upgrade
- name: custom tmp folder present
  file:
    path: "{{ tmp_dir }}"
    state: directory
    mode: u=rwx,g=rwx,o=rwx,g+s
    owner: root
    group: root
  tags: [upgrade, tmp]
- name: custom libs folder exists
  file:
    path: "{{ custom_libs }}"
    state: directory
    mode: 0771
    owner: hadoop
    group: hadoop
  tags: [upgrade, packages, python]
- name: backports are activated
  apt_repository:
    repo: "deb http://ftp.fr.debian.org/debian {{ ansible_distribution_release }}-backports main"
  tags: upgrade
- name: install backports packages
  tags: [packages, upgrade]
  apt:
    name: "{{ item }}"
    default_release: "{{ ansible_distribution_release }}-backports"
    update_cache: true
    state: latest
  with_items: ["{{ java_package }}", "{{ java_package_spark }}", maven, r-base  ]
  notify: [set_java_default, set_java_for_R]
- name: set java home
  tags: [java, upgrade]
  lineinfile:
    path: "/etc/environment" 
    regexp: '^JAVA_HOME='
    line: "JAVA_HOME={{ java_home }}"
  notify: set_java_default
- name: Getting key for installing llvm
  tags: [packages, upgrade, python]
  apt_key:
    data: "{{ lookup('url', 'https://apt.llvm.org/llvm-snapshot.gpg.key', split_lines=False) }}" 
    state: present
  environment:
    HTTPS_PROXY: "http://{{ proxy_host }}:{{ proxy_port }}"
    HTTP_PROXY: "http://{{ proxy_host }}:{{ proxy_port }}"
- name: Ensuring a fresh version llvm is present
  tags: [packages, upgrade, python]
  apt_repository:
    repo: "deb http://apt.llvm.org/{{ ansible_distribution_release }}/ llvm-toolchain-{{ ansible_distribution_release }}-6.0 main"
    state: present
- name: get node version
  tags: [packages, upgrade, test]
  shell: "if [ -f /usr/bin/nodejs  ]; then if [ `/usr/bin/nodejs --version` == 'v10.12.0' ]; then echo 'OK'; else echo 'UPDATE'; fi; else echo 'UPDATE'; fi"
  register: updatenode
  changed_when: false
- name: install node if necessary
  tags: [packages, upgrade, test]
  shell: | 
    export http_proxy=http://{{ proxy_host }}:{{ proxy_port }}
    export https_proxy=http://{{ proxy_host }}:{{ proxy_port }}
    curl -sL https://deb.nodesource.com/setup_10.x | bash -
    apt-get install -y nodejs
  register: updatenode
  changed_when: true
  when: updatenode.stdout == 'UPDATE'
  notify: [ set_npm_proxy ]
- name: installing packages
  tags: [packages, upgrade]
  apt:
    name: "{{ item }}"
    update_cache: true
    state: latest
  with_items: [build-essential, g++, autoconf, automake, libtool, zlib1g-dev, pkg-config, libssl-dev, xmlstarlet, git, libfontconfig ,pandoc, proj-data, locales, curl, jq ]
  notify: [ set_cran_mirror ]
- name: installing backportes packages
  tags: [packages, upgrade, python, blas]
  apt:
    name: "{{ item }}"
    default_release: "{{ ansible_distribution_release }}-backports"
    update_cache: true
    state: latest
  with_items: [python-setuptools, libgdal-dev, libgeos-dev, libproj-dev, cmake, llvm-6.0, libbz2-dev, libreadline-gplv2-dev, libsqlite3-dev, libncurses5-dev, libncursesw5-dev, xz-utils,tk-dev, libgdbm-dev, libc6-dev, libssl-dev, libatlas3-base, libopenblas-base ] #libgfortran-6-dev, python3, python3-pip, python3-numpy, python3-scipy]
  notify: [ set_npm_proxy, set_cran_mirror ]
- name: download python
  tags: [python, upgrade]
  command: "wget https://www.python.org/ftp/python/{{ python_ver  }}/Python-{{ python_ver  }}.tgz"
  args:
    chdir: "{{ custom_libs }}"
    creates: "{{ custom_libs }}/Python-{{ python_ver }}.tgz"
  become_user: spark
- name: uncompress python
  tags: [python, upgrade]
  unarchive:
    src: "{{ custom_libs }}/Python-{{ python_ver }}.tgz"
    dest: "{{ custom_libs }}"
{'fraction_non_alphanumeric': 0.14335532516493874, 'fraction_numerical': 0.007728557964184732, 'mean_word_length': 3.842994066636239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6069055', 'n_tokens_mistral': 3824, 'n_tokens_neox': 3683, 'n_words': 971}