<commit_before>- name: setup the sandbox
  hosts: sandbox
  gather_facts: True
  sudo: True
  sudo_user: root
  tasks:

    - name: set the hostname to sandbox.odp.org
      shell: hostnamectl set-hostname sandbox.odp.org

    - name: ensure libsolve goes to the right IP address
      lineinfile:
        dest: /etc/hosts
        regexp: 'sandbox.odp.org$'
        line: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] + ' sandbox.odp.org' }}"
        state: present

    - name: add the ODPi repos
      copy: 
        src: "{{ 'files/'+ item }}"
        dest: "{{ '/etc/yum.repos.d/' + item }}"
      with_items:
        - ambari.repo
 
    - name: install epel-release
      yum: name=epel-release state=present

    - name: install ambari packages
      yum: name={{ item }} state=latest
      with_items:
        - ambari-agent
        - ambari-server

    - name: install api helper packages
      yum: name=python-requests state=present

    # Oracle JRE+JDK+JCE
    - name: install the oracle packages from the internet
      script: files/fetch_all_oracle_tars.sh

    # FIXME - file a bug or consult list
    - name: trick the installer into using the latest jre
      file: 
        src: /var/lib/ambari-server/resources/jre-8u66-linux-x64.tar.gz
        dest: /var/lib/ambari-server/resources/jre-8u40-linux-x64.tar.gz
        state: link
    # FIXME - file a bug or consult list
    - name: trick the installer into using the latest jdk
      file: 
        src: /var/lib/ambari-server/resources/jdk-8u66-linux-x64.tar.gz
        dest: /var/lib/ambari-server/resources/jdk-8u40-linux-x64.tar.gz
        state: link

    ##############################
    # Ambari Agent
    ##############################
    - name: set the hostname in the agent config
      lineinfile:
        dest: /etc/ambari-agent/conf/ambari-agent.ini
        regexp: '^hostname='
        line: "hostname=sandbox.odp.org"
    # FIXME - file a bug or consult list
    - name: workaround bug in recent ambari agent
      file: path=/var/lib/ambari-agent/keys state=directory
    - name: start the agent
      service: name=ambari-agent state=started
    - service: name=ambari-agent enabled=true

    ##############################
    # Postgres
    ##############################
    - name: initialize postgres
      shell: service postgresql initdb creates=/var/lib/pgsql/data/pg_hba.conf
    - service: name=postgresql enabled=true
    - service: name=postgresql state=started

    ##############################
    # Ambari server
    ##############################
    - name: check if the service was already initialized
      shell: "[ -f /var/lib/ambari-server/keys/ca.crt ]"
      ignore_errors: True
      register: ambari_created
    - shell: ambari-server reset --verbose -s
      when: ambari_created.rc != 0
    - shell: ambari-server setup --verbose -s
      when: ambari_created.rc != 0
      register: spawning

    - name: give ambari a few seconds to settle
      pause: seconds=5
      when: spawning.changed

    # systemd thinks ambari is broken, so ignore errors
    - name: turn the server on
      service: name=ambari-server state=started
      ignore_errors: True
    - name: turn the server on
      service: name=ambari-server enabled=yes

    - name: Poll until ambari is listening
      wait_for: port=8080 connect_timeout=30 timeout=300

    # Show how the service is listening
    - yum: name=net-tools state=present
    - shell: "netstat -tunlp 2>/dev/null | fgrep ':8080'"
      ignore_errors: True

    ##############################
    # Blueprint
    ##############################
    - name: copy the blueprint to the sandbox
      copy: src=files/odp-v2.blueprint dest=/root/odp-v2.blueprint      
    - name: apply the blueprint
      script: ambari_scripts/post-blueprint.py /root/odp-v2.blueprint ODP-0.9-Sandbox

    ##################################################################
    # Before the cluster can be created, we need to fix the repo
    # urls so that packages will get installed. Ambari lays down 
    # yum repo configs based on these urls and then runs yum commands
    # restricted to those repos. If the urls are invalid (the default)
    # then service installation dies a miserable death.
    ##################################################################

    - name: add the primary repo [odpi.org]
      script: ambari_scripts/post-repo-url.py ODP 0.9 redhat7 ODP-0.9 http://repo.opendataplatform.org/repository/ODP/centos7/2.x/BUILDS/0.9.0.1-70
    - name: add the utils repo [odpi.org]
      script: ambari_scripts/post-repo-url.py ODP 0.9 redhat7 ODP-UTILS-1.1.0.20 http://repo.opendataplatform.org/repository/ODP-UTILS-1.1.0.20/repos/centos7 

    # Make sure the agent checks in ...
    - service: name=ambari-agent state=restarted
    - name: give ambari agent  a few seconds to settle
      pause: seconds=5


    ##############################
    # Host Mapping -> Cluster
    ##############################
    - name: create the cluster 
      script: ambari_scripts/post-mapping.py ODP-0.9-Sandbox ODP_Sandbox

    # Wait for the cluster
    - name: wait for the cluster 
      shell: cd /opt/odpi/ansible/ambari_scripts; python wait-for-cluster.py ODP_Sandbox

    - name: opcheck some commands
      shell: "{{ item }}"
      with_items:
        - "hadoop fs -ls /"
        - "yarn node -list"

    # Stop the cluster so the system can shutdown
    - name: stop all cluster services
      shell: cd /opt/odpi/ansible/ambari_scripts; ./sandbox-script.py stop
    - name: stop ambari
      service: name={{ item }} state=stopped<commit_msg>Ignore errors on opcheck because ambari is completely non-deterministic<commit_after>- name: setup the sandbox
  hosts: sandbox
  gather_facts: True
  sudo: True
  sudo_user: root
  tasks:

    - name: set the hostname to sandbox.odp.org
      shell: hostnamectl set-hostname sandbox.odp.org

    - name: ensure libsolve goes to the right IP address
      lineinfile:
        dest: /etc/hosts
        regexp: 'sandbox.odp.org$'
        line: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] + ' sandbox.odp.org' }}"
        state: present

    - name: add the ODPi repos
      copy: 
        src: "{{ 'files/'+ item }}"
        dest: "{{ '/etc/yum.repos.d/' + item }}"
      with_items:
        - ambari.repo
 
    - name: install epel-release
      yum: name=epel-release state=present

    - name: install ambari packages
      yum: name={{ item }} state=latest
      with_items:
        - ambari-agent
        - ambari-server

    - name: install api helper packages
      yum: name=python-requests state=present

    # Oracle JRE+JDK+JCE
    - name: install the oracle packages from the internet
      script: files/fetch_all_oracle_tars.sh

    # FIXME - file a bug or consult list
    - name: trick the installer into using the latest jre
      file: 
        src: /var/lib/ambari-server/resources/jre-8u66-linux-x64.tar.gz
        dest: /var/lib/ambari-server/resources/jre-8u40-linux-x64.tar.gz
        state: link
    # FIXME - file a bug or consult list
    - name: trick the installer into using the latest jdk
      file: 
        src: /var/lib/ambari-server/resources/jdk-8u66-linux-x64.tar.gz
        dest: /var/lib/ambari-server/resources/jdk-8u40-linux-x64.tar.gz
        state: link

    ##############################
    # Ambari Agent
    ##############################
    - name: set the hostname in the agent config
      lineinfile:
        dest: /etc/ambari-agent/conf/ambari-agent.ini
        regexp: '^hostname='
        line: "hostname=sandbox.odp.org"
    # FIXME - file a bug or consult list
    - name: workaround bug in recent ambari agent
      file: path=/var/lib/ambari-agent/keys state=directory
    - name: start the agent
      service: name=ambari-agent state=started
    - service: name=ambari-agent enabled=true

    ##############################
    # Postgres
    ##############################
    - name: initialize postgres
      shell: service postgresql initdb creates=/var/lib/pgsql/data/pg_hba.conf
    - service: name=postgresql enabled=true
    - service: name=postgresql state=started

    ##############################
    # Ambari server
    ##############################
    - name: check if the service was already initialized
      shell: "[ -f /var/lib/ambari-server/keys/ca.crt ]"
      ignore_errors: True
      register: ambari_created
    - shell: ambari-server reset --verbose -s
      when: ambari_created.rc != 0
    - shell: ambari-server setup --verbose -s
      when: ambari_created.rc != 0
      register: spawning

    - name: give ambari a few seconds to settle
      pause: seconds=5
      when: spawning.changed

    # systemd thinks ambari is broken, so ignore errors
    - name: turn the server on
      service: name=ambari-server state=started
      ignore_errors: True
    - name: turn the server on
      service: name=ambari-server enabled=yes

    - name: Poll until ambari is listening
      wait_for: port=8080 connect_timeout=30 timeout=300

    # Show how the service is listening
    - yum: name=net-tools state=present
    - shell: "netstat -tunlp 2>/dev/null | fgrep ':8080'"
      ignore_errors: True

    ##############################
    # Blueprint
    ##############################
    - name: copy the blueprint to the sandbox
      copy: src=files/odp-v2.blueprint dest=/root/odp-v2.blueprint      
    - name: apply the blueprint
      script: ambari_scripts/post-blueprint.py /root/odp-v2.blueprint ODP-0.9-Sandbox

    ##################################################################
    # Before the cluster can be created, we need to fix the repo
    # urls so that packages will get installed. Ambari lays down 
    # yum repo configs based on these urls and then runs yum commands
    # restricted to those repos. If the urls are invalid (the default)
    # then service installation dies a miserable death.
    ##################################################################

    - name: add the primary repo [odpi.org]
      script: ambari_scripts/post-repo-url.py ODP 0.9 redhat7 ODP-0.9 http://repo.opendataplatform.org/repository/ODP/centos7/2.x/BUILDS/0.9.0.1-70
    - name: add the utils repo [odpi.org]
      script: ambari_scripts/post-repo-url.py ODP 0.9 redhat7 ODP-UTILS-1.1.0.20 http://repo.opendataplatform.org/repository/ODP-UTILS-1.1.0.20/repos/centos7 

    # Make sure the agent checks in ...
    - service: name=ambari-agent state=restarted
    - name: give ambari agent  a few seconds to settle
      pause: seconds=5


    ##############################
    # Host Mapping -> Cluster
    ##############################
    - name: create the cluster 
      script: ambari_scripts/post-mapping.py ODP-0.9-Sandbox ODP_Sandbox

    # Wait for the cluster
    - name: wait for the cluster 
      shell: cd /opt/odpi/ansible/ambari_scripts; python wait-for-cluster.py ODP_Sandbox
      ignore_errors: True

    - name: opcheck some commands
      shell: "{{ item }}"
      ignore_errors: True
      with_items:
        - "hadoop fs -ls /"
        - "yarn node -list"

    # Stop the cluster so the system can shutdown
    - name: stop all cluster services
      shell: cd /opt/odpi/ansible/ambari_scripts; ./sandbox-script.py stop
      ignore_errors: True
    - name: stop ambari
      service: name={{ item }} state=stopped
      ignore_errors: True
{'fraction_non_alphanumeric': 0.16413081497026932, 'fraction_numerical': 0.012766701643931445, 'mean_word_length': 2.6563299232736575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '787605', 'n_tokens_mistral': 3695, 'n_tokens_neox': 3464, 'n_words': 975}