<commit_before>{% set kms = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.kms', 'grains.items', 'compound') %}
# 
# Start the Hive service
#

# @todo move this out to its own formula
mysql-svc:
  service:
    - running
    {% if grains['os_family'] == 'Debian' %}
    - name: mysql
    {% elif grains['os_family'] == 'RedHat' %}
    - name: mysqld
    {% endif %}
    - require:
      - pkg: mysql

{% if salt['pillar.get']('cdh5:security:enable', False) %}
hdfs_kinit:
  cmd:
    - run
    - name: 'kinit -kt /etc/hadoop/conf/hdfs.keytab hdfs/{{ grains.fqdn }}'
    - user: hdfs
    - group: hdfs
    - env:
      - KRB5_CONFIG: '{{ pillar.krb5.conf_file }}'
{% endif %}

configure_metastore:
  cmd:
    - script
    - template: jinja
    - source: salt://cdh5/hive/configure_metastore.sh
    - unless: echo "show databases" | mysql -u root | grep metastore
    - require: 
      - pkg: hive
      - service: mysql-svc

create_warehouse_dir:
  cmd:
    - run
    - name: 'hdfs dfs -mkdir -p /user/{{pillar.cdh5.hive.user}}/warehouse'
    - user: hdfs
    - group: hdfs
    - require:
      - pkg: hive
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: generate_hive_keytabs 
      {% endif %}
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: hdfs_kinit
      {% endif %}



create_scratch_dir:
  cmd:
    - run
    - name: 'hdfs dfs -mkdir -p /user/{{pillar.cdh5.hive.user}}/tmp'
    - user: hdfs
    - group: hdfs
    - require:
      - pkg: hive
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: generate_hive_keytabs
      {% endif %}
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: hdfs_kinit
      {% endif %}

warehouse_dir_owner:
  cmd:
    - run
    - name: 'hdfs dfs -chown -R {{pillar.cdh5.hive.user}}:{{pillar.cdh5.hive.user}} /user/{{pillar.cdh5.hive.user}}'
    - user: hdfs
    - group: hdfs
    - require:
      - cmd: create_warehouse_dir
      - cmd: create_scratch_dir

# This was chmodding the dir to 771 permissions, and it was breaking things
warehouse_dir_permissions:
  cmd:
    - run
    - name: 'hdfs dfs -chmod 1777 /user/{{pillar.cdh5.hive.user}}/warehouse'
    - user: hdfs
    - group: hdfs
    - require:
      - cmd: warehouse_dir_owner

scratch_dir_permissions:
  cmd:
    - run
    - name: 'hdfs dfs -chmod 1777 /user/{{pillar.cdh5.hive.user}}/tmp'
    - user: hdfs
    - group: hdfs
    - require:
      - cmd: warehouse_dir_owner


{% if kms %}
create_hive_key:
  cmd:
    - run
    - user: root
    - name: 'hadoop key create hive'
    - unless: 'hadoop key list | grep hive'
    {% if salt['pillar.get']('cdh5:security:enable', False) %}
    - require:
      - cmd: hdfs_kinit
    {% endif %}

create_hive_zone:
  cmd:
    - run
    - user: hdfs
    - name: 'hdfs crypto -createZone -keyName hive -path /user/{{ pillar.cdh5.hive.user }}'
    - unless: 'hdfs crypto -listZones | grep /user/{{ pillar.cdh5.hive.user }}'
    - require:
      - cmd: create_hive_key
      - cmd: warehouse_dir_permissions
      - cmd: scratch_dir_permissions
    - require_in:
      - service: hive-metastore
{% endif %}


hive-metastore:
  service:
    - running
    - require: 
      - pkg: hive
      - cmd: configure_metastore
      - cmd: warehouse_dir_permissions
      - cmd: scratch_dir_permissions
      - service: mysql-svc
      - file: /usr/lib/hive/lib/mysql-connector-java.jar
      - file: /etc/hive/conf/hive-site.xml
      - file: /mnt/tmp/
    - watch:
      - file: /etc/hive/conf/hive-site.xml

hive-server2:
  service:
    - running
    - require: 
      - service: hive-metastore<commit_msg>BLUE-56 fixed hive dependencies<commit_after>{% set kms = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cd<EMAIL>', 'grains.items', 'compound') %}
# 
# Start the Hive service
#

# @todo move this out to its own formula
mysql-svc:
  service:
    - running
    {% if grains['os_family'] == 'Debian' %}
    - name: mysql
    {% elif grains['os_family'] == 'RedHat' %}
    - name: mysqld
    {% endif %}
    - require:
      - pkg: mysql

{% if salt['pillar.get']('cdh5:security:enable', False) %}
hdfs_kinit:
  cmd:
    - run
    - name: 'kinit -kt /etc/hadoop/conf/hdfs.keytab hdfs/{{ grains.fqdn }}'
    - user: hdfs
    - env:
      - KRB5_CONFIG: '{{ pillar.krb5.conf_file }}'

hive_kinit:
  cmd:
    - run
    - name: 'kinit -kt /etc/hive/conf/hive.keytab hive/{{ grains.fqdn }}'
    - user: hive
    - env:
      - KRB5_CONFIG: '{{ pillar.krb5.conf_file }}'
    - require:
      - cmd: generate_hive_keytabs
{% endif %}

configure_metastore:
  cmd:
    - script
    - template: jinja
    - source: salt://cdh5/hive/configure_metastore.sh
    - unless: echo "show databases" | mysql -u root | grep metastore
    - require: 
      - pkg: hive
      - service: mysql-svc

create_hive_dir:
  cmd:
    - run
    - user: hdfs
    - name: 'hdfs dfs -mkdir -p /user/{{ pillar.cdh5.hive.user }} && hdfs dfs -chown -R {{pillar.cdh5.hive.user}}:{{pillar.cdh5.hive.user}} /user/{{pillar.cdh5.hive.user}}'
    {% if salt['pillar.get']('cdh5:security:enable', False) %}
    - require:
      - cmd: hdfs_kinit
    {% endif %}

{% if kms %}
create_hive_key:
  cmd:
    - run
    - user: root
    - name: 'hadoop key create hive'
    - unless: 'hadoop key list | grep hive'

create_hive_zone:
  cmd:
    - run
    - user: hdfs
    - name: 'hdfs crypto -createZone -keyName hive -path /user/{{ pillar.cdh5.hive.user }}'
    - unless: 'hdfs crypto -listZones | grep /user/{{ pillar.cdh5.hive.user }}'
    - require:
      - cmd: create_hive_key
      - cmd: create_hive_dir
    - require_in:
      - service: hive-metastore
      - cmd: create_warehouse_dir
      - cmd: create_scratch_dir
{% endif %}

create_warehouse_dir:
  cmd:
    - run
    - name: 'hdfs dfs -mkdir -p /user/{{pillar.cdh5.hive.user}}/warehouse'
    - user: hive
    - require:
      - pkg: hive
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: hive_kinit
      {% endif %}

create_scratch_dir:
  cmd:
    - run
    - name: 'hdfs dfs -mkdir -p /user/{{pillar.cdh5.hive.user}}/tmp'
    - user: hive
    - require:
      - pkg: hive
      {% if salt['pillar.get']('cdh5:security:enable', False) %}
      - cmd: hive_kinit
      {% endif %}

# This was chmodding the dir to 771 permissions, and it was breaking things
warehouse_dir_permissions:
  cmd:
    - run
    - name: 'hdfs dfs -chmod 1777 /user/{{pillar.cdh5.hive.user}}/warehouse'
    - user: hive
    - require:
      - cmd: create_warehouse_dir

scratch_dir_permissions:
  cmd:
    - run
    - name: 'hdfs dfs -chmod 1777 /user/{{pillar.cdh5.hive.user}}/tmp'
    - user: hive
    - require:
      - cmd: create_scratch_dir

hive-metastore:
  service:
    - running
    - require: 
      - pkg: hive
      - cmd: configure_metastore
      - cmd: warehouse_dir_permissions
      - cmd: scratch_dir_permissions
      - service: mysql-svc
      - file: /usr/lib/hive/lib/mysql-connector-java.jar
      - file: /etc/hive/conf/hive-site.xml
      - file: /mnt/tmp/
    - watch:
      - file: /etc/hive/conf/hive-site.xml

hive-server2:
  service:
    - running
    - require: 
      - service: hive-metastore
{'fraction_non_alphanumeric': 0.15211384360320532, 'fraction_numerical': 0.008842221608179055, 'mean_word_length': 2.259342638451148, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2809451', 'n_tokens_mistral': 2982, 'n_tokens_neox': 2814, 'n_words': 568}