<commit_before>
# Hive Server Install

TODO: Implement lock for Hive Server2
http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/4.2.0/CDH4-Installation-Guide/cdh4ig_topic_18_5.html

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/bootstrap/utils'
    module.exports.push 'masson/core/krb5_client'
    module.exports.push 'masson/core/iptables'
    module.exports.push 'masson/commons/mysql_client' # Install the mysql connector
    module.exports.push 'ryba/hadoop/mapred_client'
    module.exports.push 'ryba/tez'
    module.exports.push 'ryba/hive/client_install' # Install the Hive and HCatalog service
    module.exports.push 'ryba/hadoop/hdfs_dn_wait'
    module.exports.push 'ryba/hbase/client'
    module.exports.push require('./server').configure

## IPTables

| Service        | Port  | Proto | Parameter            |
|----------------|-------|-------|----------------------|
| Hive Metastore | 9083  | http  | hive.metastore.uris  |
| Hive Web UI    | 9999  | http  | hive.hwi.listen.port |
| Hive Server    | 10001 | tcp   | env[HIVE_PORT]       |


IPTables rules are only inserted if the parameter "iptables.action" is set to 
"start" (default value).

    module.exports.push name: 'Hive & HCat Server # IPTables', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      hive_server_port = if hive.site['hive.server2.transport.mode'] is 'binary'
      then hive.site['hive.server2.thrift.port']
      else hive.site['hive.server2.thrift.http.port']
      ctx.iptables
        rules: [
          { chain: 'INPUT', jump: 'ACCEPT', dport: 9083, protocol: 'tcp', state: 'NEW', comment: "Hive Metastore" }
          { chain: 'INPUT', jump: 'ACCEPT', dport: 9999, protocol: 'tcp', state: 'NEW', comment: "Hive Web UI" }
          { chain: 'INPUT', jump: 'ACCEPT', dport: hive_server_port, protocol: 'tcp', state: 'NEW', comment: "Hive Server" }
        ]
        if: ctx.config.iptables.action is 'start'
      , next

## Startup

Install and configure the startup script in "/etc/init.d/hive-hcatalog-server"
and "/etc/init.d/hive-server2".

The servers are not activated on startup but they endup as zombies if HDFS isnt
yet started.

    module.exports.push name: 'Hive & HCat Server # Startup', handler: (ctx, next) ->
      {site} = ctx.config.ryba.hive
      {engine} = parse_jdbc site['javax.jdo.option.ConnectionURL']
      ctx.service [
        name: 'hive-hcatalog-server'
        startup: false
      ,
        name: 'hive-server2'
        startup: false
      # ,
      #   name: 'mysql'
      #   if: engine is 'mysql'
      ], next

    module.exports.push name: 'Hive & HCat Server # Fix Startup', handler: (ctx, next) ->
      ctx.write [
        destination: '/etc/init.d/hive-hcatalog-server'
        match: /^.*# Ryba: clean pidfile if pid not running$/m
        replace: """
        if pid=`cat $PIDFILE`; then if ! ps -e -o pid | grep -v grep | grep -w $pid; then rm $PIDFILE; fi; fi; \# Ryba: clean pidfile if pid not running
        """
        append: /^PIDFILE=.*$/m
      ,
        destination: '/etc/init.d/hive-server2'
        match: /^.*# Ryba: clean pidfile if pid not running$/m
        replace: """
        if pid=`cat $PIDFILE`; then if ! ps -e -o pid | grep -v grep | grep -w $pid; then rm $PIDFILE; fi; fi; \# Ryba: clean pidfile if pid not running
        """
        append: /^PIDFILE=.*$/m
      ], next

    module.exports.push name: 'Hive & HCat Server # Database', handler: (ctx, next) ->
      {hive, db_admin} = ctx.config.ryba
      username = hive.site['javax.jdo.option.ConnectionUserName']
      password = hive.site['<PASSWORD>.ConnectionPassword']
      {engine, host, db} = parse_jdbc hive.site['javax.jdo.option.ConnectionURL']
      engines = 
        mysql: ->
          do_exists = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            ctx.execute
              cmd: "if ! #{cmd} -e \"use #{db}\"; then exit 3; fi"
              code_skipped: 3
            , (err, exists) ->
              return next err if err
              if exists then do_upgrade() else do_db()
          do_db = ->
            cmd = "#{db_admin.path} -u#{db_admin.username} -p#{db_admin.password} -h#{db_admin.host} -P#{db_admin.port}"
            ctx.execute
              cmd: """
              #{cmd} -e "
              create database if not exists #{db};
              grant all privileges on #{db}.* to '#{username}'@'localhost' identified by '#{password}';
              grant all privileges on #{db}.* to '#{username}'@'%' identified by '#{password}';
              flush privileges;
              "
              """
            , (err) ->
              return next err if err
              do_create()
          do_create = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            target_version = 'ls /usr/lib/hive/lib | grep hive-common- | sed \'s/^hive-common-\\([0-9]\\+.[0-9]\\+.[0-9]\\+\\).*\\.jar$/\\1/g\''
            ctx.execute
              cmd: """
              target=`#{target_version}`
              create=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-${target}.mysql.sql
              if ! test -f $create; then exit 1; fi
              # Create schema
              #{cmd} #{db} < $create
              # Import transaction schema
              trnx=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-txn-schema-${target}.mysql.sql
              if test -f $trnx; then #{cmd} #{db} < $trnx; fi
              """
            , next
          do_upgrade = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            current_version = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port} -e 'select SCHEMA_VERSION from hive.VERSION' --skip-column-names"
            target_version = 'ls /usr/lib/hive/lib | grep hive-common- | sed \'s/^hive-common-\\([0-9]\\+.[0-9]\\+.[0-9]\\+\\).*\\.jar$/\\1/g\''
            ctx.execute
              cmd: """
              current=`#{current_version}`
              target=`#{target_version}`
              if [ $current == $target ]; then exit 3; fi
              # Upgrade schema
              upgrade=/usr/lib/hive/scripts/metastore/upgrade/mysql/upgrade-${current}-to-${target}.mysql.sql
              if ! test -f $upgrade; then exit 1; fi
              #{cmd} #{db} < $upgrade
              # Import transaction schema
              trnx=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-txn-schema-${target}.mysql.sql
              if test -f $trnx; then #{cmd} #{db} < $trnx; fi
              """
              code_skipped: 3
            , next
          do_exists()
      return next new Error 'Database engine not supported' unless engines[engine]
      engines[engine]()

    module.exports.push name: 'Hive & HCat Server # Configure', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      ctx.hconfigure
        destination: "#{hive.conf_dir}/hive-site.xml"
        default: "#{__dirname}/../resources/hive/hive-site.xml"
        local_default: true
        properties: hive.site
        merge: true
      , (err, configured) ->
        return next err if err
        ctx.execute
          cmd: """
          chown -R #{hive.user.name}:#{hive.group.name} #{hive.conf_dir}/
          chmod -R 755 #{hive.conf_dir}
          """
        , (err) ->
          next err, configured

    module.exports.push name: 'Hive & HCat Server # Fix', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      ctx.write
        destination: "#{hive.conf_dir}/hive-env.sh"
        match: /^export HIVE_AUX_JARS_PATH=.*$/mg
        replace: 'export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH:-/usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar}'
      , next

    module.exports.push name: 'Hive & HCat Server # Libs', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      return next() unless hive.libs.length
      uploads = for lib in hive.libs
        source: lib
        destination: "/usr/lib/hive/lib/#{path.basename lib}"
      ctx.upload uploads, next

    module.exports.push name: 'Hive & HCat Server # Driver', handler: (ctx, next) ->
      ctx.link
        source: '/usr/share/java/mysql-connector-java.jar'
        destination: '/usr/lib/hive/lib/mysql-connector-java.jar'
      , next

    module.exports.push name: 'Hive & HCat Server # Kerberos', handler: (ctx, next) ->
      {hive, realm} = ctx.config.ryba
      {kadmin_principal, kadmin_password, admin_server} = ctx.config.krb5.etc_krb5_conf.realms[realm]
      modified = false
      do_metastore = ->
        ctx.krb5_addprinc
          principal: hive.site['hive.metastore.kerberos.principal'].replace '_HOST', ctx.config.host
          randkey: true
          keytab: hive.site['hive.metastore.kerberos.keytab.file']
          uid: hive.user.name
          gid: hive.group.name
          kadmin_principal: kadmin_principal
          kadmin_password: <PASSWORD>
          kadmin_server: admin_server
        , (err, created) ->
          return next err if err
          modified = true if created
          do_server2()
      do_server2 = ->
        return do_end() if hive.site['hive.metastore.kerberos.principal'] is hive.site['hive.server2.authentication.kerberos.principal']
        ctx.krb5_addprinc
          principal: hive.site['hive.server2.authentication.kerberos.principal'].replace '_HOST', ctx.config.host
          randkey: true
          keytab: hive.site['hive.server2.authentication.kerberos.keytab']
          uid: hive.user.name
          gid: hive.group.name
          kadmin_principal: kadmin_principal
          kadmin_password: <PASSWORD>
          kadmin_server: admin_server
        , (err, created) ->
          return next err if err
          modified = true if created
          do_end()
      do_end = ->
        next null, modified
      do_metastore()

    module.exports.push name: 'Hive & HCat Server # Logs', handler: (ctx, next) ->
      ctx.write [
        source: "#{__dirname}/../resources/hive/hive-exec-log4j.properties.template"
        local_source: true
        destination: '/etc/hive/conf/hive-exec-log4j.properties'
      ,
        source: "#{__dirname}/../resources/hive/hive-log4j.properties.template"
        local_source: true
        destination: '/etc/hive/conf/hive-log4j.properties'
      ], next

    module.exports.push name: 'Hive & HCat Server # Layout', timeout: -1, handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      # Required by service "hive-hcatalog-server"
      ctx.mkdir
        destination: '/var/log/hive-hcatalog'
        uid: hive.user.name
        gid: hive.group.name
      , next

    module.exports.push name: 'Hive & HCat Server # HDFS Layout', timeout: -1, handler: (ctx, next) ->
      # todo: this isnt pretty, ok that we need to execute hdfs command from an hadoop client
      # enabled environment, but there must be a better way
      {active_nn_host, hdfs, hive} = ctx.config.ryba
      hive_user = hive.user.name
      hive_group = hive.group.name
      cmd = mkcmd.hdfs ctx, "hdfs dfs -test -d /user && hdfs dfs -test -d /apps && hdfs dfs -test -d /tmp"
      ctx.waitForExecution cmd, code_skipped: 1, (err) ->
        modified = false
        do_user = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /user/#{hive_user} &>/dev/null; then exit 1; fi
            hdfs dfs -mkdir /user/#{hive_user}
            hdfs dfs -chown #{hive_user}:#{hdfs.user.name} /user/#{hive_user}
            """
            code_skipped: 1
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_warehouse()
        do_warehouse = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /apps/#{hive_user}/warehouse &>/dev/null; then exit 3; fi
            hdfs dfs -mkdir /apps/#{hive_user}
            hdfs dfs -mkdir /apps/#{hive_user}/warehouse
            hdfs dfs -chown -R #{hive_user}:#{hdfs.user.name} /apps/#{hive_user}
            hdfs dfs -chmod 755 /apps/#{hive_user}
            hdfs dfs -chmod 1777 /apps/#{hive_user}/warehouse
            """
            code_skipped: 3
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_scratch()
        do_scratch = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /tmp/scratch &> /dev/null; then exit 1; fi
            hdfs dfs -mkdir /tmp 2>/dev/null
            hdfs dfs -mkdir /tmp/scratch
            hdfs dfs -chown #{hive_user}:#{hdfs.user.name} /tmp/scratch
            hdfs dfs -chmod -R 1777 /tmp/scratch
            """
            code_skipped: 1
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_end()
        do_end = ->
          next null, modified
        do_warehouse()

    module.exports.push name: 'Hive & HCat Server # Tez Package', timeout: -1, handler: (ctx, next) ->
      return next() unless ctx.hosts_with_module 'ryba/tez'
      ctx.service
        name: 'tez'
      , next

    module.exports.push name: 'Hive & HCat Server # Tez Layout', timeout: -1, handler: (ctx, next) ->
      return next() unless ctx.hosts_with_module 'ryba/tez'
      {hive, hadoop_group} = ctx.config.ryba
      version_local = 'ls /usr/lib/hive/lib | grep hive-exec- | sed \'s/^hive-exec-\\(.*\\)\\.jar$/\\1/g\''
      version_remote = 'hdfs dfs -ls /apps/hive/install/hive-exec- | sed \'s/.*\\/hive-exec-\\(.*\\)$/\\1/g\''
      ctx.execute
        cmd: mkcmd.hdfs ctx, """
        if hdfs dfs -ls /apps/hive/install &> /dev/null; then exit 1; fi
        hdfs dfs -mkdir /apps/hive/install 2>/dev/null
        hdfs dfs -chown #{hive.user.name}:#{hadoop_group.name} /apps/hive/install
        hdfs dfs -chmod -R 1777 /apps/hive/install
        hdfs dfs -copyFromLocal /usr/lib/hive/lib/hive-exec-* /apps/hive/install/hive-exec-0.13.0.jar
        """
        code_skipped: 1
        not_if_exec: "[[ `#{version_local}` == `#{version_remote}` ]]"
      , next

# Module Dependencies

    path = require 'path'
    parse_jdbc = require '../lib/parse_jdbc'
    mkcmd = require '../lib/mkcmd'
<commit_msg>hive server: upgrade between minor versions<commit_after>
# Hive Server Install

TODO: Implement lock for Hive Server2
http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/4.2.0/CDH4-Installation-Guide/cdh4ig_topic_18_5.html

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/bootstrap/utils'
    module.exports.push 'masson/core/krb5_client'
    module.exports.push 'masson/core/iptables'
    module.exports.push 'masson/commons/mysql_client' # Install the mysql connector
    module.exports.push 'ryba/hadoop/mapred_client'
    module.exports.push 'ryba/tez'
    module.exports.push 'ryba/hive/client_install' # Install the Hive and HCatalog service
    module.exports.push 'ryba/hadoop/hdfs_dn_wait'
    module.exports.push 'ryba/hbase/client'
    module.exports.push require('./server').configure

## IPTables

| Service        | Port  | Proto | Parameter            |
|----------------|-------|-------|----------------------|
| Hive Metastore | 9083  | http  | hive.metastore.uris  |
| Hive Web UI    | 9999  | http  | hive.hwi.listen.port |
| Hive Server    | 10001 | tcp   | env[HIVE_PORT]       |


IPTables rules are only inserted if the parameter "iptables.action" is set to 
"start" (default value).

    module.exports.push name: 'Hive & HCat Server # IPTables', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      hive_server_port = if hive.site['hive.server2.transport.mode'] is 'binary'
      then hive.site['hive.server2.thrift.port']
      else hive.site['hive.server2.thrift.http.port']
      ctx.iptables
        rules: [
          { chain: 'INPUT', jump: 'ACCEPT', dport: 9083, protocol: 'tcp', state: 'NEW', comment: "Hive Metastore" }
          { chain: 'INPUT', jump: 'ACCEPT', dport: 9999, protocol: 'tcp', state: 'NEW', comment: "Hive Web UI" }
          { chain: 'INPUT', jump: 'ACCEPT', dport: hive_server_port, protocol: 'tcp', state: 'NEW', comment: "Hive Server" }
        ]
        if: ctx.config.iptables.action is 'start'
      , next

## Startup

Install and configure the startup script in "/etc/init.d/hive-hcatalog-server"
and "/etc/init.d/hive-server2".

The servers are not activated on startup but they endup as zombies if HDFS isnt
yet started.

    module.exports.push name: 'Hive & HCat Server # Startup', handler: (ctx, next) ->
      {site} = ctx.config.ryba.hive
      {engine} = parse_jdbc site['javax.jdo.option.ConnectionURL']
      ctx.service [
        name: 'hive-hcatalog-server'
        startup: false
      ,
        name: 'hive-server2'
        startup: false
      # ,
      #   name: 'mysql'
      #   if: engine is 'mysql'
      ], next

    module.exports.push name: 'Hive & HCat Server # Fix Startup', handler: (ctx, next) ->
      ctx.write [
        destination: '/etc/init.d/hive-hcatalog-server'
        match: /^.*# Ryba: clean pidfile if pid not running$/m
        replace: """
        if pid=`cat $PIDFILE`; then if ! ps -e -o pid | grep -v grep | grep -w $pid; then rm $PIDFILE; fi; fi; \# Ryba: clean pidfile if pid not running
        """
        append: /^PIDFILE=.*$/m
      ,
        destination: '/etc/init.d/hive-server2'
        match: /^.*# Ryba: clean pidfile if pid not running$/m
        replace: """
        if pid=`cat $PIDFILE`; then if ! ps -e -o pid | grep -v grep | grep -w $pid; then rm $PIDFILE; fi; fi; \# Ryba: clean pidfile if pid not running
        """
        append: /^PIDFILE=.*$/m
      ], next

    module.exports.push name: 'Hive & HCat Server # Database', handler: (ctx, next) ->
      {hive, db_admin} = ctx.config.ryba
      username = hive.site['javax.jdo.option.ConnectionUserName']
      password = hive.site['<PASSWORD>']
      {engine, host, db} = parse_jdbc hive.site['javax.jdo.option.ConnectionURL']
      engines = 
        mysql: ->
          do_exists = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            ctx.execute
              cmd: "if ! #{cmd} -e \"use #{db}\"; then exit 3; fi"
              code_skipped: 3
            , (err, exists) ->
              return next err if err
              if exists then do_upgrade() else do_db()
          do_db = ->
            cmd = "#{db_admin.path} -u#{db_admin.username} -p#{db_admin.password} -h#{db_admin.host} -P#{db_admin.port}"
            ctx.execute
              cmd: """
              #{cmd} -e "
              create database if not exists #{db};
              grant all privileges on #{db}.* to '#{username}'@'localhost' identified by '#{password}';
              grant all privileges on #{db}.* to '#{username}'@'%' identified by '#{password}';
              flush privileges;
              "
              """
            , (err) ->
              return next err if err
              do_create()
          do_create = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            target_version = 'ls /usr/lib/hive/lib | grep hive-common- | sed \'s/^hive-common-\\([0-9]\\+.[0-9]\\+.[0-9]\\+\\).*\\.jar$/\\1/g\''
            ctx.execute
              cmd: """
              target=`#{target_version}`
              create=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-${target}.mysql.sql
              if ! test -f $create; then exit 1; fi
              # Create schema
              #{cmd} #{db} < $create
              # Import transaction schema
              trnx=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-txn-schema-${target}.mysql.sql
              if test -f $trnx; then #{cmd} #{db} < $trnx; fi
              """
            , next
          do_upgrade = ->
            cmd = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port}"
            current_version = "#{db_admin.path} -u#{username} -p#{password} -h#{db_admin.host} -P#{db_admin.port} -e 'select SCHEMA_VERSION from hive.VERSION' --skip-column-names"
            target_version = 'ls /usr/lib/hive/lib | grep hive-common- | sed \'s/^hive-common-\\([0-9]\\+.[0-9]\\+.[0-9]\\+\\).*\\.jar$/\\1/g\''
            ctx.execute
              cmd: """
              current=`#{current_version}`
              target=`#{target_version}`
              if [ $current == $target ]; then exit 3; fi
              # Upgrade schema
              upgrade=/usr/lib/hive/scripts/metastore/upgrade/mysql/upgrade-${current}-to-${target}.mysql.sql
              if ! test -f $upgrade; then
                current_major=`echo $target | sed \'s/^\\([0-9]\\+\\).\\([0-9]\\+\\).\\([0-9]\\+\\)$/\\1.\\2/g\'`
                target_major=`echo $target | sed \'s/^\\([0-9]\\+\\).\\([0-9]\\+\\).\\([0-9]\\+\\)$/\\1.\\2/g\'`
                if [ $current_major == $target_major ]; then exit 0; fi
                exit 1;
              fi
              #{cmd} #{db} < $upgrade
              # Import transaction schema
              trnx=/usr/lib/hive/scripts/metastore/upgrade/mysql/hive-txn-schema-${target}.mysql.sql
              if test -f $trnx; then #{cmd} #{db} < $trnx; fi
              """
              code_skipped: 3
            , next
          do_exists()
      return next new Error 'Database engine not supported' unless engines[engine]
      engines[engine]()

    module.exports.push name: 'Hive & HCat Server # Configure', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      ctx.hconfigure
        destination: "#{hive.conf_dir}/hive-site.xml"
        default: "#{__dirname}/../resources/hive/hive-site.xml"
        local_default: true
        properties: hive.site
        merge: true
      , (err, configured) ->
        return next err if err
        ctx.execute
          cmd: """
          chown -R #{hive.user.name}:#{hive.group.name} #{hive.conf_dir}/
          chmod -R 755 #{hive.conf_dir}
          """
        , (err) ->
          next err, configured

    module.exports.push name: 'Hive & HCat Server # Fix', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      ctx.write
        destination: "#{hive.conf_dir}/hive-env.sh"
        match: /^export HIVE_AUX_JARS_PATH=.*$/mg
        replace: 'export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH:-/usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar}'
      , next

    module.exports.push name: 'Hive & HCat Server # Libs', handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      return next() unless hive.libs.length
      uploads = for lib in hive.libs
        source: lib
        destination: "/usr/lib/hive/lib/#{path.basename lib}"
      ctx.upload uploads, next

    module.exports.push name: 'Hive & HCat Server # Driver', handler: (ctx, next) ->
      ctx.link
        source: '/usr/share/java/mysql-connector-java.jar'
        destination: '/usr/lib/hive/lib/mysql-connector-java.jar'
      , next

    module.exports.push name: 'Hive & HCat Server # Kerberos', handler: (ctx, next) ->
      {hive, realm} = ctx.config.ryba
      {kadmin_principal, kadmin_password, admin_server} = ctx.config.krb5.etc_krb5_conf.realms[realm]
      modified = false
      do_metastore = ->
        ctx.krb5_addprinc
          principal: hive.site['hive.metastore.kerberos.principal'].replace '_HOST', ctx.config.host
          randkey: true
          keytab: hive.site['hive.metastore.kerberos.keytab.file']
          uid: hive.user.name
          gid: hive.group.name
          kadmin_principal: kadmin_principal
          kadmin_password: <PASSWORD>
          kadmin_server: admin_server
        , (err, created) ->
          return next err if err
          modified = true if created
          do_server2()
      do_server2 = ->
        return do_end() if hive.site['hive.metastore.kerberos.principal'] is hive.site['hive.server2.authentication.kerberos.principal']
        ctx.krb5_addprinc
          principal: hive.site['hive.server2.authentication.kerberos.principal'].replace '_HOST', ctx.config.host
          randkey: true
          keytab: hive.site['hive.server2.authentication.kerberos.keytab']
          uid: hive.user.name
          gid: hive.group.name
          kadmin_principal: kadmin_principal
          kadmin_password: <PASSWORD>
          kadmin_server: admin_server
        , (err, created) ->
          return next err if err
          modified = true if created
          do_end()
      do_end = ->
        next null, modified
      do_metastore()

    module.exports.push name: 'Hive & HCat Server # Logs', handler: (ctx, next) ->
      ctx.write [
        source: "#{__dirname}/../resources/hive/hive-exec-log4j.properties.template"
        local_source: true
        destination: '/etc/hive/conf/hive-exec-log4j.properties'
      ,
        source: "#{__dirname}/../resources/hive/hive-log4j.properties.template"
        local_source: true
        destination: '/etc/hive/conf/hive-log4j.properties'
      ], next

    module.exports.push name: 'Hive & HCat Server # Layout', timeout: -1, handler: (ctx, next) ->
      {hive} = ctx.config.ryba
      # Required by service "hive-hcatalog-server"
      ctx.mkdir
        destination: '/var/log/hive-hcatalog'
        uid: hive.user.name
        gid: hive.group.name
      , next

    module.exports.push name: 'Hive & HCat Server # HDFS Layout', timeout: -1, handler: (ctx, next) ->
      # todo: this isnt pretty, ok that we need to execute hdfs command from an hadoop client
      # enabled environment, but there must be a better way
      {active_nn_host, hdfs, hive} = ctx.config.ryba
      hive_user = hive.user.name
      hive_group = hive.group.name
      cmd = mkcmd.hdfs ctx, "hdfs dfs -test -d /user && hdfs dfs -test -d /apps && hdfs dfs -test -d /tmp"
      ctx.waitForExecution cmd, code_skipped: 1, (err) ->
        modified = false
        do_user = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /user/#{hive_user} &>/dev/null; then exit 1; fi
            hdfs dfs -mkdir /user/#{hive_user}
            hdfs dfs -chown #{hive_user}:#{hdfs.user.name} /user/#{hive_user}
            """
            code_skipped: 1
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_warehouse()
        do_warehouse = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /apps/#{hive_user}/warehouse &>/dev/null; then exit 3; fi
            hdfs dfs -mkdir /apps/#{hive_user}
            hdfs dfs -mkdir /apps/#{hive_user}/warehouse
            hdfs dfs -chown -R #{hive_user}:#{hdfs.user.name} /apps/#{hive_user}
            hdfs dfs -chmod 755 /apps/#{hive_user}
            hdfs dfs -chmod 1777 /apps/#{hive_user}/warehouse
            """
            code_skipped: 3
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_scratch()
        do_scratch = ->
          ctx.execute
            cmd: mkcmd.hdfs ctx, """
            if hdfs dfs -ls /tmp/scratch &> /dev/null; then exit 1; fi
            hdfs dfs -mkdir /tmp 2>/dev/null
            hdfs dfs -mkdir /tmp/scratch
            hdfs dfs -chown #{hive_user}:#{hdfs.user.name} /tmp/scratch
            hdfs dfs -chmod -R 1777 /tmp/scratch
            """
            code_skipped: 1
          , (err, executed, stdout) ->
            return next err if err
            modified = true if executed
            do_end()
        do_end = ->
          next null, modified
        do_warehouse()

    module.exports.push name: 'Hive & HCat Server # Tez Package', timeout: -1, handler: (ctx, next) ->
      return next() unless ctx.hosts_with_module 'ryba/tez'
      ctx.service
        name: 'tez'
      , next

    module.exports.push name: 'Hive & HCat Server # Tez Layout', timeout: -1, handler: (ctx, next) ->
      return next() unless ctx.hosts_with_module 'ryba/tez'
      {hive, hadoop_group} = ctx.config.ryba
      version_local = 'ls /usr/lib/hive/lib | grep hive-exec- | sed \'s/^hive-exec-\\(.*\\)\\.jar$/\\1/g\''
      version_remote = 'hdfs dfs -ls /apps/hive/install/hive-exec- | sed \'s/.*\\/hive-exec-\\(.*\\)$/\\1/g\''
      ctx.execute
        cmd: mkcmd.hdfs ctx, """
        if hdfs dfs -ls /apps/hive/install &> /dev/null; then exit 1; fi
        hdfs dfs -mkdir /apps/hive/install 2>/dev/null
        hdfs dfs -chown #{hive.user.name}:#{hadoop_group.name} /apps/hive/install
        hdfs dfs -chmod -R 1777 /apps/hive/install
        hdfs dfs -copyFromLocal /usr/lib/hive/lib/hive-exec-* /apps/hive/install/hive-exec-0.13.0.jar
        """
        code_skipped: 1
        not_if_exec: "[[ `#{version_local}` == `#{version_remote}` ]]"
      , next

# Module Dependencies

    path = require 'path'
    parse_jdbc = require '../lib/parse_jdbc'
    mkcmd = require '../lib/mkcmd'

{'fraction_non_alphanumeric': 0.14805141246821088, 'fraction_numerical': 0.008144889683139735, 'mean_word_length': 1.9995876713740852, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6619620', 'n_tokens_mistral': 9900, 'n_tokens_neox': 9642, 'n_words': 2212}