<commit_before>---
title: 
layout: module
---

# HBase

    hproperties = require '../lib/properties'
    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/core/yum'
    module.exports.push 'ryba/hadoop/core'

## Configure

*   `hbase_user` (object|string)   
    The Unix HBase login name or a user object (see Mecano User documentation).   
*   `hbase_group` (object|string)   
    The Unix HBase group name or a group object (see Mecano Group documentation).   

Example

```json
    "hbase_user": {
      "name": "hbase", "system": true, "gid": "hbase",
      "comment": "HBase User", "home": "/var/run/hbase"
    }
    "hbase_group": {
      "name": "HBase", "system": true
    }
```

    module.exports.push module.exports.configure = (ctx) ->
      return if ctx.hbase_configured
      ctx.hbase_configured = true
      require('masson/commons/java').configure ctx
      require('../hadoop/core').configure ctx
      {java_home} = ctx.config.java
      {static_host, realm} = ctx.config.ryba
      zookeeper_hosts = ctx.hosts_with_module('ryba/zookeeper/server').join ','
      # User
      ctx.config.ryba.hbase_user = name: ctx.config.ryba.hbase_user if typeof ctx.config.ryba.hbase_user is 'string'
      ctx.config.ryba.hbase_user ?= {}
      ctx.config.ryba.hbase_user.name ?= 'hbase'
      ctx.config.ryba.hbase_user.system ?= true
      ctx.config.ryba.hbase_user.gid ?= 'hbase'
      ctx.config.ryba.hbase_user.comment ?= 'HBase User'
      ctx.config.ryba.hbase_user.home ?= '/var/run/hbase'
      # Group
      ctx.config.ryba.hbase_group = name: ctx.config.ryba.hbase_group if typeof ctx.config.ryba.hbase_group is 'string'
      ctx.config.ryba.hbase_group ?= {}
      ctx.config.ryba.hbase_group.name ?= 'hbase'
      ctx.config.ryba.hbase_group.system ?= true
      # Layout
      hbase_conf_dir = ctx.config.ryba.hbase_conf_dir ?= '/etc/hbase/conf'
      hbase_log_dir = ctx.config.ryba.hbase_log_dir ?= '/var/log/hbase'
      ctx.config.ryba.hbase_pid_dir ?= '/var/run/hbase'
      # Configuration
      hbase_site = ctx.config.ryba.hbase_site ?= {}
      hbase_site['zookeeper.znode.parent'] ?= '/hbase'
      # The mode the cluster will be in. Possible values are
      # false: standalone and pseudo-distributed setups with managed Zookeeper
      # true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
      hbase_site['hbase.cluster.distributed'] = 'true'
      # Enter the HBase NameNode server hostname
      # http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-High-Availability-Guide/cdh4hag_topic_2_6.html
      nn_host = if secondary_namenode = ctx.host_with_module 'ryba/hadoop/hdfs_snn'
      then ctx.host_with_module 'ryba/hadoop/hdfs_nn'
      else ctx.config.ryba.nameservice
      hbase_site['hbase.rootdir'] ?= "hdfs://#{nn_host}:8020/apps/hbase/data"
      # hbase_site['hbase.rootdir'] ?= "hdfs://#{namenode}:8020/apps/hbase/data"
      # The bind address for the HBase Master web UI, [Official doc](http://hbase.apache.org/configuration.html)
      # Enter the HBase Master server hostname, [HDP DOC](http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-192.168.127.12/bk_installing_manually_book/content/rpm-chap9-3.html)
      hbase_site['hbase.master.info.bindAddress'] ?= '0.0.0.0'
      # Comma separated list of Zookeeper servers (match to
      # what is specified in zoo.cfg but without portnumbers)
      hbase_site['hbase.zookeeper.quorum'] ?= "#{zookeeper_hosts}"
      # Short-circuit are true but socket.path isnt defined for hbase, only for hdfs, see http://osdir.com/ml/hbase-user-hadoop-apache/2013-03/msg00007.html
      hbase_site['dfs.domain.socket.path'] ?= '/var/lib/hadoop-hdfs/dn_socket'
      # Security
      hbase_site['hbase.security.authentication'] ?= 'kerberos'
      hbase_site['hbase.security.authorization'] ?= 'true'
      hbase_site['hbase.rpc.engine'] ?= 'org.apache.hadoop.hbase.ipc.SecureRpcEngine'
      hbase_site['hbase.master.keytab.file'] ?= '/etc/hbase/conf/hm.service.keytab' # was hm.service.keytab
      hbase_site['hbase.master.kerberos.principal'] ?= "hbase/_HOST@#{realm}"
      hbase_site['hbase.regionserver.keytab.file'] ?= '/etc/hbase/conf/rs.service.keytab' # was rs.service.keytab
      hbase_site['hbase.regionserver.kerberos.principal'] ?= "hbase/_HOST@#{realm}"
      hbase_site['hbase.superuser'] ?= 'hbase'
      hbase_site['hbase.coprocessor.master.classes'] ?= 'org.apache.hadoop.hbase.security.access.AccessController'
      hbase_site['hbase.bulkload.staging.dir'] ?= '/apps/hbase/staging'
      # Secure Server Configuration
      hbase_site['hbase.coprocessor.region.classes'] ?= [
        'org.apache.hadoop.hbase.security.token.TokenProvider'
        'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint'
        'org.apache.hadoop.hbase.security.access.AccessController'
      ]
      hbase_site['hbase.coprocessor.region.classes'] = hbase_site['hbase.coprocessor.region.classes'].join ',' if Array.isArray hbase_site['hbase.coprocessor.region.classes']
      # Secure Client Configuration
      hbase_site['hbase.thrift.keytab.file'] ?= '/etc/hbase/conf/hbase.service.keytab'
      # TODO: add acl (http://hbase.apache.org/book.html#d3314e6371)
      hbase_site['hbase.thrift.kerberos.principal'] ?= "HTTP/_HOST@#{realm}" # Dont forget `grant 'thrift_server', 'RWCA'`
      hbase_site['hbase.rest.keytab.file'] ?= '/etc/hbase/conf/hbase.service.keytab'
      hbase_site['hbase.rest.kerberos.principal'] ?= "HTTP/_HOST@#{realm}" # Dont forget `grant 'rest_server', 'RWCA'`
      hbase_site['hbase.rest.authentication.type'] ?= 'kerberos'
      hbase_site['hbase.rest.authentication.kerberos.principal'] ?= "HTTP/_HOST@#{realm}"
      hbase_site['hbase.rest.authentication.kerberos.keytab'] ?= '/etc/hbase/conf/hbase.service.keytab'
      # Environment
      hbase_env = ctx.config.ryba.hbase_env ?=  {}
      hbase_env['JAVA_HOME'] ?= "#{java_home}"
      hbase_env['HBASE_LOG_DIR'] ?= "#{hbase_log_dir}"
      hbase_env['HBASE_OPTS'] ?= '-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode'
      hbase_env['HBASE_MASTER_OPTS'] ?= '-Xmx1024m'
      hbase_env['HBASE_REGIONSERVER_OPTS'] ?= '-Xmx1024m'
      if ctx.has_module 'ryba/hbase/client'
      # if ctx.has_any_modules ['ryba/hbase/client', 'ryba/hbase/master', 'ryba/hbase/regionserver']
        hbase_env['HBASE_OPTS'] =  hbase_env['HBASE_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-client.jaas"
      if ctx.has_module 'ryba/hbase/master'
        hbase_env['HBASE_MASTER_OPTS'] = hbase_env['HBASE_MASTER_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-master.jaas"
      if ctx.has_module 'ryba/hbase/regionserver'
        hbase_env['HBASE_REGIONSERVER_OPTS'] = hbase_env['HBASE_REGIONSERVER_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-regionserver.jaas"
      ctx.config.ryba.jaas_server ?= """
      Server {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=true
      storeKey=true
      useTicketCache=false
      keyTab="/etc/hbase/conf/hbase.service.keytab"
      principal="hbase/#{ctx.config.host}";
      };
      """
      ctx.config.ryba.jaas_client ?= """
      Client {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=false
      useTicketCache=true;
      };
      """

## Users & Groups

By default, the "hbase" package create the following entries:

```bash
cat /etc/passwd | grep hbase
hbase:x:492:492:HBase:/var/run/hbase:/bin/bash
cat /etc/group | grep hbase
hbase:x:492:
```

    module.exports.push name: 'HBase # Users & Groups', callback: (ctx, next) ->
      {hbase_group, hbase_user} = ctx.config.ryba
      ctx.group hbase_group, (err, gmodified) ->
        return next err if err
        ctx.user hbase_user, (err, umodified) ->
          next err, gmodified or umodified

## Install

Instructions to [install the HBase RPMs](http://docs.hortonworks.com/HDPDocuments/HDP1/HDP-1.3.2/bk_installing_manually_book/content/rpm-chap9-1.html)

    module.exports.push name: 'HBase # Install', timeout: -1, callback: (ctx, next) ->
      ctx.service name: 'hbase', (err, serviced) ->
        next err, if serviced then ctx.OK else ctx.PASS

    module.exports.push name: 'HBase # Layout', timeout: -1, callback: (ctx, next) ->
      {hadoop_group, hbase_user, hbase_pid_dir, hbase_log_dir} = ctx.config.ryba
      ctx.mkdir [
        destination: hbase_pid_dir
        uid: hbase_user.name
        gid: hadoop_group.name
        mode: '755'
      ,
        destination: hbase_log_dir
        uid: hbase_user.name
        gid: hadoop_group.name
        mode: '755'
      ], next

    module.exports.push name: 'HBase # Configure', callback: (ctx, next) ->
      {hbase_conf_dir, hbase_site} = ctx.config.ryba
      ctx.log 'Configure hbase-site.xml'
      ctx.hconfigure
        destination: "#{hbase_conf_dir}/hbase-site.xml"
        default: "#{__dirname}/../resources/hbase/hbase-site.xml"
        local_default: true
        properties: hbase_site
        merge: true
      , next

    module.exports.push name: 'HBase # Env', callback: (ctx, next) ->
      {hbase_conf_dir, hbase_env} = ctx.config.ryba<commit_msg>hbase: octal permissions<commit_after>---
title: 
layout: module
---

# HBase

    hproperties = require '../lib/properties'
    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/core/yum'
    module.exports.push 'ryba/hadoop/core'

## Configure

*   `hbase_user` (object|string)   
    The Unix HBase login name or a user object (see Mecano User documentation).   
*   `hbase_group` (object|string)   
    The Unix HBase group name or a group object (see Mecano Group documentation).   

Example

```json
    "hbase_user": {
      "name": "hbase", "system": true, "gid": "hbase",
      "comment": "HBase User", "home": "/var/run/hbase"
    }
    "hbase_group": {
      "name": "HBase", "system": true
    }
```

    module.exports.push module.exports.configure = (ctx) ->
      return if ctx.hbase_configured
      ctx.hbase_configured = true
      require('masson/commons/java').configure ctx
      require('../hadoop/core').configure ctx
      {java_home} = ctx.config.java
      {static_host, realm} = ctx.config.ryba
      zookeeper_hosts = ctx.hosts_with_module('ryba/zookeeper/server').join ','
      # User
      ctx.config.ryba.hbase_user = name: ctx.config.ryba.hbase_user if typeof ctx.config.ryba.hbase_user is 'string'
      ctx.config.ryba.hbase_user ?= {}
      ctx.config.ryba.hbase_user.name ?= 'hbase'
      ctx.config.ryba.hbase_user.system ?= true
      ctx.config.ryba.hbase_user.gid ?= 'hbase'
      ctx.config.ryba.hbase_user.comment ?= 'HBase User'
      ctx.config.ryba.hbase_user.home ?= '/var/run/hbase'
      # Group
      ctx.config.ryba.hbase_group = name: ctx.config.ryba.hbase_group if typeof ctx.config.ryba.hbase_group is 'string'
      ctx.config.ryba.hbase_group ?= {}
      ctx.config.ryba.hbase_group.name ?= 'hbase'
      ctx.config.ryba.hbase_group.system ?= true
      # Layout
      hbase_conf_dir = ctx.config.ryba.hbase_conf_dir ?= '/etc/hbase/conf'
      hbase_log_dir = ctx.config.ryba.hbase_log_dir ?= '/var/log/hbase'
      ctx.config.ryba.hbase_pid_dir ?= '/var/run/hbase'
      # Configuration
      hbase_site = ctx.config.ryba.hbase_site ?= {}
      hbase_site['zookeeper.znode.parent'] ?= '/hbase'
      # The mode the cluster will be in. Possible values are
      # false: standalone and pseudo-distributed setups with managed Zookeeper
      # true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
      hbase_site['hbase.cluster.distributed'] = 'true'
      # Enter the HBase NameNode server hostname
      # http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/latest/CDH4-High-Availability-Guide/cdh4hag_topic_2_6.html
      nn_host = if secondary_namenode = ctx.host_with_module 'ryba/hadoop/hdfs_snn'
      then ctx.host_with_module 'ryba/hadoop/hdfs_nn'
      else ctx.config.ryba.nameservice
      hbase_site['hbase.rootdir'] ?= "hdfs://#{nn_host}:8020/apps/hbase/data"
      # hbase_site['hbase.rootdir'] ?= "hdfs://#{namenode}:8020/apps/hbase/data"
      # The bind address for the HBase Master web UI, [Official doc](http://hbase.apache.org/configuration.html)
      # Enter the HBase Master server hostname, [HDP DOC](http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.0.6.0/bk_installing_manually_book/content/rpm-chap9-3.html)
      hbase_site['hbase.master.info.bindAddress'] ?= '0.0.0.0'
      # Comma separated list of Zookeeper servers (match to
      # what is specified in zoo.cfg but without portnumbers)
      hbase_site['hbase.zookeeper.quorum'] ?= "#{zookeeper_hosts}"
      # Short-circuit are true but socket.path isnt defined for hbase, only for hdfs, see http://osdir.com/ml/hbase-user-hadoop-apache/2013-03/msg00007.html
      hbase_site['dfs.domain.socket.path'] ?= '/var/lib/hadoop-hdfs/dn_socket'
      # Security
      hbase_site['hbase.security.authentication'] ?= 'kerberos'
      hbase_site['hbase.security.authorization'] ?= 'true'
      hbase_site['hbase.rpc.engine'] ?= 'org.apache.hadoop.hbase.ipc.SecureRpcEngine'
      hbase_site['hbase.master.keytab.file'] ?= '/etc/hbase/conf/hm.service.keytab' # was hm.service.keytab
      hbase_site['hbase.master.kerberos.principal'] ?= "hbase/_HOST@#{realm}"
      hbase_site['hbase.regionserver.keytab.file'] ?= '/etc/hbase/conf/rs.service.keytab' # was rs.service.keytab
      hbase_site['hbase.regionserver.kerberos.principal'] ?= "hbase/_HOST@#{realm}"
      hbase_site['hbase.superuser'] ?= 'hbase'
      hbase_site['hbase.coprocessor.master.classes'] ?= 'org.apache.hadoop.hbase.security.access.AccessController'
      hbase_site['hbase.bulkload.staging.dir'] ?= '/apps/hbase/staging'
      # Secure Server Configuration
      hbase_site['hbase.coprocessor.region.classes'] ?= [
        'org.apache.hadoop.hbase.security.token.TokenProvider'
        'org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint'
        'org.apache.hadoop.hbase.security.access.AccessController'
      ]
      hbase_site['hbase.coprocessor.region.classes'] = hbase_site['hbase.coprocessor.region.classes'].join ',' if Array.isArray hbase_site['hbase.coprocessor.region.classes']
      # Secure Client Configuration
      hbase_site['hbase.thrift.keytab.file'] ?= '/etc/hbase/conf/hbase.service.keytab'
      # TODO: add acl (http://hbase.apache.org/book.html#d3314e6371)
      hbase_site['hbase.thrift.kerberos.principal'] ?= "HTTP/_HOST@#{realm}" # Dont forget `grant 'thrift_server', 'RWCA'`
      hbase_site['hbase.rest.keytab.file'] ?= '/etc/hbase/conf/hbase.service.keytab'
      hbase_site['hbase.rest.kerberos.principal'] ?= "HTTP/_HOST@#{realm}" # Dont forget `grant 'rest_server', 'RWCA'`
      hbase_site['hbase.rest.authentication.type'] ?= 'kerberos'
      hbase_site['hbase.rest.authentication.kerberos.principal'] ?= "HTTP/_HOST@#{realm}"
      hbase_site['hbase.rest.authentication.kerberos.keytab'] ?= '/etc/hbase/conf/hbase.service.keytab'
      # Environment
      hbase_env = ctx.config.ryba.hbase_env ?=  {}
      hbase_env['JAVA_HOME'] ?= "#{java_home}"
      hbase_env['HBASE_LOG_DIR'] ?= "#{hbase_log_dir}"
      hbase_env['HBASE_OPTS'] ?= '-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode'
      hbase_env['HBASE_MASTER_OPTS'] ?= '-Xmx1024m'
      hbase_env['HBASE_REGIONSERVER_OPTS'] ?= '-Xmx1024m'
      if ctx.has_module 'ryba/hbase/client'
      # if ctx.has_any_modules ['ryba/hbase/client', 'ryba/hbase/master', 'ryba/hbase/regionserver']
        hbase_env['HBASE_OPTS'] =  hbase_env['HBASE_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-client.jaas"
      if ctx.has_module 'ryba/hbase/master'
        hbase_env['HBASE_MASTER_OPTS'] = hbase_env['HBASE_MASTER_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-master.jaas"
      if ctx.has_module 'ryba/hbase/regionserver'
        hbase_env['HBASE_REGIONSERVER_OPTS'] = hbase_env['HBASE_REGIONSERVER_OPTS'] + " -Djava.security.auth.login.config=#{hbase_conf_dir}/hbase-regionserver.jaas"
      ctx.config.ryba.jaas_server ?= """
      Server {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=true
      storeKey=true
      useTicketCache=false
      keyTab="/etc/hbase/conf/hbase.service.keytab"
      principal="hbase/#{ctx.config.host}";
      };
      """
      ctx.config.ryba.jaas_client ?= """
      Client {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=false
      useTicketCache=true;
      };
      """

## Users & Groups

By default, the "hbase" package create the following entries:

```bash
cat /etc/passwd | grep hbase
hbase:x:492:492:HBase:/var/run/hbase:/bin/bash
cat /etc/group | grep hbase
hbase:x:492:
```

    module.exports.push name: 'HBase # Users & Groups', callback: (ctx, next) ->
      {hbase_group, hbase_user} = ctx.config.ryba
      ctx.group hbase_group, (err, gmodified) ->
        return next err if err
        ctx.user hbase_user, (err, umodified) ->
          next err, gmodified or umodified

## Install

Instructions to [install the HBase RPMs](http://docs.hortonworks.com/HDPDocuments/HDP1/HDP-1.3.2/bk_installing_manually_book/content/rpm-chap9-1.html)

    module.exports.push name: 'HBase # Install', timeout: -1, callback: (ctx, next) ->
      ctx.service name: 'hbase', (err, serviced) ->
        next err, if serviced then ctx.OK else ctx.PASS

    module.exports.push name: 'HBase # Layout', timeout: -1, callback: (ctx, next) ->
      {hadoop_group, hbase_user, hbase_pid_dir, hbase_log_dir} = ctx.config.ryba
      ctx.mkdir [
        destination: hbase_pid_dir
        uid: hbase_user.name
        gid: hadoop_group.name
        mode: 0o0755
      ,
        destination: hbase_log_dir
        uid: hbase_user.name
        gid: hadoop_group.name
        mode: 0o0755
      ], next

    module.exports.push name: 'HBase # Configure', callback: (ctx, next) ->
      {hbase_conf_dir, hbase_site} = ctx.config.ryba
      ctx.log 'Configure hbase-site.xml'
      ctx.hconfigure
        destination: "#{hbase_conf_dir}/hbase-site.xml"
        default: "#{__dirname}/../resources/hbase/hbase-site.xml"
        local_default: true
        properties: hbase_site
        merge: true
      , next

    module.exports.push name: 'HBase # Env', callback: (ctx, next) ->
      {hbase_conf_dir, hbase_env} = ctx.config.ryba
{'fraction_non_alphanumeric': 0.14420893262679788, 'fraction_numerical': 0.008813669298150752, 'mean_word_length': 3.3121939846117976, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2923722', 'n_tokens_mistral': 6825, 'n_tokens_neox': 6705, 'n_words': 1011}