<commit_before>---
title: 
layout: module
---

# Core

The Hadoop distribution used is the Hortonwork distribution named HDP. The
installation is leveraging the Yum repositories. [Individual tarballs][tar] 
are also available as an alternative with the benefit of including the source 
code.

    
TODO: topologie

*   http://bigdataprocessing.wordpress.com/2013/07/30/hadoop-rack-awareness-and-configuration/
*   http://ofirm.wordpress.com/2014/01/09/exploring-the-hadoop-network-topology/

[tar]: http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.0.9.1/bk_installing_manually_book/content/rpm-chap13.html

    url = require 'url'
    path = require 'path'
    misc = require 'mecano/lib/misc'
    each = require 'each'
    hconfigure = require './lib/hconfigure'

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/core/yum'
    # Install kerberos clients to create/test new HDFS and Yarn principals
    module.exports.push 'masson/core/krb5_client'
    module.exports.push 'masson/commons/java'

## Configuration

*   `hdp.static_host` (boolean)   
    Write the host name of the server instead of the Hadoop "_HOST" 
    placeholder accross all the configuration files, default to false.   
*   `hdfs_user` (object|string)   
    The Unix HDFS login name or a user object (see Mecano User documentation).   
*   `yarn_user` (object|string)   
    The Unix YARN login name or a user object (see Mecano User documentation).   
*   `mapred_user` (object|string)   
    The Unix MapReduce login name or a user object (see Mecano User documentation).   
*   `test_user` (object|string)   
    The Unix Test user name or a user object (see Mecano User documentation).   
*   `hadoop_group` (object|string)   
    The Unix Hadoop group name or a group object (see Mecano Group documentation).   
*   `hdfs_group` (object|string)   
    The Unix HDFS group name or a group object (see Mecano Group documentation).   
*   `yarn_group` (object|string)   
    The Unix YARN group name or a group object (see Mecano Group documentation).   
*   `mapred_group` (object|string)   
    The Unix MapReduce group name or a group object (see Mecano Group documentation).   
*   `test_group` (object|string)   
    The Unix Test group name or a group object (see Mecano Group documentation).   

Default configuration:

```json
{
  "hdp": {
    "hdfs_user": {
      "name": "hdfs", "system": true, "gid": "hdfs",
      "comment": "HDFS User", "home": "/var/lib/hadoop-hdfs"
    },
    "yarn_user": {
      "name": "yarn", "system": true, "gid": "yarn",
      "comment": "YARN User", "home": "/var/lib/hadoop-yarn"
    },
    "mapred_user": {
      "name": "mapred", "system": true, "gid": "mapred",
      "comment": "MapReduce User", "home": "/var/lib/hadoop-mapreduce"
    },
    "test_user": {
      "name": "ryba", "system": true, "gid": "ryba",
      "comment": "ryba User", "home": "/home/ryba"
    },
    "hadoop_group": {
      "name": "hadoop", "system": true
    },
    "hdfs_group": {
      "name": "hdfs", "system": true
    },
    "yarn_group": {
      "name": "yarn", "system": true
    },
    "mapred_group": {
      "name": "mapred", "system": true
    },
    "test_group": {
      "name": "ryba", "system": true
    }
  }
}
```

    module.exports.push module.exports.configure = (ctx) ->
      return if ctx.core_configured
      ctx.core_configured = true
      require('masson/core/proxy').configure ctx
      require('masson/commons/java').configure ctx
      require('./zookeeper').configure ctx
      ctx.config.hdp ?= {}
      ctx.config.hdp.format ?= false
      ctx.config.hdp.force_check ?= false
      # User
      ctx.config.hdp.hdfs_user = name: ctx.config.hdp.hdfs_user if typeof ctx.config.hdp.hdfs_user is 'string'
      ctx.config.hdp.hdfs_user ?= {}
      ctx.config.hdp.hdfs_user.name ?= 'hdfs'
      ctx.config.hdp.hdfs_user.system ?= true
      ctx.config.hdp.hdfs_user.gid ?= 'hdfs'
      ctx.config.hdp.hdfs_user.groups ?= 'hadoop'
      ctx.config.hdp.hdfs_user.comment ?= 'Hadoop HDFS User'
      ctx.config.hdp.hdfs_user.home ?= '/var/lib/hadoop-hdfs'
      ctx.config.hdp.yarn_user = name: ctx.config.hdp.yarn_user if typeof ctx.config.hdp.yarn_user is 'string'
      ctx.config.hdp.yarn_user ?= {}
      ctx.config.hdp.yarn_user.name ?= 'yarn'
      ctx.config.hdp.yarn_user.system ?= true
      ctx.config.hdp.yarn_user.gid ?= 'yarn'
      ctx.config.hdp.yarn_user.groups ?= 'hadoop'
      ctx.config.hdp.yarn_user.comment ?= 'Hadoop YARN User'
      ctx.config.hdp.yarn_user.home ?= '/var/lib/hadoop-yarn'
      ctx.config.hdp.mapred_user = name: ctx.config.hdp.mapred_user if typeof ctx.config.hdp.mapred_user is 'string'
      ctx.config.hdp.mapred_user ?= {}
      ctx.config.hdp.mapred_user.name ?= 'mapred'
      ctx.config.hdp.mapred_user.system ?= true
      ctx.config.hdp.mapred_user.gid ?= 'mapred'
      ctx.config.hdp.mapred_user.groups ?= 'hadoop'
      ctx.config.hdp.mapred_user.comment ?= 'Hadoop MapReduce User'
      ctx.config.hdp.mapred_user.home ?= '/var/lib/hadoop-mapreduce'
      ctx.config.hdp.test_user = name: ctx.config.hdp.test_user if typeof ctx.config.hdp.test_user is 'string'
      ctx.config.hdp.test_user ?= {}
      ctx.config.hdp.test_user.name ?= 'ryba'
      ctx.config.hdp.test_user.system ?= true
      ctx.config.hdp.test_user.gid ?= 'ryba'
      ctx.config.hdp.test_user.comment ?= 'ryba User'
      ctx.config.hdp.test_user.home ?= '/home/ryba'
      # Groups
      ctx.config.hdp.hadoop_group = name: ctx.config.hdp.hadoop_group if typeof ctx.config.hdp.hadoop_group is 'string'
      ctx.config.hdp.hadoop_group ?= {}
      ctx.config.hdp.hadoop_group.name ?= 'hadoop'
      ctx.config.hdp.hadoop_group.system ?= true
      ctx.config.hdp.hdfs_group = name: ctx.config.hdp.hdfs_group if typeof ctx.config.hdp.hdfs_group is 'string'
      ctx.config.hdp.hdfs_group ?= {}
      ctx.config.hdp.hdfs_group.name ?= 'hdfs'
      ctx.config.hdp.hdfs_group.system ?= true
      ctx.config.hdp.yarn_group = name: ctx.config.hdp.yarn_group if typeof ctx.config.hdp.yarn_group is 'string'
      ctx.config.hdp.yarn_group ?= {}
      ctx.config.hdp.yarn_group.name ?= 'yarn'
      ctx.config.hdp.yarn_group.system ?= true
      ctx.config.hdp.mapred_group = name: ctx.config.hdp.mapred_group if typeof ctx.config.hdp.mapred_group is 'string'
      ctx.config.hdp.mapred_group ?= {}
      ctx.config.hdp.mapred_group.name ?= 'mapred'
      ctx.config.hdp.mapred_group.system ?= true
      ctx.config.hdp.test_group = name: ctx.config.hdp.test_group if typeof ctx.config.hdp.test_group is 'string'
      ctx.config.hdp.test_group ?= {}
      ctx.config.hdp.test_group.name ?= 'ryba'
      ctx.config.hdp.test_group.system ?= true
      # Layout
      ctx.config.hdp.hadoop_conf_dir ?= '/etc/hadoop/conf'
      ctx.config.hdp.hdfs_log_dir ?= '/var/log/hadoop-hdfs'
      ctx.config.hdp.hdfs_pid_dir ?= '/var/run/hadoop-hdfs'
      ctx.config.hdp.mapred_log_dir ?= '/var/log/hadoop-mapreduce' # required by hadoop-env.sh
      # Repository
      ctx.config.hdp.proxy = ctx.config.proxy.http_proxy if typeof ctx.config.hdp.http_proxy is 'undefined'
      ctx.config.hdp.hdp_repo ?= 'http://public-repo-1.hortonworks.com/HDP/centos5/2.x/GA/2.1-latest/hdp.repo'
      # HA Configuration
      ctx.config.hdp.nameservice ?= null
      ctx.config.hdp.active_nn ?= false
      throw new Error "Invalid Service Name" unless ctx.config.hdp.nameservice
      namenodes = ctx.hosts_with_module 'ryba/hadoop/hdfs_nn'
      throw new Error "Need at least 2 namenodes" if namenodes.length < 2
      # active_nn_hosts = ctx.config.servers.filter( (server) -> server.hdp?.active_nn ).map( (server) -> server.host )
      active_nn_hosts = namenodes.filter( (server) -> ctx.config.servers[server].hdp?.active_nn )
      throw new Error "Invalid Number of Active NameNodes: #{active_nn_hosts.length}" unless active_nn_hosts.length is 1
      ctx.config.hdp.active_nn_host = active_nn_hosts[0]
      # standby_nn_hosts = ctx.config.servers.filter( (server) -> ! server.hdp?.active_nn ).map( (server) -> server.host )
      standby_nn_hosts = namenodes.filter( (server) -> ! ctx.config.servers[server].hdp?.active_nn )
      throw new Error "Invalid Number of Passive NameNodes: #{standby_nn_hosts.length}" unless standby_nn_hosts.length is 1
      ctx.config.hdp.standby_nn_host = standby_nn_hosts[0]
      ctx.config.hdp.static_host = 
        if ctx.config.hdp.static_host and ctx.config.hdp.static_host isnt '_HOST'
        then ctx.config.host
        else '_HOST'
      # Configuration
      ctx.config.hdp.core_site ?= {}
      ctx.config.hdp.core_site['fs.defaultFS'] ?= "hdfs://#{ctx.config.hdp.nameservice}"
      # Context
      ctx.hconfigure = (options, callback) ->
        options.ssh = ctx.ssh if typeof options.ssh is 'undefined'
        options.log ?= ctx.log
        options.stdout ?= ctx.stdout
        options.stderr ?= ctx.stderr
        hconfigure options, callback
      # Environment
      ctx.config.hdp.hadoop_opts ?= 'java.net.preferIPv4Stack': true
      hadoop_opts = "export HADOOP_OPTS=\""
      for k, v of ctx.config.hdp.hadoop_opts
        hadoop_opts += "-D#{k}=#{v} "
      hadoop_opts += "${HADOOP_OPTS}\""
      ctx.config.hdp.hadoop_opts = hadoop_opts
      # Database administration
      # todo: `require('masson/commons/mysql_server').configure ctx` and use returned values as default values
      ctx.config.hdp.db_admin ?= {}
      ctx.config.hdp.db_admin.engine ?= 'mysql'
      switch ctx.config.hdp.db_admin.engine
        when 'mysql'
          unless ctx.config.hdp.db_admin.host
            mysql_hosts = ctx.hosts_with_module 'masson/commons/mysql_server'
            throw new Error "Expect at least one server with action \"masson/commons/mysql_server\"" if mysql_hosts.length is 0
            mysql_host = ctx.config.hdp.db_admin.host = if mysql_hosts.length is 1 then mysql_hosts[0] else
              i = mysql_hosts.indexOf(ctx.config.host)
              if i isnt -1 then mysql_hosts[i] else throw new Error "Failed to find a Mysql Server"
            mysql_conf = ctx.hosts[mysql_host].config.mysql_server
          ctx.config.hdp.db_admin.path ?= 'mysql'
          ctx.config.hdp.db_admin.port ?= '3306'
          ctx.config.hdp.db_admin.username ?= 'root'
          ctx.config.hdp.db_admin.password ?= <PASSWORD><commit_msg>add port to defaultFS url<commit_after>---
title: 
layout: module
---

# Core

The Hadoop distribution used is the Hortonwork distribution named HDP. The
installation is leveraging the Yum repositories. [Individual tarballs][tar] 
are also available as an alternative with the benefit of including the source 
code.

    
TODO: topologie

*   http://bigdataprocessing.wordpress.com/2013/07/30/hadoop-rack-awareness-and-configuration/
*   http://ofirm.wordpress.com/2014/01/09/exploring-the-hadoop-network-topology/

[tar]: http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.0.9.1/bk_installing_manually_book/content/rpm-chap13.html

    url = require 'url'
    path = require 'path'
    misc = require 'mecano/lib/misc'
    each = require 'each'
    hconfigure = require './lib/hconfigure'

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push 'masson/core/yum'
    # Install kerberos clients to create/test new HDFS and Yarn principals
    module.exports.push 'masson/core/krb5_client'
    module.exports.push 'masson/commons/java'

## Configuration

*   `hdp.static_host` (boolean)   
    Write the host name of the server instead of the Hadoop "_HOST" 
    placeholder accross all the configuration files, default to false.   
*   `hdfs_user` (object|string)   
    The Unix HDFS login name or a user object (see Mecano User documentation).   
*   `yarn_user` (object|string)   
    The Unix YARN login name or a user object (see Mecano User documentation).   
*   `mapred_user` (object|string)   
    The Unix MapReduce login name or a user object (see Mecano User documentation).   
*   `test_user` (object|string)   
    The Unix Test user name or a user object (see Mecano User documentation).   
*   `hadoop_group` (object|string)   
    The Unix Hadoop group name or a group object (see Mecano Group documentation).   
*   `hdfs_group` (object|string)   
    The Unix HDFS group name or a group object (see Mecano Group documentation).   
*   `yarn_group` (object|string)   
    The Unix YARN group name or a group object (see Mecano Group documentation).   
*   `mapred_group` (object|string)   
    The Unix MapReduce group name or a group object (see Mecano Group documentation).   
*   `test_group` (object|string)   
    The Unix Test group name or a group object (see Mecano Group documentation).   

Default configuration:

```json
{
  "hdp": {
    "hdfs_user": {
      "name": "hdfs", "system": true, "gid": "hdfs",
      "comment": "HDFS User", "home": "/var/lib/hadoop-hdfs"
    },
    "yarn_user": {
      "name": "yarn", "system": true, "gid": "yarn",
      "comment": "YARN User", "home": "/var/lib/hadoop-yarn"
    },
    "mapred_user": {
      "name": "mapred", "system": true, "gid": "mapred",
      "comment": "MapReduce User", "home": "/var/lib/hadoop-mapreduce"
    },
    "test_user": {
      "name": "ryba", "system": true, "gid": "ryba",
      "comment": "ryba User", "home": "/home/ryba"
    },
    "hadoop_group": {
      "name": "hadoop", "system": true
    },
    "hdfs_group": {
      "name": "hdfs", "system": true
    },
    "yarn_group": {
      "name": "yarn", "system": true
    },
    "mapred_group": {
      "name": "mapred", "system": true
    },
    "test_group": {
      "name": "ryba", "system": true
    }
  }
}
```

    module.exports.push module.exports.configure = (ctx) ->
      return if ctx.core_configured
      ctx.core_configured = true
      require('masson/core/proxy').configure ctx
      require('masson/commons/java').configure ctx
      require('./zookeeper').configure ctx
      ctx.config.hdp ?= {}
      ctx.config.hdp.format ?= false
      ctx.config.hdp.force_check ?= false
      # User
      ctx.config.hdp.hdfs_user = name: ctx.config.hdp.hdfs_user if typeof ctx.config.hdp.hdfs_user is 'string'
      ctx.config.hdp.hdfs_user ?= {}
      ctx.config.hdp.hdfs_user.name ?= 'hdfs'
      ctx.config.hdp.hdfs_user.system ?= true
      ctx.config.hdp.hdfs_user.gid ?= 'hdfs'
      ctx.config.hdp.hdfs_user.groups ?= 'hadoop'
      ctx.config.hdp.hdfs_user.comment ?= 'Hadoop HDFS User'
      ctx.config.hdp.hdfs_user.home ?= '/var/lib/hadoop-hdfs'
      ctx.config.hdp.yarn_user = name: ctx.config.hdp.yarn_user if typeof ctx.config.hdp.yarn_user is 'string'
      ctx.config.hdp.yarn_user ?= {}
      ctx.config.hdp.yarn_user.name ?= 'yarn'
      ctx.config.hdp.yarn_user.system ?= true
      ctx.config.hdp.yarn_user.gid ?= 'yarn'
      ctx.config.hdp.yarn_user.groups ?= 'hadoop'
      ctx.config.hdp.yarn_user.comment ?= 'Hadoop YARN User'
      ctx.config.hdp.yarn_user.home ?= '/var/lib/hadoop-yarn'
      ctx.config.hdp.mapred_user = name: ctx.config.hdp.mapred_user if typeof ctx.config.hdp.mapred_user is 'string'
      ctx.config.hdp.mapred_user ?= {}
      ctx.config.hdp.mapred_user.name ?= 'mapred'
      ctx.config.hdp.mapred_user.system ?= true
      ctx.config.hdp.mapred_user.gid ?= 'mapred'
      ctx.config.hdp.mapred_user.groups ?= 'hadoop'
      ctx.config.hdp.mapred_user.comment ?= 'Hadoop MapReduce User'
      ctx.config.hdp.mapred_user.home ?= '/var/lib/hadoop-mapreduce'
      ctx.config.hdp.test_user = name: ctx.config.hdp.test_user if typeof ctx.config.hdp.test_user is 'string'
      ctx.config.hdp.test_user ?= {}
      ctx.config.hdp.test_user.name ?= 'ryba'
      ctx.config.hdp.test_user.system ?= true
      ctx.config.hdp.test_user.gid ?= 'ryba'
      ctx.config.hdp.test_user.comment ?= 'ryba User'
      ctx.config.hdp.test_user.home ?= '/home/ryba'
      # Groups
      ctx.config.hdp.hadoop_group = name: ctx.config.hdp.hadoop_group if typeof ctx.config.hdp.hadoop_group is 'string'
      ctx.config.hdp.hadoop_group ?= {}
      ctx.config.hdp.hadoop_group.name ?= 'hadoop'
      ctx.config.hdp.hadoop_group.system ?= true
      ctx.config.hdp.hdfs_group = name: ctx.config.hdp.hdfs_group if typeof ctx.config.hdp.hdfs_group is 'string'
      ctx.config.hdp.hdfs_group ?= {}
      ctx.config.hdp.hdfs_group.name ?= 'hdfs'
      ctx.config.hdp.hdfs_group.system ?= true
      ctx.config.hdp.yarn_group = name: ctx.config.hdp.yarn_group if typeof ctx.config.hdp.yarn_group is 'string'
      ctx.config.hdp.yarn_group ?= {}
      ctx.config.hdp.yarn_group.name ?= 'yarn'
      ctx.config.hdp.yarn_group.system ?= true
      ctx.config.hdp.mapred_group = name: ctx.config.hdp.mapred_group if typeof ctx.config.hdp.mapred_group is 'string'
      ctx.config.hdp.mapred_group ?= {}
      ctx.config.hdp.mapred_group.name ?= 'mapred'
      ctx.config.hdp.mapred_group.system ?= true
      ctx.config.hdp.test_group = name: ctx.config.hdp.test_group if typeof ctx.config.hdp.test_group is 'string'
      ctx.config.hdp.test_group ?= {}
      ctx.config.hdp.test_group.name ?= 'ryba'
      ctx.config.hdp.test_group.system ?= true
      # Layout
      ctx.config.hdp.hadoop_conf_dir ?= '/etc/hadoop/conf'
      ctx.config.hdp.hdfs_log_dir ?= '/var/log/hadoop-hdfs'
      ctx.config.hdp.hdfs_pid_dir ?= '/var/run/hadoop-hdfs'
      ctx.config.hdp.mapred_log_dir ?= '/var/log/hadoop-mapreduce' # required by hadoop-env.sh
      # Repository
      ctx.config.hdp.proxy = ctx.config.proxy.http_proxy if typeof ctx.config.hdp.http_proxy is 'undefined'
      ctx.config.hdp.hdp_repo ?= 'http://public-repo-1.hortonworks.com/HDP/centos5/2.x/GA/2.1-latest/hdp.repo'
      # HA Configuration
      ctx.config.hdp.nameservice ?= null
      ctx.config.hdp.active_nn ?= false
      throw new Error "Invalid Service Name" unless ctx.config.hdp.nameservice
      namenodes = ctx.hosts_with_module 'ryba/hadoop/hdfs_nn'
      throw new Error "Need at least 2 namenodes" if namenodes.length < 2
      # active_nn_hosts = ctx.config.servers.filter( (server) -> server.hdp?.active_nn ).map( (server) -> server.host )
      active_nn_hosts = namenodes.filter( (server) -> ctx.config.servers[server].hdp?.active_nn )
      throw new Error "Invalid Number of Active NameNodes: #{active_nn_hosts.length}" unless active_nn_hosts.length is 1
      ctx.config.hdp.active_nn_host = active_nn_hosts[0]
      # standby_nn_hosts = ctx.config.servers.filter( (server) -> ! server.hdp?.active_nn ).map( (server) -> server.host )
      standby_nn_hosts = namenodes.filter( (server) -> ! ctx.config.servers[server].hdp?.active_nn )
      throw new Error "Invalid Number of Passive NameNodes: #{standby_nn_hosts.length}" unless standby_nn_hosts.length is 1
      ctx.config.hdp.standby_nn_host = standby_nn_hosts[0]
      ctx.config.hdp.static_host = 
        if ctx.config.hdp.static_host and ctx.config.hdp.static_host isnt '_HOST'
        then ctx.config.host
        else '_HOST'
      # Configuration
      ctx.config.hdp.core_site ?= {}
      ctx.config.hdp.core_site['fs.defaultFS'] ?= "hdfs://#{ctx.config.hdp.nameservice}:8020"
      # Context
      ctx.hconfigure = (options, callback) ->
        options.ssh = ctx.ssh if typeof options.ssh is 'undefined'
        options.log ?= ctx.log
        options.stdout ?= ctx.stdout
        options.stderr ?= ctx.stderr
        hconfigure options, callback
      # Environment
      ctx.config.hdp.hadoop_opts ?= 'java.net.preferIPv4Stack': true
      hadoop_opts = "export HADOOP_OPTS=\""
      for k, v of ctx.config.hdp.hadoop_opts
        hadoop_opts += "-D#{k}=#{v} "
      hadoop_opts += "${HADOOP_OPTS}\""
      ctx.config.hdp.hadoop_opts = hadoop_opts
      # Database administration
      # todo: `require('masson/commons/mysql_server').configure ctx` and use returned values as default values
      ctx.config.hdp.db_admin ?= {}
      ctx.config.hdp.db_admin.engine ?= 'mysql'
      switch ctx.config.hdp.db_admin.engine
        when 'mysql'
          unless ctx.config.hdp.db_admin.host
            mysql_hosts = ctx.hosts_with_module 'masson/commons/mysql_server'
            throw new Error "Expect at least one server with action \"masson/commons/mysql_server\"" if mysql_hosts.length is 0
            mysql_host = ctx.config.hdp.db_admin.host = if mysql_hosts.length is 1 then mysql_hosts[0] else
              i = mysql_hosts.indexOf(ctx.config.host)
              if i isnt -1 then mysql_hosts[i] else throw new Error "Failed to find a Mysql Server"
            mysql_conf = ctx.hosts[mysql_host].config.mysql_server
          ctx.config.hdp.db_admin.path ?= 'mysql'
          ctx.config.hdp.db_admin.port ?= '3306'
          ctx.config.hdp.db_admin.username ?= 'root'
          ctx.config.hdp.db_admin.password ?= <PASSWORD>_conf.password
{'fraction_non_alphanumeric': 0.13285872336331714, 'fraction_numerical': 0.00442541728798884, 'mean_word_length': 2.921161825726141, 'pattern_counts': {'":': 80, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5539674', 'n_tokens_mistral': 7456, 'n_tokens_neox': 7263, 'n_words': 1443}