<commit_before>
# Hadoop HDFS NameNode

NameNode’s primary responsibility is storing the HDFS namespace. This means things
like the directory tree, file permissions, and the mapping of files to block
IDs. It tracks where across the cluster the file data is kept on the DataNodes. It
does not store the data of these files itself. It’s important that this metadata
(and all changes to it) are safely persisted to stable storage for fault tolerance.

    module.exports = []

## Configuration

Look at the file [DFSConfigKeys.java][keys] for an exhaustive list of supported
properties.

*   `ryba.hdfs.site` (object)
    Properties added to the "hdfs-site.xml" file.
*   `ryba.hdfs.namenode_opts` (string)
    NameNode options.

Example:

```json
{
  "ryba": {
    "hdfs": {
      "namenode_opts": "-Xms1024m -Xmx1024m",
      "include": ["in.my.cluster"],
      "exclude": "not.in.my.cluster"
    }
  }
}
```

    module.exports.configure = (ctx) ->
      if ctx.hdfs_nn_configured then return else ctx.hdfs_nn_configured = true
      require('masson/core/iptables').configure ctx
      require('../core').configure ctx
      {ryba} = ctx.config
      ryba.hdfs ?= {}
      ryba.hdfs.site ?= {}
      ryba.hdfs.site['dfs.http.policy'] ?= 'HTTPS_ONLY' # HTTP_ONLY or HTTPS_ONLY or HTTP_AND_HTTPS
      # throw Error "Missing \"ryba.zkfc_password\" property" unless ryba.zkfc_password
      # Data
      # Comma separated list of paths. Use the list of directories.
      # For example, /data/1/hdfs/nn,/data/2/hdfs/nn.
      ryba.hdfs.site['dfs.namenode.name.dir'] ?= ['file:///var/hdfs/name']
      ryba.hdfs.site['dfs.namenode.name.dir'] = ryba.hdfs.site['dfs.namenode.name.dir'].join ',' if Array.isArray ryba.hdfs.site['dfs.namenode.name.dir']
      # Network
      ryba.hdfs.site['dfs.hosts'] ?= "#{ryba.hadoop_conf_dir}/dfs.include"
      ryba.hdfs.include ?= ctx.hosts_with_module 'ryba/hadoop/hdfs_dn'
      ryba.hdfs.include = string.lines ryba.hdfs.include if typeof ryba.hdfs.include is 'string'
      ryba.hdfs.site['dfs.hosts.exclude'] ?= "#{ryba.hadoop_conf_dir}/dfs.exclude"
      ryba.hdfs.exclude ?= []
      ryba.hdfs.exclude = string.lines ryba.hdfs.exclude if typeof ryba.hdfs.exclude is 'string'
      ryba.hdfs.namenode_opts ?= ''
      ryba.hdfs.site['fs.permissions.umask-mode'] ?= '027' # 0750
      # If "true", access tokens are used as capabilities
      # for accessing datanodes. If "false", no access tokens are checked on
      # accessing datanodes.
      ryba.hdfs.site['dfs.block.access.token.enable'] ?= 'true'
      # Kerberos
      ryba.hdfs.site['dfs.namenode.kerberos.principal'] ?= "nn/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.namenode.keytab.file'] ?= '/etc/security/keytabs/nn.service.keytab'
      ryba.hdfs.site['dfs.namenode.kerberos.internal.spnego.principal'] ?= "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.namenode.kerberos.https.principal'] = "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.web.authentication.kerberos.principal'] ?= "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.web.authentication.kerberos.keytab'] ?= '/etc/security/keytabs/spnego.service.keytab'
      # Fix HDP Companion File bug
      ryba.hdfs.site['dfs.https.namenode.https-address'] = null
      # Activate ACLs
      ryba.hdfs.site['dfs.namenode.acls.enabled'] ?= 'true'
      ryba.hdfs.site['dfs.namenode.accesstime.precision'] ?= null
      nn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_nn'
      jn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_jn'
      dn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_dn'

## Configuration for HDFS High Availability (HA)

Add High Availability specific properties to the "hdfs-site.xml" file. The
inserted properties are similar than the ones for a client or slave
configuration with the additionnal "dfs.namenode.shared.edits.dir" property.

The default configuration implement the "sshfence" fencing method. This method
SSHes to the target node and uses fuser to kill the process listening on the
service's TCP port.

      if nn_ctxs.length is 1
        ryba.hdfs.site['dfs.ha.automatic-failover.enabled'] ?= 'false'
        ryba.hdfs.site['dfs.namenode.http-address'] ?= '0.0.0.0:50070'
        ryba.hdfs.site['dfs.namenode.https-address'] ?= '0.0.0.0:50470'
      else<commit_msg>hdfs: improve token enable default value<commit_after>
# Hadoop HDFS NameNode

NameNode’s primary responsibility is storing the HDFS namespace. This means things
like the directory tree, file permissions, and the mapping of files to block
IDs. It tracks where across the cluster the file data is kept on the DataNodes. It
does not store the data of these files itself. It’s important that this metadata
(and all changes to it) are safely persisted to stable storage for fault tolerance.

    module.exports = []

## Configuration

Look at the file [DFSConfigKeys.java][keys] for an exhaustive list of supported
properties.

*   `ryba.hdfs.site` (object)
    Properties added to the "hdfs-site.xml" file.
*   `ryba.hdfs.namenode_opts` (string)
    NameNode options.

Example:

```json
{
  "ryba": {
    "hdfs": {
      "namenode_opts": "-Xms1024m -Xmx1024m",
      "include": ["in.my.cluster"],
      "exclude": "not.in.my.cluster"
    }
  }
}
```

    module.exports.configure = (ctx) ->
      if ctx.hdfs_nn_configured then return else ctx.hdfs_nn_configured = true
      require('masson/core/iptables').configure ctx
      require('../core').configure ctx
      {ryba} = ctx.config
      ryba.hdfs ?= {}
      ryba.hdfs.site ?= {}
      ryba.hdfs.site['dfs.http.policy'] ?= 'HTTPS_ONLY' # HTTP_ONLY or HTTPS_ONLY or HTTP_AND_HTTPS
      # throw Error "Missing \"ryba.zkfc_password\" property" unless ryba.zkfc_password
      # Data
      # Comma separated list of paths. Use the list of directories.
      # For example, /data/1/hdfs/nn,/data/2/hdfs/nn.
      ryba.hdfs.site['dfs.namenode.name.dir'] ?= ['file:///var/hdfs/name']
      ryba.hdfs.site['dfs.namenode.name.dir'] = ryba.hdfs.site['dfs.namenode.name.dir'].join ',' if Array.isArray ryba.hdfs.site['dfs.namenode.name.dir']
      # Network
      ryba.hdfs.site['dfs.hosts'] ?= "#{ryba.hadoop_conf_dir}/dfs.include"
      ryba.hdfs.include ?= ctx.hosts_with_module 'ryba/hadoop/hdfs_dn'
      ryba.hdfs.include = string.lines ryba.hdfs.include if typeof ryba.hdfs.include is 'string'
      ryba.hdfs.site['dfs.hosts.exclude'] ?= "#{ryba.hadoop_conf_dir}/dfs.exclude"
      ryba.hdfs.exclude ?= []
      ryba.hdfs.exclude = string.lines ryba.hdfs.exclude if typeof ryba.hdfs.exclude is 'string'
      ryba.hdfs.namenode_opts ?= ''
      ryba.hdfs.site['fs.permissions.umask-mode'] ?= '027' # 0750
      # If "true", access tokens are used as capabilities
      # for accessing datanodes. If "false", no access tokens are checked on
      # accessing datanodes.
      ryba.hdfs.site['dfs.block.access.token.enable'] ?= if ryba.core_site['hadoop.security.authentication'] is 'kerberos' then 'true' else 'false'
      # Kerberos
      ryba.hdfs.site['dfs.namenode.kerberos.principal'] ?= "nn/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.namenode.keytab.file'] ?= '/etc/security/keytabs/nn.service.keytab'
      ryba.hdfs.site['dfs.namenode.kerberos.internal.spnego.principal'] ?= "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.namenode.kerberos.https.principal'] = "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.web.authentication.kerberos.principal'] ?= "HTTP/#{ryba.static_host}@#{ryba.realm}"
      ryba.hdfs.site['dfs.web.authentication.kerberos.keytab'] ?= '/etc/security/keytabs/spnego.service.keytab'
      # Fix HDP Companion File bug
      ryba.hdfs.site['dfs.https.namenode.https-address'] = null
      # Activate ACLs
      ryba.hdfs.site['dfs.namenode.acls.enabled'] ?= 'true'
      ryba.hdfs.site['dfs.namenode.accesstime.precision'] ?= null
      nn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_nn'
      jn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_jn'
      dn_ctxs = ctx.contexts 'ryba/hadoop/hdfs_dn'

## Configuration for HDFS High Availability (HA)

Add High Availability specific properties to the "hdfs-site.xml" file. The
inserted properties are similar than the ones for a client or slave
configuration with the additionnal "dfs.namenode.shared.edits.dir" property.

The default configuration implement the "sshfence" fencing method. This method
SSHes to the target node and uses fuser to kill the process listening on the
service's TCP port.

      if nn_ctxs.length is 1
        ryba.hdfs.site['dfs.ha.automatic-failover.enabled'] ?= 'false'
        ryba.hdfs.site['dfs.namenode.http-address'] ?= '0.0.0.0:50070'
        ryba.hdfs.site['dfs.namenode.https-address'] ?= '0.0.0.0:50470'
      else
{'fraction_non_alphanumeric': 0.1387839230504981, 'fraction_numerical': 0.008244589488148402, 'mean_word_length': 3.4200404858299596, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2737507', 'n_tokens_mistral': 3210, 'n_tokens_neox': 3129, 'n_words': 653}