<commit_before>
# Druid Install

    module.exports = header: 'Druid Install', handler: ->
      {druid, realm, db_admin} = @config.ryba
      krb5 = @config.krb5.etc_krb5_conf.realms[realm]

## Register and load

      @register 'hconfigure', 'ryba/lib/hconfigure'
      @register 'hdfs_mkdir', 'ryba/lib/hdfs_mkdir'

## IPTables

| Service   | Port       | Proto     | Parameter                   |
|-----------|------------|-----------|-----------------------------|
| Druid Standalone Realtime    | 8084      | tcp/http  |  |
| Druid Router    | 8088      | tcp/http  |  |

Note, this hasnt been verified.

      # @iptables
      #   header: 'IPTables'
      #   rules: [
      #     { chain: 'INPUT', jump: 'ACCEPT', dport: port, protocol: 'tcp', state: 'NEW', comment: "Falcon Prism Local EndPoint" }
      #   ]
      #   if: @config.iptables.action is 'start'

## Users & Groups

By default, the "druid" package create the following entries:

```bash
cat /etc/passwd | grep druid
druid:x:2435:2435:druid User:/var/lib/druid:/bin/bash
cat /etc/group | grep druid
druid:x:2435:
```

      @group druid.group
      @user druid.user

## Packages

Download and unpack the release archive.

      @call if: druid.db.engine is 'postgres', ->
        # @call once: true, 'masson/commons/postgres/server/wait' # Not yet ready
        @service 'postgresql'
      @call if: druid.db.engine is 'mysql', ->
        # @call once: true, 'ryba/commons/mysql/server/wait' # Not yet ready
        @service 'mysql'
      @file.download
        header: 'Packages'
        source: "#{druid.source}"
        target: "/var/tmp/#{path.basename druid.source}"
      # TODO, could be improved
      # current implementation prevent any further attempt if download status is true and extract fails
      @extract
        source: "/var/tmp/#{path.basename druid.source}"
        target: '/opt'
        if: -> @status -1
      @link
        source: "/opt/druid-#{druid.version}"
        target: "#{druid.dir}"
      @execute
        cmd: """
        if [ $(stat -c "%U" /opt/druid-#{druid.version}) == '#{druid.user.name}' ]; then exit 3; fi
        chown -R #{druid.user.name}:#{druid.group.name} /opt/druid-#{druid.version}
        """
        code_skipped: 3

## Layout

Pid files are stored inside "/var/run/druid" by default.
Log files are stored inside "/var/log/druid" by default.

      @call header: 'Layout', handler: ->
        @mkdir
          target: "#{druid.pid_dir}"
          uid: "#{druid.user.name}"
          gid: "#{druid.group.name}"
        @link
          target: "#{druid.dir}/var/druid/pids"
          source: "#{druid.pid_dir}"
        @mkdir
          target: "#{druid.log_dir}"
          uid: "#{druid.user.name}"
          gid: "#{druid.group.name}"
          parent: true
        @link
          source: "#{druid.log_dir}"
          target: "#{druid.dir}/log"

## Kerberos

Create a service principal for this NameNode. The principal is named after
"nn/#{@config.host}@#{realm}".

      @krb5_addprinc krb5,
        header: 'Kerberos Admin Principal'
        principal: "#{druid.krb5_admin.principal}"
        password: "#{<PASSWORD>}"
        randkey: true
        uid: 'druid'
        gid: 'druid'
        mode: 0o0600
      @krb5_addprinc krb5,
        header: 'Kerberos Service Principal'
        principal: "#{druid.krb5_service.principal}"
        keytab: "#{druid.krb5_service.keytab}"
        randkey: true
        uid: "#{druid.user.name}"
        gid: "#{druid.group.name}"
        mode: 0o0600

## Cron-ed Kinit

Druid has no mechanism to renew its keytab. For that, we use a cron daemon
We then ask a first TGT.

      @cron.add
        header: 'Cron-ed kinit'
        cmd: "/usr/bin/kinit #{druid.krb5_service.principal} -kt #{druid.krb5_service.keytab}"
        when: '0 */9 * * *'
        user: 'druid'
        exec: true

## Configuration

Configure deep storage.

      @db.user druid.db, database: null,
        if: druid.db.engine in ['mysql', 'postgres']
      @db.database druid.db,
        if: druid.db.engine in ['mysql', 'postgres']
        user: druid.db.username
      @file.properties
        target: "/opt/druid-#{druid.version}/conf/druid/_common/common.runtime.properties"
        content: druid.common_runtime
        backup: true
      @copy
        target: "/opt/druid-#{druid.version}/conf/druid/_common/core-site.xml"
        source: "#{druid.hadoop_conf_dir}/core-site.xml"
      @copy
        target: "/opt/druid-#{druid.version}/conf/druid/_common/hdfs-site.xml"
        source: "#{druid.hadoop_conf_dir}/hdfs-site.xml"
      @hconfigure
        target: "/opt/druid-#{druid.version}/conf/druid/_common/yarn-site.xml"
        source: "#{druid.hadoop_conf_dir}/yarn-site.xml"
        transform: (properties) ->
          if properties['yarn.resourcemanager.ha.rm-ids']
            [id] = properties['yarn.resourcemanager.ha.rm-ids'].split ','
            properties['yarn.resourcemanager.address'] = properties["yarn.resourcemanager.address.#{id}"]
          properties
      @hconfigure
        target: "/opt/druid-#{druid.version}/conf/druid/_common/mapred-site.xml"
        source: "#{druid.hadoop_conf_dir}/mapred-site.xml"
        transform: (properties) ->
          classpath = properties['mapreduce.application.classpath'].split ','
          jar_validation = "/opt/druid-#{druid.version}/lib/validation-api-1.1.0.Final.jar"
          classpath.push jar_validation unless jar_validation in classpath<commit_msg>druid: db headers<commit_after>
# Druid Install

    module.exports = header: 'Druid Install', handler: ->
      {druid, realm, db_admin} = @config.ryba
      krb5 = @config.krb5.etc_krb5_conf.realms[realm]

## Register and load

      @register 'hconfigure', 'ryba/lib/hconfigure'
      @register 'hdfs_mkdir', 'ryba/lib/hdfs_mkdir'

## IPTables

| Service   | Port       | Proto     | Parameter                   |
|-----------|------------|-----------|-----------------------------|
| Druid Standalone Realtime    | 8084      | tcp/http  |  |
| Druid Router    | 8088      | tcp/http  |  |

Note, this hasnt been verified.

      # @iptables
      #   header: 'IPTables'
      #   rules: [
      #     { chain: 'INPUT', jump: 'ACCEPT', dport: port, protocol: 'tcp', state: 'NEW', comment: "Falcon Prism Local EndPoint" }
      #   ]
      #   if: @config.iptables.action is 'start'

## Users & Groups

By default, the "druid" package create the following entries:

```bash
cat /etc/passwd | grep druid
druid:x:2435:2435:druid User:/var/lib/druid:/bin/bash
cat /etc/group | grep druid
druid:x:2435:
```

      @group druid.group
      @user druid.user

## Packages

Download and unpack the release archive.

      @call if: druid.db.engine is 'postgres', ->
        # @call once: true, 'masson/commons/postgres/server/wait' # Not yet ready
        @service 'postgresql'
      @call if: druid.db.engine is 'mysql', ->
        # @call once: true, 'ryba/commons/mysql/server/wait' # Not yet ready
        @service 'mysql'
      @file.download
        header: 'Packages'
        source: "#{druid.source}"
        target: "/var/tmp/#{path.basename druid.source}"
      # TODO, could be improved
      # current implementation prevent any further attempt if download status is true and extract fails
      @extract
        source: "/var/tmp/#{path.basename druid.source}"
        target: '/opt'
        if: -> @status -1
      @link
        source: "/opt/druid-#{druid.version}"
        target: "#{druid.dir}"
      @execute
        cmd: """
        if [ $(stat -c "%U" /opt/druid-#{druid.version}) == '#{druid.user.name}' ]; then exit 3; fi
        chown -R #{druid.user.name}:#{druid.group.name} /opt/druid-#{druid.version}
        """
        code_skipped: 3

## Layout

Pid files are stored inside "/var/run/druid" by default.
Log files are stored inside "/var/log/druid" by default.

      @call header: 'Layout', handler: ->
        @mkdir
          target: "#{druid.pid_dir}"
          uid: "#{druid.user.name}"
          gid: "#{druid.group.name}"
        @link
          target: "#{druid.dir}/var/druid/pids"
          source: "#{druid.pid_dir}"
        @mkdir
          target: "#{druid.log_dir}"
          uid: "#{druid.user.name}"
          gid: "#{druid.group.name}"
          parent: true
        @link
          source: "#{druid.log_dir}"
          target: "#{druid.dir}/log"

## Kerberos

Create a service principal for this NameNode. The principal is named after
"nn/#{@config.host}@#{realm}".

      @krb5_addprinc krb5,
        header: 'Kerberos Admin Principal'
        principal: "#{druid.krb5_admin.principal}"
        password: "#{<PASSWORD>}"
        randkey: true
        uid: 'druid'
        gid: 'druid'
        mode: 0o0600
      @krb5_addprinc krb5,
        header: 'Kerberos Service Principal'
        principal: "#{druid.krb5_service.principal}"
        keytab: "#{druid.krb5_service.keytab}"
        randkey: true
        uid: "#{druid.user.name}"
        gid: "#{druid.group.name}"
        mode: 0o0600

## Cron-ed Kinit

Druid has no mechanism to renew its keytab. For that, we use a cron daemon
We then ask a first TGT.

      @cron.add
        header: 'Cron-ed kinit'
        cmd: "/usr/bin/kinit #{druid.krb5_service.principal} -kt #{druid.krb5_service.keytab}"
        when: '0 */9 * * *'
        user: 'druid'
        exec: true

## Database

      @db.user druid.db, database: null, header: 'DB User',
        if: druid.db.engine in ['mysql', 'postgres']
      @db.database druid.db, header: 'Database',
        if: druid.db.engine in ['mysql', 'postgres']
        user: druid.db.username

## Configuration

Configure deep storage.

      @file.properties
        target: "/opt/druid-#{druid.version}/conf/druid/_common/common.runtime.properties"
        content: druid.common_runtime
        backup: true
      @copy
        target: "/opt/druid-#{druid.version}/conf/druid/_common/core-site.xml"
        source: "#{druid.hadoop_conf_dir}/core-site.xml"
      @copy
        target: "/opt/druid-#{druid.version}/conf/druid/_common/hdfs-site.xml"
        source: "#{druid.hadoop_conf_dir}/hdfs-site.xml"
      @hconfigure
        target: "/opt/druid-#{druid.version}/conf/druid/_common/yarn-site.xml"
        source: "#{druid.hadoop_conf_dir}/yarn-site.xml"
        transform: (properties) ->
          if properties['yarn.resourcemanager.ha.rm-ids']
            [id] = properties['yarn.resourcemanager.ha.rm-ids'].split ','
            properties['yarn.resourcemanager.address'] = properties["yarn.resourcemanager.address.#{id}"]
          properties
      @hconfigure
        target: "/opt/druid-#{druid.version}/conf/druid/_common/mapred-site.xml"
        source: "#{druid.hadoop_conf_dir}/mapred-site.xml"
        transform: (properties) ->
          classpath = properties['mapreduce.application.classpath'].split ','
          jar_validation = "/opt/druid-#{druid.version}/lib/validation-api-1.1.0.Final.jar"
          classpath.push jar_validation unless jar_validation in classpath
{'fraction_non_alphanumeric': 0.156975156975157, 'fraction_numerical': 0.009100009100009101, 'mean_word_length': 2.26888756692445, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3271508', 'n_tokens_mistral': 3739, 'n_tokens_neox': 3774, 'n_words': 789}