<commit_before># Apache Spark History Server

The history servers comes with the spark-client package. The single difference is in the configuration
for  kerberos properties.
We does not recommand using the spark WEB UI because it does not support SSL. Moreover it does make Yarn
redirect the tracking URL to the WEBUI which prevents the user to see the log after the job has finished
in the resource Manager web interface.



    module.exports = []
    module.exports.push 'masson/bootstrap'
    module.exports.push 'masson/commons/java'
    module.exports.push 'ryba/hadoop/hdfs_client'
    module.exports.push require('./index').configure
    module.exports.push 'ryba/spark/client/install'

## IPTables

| Service              | Port  | Proto | Info              |
|----------------------|-------|-------|-------------------|
| spark history server | 18080 | http  | Spark HTTP server |

IPTables rules are only inserted if the parameter "iptables.action" is set to
"start" (default value).

    module.exports.push name: 'Oozie Server # IPTables', handler: (ctx, next) ->
      {spark} = ctx.config.ryba
      ctx.iptables
        rules: [
          { chain: 'INPUT', jump: 'ACCEPT', dport: spark.conf['spark.history.ui.port'], protocol: 'tcp', state: 'NEW', comment: "Oozie HTTP Server" }
        ]
        if: ctx.config.iptables.action is 'start'
      .then next

    module.exports.push name: 'Spark HS # Layout', handler: (ctx, next) ->
      {spark} = ctx.config.ryba
      fs_log_dir = spark.conf['spark.history.fs.logDirectory']
      fs_log_dir = if fs_log_dir.indexOf('file:/') is 0 then  path.join('/', fs_log_dir.substr(6)) else fs_log_dir
      ctx
      .mkdir
        destination: spark.pid_dir
        uid: spark.user.name
        gid: spark.group.name<commit_msg>spark: simplify hs log dir<commit_after># Apache Spark History Server

The history servers comes with the spark-client package. The single difference is in the configuration
for  kerberos properties.
We does not recommand using the spark WEB UI because it does not support SSL. Moreover it does make Yarn
redirect the tracking URL to the WEBUI which prevents the user to see the log after the job has finished
in the resource Manager web interface.



    module.exports = []
    module.exports.push 'masson/bootstrap'
    module.exports.push 'masson/commons/java'
    module.exports.push 'ryba/hadoop/hdfs_client'
    module.exports.push require('./index').configure
    module.exports.push 'ryba/spark/client/install'

## IPTables

| Service              | Port  | Proto | Info              |
|----------------------|-------|-------|-------------------|
| spark history server | 18080 | http  | Spark HTTP server |

IPTables rules are only inserted if the parameter "iptables.action" is set to
"start" (default value).

    module.exports.push name: 'Oozie Server # IPTables', handler: (ctx, next) ->
      {spark} = ctx.config.ryba
      ctx.iptables
        rules: [
          { chain: 'INPUT', jump: 'ACCEPT', dport: spark.conf['spark.history.ui.port'], protocol: 'tcp', state: 'NEW', comment: "Oozie HTTP Server" }
        ]
        if: ctx.config.iptables.action is 'start'
      .then next

    module.exports.push name: 'Spark HS # Layout', handler: (ctx, next) ->
      {spark} = ctx.config.ryba
      fs_log_dir = spark.conf['spark.history.fs.logDirectory']
      fs_log_dir = if fs_log_dir.indexOf('file:/') is 0 then fs_log_dir.substr(5) else fs_log_dir
      ctx
      .mkdir
        destination: spark.pid_dir
        uid: spark.user.name
        gid: spark.group.name
{'fraction_non_alphanumeric': 0.13277310924369748, 'fraction_numerical': 0.00392156862745098, 'mean_word_length': 2.6401630988786953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808839', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1021, 'n_words': 342}