<commit_before>
# Oozie Client Check

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push require('./client').configure

    module.exports.push name: 'Oozie Client # Wait Server', timeout: -1, callback: (ctx, next) ->
      {hostname, port} = url.parse ctx.config.ryba.oozie_site['oozie.base.url'] 
      ctx.waitIsOpen hostname, port, (err) -> next err

## Check Client

    module.exports.push name: 'Oozie Client # Check Client', timeout: -1, label_true: 'CHECKED', callback: (ctx, next) ->
      {realm, test_user, oozie_site} = ctx.config.ryba
      ctx.execute
        cmd: mkcmd.test ctx, """
        oozie admin -oozie #{oozie_site['oozie.base.url']} -status
        """
      , (err, executed, stdout) ->
        return next err if err
        return next new Error "Oozie not ready, got: #{JSON.stringify stdout}" if stdout.trim() isnt 'System mode: NORMAL'
        return next null, true

## Check REST

    module.exports.push name: 'Oozie Client # Check REST', timeout: -1, label_true: 'CHECKED', callback: (ctx, next) ->
      {realm, test_user, oozie_site} = ctx.config.ryba
      ctx.execute
        cmd: mkcmd.test ctx, """
        curl -s -k --negotiate -u : #{oozie_site['oozie.base.url']}/v1/admin/status
        """
      , (err, executed, stdout) ->
        return next err if err
        return next new Error "Oozie not ready" if stdout.trim() isnt '{"systemMode":"NORMAL"}'
        return next null, true

## Check HDFS Workflow

    module.exports.push name: 'Oozie Client # Check HDFS Workflow', timeout: -1, label_true: 'CHECKED', label_false: 'SKIPPED', callback: (ctx, next) ->
      {force_check, test_user, core_site, oozie_site} = ctx.config.ryba
      rm_ctxs = ctx.contexts 'ryba/hadoop/yarn_rm', require('../hadoop/yarn').configure
      if rm_ctxs.length > 1
        rm_ctx = ctx.context active_rm_host, require('../hadoop/yarn').configure
        shortname = ".#{rm_ctx.config.shortname}"
      else
        rm_ctx = rm_ctxs[0]
        shortname = ''
      rm_address = rm_ctx.config.ryba.yarn_site["yarn.resourcemanager.address#{shortname}"]
      ctx.write [
        content: """
          nameNode=#{core_site['fs.defaultFS']}
          jobTracker=#{rm_address}:8050
          queueName=default
          basedir=${nameNode}/user/#{test_user.name}/check-#{ctx.config.shortname}-oozie-fs
          oozie.wf.application.path=${basedir}
        """
        destination: "#{test_user.home}/check_oozie_fs/job.properties"
        uid: test_user.name
        gid: test_user.group
        eof: true
      ,
        content: """
        <workflow-app xmlns="uri:oozie:workflow:0.2" name="test-oozie-wf">
          <start to="move"/>
          <action name="move">
            <fs>
              <move source='${basedir}/source' target='${basedir}/target'/>
            </fs>
            <ok to="end"/>
            <error to="fail"/>
          </action>
          <kill name="fail">
              <message>Error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
          </kill>
          <end name="end"/>
        </workflow-app>
        """
        destination: "#{test_user.home}/check_oozie_fs/workflow.xml"
        uid: test_user.name
        gid: test_user.group
        eof: true
      ], (err, written) ->
        return next err if err
        ctx.execute
          cmd: """
          hdfs dfs -rm -r -skipTrash check-#{ctx.config.shortname}-oozie-fs 2>/dev/null
          hdfs dfs -mkdir -p check-#{ctx.config.shortname}-oozie-fs
          hdfs dfs -touchz check-#{ctx.config.shortname}-oozie-fs/source
          hdfs dfs -put -f #{test_user.home}/check_oozie_fs/job.properties check-#{ctx.config.shortname}-oozie-fs
          hdfs dfs -put -f #{test_user.home}/check_oozie_fs/workflow.xml check-#{ctx.config.shortname}-oozie-fs
          export OOZIE_URL=#{oozie_site['oozie.base.url']}
          oozie job -dryrun -config #{test_user.home}/check_oozie_fs/job.properties
          jobid=`oozie job -run -config #{test_user.home}/check_oozie_fs/job.properties | grep job: | sed 's/job: \\(.*\\)/\\1/'`
          i=0
          while [[ $i -lt 1000 ]] && [[ `oozie job -info $jobid | grep -e '^Status' | sed 's/^Status\\s\\+:\\s\\+\\(.*\\)$/\\1/'` == 'RUNNING' ]]
          do ((i++)); sleep 1; done
          oozie job -info $jobid | grep -e '^Status\\s\\+:\\s\\+SUCCEEDED'
          """
          code_skipped: 2
          not_if_exec: unless force_check then mkcmd.test ctx, "hdfs dfs -test -f check-#{ctx.config.shortname}-oozie-fs/target"
        , next

## Check Pig Workflow

    module.exports.push name: 'Oozie Client # Check Pig Workflow', timeout: -1, label_true: 'CHECKED', label_false: 'SKIPPED', callback: (ctx, next) ->
      {force_check, test_user, core_site, active_rm_host, oozie_site} = ctx.config.ryba
      rm_ctxs = ctx.contexts 'ryba/hadoop/yarn_rm', require('../hadoop/yarn').configure
      if rm_ctxs.length > 1
        rm_ctx = ctx.context active_rm_host, require('../hadoop/yarn').configure
        shortname = ".#{rm_ctx.config.shortname}"
      else
        rm_ctx = rm_ctxs[0]
        shortname = ''
      rm_address = rm_ctx.config.ryba.yarn_site["yarn.resourcemanager.address#{shortname}"]
      # Get the name of the user running the Oozie Server
      os_ctxs = ctx.contexts 'ryba/oozie/server', require('./server').configure
      {oozie_user} = os_ctxs[0].config.ryba
      ctx.write [
        content: """
          nameNode=#{core_site['fs.defaultFS']}
          jobTracker=#{rm_address}
          oozie.libpath=/user/#{oozie_user.name}/share/lib
          queueName=default
          basedir=${nameNode}/user/#{test_user.name}/check-#{ctx.config.shortname}-oozie-pig
          oozie.wf.application.path=${basedir}
          oozie.use.system.libpath=true
        """
        destination: "#{test_user.home}/check_oozie_pig/job.properties"
        uid: test_user.name
        gid: test_user.group
        eof: true<commit_msg>oozie client: skip pig test<commit_after>
# Oozie Client Check

    module.exports = []
    module.exports.push 'masson/bootstrap/'
    module.exports.push require('./client').configure

    module.exports.push name: 'Oozie Client # Wait Server', timeout: -1, callback: (ctx, next) ->
      {hostname, port} = url.parse ctx.config.ryba.oozie_site['oozie.base.url'] 
      ctx.waitIsOpen hostname, port, (err) -> next err

## Check Client

    module.exports.push name: 'Oozie Client # Check Client', timeout: -1, label_true: 'CHECKED', callback: (ctx, next) ->
      {realm, test_user, oozie_site} = ctx.config.ryba
      ctx.execute
        cmd: mkcmd.test ctx, """
        oozie admin -oozie #{oozie_site['oozie.base.url']} -status
        """
      , (err, executed, stdout) ->
        return next err if err
        return next new Error "Oozie not ready, got: #{JSON.stringify stdout}" if stdout.trim() isnt 'System mode: NORMAL'
        return next null, true

## Check REST

    module.exports.push name: 'Oozie Client # Check REST', timeout: -1, label_true: 'CHECKED', callback: (ctx, next) ->
      {realm, test_user, oozie_site} = ctx.config.ryba
      ctx.execute
        cmd: mkcmd.test ctx, """
        curl -s -k --negotiate -u : #{oozie_site['oozie.base.url']}/v1/admin/status
        """
      , (err, executed, stdout) ->
        return next err if err
        return next new Error "Oozie not ready" if stdout.trim() isnt '{"systemMode":"NORMAL"}'
        return next null, true

## Check HDFS Workflow

    module.exports.push name: 'Oozie Client # Check HDFS Workflow', timeout: -1, label_true: 'CHECKED', label_false: 'SKIPPED', callback: (ctx, next) ->
      {force_check, test_user, core_site, oozie_site} = ctx.config.ryba
      rm_ctxs = ctx.contexts 'ryba/hadoop/yarn_rm', require('../hadoop/yarn').configure
      if rm_ctxs.length > 1
        rm_ctx = ctx.context active_rm_host, require('../hadoop/yarn').configure
        shortname = ".#{rm_ctx.config.shortname}"
      else
        rm_ctx = rm_ctxs[0]
        shortname = ''
      rm_address = rm_ctx.config.ryba.yarn_site["yarn.resourcemanager.address#{shortname}"]
      ctx.write [
        content: """
          nameNode=#{core_site['fs.defaultFS']}
          jobTracker=#{rm_address}:8050
          queueName=default
          basedir=${nameNode}/user/#{test_user.name}/check-#{ctx.config.shortname}-oozie-fs
          oozie.wf.application.path=${basedir}
        """
        destination: "#{test_user.home}/check_oozie_fs/job.properties"
        uid: test_user.name
        gid: test_user.group
        eof: true
      ,
        content: """
        <workflow-app xmlns="uri:oozie:workflow:0.2" name="test-oozie-wf">
          <start to="move"/>
          <action name="move">
            <fs>
              <move source='${basedir}/source' target='${basedir}/target'/>
            </fs>
            <ok to="end"/>
            <error to="fail"/>
          </action>
          <kill name="fail">
              <message>Error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
          </kill>
          <end name="end"/>
        </workflow-app>
        """
        destination: "#{test_user.home}/check_oozie_fs/workflow.xml"
        uid: test_user.name
        gid: test_user.group
        eof: true
      ], (err, written) ->
        return next err if err
        ctx.execute
          cmd: """
          hdfs dfs -rm -r -skipTrash check-#{ctx.config.shortname}-oozie-fs 2>/dev/null
          hdfs dfs -mkdir -p check-#{ctx.config.shortname}-oozie-fs
          hdfs dfs -touchz check-#{ctx.config.shortname}-oozie-fs/source
          hdfs dfs -put -f #{test_user.home}/check_oozie_fs/job.properties check-#{ctx.config.shortname}-oozie-fs
          hdfs dfs -put -f #{test_user.home}/check_oozie_fs/workflow.xml check-#{ctx.config.shortname}-oozie-fs
          export OOZIE_URL=#{oozie_site['oozie.base.url']}
          oozie job -dryrun -config #{test_user.home}/check_oozie_fs/job.properties
          jobid=`oozie job -run -config #{test_user.home}/check_oozie_fs/job.properties | grep job: | sed 's/job: \\(.*\\)/\\1/'`
          i=0
          while [[ $i -lt 1000 ]] && [[ `oozie job -info $jobid | grep -e '^Status' | sed 's/^Status\\s\\+:\\s\\+\\(.*\\)$/\\1/'` == 'RUNNING' ]]
          do ((i++)); sleep 1; done
          oozie job -info $jobid | grep -e '^Status\\s\\+:\\s\\+SUCCEEDED'
          """
          code_skipped: 2
          not_if_exec: unless force_check then mkcmd.test ctx, "hdfs dfs -test -f check-#{ctx.config.shortname}-oozie-fs/target"
        , next

## Check Pig Workflow

    module.exports.push name: 'Oozie Client # Check Pig Workflow', skip: true, timeout: -1, label_true: 'CHECKED', label_false: 'SKIPPED', callback: (ctx, next) ->
      {force_check, test_user, core_site, active_rm_host, oozie_site} = ctx.config.ryba
      rm_ctxs = ctx.contexts 'ryba/hadoop/yarn_rm', require('../hadoop/yarn').configure
      if rm_ctxs.length > 1
        rm_ctx = ctx.context active_rm_host, require('../hadoop/yarn').configure
        shortname = ".#{rm_ctx.config.shortname}"
      else
        rm_ctx = rm_ctxs[0]
        shortname = ''
      rm_address = rm_ctx.config.ryba.yarn_site["yarn.resourcemanager.address#{shortname}"]
      # Get the name of the user running the Oozie Server
      os_ctxs = ctx.contexts 'ryba/oozie/server', require('./server').configure
      {oozie_user} = os_ctxs[0].config.ryba
      ctx.write [
        content: """
          nameNode=#{core_site['fs.defaultFS']}
          jobTracker=#{rm_address}
          oozie.libpath=/user/#{oozie_user.name}/share/lib
          queueName=default
          basedir=${nameNode}/user/#{test_user.name}/check-#{ctx.config.shortname}-oozie-pig
          oozie.wf.application.path=${basedir}
          oozie.use.system.libpath=true
        """
        destination: "#{test_user.home}/check_oozie_pig/job.properties"
        uid: test_user.name
        gid: test_user.group
        eof: true
{'fraction_non_alphanumeric': 0.15323391420911528, 'fraction_numerical': 0.004524128686327078, 'mean_word_length': 2.42918701522551, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4200185', 'n_tokens_mistral': 4059, 'n_tokens_neox': 3980, 'n_words': 809}