<commit_before>const _ = require('lodash')
const config = require('config')
const resolve = require('resolve-dir')

const source = './data'
const destination = resolve(config.data_sync_destination)
const experiments = config.experiments
const experimentTasks = _.map(experiments, function(name) {
  return `shell:local:${name}`
})

function composeCommand(experiment) {
  // override with custom command if has 'python'
  var cmd = _.includes(experiment, 'python') ? experiment : `python3 main.py -bgp -e ${experiment} -t 5`
  return `(${cmd} | tee -a ./data/terminal.log) & NOTI_SLACK_DEST='${config.NOTI_SLACK_DEST}' NOTI_SLACK_TOK='${config.NOTI_SLACK_TOK}' noti -k -t '${experiment}' -m '[${new Date().toISOString()}] Experiment completed on ${process.env.USER}' -pwatch $! &`
}

module.exports = function(grunt) {
  require('load-grunt-tasks')(grunt)

  grunt.initConfig({
    sync: {
      main: {
        files: [{
          cwd: source,
          src: ['**'],
          dest: destination,
        }],
        // pretend: true, // Don't do real IO; log only
        // verbose: true // Display log messages when copying files
      }
    },

    watch: {
      data: {
        files: `${source}/**`,
        tasks: ['sync']
      }
    },

    shell: {
      options: {
        execOptions: {
          killSignal: 'SIGINT',
          env: process.env
        }
      },
      local: {
        command(experiment) {
          return composeCommand(experiment)
        }
      },
      remote: {
        command(experiment) {
          return 'xvfb-run -a -s "-screen 0 1400x900x24" -- grunt'
        }
      },
    },

    concurrent: {
      local: ['watch', 'lab'],
      options: {
        logConcurrentOutput: true
      }
    },
  })

  grunt.registerTask('lab', 'run all the experiments', experimentTasks)
  grunt.registerTask('lab_sync', 'run lab with auto file syncing', ['concurrent:local'])
  grunt.registerTask('default', ['lab_sync'])

  grunt.registerTask('remote', ['shell:remote'])
}<commit_msg>remove fail, use better exit msg. wrap up grunt for now<commit_after>const _ = require('lodash')
const config = require('config')
const resolve = require('resolve-dir')

const source = './data'
const destination = resolve(config.data_sync_destination)
const experiments = config.experiments
const experimentTasks = _.map(experiments, function(name) {
  return `shell:local:${name}`
})

function composeCommand(experiment) {
  // override with custom command if has 'python'
  var cmd = _.includes(experiment, 'python') ? experiment : `python3 main.py -bgp -e ${experiment} -t 5`
  return `${cmd} | tee -a ./data/terminal.log; NOTI_SLACK_DEST='${config.NOTI_SLACK_DEST}' NOTI_SLACK_TOK='${config.NOTI_SLACK_TOK}' noti -k -t 'Experiment completed' -m '[${new Date().toISOString()}] ${experiment} on ${process.env.USER}'`
}

const finishMsg = `
==========================================
Experiments complete. Press Cmd+C to exit.
==========================================
`

module.exports = function(grunt) {
  require('load-grunt-tasks')(grunt)

  grunt.initConfig({
    sync: {
      main: {
        files: [{
          cwd: source,
          src: ['**'],
          dest: destination,
        }],
        // pretend: true, // Don't do real IO; log only
        // verbose: true // Display log messages when copying files
      }
    },

    watch: {
      data: {
        files: `${source}/**`,
        tasks: ['sync']
      }
    },

    shell: {
      options: {
        execOptions: {
          killSignal: 'SIGINT',
          env: process.env
        }
      },
      local: {
        command(experiment) {
          return composeCommand(experiment)
        }
      },
      remote: 'xvfb-run -a -s "-screen 0 1400x900x24" -- grunt',
      killnoti: {
        command: 'killall noti',
        options: {
          failOnError: false
        }
      },
      finish: `echo "${finishMsg}"`
    },

    concurrent: {
      local: ['watch', ['lab', 'kill_noti', 'shell:finish']],
      options: {
        logConcurrentOutput: true
      }
    },
  })

  grunt.registerTask('lab', 'run all the experiments', experimentTasks)
  grunt.registerTask('lab_sync', 'run lab with auto file syncing', ['concurrent:local'])
  grunt.registerTask('default', ['lab_sync'])

  grunt.registerTask('remote', ['shell:remote'])
  grunt.registerTask('kill_noti', ['shell:killnoti'])
}
{'fraction_non_alphanumeric': 0.17062043795620438, 'fraction_numerical': 0.005474452554744526, 'mean_word_length': 2.6848739495798317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3980589', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1346, 'n_words': 315}