<commit_before>const mongojs = require('../app/js/infrastructure/mongojs')
const { db } = mongojs
const async = require('async')
const minilist = require('minimist')

const newTimeout = 240
const oldTimeoutLimits = {$gt: 60, $lt: 240}

const updateUser = function (user, callback) {
  console.log(`Updating user ${user._id}`)
  const update = {
    $set: {
      'features.compileTimeout': newTimeout
    }
  }
  db.users.update({
    _id: user._id,
    'features.compileTimeout': oldTimeoutLimits
  }, update, callback)
}

const updateUsers = (users, callback) =>
  async.eachLimit(users, ASYNC_LIMIT, updateUser, function (error) {
    if (error) {
      callback(error)
      return
    }
    counter += users.length
    console.log(`${counter} users updated`)
    loopForUsers(callback)
  })

var loopForUsers = callback =>
  db.users.find(
    { 'features.compileTimeout': oldTimeoutLimits },
    { 'features.compileTimeout': 1 }
  ).limit(FETCH_LIMIT, function (error, users) {
    if (error) {
      callback(error)
      return
    }
    if (users.length === 0) {
      console.log(`DONE (${counter} users updated)`)
      return callback()
    }
    updateUsers(users, callback)
  })

var counter = 0
var run = () =>
  loopForUsers(function (error) {
    if (error) { throw error }
    process.exit()
  })

let FETCH_LIMIT, ASYNC_LIMIT
var setup = function () {
  let args = minilist(process.argv.slice(2))
  FETCH_LIMIT = (args.fetch) ? args.fetch : 100
  ASYNC_LIMIT = (args.async) ? args.async : 10
}

setup()
run()<commit_msg>add --all option for increase_compile_timeouts script<commit_after>const mongojs = require('../app/js/infrastructure/mongojs')
const { db } = mongojs
const async = require('async')
const minilist = require('minimist')

const newTimeout = 240
const oldTimeoutLimits = {$gt: 60, $lt: 240}

const updateUser = function (user, callback) {
  console.log(`Updating user ${user._id}`)
  const update = {
    $set: {
      'features.compileTimeout': newTimeout
    }
  }
  db.users.update({
    _id: user._id,
    'features.compileTimeout': oldTimeoutLimits
  }, update, callback)
}

const updateUsers = (users, callback) =>
  async.eachLimit(users, ASYNC_LIMIT, updateUser, function (error) {
    if (error) {
      callback(error)
      return
    }
    counter += users.length
    console.log(`${counter} users updated`);
    if (DO_ALL) {
      return loopForUsers(callback)
    } else {
      console.log('run again to continue updating');
      return callback()
    }
  })

var loopForUsers = callback =>
  db.users.find(
    { 'features.compileTimeout': oldTimeoutLimits },
    { 'features.compileTimeout': 1 }
  ).limit(FETCH_LIMIT, function (error, users) {
    if (error) {
      callback(error)
      return
    }
    if (users.length === 0) {
      console.log(`DONE (${counter} users updated)`)
      return callback()
    }
    updateUsers(users, callback)
  })

var counter = 0
var run = () =>
  loopForUsers(function (error) {
    if (error) { throw error }
    process.exit()
  })

let FETCH_LIMIT, ASYNC_LIMIT, DO_ALL
var setup = function () {
  let args = minilist(process.argv.slice(2))
  // --fetch N  get N users each time
  FETCH_LIMIT = (args.fetch) ? args.fetch : 100
  // --async M  run M updates in parallel
  ASYNC_LIMIT = (args.async) ? args.async : 10
  // --all means run to completion
  DO_ALL = (args.all)
}

setup()
run()
{'fraction_non_alphanumeric': 0.13994685562444642, 'fraction_numerical': 0.010038382049010924, 'mean_word_length': 3.198265179677819, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1454486', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1125, 'n_words': 273}