<commit_before>/* compile the language files into dist/localization.build.js */

const path = require('path')
const fs = require('fs')
const decomment = require('decomment')

const outputDir = path.join(__dirname, '../dist')
const outputFile = path.join(outputDir, 'localization.build.js')

const languageFileDir = path.join(__dirname, '../localization/languages')

function buildLocalization () {
  // read all the files from the "languages" directory

  const languageFiles = fs.readdirSync(languageFileDir)

  // build languages object

  const languages = {}

  languageFiles.forEach(function (file) {
    const data = fs.readFileSync(path.join(languageFileDir, file), 'utf-8')

    let obj
    try {
      obj = JSON.parse(decomment(data))
    } catch (e) {
      console.error('parsing language file "' + file + '" failed.')
      console.error(e.toString())
      const loc = parseInt(/at position (\d+)/g.exec(e)[1])
      console.info('"' + decomment(data).substring(loc - 40, loc + 40) + '"')
      process.exit()
    }

    languages[obj.identifier] = obj
  })

  let fileContents = 'var languages = ' + JSON.stringify(languages) + ';\n'

  // add contents of localization.js (helper functions, ...)

  fileContents += fs.readFileSync(path.join(__dirname, '../localization/localizationHelpers.js'))

  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir)
  }

  fs.writeFileSync(outputFile, fileContents)
}

if (module.parent) {
  module.exports = buildLocalization
} else {
  buildLocalization()
}<commit_msg>Improve error handling for invalid JSON (#1749)<commit_after>/* compile the language files into dist/localization.build.js */

const path = require('path')
const fs = require('fs')
const decomment = require('decomment')

const outputDir = path.join(__dirname, '../dist')
const outputFile = path.join(outputDir, 'localization.build.js')

const languageFileDir = path.join(__dirname, '../localization/languages')

function buildLocalization () {
  // read all the files from the "languages" directory

  const languageFiles = fs.readdirSync(languageFileDir)

  // build languages object

  const languages = {}

  languageFiles.forEach(function (file) {
    const data = fs.readFileSync(path.join(languageFileDir, file), 'utf-8')

    let obj
    let decommented = null

    try {
      decommented = decomment(data)
      obj = JSON.parse(decommented)
    } catch (e) {
      console.error('parsing language file "' + file + '" failed.')
      console.error(e.toString())

      if (decommented !== null) {
        const msg = e.message
        const match = msg.match(/at position (\d+)/)

        if (match !== null) {
          const loc = parseInt(match[1])
          console.info('"' + decommented.substring(loc - 40, loc + 40) + '"')
        }
      }

      process.exit()
    }

    languages[obj.identifier] = obj
  })

  let fileContents = 'var languages = ' + JSON.stringify(languages) + ';\n'

  // add contents of localization.js (helper functions, ...)

  fileContents += fs.readFileSync(path.join(__dirname, '../localization/localizationHelpers.js'))

  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir)
  }

  fs.writeFileSync(outputFile, fileContents)
}

if (module.parent) {
  module.exports = buildLocalization
} else {
  buildLocalization()
}
{'fraction_non_alphanumeric': 0.1423810966373826, 'fraction_numerical': 0.004847016055740685, 'mean_word_length': 3.4441453566621805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6512337', 'n_tokens_mistral': 1026, 'n_tokens_neox': 981, 'n_words': 233}