<commit_before>const fs = require('fs')
const path = require('path')
const dsvFormat = require('d3-dsv').dsvFormat
const dsv = dsvFormat(',')
const { currentBuildLocales } = require('../helpers')

/**
 *  Converts an Array to an Object
 */
function convertToObject (array) {
  const obj = []
  array.forEach((element, index, array) => {
    obj[element[0]] = element[1]
  })
  return obj
}

module.exports = function (csvDirectories, config = null) {
  const currentLocales = currentBuildLocales()
  const fallbackLocale = 'en-US'
  let messages = {}
  let languages = []

  // get messages from CSV files
  csvDirectories.forEach(directory => {
    fs.readdirSync(directory).forEach(file => {
      const fullFileName = path.join(directory, file)
      const extName = path.extname(fullFileName)
      const baseName = path.posix.basename(file, extName)

      if (extName === '.csv') {
        const fileContent = fs.readFileSync(fullFileName, 'utf8')
        if (languages.indexOf(baseName) === -1) {
          languages.push(baseName)
        }
        console.debug(`Processing translation file: ${fullFileName}`)
        messages[baseName] = Object.assign(messages[baseName] ? messages[baseName] : {}, convertToObject(dsv.parseRows(fileContent)))
      }
    })
  })

  // create fallback
  console.debug(`Writing JSON file fallback: ${fallbackLocale}.json`)
  fs.writeFileSync(path.join(__dirname, '../resource/i18n', `${fallbackLocale}.json`), JSON.stringify(messages[fallbackLocale]))

  // bundle all messages in one file
  if (config && config.i18n.bundleAllStoreviewLanguages) {
    const bundledLanguages = { [fallbackLocale]: messages[fallbackLocale] } // fallback locale
    bundledLanguages[config.i18n.defaultLocale] = messages[config.i18n.defaultLocale] // default locale
    currentLocales.forEach((locale) => {
      bundledLanguages[locale] = messages[locale]
    })

    console.debug(`Writing JSON file multistoreLanguages`)
    fs.writeFileSync(path.join(__dirname, '../resource/i18n', `multistoreLanguages.json`), JSON.stringify(bundledLanguages))
  } else {
    currentLocales.forEach((language) => {
      if (language !== fallbackLocale) return // it's already loaded
      const filePath = path.join(__dirname, '../resource/i18n', `${language}.json`)
      console.debug(`Writing JSON file: ${language}.json`)
      fs.writeFileSync(filePath, JSON.stringify(messages[language]))
    })
    fs.writeFileSync(path.join(__dirname, '../resource/i18n', `multistoreLanguages.json`), JSON.stringify({})) // fix for webpack compilation error in case of `bundleAllStoreviewLanguages` = `false` (#3188)
  }
}<commit_msg>fix logic for generating lang.json files<commit_after>const fs = require('fs')
const path = require('path')
const dsvFormat = require('d3-dsv').dsvFormat
const dsv = dsvFormat(',')
const { currentBuildLocales } = require('../helpers')

/**
 *  Converts an Array to an Object
 */
function convertToObject (array) {
  const obj = []
  array.forEach((element, index, array) => {
    obj[element[0]] = element[1]
  })
  return obj
}

module.exports = function (csvDirectories, config = null) {
  const currentLocales = currentBuildLocales()
  const fallbackLocale = 'en-US'
  let messages = {}
  let languages = []

  // get messages from CSV files
  csvDirectories.forEach(directory => {
    fs.readdirSync(directory).forEach(file => {
      const fullFileName = path.join(directory, file)
      const extName = path.extname(fullFileName)
      const baseName = path.posix.basename(file, extName)

      if (extName === '.csv') {
        const fileContent = fs.readFileSync(fullFileName, 'utf8')
        if (languages.indexOf(baseName) === -1) {
          languages.push(baseName)
        }
        console.debug(`Processing translation file: ${fullFileName}`)
        messages[baseName] = Object.assign(messages[baseName] ? messages[baseName] : {}, convertToObject(dsv.parseRows(fileContent)))
      }
    })
  })

  // create fallback
  console.debug(`Writing JSON file fallback: ${fallbackLocale}.json`)
  fs.writeFileSync(path.join(__dirname, '../resource/i18n', `${fallbackLocale}.json`), JSON.stringify(messages[fallbackLocale]))

  // bundle all messages in one file
  if (config && config.i18n.bundleAllStoreviewLanguages) {
    const bundledLanguages = { [fallbackLocale]: messages[fallbackLocale] } // fallback locale
    bundledLanguages[config.i18n.defaultLocale] = messages[config.i18n.defaultLocale] // default locale
    currentLocales.forEach((locale) => {
      bundledLanguages[locale] = messages[locale]
    })

    console.debug(`Writing JSON file multistoreLanguages`)
    fs.writeFileSync(path.join(__dirname, '../resource/i18n', `multistoreLanguages.json`), JSON.stringify(bundledLanguages))
  } else {
    currentLocales.forEach((language) => {
      if (language === fallbackLocale) return // it's already loaded
      const filePath = path.join(__dirname, '../resource/i18n', `${language}.json`)
      console.debug(`Writing JSON file: ${language}.json`)
      fs.writeFileSync(filePath, JSON.stringify(messages[language]))
    })
    fs.writeFileSync(path.join(__dirname, '../resource/i18n', `multistoreLanguages.json`), JSON.stringify({})) // fix for webpack compilation error in case of `bundleAllStoreviewLanguages` = `false` (#3188)
  }
}
{'fraction_non_alphanumeric': 0.14344957587181903, 'fraction_numerical': 0.008671065032987747, 'mean_word_length': 4.048525214081827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1780066', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1572, 'n_words': 327}