<commit_before>var fs = require('fs-extra');
var path = require('path');
var csvParse = require('csv-parse');
var getByPath = require('getbypath');
var extendObj = require('extend-object');
var UNSAFE_SYMBOLS = [
  {
    pattern: / \n/g,
    replace: ' '
  }
];

/**
 * Replaces some unsafe symbols to spaces
 * @param {string} csvContent
 * @returns {string}
 */
function processCsv (csvContent) {
  UNSAFE_SYMBOLS.forEach(function (symbolParams) {
    csvContent = csvContent.replace(symbolParams.pattern, symbolParams.replace);
  });
  return csvContent;
}

/**
 * Creates all directories and files from json
 * @param {Array} json
 * @param {object} params
 * @param {function} callback
 */
function createLocales (json, params, callback) {
  var locales = registerLocales(json.shift());
  var indexes = locales.indexes;
  var messageName;

  delete locales.indexes;

  json.forEach(function (messageLine) {
    var messageProp, lineValue, propPath;

    for (var i = 0, len = messageLine.length; i < len; i++) {
      lineValue = messageLine[i];
      if (i === 0) {
        // The first item is a message name
        // or an empty string for a "description"
        messageName = lineValue || messageName;

      } else if (i === 1) {
        // The second is "message" or "description"
        messageProp = lineValue;

      } else {
        // All others are locales
        if (lineValue === '' && messageProp !== 'description') {
          continue;
        }
        propPath = [
          indexes[i], 'messages', messageName
        ].join('.');
        getByPath(locales, propPath, true)[messageProp] =
          messageProp !== 'placeholders' ?
            lineValue || ''
            : parsePlaceholders(lineValue);
      }
    }
  });
  if (params.debug) {
    log('Generated locales:', locales);
  }
  writeLocales(locales, params, callback);
}

/**
 * Generates a placeholders object from a source string
 * @param {string} source
 * @returns {object}
 */
function parsePlaceholders (source) {
  return source.match(/\([^\)]+\)/g).reduce(function (result, placeholder) {
    placeholder = placeholder.replace(/^\(([^\)]+)\)$/, '$1');
    var keyValue = placeholder.split(':');
    result[keyValue[0]] = {
      content: keyValue[1]
    };
    return result;
  }, {});
}

/**
 * Writes locales to a file system
 * @param {object} locales
 * @param {object} params
 * @param {function} callback
 */
function writeLocales (locales, params, callback) {
  try {
    var jsonPath, json;
    for (var key in locales) {
      if (locales.hasOwnProperty(key)) {
        json = locales[key];
        jsonPath = path.join(params.dirPath, json.localeName, 'messages.json');
        fs.outputJsonSync(jsonPath, json.messages);
        if (params.debug) {
          log('Created file', jsonPath, json.messages);
        }
      }
    }
    callback(null);
  } catch (err) {
    callback(err);
  }
}
<commit_msg>fix: untranslated strings made it into messages.json<commit_after>var fs = require('fs-extra');
var path = require('path');
var csvParse = require('csv-parse');
var getByPath = require('getbypath');
var extendObj = require('extend-object');
var UNSAFE_SYMBOLS = [
  {
    pattern: / \n/g,
    replace: ' '
  }
];

/**
 * Replaces some unsafe symbols to spaces
 * @param {string} csvContent
 * @returns {string}
 */
function processCsv (csvContent) {
  UNSAFE_SYMBOLS.forEach(function (symbolParams) {
    csvContent = csvContent.replace(symbolParams.pattern, symbolParams.replace);
  });
  return csvContent;
}

/**
 * Creates all directories and files from json
 * @param {Array} json
 * @param {object} params
 * @param {function} callback
 */
function createLocales (json, params, callback) {
  var locales = registerLocales(json.shift());
  var indexes = locales.indexes;
  var messageName;

  delete locales.indexes;

  json.forEach(function (messageLine) {
    var messageProp, lineValue, propPath;

    for (var i = 0, len = messageLine.length; i < len; i++) {
      lineValue = messageLine[i];
      if (i === 0) {
        // The first item is a message name
        // or an empty string for a "description"
        messageName = lineValue || messageName;

      } else if (i === 1) {
        // The second is "message" or "description"
        messageProp = lineValue;

      } else {
        // All others are locales
        if (lineValue === '' && messageProp !== 'description') {
          continue;
        }
        propPath = [
          indexes[i], 'messages', messageName
        ].join('.');
        getByPath(locales, propPath, true)[messageProp] =
          messageProp !== 'placeholders' ?
            lineValue || ''
            : parsePlaceholders(lineValue);
      }
    }
  });
  if (params.debug) {
    log('Generated locales:', locales);
  }
  writeLocales(locales, params, callback);
}

/**
 * Generates a placeholders object from a source string
 * @param {string} source
 * @returns {object}
 */
function parsePlaceholders (source) {
  return source.match(/\([^\)]+\)/g).reduce(function (result, placeholder) {
    placeholder = placeholder.replace(/^\(([^\)]+)\)$/, '$1');
    var keyValue = placeholder.split(':');
    result[keyValue[0]] = {
      content: keyValue[1]
    };
    return result;
  }, {});
}

/**
 * Writes locales to a file system
 * @param {object} locales
 * @param {object} params
 * @param {function} callback
 */
function writeLocales (locales, params, callback) {
  try {
    var jsonPath, json;
    for (var key in locales) {
      if (locales.hasOwnProperty(key)) {
        json = locales[key];
        jsonPath = path.join(params.dirPath, json.localeName, 'messages.json');

        // remove empty messages
        for (var msg_key in json.messages) {
          if (json.messages.hasOwnProperty(msg_key)) {
            if (json.messages[msg_key]['message'] == undefined) {
              delete json.messages[msg_key];
            }
          }
        }

        fs.outputJsonSync(jsonPath, json.messages);
        if (params.debug) {
          log('Created file', jsonPath, json.messages);
        }
      }
    }
    callback(null);
  } catch (err) {
    callback(err);
  }
}

{'fraction_non_alphanumeric': 0.14552541822316062, 'fraction_numerical': 0.0019490011369173298, 'mean_word_length': 2.7141133896260556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2699655', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1855, 'n_words': 483}