<commit_before>
/**
 * Module dependencies.
 */

var fs = require('fs')
  , Q = require('q')
  , parser = require('./parser')
  , syntax = require('./syntax')
  , _ = require('underscore')
  , merger = require('./merger')
  , readline = require('readline')
  , file = require('./file')
  , log = require('./_log')
  , Hashids = require('hashids')
  , hashids = new Hashids(program.LOCALIZATION_ID_HASH_SECRET, program.LOCALIZATION_ID_CHARACTER_LENGTH);

/**
 * Add terminal colors
 */

require('terminal-colors');

/**
 * Update
 *
 * @constructor
 */

function Update() {
  this.isWaitingUserInput = false;
  this.deletedKeys = [];
  this.addedKeys = [];
  this.migratedKeys = [];
  this.rl = null;
}

/**
 * Update keys
 *
 * @return {void}
 * @api public
 */

Update.prototype.run = function() {
  var _this = this;

  this.getNewLocalizations()
    .then(function(newLocalizations) {
      return _this._mergeWithOldLocalizations(newLocalizations);
    })
    .then(function(mergedLocalizations) {
      return file.writeLocalizations(mergedLocalizations)
    })
    .fail(function(error) {
      if(commands.stack && error) {
        console.log(error.stack);
      }

      log.error('Localizations update failed');
    });
};

/**
 * Strip inner functions calls. All function calls that is not from l10ns
 * needs to be removed. Because they can cause updating error
 * when they are defined inside variables like below
 *
 *    Example:
 *
 *      gt('SOME_TRANSLATION_KEY', {
 *        prop1 : test() // Becomes ''
 *      });
 *
 * @param {string} content, file content
 * @api private
 */

Update.prototype._stripInnerFunctionCalls = function(content) {
  var getLocalizationStringFunctionSyntax = new RegExp(language.GET_LOCALIZATION_STRING_FUNCTION_NAME + '\\(', 'g')
    , getLocalizationStringUsingDataFunctionSyntax = new RegExp(language.GET_LOCALIZATION_STRING_USING_DATA_FUNCTION_NAME + '\\(', 'g');

  return content.replace(language.GET_LOCALIZATION_INNER_FUNCTION_CALL_SYNTAX, function(match) {
    if(getLocalizationStringFunctionSyntax.test(match) || getLocalizationStringUsingDataFunctionSyntax.test(match)) {
      return match;
    } else {
      return '';
    }
  });
};

/**
 * Get translation function calls form source
 *
 * @return {Object} newLocalizations
 * @api private
 */

Update.prototype.getNewLocalizations = function() {
  var _this = this
    , deferred = Q.defer()
    , now = parseInt(Date.now() / 1000, 10)
    , newLocalizations = {}
    , hashCount = 0
    , fileCount = 0
    , rejected = false;

  project.source.forEach(function(file) {
    if(fs.lstatSync(file).isDirectory()) {
      fileCount++;
      return;
    }

    fs.readFile(file, 'utf8', function(error, content) {
        fileCount++;

        if(rejected) {
          return;
        }

        if(error) {
          rejected = true;
          return deferred.reject(error);
        }

        content = _this._stripInnerFunctionCalls(content);
        // Match all gt() calls
        var calls = content.match(language.GET_LOCALIZATION_FUNCTION_CALL_SYNTAX);
        if(calls !== null) {
          calls.forEach(function(call) {
            var key  = parser.getKey(call)
              , variables = parser.getVariables(call);
            if(!(key in newLocalizations)) {
              newLocalizations[key] = {};
              newLocalizations[key].id = hashids.encrypt(now, hashCount);
              newLocalizations[key].key = key;
              newLocalizations[key].variables = variables;
              newLocalizations[key].files = [file];
              hashCount++;
            }
            else {
              if(syntax.hasErrorDuplicate(newLocalizations, key, variables)) {
                rejected = true;
                return deferred.reject(new TypeError('You have defined a localization key ('
                  + key + ') with different variables.\n In file:' + file));
              }
              newLocalizations[key].files.push(file);
            }
          });<commit_msg>Adds check for duplicate files<commit_after>
/**
 * Module dependencies.
 */

var fs = require('fs')
  , Q = require('q')
  , parser = require('./parser')
  , syntax = require('./syntax')
  , _ = require('underscore')
  , merger = require('./merger')
  , readline = require('readline')
  , file = require('./file')
  , log = require('./_log')
  , Hashids = require('hashids')
  , hashids = new Hashids(program.LOCALIZATION_ID_HASH_SECRET, program.LOCALIZATION_ID_CHARACTER_LENGTH);

/**
 * Add terminal colors
 */

require('terminal-colors');

/**
 * Update
 *
 * @constructor
 */

function Update() {
  this.isWaitingUserInput = false;
  this.deletedKeys = [];
  this.addedKeys = [];
  this.migratedKeys = [];
  this.rl = null;
}

/**
 * Update keys
 *
 * @return {void}
 * @api public
 */

Update.prototype.run = function() {
  var _this = this;

  this.getNewLocalizations()
    .then(function(newLocalizations) {
      return _this._mergeWithOldLocalizations(newLocalizations);
    })
    .then(function(mergedLocalizations) {
      return file.writeLocalizations(mergedLocalizations)
    })
    .fail(function(error) {
      if(commands.stack && error) {
        console.log(error.stack);
      }

      log.error('Localizations update failed');
    });
};

/**
 * Strip inner functions calls. All function calls that is not from l10ns
 * needs to be removed. Because they can cause updating error
 * when they are defined inside variables like below
 *
 *    Example:
 *
 *      gt('SOME_TRANSLATION_KEY', {
 *        prop1 : test() // Becomes ''
 *      });
 *
 * @param {string} content, file content
 * @api private
 */

Update.prototype._stripInnerFunctionCalls = function(content) {
  var getLocalizationStringFunctionSyntax = new RegExp(language.GET_LOCALIZATION_STRING_FUNCTION_NAME + '\\(', 'g')
    , getLocalizationStringUsingDataFunctionSyntax = new RegExp(language.GET_LOCALIZATION_STRING_USING_DATA_FUNCTION_NAME + '\\(', 'g');

  return content.replace(language.GET_LOCALIZATION_INNER_FUNCTION_CALL_SYNTAX, function(match) {
    if(getLocalizationStringFunctionSyntax.test(match) || getLocalizationStringUsingDataFunctionSyntax.test(match)) {
      return match;
    } else {
      return '';
    }
  });
};

/**
 * Get translation function calls form source
 *
 * @return {Object} newLocalizations
 * @api private
 */

Update.prototype.getNewLocalizations = function() {
  var _this = this
    , deferred = Q.defer()
    , now = parseInt(Date.now() / 1000, 10)
    , newLocalizations = {}
    , hashCount = 0
    , fileCount = 0
    , rejected = false;

  project.source.forEach(function(file) {
    if(fs.lstatSync(file).isDirectory()) {
      fileCount++;
      return;
    }

    fs.readFile(file, 'utf8', function(error, content) {
        fileCount++;

        if(rejected) {
          return;
        }

        if(error) {
          rejected = true;
          return deferred.reject(error);
        }

        content = _this._stripInnerFunctionCalls(content);
        // Match all gt() calls
        var calls = content.match(language.GET_LOCALIZATION_FUNCTION_CALL_SYNTAX);
        if(calls !== null) {
          calls.forEach(function(call) {
            var key  = parser.getKey(call)
              , variables = parser.getVariables(call);
            if(!(key in newLocalizations)) {
              newLocalizations[key] = {};
              newLocalizations[key].id = hashids.encrypt(now, hashCount);
              newLocalizations[key].key = key;
              newLocalizations[key].variables = variables;
              newLocalizations[key].files = [file];
              hashCount++;
            }
            else {
              if(syntax.hasErrorDuplicate(newLocalizations, key, variables)) {
                rejected = true;
                return deferred.reject(new TypeError('You have defined a localization key ('
                  + key + ') with different variables.\n In file:' + file));
              }
              if(newLocalizations[key].files.indexOf(file) === -1) {
                newLocalizations[key].files.push(file);
              }
            }
          });
{'fraction_non_alphanumeric': 0.13013530135301354, 'fraction_numerical': 0.003075030750307503, 'mean_word_length': 2.6282909415439537, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6158151', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2365, 'n_words': 460}